ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-02-08 12:00:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #492 from mapbox/rate-clarify

Browse Source 
Clarify documentation; add --no-progress-indicator option; fix many compiler warnings
This commit is contained in:
Eric Fischer 2017-11-17 14:11:22 -08:00 committed by GitHub
commit 4754084130
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 523 additions and 468 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -1,3 +1,8 @@
## 1.26.7
* Add an option to quiet the progress indicator but not warnings
* Enable more compiler warnings and fix related problems
## 1.26.6 ## 1.26.6
* Be more careful about checking for overflow when parsing numbers * Be more careful about checking for overflow when parsing numbers

2
Makefile
View File

@ -9,7 +9,7 @@ CXX := $(CXX)
CFLAGS := $(CFLAGS) CFLAGS := $(CFLAGS)
CXXFLAGS := $(CXXFLAGS) -std=c++11 CXXFLAGS := $(CXXFLAGS) -std=c++11
LDFLAGS := $(LDFLAGS) LDFLAGS := $(LDFLAGS)
WARNING_FLAGS := -Wall -Wshadow -Wsign-compare WARNING_FLAGS := -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow
RELEASE_FLAGS := -O3 -DNDEBUG RELEASE_FLAGS := -O3 -DNDEBUG
DEBUG_FLAGS := -O0 -DDEBUG -fno-inline-functions -fno-omit-frame-pointer DEBUG_FLAGS := -O0 -DDEBUG -fno-inline-functions -fno-omit-frame-pointer

17
README.md
View File

@ -96,13 +96,15 @@ delete the file that already exists with that name.
If you aren't sure what the right maxzoom is for your data, `-zg` will guess one for you If you aren't sure what the right maxzoom is for your data, `-zg` will guess one for you
based on the density of features. based on the density of features.
If you are mapping point features, you will often want to use `-Bg` to automatically choose Tippecanoe will normally drop a fraction of point features at zooms below the maxzoom,
a base zoom level for dot dropping. If that doesn't work out for you, try to keep the low-zoom tiles from getting too big. If you have a smaller data set where
`-r1 --drop-fraction-as-needed` to turn off the normal dot dropping and instead all the points would fit without dropping any of them, use `-r1` to keep them all.
only drop features if the tiles get too big. If you do want point dropping, but you still want the tiles to be denser than `-zg`
thinks they should be, use `-B` to set a basezoom lower than the maxzoom.
If you are mapping points or polygons, you will often want to use `--drop-densest-as-needed` If some of your tiles are coming out too big in spite of the settings above, you will
to drop some of them if necessary to make the low zoom levels work. often want to use `--drop-densest-as-needed` to drop whatever fraction of the features
is necessary at each zoom level to make that zoom level's tiles work.
If your features have a lot of attributes, use `-y` to keep only the ones you really need. If your features have a lot of attributes, use `-y` to keep only the ones you really need.
@ -262,7 +264,8 @@ tippecanoe -z5 -o filtered.mbtiles -j '{ "ne_10m_admin_0_countries": [ "all", [
### Progress indicator ### Progress indicator
* `-q` or `--quiet`: Work quietly instead of reporting progress * `-q` or `--quiet`: Work quietly instead of reporting progress or warning messages
* `-Q` or `--no-progress-indicator`: Don't report progress, but still give warnings
* `-v` or `--version`: Report Tippecanoe's version number * `-v` or `--version`: Report Tippecanoe's version number
### Filters ### Filters

2
decode.cpp
View File

@ -120,7 +120,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st
} }
// X and Y are unsigned, so no need to check <0 // X and Y are unsigned, so no need to check <0
if (x > (1 << z) || y > (1 << z)) { if (x > (1ULL << z) || y > (1ULL << z)) {
fprintf(stderr, "Impossible tile %d/%u/%u\n", z, x, y); fprintf(stderr, "Impossible tile %d/%u/%u\n", z, x, y);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }

80
geobuf.cpp
View File

@ -22,13 +22,13 @@
#define MULTIPOLYGON 5 #define MULTIPOLYGON 5
struct queued_feature { struct queued_feature {
protozero::pbf_reader pbf; protozero::pbf_reader pbf{};
size_t dim; size_t dim = 0;
double e; double e = 0;
std::vector<std::string> *keys; std::vector<std::string> *keys = NULL;
struct serialization_state *sst; std::vector<struct serialization_state> *sst = NULL;
int layer; int layer = 0;
std::string layername; std::string layername = "";
}; };
static std::vector<queued_feature> feature_queue; static std::vector<queued_feature> feature_queue;
@ -40,7 +40,7 @@ void ensureDim(size_t dim) {
} }
} }
serial_val readValue(protozero::pbf_reader &pbf, std::vector<std::string> &keys) { serial_val readValue(protozero::pbf_reader &pbf) {
serial_val sv; serial_val sv;
sv.type = mvt_null; sv.type = mvt_null;
sv.s = "null"; sv.s = "null";
@ -94,7 +94,7 @@ serial_val readValue(protozero::pbf_reader &pbf, std::vector<std::string> &keys)
return sv; return sv;
} }
drawvec readPoint(std::vector<long long> &coords, std::vector<int> &lengths, size_t dim, double e) { drawvec readPoint(std::vector<long long> &coords, size_t dim, double e) {
ensureDim(dim); ensureDim(dim);
long long x, y; long long x, y;
@ -104,7 +104,7 @@ drawvec readPoint(std::vector<long long> &coords, std::vector<int> &lengths, siz
return dv; return dv;
} }
drawvec readLinePart(std::vector<long long> &coords, std::vector<int> &lengths, size_t dim, double e, size_t start, size_t end, bool closed) { drawvec readLinePart(std::vector<long long> &coords, size_t dim, double e, size_t start, size_t end, bool closed) {
ensureDim(dim); ensureDim(dim);
drawvec dv; drawvec dv;
@ -141,19 +141,19 @@ drawvec readLinePart(std::vector<long long> &coords, std::vector<int> &lengths,
return dv; return dv;
} }
drawvec readLine(std::vector<long long> &coords, std::vector<int> &lengths, size_t dim, double e, bool closed) { drawvec readLine(std::vector<long long> &coords, size_t dim, double e, bool closed) {
return readLinePart(coords, lengths, dim, e, 0, coords.size(), closed); return readLinePart(coords, dim, e, 0, coords.size(), closed);
} }
drawvec readMultiLine(std::vector<long long> &coords, std::vector<int> &lengths, size_t dim, double e, bool closed) { drawvec readMultiLine(std::vector<long long> &coords, std::vector<int> &lengths, size_t dim, double e, bool closed) {
if (lengths.size() == 0) { if (lengths.size() == 0) {
return readLinePart(coords, lengths, dim, e, 0, coords.size(), closed); return readLinePart(coords, dim, e, 0, coords.size(), closed);
} }
drawvec dv; drawvec dv;
size_t here = 0; size_t here = 0;
for (size_t i = 0; i < lengths.size(); i++) { for (size_t i = 0; i < lengths.size(); i++) {
drawvec dv2 = readLinePart(coords, lengths, dim, e, here, here + lengths[i] * dim, closed); drawvec dv2 = readLinePart(coords, dim, e, here, here + lengths[i] * dim, closed);
here += lengths[i] * dim; here += lengths[i] * dim;
for (size_t j = 0; j < dv2.size(); j++) { for (size_t j = 0; j < dv2.size(); j++) {
@ -168,7 +168,7 @@ drawvec readMultiPolygon(std::vector<long long> &coords, std::vector<int> &lengt
ensureDim(dim); ensureDim(dim);
if (lengths.size() == 0) { if (lengths.size() == 0) {
return readLinePart(coords, lengths, dim, e, 0, coords.size(), true); return readLinePart(coords, dim, e, 0, coords.size(), true);
} }
size_t polys = lengths[0]; size_t polys = lengths[0];
@ -180,7 +180,7 @@ drawvec readMultiPolygon(std::vector<long long> &coords, std::vector<int> &lengt
size_t rings = lengths[n++]; size_t rings = lengths[n++];
for (size_t j = 0; j < rings; j++) { for (size_t j = 0; j < rings; j++) {
drawvec dv2 = readLinePart(coords, lengths, dim, e, here, here + lengths[n] * dim, true); drawvec dv2 = readLinePart(coords, dim, e, here, here + lengths[n] * dim, true);
here += lengths[n] * dim; here += lengths[n] * dim;
n++; n++;
@ -196,8 +196,8 @@ drawvec readMultiPolygon(std::vector<long long> &coords, std::vector<int> &lengt
} }
struct drawvec_type { struct drawvec_type {
drawvec dv; drawvec dv{};
int type; int type = 0;
}; };
std::vector<drawvec_type> readGeometry(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys) { std::vector<drawvec_type> readGeometry(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys) {
@ -246,11 +246,11 @@ std::vector<drawvec_type> readGeometry(protozero::pbf_reader &pbf, size_t dim, d
drawvec_type dv; drawvec_type dv;
if (type == POINT) { if (type == POINT) {
dv.dv = readPoint(coords, lengths, dim, e); dv.dv = readPoint(coords, dim, e);
} else if (type == MULTIPOINT) { } else if (type == MULTIPOINT) {
dv.dv = readLine(coords, lengths, dim, e, false); dv.dv = readLine(coords, dim, e, false);
} else if (type == LINESTRING) { } else if (type == LINESTRING) {
dv.dv = readLine(coords, lengths, dim, e, false); dv.dv = readLine(coords, dim, e, false);
} else if (type == POLYGON) { } else if (type == POLYGON) {
dv.dv = readMultiLine(coords, lengths, dim, e, true); dv.dv = readMultiLine(coords, lengths, dim, e, true);
} else if (type == MULTIPOLYGON) { } else if (type == MULTIPOLYGON) {
@ -311,7 +311,7 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<s
case 13: { case 13: {
protozero::pbf_reader value_reader(pbf.get_message()); protozero::pbf_reader value_reader(pbf.get_message());
values.push_back(readValue(value_reader, keys)); values.push_back(readValue(value_reader));
break; break;
} }
@ -425,6 +425,10 @@ struct queue_run_arg {
size_t start; size_t start;
size_t end; size_t end;
size_t segment; size_t segment;
queue_run_arg(size_t start1, size_t end1, size_t segment1)
: start(start1), end(end1), segment(segment1) {
}
}; };
void *run_parse_feature(void *v) { void *run_parse_feature(void *v) {
@ -432,7 +436,7 @@ void *run_parse_feature(void *v) {
for (size_t i = qra->start; i < qra->end; i++) { for (size_t i = qra->start; i < qra->end; i++) {
struct queued_feature &qf = feature_queue[i]; struct queued_feature &qf = feature_queue[i];
readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &qf.sst[qra->segment], qf.layer, qf.layername); readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &(*qf.sst)[qra->segment], qf.layer, qf.layername);
} }
return NULL; return NULL;
@ -443,16 +447,21 @@ void runQueue() {
return; return;
} }
struct queue_run_arg qra[CPUS]; std::vector<struct queue_run_arg> qra;
pthread_t pthreads[CPUS];
std::vector<pthread_t> pthreads;
pthreads.resize(CPUS);
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
*(feature_queue[0].sst[i].layer_seq) = *(feature_queue[0].sst[0].layer_seq) + feature_queue.size() * i / CPUS; *((*(feature_queue[0].sst))[i].layer_seq) = *((*(feature_queue[0].sst))[0].layer_seq) + feature_queue.size() * i / CPUS;
qra[i].start = feature_queue.size() * i / CPUS; qra.push_back(queue_run_arg(
qra[i].end = feature_queue.size() * (i + 1) / CPUS; feature_queue.size() * i / CPUS,
qra[i].segment = i; feature_queue.size() * (i + 1) / CPUS,
i));
}
for (size_t i = 0; i < CPUS; i++) {
if (pthread_create(&pthreads[i], NULL, run_parse_feature, &qra[i]) != 0) { if (pthread_create(&pthreads[i], NULL, run_parse_feature, &qra[i]) != 0) {
perror("pthread_create"); perror("pthread_create");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
@ -467,11 +476,11 @@ void runQueue() {
} }
} }
*(feature_queue[0].sst[0].layer_seq) = *(feature_queue[0].sst[CPUS - 1].layer_seq); *((*(feature_queue[0].sst))[0].layer_seq) = *((*(feature_queue[0].sst))[CPUS - 1].layer_seq);
feature_queue.clear(); feature_queue.clear();
} }
void queueFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) { void queueFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, std::vector<struct serialization_state> *sst, int layer, std::string layername) {
struct queued_feature qf; struct queued_feature qf;
qf.pbf = pbf; qf.pbf = pbf;
qf.dim = dim; qf.dim = dim;
@ -488,7 +497,7 @@ void queueFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<
} }
} }
void outBareGeometry(drawvec const &dv, int type, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) { void outBareGeometry(drawvec const &dv, int type, struct serialization_state *sst, int layer, std::string layername) {
serial_feature sf; serial_feature sf;
sf.layer = layer; sf.layer = layer;
@ -506,7 +515,7 @@ void outBareGeometry(drawvec const &dv, int type, size_t dim, double e, std::vec
serialize_feature(sst, sf); serialize_feature(sst, sf);
} }
void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) { void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, std::vector<struct serialization_state> *sst, int layer, std::string layername) {
while (pbf.next()) { while (pbf.next()) {
switch (pbf.tag()) { switch (pbf.tag()) {
case 1: { case 1: {
@ -521,7 +530,7 @@ void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std
} }
} }
void parse_geobuf(struct serialization_state *sst, const char *src, size_t len, int layer, std::string layername) { void parse_geobuf(std::vector<struct serialization_state> *sst, const char *src, size_t len, int layer, std::string layername) {
protozero::pbf_reader pbf(src, len); protozero::pbf_reader pbf(src, len);
size_t dim = 2; size_t dim = 2;
@ -558,7 +567,8 @@ void parse_geobuf(struct serialization_state *sst, const char *src, size_t len,
protozero::pbf_reader geometry_reader(pbf.get_message()); protozero::pbf_reader geometry_reader(pbf.get_message());
std::vector<drawvec_type> dv = readGeometry(geometry_reader, dim, e, keys); std::vector<drawvec_type> dv = readGeometry(geometry_reader, dim, e, keys);
for (size_t i = 0; i < dv.size(); i++) { for (size_t i = 0; i < dv.size(); i++) {
outBareGeometry(dv[i].dv, dv[i].type, dim, e, keys, sst, layer, layername); // Always on thread 0
outBareGeometry(dv[i].dv, dv[i].type, &(*sst)[0], layer, layername);
} }
break; break;
} }

2
geobuf.hpp
View File

@ -8,6 +8,6 @@
#include "mbtiles.hpp" #include "mbtiles.hpp"
#include "serial.hpp" #include "serial.hpp"
void parse_geobuf(struct serialization_state *sst, const char *s, size_t len, int layer, std::string layername); void parse_geobuf(std::vector<struct serialization_state> *sst, const char *s, size_t len, int layer, std::string layername);
#endif #endif

11
geojson.cpp
View File

@ -146,10 +146,15 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
nprop = properties->length; nprop = properties->length;
} }
char *metakey[nprop]; std::vector<char *> metakey;
metakey.resize(nprop);
std::vector<std::string> metaval; std::vector<std::string> metaval;
metaval.resize(nprop); metaval.resize(nprop);
int metatype[nprop];
std::vector<int> metatype;
metatype.resize(nprop);
size_t m = 0; size_t m = 0;
for (size_t i = 0; i < nprop; i++) { for (size_t i = 0; i < nprop; i++) {
@ -158,7 +163,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
int type = -1; int type = -1;
std::string val; std::string val;
stringify_value(properties->values[i], type, val, sst->fname, sst->line, feature, properties->keys[i]->string); stringify_value(properties->values[i], type, val, sst->fname, sst->line, feature);
if (type >= 0) { if (type >= 0) {
metakey[m] = properties->keys[i]->string; metakey[m] = properties->keys[i]->string;

6
geojson.hpp
View File

@ -13,10 +13,12 @@ struct parse_json_args {
json_pull *jp; json_pull *jp;
int layer; int layer;
std::string *layername; std::string *layername;
std::map<std::string, int> const *attribute_types;
bool want_dist;
struct serialization_state *sst; struct serialization_state *sst;
parse_json_args(json_pull *jp1, int layer1, std::string *layername1, struct serialization_state *sst1)
: jp(jp1), layer(layer1), layername(layername1), sst(sst1) {
}
}; };
struct json_pull *json_begin_map(char *map, long long len); struct json_pull *json_begin_map(char *map, long long len);

6
geometry.cpp
View File

@ -219,7 +219,7 @@ static void decode_clipped(mapbox::geometry::multi_polygon<long long> &t, drawve
} }
} }
drawvec clean_or_clip_poly(drawvec &geom, int z, int detail, int buffer, bool clip) { drawvec clean_or_clip_poly(drawvec &geom, int z, int buffer, bool clip) {
mapbox::geometry::wagyu::wagyu<long long> wagyu; mapbox::geometry::wagyu::wagyu<long long> wagyu;
geom = remove_noop(geom, VT_POLYGON, 0); geom = remove_noop(geom, VT_POLYGON, 0);
@ -345,7 +345,7 @@ static int pnpoly(drawvec &vert, size_t start, size_t nvert, long long testx, lo
return c; return c;
} }
void check_polygon(drawvec &geom, drawvec &before) { void check_polygon(drawvec &geom) {
geom = remove_noop(geom, VT_POLYGON, 0); geom = remove_noop(geom, VT_POLYGON, 0);
mapbox::geometry::multi_polygon<long long> mp; mapbox::geometry::multi_polygon<long long> mp;
@ -635,7 +635,7 @@ int quick_check(long long *bbox, int z, long long buffer) {
return 2; return 2;
} }
bool point_within_tile(long long x, long long y, int z, long long buffer) { bool point_within_tile(long long x, long long y, int z) {
// No adjustment for buffer, because the point must be // No adjustment for buffer, because the point must be
// strictly within the tile to appear exactly once // strictly within the tile to appear exactly once

26
geometry.hpp
View File

@ -22,18 +22,18 @@ struct draw {
long long y : 40; long long y : 40;
signed char necessary; signed char necessary;
draw(int nop, long long nx, long long ny) { draw(int nop, long long nx, long long ny)
this->op = nop; : x(nx),
this->x = nx; op(nop),
this->y = ny; y(ny),
this->necessary = 0; necessary(0) {
} }
draw() { draw()
this->op = 0; : x(0),
this->x = 0; op(0),
this->y = 0; y(0),
this->necessary = 0; necessary(0) {
} }
bool operator<(draw const &s) const { bool operator<(draw const &s) const {
@ -59,19 +59,19 @@ drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsi
void to_tile_scale(drawvec &geom, int z, int detail); void to_tile_scale(drawvec &geom, int z, int detail);
drawvec remove_noop(drawvec geom, int type, int shift); drawvec remove_noop(drawvec geom, int type, int shift);
drawvec clip_point(drawvec &geom, int z, long long buffer); drawvec clip_point(drawvec &geom, int z, long long buffer);
drawvec clean_or_clip_poly(drawvec &geom, int z, int detail, int buffer, bool clip); drawvec clean_or_clip_poly(drawvec &geom, int z, int buffer, bool clip);
drawvec simple_clip_poly(drawvec &geom, int z, int buffer); drawvec simple_clip_poly(drawvec &geom, int z, int buffer);
drawvec close_poly(drawvec &geom); drawvec close_poly(drawvec &geom);
drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double *accum_area); drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double *accum_area);
drawvec clip_lines(drawvec &geom, int z, long long buffer); drawvec clip_lines(drawvec &geom, int z, long long buffer);
drawvec stairstep(drawvec &geom, int z, int detail); drawvec stairstep(drawvec &geom, int z, int detail);
bool point_within_tile(long long x, long long y, int z, long long buffer); bool point_within_tile(long long x, long long y, int z);
int quick_check(long long *bbox, int z, long long buffer); int quick_check(long long *bbox, int z, long long buffer);
drawvec simplify_lines(drawvec &geom, int z, int detail, bool mark_tile_bounds, double simplification, size_t retain); drawvec simplify_lines(drawvec &geom, int z, int detail, bool mark_tile_bounds, double simplification, size_t retain);
drawvec reorder_lines(drawvec &geom); drawvec reorder_lines(drawvec &geom);
drawvec fix_polygon(drawvec &geom); drawvec fix_polygon(drawvec &geom);
std::vector<drawvec> chop_polygon(std::vector<drawvec> &geoms); std::vector<drawvec> chop_polygon(std::vector<drawvec> &geoms);
void check_polygon(drawvec &geom, drawvec &before); void check_polygon(drawvec &geom);
double get_area(drawvec &geom, size_t i, size_t j); double get_area(drawvec &geom, size_t i, size_t j);
double get_mp_area(drawvec &geom); double get_mp_area(drawvec &geom);

201
main.cpp
View File

@ -63,6 +63,7 @@ static int full_detail = -1;
static int min_detail = 7; static int min_detail = 7;
int quiet = 0; int quiet = 0;
int quiet_progress = 0;
int geometry_scale = 0; int geometry_scale = 0;
double simplification = 1; double simplification = 1;
size_t max_tile_size = 500000; size_t max_tile_size = 500000;
@ -71,8 +72,8 @@ int prevent[256];
int additional[256]; int additional[256];
struct source { struct source {
std::string layer; std::string layer = "";
std::string file; std::string file = "";
}; };
size_t CPUS; size_t CPUS;
@ -80,13 +81,12 @@ size_t TEMP_FILES;
long long MAX_FILES; long long MAX_FILES;
static long long diskfree; static long long diskfree;
void checkdisk(struct reader *r, int nreader) { void checkdisk(std::vector<struct reader> *r) {
long long used = 0; long long used = 0;
int i; for (size_t i = 0; i < r->size(); i++) {
for (i = 0; i < nreader; i++) {
// Meta, pool, and tree are used once. // Meta, pool, and tree are used once.
// Geometry and index will be duplicated during sorting and tiling. // Geometry and index will be duplicated during sorting and tiling.
used += r[i].metapos + 2 * r[i].geompos + 2 * r[i].indexpos + r[i].poolfile->len + r[i].treefile->len; used += (*r)[i].metapos + 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len;
} }
static int warned = 0; static int warned = 0;
@ -167,9 +167,9 @@ int indexcmp(const void *v1, const void *v2) {
const struct index *i1 = (const struct index *) v1; const struct index *i1 = (const struct index *) v1;
const struct index *i2 = (const struct index *) v2; const struct index *i2 = (const struct index *) v2;
if (i1->index < i2->index) { if (i1->ix < i2->ix) {
return -1; return -1;
} else if (i1->index > i2->index) { } else if (i1->ix > i2->ix) {
return 1; return 1;
} }
@ -209,10 +209,10 @@ struct drop_state {
unsigned y; unsigned y;
}; };
int calc_feature_minzoom(struct index *ix, struct drop_state *ds, int maxzoom, int basezoom, double droprate, double gamma) { int calc_feature_minzoom(struct index *ix, struct drop_state *ds, int maxzoom, double gamma) {
int feature_minzoom = 0; int feature_minzoom = 0;
unsigned xx, yy; unsigned xx, yy;
decode(ix->index, &xx, &yy); decode(ix->ix, &xx, &yy);
if (gamma >= 0 && (ix->t == VT_POINT || if (gamma >= 0 && (ix->t == VT_POINT ||
(additional[A_LINE_DROP] && ix->t == VT_LINE) || (additional[A_LINE_DROP] && ix->t == VT_LINE) ||
@ -255,7 +255,7 @@ int calc_feature_minzoom(struct index *ix, struct drop_state *ds, int maxzoom, i
return feature_minzoom; return feature_minzoom;
} }
static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map, FILE *indexfile, int bytes, long long nrec, char *geom_map, FILE *geom_out, long long *geompos, long long *progress, long long *progress_max, long long *progress_reported, int maxzoom, int basezoom, double droprate, double gamma, struct drop_state *ds) { static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map, FILE *indexfile, int bytes, char *geom_map, FILE *geom_out, long long *geompos, long long *progress, long long *progress_max, long long *progress_reported, int maxzoom, double gamma, struct drop_state *ds) {
struct mergelist *head = NULL; struct mergelist *head = NULL;
for (size_t i = 0; i < nmerges; i++) { for (size_t i = 0; i < nmerges; i++) {
@ -269,12 +269,12 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map,
long long pos = *geompos; long long pos = *geompos;
fwrite_check(geom_map + ix.start, 1, ix.end - ix.start, geom_out, "merge geometry"); fwrite_check(geom_map + ix.start, 1, ix.end - ix.start, geom_out, "merge geometry");
*geompos += ix.end - ix.start; *geompos += ix.end - ix.start;
int feature_minzoom = calc_feature_minzoom(&ix, ds, maxzoom, basezoom, droprate, gamma); int feature_minzoom = calc_feature_minzoom(&ix, ds, maxzoom, gamma);
serialize_byte(geom_out, feature_minzoom, geompos, "merge geometry"); serialize_byte(geom_out, feature_minzoom, geompos, "merge geometry");
// Count this as an 75%-accomplishment, since we already 25%-counted it // Count this as an 75%-accomplishment, since we already 25%-counted it
*progress += (ix.end - ix.start) * 3 / 4; *progress += (ix.end - ix.start) * 3 / 4;
if (!quiet && 100 * *progress / *progress_max != *progress_reported) { if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) {
fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
*progress_reported = 100 * *progress / *progress_max; *progress_reported = 100 * *progress / *progress_max;
} }
@ -303,6 +303,10 @@ struct sort_arg {
size_t nmerges; size_t nmerges;
long long unit; long long unit;
int bytes; int bytes;
sort_arg(int task1, int cpus1, long long indexpos1, struct mergelist *merges1, int indexfd1, size_t nmerges1, long long unit1, int bytes1)
: task(task1), cpus(cpus1), indexpos(indexpos1), merges(merges1), indexfd(indexfd1), nmerges(nmerges1), unit(unit1), bytes(bytes1) {
}
}; };
void *run_sort(void *v) { void *run_sort(void *v) {
@ -350,7 +354,7 @@ void *run_sort(void *v) {
return NULL; return NULL;
} }
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > *layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector<struct reader> *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) {
long long segs[CPUS + 1]; long long segs[CPUS + 1];
segs[0] = 0; segs[0] = 0;
segs[CPUS] = len; segs[CPUS] = len;
@ -374,8 +378,11 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
dist_sums[i] = dist_counts[i] = 0; dist_sums[i] = dist_counts[i] = 0;
} }
struct parse_json_args pja[CPUS]; std::vector<parse_json_args> pja;
struct serialization_state sst[CPUS];
std::vector<serialization_state> sst;
sst.resize(CPUS);
pthread_t pthreads[CPUS]; pthread_t pthreads[CPUS];
std::vector<std::set<type_and_string> > file_subkeys; std::vector<std::set<type_and_string> > file_subkeys;
@ -407,12 +414,14 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
sst[i].basezoom = basezoom; sst[i].basezoom = basezoom;
sst[i].attribute_types = attribute_types; sst[i].attribute_types = attribute_types;
pja[i].jp = json_begin_map(map + segs[i], segs[i + 1] - segs[i]); pja.push_back(parse_json_args(
pja[i].layer = source; json_begin_map(map + segs[i], segs[i + 1] - segs[i]),
pja[i].layername = &layername; source,
&layername,
pja[i].sst = &sst[i]; &sst[i]));
}
for (size_t i = 0; i < CPUS; i++) {
if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) { if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) {
perror("pthread_create"); perror("pthread_create");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
@ -434,37 +443,34 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
} }
struct read_parallel_arg { struct read_parallel_arg {
int fd; int fd = 0;
FILE *fp; FILE *fp = NULL;
long long offset; long long offset = 0;
long long len; long long len = 0;
volatile int *is_parsing; volatile int *is_parsing = NULL;
int separator; int separator = 0;
const char *reading; const char *reading = NULL;
struct reader *readers; std::vector<struct reader> *readers = NULL;
volatile long long *progress_seq; volatile long long *progress_seq = NULL;
std::set<std::string> *exclude; std::set<std::string> *exclude = NULL;
std::set<std::string> *include; std::set<std::string> *include = NULL;
int exclude_all; int exclude_all = 0;
json_object *filter; json_object *filter = NULL;
char *fname; int maxzoom = 0;
int maxzoom; int basezoom = 0;
int basezoom; int source = 0;
int source; std::vector<std::map<std::string, layermap_entry> > *layermaps = NULL;
int nlayers; int *initialized = NULL;
std::vector<std::map<std::string, layermap_entry> > *layermaps; unsigned *initial_x = NULL;
double droprate; unsigned *initial_y = NULL;
int *initialized; std::string layername = "";
unsigned *initial_x; bool uses_gamma = false;
unsigned *initial_y; std::map<std::string, int> const *attribute_types = NULL;
std::string layername; double *dist_sum = NULL;
bool uses_gamma; size_t *dist_count = NULL;
std::map<std::string, int> const *attribute_types; bool want_dist = false;
double *dist_sum; bool filters = false;
size_t *dist_count;
bool want_dist;
bool filters;
}; };
void *run_read_parallel(void *v) { void *run_read_parallel(void *v) {
@ -486,7 +492,7 @@ void *run_read_parallel(void *v) {
} }
madvise(map, rpa->len, MADV_RANDOM); // sequential, but from several pointers at once madvise(map, rpa->len, MADV_RANDOM); // sequential, but from several pointers at once
do_read_parallel(map, rpa->len, rpa->offset, rpa->reading, rpa->readers, rpa->progress_seq, rpa->exclude, rpa->include, rpa->exclude_all, rpa->filter, rpa->fname, rpa->basezoom, rpa->source, rpa->nlayers, rpa->layermaps, rpa->droprate, rpa->initialized, rpa->initial_x, rpa->initial_y, rpa->maxzoom, rpa->layername, rpa->uses_gamma, rpa->attribute_types, rpa->separator, rpa->dist_sum, rpa->dist_count, rpa->want_dist, rpa->filters); do_read_parallel(map, rpa->len, rpa->offset, rpa->reading, rpa->readers, rpa->progress_seq, rpa->exclude, rpa->include, rpa->exclude_all, rpa->filter, rpa->basezoom, rpa->source, rpa->layermaps, rpa->initialized, rpa->initial_x, rpa->initial_y, rpa->maxzoom, rpa->layername, rpa->uses_gamma, rpa->attribute_types, rpa->separator, rpa->dist_sum, rpa->dist_count, rpa->want_dist, rpa->filters);
madvise(map, rpa->len, MADV_DONTNEED); madvise(map, rpa->len, MADV_DONTNEED);
if (munmap(map, rpa->len) != 0) { if (munmap(map, rpa->len) != 0) {
@ -503,7 +509,7 @@ void *run_read_parallel(void *v) {
return NULL; return NULL;
} }
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > &layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector<struct reader> *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) {
// This has to kick off an intermediate thread to start the parser threads, // This has to kick off an intermediate thread to start the parser threads,
// so the main thread can get back to reading the next input stage while // so the main thread can get back to reading the next input stage while
// the intermediate thread waits for the completion of the parser threads. // the intermediate thread waits for the completion of the parser threads.
@ -530,12 +536,9 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i
rpa->include = include; rpa->include = include;
rpa->exclude_all = exclude_all; rpa->exclude_all = exclude_all;
rpa->filter = filter; rpa->filter = filter;
rpa->fname = fname;
rpa->basezoom = basezoom; rpa->basezoom = basezoom;
rpa->source = source; rpa->source = source;
rpa->nlayers = nlayers;
rpa->layermaps = &layermaps; rpa->layermaps = &layermaps;
rpa->droprate = droprate;
rpa->initialized = initialized; rpa->initialized = initialized;
rpa->initial_x = initial_x; rpa->initial_x = initial_x;
rpa->initial_y = initial_y; rpa->initial_y = initial_y;
@ -633,7 +636,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
for (size_t a = 0; a < indexst.st_size / sizeof(struct index); a++) { for (size_t a = 0; a < indexst.st_size / sizeof(struct index); a++) {
struct index ix = indexmap[a]; struct index ix = indexmap[a];
unsigned long long which = (ix.index << prefix) >> (64 - splitbits); unsigned long long which = (ix.ix << prefix) >> (64 - splitbits);
long long pos = sub_geompos[which]; long long pos = sub_geompos[which];
fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], "geom"); fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], "geom");
@ -641,7 +644,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
// Count this as a 25%-accomplishment, since we will copy again // Count this as a 25%-accomplishment, since we will copy again
*progress += (ix.end - ix.start) / 4; *progress += (ix.end - ix.start) / 4;
if (!quiet && 100 * *progress / *progress_max != *progress_reported) { if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) {
fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
*progress_reported = 100 * *progress / *progress_max; *progress_reported = 100 * *progress / *progress_max;
} }
@ -728,18 +731,21 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
} }
pthread_t pthreads[CPUS]; pthread_t pthreads[CPUS];
struct sort_arg args[CPUS]; std::vector<sort_arg> args;
for (size_t a = 0; a < CPUS; a++) { for (size_t a = 0; a < CPUS; a++) {
args[a].task = a; args.push_back(sort_arg(
args[a].cpus = CPUS; a,
args[a].indexpos = indexpos; CPUS,
args[a].merges = merges; indexpos,
args[a].indexfd = indexfds[i]; merges,
args[a].nmerges = nmerges; indexfds[i],
args[a].unit = unit; nmerges,
args[a].bytes = bytes; unit,
bytes));
}
for (size_t a = 0; a < CPUS; a++) {
if (pthread_create(&pthreads[a], NULL, run_sort, &args[a]) != 0) { if (pthread_create(&pthreads[a], NULL, run_sort, &args[a]) != 0) {
perror("pthread_create"); perror("pthread_create");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
@ -770,7 +776,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
madvise(geommap, geomst.st_size, MADV_RANDOM); madvise(geommap, geomst.st_size, MADV_RANDOM);
madvise(geommap, geomst.st_size, MADV_WILLNEED); madvise(geommap, geomst.st_size, MADV_WILLNEED);
merge(merges, nmerges, (unsigned char *) indexmap, indexfile, bytes, indexpos / bytes, geommap, geomfile, geompos_out, progress, progress_max, progress_reported, maxzoom, basezoom, droprate, gamma, ds); merge(merges, nmerges, (unsigned char *) indexmap, indexfile, bytes, geommap, geomfile, geompos_out, progress, progress_max, progress_reported, maxzoom, gamma, ds);
madvise(indexmap, indexst.st_size, MADV_DONTNEED); madvise(indexmap, indexst.st_size, MADV_DONTNEED);
if (munmap(indexmap, indexst.st_size) < 0) { if (munmap(indexmap, indexst.st_size) < 0) {
@ -805,12 +811,12 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfile, "geom"); fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfile, "geom");
*geompos_out += ix.end - ix.start; *geompos_out += ix.end - ix.start;
int feature_minzoom = calc_feature_minzoom(&ix, ds, maxzoom, basezoom, droprate, gamma); int feature_minzoom = calc_feature_minzoom(&ix, ds, maxzoom, gamma);
serialize_byte(geomfile, feature_minzoom, geompos_out, "merge geometry"); serialize_byte(geomfile, feature_minzoom, geompos_out, "merge geometry");
// Count this as an 75%-accomplishment, since we already 25%-counted it // Count this as an 75%-accomplishment, since we already 25%-counted it
*progress += (ix.end - ix.start) * 3 / 4; *progress += (ix.end - ix.start) * 3 / 4;
if (!quiet && 100 * *progress / *progress_max != *progress_reported) { if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) {
fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
*progress_reported = 100 * *progress / *progress_max; *progress_reported = 100 * *progress / *progress_max;
} }
@ -877,7 +883,7 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d
} }
} }
void radix(struct reader *readers, int nreaders, FILE *geomfile, int geomfd, FILE *indexfile, int indexfd, const char *tmpdir, long long *geompos, int maxzoom, int basezoom, double droprate, double gamma) { void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, long long *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
// Run through the index and geometry for each reader, // Run through the index and geometry for each reader,
// splitting the contents out by index into as many // splitting the contents out by index into as many
// sub-files as we can write to simultaneously. // sub-files as we can write to simultaneously.
@ -954,7 +960,7 @@ void radix(struct reader *readers, int nreaders, FILE *geomfile, int geomfd, FIL
} }
} }
void choose_first_zoom(long long *file_bbox, struct reader *readers, unsigned *iz, unsigned *ix, unsigned *iy, int minzoom, int buffer) { void choose_first_zoom(long long *file_bbox, std::vector<struct reader> &readers, unsigned *iz, unsigned *ix, unsigned *iy, int minzoom, int buffer) {
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
if (readers[i].file_bbox[0] < file_bbox[0]) { if (readers[i].file_bbox[0] < file_bbox[0]) {
file_bbox[0] = readers[i].file_bbox[0]; file_bbox[0] = readers[i].file_bbox[0];
@ -1008,7 +1014,8 @@ void choose_first_zoom(long long *file_bbox, struct reader *readers, unsigned *i
int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm) { int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm) {
int ret = EXIT_SUCCESS; int ret = EXIT_SUCCESS;
struct reader readers[CPUS]; std::vector<struct reader> readers;
readers.resize(CPUS);
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
struct reader *r = &readers[i]; struct reader *r = &readers[i];
@ -1215,7 +1222,8 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
long long layer_seq[CPUS]; long long layer_seq[CPUS];
double dist_sums[CPUS]; double dist_sums[CPUS];
size_t dist_counts[CPUS]; size_t dist_counts[CPUS];
struct serialization_state sst[CPUS]; std::vector<struct serialization_state> sst;
sst.resize(CPUS);
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
layer_seq[i] = overall_offset; layer_seq[i] = overall_offset;
@ -1226,7 +1234,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
sst[i].line = 0; sst[i].line = 0;
sst[i].layer_seq = &layer_seq[i]; sst[i].layer_seq = &layer_seq[i];
sst[i].progress_seq = &progress_seq; sst[i].progress_seq = &progress_seq;
sst[i].readers = readers; sst[i].readers = &readers;
sst[i].segment = i; sst[i].segment = i;
sst[i].initial_x = &initial_x[i]; sst[i].initial_x = &initial_x[i];
sst[i].initial_y = &initial_y[i]; sst[i].initial_y = &initial_y[i];
@ -1246,7 +1254,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
sst[i].attribute_types = attribute_types; sst[i].attribute_types = attribute_types;
} }
parse_geobuf(sst, map, st.st_size, layer, sources[layer].layer); parse_geobuf(&sst, map, st.st_size, layer, sources[layer].layer);
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
dist_sum += dist_sums[i]; dist_sum += dist_sums[i];
@ -1263,7 +1271,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
overall_offset = layer_seq[0]; overall_offset = layer_seq[0];
checkdisk(readers, CPUS); checkdisk(&readers);
continue; continue;
} }
@ -1304,9 +1312,9 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
if (map != NULL && map != MAP_FAILED && read_parallel_this) { if (map != NULL && map != MAP_FAILED && read_parallel_this) {
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), readers, &progress_seq, exclude, include, exclude_all, filter, fname, basezoom, layer, nlayers, &layermaps, droprate, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, uses_gamma, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, filter, basezoom, layer, &layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, uses_gamma, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL);
overall_offset += st.st_size - off; overall_offset += st.st_size - off;
checkdisk(readers, CPUS); checkdisk(&readers);
if (munmap(map, st.st_size - off) != 0) { if (munmap(map, st.st_size - off) != 0) {
perror("munmap source file"); perror("munmap source file");
@ -1380,11 +1388,11 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
fflush(readfp); fflush(readfp);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), readers, &progress_seq, exclude, include, exclude_all, filter, fname, basezoom, layer, nlayers, layermaps, droprate, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, filter, basezoom, layer, layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL);
initial_offset += ahead; initial_offset += ahead;
overall_offset += ahead; overall_offset += ahead;
checkdisk(readers, CPUS); checkdisk(&readers);
ahead = 0; ahead = 0;
sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX"); sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX");
@ -1417,7 +1425,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
fflush(readfp); fflush(readfp);
if (ahead > 0) { if (ahead > 0) {
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), readers, &progress_seq, exclude, include, exclude_all, filter, fname, basezoom, layer, nlayers, layermaps, droprate, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, filter, basezoom, layer, layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL);
if (parser_created) { if (parser_created) {
if (pthread_join(parallel_parser, NULL) != 0) { if (pthread_join(parallel_parser, NULL) != 0) {
@ -1427,7 +1435,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
overall_offset += ahead; overall_offset += ahead;
checkdisk(readers, CPUS); checkdisk(&readers);
} }
} else { } else {
// Plain serial reading // Plain serial reading
@ -1440,7 +1448,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
sst.line = 0; sst.line = 0;
sst.layer_seq = &layer_seq; sst.layer_seq = &layer_seq;
sst.progress_seq = &progress_seq; sst.progress_seq = &progress_seq;
sst.readers = readers; sst.readers = &readers;
sst.segment = 0; sst.segment = 0;
sst.initial_x = &initial_x[0]; sst.initial_x = &initial_x[0];
sst.initial_y = &initial_y[0]; sst.initial_y = &initial_y[0];
@ -1462,7 +1470,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
parse_json(&sst, jp, layer, sources[layer].layer); parse_json(&sst, jp, layer, sources[layer].layer);
json_end(jp); json_end(jp);
overall_offset = layer_seq; overall_offset = layer_seq;
checkdisk(readers, CPUS); checkdisk(&readers);
} }
if (fclose(fp) != 0) { if (fclose(fp) != 0) {
@ -1654,7 +1662,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
serialize_uint(geomfile, ix, &geompos, fname); serialize_uint(geomfile, ix, &geompos, fname);
serialize_uint(geomfile, iy, &geompos, fname); serialize_uint(geomfile, iy, &geompos, fname);
radix(readers, CPUS, geomfile, geomfd, indexfile, indexfd, tmpdir, &geompos, maxzoom, basezoom, droprate, gamma); radix(readers, CPUS, geomfile, indexfile, tmpdir, &geompos, maxzoom, basezoom, droprate, gamma);
/* end of tile */ /* end of tile */
serialize_byte(geomfile, -2, &geompos, fname); serialize_byte(geomfile, -2, &geompos, fname);
@ -1705,15 +1713,15 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
long long progress = -1; long long progress = -1;
long long ip; long long ip;
for (ip = 1; ip < indices; ip++) { for (ip = 1; ip < indices; ip++) {
if (map[ip].index != map[ip - 1].index) { if (map[ip].ix != map[ip - 1].ix) {
count++; count++;
sum += log(map[ip].index - map[ip - 1].index); sum += log(map[ip].ix - map[ip - 1].ix);
} }
long long nprogress = 100 * ip / indices; long long nprogress = 100 * ip / indices;
if (nprogress != progress) { if (nprogress != progress) {
progress = nprogress; progress = nprogress;
if (!quiet) { if (!quiet && !quiet_progress) {
fprintf(stderr, "Maxzoom: %lld%% \r", progress); fprintf(stderr, "Maxzoom: %lld%% \r", progress);
} }
} }
@ -1804,12 +1812,12 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
long long ip; long long ip;
for (ip = 0; ip < indices; ip++) { for (ip = 0; ip < indices; ip++) {
unsigned xx, yy; unsigned xx, yy;
decode(map[ip].index, &xx, &yy); decode(map[ip].ix, &xx, &yy);
long long nprogress = 100 * ip / indices; long long nprogress = 100 * ip / indices;
if (nprogress != progress) { if (nprogress != progress) {
progress = nprogress; progress = nprogress;
if (!quiet) { if (!quiet && !quiet_progress) {
fprintf(stderr, "Base zoom/drop rate: %lld%% \r", progress); fprintf(stderr, "Base zoom/drop rate: %lld%% \r", progress);
} }
} }
@ -1839,7 +1847,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
tile[z].fullcount++; tile[z].fullcount++;
if (manage_gap(map[ip].index, &tile[z].previndex, scale, gamma, &tile[z].gap)) { if (manage_gap(map[ip].ix, &tile[z].previndex, scale, gamma, &tile[z].gap)) {
continue; continue;
} }
@ -1968,7 +1976,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
if (ip > 0 && map[ip].start != map[ip - 1].end) { if (ip > 0 && map[ip].start != map[ip - 1].end) {
fprintf(stderr, "Mismatched index at %lld: %lld vs %lld\n", ip, map[ip].start, map[ip].end); fprintf(stderr, "Mismatched index at %lld: %lld vs %lld\n", ip, map[ip].start, map[ip].end);
} }
int feature_minzoom = calc_feature_minzoom(&map[ip], ds, maxzoom, basezoom, droprate, gamma); int feature_minzoom = calc_feature_minzoom(&map[ip], ds, maxzoom, gamma);
geom[map[ip].end - 1] = feature_minzoom; geom[map[ip].end - 1] = feature_minzoom;
} }
@ -2002,7 +2010,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
unsigned midx = 0, midy = 0; unsigned midx = 0, midy = 0;
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, outdir, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter); int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter);
if (maxzoom != written) { if (maxzoom != written) {
fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written);
@ -2256,6 +2264,7 @@ int main(int argc, char **argv) {
{"Progress indicator", 0, 0, 0}, {"Progress indicator", 0, 0, 0},
{"quiet", no_argument, 0, 'q'}, {"quiet", no_argument, 0, 'q'},
{"no-progress-indicator", no_argument, 0, 'Q'},
{"version", no_argument, 0, 'v'}, {"version", no_argument, 0, 'v'},
{"", 0, 0, 0}, {"", 0, 0, 0},
@ -2479,6 +2488,10 @@ int main(int argc, char **argv) {
quiet = 1; quiet = 1;
break; break;
case 'Q':
quiet_progress = 1;
break;
case 'p': { case 'p': {
char *cp; char *cp;
for (cp = optarg; *cp != '\0'; cp++) { for (cp = optarg; *cp != '\0'; cp++) {

16
main.hpp
View File

@ -4,18 +4,24 @@
#include <stddef.h> #include <stddef.h>
struct index { struct index {
long long start; long long start = 0;
long long end; long long end = 0;
unsigned long long index; unsigned long long ix = 0;
short segment; short segment = 0;
unsigned short t : 2; unsigned short t : 2;
unsigned long long seq : (64 - 18); // pack with segment and t to stay in 32 bytes unsigned long long seq : (64 - 18); // pack with segment and t to stay in 32 bytes
index()
: t(0),
seq(0) {
}
}; };
void checkdisk(struct reader *r, int nreader); void checkdisk(std::vector<struct reader> *r);
extern int geometry_scale; extern int geometry_scale;
extern int quiet; extern int quiet;
extern int quiet_progress;
extern size_t CPUS; extern size_t CPUS;
extern size_t TEMP_FILES; extern size_t TEMP_FILES;

18
man/tippecanoe.1
View File

@ -93,13 +93,15 @@ delete the file that already exists with that name.
If you aren't sure what the right maxzoom is for your data, \fB\fC\-zg\fR will guess one for you If you aren't sure what the right maxzoom is for your data, \fB\fC\-zg\fR will guess one for you
based on the density of features. based on the density of features.
.PP .PP
If you are mapping point features, you will often want to use \fB\fC\-Bg\fR to automatically choose Tippecanoe will normally drop a fraction of point features at zooms below the maxzoom,
a base zoom level for dot dropping. If that doesn't work out for you, try to keep the low\-zoom tiles from getting too big. If you have a smaller data set where
\fB\fC\-r1 \-\-drop\-fraction\-as\-needed\fR to turn off the normal dot dropping and instead all the points would fit without dropping any of them, use \fB\fC\-r1\fR to keep them all.
only drop features if the tiles get too big. If you do want point dropping, but you still want the tiles to be denser than \fB\fC\-zg\fR
thinks they should be, use \fB\fC\-B\fR to set a basezoom lower than the maxzoom.
.PP .PP
If you are mapping points or polygons, you will often want to use \fB\fC\-\-drop\-densest\-as\-needed\fR If some of your tiles are coming out too big in spite of the settings above, you will
to drop some of them if necessary to make the low zoom levels work. often want to use \fB\fC\-\-drop\-densest\-as\-needed\fR to drop whatever fraction of the features
is necessary at each zoom level to make that zoom level's tiles work.
.PP .PP
If your features have a lot of attributes, use \fB\fC\-y\fR to keep only the ones you really need. If your features have a lot of attributes, use \fB\fC\-y\fR to keep only the ones you really need.
.PP .PP
@ -319,7 +321,9 @@ If you don't specify, it will use \fB\fC/tmp\fR\&.
.SS Progress indicator .SS Progress indicator
.RS .RS
.IP \(bu 2 .IP \(bu 2
\fB\fC\-q\fR or \fB\fC\-\-quiet\fR: Work quietly instead of reporting progress \fB\fC\-q\fR or \fB\fC\-\-quiet\fR: Work quietly instead of reporting progress or warning messages
.IP \(bu 2
\fB\fC\-Q\fR or \fB\fC\-\-no\-progress\-indicator\fR: Don't report progress, but still give warnings
.IP \(bu 2 .IP \(bu 2
\fB\fC\-v\fR or \fB\fC\-\-version\fR: Report Tippecanoe's version number \fB\fC\-v\fR or \fB\fC\-\-version\fR: Report Tippecanoe's version number
.RE .RE

14
mbtiles.hpp
View File

@ -6,25 +6,25 @@
#include "mvt.hpp" #include "mvt.hpp"
struct type_and_string { struct type_and_string {
int type; int type = 0;
std::string string; std::string string = "";
bool operator<(const type_and_string &o) const; bool operator<(const type_and_string &o) const;
bool operator!=(const type_and_string &o) const; bool operator!=(const type_and_string &o) const;
}; };
struct type_and_string_stats { struct type_and_string_stats {
std::vector<type_and_string> sample_values; // sorted std::vector<type_and_string> sample_values = std::vector<type_and_string>(); // sorted
double min = INFINITY; double min = INFINITY;
double max = -INFINITY; double max = -INFINITY;
int type = 0; int type = 0;
}; };
struct layermap_entry { struct layermap_entry {
size_t id; size_t id = 0;
std::map<std::string, type_and_string_stats> file_keys; std::map<std::string, type_and_string_stats> file_keys{};
int minzoom; int minzoom = 0;
int maxzoom; int maxzoom = 0;
size_t points = 0; size_t points = 0;
size_t lines = 0; size_t lines = 0;

10
memfile.hpp
View File

@ -2,11 +2,11 @@
#define MEMFILE_HPP #define MEMFILE_HPP
struct memfile { struct memfile {
int fd; int fd = 0;
char *map; char *map = NULL;
long long len; long long len = 0;
long long off; long long off = 0;
unsigned long tree; unsigned long tree = 0;
}; };
struct memfile *memfile_open(int fd); struct memfile *memfile_open(int fd);

40
mvt.hpp
View File

@ -17,9 +17,9 @@ enum mvt_operation {
}; };
struct mvt_geometry { struct mvt_geometry {
long long x; long long x = 0;
long long y; long long y = 0;
int /* mvt_operation */ op; int /* mvt_operation */ op = 0;
mvt_geometry(int op, long long x, long long y); mvt_geometry(int op, long long x, long long y);
@ -43,11 +43,11 @@ enum mvt_geometry_type {
}; };
struct mvt_feature { struct mvt_feature {
std::vector<unsigned> tags; std::vector<unsigned> tags{};
std::vector<mvt_geometry> geometry; std::vector<mvt_geometry> geometry{};
int /* mvt_geometry_type */ type; int /* mvt_geometry_type */ type = 0;
unsigned long long id; unsigned long long id = 0;
bool has_id; bool has_id = false;
mvt_feature() { mvt_feature() {
has_id = false; has_id = false;
@ -80,26 +80,32 @@ struct mvt_value {
bool operator<(const mvt_value &o) const; bool operator<(const mvt_value &o) const;
std::string toString(); std::string toString();
mvt_value() {
this->type = mvt_double;
this->string_value = "";
this->numeric_value.double_value = 0;
}
}; };
struct mvt_layer { struct mvt_layer {
int version; int version = 0;
std::string name; std::string name = "";
std::vector<mvt_feature> features; std::vector<mvt_feature> features{};
std::vector<std::string> keys; std::vector<std::string> keys{};
std::vector<mvt_value> values; std::vector<mvt_value> values{};
long long extent; long long extent = 0;
// Add a key-value pair to a feature, using this layer's constant pool // Add a key-value pair to a feature, using this layer's constant pool
void tag(mvt_feature &feature, std::string key, mvt_value value); void tag(mvt_feature &feature, std::string key, mvt_value value);
// For tracking the key-value constants already used in this layer // For tracking the key-value constants already used in this layer
std::map<std::string, size_t> key_map; std::map<std::string, size_t> key_map{};
std::map<mvt_value, size_t> value_map; std::map<mvt_value, size_t> value_map{};
}; };
struct mvt_tile { struct mvt_tile {
std::vector<mvt_layer> layers; std::vector<mvt_layer> layers{};
std::string encode(); std::string encode();
bool decode(std::string &message, bool &was_compressed); bool decode(std::string &message, bool &was_compressed);

6
plugin.cpp
View File

@ -198,7 +198,7 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
} }
if (mb_geometry[t] == VT_POLYGON) { if (mb_geometry[t] == VT_POLYGON) {
dv = clean_or_clip_poly(dv, 0, 0, 0, false); dv = clean_or_clip_poly(dv, 0, 0, false);
if (dv.size() < 3) { if (dv.size() < 3) {
dv.clear(); dv.clear();
} }
@ -257,7 +257,7 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
int tp = -1; int tp = -1;
std::string s; std::string s;
stringify_value(properties->values[i], tp, s, "Filter output", jp->line, j, ""); stringify_value(properties->values[i], tp, s, "Filter output", jp->line, j);
if (tp >= 0) { if (tp >= 0) {
mvt_value v = stringified_to_mvt_value(tp, s.c_str()); mvt_value v = stringified_to_mvt_value(tp, s.c_str());
l->second.tag(feature, std::string(properties->keys[i]->string), v); l->second.tag(feature, std::string(properties->keys[i]->string), v);
@ -486,7 +486,7 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
serial_val v; serial_val v;
v.type = -1; v.type = -1;
stringify_value(properties->values[i], v.type, v.s, "Filter output", jp->line, j, ""); stringify_value(properties->values[i], v.type, v.s, "Filter output", jp->line, j);
if (v.type >= 0) { if (v.type >= 0) {
sf.full_keys.push_back(std::string(properties->keys[i]->string)); sf.full_keys.push_back(std::string(properties->keys[i]->string));

6
pool.hpp
View File

@ -2,9 +2,9 @@
#define POOL_HPP #define POOL_HPP
struct stringpool { struct stringpool {
unsigned long left; unsigned long left = 0;
unsigned long right; unsigned long right = 0;
unsigned long off; unsigned long off = 0;
}; };
long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type); long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type);

2
projection.cpp
View File

@ -7,7 +7,7 @@
struct projection projections[] = { struct projection projections[] = {
{"EPSG:4326", lonlat2tile, tile2lonlat, "urn:ogc:def:crs:OGC:1.3:CRS84"}, {"EPSG:4326", lonlat2tile, tile2lonlat, "urn:ogc:def:crs:OGC:1.3:CRS84"},
{"EPSG:3857", epsg3857totile, tiletoepsg3857, "urn:ogc:def:crs:EPSG::3857"}, {"EPSG:3857", epsg3857totile, tiletoepsg3857, "urn:ogc:def:crs:EPSG::3857"},
{NULL, NULL}, {NULL, NULL, NULL, NULL},
}; };
struct projection *projection = &projections[0]; struct projection *projection = &projections[0];

2
read_json.cpp
View File

@ -129,7 +129,7 @@ void canonicalize(json_object *o) {
} }
} }
void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key) { void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature) {
if (value != NULL) { if (value != NULL) {
int vt = value->type; int vt = value->type;
std::string val; std::string val;

2
read_json.hpp
View File

@ -13,4 +13,4 @@ extern int mb_geometry[GEOM_TYPES];
void json_context(json_object *j); void json_context(json_object *j);
void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature); void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature);
void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key); void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature);

8
serial.cpp
View File

@ -379,7 +379,7 @@ static long long scale_geometry(struct serialization_state *sst, long long *bbox
} }
int serialize_feature(struct serialization_state *sst, serial_feature &sf) { int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
struct reader *r = &(sst->readers[sst->segment]); struct reader *r = &(*sst->readers)[sst->segment];
sf.bbox[0] = LLONG_MAX; sf.bbox[0] = LLONG_MAX;
sf.bbox[1] = LLONG_MAX; sf.bbox[1] = LLONG_MAX;
@ -611,7 +611,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
index.segment = sst->segment; index.segment = sst->segment;
index.seq = *(sst->layer_seq); index.seq = *(sst->layer_seq);
index.t = sf.t; index.t = sf.t;
index.index = bbox_index; index.ix = bbox_index;
fwrite_check(&index, sizeof(struct index), 1, r->indexfile, sst->fname); fwrite_check(&index, sizeof(struct index), 1, r->indexfile, sst->fname);
r->indexpos += sizeof(struct index); r->indexpos += sizeof(struct index);
@ -628,8 +628,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
} }
if (*(sst->progress_seq) % 10000 == 0) { if (*(sst->progress_seq) % 10000 == 0) {
checkdisk(sst->readers, CPUS); checkdisk(sst->readers);
if (!quiet) { if (!quiet && !quiet_progress) {
fprintf(stderr, "Read %.2f million features\r", *sst->progress_seq / 1000000.0); fprintf(stderr, "Read %.2f million features\r", *sst->progress_seq / 1000000.0);
} }
} }

126
serial.hpp
View File

@ -31,102 +31,102 @@ int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos);
int deserialize_byte_io(FILE *f, signed char *n, long long *geompos); int deserialize_byte_io(FILE *f, signed char *n, long long *geompos);
struct serial_val { struct serial_val {
int type; int type = 0;
std::string s; std::string s = "";
}; };
struct serial_feature { struct serial_feature {
long long layer; long long layer = 0;
int segment; int segment = 0;
long long seq; long long seq = 0;
signed char t; signed char t = 0;
signed char feature_minzoom; signed char feature_minzoom = 0;
bool has_id; bool has_id = false;
unsigned long long id; unsigned long long id = 0;
bool has_tippecanoe_minzoom; bool has_tippecanoe_minzoom = false;
int tippecanoe_minzoom; int tippecanoe_minzoom = 0;
bool has_tippecanoe_maxzoom; bool has_tippecanoe_maxzoom = false;
int tippecanoe_maxzoom; int tippecanoe_maxzoom = 0;
drawvec geometry; drawvec geometry = drawvec();
unsigned long long index; unsigned long long index = 0;
long long extent; long long extent = 0;
size_t m; size_t m = 0;
std::vector<long long> keys; std::vector<long long> keys{};
std::vector<long long> values; std::vector<long long> values{};
long long metapos; long long metapos = 0;
// XXX This isn't serialized. Should it be here? // XXX This isn't serialized. Should it be here?
long long bbox[4]; long long bbox[4] = {0, 0, 0, 0};
std::vector<std::string> full_keys; std::vector<std::string> full_keys{};
std::vector<serial_val> full_values; std::vector<serial_val> full_values{};
std::string layername; std::string layername = "";
}; };
void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, const char *fname, long long wx, long long wy, bool include_minzoom); void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, const char *fname, long long wx, long long wy, bool include_minzoom);
serial_feature deserialize_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y); serial_feature deserialize_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
struct reader { struct reader {
int metafd; int metafd = -1;
int poolfd; int poolfd = -1;
int treefd; int treefd = -1;
int geomfd; int geomfd = -1;
int indexfd; int indexfd = -1;
FILE *metafile; FILE *metafile = NULL;
struct memfile *poolfile; struct memfile *poolfile = NULL;
struct memfile *treefile; struct memfile *treefile = NULL;
FILE *geomfile; FILE *geomfile = NULL;
FILE *indexfile; FILE *indexfile = NULL;
long long metapos; long long metapos = 0;
long long geompos; long long geompos = 0;
long long indexpos; long long indexpos = 0;
long long file_bbox[4]; long long file_bbox[4] = {0, 0, 0, 0};
struct stat geomst; struct stat geomst {};
struct stat metast; struct stat metast {};
char *geom_map; char *geom_map = NULL;
}; };
struct serialization_state { struct serialization_state {
const char *fname; // source file name const char *fname = NULL; // source file name
int line; // user-oriented location within source for error reports int line = 0; // user-oriented location within source for error reports
volatile long long *layer_seq; // sequence within current layer volatile long long *layer_seq = NULL; // sequence within current layer
volatile long long *progress_seq; // overall sequence for progress indicator volatile long long *progress_seq = NULL; // overall sequence for progress indicator
struct reader *readers; // array of data for each input thread std::vector<struct reader> *readers = NULL; // array of data for each input thread
int segment; // the current input thread int segment = 0; // the current input thread
unsigned *initial_x; // relative offset of all geometries unsigned *initial_x = NULL; // relative offset of all geometries
unsigned *initial_y; unsigned *initial_y = NULL;
int *initialized; int *initialized = NULL;
double *dist_sum; // running tally for calculation of resolution within features double *dist_sum = NULL; // running tally for calculation of resolution within features
size_t *dist_count; size_t *dist_count = NULL;
bool want_dist; bool want_dist = false;
int maxzoom; int maxzoom = 0;
int basezoom; int basezoom = 0;
bool filters; bool filters = false;
bool uses_gamma; bool uses_gamma = false;
std::map<std::string, layermap_entry> *layermap; std::map<std::string, layermap_entry> *layermap = NULL;
std::map<std::string, int> const *attribute_types; std::map<std::string, int> const *attribute_types = NULL;
std::set<std::string> *exclude; std::set<std::string> *exclude = NULL;
std::set<std::string> *include; std::set<std::string> *include = NULL;
int exclude_all; int exclude_all = 0;
json_object *filter; json_object *filter = NULL;
}; };
int serialize_feature(struct serialization_state *sst, serial_feature &sf); int serialize_feature(struct serialization_state *sst, serial_feature &sf);

56
tile-join.cpp
View File

@ -322,20 +322,20 @@ double max(double a, double b) {
} }
struct reader { struct reader {
long long zoom; long long zoom = 0;
long long x; long long x = 0;
long long sorty; long long sorty = 0;
long long y; long long y = 0;
int pbf_count; int pbf_count = 0;
int z_flag; int z_flag = 0;
std::string data; std::string data = "";
std::vector<std::string> pbf_path; std::vector<std::string> pbf_path{};
std::vector<std::string> large_zoom; std::vector<std::string> large_zoom{};
sqlite3 *db; sqlite3 *db = NULL;
sqlite3_stmt *stmt; sqlite3_stmt *stmt = NULL;
struct reader *next; struct reader *next = NULL;
bool operator<(const struct reader &r) const { bool operator<(const struct reader &r) const {
if (zoom < r.zoom) { if (zoom < r.zoom) {
@ -569,10 +569,10 @@ struct zxy {
long long x; long long x;
long long y; long long y;
zxy(long long _z, long long _x, long long _y) { zxy(long long _z, long long _x, long long _y)
z = _z; : z(_z),
x = _x; x(_x),
y = _y; y(_y) {
} }
bool operator<(zxy const &other) const { bool operator<(zxy const &other) const {
@ -599,18 +599,18 @@ struct zxy {
}; };
struct arg { struct arg {
std::map<zxy, std::vector<std::string>> inputs; std::map<zxy, std::vector<std::string>> inputs{};
std::map<zxy, std::string> outputs; std::map<zxy, std::string> outputs{};
std::map<std::string, layermap_entry> *layermap; std::map<std::string, layermap_entry> *layermap = NULL;
std::vector<std::string> *header; std::vector<std::string> *header = NULL;
std::map<std::string, std::vector<std::string>> *mapping; std::map<std::string, std::vector<std::string>> *mapping = NULL;
std::set<std::string> *exclude; std::set<std::string> *exclude = NULL;
std::set<std::string> *keep_layers; std::set<std::string> *keep_layers = NULL;
std::set<std::string> *remove_layers; std::set<std::string> *remove_layers = NULL;
int ifmatched; int ifmatched = 0;
json_object *filter; json_object *filter = NULL;
}; };
void *join_worker(void *v) { void *join_worker(void *v) {
@ -711,7 +711,7 @@ void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<st
} }
} }
void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name, json_object *filter) { void decode(struct reader *readers, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name, json_object *filter) {
std::vector<std::map<std::string, layermap_entry>> layermaps; std::vector<std::map<std::string, layermap_entry>> layermaps;
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
layermaps.push_back(std::map<std::string, layermap_entry>()); layermaps.push_back(std::map<std::string, layermap_entry>());
@ -1205,7 +1205,7 @@ int main(int argc, char **argv) {
*rr = r; *rr = r;
} }
decode(readers, csv, layermap, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name, filter); decode(readers, layermap, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name, filter);
if (set_attribution.size() != 0) { if (set_attribution.size() != 0) {
attribution = set_attribution; attribution = set_attribution;

311
tile.cpp
View File

@ -76,21 +76,21 @@ int metacmp(int m1, const std::vector<long long> &keys1, const std::vector<long
int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2); int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2);
struct coalesce { struct coalesce {
char *stringpool; char *stringpool = NULL;
std::vector<long long> keys; std::vector<long long> keys = std::vector<long long>();
std::vector<long long> values; std::vector<long long> values = std::vector<long long>();
std::vector<std::string> full_keys; std::vector<std::string> full_keys = std::vector<std::string>();
std::vector<serial_val> full_values; std::vector<serial_val> full_values = std::vector<serial_val>();
drawvec geom; drawvec geom = drawvec();
unsigned long long index; unsigned long long index = 0;
unsigned long long index2; unsigned long long index2 = 0;
long long original_seq; long long original_seq = 0;
int type; int type = 0;
int m; int m = 0;
bool coalesced; bool coalesced = false;
double spacing; double spacing = 0;
bool has_id; bool has_id = false;
unsigned long long id; unsigned long long id = 0;
bool operator<(const coalesce &o) const { bool operator<(const coalesce &o) const {
int cmp = coalindexcmp(this, &o); int cmp = coalindexcmp(this, &o);
@ -200,7 +200,7 @@ int metacmp(int m1, const std::vector<long long> &keys1, const std::vector<long
} }
} }
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) { void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) {
if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) { if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) {
int xo, yo; int xo, yo;
int span = 1 << (nextzoom - z); int span = 1 << (nextzoom - z);
@ -309,34 +309,34 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
} }
struct partial { struct partial {
std::vector<drawvec> geoms; std::vector<drawvec> geoms = std::vector<drawvec>();
std::vector<long long> keys; std::vector<long long> keys = std::vector<long long>();
std::vector<long long> values; std::vector<long long> values = std::vector<long long>();
std::vector<std::string> full_keys; std::vector<std::string> full_keys = std::vector<std::string>();
std::vector<serial_val> full_values; std::vector<serial_val> full_values = std::vector<serial_val>();
std::vector<ssize_t> arc_polygon; std::vector<ssize_t> arc_polygon = std::vector<ssize_t>();
long long layer; long long layer = 0;
long long original_seq; long long original_seq = 0;
unsigned long long index; unsigned long long index = 0;
unsigned long long index2; unsigned long long index2 = 0;
int m; int m = 0;
int segment; int segment = 0;
bool reduced; bool reduced = 0;
int z; int z = 0;
int line_detail; int line_detail = 0;
int maxzoom; int maxzoom = 0;
double spacing; double spacing = 0;
double simplification; double simplification = 0;
signed char t; signed char t = 0;
unsigned long long id; unsigned long long id = 0;
bool has_id; bool has_id = 0;
ssize_t renamed; ssize_t renamed = 0;
}; };
struct partial_arg { struct partial_arg {
std::vector<struct partial> *partials; std::vector<struct partial> *partials = NULL;
int task; int task = 0;
int tasks; int tasks = 0;
}; };
drawvec revive_polygon(drawvec &geom, double area, int z, int detail) { drawvec revive_polygon(drawvec &geom, double area, int z, int detail) {
@ -442,9 +442,9 @@ void *partial_feature_worker(void *v) {
// Give Clipper a chance to try to fix it. // Give Clipper a chance to try to fix it.
for (size_t g = 0; g < geoms.size(); g++) { for (size_t g = 0; g < geoms.size(); g++) {
drawvec before = geoms[g]; drawvec before = geoms[g];
geoms[g] = clean_or_clip_poly(geoms[g], 0, 0, 0, false); geoms[g] = clean_or_clip_poly(geoms[g], 0, 0, false);
if (additional[A_DEBUG_POLYGON]) { if (additional[A_DEBUG_POLYGON]) {
check_polygon(geoms[g], before); check_polygon(geoms[g]);
} }
if (geoms[g].size() < 3) { if (geoms[g].size() < 3) {
@ -523,11 +523,11 @@ static drawvec reverse_subring(drawvec const &dv) {
} }
struct edge { struct edge {
unsigned x1; unsigned x1 = 0;
unsigned y1; unsigned y1 = 0;
unsigned x2; unsigned x2 = 0;
unsigned y2; unsigned y2 = 0;
unsigned ring; unsigned ring = 0;
edge(unsigned _x1, unsigned _y1, unsigned _x2, unsigned _y2, unsigned _ring) { edge(unsigned _x1, unsigned _y1, unsigned _x2, unsigned _y2, unsigned _ring) {
x1 = _x1; x1 = _x1;
@ -872,10 +872,10 @@ bool find_common_edges(std::vector<partial> &partials, int z, int line_detail, d
// If necessary, merge some adjacent polygons into some other polygons // If necessary, merge some adjacent polygons into some other polygons
struct merge_order { struct merge_order {
ssize_t edge; ssize_t edge = 0;
unsigned long long gap; unsigned long long gap = 0;
size_t p1; size_t p1 = 0;
size_t p2; size_t p2 = 0;
bool operator<(const merge_order &m) const { bool operator<(const merge_order &m) const {
return gap < m.gap; return gap < m.gap;
@ -1127,53 +1127,51 @@ long long choose_minextent(std::vector<long long> &extents, double f) {
} }
struct write_tile_args { struct write_tile_args {
struct task *tasks; struct task *tasks = NULL;
char *metabase; char *metabase = NULL;
char *stringpool; char *stringpool = NULL;
int min_detail; int min_detail = 0;
int basezoom; sqlite3 *outdb = NULL;
sqlite3 *outdb; const char *outdir = NULL;
const char *outdir; int buffer = 0;
double droprate; const char *fname = NULL;
int buffer; FILE **geomfile = NULL;
const char *fname; double todo = 0;
FILE **geomfile; volatile long long *along = NULL;
double todo; double gamma = 0;
volatile long long *along; double gamma_out = 0;
double gamma; int child_shards = 0;
double gamma_out; int *geomfd = NULL;
int child_shards; off_t *geom_size = NULL;
int *geomfd; volatile unsigned *midx = NULL;
off_t *geom_size; volatile unsigned *midy = NULL;
volatile unsigned *midx; int maxzoom = 0;
volatile unsigned *midy; int minzoom = 0;
int maxzoom; int full_detail = 0;
int minzoom; int low_detail = 0;
int full_detail; double simplification = 0;
int low_detail; volatile long long *most = NULL;
double simplification; long long *meta_off = NULL;
volatile long long *most; long long *pool_off = NULL;
long long *meta_off; unsigned *initial_x = NULL;
long long *pool_off; unsigned *initial_y = NULL;
unsigned *initial_x; volatile int *running = NULL;
unsigned *initial_y; int err = 0;
volatile int *running; std::vector<std::map<std::string, layermap_entry>> *layermaps = NULL;
int err; std::vector<std::vector<std::string>> *layer_unmaps = NULL;
std::vector<std::map<std::string, layermap_entry>> *layermaps; size_t pass = 0;
std::vector<std::vector<std::string>> *layer_unmaps; size_t passes = 0;
size_t pass; unsigned long long mingap = 0;
size_t passes; unsigned long long mingap_out = 0;
unsigned long long mingap; long long minextent = 0;
unsigned long long mingap_out; long long minextent_out = 0;
long long minextent; double fraction = 0;
long long minextent_out; double fraction_out = 0;
double fraction; const char *prefilter = NULL;
double fraction_out; const char *postfilter = NULL;
const char *prefilter; bool still_dropping = false;
const char *postfilter; int wrote_zoom = 0;
bool still_dropping; size_t tiling_seg = 0;
int wrote_zoom;
size_t tiling_seg;
}; };
bool clip_to_tile(serial_feature &sf, int z, long long buffer) { bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
@ -1233,7 +1231,7 @@ bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
// that are duplicated across the date line // that are duplicated across the date line
if (prevent[P_DUPLICATION] && z != 0) { if (prevent[P_DUPLICATION] && z != 0) {
if (point_within_tile((sf.bbox[0] + sf.bbox[2]) / 2, (sf.bbox[1] + sf.bbox[3]) / 2, z, buffer)) { if (point_within_tile((sf.bbox[0] + sf.bbox[2]) / 2, (sf.bbox[1] + sf.bbox[3]) / 2, z)) {
// sf.geometry is unchanged // sf.geometry is unchanged
} else { } else {
sf.geometry.clear(); sf.geometry.clear();
@ -1252,7 +1250,7 @@ bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
return false; return false;
} }
serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, volatile long long *along, long long alongminus, int buffer, int *within, bool *first_time, int line_detail, FILE **geomfile, long long *geompos, volatile double *oprogress, double todo, const char *fname, int child_shards) { serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, volatile long long *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, long long *geompos, volatile double *oprogress, double todo, const char *fname, int child_shards) {
while (1) { while (1) {
serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y); serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y);
if (sf.t < 0) { if (sf.t < 0) {
@ -1261,7 +1259,7 @@ serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase,
double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10; double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10;
if (progress >= *oprogress + 0.1) { if (progress >= *oprogress + 0.1) {
if (!quiet) { if (!quiet && !quiet_progress) {
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
} }
*oprogress = progress; *oprogress = progress;
@ -1279,7 +1277,7 @@ serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase,
if (*first_time && pass == 1) { /* only write out the next zoom once, even if we retry */ if (*first_time && pass == 1) { /* only write out the next zoom once, even if we retry */
if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) { if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) {
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.m, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent); rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.m, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent);
} }
} }
@ -1302,46 +1300,45 @@ serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase,
} }
struct run_prefilter_args { struct run_prefilter_args {
FILE *geoms; FILE *geoms = NULL;
long long *geompos_in; long long *geompos_in = NULL;
char *metabase; char *metabase = NULL;
long long *meta_off; long long *meta_off = NULL;
int z; int z = 0;
unsigned tx; unsigned tx = 0;
unsigned ty; unsigned ty = 0;
unsigned *initial_x; unsigned *initial_x = 0;
unsigned *initial_y; unsigned *initial_y = 0;
long long *original_features; long long *original_features = 0;
long long *unclipped_features; long long *unclipped_features = 0;
int nextzoom; int nextzoom = 0;
int maxzoom; int maxzoom = 0;
int minzoom; int minzoom = 0;
int max_zoom_increment; int max_zoom_increment = 0;
size_t pass; size_t pass = 0;
size_t passes; size_t passes = 0;
volatile long long *along; volatile long long *along = 0;
long long alongminus; long long alongminus = 0;
int buffer; int buffer = 0;
int *within; int *within = NULL;
bool *first_time; bool *first_time = NULL;
int line_detail; FILE **geomfile = NULL;
FILE **geomfile; long long *geompos = NULL;
long long *geompos; volatile double *oprogress = NULL;
volatile double *oprogress; double todo = 0;
double todo; const char *fname = 0;
const char *fname; int child_shards = 0;
int child_shards; std::vector<std::vector<std::string>> *layer_unmaps = NULL;
std::vector<std::vector<std::string>> *layer_unmaps; char *stringpool = NULL;
char *stringpool; long long *pool_off = NULL;
long long *pool_off; FILE *prefilter_fp = NULL;
FILE *prefilter_fp;
}; };
void *run_prefilter(void *v) { void *run_prefilter(void *v) {
run_prefilter_args *rpa = (run_prefilter_args *) v; run_prefilter_args *rpa = (run_prefilter_args *) v;
while (1) { while (1) {
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->line_detail, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards); serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards);
if (sf.t < 0) { if (sf.t < 0) {
break; break;
} }
@ -1392,7 +1389,7 @@ void *run_prefilter(void *v) {
return NULL; return NULL;
} }
long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) { long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) {
int line_detail; int line_detail;
double merge_fraction = 1; double merge_fraction = 1;
double mingap_fraction = 1; double mingap_fraction = 1;
@ -1497,7 +1494,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
rpa.buffer = buffer; rpa.buffer = buffer;
rpa.within = within; rpa.within = within;
rpa.first_time = &first_time; rpa.first_time = &first_time;
rpa.line_detail = line_detail;
rpa.geomfile = geomfile; rpa.geomfile = geomfile;
rpa.geompos = geompos; rpa.geompos = geompos;
rpa.oprogress = &oprogress; rpa.oprogress = &oprogress;
@ -1526,7 +1522,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
serial_feature sf; serial_feature sf;
if (prefilter == NULL) { if (prefilter == NULL) {
sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, line_detail, geomfile, geompos, &oprogress, todo, fname, child_shards); sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards);
} else { } else {
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
} }
@ -1684,7 +1680,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
} }
pthread_t pthreads[tasks]; pthread_t pthreads[tasks];
partial_arg args[tasks]; std::vector<partial_arg> args;
args.resize(tasks);
for (int i = 0; i < tasks; i++) { for (int i = 0; i < tasks; i++) {
args[i].task = i; args[i].task = i;
args[i].tasks = tasks; args[i].tasks = tasks;
@ -1809,7 +1806,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
if (layer_features[x].type == VT_POLYGON) { if (layer_features[x].type == VT_POLYGON) {
if (layer_features[x].coalesced) { if (layer_features[x].coalesced) {
layer_features[x].geom = clean_or_clip_poly(layer_features[x].geom, 0, 0, 0, false); layer_features[x].geom = clean_or_clip_poly(layer_features[x].geom, 0, 0, false);
} }
layer_features[x].geom = close_poly(layer_features[x].geom); layer_features[x].geom = close_poly(layer_features[x].geom);
@ -1901,7 +1898,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10; double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10;
if (progress >= oprogress + 0.1) { if (progress >= oprogress + 0.1) {
if (!quiet) { if (!quiet && !quiet_progress) {
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
} }
oprogress = progress; oprogress = progress;
@ -2094,8 +2091,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
} }
struct task { struct task {
int fileno; int fileno = 0;
struct task *next; struct task *next = NULL;
}; };
void *run_thread(void *vargs) { void *run_thread(void *vargs) {
@ -2138,7 +2135,7 @@ void *run_thread(void *vargs) {
// fprintf(stderr, "%d/%u/%u\n", z, x, y); // fprintf(stderr, "%d/%u/%u\n", z, x, y);
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->outdir, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg); long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg);
if (len < 0) { if (len < 0) {
int *err = &arg->err; int *err = &arg->err;
@ -2203,7 +2200,7 @@ void *run_thread(void *vargs) {
return NULL; return NULL;
} }
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter) { int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter) {
// The existing layermaps are one table per input thread. // The existing layermaps are one table per input thread.
// We need to add another one per *tiling* thread so that it can be // We need to add another one per *tiling* thread so that it can be
// safely changed during tiling. // safely changed during tiling.
@ -2281,12 +2278,17 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
// Assign temporary files to threads // Assign temporary files to threads
struct task tasks[TEMP_FILES]; std::vector<struct task> tasks;
tasks.resize(TEMP_FILES);
struct dispatch { struct dispatch {
struct task *tasks; struct task *tasks = NULL;
long long todo; long long todo = 0;
struct dispatch *next; struct dispatch *next = NULL;
} dispatches[threads]; };
std::vector<struct dispatch> dispatches;
dispatches.resize(threads);
struct dispatch *dispatch_head = &dispatches[0]; struct dispatch *dispatch_head = &dispatches[0];
for (size_t j = 0; j < threads; j++) { for (size_t j = 0; j < threads; j++) {
dispatches[j].tasks = NULL; dispatches[j].tasks = NULL;
@ -2336,7 +2338,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
for (size_t pass = start; pass < 2; pass++) { for (size_t pass = start; pass < 2; pass++) {
pthread_t pthreads[threads]; pthread_t pthreads[threads];
write_tile_args args[threads]; std::vector<write_tile_args> args;
args.resize(threads);
int running = threads; int running = threads;
long long along = 0; long long along = 0;
@ -2344,10 +2347,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
args[thread].metabase = metabase; args[thread].metabase = metabase;
args[thread].stringpool = stringpool; args[thread].stringpool = stringpool;
args[thread].min_detail = min_detail; args[thread].min_detail = min_detail;
args[thread].basezoom = basezoom;
args[thread].outdb = outdb; // locked with db_lock args[thread].outdb = outdb; // locked with db_lock
args[thread].outdir = outdir; args[thread].outdir = outdir;
args[thread].droprate = droprate;
args[thread].buffer = buffer; args[thread].buffer = buffer;
args[thread].fname = fname; args[thread].fname = fname;
args[thread].geomfile = sub + thread * (TEMP_FILES / threads); args[thread].geomfile = sub + thread * (TEMP_FILES / threads);

2
tile.hpp
View File

@ -9,7 +9,7 @@
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers); long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter); int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);

2
version.hpp
View File

@ -1,6 +1,6 @@
#ifndef VERSION_HPP #ifndef VERSION_HPP
#define VERSION_HPP #define VERSION_HPP
#define VERSION "tippecanoe v1.26.6\n" #define VERSION "tippecanoe v1.26.7\n"
#endif #endif

12
write_json.cpp
View File

@ -16,12 +16,12 @@ struct lonlat {
long long x; long long x;
long long y; long long y;
lonlat(int nop, double nlon, double nlat, long long nx, long long ny) { lonlat(int nop, double nlon, double nlat, long long nx, long long ny)
this->op = nop; : op(nop),
this->lon = nlon; lon(nlon),
this->lat = nlat; lat(nlat),
this->x = nx; x(nx),
this->y = ny; y(ny) {
} }
}; };