From 9d25afa41f8982cd6aeca0c61bd934eb47989d50 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 10 Jul 2015 10:26:23 -0700 Subject: [PATCH 01/27] Fix buffer overflow in length of temporary file names --- tile.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tile.cc b/tile.cc index 1d5d8d2..f99cacc 100644 --- a/tile.cc +++ b/tile.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,9 @@ extern "C" { #define CMD_BITS 3 +#define XSTRINGIFY(s) STRINGIFY(s) +#define STRINGIFY(s) #s + // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp static inline int compress(std::string const &input, std::string &output) { z_stream deflate_s; @@ -793,7 +797,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo int subfd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; int j; for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { - char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; + char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX" XSTRINGIFY(INT_MAX)) + 1]; sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j); subfd[j] = mkstemp(geomname); // printf("%s\n", geomname); From e01ea076edf99f962717bc859f5af1360b8e0b06 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 14 Jul 2015 17:11:34 -0700 Subject: [PATCH 02/27] Fix double-free crash with unsupported property type --- geojson.c | 1 - 1 file changed, 1 deletion(-) diff --git a/geojson.c b/geojson.c index f432ad0..0793df0 100644 --- a/geojson.c +++ b/geojson.c @@ -639,7 +639,6 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max ; } else { fprintf(stderr, "%s:%d: Unsupported property type for %s\n", reading, jp->line, properties->keys[i]->string); - json_free(j); continue; } } From cc05f46fb7e5365cbc2fb154c6648d4d8d762fd6 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 27 Jul 2015 16:20:20 -0700 Subject: [PATCH 03/27] Don't crash if there are more than 128 layers --- geojson.c | 2 +- pool.c | 9 +++++++++ tile.cc | 8 ++++---- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/geojson.c b/geojson.c index 0793df0..46132ff 100644 --- a/geojson.c +++ b/geojson.c @@ -653,7 +653,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max long long geomstart = geompos; serialize_byte(geomfile, mb_geometry[t], &geompos, fname); - serialize_byte(geomfile, n, &geompos, fname); + serialize_long_long(geomfile, n, &geompos, fname); serialize_long_long(geomfile, metastart, &geompos, fname); long long wx = initial_x, wy = initial_y; parse_geometry(t, coordinates, bbox, &geompos, geomfile, VT_MOVETO, fname, jp, &wx, &wy, &initialized); diff --git a/pool.c b/pool.c index e874d23..74dd536 100644 --- a/pool.c +++ b/pool.c @@ -1,3 +1,4 @@ +#include #include #include #include "pool.h" @@ -34,6 +35,10 @@ struct pool_val *pool(struct pool *p, char *s, int type) { } *v = malloc(sizeof(struct pool_val)); + if (*v == NULL) { + fprintf(stderr, "out of memory making string pool\n"); + exit(EXIT_FAILURE); + } (*v)->left = NULL; (*v)->right = NULL; (*v)->next = NULL; @@ -104,6 +109,10 @@ void pool_free_strings(struct pool *p) { void pool_init(struct pool *p, int n) { p->n = n; p->vals = calloc(POOL_WIDTH, sizeof(struct pool_val *)); + if (p->vals == NULL) { + fprintf(stderr, "out of memory creating string pool\n"); + exit(EXIT_FAILURE); + } p->head = NULL; p->tail = NULL; } diff --git a/tile.cc b/tile.cc index f99cacc..0fba44f 100644 --- a/tile.cc +++ b/tile.cc @@ -362,7 +362,7 @@ void evaluate(std::vector &features, char *metabase, struct pool *file } #endif -void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, signed char layer, long long metastart, signed char feature_minzoom) { +void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom) { if (geom.size() > 0 && nextzoom <= file_maxzoom) { int xo, yo; int span = 1 << (nextzoom - z); @@ -428,7 +428,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb // printf("type %d, meta %lld\n", t, metastart); serialize_byte(geomfile[j], t, &geompos[j], fname); - serialize_byte(geomfile[j], layer, &geompos[j], fname); + serialize_long_long(geomfile[j], layer, &geompos[j], fname); serialize_long_long(geomfile[j], metastart, &geompos[j], fname); long long wx = initial_x, wy = initial_y; @@ -518,8 +518,8 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f break; } - signed char layer; - deserialize_byte(geoms, &layer); + long long layer; + deserialize_long_long(geoms, &layer); long long metastart; deserialize_long_long(geoms, &metastart); From 26beada6bd3f225be1c56e007b1663a04043b8a6 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 28 Jul 2015 12:38:35 -0700 Subject: [PATCH 04/27] Clarify documentation for dependencies --- README.md | 21 ++++++++++++++++----- man/tippecanoe.1 | 29 ++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c92b533..f7acfb9 100644 --- a/README.md +++ b/README.md @@ -165,10 +165,21 @@ lower resolutions before failing if it still doesn't fit. Development ----------- -Requires protoc (`brew install protobuf` or -`apt-get install libprotobuf-dev` and `protobuf-compiler`), -`md2man` (`gem install md2man`), and sqlite3 (`apt-get install libsqlite3-dev`). -To build: +Requires protoc and sqlite3. Rebuilding the manpage +uses md2man (`gem install md2man`). + +MacOS: + + brew install protobuf + brew install sqlite + +Linux: + + sudo apt-get install libprotobuf-dev + sudo apt-get install protobuf-compiler + sudo apt-get install libsqlite3-dev + +Then build: make @@ -184,4 +195,4 @@ Check out [some examples of maps made with tippecanoe](MADE_WITH.md) Name ---- -The name is [a joking reference](http://en.wikipedia.org/wiki/Tippecanoe_and_Tyler_Too) to making tiles. +The name is [a joking reference](http://en.wikipedia.org/wiki/Tippecanoe_and_Tyler_Too) to a "tiler" for making map tiles. diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 9fdbfc3..a1bf175 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -187,10 +187,29 @@ If a tile is larger than 500K, it will try encoding that tile at progressively lower resolutions before failing if it still doesn't fit. .SH Development .PP -Requires protoc (\fB\fCbrew install protobuf\fR or -\fB\fCapt\-get install libprotobuf\-dev\fR and \fB\fCprotobuf\-compiler\fR), -\fB\fCmd2man\fR (\fB\fCgem install md2man\fR), and sqlite3 (\fB\fCapt\-get install libsqlite3\-dev\fR). -To build: +Requires protoc and sqlite3. Rebuilding the manpage +uses md2man (\fB\fCgem install md2man\fR). +.PP +MacOS: +.PP +.RS +.nf +brew install protobuf +brew install sqlite +.fi +.RE +.PP +Linux: +.PP +.RS +.nf +sudo apt\-get install libprotobuf\-dev +sudo apt\-get install protobuf\-compiler +sudo apt\-get install libsqlite3\-dev +.fi +.RE +.PP +Then build: .PP .RS .nf @@ -212,4 +231,4 @@ Check out some examples of maps made with tippecanoe .SH Name .PP The name is a joking reference -\[la]http://en.wikipedia.org/wiki/Tippecanoe_and_Tyler_Too\[ra] to making tiles. +\[la]http://en.wikipedia.org/wiki/Tippecanoe_and_Tyler_Too\[ra] to a "tiler" for making map tiles. From 70d11cc335812c0b818522760711be2c64a9950c Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 28 Jul 2015 12:48:49 -0700 Subject: [PATCH 05/27] Actually using the stock sqlite on MacOS --- Makefile | 4 +++- README.md | 1 - man/tippecanoe.1 | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index efb279f..c0653b1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ PREFIX ?= /usr/local MANDIR ?= /usr/share/man/man1/ -all: tippecanoe enumerate decode man/tippecanoe.1 +all: tippecanoe enumerate decode + +docs: man/tippecanoe.1 install: tippecanoe mkdir -p $(PREFIX)/bin diff --git a/README.md b/README.md index f7acfb9..c773c34 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,6 @@ uses md2man (`gem install md2man`). MacOS: brew install protobuf - brew install sqlite Linux: diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index a1bf175..95b534a 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -195,7 +195,6 @@ MacOS: .RS .nf brew install protobuf -brew install sqlite .fi .RE .PP From 10e35c43008a6578a7eec79f65f071e7f9d2f280 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 29 Jul 2015 16:45:41 -0700 Subject: [PATCH 06/27] Fix memory leak when closing out tiles that had all features clipped away --- geojson.c | 13 +++++++++++++ tile.cc | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/geojson.c b/geojson.c index 46132ff..c8577ef 100644 --- a/geojson.c +++ b/geojson.c @@ -1,3 +1,7 @@ +#ifdef MEMTRACE + #include +#endif + #include #include #include @@ -1043,6 +1047,10 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max } int main(int argc, char **argv) { +#ifdef MTRACE + mtrace(); +#endif + extern int optind; extern char *optarg; int i; @@ -1190,5 +1198,10 @@ int main(int argc, char **argv) { ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent); mbtiles_close(outdb, argv); + +#ifdef MTRACE + muntrace(); +#endif + return ret; } diff --git a/tile.cc b/tile.cc index 0fba44f..ff20b9c 100644 --- a/tile.cc +++ b/tile.cc @@ -780,6 +780,12 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f return count; } } else { + int i; + for (i = 0; i < nlayers; i++) { + pool_free(&keys1[i]); + pool_free(&values1[i]); + } + return count; } } From a51ddbe1809e034848147d4cab7000e97bb94c34 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 29 Jul 2015 16:46:56 -0700 Subject: [PATCH 07/27] Fix misspelling in ifdef --- geojson.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geojson.c b/geojson.c index c8577ef..d65d61c 100644 --- a/geojson.c +++ b/geojson.c @@ -1,4 +1,4 @@ -#ifdef MEMTRACE +#ifdef MTRACE #include #endif From bc661ef3d5e10a55d9f3690f2caffdb8e6a8f16a Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 11 Aug 2015 16:46:20 -0700 Subject: [PATCH 08/27] Most of the way to handling GeometryCollections --- geojson.c | 314 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 161 insertions(+), 153 deletions(-) diff --git a/geojson.c b/geojson.c index d65d61c..d873966 100644 --- a/geojson.c +++ b/geojson.c @@ -409,6 +409,159 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, char *s, c return off; } +int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, json_pull *jp, long long *seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int maxzoom, int n, double droprate, unsigned *file_bbox) { + json_object *geometry_type = json_hash_get(geometry, "type"); + if (geometry_type == NULL) { + static int warned = 0; + if (!warned) { + fprintf(stderr, "%s:%d: null geometry (additional not reported)\n", reading, jp->line); + warned = 1; + } + + return 0; + } + + if (geometry_type->type != JSON_STRING) { + fprintf(stderr, "%s:%d: geometry without type\n", reading, jp->line); + return 0; + } + + json_object *coordinates = json_hash_get(geometry, "coordinates"); + if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + fprintf(stderr, "%s:%d: feature without coordinates array\n", reading, jp->line); + return 0; + } + + int t; + for (t = 0; t < GEOM_TYPES; t++) { + if (strcmp(geometry_type->string, geometry_names[t]) == 0) { + break; + } + } + if (t >= GEOM_TYPES) { + fprintf(stderr, "%s:%d: Can't handle geometry type %s\n", reading, jp->line, geometry_type->string); + return 0; + } + + unsigned bbox[] = {UINT_MAX, UINT_MAX, 0, 0}; + + int nprop = 0; + if (properties->type == JSON_HASH) { + nprop = properties->length; + } + + long long metastart = *metapos; + char *metakey[nprop]; + char *metaval[nprop]; + int metatype[nprop]; + int m = 0; + + int i; + for (i = 0; i < nprop; i++) { + if (properties->keys[i]->type == JSON_STRING) { + if (exclude_all) { + if (!is_pooled(include, properties->keys[i]->string, VT_STRING)) { + continue; + } + } else if (is_pooled(exclude, properties->keys[i]->string, VT_STRING)) { + continue; + } + + metakey[m] = properties->keys[i]->string; + + if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) { + metatype[m] = VT_STRING; + metaval[m] = properties->values[i]->string; + m++; + } else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) { + metatype[m] = VT_NUMBER; + metaval[m] = properties->values[i]->string; + m++; + } else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) { + metatype[m] = VT_BOOLEAN; + metaval[m] = properties->values[i]->type == JSON_TRUE ? "true" : "false"; + m++; + } else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) { + ; + } else { + fprintf(stderr, "%s:%d: Unsupported property type for %s\n", reading, jp->line, properties->keys[i]->string); + continue; + } + } + } + + serialize_int(metafile, m, metapos, fname); + for (i = 0; i < m; i++) { + serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname); + serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname); + } + + long long geomstart = *geompos; + + serialize_byte(geomfile, mb_geometry[t], geompos, fname); + serialize_long_long(geomfile, n, geompos, fname); + serialize_long_long(geomfile, metastart, geompos, fname); + long long wx = initial_x, wy = initial_y; + parse_geometry(t, coordinates, bbox, geompos, geomfile, VT_MOVETO, fname, jp, &wx, &wy, &initialized); + serialize_byte(geomfile, VT_END, geompos, fname); + + /* + * Note that minzoom for lines is the dimension + * of the geometry in world coordinates, but + * for points is the lowest zoom level (in tiles, + * not in pixels) at which it should be drawn. + * + * So a line that is too small for, say, z8 + * will have minzoom of 18 (if tile detail is 10), + * not 8. + */ + int minzoom = 0; + if (mb_geometry[t] == VT_LINE) { + for (minzoom = 0; minzoom < 31; minzoom++) { + unsigned mask = 1 << (32 - (minzoom + 1)); + + if (((bbox[0] & mask) != (bbox[2] & mask)) || ((bbox[1] & mask) != (bbox[3] & mask))) { + break; + } + } + } else if (mb_geometry[t] == VT_POINT) { + double r = ((double) rand()) / RAND_MAX; + if (r == 0) { + r = .00000001; + } + minzoom = maxzoom - floor(log(r) / -log(droprate)); + } + + serialize_byte(geomfile, minzoom, geompos, fname); + + struct index index; + index.start = geomstart; + index.end = *geompos; + index.index = encode(bbox[0] / 2 + bbox[2] / 2, bbox[1] / 2 + bbox[3] / 2); + fwrite_check(&index, sizeof(struct index), 1, indexfile, fname); + *indexpos += sizeof(struct index); + + for (i = 0; i < 2; i++) { + if (bbox[i] < file_bbox[i]) { + file_bbox[i] = bbox[i]; + } + } + for (i = 2; i < 4; i++) { + if (bbox[i] > file_bbox[i]) { + file_bbox[i] = bbox[i]; + } + } + + if (*seq % 10000 == 0) { + if (!quiet) { + fprintf(stderr, "Read %.2f million features\r", *seq / 1000000.0); + } + } + (*seq)++; + + return 1; +} + int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent) { int ret = EXIT_SUCCESS; @@ -536,7 +689,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max if (j->type == JSON_HASH) { found_hashes++; - if (found_hashes == 50 && found_features == 0) { + if (0 && found_hashes == 50 && found_features == 0) { fprintf(stderr, "%s:%d: Not finding any GeoJSON features in input. Is your file just bare geometries?\n", reading, jp->line); break; } @@ -556,24 +709,6 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max continue; } - json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { - static int warned = 0; - if (!warned) { - fprintf(stderr, "%s:%d: null geometry (additional not reported)\n", reading, jp->line); - warned = 1; - } - - json_free(j); - continue; - } - - if (geometry_type->type != JSON_STRING) { - fprintf(stderr, "%s:%d: geometry without type\n", reading, jp->line); - json_free(j); - continue; - } - json_object *properties = json_hash_get(j, "properties"); if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "%s:%d: feature without properties hash\n", reading, jp->line); @@ -581,141 +716,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max continue; } - json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { - fprintf(stderr, "%s:%d: feature without coordinates array\n", reading, jp->line); - json_free(j); - continue; - } - - int t; - for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->string, geometry_names[t]) == 0) { - break; + json_object *geometries = json_hash_get(geometry, "geometries"); + if (geometries != NULL) { + int g; + for (g = 0; g < geometries->length; g++) { + serialize_geometry(geometries->array[g], properties, reading, jp, &seq, &metapos, &geompos, &indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, n, droprate, file_bbox); } - } - if (t >= GEOM_TYPES) { - fprintf(stderr, "%s:%d: Can't handle geometry type %s\n", reading, jp->line, geometry_type->string); - json_free(j); - continue; - } - - { - unsigned bbox[] = {UINT_MAX, UINT_MAX, 0, 0}; - - int nprop = 0; - if (properties->type == JSON_HASH) { - nprop = properties->length; - } - - long long metastart = metapos; - char *metakey[nprop]; - char *metaval[nprop]; - int metatype[nprop]; - int m = 0; - - int i; - for (i = 0; i < nprop; i++) { - if (properties->keys[i]->type == JSON_STRING) { - if (exclude_all) { - if (!is_pooled(include, properties->keys[i]->string, VT_STRING)) { - continue; - } - } else if (is_pooled(exclude, properties->keys[i]->string, VT_STRING)) { - continue; - } - - metakey[m] = properties->keys[i]->string; - - if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) { - metatype[m] = VT_STRING; - metaval[m] = properties->values[i]->string; - m++; - } else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) { - metatype[m] = VT_NUMBER; - metaval[m] = properties->values[i]->string; - m++; - } else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) { - metatype[m] = VT_BOOLEAN; - metaval[m] = properties->values[i]->type == JSON_TRUE ? "true" : "false"; - m++; - } else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) { - ; - } else { - fprintf(stderr, "%s:%d: Unsupported property type for %s\n", reading, jp->line, properties->keys[i]->string); - continue; - } - } - } - - serialize_int(metafile, m, &metapos, fname); - for (i = 0; i < m; i++) { - serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), &metapos, fname); - serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), &metapos, fname); - } - - long long geomstart = geompos; - - serialize_byte(geomfile, mb_geometry[t], &geompos, fname); - serialize_long_long(geomfile, n, &geompos, fname); - serialize_long_long(geomfile, metastart, &geompos, fname); - long long wx = initial_x, wy = initial_y; - parse_geometry(t, coordinates, bbox, &geompos, geomfile, VT_MOVETO, fname, jp, &wx, &wy, &initialized); - serialize_byte(geomfile, VT_END, &geompos, fname); - - /* - * Note that minzoom for lines is the dimension - * of the geometry in world coordinates, but - * for points is the lowest zoom level (in tiles, - * not in pixels) at which it should be drawn. - * - * So a line that is too small for, say, z8 - * will have minzoom of 18 (if tile detail is 10), - * not 8. - */ - int minzoom = 0; - if (mb_geometry[t] == VT_LINE) { - for (minzoom = 0; minzoom < 31; minzoom++) { - unsigned mask = 1 << (32 - (minzoom + 1)); - - if (((bbox[0] & mask) != (bbox[2] & mask)) || ((bbox[1] & mask) != (bbox[3] & mask))) { - break; - } - } - } else if (mb_geometry[t] == VT_POINT) { - double r = ((double) rand()) / RAND_MAX; - if (r == 0) { - r = .00000001; - } - minzoom = maxzoom - floor(log(r) / -log(droprate)); - } - - serialize_byte(geomfile, minzoom, &geompos, fname); - - struct index index; - index.start = geomstart; - index.end = geompos; - index.index = encode(bbox[0] / 2 + bbox[2] / 2, bbox[1] / 2 + bbox[3] / 2); - fwrite_check(&index, sizeof(struct index), 1, indexfile, fname); - indexpos += sizeof(struct index); - - for (i = 0; i < 2; i++) { - if (bbox[i] < file_bbox[i]) { - file_bbox[i] = bbox[i]; - } - } - for (i = 2; i < 4; i++) { - if (bbox[i] > file_bbox[i]) { - file_bbox[i] = bbox[i]; - } - } - - if (seq % 10000 == 0) { - if (!quiet) { - fprintf(stderr, "Read %.2f million features\r", seq / 1000000.0); - } - } - seq++; + } else { + serialize_geometry(geometry, properties, reading, jp, &seq, &metapos, &geompos, &indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, n, droprate, file_bbox); } json_free(j); From 976101055003309609321f3090d9244c929e3a2e Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 11 Aug 2015 16:55:54 -0700 Subject: [PATCH 09/27] Downgrade no-features error to a warning --- geojson.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/geojson.c b/geojson.c index d873966..575b3f0 100644 --- a/geojson.c +++ b/geojson.c @@ -689,9 +689,8 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max if (j->type == JSON_HASH) { found_hashes++; - if (0 && found_hashes == 50 && found_features == 0) { - fprintf(stderr, "%s:%d: Not finding any GeoJSON features in input. Is your file just bare geometries?\n", reading, jp->line); - break; + if (found_hashes == 50 && found_features == 0) { + fprintf(stderr, "%s:%d: Not finding any GeoJSON features in input after 50 objects. Is your file just bare geometries?\n", reading, jp->line); } } From 86d5a542deb5758f369fb3c19bfd47b7983f337d Mon Sep 17 00:00:00 2001 From: Alex Barth Date: Sat, 15 Aug 2015 09:50:04 -0400 Subject: [PATCH 10/27] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c773c34..0f883e6 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ I don't know why 2.5 is the appropriate number, but the densities of many differ data sets fall off at about this same rate. You can use -r to specify a different rate. You can use the gamma option to thin out especially dense clusters of points. -For any area that where dots are closer than one pixel together (at whatever zoom level), +For any area where dots are closer than one pixel together (at whatever zoom level), a gamma of 3, for example, will reduce these clusters to the cube root of their original density. For line features, it drops any features that are too small to draw at all. From caafe2fe5cf260233cc91ed2ba9ab40e76f0b9e5 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 13:02:34 -0700 Subject: [PATCH 11/27] Checkpoint in copying data from one tileset to another --- Makefile | 5 +- tile-join.cc | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 tile-join.cc diff --git a/Makefile b/Makefile index c0653b1..f181e63 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PREFIX ?= /usr/local MANDIR ?= /usr/share/man/man1/ -all: tippecanoe enumerate decode +all: tippecanoe enumerate decode tile-join docs: man/tippecanoe.1 @@ -33,6 +33,9 @@ enumerate: enumerate.o decode: decode.o vector_tile.pb.o projection.o g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 +tile-join: tile-join.o vector_tile.pb.o projection.o pool.o + g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 + libjsonpull.a: jsonpull.o ar rc $@ $^ ranlib $@ diff --git a/tile-join.cc b/tile-join.cc new file mode 100644 index 0000000..57fa9b4 --- /dev/null +++ b/tile-join.cc @@ -0,0 +1,172 @@ +#include +#include +#include +#include +#include +#include +#include +#include "vector_tile.pb.h" + +extern "C" { +#include "projection.h" +#include "pool.h" +} + +// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp +inline bool is_compressed(std::string const &data) { + return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); +} + +// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp +inline int decompress(std::string const &input, std::string &output) { + z_stream inflate_s; + inflate_s.zalloc = Z_NULL; + inflate_s.zfree = Z_NULL; + inflate_s.opaque = Z_NULL; + inflate_s.avail_in = 0; + inflate_s.next_in = Z_NULL; + if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { + fprintf(stderr, "error: %s\n", inflate_s.msg); + } + inflate_s.next_in = (Bytef *) input.data(); + inflate_s.avail_in = input.size(); + size_t length = 0; + do { + output.resize(length + 2 * input.size()); + inflate_s.avail_out = 2 * input.size(); + inflate_s.next_out = (Bytef *) (output.data() + length); + int ret = inflate(&inflate_s, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + fprintf(stderr, "error: %s\n", inflate_s.msg); + return 0; + } + + length += (2 * input.size() - inflate_s.avail_out); + } while (inflate_s.avail_out == 0); + inflateEnd(&inflate_s); + output.resize(length); + return 1; +} + +int dezig(unsigned n) { + return (n >> 1) ^ (-(n & 1)); +} + +void handle(std::string message, int z, unsigned x, unsigned y, struct pool *file_keys) { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp + mapnik::vector::tile tile; + mapnik::vector::tile outtile; + + if (is_compressed(message)) { + std::string uncompressed; + decompress(message, uncompressed); + if (!tile.ParseFromString(uncompressed)) { + fprintf(stderr, "Couldn't decompress tile %d/%u/%u\n", z, x, y); + exit(EXIT_FAILURE); + } + } else if (!tile.ParseFromString(message)) { + fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y); + exit(EXIT_FAILURE); + } + + for (int l = 0; l < tile.layers_size(); l++) { + mapnik::vector::tile_layer layer = tile.layers(l); + mapnik::vector::tile_layer *outlayer = outtile.add_layers(); + + outlayer->set_name(layer.name()); + outlayer->set_version(layer.version()); + outlayer->set_extent(layer.extent()); + + for (int f = 0; f < layer.features_size(); f++) { + mapnik::vector::tile_feature feat = layer.features(f); + mapnik::vector::tile_feature *outfeature = outlayer->add_features(); + + outfeature->set_type(feat.type()); + + for (int g = 0; g < feat.geometry_size(); g++) { + outfeature->add_geometry(feat.geometry(g)); + } + } + } +} + +void decode(char *fname, char *map, struct pool *file_keys) { + sqlite3 *db; + + if (sqlite3_open(fname, &db) != SQLITE_OK) { + fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + + const char *sql = "SELECT zoom_level, tile_column, tile_row, tile_data from tiles;"; + sqlite3_stmt *stmt; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { + fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + + while (sqlite3_step(stmt) == SQLITE_ROW) { + long long zoom = sqlite3_column_int(stmt, 0); + long long x = sqlite3_column_int(stmt, 1); + long long y = sqlite3_column_int(stmt, 2); + y = (1LL << zoom) - 1 - y; + + int len = sqlite3_column_bytes(stmt, 3); + const char *s = (const char *) sqlite3_column_blob(stmt, 3); + + printf("found %lld/%lld/%lld\n", zoom, x, y); + + handle(std::string(s, len), zoom, x, y, file_keys); + } + + sqlite3_finalize(stmt); + + if (sqlite3_close(db) != SQLITE_OK) { + fprintf(stderr, "%s: could not close database: %s\n", fname, sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } +} + +void usage(char **argv) { + fprintf(stderr, "Usage: %s [-f] -o new.mbtiles source.mbtiles map.csv\n", argv[0]); + exit(EXIT_FAILURE); +} + +int main(int argc, char **argv) { + char *outfile = NULL; + int force = 0; + + extern int optind; + extern char *optarg; + int i; + + while ((i = getopt(argc, argv, "fo:")) != -1) { + switch (i) { + case 'o': + outfile = optarg; + break; + + case 'f': + force = 1; + break; + + default: + usage(argv); + } + } + + if (argc != optind + 2 || outfile == NULL) { + usage(argv); + } + + struct pool file_keys; + pool_init(&file_keys, 0); + + decode(argv[optind], argv[optind + 1], &file_keys); + + pool_free(&file_keys); + + return 0; +} From 224321f6c855f9457642bb18c05aac42f56900f1 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 13:42:24 -0700 Subject: [PATCH 12/27] Look up layers by name because the source mbtiles might be inconsistent --- tile-join.cc | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index 57fa9b4..15b9f00 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -52,7 +52,7 @@ int dezig(unsigned n) { return (n >> 1) ^ (-(n & 1)); } -void handle(std::string message, int z, unsigned x, unsigned y, struct pool *file_keys) { +void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers) { GOOGLE_PROTOBUF_VERIFY_VERSION; // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp @@ -79,6 +79,23 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool *fil outlayer->set_version(layer.version()); outlayer->set_extent(layer.extent()); + const char *ln = layer.name().c_str(); + + int ll; + for (ll = 0; ll < *nlayers; ll++) { + if (strcmp((*layernames)[ll], ln) == 0) { + break; + } + } + if (ll == *nlayers) { + *file_keys = (struct pool *) realloc(*file_keys, (ll + 1) * sizeof(struct pool)); + *layernames = (char **) realloc(*layernames, (ll + 1) * sizeof(char *)); + + pool_init(file_keys[ll], 0); + (*layernames)[ll] = strdup(ln); + *nlayers = ll + 1; + } + for (int f = 0; f < layer.features_size(); f++) { mapnik::vector::tile_feature feat = layer.features(f); mapnik::vector::tile_feature *outfeature = outlayer->add_features(); @@ -92,7 +109,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool *fil } } -void decode(char *fname, char *map, struct pool *file_keys) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -118,7 +135,7 @@ void decode(char *fname, char *map, struct pool *file_keys) { printf("found %lld/%lld/%lld\n", zoom, x, y); - handle(std::string(s, len), zoom, x, y, file_keys); + handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers); } sqlite3_finalize(stmt); @@ -161,12 +178,15 @@ int main(int argc, char **argv) { usage(argv); } - struct pool file_keys; - pool_init(&file_keys, 0); + struct pool *file_keys = NULL; + char **layernames = NULL; + int nlayers = 0; - decode(argv[optind], argv[optind + 1], &file_keys); + decode(argv[optind], argv[optind + 1], &file_keys, &layernames, &nlayers); - pool_free(&file_keys); + for (i = 0; i < nlayers; i++) { + printf("%s\n", layernames[i]); + } return 0; } From 863c9a59291789244ef697b9ac37b55c52326192 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 14:27:39 -0700 Subject: [PATCH 13/27] Retrieving keys and values --- pool.c | 4 ++-- pool.h | 2 +- tile-join.cc | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/pool.c b/pool.c index 74dd536..cc55df9 100644 --- a/pool.c +++ b/pool.c @@ -5,7 +5,7 @@ #define POOL_WIDTH 256 -static int hash(char *s) { +static int hash(const char *s) { int h = 0; for (; *s; s++) { h = h * 37 + *s; @@ -57,7 +57,7 @@ struct pool_val *pool(struct pool *p, char *s, int type) { return *v; } -int is_pooled(struct pool *p, char *s, int type) { +int is_pooled(struct pool *p, const char *s, int type) { int h = hash(s); struct pool_val **v = &(p->vals[h]); diff --git a/pool.h b/pool.h index 147e8c8..683b551 100644 --- a/pool.h +++ b/pool.h @@ -21,4 +21,4 @@ struct pool_val *pool(struct pool *p, char *s, int type); void pool_free(struct pool *p); void pool_free_strings(struct pool *p); void pool_init(struct pool *p, int n); -int is_pooled(struct pool *p, char *s, int type); +int is_pooled(struct pool *p, const char *s, int type); diff --git a/tile-join.cc b/tile-join.cc index 15b9f00..af02b0d 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -1,3 +1,6 @@ +// for vasprintf() on Linux +#define _GNU_SOURCE + #include #include #include @@ -6,6 +9,7 @@ #include #include #include "vector_tile.pb.h" +#include "tile.h" extern "C" { #include "projection.h" @@ -105,6 +109,50 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi for (int g = 0; g < feat.geometry_size(); g++) { outfeature->add_geometry(feat.geometry(g)); } + + for (int t = 0; t + 1 < feat.tags_size(); t += 2) { + if (feat.tags(t) >= layer.keys_size() || feat.tags(t + 1) >= layer.values_size()) { + printf("out of range: %d=%d\n", feat.tags(t), feat.tags(t + 1)); + continue; + } + + const char *key = layer.keys(feat.tags(t)).c_str(); + mapnik::vector::tile_value const &val = layer.values(feat.tags(t + 1)); + char *value; + int type; + + if (val.has_string_value()) { + value = strdup(val.string_value().c_str()); + type = VT_STRING; + } else if (val.has_int_value()) { + asprintf(&value, "%lld", val.int_value()); + type = VT_NUMBER; + } else if (val.has_double_value()) { + asprintf(&value, "%g", val.double_value()); + type = VT_NUMBER; + } else if (val.has_float_value()) { + asprintf(&value, "%g", val.float_value()); + type = VT_NUMBER; + } else if (val.has_bool_value()) { + asprintf(&value, "%s", val.bool_value() ? "true" : "false"); + type = VT_BOOLEAN; + } else if (val.has_sint_value()) { + asprintf(&value, "%lld", val.sint_value()); + type = VT_NUMBER; + } else if (val.has_uint_value()) { + asprintf(&value, "%llu", val.uint_value()); + type = VT_NUMBER; + } else { + continue; + } + + if (!is_pooled(file_keys[ll], key, type)) { + pool(file_keys[ll], strdup(key), type); + } + + printf("%d: %s=%s\n", type, key, value); + free(value); + } } } } From ed378681e455b3190c728162ba58ecd74ee1868d Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 14:50:26 -0700 Subject: [PATCH 14/27] Making the string pool in the destination tile --- pool.c | 4 ++-- pool.h | 4 ++-- tile-join.cc | 26 +++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/pool.c b/pool.c index cc55df9..4641afc 100644 --- a/pool.c +++ b/pool.c @@ -14,7 +14,7 @@ static int hash(const char *s) { return h; } -struct pool_val *pool(struct pool *p, char *s, int type) { +struct pool_val *pool(struct pool *p, const char *s, int type) { int h = hash(s); struct pool_val **v = &(p->vals[h]); @@ -83,7 +83,7 @@ int is_pooled(struct pool *p, const char *s, int type) { void pool_free1(struct pool *p, void (*func)(void *)) { while (p->head != NULL) { if (func != NULL) { - func(p->head->s); + func((void *) p->head->s); } struct pool_val *next = p->head->next; diff --git a/pool.h b/pool.h index 683b551..100dd76 100644 --- a/pool.h +++ b/pool.h @@ -1,5 +1,5 @@ struct pool_val { - char *s; + const char *s; int type; int n; @@ -17,7 +17,7 @@ struct pool { int n; }; -struct pool_val *pool(struct pool *p, char *s, int type); +struct pool_val *pool(struct pool *p, const char *s, int type); void pool_free(struct pool *p); void pool_free_strings(struct pool *p); void pool_init(struct pool *p, int n); diff --git a/tile-join.cc b/tile-join.cc index af02b0d..89f9e8c 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -100,6 +100,10 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi *nlayers = ll + 1; } + struct pool keys, values; + pool_init(&keys, 0); + pool_init(&values, 0); + for (int f = 0; f < layer.features_size(); f++) { mapnik::vector::tile_feature feat = layer.features(f); mapnik::vector::tile_feature *outfeature = outlayer->add_features(); @@ -150,10 +154,30 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi pool(file_keys[ll], strdup(key), type); } - printf("%d: %s=%s\n", type, key, value); + struct pool_val *k, *v; + + if (is_pooled(&keys, key, VT_STRING)) { + k = pool(&keys, key, VT_STRING); + } else { + k = pool(&keys, strdup(key), VT_STRING); + } + + if (is_pooled(&values, value, type)) { + v = pool(&values, value, type); + } else { + v = pool(&values, strdup(value), type); + } + + outfeature->add_tags(k->n); + outfeature->add_tags(v->n); + + printf("%d: %s=%s %d=%d\n", type, key, value, k->n, v->n); free(value); } } + + pool_free_strings(&keys); + pool_free_strings(&values); } } From 2e3f03172e2749a1fba5909cb60300192c2a9fea Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 15:01:34 -0700 Subject: [PATCH 15/27] Write out the JSON part of the metadata --- Makefile | 2 +- tile-join.cc | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f181e63..0538346 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ enumerate: enumerate.o decode: decode.o vector_tile.pb.o projection.o g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 -tile-join: tile-join.o vector_tile.pb.o projection.o pool.o +tile-join: tile-join.o vector_tile.pb.o projection.o pool.o mbtiles.o g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 libjsonpull.a: jsonpull.o diff --git a/tile-join.cc b/tile-join.cc index 89f9e8c..71ff9fc 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -14,6 +14,7 @@ extern "C" { #include "projection.h" #include "pool.h" +#include "mbtiles.h" } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp @@ -170,8 +171,6 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi outfeature->add_tags(k->n); outfeature->add_tags(v->n); - - printf("%d: %s=%s %d=%d\n", type, key, value, k->n, v->n); free(value); } } @@ -205,7 +204,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int len = sqlite3_column_bytes(stmt, 3); const char *s = (const char *) sqlite3_column_blob(stmt, 3); - printf("found %lld/%lld/%lld\n", zoom, x, y); + printf("%lld/%lld/%lld \r", zoom, x, y); handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers); } @@ -250,6 +249,12 @@ int main(int argc, char **argv) { usage(argv); } + if (force) { + unlink(outfile); + } + + sqlite3 *outdb = mbtiles_open(outfile, argv); + struct pool *file_keys = NULL; char **layernames = NULL; int nlayers = 0; @@ -260,5 +265,8 @@ int main(int argc, char **argv) { printf("%s\n", layernames[i]); } + mbtiles_write_metadata(outdb, outfile, layernames, 0, 0, 0, 0, 0, 0, 0, 0, &file_keys, nlayers); + mbtiles_close(outdb, argv); + return 0; } From 29fcbf244d23ed530b6c40fbf4a9cf66f799e83e Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 15:15:47 -0700 Subject: [PATCH 16/27] Write each tile to the destination tileset --- tile-join.cc | 62 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index 71ff9fc..f73f4bb 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -53,11 +53,35 @@ inline int decompress(std::string const &input, std::string &output) { return 1; } -int dezig(unsigned n) { - return (n >> 1) ^ (-(n & 1)); +// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp +static inline int compress(std::string const &input, std::string &output) { + z_stream deflate_s; + deflate_s.zalloc = Z_NULL; + deflate_s.zfree = Z_NULL; + deflate_s.opaque = Z_NULL; + deflate_s.avail_in = 0; + deflate_s.next_in = Z_NULL; + deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); + deflate_s.next_in = (Bytef *) input.data(); + deflate_s.avail_in = input.size(); + size_t length = 0; + do { + size_t increase = input.size() / 2 + 1024; + output.resize(length + increase); + deflate_s.avail_out = increase; + deflate_s.next_out = (Bytef *) (output.data() + length); + int ret = deflate(&deflate_s, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + return -1; + } + length += (increase - deflate_s.avail_out); + } while (deflate_s.avail_out == 0); + deflateEnd(&deflate_s); + output.resize(length); + return 0; } -void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers) { +void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { GOOGLE_PROTOBUF_VERIFY_VERSION; // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp @@ -175,12 +199,36 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi } } + struct pool_val *pv; + for (pv = keys.head; pv != NULL; pv = pv->next) { + outlayer->add_keys(pv->s, strlen(pv->s)); + } + for (pv = values.head; pv != NULL; pv = pv->next) { + mapnik::vector::tile_value *tv = outlayer->add_values(); + + if (pv->type == VT_NUMBER) { + tv->set_double_value(atof(pv->s)); + } else if (pv->type == VT_BOOLEAN) { + tv->set_bool_value(pv->s[0] == 't'); + } else { + tv->set_string_value(pv->s); + } + } + pool_free_strings(&keys); pool_free_strings(&values); } + + std::string s; + std::string compressed; + + outtile.SerializeToString(&s); + compress(s, compressed); + + mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } -void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -204,9 +252,9 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int len = sqlite3_column_bytes(stmt, 3); const char *s = (const char *) sqlite3_column_blob(stmt, 3); - printf("%lld/%lld/%lld \r", zoom, x, y); + fprintf(stderr, "%lld/%lld/%lld \r", zoom, x, y); - handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers); + handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb); } sqlite3_finalize(stmt); @@ -259,7 +307,7 @@ int main(int argc, char **argv) { char **layernames = NULL; int nlayers = 0; - decode(argv[optind], argv[optind + 1], &file_keys, &layernames, &nlayers); + decode(argv[optind], argv[optind + 1], &file_keys, &layernames, &nlayers, outdb); for (i = 0; i < nlayers; i++) { printf("%s\n", layernames[i]); From f131987f9ef2f2628bc61273012fb54488ae3d53 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 15:47:44 -0700 Subject: [PATCH 17/27] Fix my pointer confusion around the global keys per layer --- tile-join.cc | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index f73f4bb..4aaf939 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -120,7 +120,16 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi *file_keys = (struct pool *) realloc(*file_keys, (ll + 1) * sizeof(struct pool)); *layernames = (char **) realloc(*layernames, (ll + 1) * sizeof(char *)); - pool_init(file_keys[ll], 0); + if (*file_keys == NULL) { + perror("realloc file_keys"); + exit(EXIT_FAILURE); + } + if (*layernames == NULL) { + perror("realloc layernames"); + exit(EXIT_FAILURE); + } + + pool_init(&((*file_keys)[ll]), 0); (*layernames)[ll] = strdup(ln); *nlayers = ll + 1; } @@ -175,8 +184,8 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi continue; } - if (!is_pooled(file_keys[ll], key, type)) { - pool(file_keys[ll], strdup(key), type); + if (!is_pooled(&((*file_keys)[ll]), key, type)) { + pool(&((*file_keys)[ll]), strdup(key), type); } struct pool_val *k, *v; @@ -266,19 +275,20 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, } void usage(char **argv) { - fprintf(stderr, "Usage: %s [-f] -o new.mbtiles source.mbtiles map.csv\n", argv[0]); + fprintf(stderr, "Usage: %s [-f] [-c joins.csv] -o new.mbtiles source.mbtiles ...\n", argv[0]); exit(EXIT_FAILURE); } int main(int argc, char **argv) { char *outfile = NULL; + char *csv = NULL; int force = 0; extern int optind; extern char *optarg; int i; - while ((i = getopt(argc, argv, "fo:")) != -1) { + while ((i = getopt(argc, argv, "fo:c:")) != -1) { switch (i) { case 'o': outfile = optarg; @@ -288,12 +298,16 @@ int main(int argc, char **argv) { force = 1; break; + case 'c': + csv = optarg; + break; + default: usage(argv); } } - if (argc != optind + 2 || outfile == NULL) { + if (argc - optind < 1 || outfile == NULL) { usage(argv); } @@ -307,13 +321,20 @@ int main(int argc, char **argv) { char **layernames = NULL; int nlayers = 0; - decode(argv[optind], argv[optind + 1], &file_keys, &layernames, &nlayers, outdb); + for (i = optind; i < argc; i++) { + decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb); + } for (i = 0; i < nlayers; i++) { printf("%s\n", layernames[i]); } - mbtiles_write_metadata(outdb, outfile, layernames, 0, 0, 0, 0, 0, 0, 0, 0, &file_keys, nlayers); + struct pool *fk[nlayers]; + for (i = 0; i < nlayers; i++) { + fk[i] = &(file_keys[i]); + } + + mbtiles_write_metadata(outdb, outfile, layernames, 0, 0, 0, 0, 0, 0, 0, 0, fk, nlayers); mbtiles_close(outdb, argv); return 0; From 06ca52526d3a899bdba270dd65079226c7f63548 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 16:12:34 -0700 Subject: [PATCH 18/27] Copy the bounding box, center, zoom range metadata over --- tile-join.cc | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index 4aaf939..dc7f036 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -17,6 +17,13 @@ extern "C" { #include "mbtiles.h" } +struct stats { + int minzoom; + int maxzoom; + double midlat, midlon; + double minlat, minlon, maxlat, maxlon; +}; + // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp inline bool is_compressed(std::string const &data) { return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); @@ -237,7 +244,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } -void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -268,6 +275,33 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, sqlite3_finalize(stmt); + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'minzoom'", -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + st->minzoom = sqlite3_column_int(stmt, 0); + } + sqlite3_finalize(stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'maxzoom'", -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + st->maxzoom = sqlite3_column_int(stmt, 0); + } + sqlite3_finalize(stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'center'", -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(stmt, 0); + sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat); + } + sqlite3_finalize(stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'bounds'", -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(stmt, 0); + sscanf((char *) s, "%lf,%lf,%lf,%lf", &st->minlon, &st->minlat, &st->maxlon, &st->maxlat); + } + sqlite3_finalize(stmt); + } + if (sqlite3_close(db) != SQLITE_OK) { fprintf(stderr, "%s: could not close database: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_FAILURE); @@ -316,13 +350,15 @@ int main(int argc, char **argv) { } sqlite3 *outdb = mbtiles_open(outfile, argv); + struct stats st; + memset(&st, 0, sizeof(st)); struct pool *file_keys = NULL; char **layernames = NULL; int nlayers = 0; for (i = optind; i < argc; i++) { - decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb); + decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st); } for (i = 0; i < nlayers; i++) { @@ -334,7 +370,7 @@ int main(int argc, char **argv) { fk[i] = &(file_keys[i]); } - mbtiles_write_metadata(outdb, outfile, layernames, 0, 0, 0, 0, 0, 0, 0, 0, fk, nlayers); + mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, fk, nlayers); mbtiles_close(outdb, argv); return 0; From 514da467e145a9155ec9dae4f13b3a8493c5b61a Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 16:15:09 -0700 Subject: [PATCH 19/27] Only allow one input .mbtiles because multiple really won't work --- tile-join.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index dc7f036..d0c05f8 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -309,7 +309,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, } void usage(char **argv) { - fprintf(stderr, "Usage: %s [-f] [-c joins.csv] -o new.mbtiles source.mbtiles ...\n", argv[0]); + fprintf(stderr, "Usage: %s [-f] [-c joins.csv] -o new.mbtiles source.mbtiles\n", argv[0]); exit(EXIT_FAILURE); } @@ -341,7 +341,7 @@ int main(int argc, char **argv) { } } - if (argc - optind < 1 || outfile == NULL) { + if (argc - optind != 1 || outfile == NULL) { usage(argv); } From 9a4953409819b64dbffa7d1f3b8b5ba4342daaca Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 16:45:45 -0700 Subject: [PATCH 20/27] Parse the CSV --- tile-join.cc | 71 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index d0c05f8..bad0c78 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include #include #include "vector_tile.pb.h" @@ -88,7 +90,7 @@ static inline int compress(std::string const &input, std::string &output) { return 0; } -void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { +void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, std::vector &header, std::map > &mapping) { GOOGLE_PROTOBUF_VERIFY_VERSION; // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp @@ -244,7 +246,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } -void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector &header, std::map > &mapping) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -270,7 +272,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, fprintf(stderr, "%lld/%lld/%lld \r", zoom, x, y); - handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb); + handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb, header, mapping); } sqlite3_finalize(stmt); @@ -313,11 +315,66 @@ void usage(char **argv) { exit(EXIT_FAILURE); } +#define MAXLINE 10000 /* XXX */ + +std::vector split(char *s) { + std::vector ret; + + while (*s && *s != '\n') { + char *start = s; + int within = 0; + + for (; *s && *s != '\n'; s++) { + if (*s == '"') { + within = !within; + } + + if (*s == ',' && !within) { + break; + } + } + + std::string v = std::string(start, s - start); + ret.push_back(v); + + if (*s == ',') { + s++; + } + } + + return ret; +} + +void readcsv(char *fn, std::vector &header, std::map > &mapping) { + FILE *f = fopen(fn, "r"); + if (f == NULL) { + perror(fn); + exit(EXIT_FAILURE); + } + + char s[MAXLINE]; + if (fgets(s, MAXLINE, f)) { + header = split(s); + } + while (fgets(s, MAXLINE, f)) { + std::vector line = split(s); + + for (int i = 0; i < line.size() && i < header.size(); i++) { + mapping.insert(std::pair >(line[0], line)); + } + } + + fclose(f); +} + int main(int argc, char **argv) { char *outfile = NULL; char *csv = NULL; int force = 0; + std::vector header; + std::map > mapping; + extern int optind; extern char *optarg; int i; @@ -333,7 +390,13 @@ int main(int argc, char **argv) { break; case 'c': + if (csv != NULL) { + fprintf(stderr, "Only one -c for now\n"); + exit(EXIT_FAILURE); + } + csv = optarg; + readcsv(csv, header, mapping); break; default: @@ -358,7 +421,7 @@ int main(int argc, char **argv) { int nlayers = 0; for (i = optind; i < argc; i++) { - decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st); + decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping); } for (i = 0; i < nlayers; i++) { From 529c9aedc9f707a85f67f9ecd367d16fd99f2c0f Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 17:32:58 -0700 Subject: [PATCH 21/27] Join attributes from CSV into tiles --- tile-join.cc | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tile-join.cc b/tile-join.cc index bad0c78..7f0d1be 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -19,6 +19,8 @@ extern "C" { #include "mbtiles.h" } +std::string dequote(std::string s); + struct stats { int minzoom; int maxzoom; @@ -213,6 +215,52 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi outfeature->add_tags(k->n); outfeature->add_tags(v->n); + + if (strcmp(key, header[0].c_str()) == 0) { + std::map >::iterator ii = mapping.find(std::string(value)); + + if (ii != mapping.end()) { + std::vector fields = ii->second; + + for (int i = 1; i < fields.size(); i++) { + std::string joinkey = header[i]; + std::string joinval = fields[i]; + int type = VT_STRING; + + if (joinval.size() > 0) { + if (joinval[0] == '"') { + joinval = dequote(joinval); + } else if ((joinval[0] >= '0' && joinval[0] <= '9') || joinval[0] == '-') { + type = VT_NUMBER; + } + } + + const char *sjoinkey = joinkey.c_str(); + const char *sjoinval = joinval.c_str(); + + if (!is_pooled(&((*file_keys)[ll]), sjoinkey, type)) { + pool(&((*file_keys)[ll]), strdup(sjoinkey), type); + } + + if (is_pooled(&keys, sjoinkey, VT_STRING)) { + k = pool(&keys, sjoinkey, VT_STRING); + } else { + k = pool(&keys, strdup(sjoinkey), VT_STRING); + } + + if (is_pooled(&values, sjoinval, type)) { + v = pool(&values, sjoinval, type); + } else { + v = pool(&values, strdup(sjoinval), type); + } + + outfeature->add_tags(k->n); + outfeature->add_tags(v->n); + } + } + + } + free(value); } } @@ -345,6 +393,21 @@ std::vector split(char *s) { return ret; } +std::string dequote(std::string s) { + std::string out; + int i; + for (i = 0; i < s.size(); i++) { + if (s[i] == '"') { + if (i + 1 < s.size() && s[i + 1] == '"') { + out.push_back('"'); + } + } else { + out.push_back(s[i]); + } + } + return out; +} + void readcsv(char *fn, std::vector &header, std::map > &mapping) { FILE *f = fopen(fn, "r"); if (f == NULL) { @@ -355,11 +418,19 @@ void readcsv(char *fn, std::vector &header, std::map line = split(s); + if (line.size() > 0) { + line[0] = dequote(line[0]); + } for (int i = 0; i < line.size() && i < header.size(); i++) { + // printf("putting %s\n", line[0].c_str()); mapping.insert(std::pair >(line[0], line)); } } From 88b9750959c14224877ec11c71e91b7c0be62ec3 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 21 Aug 2015 10:33:57 -0700 Subject: [PATCH 22/27] Fix a bunch of compiler warnings --- tile-join.cc | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index 7f0d1be..84822d7 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -1,6 +1,3 @@ -// for vasprintf() on Linux -#define _GNU_SOURCE - #include #include #include @@ -160,41 +157,46 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi } for (int t = 0; t + 1 < feat.tags_size(); t += 2) { - if (feat.tags(t) >= layer.keys_size() || feat.tags(t + 1) >= layer.values_size()) { - printf("out of range: %d=%d\n", feat.tags(t), feat.tags(t + 1)); - continue; - } - const char *key = layer.keys(feat.tags(t)).c_str(); mapnik::vector::tile_value const &val = layer.values(feat.tags(t + 1)); char *value; - int type; + int type = -1; if (val.has_string_value()) { value = strdup(val.string_value().c_str()); type = VT_STRING; } else if (val.has_int_value()) { - asprintf(&value, "%lld", val.int_value()); - type = VT_NUMBER; + if (asprintf(&value, "%lld", (long long) val.int_value()) >= 0) { + type = VT_NUMBER; + } } else if (val.has_double_value()) { - asprintf(&value, "%g", val.double_value()); - type = VT_NUMBER; + if (asprintf(&value, "%g", val.double_value()) >= 0) { + type = VT_NUMBER; + } } else if (val.has_float_value()) { - asprintf(&value, "%g", val.float_value()); - type = VT_NUMBER; + if (asprintf(&value, "%g", val.float_value()) >= 0) { + type = VT_NUMBER; + } } else if (val.has_bool_value()) { - asprintf(&value, "%s", val.bool_value() ? "true" : "false"); - type = VT_BOOLEAN; + if (asprintf(&value, "%s", val.bool_value() ? "true" : "false") >= 0) { + type = VT_BOOLEAN; + } } else if (val.has_sint_value()) { - asprintf(&value, "%lld", val.sint_value()); - type = VT_NUMBER; + if (asprintf(&value, "%lld", (long long) val.sint_value()) >= 0) { + type = VT_NUMBER; + } } else if (val.has_uint_value()) { - asprintf(&value, "%llu", val.uint_value()); - type = VT_NUMBER; + if (asprintf(&value, "%llu", (long long) val.uint_value()) >= 0) { + type = VT_NUMBER; + } } else { continue; } + if (type < 0) { + continue; + } + if (!is_pooled(&((*file_keys)[ll]), key, type)) { pool(&((*file_keys)[ll]), strdup(key), type); } @@ -222,7 +224,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi if (ii != mapping.end()) { std::vector fields = ii->second; - for (int i = 1; i < fields.size(); i++) { + for (unsigned i = 1; i < fields.size(); i++) { std::string joinkey = header[i]; std::string joinval = fields[i]; int type = VT_STRING; @@ -395,7 +397,7 @@ std::vector split(char *s) { std::string dequote(std::string s) { std::string out; - int i; + unsigned i; for (i = 0; i < s.size(); i++) { if (s[i] == '"') { if (i + 1 < s.size() && s[i + 1] == '"') { @@ -419,7 +421,7 @@ void readcsv(char *fn, std::vector &header, std::map &header, std::map >(line[0], line)); } From f99d3202533cafd31054d86a0c81481be8976e88 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 21 Aug 2015 14:27:11 -0700 Subject: [PATCH 23/27] Fix indent style --- geojson.c | 2 +- tile-join.cc | 41 ++++++++++++++++++++--------------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/geojson.c b/geojson.c index d65d61c..bd4b750 100644 --- a/geojson.c +++ b/geojson.c @@ -1,5 +1,5 @@ #ifdef MTRACE - #include +#include #endif #include diff --git a/tile-join.cc b/tile-join.cc index 84822d7..43ce2a6 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -260,7 +260,6 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi outfeature->add_tags(v->n); } } - } free(value); @@ -312,10 +311,10 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, } while (sqlite3_step(stmt) == SQLITE_ROW) { - long long zoom = sqlite3_column_int(stmt, 0); - long long x = sqlite3_column_int(stmt, 1); - long long y = sqlite3_column_int(stmt, 2); - y = (1LL << zoom) - 1 - y; + long long zoom = sqlite3_column_int(stmt, 0); + long long x = sqlite3_column_int(stmt, 1); + long long y = sqlite3_column_int(stmt, 2); + y = (1LL << zoom) - 1 - y; int len = sqlite3_column_bytes(stmt, 3); const char *s = (const char *) sqlite3_column_blob(stmt, 3); @@ -454,26 +453,26 @@ int main(int argc, char **argv) { while ((i = getopt(argc, argv, "fo:c:")) != -1) { switch (i) { - case 'o': - outfile = optarg; - break; + case 'o': + outfile = optarg; + break; - case 'f': - force = 1; - break; + case 'f': + force = 1; + break; - case 'c': - if (csv != NULL) { - fprintf(stderr, "Only one -c for now\n"); - exit(EXIT_FAILURE); - } + case 'c': + if (csv != NULL) { + fprintf(stderr, "Only one -c for now\n"); + exit(EXIT_FAILURE); + } - csv = optarg; - readcsv(csv, header, mapping); - break; + csv = optarg; + readcsv(csv, header, mapping); + break; - default: - usage(argv); + default: + usage(argv); } } From 2814f1ccf22b953707925607db50ae7efaaa5522 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 21 Aug 2015 14:37:55 -0700 Subject: [PATCH 24/27] Check for tiles that are too large to upload. --- tile-join.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tile-join.cc b/tile-join.cc index 43ce2a6..447c67e 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -292,6 +292,11 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi outtile.SerializeToString(&s); compress(s, compressed); + if (compressed.size() > 500000) { + fprintf(stderr, "Tile %d/%u/%u size is %lld, >500000. Skipping this tile\n.", z, x, y, (long long) compressed.size()); + return; + } + mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } From a57b0102450653669ddc280c37570804e9cab85e Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 21 Aug 2015 17:32:44 -0700 Subject: [PATCH 25/27] Add a feature to exclude a key from the joined tileset --- tile-join.cc | 93 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 40 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index 447c67e..e1c91a5 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -89,7 +89,7 @@ static inline int compress(std::string const &input, std::string &output) { return 0; } -void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, std::vector &header, std::map > &mapping) { +void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, std::vector &header, std::map > &mapping, struct pool *exclude) { GOOGLE_PROTOBUF_VERIFY_VERSION; // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp @@ -197,27 +197,29 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi continue; } - if (!is_pooled(&((*file_keys)[ll]), key, type)) { - pool(&((*file_keys)[ll]), strdup(key), type); + if (!is_pooled(exclude, key, VT_STRING)) { + if (!is_pooled(&((*file_keys)[ll]), key, type)) { + pool(&((*file_keys)[ll]), strdup(key), type); + } + + struct pool_val *k, *v; + + if (is_pooled(&keys, key, VT_STRING)) { + k = pool(&keys, key, VT_STRING); + } else { + k = pool(&keys, strdup(key), VT_STRING); + } + + if (is_pooled(&values, value, type)) { + v = pool(&values, value, type); + } else { + v = pool(&values, strdup(value), type); + } + + outfeature->add_tags(k->n); + outfeature->add_tags(v->n); } - struct pool_val *k, *v; - - if (is_pooled(&keys, key, VT_STRING)) { - k = pool(&keys, key, VT_STRING); - } else { - k = pool(&keys, strdup(key), VT_STRING); - } - - if (is_pooled(&values, value, type)) { - v = pool(&values, value, type); - } else { - v = pool(&values, strdup(value), type); - } - - outfeature->add_tags(k->n); - outfeature->add_tags(v->n); - if (strcmp(key, header[0].c_str()) == 0) { std::map >::iterator ii = mapping.find(std::string(value)); @@ -240,24 +242,28 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi const char *sjoinkey = joinkey.c_str(); const char *sjoinval = joinval.c_str(); - if (!is_pooled(&((*file_keys)[ll]), sjoinkey, type)) { - pool(&((*file_keys)[ll]), strdup(sjoinkey), type); - } + if (!is_pooled(exclude, sjoinkey, VT_STRING)) { + if (!is_pooled(&((*file_keys)[ll]), sjoinkey, type)) { + pool(&((*file_keys)[ll]), strdup(sjoinkey), type); + } - if (is_pooled(&keys, sjoinkey, VT_STRING)) { - k = pool(&keys, sjoinkey, VT_STRING); - } else { - k = pool(&keys, strdup(sjoinkey), VT_STRING); - } + struct pool_val *k, *v; - if (is_pooled(&values, sjoinval, type)) { - v = pool(&values, sjoinval, type); - } else { - v = pool(&values, strdup(sjoinval), type); - } + if (is_pooled(&keys, sjoinkey, VT_STRING)) { + k = pool(&keys, sjoinkey, VT_STRING); + } else { + k = pool(&keys, strdup(sjoinkey), VT_STRING); + } - outfeature->add_tags(k->n); - outfeature->add_tags(v->n); + if (is_pooled(&values, sjoinval, type)) { + v = pool(&values, sjoinval, type); + } else { + v = pool(&values, strdup(sjoinval), type); + } + + outfeature->add_tags(k->n); + outfeature->add_tags(v->n); + } } } } @@ -300,7 +306,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } -void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector &header, std::map > &mapping) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector &header, std::map > &mapping, struct pool *exclude) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -326,7 +332,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, fprintf(stderr, "%lld/%lld/%lld \r", zoom, x, y); - handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb, header, mapping); + handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb, header, mapping, exclude); } sqlite3_finalize(stmt); @@ -365,7 +371,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, } void usage(char **argv) { - fprintf(stderr, "Usage: %s [-f] [-c joins.csv] -o new.mbtiles source.mbtiles\n", argv[0]); + fprintf(stderr, "Usage: %s [-f] [-c joins.csv] [-x exclude ...] -o new.mbtiles source.mbtiles\n", argv[0]); exit(EXIT_FAILURE); } @@ -452,11 +458,14 @@ int main(int argc, char **argv) { std::vector header; std::map > mapping; + struct pool exclude; + pool_init(&exclude, 0); + extern int optind; extern char *optarg; int i; - while ((i = getopt(argc, argv, "fo:c:")) != -1) { + while ((i = getopt(argc, argv, "fo:c:x:")) != -1) { switch (i) { case 'o': outfile = optarg; @@ -476,6 +485,10 @@ int main(int argc, char **argv) { readcsv(csv, header, mapping); break; + case 'x': + pool(&exclude, optarg, VT_STRING); + break; + default: usage(argv); } @@ -498,7 +511,7 @@ int main(int argc, char **argv) { int nlayers = 0; for (i = optind; i < argc; i++) { - decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping); + decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping, &exclude); } for (i = 0; i < nlayers; i++) { From b3302c30e5d68852121604f54e633b48bfe4077f Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 24 Aug 2015 14:12:46 -0700 Subject: [PATCH 26/27] Add a flag to retain only the features that match the join CSV --- tile-join.cc | 52 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index e1c91a5..8a4dd6a 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -89,7 +89,7 @@ static inline int compress(std::string const &input, std::string &output) { return 0; } -void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, std::vector &header, std::map > &mapping, struct pool *exclude) { +void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, std::vector &header, std::map > &mapping, struct pool *exclude, int ifmatched) { GOOGLE_PROTOBUF_VERIFY_VERSION; // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp @@ -148,13 +148,8 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi for (int f = 0; f < layer.features_size(); f++) { mapnik::vector::tile_feature feat = layer.features(f); - mapnik::vector::tile_feature *outfeature = outlayer->add_features(); - - outfeature->set_type(feat.type()); - - for (int g = 0; g < feat.geometry_size(); g++) { - outfeature->add_geometry(feat.geometry(g)); - } + std::vector feature_tags; + int matched = 0; for (int t = 0; t + 1 < feat.tags_size(); t += 2) { const char *key = layer.keys(feat.tags(t)).c_str(); @@ -216,8 +211,8 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi v = pool(&values, strdup(value), type); } - outfeature->add_tags(k->n); - outfeature->add_tags(v->n); + feature_tags.push_back(k->n); + feature_tags.push_back(v->n); } if (strcmp(key, header[0].c_str()) == 0) { @@ -225,6 +220,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi if (ii != mapping.end()) { std::vector fields = ii->second; + matched = 1; for (unsigned i = 1; i < fields.size(); i++) { std::string joinkey = header[i]; @@ -261,8 +257,8 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi v = pool(&values, strdup(sjoinval), type); } - outfeature->add_tags(k->n); - outfeature->add_tags(v->n); + feature_tags.push_back(k->n); + feature_tags.push_back(v->n); } } } @@ -270,6 +266,19 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi free(value); } + + if (matched || !ifmatched) { + mapnik::vector::tile_feature *outfeature = outlayer->add_features(); + outfeature->set_type(feat.type()); + + for (int g = 0; g < feat.geometry_size(); g++) { + outfeature->add_geometry(feat.geometry(g)); + } + + for (unsigned i = 0; i < feature_tags.size(); i++) { + outfeature->add_tags(feature_tags[i]); + } + } } struct pool_val *pv; @@ -306,7 +315,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } -void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector &header, std::map > &mapping, struct pool *exclude) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector &header, std::map > &mapping, struct pool *exclude, int ifmatched) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -332,7 +341,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, fprintf(stderr, "%lld/%lld/%lld \r", zoom, x, y); - handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb, header, mapping, exclude); + handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb, header, mapping, exclude, ifmatched); } sqlite3_finalize(stmt); @@ -371,7 +380,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, } void usage(char **argv) { - fprintf(stderr, "Usage: %s [-f] [-c joins.csv] [-x exclude ...] -o new.mbtiles source.mbtiles\n", argv[0]); + fprintf(stderr, "Usage: %s [-f] [-i] [-c joins.csv] [-x exclude ...] -o new.mbtiles source.mbtiles\n", argv[0]); exit(EXIT_FAILURE); } @@ -454,6 +463,7 @@ int main(int argc, char **argv) { char *outfile = NULL; char *csv = NULL; int force = 0; + int ifmatched = 0; std::vector header; std::map > mapping; @@ -465,7 +475,7 @@ int main(int argc, char **argv) { extern char *optarg; int i; - while ((i = getopt(argc, argv, "fo:c:x:")) != -1) { + while ((i = getopt(argc, argv, "fo:c:x:i")) != -1) { switch (i) { case 'o': outfile = optarg; @@ -475,6 +485,10 @@ int main(int argc, char **argv) { force = 1; break; + case 'i': + ifmatched = 1; + break; + case 'c': if (csv != NULL) { fprintf(stderr, "Only one -c for now\n"); @@ -511,11 +525,7 @@ int main(int argc, char **argv) { int nlayers = 0; for (i = optind; i < argc; i++) { - decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping, &exclude); - } - - for (i = 0; i < nlayers; i++) { - printf("%s\n", layernames[i]); + decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping, &exclude, ifmatched); } struct pool *fk[nlayers]; From 541d3518c2477dbb7f9461942c5e9203ce0a61c4 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 27 Aug 2015 16:02:04 -0700 Subject: [PATCH 27/27] Change some defaults that keep tripping people up: * Default detail with low -z is only 13, so it works with GL * No coalescing, line-reversing, or reordering by attributes except by request * Dropping lines like points moves from -d to -a --- README.md | 14 +++++++++----- geojson.c | 28 +++++++++++++++++++++------- man/tippecanoe.1 | 25 +++++++++++++++---------- tile.cc | 14 +++++++------- tile.h | 4 ++-- 5 files changed, 54 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 0f883e6..128dc90 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,13 @@ Options * -o _file_.mbtiles: Name the output file. * -f: Delete the mbtiles file if it already exists instead of giving an error + * -t _directory_: Put the temporary files in _directory_. ### Zoom levels and resolution * -z _zoom_: Base (maxzoom) zoom level (default 14) * -Z _zoom_: Lowest (minzoom) zoom level (default 0) - * -d _detail_: Detail at base zoom level (default 26-basezoom, ~0.5m, for tile resolution of 4096 if -z14) + * -d _detail_: Detail at base zoom level (default 12 at -z14 or higher, or 13 at -z13 or lower. Detail beyond 13 has rendering problems with Mapbox GL.) * -D _detail_: Detail at lower zoom levels (default 10, for tile resolution of 1024) * -m _detail_: Minimum detail that it will try if tiles are too big at regular detail (default 7) * -b _pixels_: Buffer size where features are duplicated from adjacent tiles. Units are "screen pixels"--1/256th of the tile width or height. (default 5) @@ -82,15 +83,18 @@ Options * -r _rate_: Rate at which dots are dropped at lower zoom levels (default 2.5) * -g _gamma_: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number. +### Doing more + + * -ac: Coalesce adjacent line and polygon features that have the same properties + * -ar: Try reversing the directions of lines to make them coalesce and compress better + * -ao: Reorder features to put ones with the same properties in sequence, to try to get them to coalesce + * -al: Let "dot" dropping at lower zooms apply to lines too + ### Doing less * -ps: Don't simplify lines - * -pr: Don't reverse the direction of lines to make them coalesce better - * -pc: Don't coalesce features with the same properties * -pf: Don't limit tiles to 200,000 features * -pk: Don't limit tiles to 500K bytes - * -po: Don't reorder features to put the same properties in sequence - * -pl: Let "dot" simplification apply to lines too * -pd: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. * -q: Work quietly instead of reporting progress diff --git a/geojson.c b/geojson.c index 44e90be..8b24d42 100644 --- a/geojson.c +++ b/geojson.c @@ -562,7 +562,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha return 1; } -int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent) { +int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional) { int ret = EXIT_SUCCESS; char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; @@ -1001,7 +1001,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", seq, (long long) geomst.st_size, (long long) metast.st_size, poolfile->off); } - int written = traverse_zooms(fd, size, meta, stringpool, file_bbox, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, full_detail, low_detail, min_detail); + int written = traverse_zooms(fd, size, meta, stringpool, file_bbox, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); @@ -1073,6 +1073,7 @@ int main(int argc, char **argv) { int buffer = 5; const char *tmpdir = "/tmp"; char prevent[256]; + char additional[256]; struct pool exclude, include; pool_init(&exclude, 0); @@ -1081,9 +1082,10 @@ int main(int argc, char **argv) { for (i = 0; i < 256; i++) { prevent[i] = 0; + additional[i] = 0; } - while ((i = getopt(argc, argv, "l:n:z:Z:d:D:m:o:x:y:r:b:fXt:g:p:vq")) != -1) { + while ((i = getopt(argc, argv, "l:n:z:Z:d:D:m:o:x:y:r:b:fXt:g:p:vqa:")) != -1) { switch (i) { case 'n': name = optarg; @@ -1161,12 +1163,19 @@ int main(int argc, char **argv) { } } break; + case 'a': { + char *cp; + for (cp = optarg; *cp != '\0'; cp++) { + additional[*cp & 0xFF] = 1; + } + } break; + case 'v': fprintf(stderr, VERSION); exit(EXIT_FAILURE); default: - fprintf(stderr, "Usage: %s -o out.mbtiles [-n name] [-l layername] [-z maxzoom] [-Z minzoom] [-d detail] [-D lower-detail] [-m min-detail] [-x excluded-field ...] [-y included-field ...] [-X] [-r droprate] [-b buffer] [-t tmpdir] [-p rcfs] [file.json ...]\n", argv[0]); + fprintf(stderr, "Usage: %s -o out.mbtiles [-n name] [-l layername] [-z maxzoom] [-Z minzoom] [-d detail] [-D lower-detail] [-m min-detail] [-x excluded-field ...] [-y included-field ...] [-X] [-r droprate] [-b buffer] [-t tmpdir] [-a rco] [-p sfkld] [-q] [file.json ...]\n", argv[0]); exit(EXIT_FAILURE); } } @@ -1177,8 +1186,13 @@ int main(int argc, char **argv) { } if (full_detail <= 0) { - // ~0.5m accuracy at whatever zoom - // 12 bits (4096 units) at z14 + if (maxzoom >= 14) { + // ~0.5m accuracy at z14 + full_detail = 12; + } else { + // as good as we can get without breaking GL + full_detail = 13; + } full_detail = 26 - maxzoom; } @@ -1202,7 +1216,7 @@ int main(int argc, char **argv) { sqlite3 *outdb = mbtiles_open(outdir, argv); int ret = EXIT_SUCCESS; - ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent); + ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional); mbtiles_close(outdb, argv); diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 95b534a..44a69c0 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -64,6 +64,8 @@ it encounters. \-o \fIfile\fP\&.mbtiles: Name the output file. .IP \(bu 2 \-f: Delete the mbtiles file if it already exists instead of giving an error +.IP \(bu 2 +\-t \fIdirectory\fP: Put the temporary files in \fIdirectory\fP\&. .RE .SS Zoom levels and resolution .RS @@ -72,7 +74,7 @@ it encounters. .IP \(bu 2 \-Z \fIzoom\fP: Lowest (minzoom) zoom level (default 0) .IP \(bu 2 -\-d \fIdetail\fP: Detail at base zoom level (default 26\-basezoom, ~0.5m, for tile resolution of 4096 if \-z14) +\-d \fIdetail\fP: Detail at base zoom level (default 12 at \-z14 or higher, or 13 at \-z13 or lower. Detail beyond 13 has rendering problems with Mapbox GL.) .IP \(bu 2 \-D \fIdetail\fP: Detail at lower zoom levels (default 10, for tile resolution of 1024) .IP \(bu 2 @@ -96,23 +98,26 @@ it encounters. .IP \(bu 2 \-g \fIgamma\fP: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number. .RE +.SS Doing more +.RS +.IP \(bu 2 +\-ac: Coalesce adjacent line and polygon features that have the same properties +.IP \(bu 2 +\-ar: Try reversing the directions of lines to make them coalesce and compress better +.IP \(bu 2 +\-ao: Reorder features to put ones with the same properties in sequence, to try to get them to coalesce +.IP \(bu 2 +\-al: Let "dot" dropping at lower zooms apply to lines too +.RE .SS Doing less .RS .IP \(bu 2 \-ps: Don't simplify lines .IP \(bu 2 -\-pr: Don't reverse the direction of lines to make them coalesce better -.IP \(bu 2 -\-pc: Don't coalesce features with the same properties -.IP \(bu 2 \-pf: Don't limit tiles to 200,000 features .IP \(bu 2 \-pk: Don't limit tiles to 500K bytes .IP \(bu 2 -\-po: Don't reorder features to put the same properties in sequence -.IP \(bu 2 -\-pl: Let "dot" simplification apply to lines too -.IP \(bu 2 \-pd: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. .IP \(bu 2 \-q: Work quietly instead of reporting progress @@ -167,7 +172,7 @@ I don't know why 2.5 is the appropriate number, but the densities of many differ data sets fall off at about this same rate. You can use \-r to specify a different rate. .PP You can use the gamma option to thin out especially dense clusters of points. -For any area that where dots are closer than one pixel together (at whatever zoom level), +For any area where dots are closer than one pixel together (at whatever zoom level), a gamma of 3, for example, will reduce these clusters to the cube root of their original density. .PP For line features, it drops any features that are too small to draw at all. diff --git a/tile.cc b/tile.cc index ff20b9c..88dbff3 100644 --- a/tile.cc +++ b/tile.cc @@ -451,7 +451,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb } } -long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) { +long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional) { int line_detail; static bool evaluated = false; double oprogress = 0; @@ -580,7 +580,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f continue; } - if (gamma >= 0 && (t == VT_POINT || (prevent['l' & 0xFF] && t == VT_LINE))) { + if (gamma >= 0 && (t == VT_POINT || (additional['l' & 0xFF] && t == VT_LINE))) { seq++; if (seq >= 0) { seq -= interval; @@ -645,7 +645,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } #endif - if (t == VT_LINE && !prevent['r' & 0xFF]) { + if (t == VT_LINE && additional['r' & 0xFF]) { geom = reorder_lines(geom); } @@ -687,7 +687,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } for (j = 0; j < nlayers; j++) { - if (!prevent['o' & 0xFF]) { + if (additional['o' & 0xFF]) { std::sort(features[j].begin(), features[j].end()); } @@ -702,7 +702,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } #endif - if (!prevent['c' & 0xFF] && out.size() > 0 && out[y].geom.size() + features[j][x].geom.size() < 20000 && coalcmp(&features[j][x], &out[y]) == 0 && features[j][x].type != VT_POINT) { + if (additional['c' & 0xFF] && out.size() > 0 && out[y].geom.size() + features[j][x].geom.size() < 20000 && coalcmp(&features[j][x], &out[y]) == 0 && features[j][x].type != VT_POINT) { unsigned z; for (z = 0; z < features[j][x].geom.size(); z++) { out[y].geom.push_back(features[j][x].geom[z]); @@ -794,7 +794,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f return -1; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail) { int i; for (i = 0; i <= maxzoom; i++) { long long most = 0; @@ -855,7 +855,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(&geom, metabase, stringpool, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, min_detail, maxzoom, file_keys, layernames, outdb, droprate, buffer, fname, sub, minzoom, maxzoom, todo, geomstart, along, gamma, nlayers, prevent); + long long len = write_tile(&geom, metabase, stringpool, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, min_detail, maxzoom, file_keys, layernames, outdb, droprate, buffer, fname, sub, minzoom, maxzoom, todo, geomstart, along, gamma, nlayers, prevent, additional); if (len < 0) { return i - 1; diff --git a/tile.h b/tile.h index 87d2506..3094fe6 100644 --- a/tile.h +++ b/tile.h @@ -25,9 +25,9 @@ void deserialize_uint(char **f, unsigned *n); void deserialize_byte(char **f, signed char *n); struct pool_val *deserialize_string(char **f, struct pool *p, int type); -long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent); +long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail); extern unsigned initial_x, initial_y; extern int geometry_scale;