From 748ef3b1d5f94ce33d772f20aca667d18ba2e38d Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 9 Oct 2017 17:05:29 -0700 Subject: [PATCH] Add a tippecanoe-decode option to report tile size and feature count --- Makefile | 4 +- README.md | 2 + decode.cpp | 73 ++++++++++++++++++---- man/tippecanoe.1 | 4 ++ tests/muni/decode/multi.mbtiles.stats.json | 9 +++ 5 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 tests/muni/decode/multi.mbtiles.stats.json diff --git a/Makefile b/Makefile index b4b2773..3064920 100644 --- a/Makefile +++ b/Makefile @@ -141,9 +141,11 @@ decode-test: ./tippecanoe -z11 -Z11 -f -o tests/muni/decode/multi.mbtiles tests/muni/*.json ./tippecanoe-decode -l subway tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.json.check ./tippecanoe-decode -c tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.pipeline.json.check + ./tippecanoe-decode --stats tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.stats.json.check cmp tests/muni/decode/multi.mbtiles.json.check tests/muni/decode/multi.mbtiles.json cmp tests/muni/decode/multi.mbtiles.pipeline.json.check tests/muni/decode/multi.mbtiles.pipeline.json - rm -f tests/muni/decode/multi.mbtiles.json.check tests/muni/decode/multi.mbtiles tests/muni/decode/multi.mbtiles.pipeline.json.check + cmp tests/muni/decode/multi.mbtiles.stats.json.check tests/muni/decode/multi.mbtiles.stats.json + rm -f tests/muni/decode/multi.mbtiles.json.check tests/muni/decode/multi.mbtiles tests/muni/decode/multi.mbtiles.pipeline.json.check tests/muni/decode/multi.mbtiles.stats.json.check pbf-test: ./tippecanoe-decode tests/pbf/11-328-791.vector.pbf 11 328 791 > tests/pbf/11-328-791.vector.pbf.out diff --git a/README.md b/README.md index da0bf69..af3ae2d 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,7 @@ tippecanoe -z5 -o filtered.mbtiles -j '{ "ne_10m_admin_0_countries": [ "all", [ * `-as` or `--drop-densest-as-needed`: If a tile is too large, try to reduce it to under 500K by increasing the minimum spacing between features. The discovered spacing applies to the entire zoom level. * `-ad` or `--drop-fraction-as-needed`: Dynamically drop some fraction of features from each zoom level to keep large tiles under the 500K size limit. (This is like `-pd` but applies to the entire zoom level, not to each tile.) * `-an` or `--drop-smallest-as-needed`: Dynamically drop the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level to keep large tiles under the 500K size limit. This option will not work for point features. + * `-aN` or `--coalesce-smallest-as-needed`: Dynamically combine the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level into other nearby features to keep large tiles under the 500K size limit. This option will not work for point features, and will probably not help very much with LineStrings. It is mostly intended for polygons, to maintain the full original area covered by polygons while still reducing the feature count somehow. The attributes of the small polygons are *not* preserved into the combined features, only their geometry. * `-pd` or `--force-feature-limit`: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. (This is like `-ad` but applies to each tile individually, not to the entire zoom level.) You probably don't want to use this. ### Dropping tightly overlapping features @@ -612,4 +613,5 @@ resolutions. * `-Z` _minzoom_ or `--minimum-zoom=`*minzoom*: Specify the lowest zoom level to decode from the tileset * `-l` _layer_ or `--layer=`*layer*: Decode only layers with the specified names. (Multiple `-l` options can be specified.) * `-c` or `--tag-layer-and-zoom`: Include each feature's layer and zoom level as part of its `tippecanoe` object rather than as a FeatureCollection wrapper + * `-S` or `--stats`: Just report statistics about each tile's size and the number of features in it, as a JSON structure. * `-f` or `--force`: Decode tiles even if polygon ring order or closure problems are detected diff --git a/decode.cpp b/decode.cpp index dd3cc36..7079a98 100644 --- a/decode.cpp +++ b/decode.cpp @@ -23,7 +23,34 @@ int minzoom = 0; int maxzoom = 32; bool force = false; -void handle(std::string message, int z, unsigned x, unsigned y, int describe, std::set const &to_decode, bool pipeline) { +void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, unsigned y) { + printf("{ \"zoom\": %d, \"x\": %u, \"y\": %u, \"bytes\": %zu, \"compressed\": %s", z, x, y, size, compressed ? "true" : "false"); + + printf(", \"layers\": { "); + for (size_t i = 0; i < tile.layers.size(); i++) { + if (i != 0) { + printf(", "); + } + fprintq(stdout, tile.layers[i].name.c_str()); + + int points = 0, lines = 0, polygons = 0; + for (size_t j = 0; j < tile.layers[i].features.size(); j++) { + if (tile.layers[i].features[j].type == mvt_point) { + points++; + } else if (tile.layers[i].features[j].type == mvt_linestring) { + lines++; + } else if (tile.layers[i].features[j].type == mvt_polygon) { + polygons++; + } + } + + printf(": { \"points\": %d, \"lines\": %d, \"polygons\": %d, \"extent\": %lld }", points, lines, polygons, tile.layers[i].extent); + } + + printf(" } }\n"); +} + +void handle(std::string message, int z, unsigned x, unsigned y, int describe, std::set const &to_decode, bool pipeline, bool stats) { mvt_tile tile; bool was_compressed; @@ -37,6 +64,11 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st exit(EXIT_FAILURE); } + if (stats) { + do_stats(tile, message.size(), was_compressed, z, x, y); + return; + } + if (!pipeline) { printf("{ \"type\": \"FeatureCollection\""); @@ -96,7 +128,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st } } -void decode(char *fname, int z, unsigned x, unsigned y, std::set const &to_decode, bool pipeline) { +void decode(char *fname, int z, unsigned x, unsigned y, std::set const &to_decode, bool pipeline, bool stats) { sqlite3 *db; int oz = z; unsigned ox = x, oy = y; @@ -111,7 +143,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co if (strcmp(map, "SQLite format 3") != 0) { if (z >= 0) { std::string s = std::string(map, st.st_size); - handle(s, z, x, y, 1, to_decode, pipeline); + handle(s, z, x, y, 1, to_decode, pipeline, stats); munmap(map, st.st_size); return; } else { @@ -141,7 +173,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co if (z < 0) { int within = 0; - if (!pipeline) { + if (!pipeline && !stats) { printf("{ \"type\": \"FeatureCollection\", \"properties\": {\n"); const char *sql2 = "SELECT name, value from metadata order by name;"; @@ -168,6 +200,10 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co sqlite3_finalize(stmt2); } + if (stats) { + printf("[\n"); + } + const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;"; sqlite3_stmt *stmt; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { @@ -178,13 +214,19 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co sqlite3_bind_int(stmt, 1, minzoom); sqlite3_bind_int(stmt, 2, maxzoom); - if (!pipeline) { + if (!pipeline && !stats) { printf("\n}, \"features\": [\n"); } within = 0; while (sqlite3_step(stmt) == SQLITE_ROW) { - if (!pipeline) { + if (!pipeline && !stats) { + if (within) { + printf(",\n"); + } + within = 1; + } + if (stats) { if (within) { printf(",\n"); } @@ -198,12 +240,15 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co ty = (1LL << tz) - 1 - ty; const char *s = (const char *) sqlite3_column_blob(stmt, 0); - handle(std::string(s, len), tz, tx, ty, 1, to_decode, pipeline); + handle(std::string(s, len), tz, tx, ty, 1, to_decode, pipeline, stats); } - if (!pipeline) { + if (!pipeline && !stats) { printf("] }\n"); } + if (stats) { + printf("]\n"); + } sqlite3_finalize(stmt); } else { @@ -228,7 +273,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy); } - handle(std::string(s, len), z, x, y, 0, to_decode, pipeline); + handle(std::string(s, len), z, x, y, 0, to_decode, pipeline, stats); handled = 1; } @@ -257,6 +302,7 @@ int main(int argc, char **argv) { int i; std::set to_decode; bool pipeline = false; + bool stats = false; struct option long_options[] = { {"projection", required_argument, 0, 's'}, @@ -264,6 +310,7 @@ int main(int argc, char **argv) { {"minimum-zoom", required_argument, 0, 'Z'}, {"layer", required_argument, 0, 'l'}, {"tag-layer-and-zoom", no_argument, 0, 'c'}, + {"stats", no_argument, 0, 'S'}, {"force", no_argument, 0, 'f'}, {0, 0, 0, 0}, }; @@ -304,6 +351,10 @@ int main(int argc, char **argv) { pipeline = true; break; + case 'S': + stats = true; + break; + case 'f': force = true; break; @@ -314,9 +365,9 @@ int main(int argc, char **argv) { } if (argc == optind + 4) { - decode(argv[optind], atoi(argv[optind + 1]), atoi(argv[optind + 2]), atoi(argv[optind + 3]), to_decode, pipeline); + decode(argv[optind], atoi(argv[optind + 1]), atoi(argv[optind + 2]), atoi(argv[optind + 3]), to_decode, pipeline, stats); } else if (argc == optind + 1) { - decode(argv[optind], -1, -1, -1, to_decode, pipeline); + decode(argv[optind], -1, -1, -1, to_decode, pipeline, stats); } else { usage(argv); } diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 880cf80..9c33727 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -237,6 +237,8 @@ compensate for the larger marker, or \fB\fC\-Bf\fR\fInumber\fP to allow at most .IP \(bu 2 \fB\fC\-an\fR or \fB\fC\-\-drop\-smallest\-as\-needed\fR: Dynamically drop the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level to keep large tiles under the 500K size limit. This option will not work for point features. .IP \(bu 2 +\fB\fC\-aN\fR or \fB\fC\-\-coalesce\-smallest\-as\-needed\fR: Dynamically combine the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level into other nearby features to keep large tiles under the 500K size limit. This option will not work for point features, and will probably not help very much with LineStrings. It is mostly intended for polygons, to maintain the full original area covered by polygons while still reducing the feature count somehow. The attributes of the small polygons are \fInot\fP preserved into the combined features, only their geometry. +.IP \(bu 2 \fB\fC\-pd\fR or \fB\fC\-\-force\-feature\-limit\fR: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. (This is like \fB\fC\-ad\fR but applies to each tile individually, not to the entire zoom level.) You probably don't want to use this. .RE .SS Dropping tightly overlapping features @@ -726,5 +728,7 @@ resolutions. .IP \(bu 2 \fB\fC\-c\fR or \fB\fC\-\-tag\-layer\-and\-zoom\fR: Include each feature's layer and zoom level as part of its \fB\fCtippecanoe\fR object rather than as a FeatureCollection wrapper .IP \(bu 2 +\fB\fC\-S\fR or \fB\fC\-\-stats\fR: Just report statistics about each tile's size and the number of features in it, as a JSON structure. +.IP \(bu 2 \fB\fC\-f\fR or \fB\fC\-\-force\fR: Decode tiles even if polygon ring order or closure problems are detected .RE diff --git a/tests/muni/decode/multi.mbtiles.stats.json b/tests/muni/decode/multi.mbtiles.stats.json new file mode 100644 index 0000000..37215ba --- /dev/null +++ b/tests/muni/decode/multi.mbtiles.stats.json @@ -0,0 +1,9 @@ +[ +{ "zoom": 11, "x": 326, "y": 791, "bytes": 372, "compressed": true, "layers": { "muni": { "points": 14, "lines": 0, "polygons": 0, "extent": 4096 } } } +, +{ "zoom": 11, "x": 327, "y": 792, "bytes": 6481, "compressed": true, "layers": { "muni": { "points": 528, "lines": 0, "polygons": 0, "extent": 4096 } } } +, +{ "zoom": 11, "x": 327, "y": 791, "bytes": 44376, "compressed": true, "layers": { "muni": { "points": 4285, "lines": 0, "polygons": 0, "extent": 4096 }, "subway": { "points": 19, "lines": 0, "polygons": 0, "extent": 4096 } } } +, +{ "zoom": 11, "x": 954, "y": 791, "bytes": 75, "compressed": true, "layers": { "muni": { "points": 12, "lines": 0, "polygons": 0, "extent": 4096 } } } +]