From 6ec453229c2e7d7cbc507dadbcddddb390e4b48f Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 4 May 2017 13:30:45 -0700 Subject: [PATCH 1/8] Preserve the tileset names from the source mbtiles in tile-join --- CHANGELOG.md | 4 ++++ tests/join-population/joined-i.mbtiles.json | 2 +- tests/join-population/joined.mbtiles.json | 2 +- .../join-population/just-macarthur.mbtiles.json | 2 +- tests/join-population/merged.mbtiles.json | 2 +- tests/join-population/no-macarthur.mbtiles.json | 2 +- tests/join-population/windows.mbtiles.json | 2 +- tile-join.cpp | 17 ++++++++++++++--- version.hpp | 2 +- 9 files changed, 25 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0ffb96..2d08ebd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.17.5 + +* Preserve the tileset names from the source mbtiles in tile-join + ## 1.17.4 * Fix RFC 8142 support: Don't try to split *all* memory mapped files diff --git a/tests/join-population/joined-i.mbtiles.json b/tests/join-population/joined-i.mbtiles.json index d69dd18..e32ec68 100644 --- a/tests/join-population/joined-i.mbtiles.json +++ b/tests/join-population/joined-i.mbtiles.json @@ -6,7 +6,7 @@ "json": "{\"vector_layers\": [ { \"id\": \"tabblock_06001420\", \"description\": \"\", \"minzoom\": 3, \"maxzoom\": 12, \"fields\": {\"ALAND10\": \"Number\", \"AWATER10\": \"Number\", \"BLOCKCE10\": \"String\", \"COUNTYFP10\": \"String\", \"FUNCSTAT10\": \"String\", \"INTPTLAT10\": \"String\", \"INTPTLON10\": \"String\", \"MTFCC10\": \"String\", \"NAME10\": \"String\", \"STATEFP10\": \"String\", \"TRACTCE10\": \"String\", \"UACE10\": \"String\", \"UATYP10\": \"String\", \"UR10\": \"String\", \"population\": \"Number\"} } ] }", "maxzoom": "12", "minzoom": "0", -"name": "tests/join-population/joined-i.mbtiles", +"name": "tests/join-population/tabblock_06001420.mbtiles", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/join-population/joined.mbtiles.json b/tests/join-population/joined.mbtiles.json index da4ed6c..0e45d02 100644 --- a/tests/join-population/joined.mbtiles.json +++ b/tests/join-population/joined.mbtiles.json @@ -6,7 +6,7 @@ "json": "{\"vector_layers\": [ { \"id\": \"tabblock_06001420\", \"description\": \"\", \"minzoom\": 3, \"maxzoom\": 12, \"fields\": {\"ALAND10\": \"Number\", \"AWATER10\": \"Number\", \"BLOCKCE10\": \"String\", \"COUNTYFP10\": \"String\", \"FUNCSTAT10\": \"String\", \"INTPTLAT10\": \"String\", \"INTPTLON10\": \"String\", \"MTFCC10\": \"String\", \"NAME10\": \"String\", \"STATEFP10\": \"String\", \"TRACTCE10\": \"String\", \"UACE10\": \"String\", \"UATYP10\": \"String\", \"UR10\": \"String\", \"population\": \"Number\"} } ] }", "maxzoom": "12", "minzoom": "0", -"name": "tests/join-population/joined.mbtiles", +"name": "tests/join-population/tabblock_06001420.mbtiles", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/join-population/just-macarthur.mbtiles.json b/tests/join-population/just-macarthur.mbtiles.json index babe200..3c2f362 100644 --- a/tests/join-population/just-macarthur.mbtiles.json +++ b/tests/join-population/just-macarthur.mbtiles.json @@ -6,7 +6,7 @@ "json": "{\"vector_layers\": [ { \"id\": \"macarthur\", \"description\": \"\", \"minzoom\": 5, \"maxzoom\": 11, \"fields\": {\"FULLNAME\": \"String\", \"LINEARID\": \"String\", \"MTFCC\": \"String\", \"RTTYP\": \"String\"} } ] }", "maxzoom": "12", "minzoom": "0", -"name": "tests/join-population/just-macarthur.mbtiles", +"name": "tests/join-population/macarthur.mbtiles + tests/join-population/macarthur2.mbtiles + tests/join-population/tabblock_06001420.mbtiles", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/join-population/merged.mbtiles.json b/tests/join-population/merged.mbtiles.json index ff5f3c8..1cb3323 100644 --- a/tests/join-population/merged.mbtiles.json +++ b/tests/join-population/merged.mbtiles.json @@ -6,7 +6,7 @@ "json": "{\"vector_layers\": [ { \"id\": \"macarthur\", \"description\": \"\", \"minzoom\": 5, \"maxzoom\": 11, \"fields\": {\"FULLNAME\": \"String\", \"LINEARID\": \"String\", \"MTFCC\": \"String\", \"RTTYP\": \"String\"} }, { \"id\": \"tabblock_06001420\", \"description\": \"\", \"minzoom\": 3, \"maxzoom\": 12, \"fields\": {\"ALAND10\": \"Number\", \"AWATER10\": \"Number\", \"BLOCKCE10\": \"String\", \"COUNTYFP10\": \"String\", \"FUNCSTAT10\": \"String\", \"GEOID10\": \"String\", \"INTPTLAT10\": \"String\", \"INTPTLON10\": \"String\", \"MTFCC10\": \"String\", \"NAME10\": \"String\", \"STATEFP10\": \"String\", \"TRACTCE10\": \"String\", \"UACE10\": \"String\", \"UATYP10\": \"String\", \"UR10\": \"String\"} } ] }", "maxzoom": "12", "minzoom": "0", -"name": "tests/join-population/merged.mbtiles", +"name": "tests/join-population/macarthur.mbtiles + tests/join-population/macarthur2.mbtiles + tests/join-population/tabblock_06001420.mbtiles", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/join-population/no-macarthur.mbtiles.json b/tests/join-population/no-macarthur.mbtiles.json index 0288448..5dcb5e8 100644 --- a/tests/join-population/no-macarthur.mbtiles.json +++ b/tests/join-population/no-macarthur.mbtiles.json @@ -6,7 +6,7 @@ "json": "{\"vector_layers\": [ { \"id\": \"tabblock_06001420\", \"description\": \"\", \"minzoom\": 3, \"maxzoom\": 12, \"fields\": {\"ALAND10\": \"Number\", \"AWATER10\": \"Number\", \"BLOCKCE10\": \"String\", \"COUNTYFP10\": \"String\", \"FUNCSTAT10\": \"String\", \"GEOID10\": \"String\", \"INTPTLAT10\": \"String\", \"INTPTLON10\": \"String\", \"MTFCC10\": \"String\", \"NAME10\": \"String\", \"STATEFP10\": \"String\", \"TRACTCE10\": \"String\", \"UACE10\": \"String\", \"UATYP10\": \"String\", \"UR10\": \"String\"} } ] }", "maxzoom": "12", "minzoom": "0", -"name": "tests/join-population/no-macarthur.mbtiles", +"name": "tests/join-population/macarthur.mbtiles + tests/join-population/macarthur2.mbtiles + tests/join-population/tabblock_06001420.mbtiles", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/join-population/windows.mbtiles.json b/tests/join-population/windows.mbtiles.json index 2874056..825bae6 100644 --- a/tests/join-population/windows.mbtiles.json +++ b/tests/join-population/windows.mbtiles.json @@ -6,7 +6,7 @@ "json": "{\"vector_layers\": [ { \"id\": \"macarthur\", \"description\": \"\", \"minzoom\": 5, \"maxzoom\": 10, \"fields\": {\"FULLNAME\": \"String\", \"LINEARID\": \"String\", \"MTFCC\": \"String\", \"RTTYP\": \"String\"} } ] }", "maxzoom": "10", "minzoom": "5", -"name": "tests/join-population/windows.mbtiles", +"name": "tests/join-population/macarthur.mbtiles", "type": "overlay", "version": "2" }, "features": [ diff --git a/tile-join.cpp b/tile-join.cpp index 40229aa..4a6a99e 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -464,7 +464,7 @@ void handle_tasks(std::map> &tasks, std::vector &layermap, sqlite3 *outdb, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers) { +void decode(struct reader *readers, char *map, std::map &layermap, sqlite3 *outdb, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name) { std::vector> layermaps; for (size_t i = 0; i < CPUS; i++) { layermaps.push_back(std::map()); @@ -558,6 +558,16 @@ void decode(struct reader *readers, char *map, std::mapstmt); } + if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'name'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + if (name.size() == 0) { + name = std::string((char *) sqlite3_column_text(r->stmt, 0)); + } else { + name += " + " + std::string((char *) sqlite3_column_text(r->stmt, 0)); + } + } + sqlite3_finalize(r->stmt); + } if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'bounds'", -1, &r->stmt, NULL) == SQLITE_OK) { if (sqlite3_step(r->stmt) == SQLITE_ROW) { const unsigned char *s = sqlite3_column_text(r->stmt, 0); @@ -752,6 +762,7 @@ int main(int argc, char **argv) { std::map layermap; std::string attribution; std::string description; + std::string name; struct reader *readers = NULL; @@ -769,9 +780,9 @@ int main(int argc, char **argv) { *rr = r; } - decode(readers, csv, layermap, outdb, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers); + decode(readers, csv, layermap, outdb, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name); - mbtiles_write_metadata(outdb, NULL, outfile, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str()); + mbtiles_write_metadata(outdb, NULL, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str()); mbtiles_close(outdb, argv); return 0; diff --git a/version.hpp b/version.hpp index 65d04a1..5d1bd52 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.17.4\n" +#define VERSION "tippecanoe v1.17.5\n" From f87a9d166008fb0d0980ca1e6ac4229337ab60c5 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 4 May 2017 15:08:31 -0700 Subject: [PATCH 2/8] Add tile-join options to set tileset name, description, attribution --- CHANGELOG.md | 4 +++ Makefile | 2 +- README.md | 3 +++ man/tippecanoe.1 | 6 +++++ .../just-macarthur.mbtiles.json | 5 ++-- tile-join.cpp | 26 ++++++++++++++++++- version.hpp | 2 +- 7 files changed, 43 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d08ebd..67ef478 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.17.6 + +* Add tile-join options to set name, attribution, description + ## 1.17.5 * Preserve the tileset names from the source mbtiles in tile-join diff --git a/Makefile b/Makefile index 0024ce5..47388c9 100644 --- a/Makefile +++ b/Makefile @@ -157,7 +157,7 @@ join-test: cmp tests/join-population/joined-i.mbtiles.json.check tests/join-population/joined-i.mbtiles.json cmp tests/join-population/merged.mbtiles.json.check tests/join-population/merged.mbtiles.json cmp tests/join-population/windows.mbtiles.json.check tests/join-population/windows.mbtiles.json - ./tile-join -f -l macarthur -o tests/join-population/just-macarthur.mbtiles tests/join-population/merged.mbtiles + ./tile-join -f -l macarthur -n "macarthur name" -N "macarthur description" -A "macarthur attribution" -o tests/join-population/just-macarthur.mbtiles tests/join-population/merged.mbtiles ./tile-join -f -L macarthur -o tests/join-population/no-macarthur.mbtiles tests/join-population/merged.mbtiles ./tippecanoe-decode tests/join-population/just-macarthur.mbtiles > tests/join-population/just-macarthur.mbtiles.json.check ./tippecanoe-decode tests/join-population/no-macarthur.mbtiles > tests/join-population/no-macarthur.mbtiles.json.check diff --git a/README.md b/README.md index 00f799b..c536122 100644 --- a/README.md +++ b/README.md @@ -392,6 +392,9 @@ The options are: * `-pk`: Don't skip tiles larger than 500K. * `-l` *layer*: Include the named layer in the output. You can specify multiple `-l` options to keep multiple layers. If you don't specify, they will all be retained. * `-L` *layer*: Remove the named layer from the output. You can specify multiple `-L` options to remove multiple layers. + * `-A` *attribution*: Set the attribution string. + * `-n` *name*: Set the tileset name. + * `-N` *description*: Set the tileset description. Because tile-join just copies the geometries to the new .mbtiles without processing them (except to rescale the extents if necessary), diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index d80b17e..010a659 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -457,6 +457,12 @@ The options are: \fB\fC\-l\fR \fIlayer\fP: Include the named layer in the output. You can specify multiple \fB\fC\-l\fR options to keep multiple layers. If you don't specify, they will all be retained. .IP \(bu 2 \fB\fC\-L\fR \fIlayer\fP: Remove the named layer from the output. You can specify multiple \fB\fC\-L\fR options to remove multiple layers. +.IP \(bu 2 +\fB\fC\-A\fR \fIattribution\fP: Set the attribution string. +.IP \(bu 2 +\fB\fC\-n\fR \fIname\fP: Set the tileset name. +.IP \(bu 2 +\fB\fC\-N\fR \fIdescription\fP: Set the tileset description. .RE .PP Because tile\-join just copies the geometries to the new .mbtiles without processing them diff --git a/tests/join-population/just-macarthur.mbtiles.json b/tests/join-population/just-macarthur.mbtiles.json index 3c2f362..faeed2b 100644 --- a/tests/join-population/just-macarthur.mbtiles.json +++ b/tests/join-population/just-macarthur.mbtiles.json @@ -1,12 +1,13 @@ { "type": "FeatureCollection", "properties": { +"attribution": "macarthur attribution", "bounds": "-122.334735,37.695438,-122.104097,37.898925", "center": "-122.299805,37.892187,12", -"description": "tests/join-population/tabblock_06001420.mbtiles", +"description": "macarthur description", "format": "pbf", "json": "{\"vector_layers\": [ { \"id\": \"macarthur\", \"description\": \"\", \"minzoom\": 5, \"maxzoom\": 11, \"fields\": {\"FULLNAME\": \"String\", \"LINEARID\": \"String\", \"MTFCC\": \"String\", \"RTTYP\": \"String\"} } ] }", "maxzoom": "12", "minzoom": "0", -"name": "tests/join-population/macarthur.mbtiles + tests/join-population/macarthur2.mbtiles + tests/join-population/tabblock_06001420.mbtiles", +"name": "macarthur name", "type": "overlay", "version": "2" }, "features": [ diff --git a/tile-join.cpp b/tile-join.cpp index 4a6a99e..3454e5e 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -691,11 +691,13 @@ int main(int argc, char **argv) { std::set keep_layers; std::set remove_layers; + std::string set_name, set_description, set_attribution; + extern int optind; extern char *optarg; int i; - while ((i = getopt(argc, argv, "fo:c:x:ip:l:L:")) != -1) { + while ((i = getopt(argc, argv, "fo:c:x:ip:l:L:A:N:n:")) != -1) { switch (i) { case 'o': outfile = optarg; @@ -709,6 +711,18 @@ int main(int argc, char **argv) { ifmatched = 1; break; + case 'A': + set_attribution = optarg; + break; + + case 'n': + set_name = optarg; + break; + + case 'N': + set_description = optarg; + break; + case 'p': if (strcmp(optarg, "k") == 0) { pk = true; @@ -782,6 +796,16 @@ int main(int argc, char **argv) { decode(readers, csv, layermap, outdb, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name); + if (set_attribution.size() != 0) { + attribution = set_attribution; + } + if (set_description.size() != 0) { + description = set_description; + } + if (set_name.size() != 0) { + name = set_name; + } + mbtiles_write_metadata(outdb, NULL, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str()); mbtiles_close(outdb, argv); diff --git a/version.hpp b/version.hpp index 5d1bd52..22946d4 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.17.5\n" +#define VERSION "tippecanoe v1.17.6\n" From 9eb3a7f7ecfa84852d70a98dbd4070f1bcc502b8 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 5 May 2017 10:56:50 -0700 Subject: [PATCH 3/8] Enforce polygon winding and closure rules in tippecanoe-decode --- CHANGELOG.md | 4 ++++ README.md | 1 + decode.cpp | 23 ++++++++++++++++++++++- man/tippecanoe.1 | 2 ++ version.hpp | 2 +- 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67ef478..00cf0ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.17.7 + +* Enforce polygon winding and closure rules in tippecanoe-decode + ## 1.17.6 * Add tile-join options to set name, attribution, description diff --git a/README.md b/README.md index c536122..0d5745e 100644 --- a/README.md +++ b/README.md @@ -482,3 +482,4 @@ resolutions. * `-z` _maxzoom_: Specify the highest zoom level to decode from the tileset * `-Z` _minzoom_: Specify the lowest zoom level to decode from the tileset * `-l` _layer_: Decode only layers with the specified names. (Multiple `-l` options can be specified.) + * `-f`: Decode tiles even if polygon ring order or closure problems are detected diff --git a/decode.cpp b/decode.cpp index cc84daf..c55288e 100644 --- a/decode.cpp +++ b/decode.cpp @@ -19,6 +19,7 @@ int minzoom = 0; int maxzoom = 32; +bool force = false; void printq(const char *s) { putchar('"'); @@ -261,6 +262,15 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st rings[n].push_back(ops[i]); } } + + if (i >= ops.size() || ops[i + 1].op == VT_MOVETO) { + if (ops[i].op != VT_CLOSEPATH) { + fprintf(stderr, "Ring does not end with closepath (ends with %d)\n", ops[i].op); + if (!force) { + exit(EXIT_FAILURE); + } + } + } } int outer = 0; @@ -290,6 +300,13 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st int state = 0; for (size_t i = 0; i < rings.size(); i++) { + if (i == 0 && areas[i] < 0) { + fprintf(stderr, "Polygon begins with an inner ring\n"); + if (!force) { + exit(EXIT_FAILURE); + } + } + if (areas[i] >= 0) { if (state != 0) { // new multipolygon @@ -489,7 +506,7 @@ int main(int argc, char **argv) { int i; std::set to_decode; - while ((i = getopt(argc, argv, "t:Z:z:l:")) != -1) { + while ((i = getopt(argc, argv, "t:Z:z:l:f")) != -1) { switch (i) { case 't': set_projection_or_exit(optarg); @@ -507,6 +524,10 @@ int main(int argc, char **argv) { to_decode.insert(optarg); break; + case 'f': + force = true; + break; + default: usage(argv); } diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 010a659..167a114 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -568,4 +568,6 @@ resolutions. \fB\fC\-Z\fR \fIminzoom\fP: Specify the lowest zoom level to decode from the tileset .IP \(bu 2 \fB\fC\-l\fR \fIlayer\fP: Decode only layers with the specified names. (Multiple \fB\fC\-l\fR options can be specified.) +.IP \(bu 2 +\fB\fC\-f\fR: Decode tiles even if polygon ring order or closure problems are detected .RE diff --git a/version.hpp b/version.hpp index 22946d4..c70088e 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.17.6\n" +#define VERSION "tippecanoe v1.17.7\n" From 922bef72c83a0b223f6483faf56a2e12f6814030 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 5 May 2017 11:22:40 -0700 Subject: [PATCH 4/8] Fix array bounds error found by -fsanitize=address --- decode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decode.cpp b/decode.cpp index c55288e..ecc7232 100644 --- a/decode.cpp +++ b/decode.cpp @@ -263,7 +263,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st } } - if (i >= ops.size() || ops[i + 1].op == VT_MOVETO) { + if (i + 1 >= ops.size() || ops[i + 1].op == VT_MOVETO) { if (ops[i].op != VT_CLOSEPATH) { fprintf(stderr, "Ring does not end with closepath (ends with %d)\n", ops[i].op); if (!force) { From 30d54ff50d5c6f50965987051b1c200440375b0b Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 11 May 2017 12:08:47 -0700 Subject: [PATCH 5/8] Tile-join had not been compressing its tiles. --- decode.cpp | 10 ++++++++-- mvt.cpp | 4 +++- mvt.hpp | 2 +- tile-join.cpp | 7 +++++-- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/decode.cpp b/decode.cpp index ecc7232..dca817a 100644 --- a/decode.cpp +++ b/decode.cpp @@ -54,9 +54,10 @@ struct lonlat { void handle(std::string message, int z, unsigned x, unsigned y, int describe, std::set const &to_decode) { int within = 0; mvt_tile tile; + bool was_compressed; try { - if (!tile.decode(message)) { + if (!tile.decode(message, was_compressed)) { fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y); exit(EXIT_FAILURE); } @@ -68,7 +69,12 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st printf("{ \"type\": \"FeatureCollection\""); if (describe) { - printf(", \"properties\": { \"zoom\": %d, \"x\": %d, \"y\": %d }", z, x, y); + printf(", \"properties\": { \"zoom\": %d, \"x\": %d, \"y\": %d", z, x, y); + if (!was_compressed) { + printf(", \"compressed\": false"); + } + + printf(" }"); if (projection != projections) { printf(", \"crs\": { \"type\": \"name\", \"properties\": { \"name\": "); diff --git a/mvt.cpp b/mvt.cpp index 22198e9..a7bf259 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -79,7 +79,7 @@ int compress(std::string const &input, std::string &output) { return 0; } -bool mvt_tile::decode(std::string &message) { +bool mvt_tile::decode(std::string &message, bool &was_compressed) { layers.clear(); std::string src; @@ -87,8 +87,10 @@ bool mvt_tile::decode(std::string &message) { std::string uncompressed; decompress(message, uncompressed); src = uncompressed; + was_compressed = true; } else { src = message; + was_compressed = false; } protozero::pbf_reader reader(src); diff --git a/mvt.hpp b/mvt.hpp index 6e9d792..7d5f334 100644 --- a/mvt.hpp +++ b/mvt.hpp @@ -91,7 +91,7 @@ struct mvt_tile { std::vector layers; std::string encode(); - bool decode(std::string &message); + bool decode(std::string &message, bool &was_compressed); }; bool is_compressed(std::string const &data); diff --git a/tile-join.cpp b/tile-join.cpp index 3454e5e..3677337 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -32,8 +32,9 @@ struct stats { void handle(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, std::set &exclude, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile) { mvt_tile tile; int features_added = 0; + bool was_compressed; - if (!tile.decode(message)) { + if (!tile.decode(message, was_compressed)) { fprintf(stderr, "Couldn't decompress tile %d/%u/%u\n", z, x, y); exit(EXIT_FAILURE); } @@ -402,7 +403,9 @@ void *join_worker(void *v) { } if (anything) { - std::string compressed = tile.encode(); + std::string pbf = tile.encode(); + std::string compressed; + compress(pbf, compressed); if (!pk && compressed.size() > 500000) { fprintf(stderr, "Tile %lld/%lld/%lld size is %lld, >500000. Skipping this tile\n.", ai->first.z, ai->first.x, ai->first.y, (long long) compressed.size()); From fb6551c59ec913d8c7df087069b3b6806cda4e76 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 11 May 2017 12:36:35 -0700 Subject: [PATCH 6/8] Calculate the tileset bounding box in tile-join from the tile boundaries --- CHANGELOG.md | 5 +++ tests/join-population/joined-i.mbtiles.json | 2 +- tests/join-population/joined.mbtiles.json | 2 +- .../just-macarthur.mbtiles.json | 2 +- tests/join-population/merged.mbtiles.json | 2 +- .../join-population/no-macarthur.mbtiles.json | 2 +- tests/join-population/windows.mbtiles.json | 2 +- tile-join.cpp | 40 ++++++++++++++++--- version.hpp | 2 +- 9 files changed, 46 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00cf0ba..2f1f232 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.18.0 + +* Fix compression of tiles in tile-join +* Calculate the tileset bounding box in tile-join from the tile boundaries + ## 1.17.7 * Enforce polygon winding and closure rules in tippecanoe-decode diff --git a/tests/join-population/joined-i.mbtiles.json b/tests/join-population/joined-i.mbtiles.json index e32ec68..6aa8bae 100644 --- a/tests/join-population/joined-i.mbtiles.json +++ b/tests/join-population/joined-i.mbtiles.json @@ -1,5 +1,5 @@ { "type": "FeatureCollection", "properties": { -"bounds": "-122.334735,37.877356,-122.281639,37.898925", +"bounds": "-122.343750,37.857507,-122.255859,37.926868", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined.mbtiles.json b/tests/join-population/joined.mbtiles.json index 0e45d02..8aa68e2 100644 --- a/tests/join-population/joined.mbtiles.json +++ b/tests/join-population/joined.mbtiles.json @@ -1,5 +1,5 @@ { "type": "FeatureCollection", "properties": { -"bounds": "-122.334735,37.877356,-122.281639,37.898925", +"bounds": "-122.343750,37.857507,-122.255859,37.926868", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/just-macarthur.mbtiles.json b/tests/join-population/just-macarthur.mbtiles.json index faeed2b..a902f12 100644 --- a/tests/join-population/just-macarthur.mbtiles.json +++ b/tests/join-population/just-macarthur.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { "attribution": "macarthur attribution", -"bounds": "-122.334735,37.695438,-122.104097,37.898925", +"bounds": "-122.343750,37.695438,-122.104097,37.926868", "center": "-122.299805,37.892187,12", "description": "macarthur description", "format": "pbf", diff --git a/tests/join-population/merged.mbtiles.json b/tests/join-population/merged.mbtiles.json index 1cb3323..ea26226 100644 --- a/tests/join-population/merged.mbtiles.json +++ b/tests/join-population/merged.mbtiles.json @@ -1,5 +1,5 @@ { "type": "FeatureCollection", "properties": { -"bounds": "-122.334735,37.695438,-122.104097,37.898925", +"bounds": "-122.343750,37.695438,-122.104097,37.926868", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/no-macarthur.mbtiles.json b/tests/join-population/no-macarthur.mbtiles.json index 5dcb5e8..fa5a04c 100644 --- a/tests/join-population/no-macarthur.mbtiles.json +++ b/tests/join-population/no-macarthur.mbtiles.json @@ -1,5 +1,5 @@ { "type": "FeatureCollection", "properties": { -"bounds": "-122.334735,37.695438,-122.104097,37.898925", +"bounds": "-122.343750,37.695438,-122.104097,37.926868", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/windows.mbtiles.json b/tests/join-population/windows.mbtiles.json index 825bae6..acdd7e9 100644 --- a/tests/join-population/windows.mbtiles.json +++ b/tests/join-population/windows.mbtiles.json @@ -1,5 +1,5 @@ { "type": "FeatureCollection", "properties": { -"bounds": "-122.294563,37.695438,-122.104097,37.833010", +"bounds": "-122.343750,37.439974,-121.992188,37.996163", "center": "-122.167969,37.833010,10", "description": "tests/join-population/macarthur.mbtiles", "format": "pbf", diff --git a/tile-join.cpp b/tile-join.cpp index 3677337..f06107c 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -475,11 +475,34 @@ void decode(struct reader *readers, char *map, std::map> tasks; + double minlat = INT_MAX; + double minlon = INT_MAX; + double maxlat = INT_MIN; + double maxlon = INT_MIN; + int zoom_for_bbox = -1; + while (readers != NULL && readers->zoom < 32) { reader *r = readers; readers = readers->next; r->next = NULL; + if (r->zoom != zoom_for_bbox) { + // Only use highest zoom for bbox calculation + // to avoid z0 always covering the world + + minlat = minlon = INT_MAX; + maxlat = maxlon = INT_MIN; + zoom_for_bbox = r->zoom; + } + + double lat1, lon1, lat2, lon2; + tile2lonlat(r->x, r->y, r->zoom, &lon1, &lat1); + tile2lonlat(r->x + 1, r->y + 1, r->zoom, &lon2, &lat2); + minlat = min(lat2, minlat); + minlon = min(lon1, minlon); + maxlat = max(lat1, maxlat); + maxlon = max(lon2, maxlon); + zxy tile = zxy(r->zoom, r->x, r->y); if (tasks.count(tile) == 0) { tasks.insert(std::pair>(tile, std::vector())); @@ -520,6 +543,11 @@ void decode(struct reader *readers, char *map, std::mapminlon = min(minlon, st->minlon); + st->maxlon = max(maxlon, st->maxlon); + st->minlat = min(minlat, st->minlat); + st->maxlat = max(maxlat, st->maxlat); + handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched, keep_layers, remove_layers); layermap = merge_layermaps(layermaps); @@ -574,12 +602,12 @@ void decode(struct reader *readers, char *map, std::mapdb, "SELECT value from metadata where name = 'bounds'", -1, &r->stmt, NULL) == SQLITE_OK) { if (sqlite3_step(r->stmt) == SQLITE_ROW) { const unsigned char *s = sqlite3_column_text(r->stmt, 0); - double minlon, minlat, maxlon, maxlat; - sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat); - st->minlon = min(minlon, st->minlon); - st->maxlon = max(maxlon, st->maxlon); - st->minlat = min(minlat, st->minlat); - st->maxlat = max(maxlat, st->maxlat); + if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) { + st->minlon = min(minlon, st->minlon); + st->maxlon = max(maxlon, st->maxlon); + st->minlat = min(minlat, st->minlat); + st->maxlat = max(maxlat, st->maxlat); + } } sqlite3_finalize(r->stmt); } diff --git a/version.hpp b/version.hpp index c70088e..babfe73 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.17.7\n" +#define VERSION "tippecanoe v1.18.0\n" From 9632c14551665c63505f78532b5f5373533eeb14 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 12 May 2017 11:15:27 -0700 Subject: [PATCH 7/8] Only warn once about invalid polygon encoding in tippecanoe-decode --- CHANGELOG.md | 4 ++++ decode.cpp | 24 ++++++++++++++++++------ version.hpp | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f1f232..11d885c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.18.1 + +* Only warn once about invalid polygons in tippecanoe-decode + ## 1.18.0 * Fix compression of tiles in tile-join diff --git a/decode.cpp b/decode.cpp index dca817a..e90a4bc 100644 --- a/decode.cpp +++ b/decode.cpp @@ -271,9 +271,15 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st if (i + 1 >= ops.size() || ops[i + 1].op == VT_MOVETO) { if (ops[i].op != VT_CLOSEPATH) { - fprintf(stderr, "Ring does not end with closepath (ends with %d)\n", ops[i].op); - if (!force) { - exit(EXIT_FAILURE); + static bool warned = false; + + if (!warned) { + fprintf(stderr, "Ring does not end with closepath (ends with %d)\n", ops[i].op); + if (!force) { + exit(EXIT_FAILURE); + } + + warned = true; } } } @@ -307,9 +313,15 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st int state = 0; for (size_t i = 0; i < rings.size(); i++) { if (i == 0 && areas[i] < 0) { - fprintf(stderr, "Polygon begins with an inner ring\n"); - if (!force) { - exit(EXIT_FAILURE); + static bool warned = false; + + if (!warned) { + fprintf(stderr, "Polygon begins with an inner ring\n"); + if (!force) { + exit(EXIT_FAILURE); + } + + warned = true; } } diff --git a/version.hpp b/version.hpp index babfe73..74600d9 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.18.0\n" +#define VERSION "tippecanoe v1.18.1\n" From 2f7b5d8afd466127921a694c969ba7560606c224 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 12 May 2017 16:21:25 -0700 Subject: [PATCH 8/8] There should be a test for the no-compression mbtiles output format --- tests/overlap/out/-z0_-pC.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/overlap/out/-z0_-pC.json diff --git a/tests/overlap/out/-z0_-pC.json b/tests/overlap/out/-z0_-pC.json new file mode 100644 index 0000000..332ef69 --- /dev/null +++ b/tests/overlap/out/-z0_-pC.json @@ -0,0 +1,20 @@ +{ "type": "FeatureCollection", "properties": { +"bounds": "-135.000000,-31.000000,-3.000000,79.000000", +"center": "-3.000000,0.000000,0", +"description": "tests/overlap/out/-z0_-pC.json.check.mbtiles", +"format": "pbf", +"json": "{\"vector_layers\": [ { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} } ] }", +"maxzoom": "0", +"minzoom": "0", +"name": "tests/overlap/out/-z0_-pC.json.check.mbtiles", +"type": "overlay", +"version": "2" +}, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0, "compressed": false }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -33.046875, 79.004962 ], [ -3.076172, 61.015725 ], [ -37.001953, 34.016242 ], [ -71.015625, 29.075375 ], [ -103.007812, 39.027719 ], [ -117.070312, 60.020952 ], [ -113.027344, 69.005675 ], [ -93.076172, 75.004940 ], [ -74.003906, 79.004962 ], [ -33.046875, 79.004962 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -81.035156, 63.035039 ], [ -28.037109, 59.040555 ], [ -10.019531, 50.007739 ], [ -5.009766, 8.059230 ], [ -17.050781, -21.943046 ], [ -75.058594, -30.977609 ], [ -118.037109, -8.928487 ], [ -135.000000, 29.075375 ], [ -113.027344, 53.014783 ], [ -81.035156, 63.035039 ] ] ] } } +] } +] } +] }