From 53de8a2014e263453cdec39f2cc22656076552d0 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 1 Nov 2018 17:07:33 -0700 Subject: [PATCH] Add an option to convert stringified number feature IDs to numbers --- CHANGELOG.md | 4 +++ README.md | 1 + geojson.cpp | 47 ++++++++++++++++++++++++++++----- main.cpp | 1 + man/tippecanoe.1 | 2 ++ options.hpp | 1 + tests/stringid/in.json | 6 +++++ tests/stringid/out/-z0.json | 28 ++++++++++++++++++++ tests/stringid/out/-z0_-aI.json | 28 ++++++++++++++++++++ version.hpp | 2 +- 10 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 tests/stringid/in.json create mode 100644 tests/stringid/out/-z0.json create mode 100644 tests/stringid/out/-z0_-aI.json diff --git a/CHANGELOG.md b/CHANGELOG.md index fc46ab8..f947fc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.32.3 + +* Add an option to convert stringified number feature IDs to numbers + ## 1.32.2 * Warn in tile-join if tilesets being joined have inconsistent maxzooms diff --git a/README.md b/README.md index 416fa18..af0c454 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,7 @@ resolution is obtained than by using a smaller _maxzoom_ or _detail_. `sum`, `product`, `mean`, `max`, `min`, `concat`, or `comma` to specify how the named _attribute_ is accumulated onto the attribute of the same name in a feature that does survive. * `-pe` or `--empty-csv-columns-are-null`: Treat empty CSV columns as nulls rather than as empty strings. + * `-aI` or `--convert-stringified-ids-to-numbers`: If a feature ID is the string representation of a number, convert it to a plain number to use as the feature ID. ### Filtering features by attributes diff --git a/geojson.cpp b/geojson.cpp index 6222219..67fe653 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -113,6 +113,13 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->string); warned_frac = true; } + } else if (std::to_string(id_value) != id->string) { + static bool warned = false; + + if (!warned) { + fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->string); + warned = true; + } } else { has_id = true; } @@ -125,13 +132,41 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom } } } else { - static bool warned_nan = false; + bool converted = false; - if (!warned_nan) { - char *s = json_stringify(id); - fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", s); - free(s); // stringify - warned_nan = true; + if (additional[A_CONVERT_NUMERIC_IDS] && id->type == JSON_STRING) { + char *err = NULL; + id_value = strtoull(id->string, &err, 10); + + if (err != NULL && *err != '\0') { + static bool warned_frac = false; + + if (!warned_frac) { + fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->string); + warned_frac = true; + } + } else if (std::to_string(id_value) != id->string) { + static bool warned = false; + + if (!warned) { + fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->string); + warned = true; + } + } else { + has_id = true; + converted = true; + } + } + + if (!converted) { + static bool warned_nan = false; + + if (!warned_nan) { + char *s = json_stringify(id); + fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", s); + free(s); // stringify + warned_nan = true; + } } } } diff --git a/main.cpp b/main.cpp index 4031ee4..56962a1 100644 --- a/main.cpp +++ b/main.cpp @@ -2541,6 +2541,7 @@ int main(int argc, char **argv) { {"attribute-description", required_argument, 0, 'Y'}, {"accumulate-attribute", required_argument, 0, 'E'}, {"empty-csv-columns-are-null", no_argument, &prevent[P_EMPTY_CSV_COLUMNS], 1}, + {"convert-stringified-ids-to-numbers", no_argument, &additional[A_CONVERT_NUMERIC_IDS], 1}, {"Filtering features by attributes", 0, 0, 0}, {"feature-filter-file", required_argument, 0, 'J'}, diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 61f8bff..1e83082 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -424,6 +424,8 @@ that are dropped, coalesced\-as\-needed, or clustered. The \fIoperation\fP may b to specify how the named \fIattribute\fP is accumulated onto the attribute of the same name in a feature that does survive. .IP \(bu 2 \fB\fC\-pe\fR or \fB\fC\-\-empty\-csv\-columns\-are\-null\fR: Treat empty CSV columns as nulls rather than as empty strings. +.IP \(bu 2 +\fB\fC\-aI\fR or \fB\fC\-\-convert\-stringified\-ids\-to\-numbers\fR: If a feature ID is the string representation of a number, convert it to a plain number to use as the feature ID. .RE .SS Filtering features by attributes .RS diff --git a/options.hpp b/options.hpp index 0f5e782..b3b7a35 100644 --- a/options.hpp +++ b/options.hpp @@ -23,6 +23,7 @@ #define A_EXTEND_ZOOMS ((int) 'e') #define A_CLUSTER_DENSEST_AS_NEEDED ((int) 'C') #define A_GENERATE_IDS ((int) 'i') +#define A_CONVERT_NUMERIC_IDS ((int) 'I') #define P_SIMPLIFY ((int) 's') #define P_SIMPLIFY_LOW ((int) 'S') diff --git a/tests/stringid/in.json b/tests/stringid/in.json new file mode 100644 index 0000000..ff9db89 --- /dev/null +++ b/tests/stringid/in.json @@ -0,0 +1,6 @@ +{ "type": "Feature", "id": 12345, "properties": {}, "geometry": { "type": "Point", "coordinates": [ 1, 1 ] } } +{ "type": "Feature", "id": "12345", "properties": {}, "geometry": { "type": "Point", "coordinates": [ 2, 1 ] } } +{ "type": "Feature", "id": "12345.6789", "properties": {}, "geometry": { "type": "Point", "coordinates": [ 3, 1 ] } } +{ "type": "Feature", "id": "9837489273489273894728943728903480989080938597489274389", "properties": {}, "geometry": { "type": "Point", "coordinates": [ 4, 1 ] } } +{ "type": "Feature", "id": 9837489273489273894728943728903480989080938597489274389, "properties": {}, "geometry": { "type": "Point", "coordinates": [ 5, 1 ] } } +{ "type": "Feature", "id": "1e5", "properties": {}, "geometry": { "type": "Point", "coordinates": [ 6, 1 ] } } diff --git a/tests/stringid/out/-z0.json b/tests/stringid/out/-z0.json new file mode 100644 index 0000000..c48babd --- /dev/null +++ b/tests/stringid/out/-z0.json @@ -0,0 +1,28 @@ +{ "type": "FeatureCollection", "properties": { +"bounds": "1.000000,1.000000,6.000000,1.000000", +"center": "1.000000,1.000000,0", +"description": "tests/stringid/out/-z0.json.check.mbtiles", +"format": "pbf", +"json": "{\"vector_layers\": [ { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} } ],\"tilestats\": {\"layerCount\": 1,\"layers\": [{\"layer\": \"in\",\"count\": 6,\"geometry\": \"Point\",\"attributeCount\": 0,\"attributes\": []}]}}", +"maxzoom": "0", +"minzoom": "0", +"name": "tests/stringid/out/-z0.json.check.mbtiles", +"type": "overlay", +"version": "2" +}, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "id": 12345, "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 1.054628 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 1.933594, 1.054628 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 2.988281, 1.054628 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 3.955078, 1.054628 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 4.921875, 1.054628 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 5.976562, 1.054628 ] } } +] } +] } +] } diff --git a/tests/stringid/out/-z0_-aI.json b/tests/stringid/out/-z0_-aI.json new file mode 100644 index 0000000..ec520af --- /dev/null +++ b/tests/stringid/out/-z0_-aI.json @@ -0,0 +1,28 @@ +{ "type": "FeatureCollection", "properties": { +"bounds": "1.000000,1.000000,6.000000,1.000000", +"center": "1.000000,1.000000,0", +"description": "tests/stringid/out/-z0_-aI.json.check.mbtiles", +"format": "pbf", +"json": "{\"vector_layers\": [ { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} } ],\"tilestats\": {\"layerCount\": 1,\"layers\": [{\"layer\": \"in\",\"count\": 6,\"geometry\": \"Point\",\"attributeCount\": 0,\"attributes\": []}]}}", +"maxzoom": "0", +"minzoom": "0", +"name": "tests/stringid/out/-z0_-aI.json.check.mbtiles", +"type": "overlay", +"version": "2" +}, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "id": 12345, "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 1.054628 ] } } +, +{ "type": "Feature", "id": 12345, "properties": { }, "geometry": { "type": "Point", "coordinates": [ 1.933594, 1.054628 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 2.988281, 1.054628 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 3.955078, 1.054628 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 4.921875, 1.054628 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 5.976562, 1.054628 ] } } +] } +] } +] } diff --git a/version.hpp b/version.hpp index 51b494b..d6eb94c 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v1.32.2" +#define VERSION "v1.32.3" #endif