From aa7191b1eeb2de1f25a7a4730e43a089df7f3beb Mon Sep 17 00:00:00 2001
From: Eric Fischer <enf@pobox.com>
Date: Thu, 9 Nov 2017 12:49:09 -0800
Subject: [PATCH] Also test large integers. Work around an apparent bug in
strtoull.
---
mvt.cpp | 44 +++++++++++++++++++++++++++++++++++--
mvt.hpp | 3 +++
read_json.cpp | 24 ++++++++++++++++++--
tests/overflow/in.json | 12 ++++++++++
tests/overflow/out/-z0.json | 26 +++++++++++++++++++++-
5 files changed, 104 insertions(+), 5 deletions(-)
diff --git a/mvt.cpp b/mvt.cpp
index 27ee21a..d1d7452 100644
--- a/mvt.cpp
+++ b/mvt.cpp
@@ -472,7 +472,7 @@ void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) {
feature.tags.push_back(vo);
}
-static int is_integer(const char *s, long long *v) {
+bool is_integer(const char *s, long long *v) {
errno = 0;
char *endptr;
@@ -480,7 +480,47 @@ static int is_integer(const char *s, long long *v) {
if (*v == 0 && errno != 0) {
return 0;
}
- if ((*v == LLONG_MIN || *v == LLONG_MAX) && (errno == ERANGE)) {
+ if ((*v == LLONG_MIN || *v == LLONG_MAX) && (errno == ERANGE || errno == EINVAL)) {
+ return 0;
+ }
+ if (*endptr != '\0') {
+ // Special case: If it is an integer followed by .0000 or similar,
+ // it is still an integer
+
+ if (*endptr != '.') {
+ return 0;
+ }
+ endptr++;
+ for (; *endptr != '\0'; endptr++) {
+ if (*endptr != '0') {
+ return 0;
+ }
+ }
+
+ return 1;
+ }
+
+ return 1;
+}
+
+bool is_unsigned_integer(const char *s, unsigned long long *v) {
+ errno = 0;
+ char *endptr;
+
+ // Special check because MacOS stroull() returns 1
+ // for -18446744073709551615
+ while (isspace(*s)) {
+ s++;
+ }
+ if (*s == '-') {
+ return 0;
+ }
+
+ *v = strtoull(s, &endptr, 0);
+ if (*v == 0 && errno != 0) {
+ return 0;
+ }
+ if ((*v == ULLONG_MAX) && (errno == ERANGE || errno == EINVAL)) {
return 0;
}
if (*endptr != '\0') {
diff --git a/mvt.hpp b/mvt.hpp
index 32ef55f..78539a9 100644
--- a/mvt.hpp
+++ b/mvt.hpp
@@ -111,4 +111,7 @@ int compress(std::string const &input, std::string &output);
int dezig(unsigned n);
mvt_value stringified_to_mvt_value(int type, const char *s);
+
+bool is_integer(const char *s, long long *v);
+bool is_unsigned_integer(const char *s, unsigned long long *v);
#endif
diff --git a/read_json.cpp b/read_json.cpp
index bf70e78..5cc63a9 100644
--- a/read_json.cpp
+++ b/read_json.cpp
@@ -105,7 +105,17 @@ void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fna
void canonicalize(json_object *o) {
if (o->type == JSON_NUMBER) {
- std::string s = milo::dtoa_milo(o->number);
+ std::string s;
+ long long v;
+ unsigned long long uv;
+
+ if (is_integer(o->string, &v)) {
+ s = std::to_string(v);
+ } else if (is_unsigned_integer(o->string, &uv)) {
+ s = std::to_string(uv);
+ } else {
+ s = milo::dtoa_milo(o->number);
+ }
free(o->string);
o->string = strdup(s.c_str());
} else if (o->type == JSON_HASH) {
@@ -150,7 +160,17 @@ void stringify_value(json_object *value, int &type, std::string &stringified, co
}
} else if (vt == JSON_NUMBER) {
type = mvt_double;
- stringified = milo::dtoa_milo(value->number);
+
+ long long v;
+ unsigned long long uv;
+
+ if (is_integer(value->string, &v)) {
+ stringified = std::to_string(v);
+ } else if (is_unsigned_integer(value->string, &uv)) {
+ stringified = std::to_string(uv);
+ } else {
+ stringified = milo::dtoa_milo(value->number);
+ }
} else if (vt == JSON_TRUE || vt == JSON_FALSE) {
type = mvt_bool;
stringified = val;
diff --git a/tests/overflow/in.json b/tests/overflow/in.json
index cbf7d63..9aeffd2 100644
--- a/tests/overflow/in.json
+++ b/tests/overflow/in.json
@@ -1,3 +1,15 @@
{ "type": "Feature", "properties": { "excess": 2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
{ "type": "Feature", "properties": { "excess": 22e291 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
{ "type": "Feature", "properties": { "excess": 2.5 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": 2147483648 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": -2147483648 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": 2147483647 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": -2147483647 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": 18446744073709551616 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": -18446744073709551616 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": 18446744073709551615 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": -18446744073709551615 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": 9223372036854775808 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": -9223372036854775808 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": 9223372036854775807 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
+{ "type": "Feature", "properties": { "excess": -9223372036854775807 }, "geometry": { "type": "Point", "coordinates": [ 0,0 ] } }
diff --git a/tests/overflow/out/-z0.json b/tests/overflow/out/-z0.json
index 94df132..a6f6cdc 100644
--- a/tests/overflow/out/-z0.json
+++ b/tests/overflow/out/-z0.json
@@ -3,7 +3,7 @@
"center": "0.000000,0.000000,0",
"description": "tests/overflow/out/-z0.json.check.mbtiles",
"format": "pbf",
-"json": "{\"vector_layers\": [ { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {\"excess\": \"Number\"} } ],\"tilestats\": {\"layerCount\": 1,\"layers\": [{\"layer\": \"in\",\"count\": 3,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"excess\",\"count\": 3,\"type\": \"number\",\"values\": [2.2222222222222223e+291,2.2e+292,2.5],\"min\": 2.5,\"max\": 2.2e+292}]}]}}",
+"json": "{\"vector_layers\": [ { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {\"excess\": \"Number\"} } ],\"tilestats\": {\"layerCount\": 1,\"layers\": [{\"layer\": \"in\",\"count\": 15,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"excess\",\"count\": 14,\"type\": \"number\",\"values\": [-18446744073709553000,-2147483647,-2147483648,-9223372036854775807,-9223372036854775808,18446744073709551615,18446744073709553000,2.2222222222222223e+291,2.2e+292,2.5,2147483647,2147483648,9223372036854775807,9223372036854775808],\"min\": -18446744073709553000,\"max\": 2.2e+292}]}]}}",
"maxzoom": "0",
"minzoom": "0",
"name": "tests/overflow/out/-z0.json.check.mbtiles",
@@ -17,6 +17,30 @@
{ "type": "Feature", "properties": { "excess": 2.2e+292 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
,
{ "type": "Feature", "properties": { "excess": 2.5 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": 2147483648 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": -2147483648 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": 2147483647 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": -2147483647 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": 18446744073709553000 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": -18446744073709553000 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": 18446744073709553000 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": -18446744073709553000 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": 9223372036854776000 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": -9223372036854775808 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": 9223372036854775807 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "excess": -9223372036854775807 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
] }
] }
] }