From e7ee83f27b6dbb3aba791341e80b5f97e2eec1cd Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 24 Aug 2017 17:27:30 -0700 Subject: [PATCH] Move attribute type coercion out of parsing and into serialization --- geojson.cpp | 22 ++++++++++----- geojson.hpp | 2 +- main.cpp | 5 ++-- mvt.hpp | 3 ++- plugin.cpp | 6 ++--- read_json.cpp | 75 ++++++++++++++++++++++++++------------------------- read_json.hpp | 3 ++- serial.hpp | 1 + 8 files changed, 65 insertions(+), 52 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index a2f93db..f72d607 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -99,7 +99,7 @@ static long long scale_geometry(struct serialization_state *sst, long long *bbox return geom.size(); } -int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string layername, std::map const *attribute_types) { +int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string layername) { json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == NULL) { static int warned = 0; @@ -219,7 +219,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom int type = -1; std::string val; - stringify_value(properties->values[i], type, val, sst->fname, sst->line, feature, properties->keys[i]->string, attribute_types); + stringify_value(properties->values[i], type, val, sst->fname, sst->line, feature, properties->keys[i]->string); if (type >= 0) { metakey[m] = properties->keys[i]->string; @@ -406,10 +406,18 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { if (sst->include->count(sf.full_keys[i]) == 0) { sf.full_keys[i] = ""; sf.m--; + continue; } } else if (sst->exclude->count(sf.full_keys[i]) != 0) { sf.full_keys[i] = ""; sf.m--; + continue; + } + + coerce_value(sf.full_keys[i], sf.full_values[i].type, sf.full_values[i].s, sst->attribute_types); + if (sf.full_values[i].type == mvt_null) { + sf.full_keys[i] = ""; + sf.m--; } } @@ -501,7 +509,7 @@ void check_crs(json_object *j, const char *reading) { } } -void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername, std::map const *attribute_types) { +void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; @@ -569,7 +577,7 @@ void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std:: } found_geometries++; - serialize_geojson_feature(sst, j, NULL, NULL, layer, NULL, j, layername, attribute_types); + serialize_geojson_feature(sst, j, NULL, NULL, layer, NULL, j, layername); json_free(j); continue; } @@ -612,10 +620,10 @@ void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std:: if (geometries != NULL) { size_t g; for (g = 0; g < geometries->length; g++) { - serialize_geojson_feature(sst, geometries->array[g], properties, id, layer, tippecanoe, j, layername, attribute_types); + serialize_geojson_feature(sst, geometries->array[g], properties, id, layer, tippecanoe, j, layername); } } else { - serialize_geojson_feature(sst, geometry, properties, id, layer, tippecanoe, j, layername, attribute_types); + serialize_geojson_feature(sst, geometry, properties, id, layer, tippecanoe, j, layername); } json_free(j); @@ -627,7 +635,7 @@ void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std:: void *run_parse_json(void *v) { struct parse_json_args *pja = (struct parse_json_args *) v; - parse_json(pja->sst, pja->jp, pja->layer, *pja->layername, pja->attribute_types); + parse_json(pja->sst, pja->jp, pja->layer, *pja->layername); return NULL; } diff --git a/geojson.hpp b/geojson.hpp index 863d199..5f992fe 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -22,7 +22,7 @@ struct parse_json_args { struct json_pull *json_begin_map(char *map, long long len); void json_end_map(struct json_pull *jp); -void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername, std::map const *attribute_types); +void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername); void *run_parse_json(void *v); #endif diff --git a/main.cpp b/main.cpp index 1d82502..eb8d43c 100644 --- a/main.cpp +++ b/main.cpp @@ -403,11 +403,11 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const sst[i].include = include; sst[i].exclude_all = exclude_all; sst[i].basezoom = basezoom; + sst[i].attribute_types = attribute_types; pja[i].jp = json_begin_map(map + segs[i], segs[i + 1] - segs[i]); pja[i].layer = source; pja[i].layername = &layername; - pja[i].attribute_types = attribute_types; pja[i].sst = &sst[i]; @@ -1382,8 +1382,9 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo sst.include = include; sst.exclude_all = exclude_all; sst.basezoom = basezoom; + sst.attribute_types = attribute_types; - parse_json(&sst, jp, layer, sources[layer].layer, attribute_types); + parse_json(&sst, jp, layer, sources[layer].layer); json_end(jp); overall_offset = layer_seq; checkdisk(reader, CPUS); diff --git a/mvt.hpp b/mvt.hpp index d64ee53..68131ec 100644 --- a/mvt.hpp +++ b/mvt.hpp @@ -62,7 +62,8 @@ enum mvt_value_type { mvt_int, mvt_uint, mvt_sint, - mvt_bool + mvt_bool, + mvt_null, }; struct mvt_value { diff --git a/plugin.cpp b/plugin.cpp index bf44979..87a4a17 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -257,8 +257,7 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: int tp = -1; std::string s; - std::map nullmap; - stringify_value(properties->values[i], tp, s, "Filter output", jp->line, j, "", &nullmap); + stringify_value(properties->values[i], tp, s, "Filter output", jp->line, j, ""); if (tp >= 0) { mvt_value v = stringified_to_mvt_value(tp, s.c_str()); l->second.tag(feature, std::string(properties->keys[i]->string), v); @@ -487,8 +486,7 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: serial_val v; v.type = -1; - std::map nullmap; - stringify_value(properties->values[i], v.type, v.s, "Filter output", jp->line, j, "", &nullmap); + stringify_value(properties->values[i], v.type, v.s, "Filter output", jp->line, j, ""); if (v.type >= 0) { sf.full_keys.push_back(std::string(properties->keys[i]->string)); diff --git a/read_json.cpp b/read_json.cpp index 461becb..c751bb9 100644 --- a/read_json.cpp +++ b/read_json.cpp @@ -106,7 +106,7 @@ void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fna } } -void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key, std::map const *attribute_types) { +void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key) { if (value != NULL) { int vt = value->type; std::string val; @@ -125,40 +125,6 @@ void stringify_value(json_object *value, int &type, std::string &stringified, co free((void *) v); // stringify } - auto a = (*attribute_types).find(key); - if (a != attribute_types->end()) { - if (a->second == mvt_string) { - vt = JSON_STRING; - } else if (a->second == mvt_float) { - vt = JSON_NUMBER; - val = std::to_string(atof(val.c_str())); - } else if (a->second == mvt_int) { - vt = JSON_NUMBER; - if (val.size() == 0) { - val = "0"; - } - - for (size_t ii = 0; ii < val.size(); ii++) { - char c = val[ii]; - if (c < '0' || c > '9') { - val = std::to_string(round(atof(val.c_str()))); - break; - } - } - } else if (a->second == mvt_bool) { - if (val == "false" || val == "0" || val == "null" || val.size() == 0) { - vt = JSON_FALSE; - val = "false"; - } else { - vt = JSON_TRUE; - val = "true"; - } - } else { - fprintf(stderr, "Can't happen: attribute type %d\n", a->second); - exit(EXIT_FAILURE); - } - } - if (vt == JSON_STRING) { type = mvt_string; stringified = val; @@ -175,10 +141,47 @@ void stringify_value(json_object *value, int &type, std::string &stringified, co type = mvt_bool; stringified = val; } else if (vt == JSON_NULL) { - ; + type = mvt_null; + stringified = "null"; } else { type = mvt_string; stringified = val; } } } + +void coerce_value(std::string const &key, int &vt, std::string &val, std::map const *attribute_types) { + auto a = (*attribute_types).find(key); + if (a != attribute_types->end()) { + if (a->second == mvt_string) { + vt = mvt_string; + } else if (a->second == mvt_float) { + vt = mvt_double; + val = std::to_string(atof(val.c_str())); + } else if (a->second == mvt_int) { + vt = mvt_double; + if (val.size() == 0) { + val = "0"; + } + + for (size_t ii = 0; ii < val.size(); ii++) { + char c = val[ii]; + if (c < '0' || c > '9') { + val = std::to_string(round(atof(val.c_str()))); + break; + } + } + } else if (a->second == mvt_bool) { + if (val == "false" || val == "0" || val == "null" || val.size() == 0) { + vt = mvt_bool; + val = "false"; + } else { + vt = mvt_bool; + val = "true"; + } + } else { + fprintf(stderr, "Can't happen: attribute type %d\n", a->second); + exit(EXIT_FAILURE); + } + } +} diff --git a/read_json.hpp b/read_json.hpp index 3a997d6..a637747 100644 --- a/read_json.hpp +++ b/read_json.hpp @@ -13,4 +13,5 @@ extern int mb_geometry[GEOM_TYPES]; void json_context(json_object *j); void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature); -void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key, std::map const *attribute_types); +void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key); +void coerce_value(std::string const &key, int &vt, std::string &val, std::map const *attribute_types); diff --git a/serial.hpp b/serial.hpp index f033f18..f0fd590 100644 --- a/serial.hpp +++ b/serial.hpp @@ -121,6 +121,7 @@ struct serialization_state { std::map *layermap; + std::map const *attribute_types; std::set *exclude; std::set *include; int exclude_all;