From 79363e397bcd9516a72795fe064f29e00cd4eb41 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 8 Aug 2018 12:13:21 -0700 Subject: [PATCH] Factor out the GeoJSON parsing loop from feature serialization --- Makefile | 4 +- decode.cpp | 2 +- geojson-loop.cpp | 155 +++++++++++++++++++++++++++++++++++++++++++++++ geojson-loop.hpp | 11 ++++ geojson.cpp | 139 ++++++------------------------------------ 5 files changed, 189 insertions(+), 122 deletions(-) create mode 100644 geojson-loop.cpp create mode 100644 geojson-loop.hpp diff --git a/Makefile b/Makefile index 7a8d0d5..c32718c 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o +tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o @@ -59,7 +59,7 @@ tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/json tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o +tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread unit: unit.o text.o diff --git a/decode.cpp b/decode.cpp index 9302976..86e58fb 100644 --- a/decode.cpp +++ b/decode.cpp @@ -94,7 +94,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::set +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include "geojson-loop.hpp" +#include "jsonpull/jsonpull.h" + +// XXX duplicated +#define GEOM_TYPES 6 +static const char *geometry_names[GEOM_TYPES] = { + "Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon", +}; + +// XXX duplicated +static void json_context(json_object *j) { + char *s = json_stringify(j); + + if (strlen(s) >= 500) { + sprintf(s + 497, "..."); + } + + fprintf(stderr, "In JSON object %s\n", s); + free(s); // stringify +} + +void parse_json(json_feature_action *jfa, json_pull *jp) { + long long found_hashes = 0; + long long found_features = 0; + long long found_geometries = 0; + + while (1) { + json_object *j = json_read(jp); + if (j == NULL) { + if (jp->error != NULL) { + fprintf(stderr, "%s:%d: %s\n", jfa->fname.c_str(), jp->line, jp->error); + if (jp->root != NULL) { + json_context(jp->root); + } + } + + json_free(jp->root); + break; + } + + if (j->type == JSON_HASH) { + found_hashes++; + + if (found_hashes == 50 && found_features == 0 && found_geometries == 0) { + fprintf(stderr, "%s:%d: Warning: not finding any GeoJSON features or geometries in input yet after 50 objects.\n", jfa->fname.c_str(), jp->line); + } + } + + json_object *type = json_hash_get(j, "type"); + if (type == NULL || type->type != JSON_STRING) { + continue; + } + + if (found_features == 0) { + int i; + int is_geometry = 0; + for (i = 0; i < GEOM_TYPES; i++) { + if (strcmp(type->string, geometry_names[i]) == 0) { + is_geometry = 1; + break; + } + } + + if (is_geometry) { + if (j->parent != NULL) { + if (j->parent->type == JSON_ARRAY && j->parent->parent != NULL) { + if (j->parent->parent->type == JSON_HASH) { + json_object *geometries = json_hash_get(j->parent->parent, "geometries"); + if (geometries != NULL) { + // Parent of Parent must be a GeometryCollection + is_geometry = 0; + } + } + } else if (j->parent->type == JSON_HASH) { + json_object *geometry = json_hash_get(j->parent, "geometry"); + if (geometry != NULL) { + // Parent must be a Feature + is_geometry = 0; + } + } + } + } + + if (is_geometry) { + if (found_features != 0 && found_geometries == 0) { + fprintf(stderr, "%s:%d: Warning: found a mixture of features and bare geometries\n", jfa->fname.c_str(), jp->line); + } + found_geometries++; + + jfa->add_feature(j, NULL, NULL, NULL, j); + json_free(j); + continue; + } + } + + if (strcmp(type->string, "Feature") != 0) { + if (strcmp(type->string, "FeatureCollection") == 0) { + jfa->check_crs(j); + json_free(j); + } + + continue; + } + + if (found_features == 0 && found_geometries != 0) { + fprintf(stderr, "%s:%d: Warning: found a mixture of features and bare geometries\n", jfa->fname.c_str(), jp->line); + } + found_features++; + + json_object *geometry = json_hash_get(j, "geometry"); + if (geometry == NULL) { + fprintf(stderr, "%s:%d: feature with no geometry\n", jfa->fname.c_str(), jp->line); + json_context(j); + json_free(j); + continue; + } + + json_object *properties = json_hash_get(j, "properties"); + if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + fprintf(stderr, "%s:%d: feature without properties hash\n", jfa->fname.c_str(), jp->line); + json_context(j); + json_free(j); + continue; + } + + json_object *tippecanoe = json_hash_get(j, "tippecanoe"); + json_object *id = json_hash_get(j, "id"); + + json_object *geometries = json_hash_get(geometry, "geometries"); + if (geometries != NULL && geometries->type == JSON_ARRAY) { + size_t g; + for (g = 0; g < geometries->length; g++) { + jfa->add_feature(geometries->array[g], properties, id, tippecanoe, j); + } + } else { + jfa->add_feature(geometry, properties, id, tippecanoe, j); + } + + json_free(j); + + /* XXX check for any non-features in the outer object */ + } +} diff --git a/geojson-loop.hpp b/geojson-loop.hpp new file mode 100644 index 0000000..d1bed2d --- /dev/null +++ b/geojson-loop.hpp @@ -0,0 +1,11 @@ +#include +#include "jsonpull/jsonpull.h" + +struct json_feature_action { + std::string fname; + + virtual int add_feature(json_object *geometry, json_object *properties, json_object *id, json_object *tippecanoe, json_object *feature) = 0; + virtual void check_crs(json_object *j) = 0; +}; + +void parse_json(json_feature_action *action, json_pull *jp); diff --git a/geojson.cpp b/geojson.cpp index 09cdb6d..30f0ecc 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -37,6 +37,7 @@ #include "text.hpp" #include "read_json.hpp" #include "mvt.hpp" +#include "geojson-loop.hpp" int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string layername) { json_object *geometry_type = json_hash_get(geometry, "type"); @@ -227,127 +228,27 @@ void check_crs(json_object *j, const char *reading) { } } -void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername) { - long long found_hashes = 0; - long long found_features = 0; - long long found_geometries = 0; +struct json_serialize_action : json_feature_action { + serialization_state *sst; + int layer; + std::string layername; - while (1) { - json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { - fprintf(stderr, "%s:%d: %s\n", sst->fname, jp->line, jp->error); - if (jp->root != NULL) { - json_context(jp->root); - } - } - - json_free(jp->root); - break; - } - - if (j->type == JSON_HASH) { - found_hashes++; - - if (found_hashes == 50 && found_features == 0 && found_geometries == 0) { - fprintf(stderr, "%s:%d: Warning: not finding any GeoJSON features or geometries in input yet after 50 objects.\n", sst->fname, jp->line); - } - } - - json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { - continue; - } - - if (found_features == 0) { - int i; - int is_geometry = 0; - for (i = 0; i < GEOM_TYPES; i++) { - if (strcmp(type->string, geometry_names[i]) == 0) { - is_geometry = 1; - break; - } - } - - if (is_geometry) { - if (j->parent != NULL) { - if (j->parent->type == JSON_ARRAY && j->parent->parent != NULL) { - if (j->parent->parent->type == JSON_HASH) { - json_object *geometries = json_hash_get(j->parent->parent, "geometries"); - if (geometries != NULL) { - // Parent of Parent must be a GeometryCollection - is_geometry = 0; - } - } - } else if (j->parent->type == JSON_HASH) { - json_object *geometry = json_hash_get(j->parent, "geometry"); - if (geometry != NULL) { - // Parent must be a Feature - is_geometry = 0; - } - } - } - } - - if (is_geometry) { - if (found_features != 0 && found_geometries == 0) { - fprintf(stderr, "%s:%d: Warning: found a mixture of features and bare geometries\n", sst->fname, jp->line); - } - found_geometries++; - - serialize_geojson_feature(sst, j, NULL, NULL, layer, NULL, j, layername); - json_free(j); - continue; - } - } - - if (strcmp(type->string, "Feature") != 0) { - if (strcmp(type->string, "FeatureCollection") == 0) { - check_crs(j, sst->fname); - json_free(j); - } - - continue; - } - - if (found_features == 0 && found_geometries != 0) { - fprintf(stderr, "%s:%d: Warning: found a mixture of features and bare geometries\n", sst->fname, jp->line); - } - found_features++; - - json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { - fprintf(stderr, "%s:%d: feature with no geometry\n", sst->fname, jp->line); - json_context(j); - json_free(j); - continue; - } - - json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { - fprintf(stderr, "%s:%d: feature without properties hash\n", sst->fname, jp->line); - json_context(j); - json_free(j); - continue; - } - - json_object *tippecanoe = json_hash_get(j, "tippecanoe"); - json_object *id = json_hash_get(j, "id"); - - json_object *geometries = json_hash_get(geometry, "geometries"); - if (geometries != NULL && geometries->type == JSON_ARRAY) { - size_t g; - for (g = 0; g < geometries->length; g++) { - serialize_geojson_feature(sst, geometries->array[g], properties, id, layer, tippecanoe, j, layername); - } - } else { - serialize_geojson_feature(sst, geometry, properties, id, layer, tippecanoe, j, layername); - } - - json_free(j); - - /* XXX check for any non-features in the outer object */ + int add_feature(json_object *geometry, json_object *properties, json_object *id, json_object *tippecanoe, json_object *feature) { + return serialize_geojson_feature(sst, geometry, properties, id, layer, tippecanoe, feature, layername); } + + void check_crs(json_object *j) { + ::check_crs(j, fname.c_str()); + } +}; + +void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername) { + json_serialize_action jsa; + jsa.sst = sst; + jsa.layer = layer; + jsa.layername = layername; + + parse_json(&jsa, jp); } void *run_parse_json(void *v) {