From 34b1b215f4e8b24f7bac206aa50ac587fbe03272 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 24 Aug 2017 16:30:01 -0700 Subject: [PATCH] Move tilestats management out of parsing and into serialization --- geojson.cpp | 96 +++++++++++++++++++++++++---------------------------- geojson.hpp | 3 +- main.cpp | 5 +-- serial.hpp | 4 +++ 4 files changed, 53 insertions(+), 55 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index cbd1934..e1fcbbf 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -99,7 +99,7 @@ static long long scale_geometry(struct serialization_state *sst, long long *bbox return geom.size(); } -int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, std::set *exclude, std::set *include, int exclude_all, int basezoom, int layer, json_object *tippecanoe, json_object *feature, std::map *layermap, std::string layername, std::map const *attribute_types) { +int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, std::set *exclude, std::set *include, int exclude_all, int basezoom, int layer, json_object *tippecanoe, json_object *feature, std::string layername, std::map const *attribute_types) { json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == NULL) { static int warned = 0; @@ -202,42 +202,6 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom } } - if (!sst->filters) { - if (tippecanoe_layername.size() != 0) { - if (layermap->count(tippecanoe_layername) == 0) { - layermap->insert(std::pair(tippecanoe_layername, layermap_entry(layermap->size()))); - } - - auto ai = layermap->find(tippecanoe_layername); - if (ai != layermap->end()) { - layer = ai->second.id; - layername = tippecanoe_layername; - - if (mb_geometry[t] == VT_POINT) { - ai->second.points++; - } else if (mb_geometry[t] == VT_LINE) { - ai->second.lines++; - } else if (mb_geometry[t] == VT_POLYGON) { - ai->second.polygons++; - } - } else { - fprintf(stderr, "Internal error: can't find layer name %s\n", tippecanoe_layername.c_str()); - exit(EXIT_FAILURE); - } - } else { - auto fk = layermap->find(layername); - if (fk != layermap->end()) { - if (mb_geometry[t] == VT_POINT) { - fk->second.points++; - } else if (mb_geometry[t] == VT_LINE) { - fk->second.lines++; - } else if (mb_geometry[t] == VT_POLYGON) { - fk->second.polygons++; - } - } - } - } - size_t nprop = 0; if (properties != NULL && properties->type == JSON_HASH) { nprop = properties->length; @@ -270,15 +234,6 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom metatype[m] = type; metaval[m] = val; m++; - - type_and_string attrib; - attrib.type = metatype[m - 1]; - attrib.string = metaval[m - 1]; - - if (!sst->filters) { - auto fk = layermap->find(layername); - add_to_file_keys(fk->second.file_keys, metakey[m - 1], attrib); - } } } } @@ -301,6 +256,12 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom sf.feature_minzoom = 0; // Will be filled in during index merging sf.seq = *(sst->layer_seq); + if (tippecanoe_layername.size() != 0) { + sf.layername = tippecanoe_layername; + } else { + sf.layername = layername; + } + for (size_t i = 0; i < m; i++) { sf.full_keys.push_back(metakey[i]); @@ -426,6 +387,39 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { sf.index = 0; } + if (sst->layermap->count(sf.layername) == 0) { + sst->layermap->insert(std::pair(sf.layername, layermap_entry(sst->layermap->size()))); + } + + auto ai = sst->layermap->find(sf.layername); + if (ai != sst->layermap->end()) { + sf.layer = ai->second.id; + + if (!sst->filters) { + if (sf.t == VT_POINT) { + ai->second.points++; + } else if (sf.t == VT_LINE) { + ai->second.lines++; + } else if (sf.t == VT_POLYGON) { + ai->second.polygons++; + } + } + } else { + fprintf(stderr, "Internal error: can't find layer name %s\n", sf.layername.c_str()); + exit(EXIT_FAILURE); + } + + if (!sst->filters) { + for (size_t i = 0; i < sf.full_keys.size(); i++) { + type_and_string attrib; + attrib.type = sf.full_values[i].type; + attrib.string = sf.full_values[i].s; + + auto fk = sst->layermap->find(sf.layername); + add_to_file_keys(fk->second.file_keys, sf.full_keys[i], attrib); + } + } + if (inline_meta) { sf.metapos = -1; for (size_t i = 0; i < sf.full_keys.size(); i++) { @@ -493,7 +487,7 @@ void check_crs(json_object *j, const char *reading) { } } -void parse_json(struct serialization_state *sst, json_pull *jp, std::set *exclude, std::set *include, int exclude_all, int basezoom, int layer, std::map *layermap, std::string layername, std::map const *attribute_types) { +void parse_json(struct serialization_state *sst, json_pull *jp, std::set *exclude, std::set *include, int exclude_all, int basezoom, int layer, std::string layername, std::map const *attribute_types) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; @@ -561,7 +555,7 @@ void parse_json(struct serialization_state *sst, json_pull *jp, std::setlength; g++) { - serialize_geojson_feature(sst, geometries->array[g], properties, id, exclude, include, exclude_all, basezoom, layer, tippecanoe, j, layermap, layername, attribute_types); + serialize_geojson_feature(sst, geometries->array[g], properties, id, exclude, include, exclude_all, basezoom, layer, tippecanoe, j, layername, attribute_types); } } else { - serialize_geojson_feature(sst, geometry, properties, id, exclude, include, exclude_all, basezoom, layer, tippecanoe, j, layermap, layername, attribute_types); + serialize_geojson_feature(sst, geometry, properties, id, exclude, include, exclude_all, basezoom, layer, tippecanoe, j, layername, attribute_types); } json_free(j); @@ -619,7 +613,7 @@ void parse_json(struct serialization_state *sst, json_pull *jp, std::setsst, pja->jp, pja->exclude, pja->include, pja->exclude_all, pja->basezoom, pja->layer, pja->layermap, *pja->layername, pja->attribute_types); + parse_json(pja->sst, pja->jp, pja->exclude, pja->include, pja->exclude_all, pja->basezoom, pja->layer, *pja->layername, pja->attribute_types); return NULL; } diff --git a/geojson.hpp b/geojson.hpp index e42f64f..ca7bf1e 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -16,7 +16,6 @@ struct parse_json_args { int exclude_all; int basezoom; int layer; - std::map *layermap; std::string *layername; std::map const *attribute_types; bool want_dist; @@ -27,7 +26,7 @@ struct parse_json_args { struct json_pull *json_begin_map(char *map, long long len); void json_end_map(struct json_pull *jp); -void parse_json(struct serialization_state *sst, json_pull *jp, std::set *exclude, std::set *include, int exclude_all, int basezoom, int layer, std::map *layermap, std::string layername, std::map const *attribute_types); +void parse_json(struct serialization_state *sst, json_pull *jp, std::set *exclude, std::set *include, int exclude_all, int basezoom, int layer, std::string layername, std::map const *attribute_types); void *run_parse_json(void *v); #endif diff --git a/main.cpp b/main.cpp index d30b50f..f8a46ec 100644 --- a/main.cpp +++ b/main.cpp @@ -398,6 +398,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const sst[i].maxzoom = maxzoom; sst[i].uses_gamma = uses_gamma; sst[i].filters = filters; + sst[i].layermap = &(*layermaps)[i]; pja[i].jp = json_begin_map(map + segs[i], segs[i + 1] - segs[i]); pja[i].exclude = exclude; @@ -405,7 +406,6 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const pja[i].exclude_all = exclude_all; pja[i].basezoom = basezoom; pja[i].layer = source; - pja[i].layermap = &(*layermaps)[i]; pja[i].layername = &layername; pja[i].attribute_types = attribute_types; @@ -1377,8 +1377,9 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo sst.maxzoom = maxzoom; sst.filters = prefilter != NULL || postfilter != NULL; sst.uses_gamma = uses_gamma; + sst.layermap = &layermaps[0]; - parse_json(&sst, jp, exclude, include, exclude_all, basezoom, layer, &layermaps[0], sources[layer].layer, attribute_types); + parse_json(&sst, jp, exclude, include, exclude_all, basezoom, layer, sources[layer].layer, attribute_types); json_end(jp); overall_offset = layer_seq; checkdisk(reader, CPUS); diff --git a/serial.hpp b/serial.hpp index a93b53d..002e7ef 100644 --- a/serial.hpp +++ b/serial.hpp @@ -6,6 +6,7 @@ #include #include #include "geometry.hpp" +#include "mbtiles.hpp" size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname); @@ -63,6 +64,7 @@ struct serial_feature { long long bbox[4]; std::vector full_keys; std::vector full_values; + std::string layername; }; void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, const char *fname, long long wx, long long wy, bool include_minzoom); @@ -114,6 +116,8 @@ struct serialization_state { int maxzoom; bool filters; bool uses_gamma; + + std::map *layermap; }; #endif