From afe739c2f300a40c696f17461d671655b7a7a6e4 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 10 Sep 2018 11:25:52 -0700 Subject: [PATCH] String IDs are now plain strings instead of references --- geojson.cpp | 3 --- mvt.cpp | 6 ++--- mvt.hpp | 2 +- plugin.cpp | 14 +----------- serial.cpp | 8 +++---- serial.hpp | 1 - tile-join.cpp | 30 ++++--------------------- tile.cpp | 60 +++++++++++--------------------------------------- write_json.cpp | 8 ++----- 9 files changed, 27 insertions(+), 105 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index b70f9c0..843d8e0 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -105,7 +105,6 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom bool has_id = false; unsigned long long id_value = 0; - bool has_string_id = false; std::string string_id_value; if (id != NULL) { if (id->type == JSON_NUMBER) { @@ -132,7 +131,6 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom } } } else if (id->type == JSON_STRING) { - has_string_id = true; string_id_value = id->string; } else { static bool warned_nan = false; @@ -199,7 +197,6 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom sf.t = mb_geometry[t]; sf.has_id = has_id; sf.id = id_value; - sf.has_string_id = has_string_id; sf.string_id = string_id_value; sf.has_tippecanoe_minzoom = (tippecanoe_minzoom != -1); sf.tippecanoe_minzoom = tippecanoe_minzoom; diff --git a/mvt.cpp b/mvt.cpp index 0a3d1ee..959a690 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -368,7 +368,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { case 10: /* string id */ { - feature.string_id = feature_reader.get_uint32(); + feature.string_id = feature_reader.get_string(); break; } @@ -557,8 +557,8 @@ std::string mvt_tile::encode() { if (layers[i].features[f].has_id) { feature_writer.add_uint64(1, layers[i].features[f].id); } - if (layers[i].features[f].string_id >= 0) { - feature_writer.add_uint32(10, layers[i].features[f].string_id); + if (layers[i].features[f].string_id.size() != 0) { + feature_writer.add_string(10, layers[i].features[f].string_id); } std::vector geometry; diff --git a/mvt.hpp b/mvt.hpp index 9c82419..8640150 100644 --- a/mvt.hpp +++ b/mvt.hpp @@ -63,7 +63,7 @@ struct mvt_feature { int /* mvt_geometry_type */ type = 0; unsigned long long id = 0; bool has_id = false; - ssize_t string_id = -1; + std::string string_id = ""; bool dropped = false; // For use during decoding diff --git a/plugin.cpp b/plugin.cpp index c6c3508..f539d40 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -218,17 +218,7 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: feature.id = atoll(id->string); feature.has_id = true; } else if (id->type == JSON_STRING) { - mvt_value val; - val.type = mvt_string; - val.string_value = id->string; - - std::vector onto; - l->second.tag_v3_value(val, onto); - if (onto.size() != 1) { - fprintf(stderr, "Internal error: tagging a string value didn't have a length of 1\n"); - exit(EXIT_FAILURE); - } - feature.string_id = onto[0] >> 4; + feature.string_id = id->string; } } @@ -425,7 +415,6 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: sf.extent = 0; sf.metapos = 0; sf.has_id = false; - sf.has_string_id = false; std::string layername = "unknown"; json_object *tippecanoe = json_hash_get(j, "tippecanoe"); @@ -480,7 +469,6 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: sf.has_id = true; } else if (id->type == JSON_STRING) { sf.string_id = id->string; - sf.has_string_id = true; } } diff --git a/serial.cpp b/serial.cpp index 0268ee2..9a45564 100644 --- a/serial.cpp +++ b/serial.cpp @@ -240,7 +240,7 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomiclayer << 7; - layer |= (sf->has_string_id << 6); + layer |= ((sf->string_id.size() != 0) << 6); layer |= (sf->seq != 0) << 5; layer |= (sf->index != 0) << 4; layer |= (sf->extent != 0) << 3; @@ -261,7 +261,7 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomichas_id) { serialize_ulong_long(geomfile, sf->id, geompos, fname); } - if (sf->has_string_id) { + if (sf->string_id.size() != 0) { serialize_string(geomfile, sf->string_id, geompos, fname); } @@ -311,7 +311,6 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_ sf.tippecanoe_maxzoom = -1; sf.id = 0; sf.has_id = false; - sf.has_string_id = false; if (sf.layer & (1 << 1)) { deserialize_int_io(geoms, &sf.tippecanoe_minzoom, geompos_in); } @@ -323,7 +322,6 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_ deserialize_ulong_long_io(geoms, &sf.id, geompos_in); } if (sf.layer & (1 << 6)) { - sf.has_string_id = true; deserialize_string_io(geoms, sf.string_id, geompos_in); } @@ -449,7 +447,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { sf.geometry = fix_polygon(sf.geometry); } - if (!sf.has_id && !sf.has_string_id) { + if (!sf.has_id && sf.string_id.size() == 0) { if (additional[A_GENERATE_IDS]) { sf.has_id = true; sf.id = sf.seq + 1; diff --git a/serial.hpp b/serial.hpp index 6712852..7c21edb 100644 --- a/serial.hpp +++ b/serial.hpp @@ -51,7 +51,6 @@ struct serial_feature { bool has_id = false; unsigned long long id = 0; - bool has_string_id = false; std::string string_id = ""; bool has_tippecanoe_minzoom = false; diff --git a/tile-join.cpp b/tile-join.cpp index 288a07b..1b067d6 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -181,15 +181,10 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::map("$id", v)); - } else if (feat.string_id >= 0) { - if (feat.string_id >= (ssize_t) layer.string_values.string_values.size()) { - fprintf(stderr, "Internal error: Out of bounds string ID reference\n"); - exit(EXIT_FAILURE); - } - + } else if (feat.string_id.size() != 0) { mvt_value v; v.type = mvt_string; - v.string_value = layer.string_values.string_values[feat.string_id]; + v.string_value = feat.string_id; attributes.insert(std::pair("$id", v)); } @@ -225,25 +220,8 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::map= 0) { - if (feat.string_id >= (ssize_t) layer.string_values.string_values.size()) { - fprintf(stderr, "Internal error: Out of bounds string ID reference\n"); - exit(EXIT_FAILURE); - } - - mvt_value v; - v.type = mvt_string; - v.string_value = layer.string_values.string_values[feat.string_id]; - - std::vector onto; - outlayer.tag_v3_value(v, onto); - - if (onto.size() != 1) { - fprintf(stderr, "Internal error: tagging string value didn't have size of 1\n"); - exit(EXIT_FAILURE); - } - - outfeature.string_id = onto[0] >> 4; + if (feat.string_id.size() != 0) { + outfeature.string_id = feat.string_id; } std::vector> todo; diff --git a/tile.cpp b/tile.cpp index b61181d..6c48a24 100644 --- a/tile.cpp +++ b/tile.cpp @@ -85,7 +85,6 @@ struct coalesce { double spacing = 0; bool has_id = false; unsigned long long id = 0; - bool has_string_id = false; std::string string_id; bool operator<(const coalesce &o) const { @@ -116,10 +115,6 @@ int coalcmp(const void *v1, const void *v2) { if (c1->has_id != c2->has_id) { return (int) c1->has_id - (int) c2->has_id; } - if (c1->has_string_id != c2->has_string_id) { - return (int) c1->has_string_id - (int) c2->has_string_id; - } - if (c1->has_id && c2->has_id) { if (c1->id < c2->id) { return -1; @@ -128,13 +123,12 @@ int coalcmp(const void *v1, const void *v2) { return 1; } } - if (c1->has_string_id && c2->has_string_id) { - if (c1->string_id < c2->string_id) { - return -1; - } - if (c1->string_id > c2->string_id) { - return 1; - } + + if (c1->string_id < c2->string_id) { + return -1; + } + if (c1->string_id > c2->string_id) { + return 1; } cmp = metacmp(c1->keys, c1->values, c1->stringpool, c2->keys, c2->values, c2->stringpool); @@ -257,7 +251,7 @@ static int metacmp(const std::vector &keys1, const std::vector *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector &metakeys, std::vector &metavals, bool has_id, unsigned long long id, bool has_string_id, std::string string_id, unsigned long long index, long long extent) { +void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector &metakeys, std::vector &metavals, bool has_id, unsigned long long id, std::string string_id, unsigned long long index, long long extent) { if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) { int xo, yo; int span = 1 << (nextzoom - z); @@ -343,7 +337,6 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u sf.t = t; sf.has_id = has_id; sf.id = id; - sf.has_string_id = has_string_id; sf.string_id = string_id; sf.has_tippecanoe_minzoom = tippecanoe_minzoom != -1; sf.tippecanoe_minzoom = tippecanoe_minzoom; @@ -390,7 +383,6 @@ struct partial { unsigned long long id = 0; std::string string_id; bool has_id = 0; - bool has_string_id = 0; ssize_t renamed = 0; long long extent = 0; long long clustered = 0; @@ -1357,7 +1349,7 @@ serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, cha if (*first_time && pass == 1) { /* only write out the next zoom once, even if we retry */ if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) { - rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.has_string_id, sf.string_id, sf.index, sf.extent); + rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.string_id, sf.index, sf.extent); } } @@ -1402,7 +1394,7 @@ serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, cha attributes.insert(std::pair("$id", v)); } - if (sf.has_string_id) { + if (sf.string_id.size() != 0) { mvt_value v; v.type = mvt_string; v.string_value = sf.string_id; @@ -1525,20 +1517,8 @@ void *run_prefilter(void *v) { tmp_feature.has_id = sf.has_id; tmp_feature.dropped = sf.dropped; - if (sf.has_string_id) { - mvt_value sv; - sv.type = mvt_string; - sv.string_value = sf.string_id; - - std::vector onto; - tmp_layer.tag_v3_value(sv, onto); - - if (onto.size() != 1) { - fprintf(stderr, "Internal error: Expected 1 element from tagging string\n"); - exit(EXIT_FAILURE); - } - - tmp_feature.string_id = onto[0] >> 4; + if (sf.string_id.size() != 0) { + tmp_feature.string_id = sf.string_id; } // Offset from tile coordinates back to world coordinates @@ -2010,7 +1990,6 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta p.id = sf.id; p.has_id = sf.has_id; p.string_id = sf.string_id; - p.has_string_id = sf.has_string_id; p.index = sf.index; p.renamed = -1; p.extent = sf.extent; @@ -2151,7 +2130,6 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta c.id = partials[i].id; c.has_id = partials[i].has_id; c.string_id = partials[i].string_id; - c.has_string_id = partials[i].has_string_id; // printf("segment %d layer %lld is %s\n", partials[i].segment, partials[i].layer, (*layer_unmaps)[partials[i].segment][partials[i].layer].c_str()); @@ -2270,20 +2248,8 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta feature.id = layer_features[x].id; feature.has_id = layer_features[x].has_id; - if (layer_features[x].has_string_id) { - mvt_value sv; - sv.type = mvt_string; - sv.string_value = layer_features[x].string_id; - - std::vector onto; - layer.tag_v3_value(sv, onto); - - if (onto.size() != 1) { - fprintf(stderr, "Internal error: Expected 1 element from tagging string\n"); - exit(EXIT_FAILURE); - } - - feature.string_id = onto[0] >> 4; + if (layer_features[x].string_id.size() != 0) { + feature.string_id = layer_features[x].string_id; } decode_meta(layer_features[x].keys, layer_features[x].values, layer_features[x].stringpool, layer, feature, true); diff --git a/write_json.cpp b/write_json.cpp index 464dc84..4301279 100644 --- a/write_json.cpp +++ b/write_json.cpp @@ -413,13 +413,9 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y state.json_write_string("id"); state.json_write_unsigned(feat.id); } - if (feat.string_id >= 0) { + if (feat.string_id.size() != 0) { state.json_write_string("id"); - if (feat.string_id >= (ssize_t) layer.string_values.string_values.size()) { - fprintf(stderr, "Internal error: out of bounds string ID reference %zd\n", feat.string_id); - exit(EXIT_FAILURE); - } - state.json_write_string(layer.string_values.string_values[feat.string_id]); + state.json_write_string(feat.string_id); } if (name || zoom || index != 0 || sequence != 0 || extent != 0) {