From 6d6c1abc64fbedde88afe15e4f70ffbba671a17d Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 1 Aug 2016 14:29:30 -0700 Subject: [PATCH 1/8] Parse GeoJSON geometry into memory instead of straight to a file --- geojson.cpp | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index 846a374..08c10d2 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -63,7 +63,19 @@ static int mb_geometry[GEOM_TYPES] = { VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON, }; -long long parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FILE *out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) { +static void write_geometry(drawvec const &dv, long long *fpos, FILE *out, const char *fname) { + for (size_t i = 0; i < dv.size(); i++) { + if (dv[i].op == VT_CLOSEPATH) { + serialize_byte(out, dv[i].op, fpos, fname); + } else { + serialize_byte(out, dv[i].op, fpos, fname); + serialize_long_long(out, dv[i].x, fpos, fname); + serialize_long_long(out, dv[i].y, fpos, fname); + } + } +} + +long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) { long long g = 0; if (j == NULL || j->type != JSON_ARRAY) { @@ -83,7 +95,7 @@ long long parse_geometry(int t, json_object *j, long long *bbox, long long *fpos } } - g += parse_geometry(within, j->array[i], bbox, fpos, out, op, fname, line, wx, wy, initialized, initial_x, initial_y); + g += parse_geometry(within, j->array[i], bbox, out, op, fname, line, wx, wy, initialized, initial_x, initial_y); } } else { if (j->length >= 2 && j->array[0]->type == JSON_NUMBER && j->array[1]->type == JSON_NUMBER) { @@ -130,9 +142,8 @@ long long parse_geometry(int t, json_object *j, long long *bbox, long long *fpos *initialized = 1; } - serialize_byte(out, op, fpos, fname); - serialize_long_long(out, (x >> geometry_scale) - (*wx >> geometry_scale), fpos, fname); - serialize_long_long(out, (y >> geometry_scale) - (*wy >> geometry_scale), fpos, fname); + draw d(op, (x >> geometry_scale) - (*wx >> geometry_scale), (y >> geometry_scale) - (*wy >> geometry_scale)); + out.push_back(d); *wx = x; *wy = y; g++; @@ -151,7 +162,7 @@ long long parse_geometry(int t, json_object *j, long long *bbox, long long *fpos // rings come from which Polygons so that it can make the winding order // of the outer ring be the opposite of the order of the inner rings. - serialize_byte(out, VT_CLOSEPATH, fpos, fname); + out.push_back(draw(VT_CLOSEPATH, 0, 0)); } return g; @@ -314,7 +325,9 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje serialize_int(geomfile, segment, geompos, fname); long long wx = *initial_x, wy = *initial_y; - long long g = parse_geometry(t, coordinates, bbox, geompos, geomfile, VT_MOVETO, fname, line, &wx, &wy, initialized, initial_x, initial_y); + drawvec dv; + long long g = parse_geometry(t, coordinates, bbox, dv, VT_MOVETO, fname, line, &wx, &wy, initialized, initial_x, initial_y); + write_geometry(dv, geompos, geomfile, fname); serialize_byte(geomfile, VT_END, geompos, fname); bool inline_meta = true; From 67272bfe4a684da266257ff25e047bc69d9cdfc9 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 1 Aug 2016 17:35:37 -0700 Subject: [PATCH 2/8] Move polygon winding fixup out of tiling and into parsing --- geojson.cpp | 29 ++++++++++++++--------------- tile.cpp | 4 ---- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index 08c10d2..5424e4a 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -63,19 +63,21 @@ static int mb_geometry[GEOM_TYPES] = { VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON, }; -static void write_geometry(drawvec const &dv, long long *fpos, FILE *out, const char *fname) { +static void write_geometry(drawvec const &dv, long long *fpos, FILE *out, const char *fname, long long wx, long long wy) { for (size_t i = 0; i < dv.size(); i++) { if (dv[i].op == VT_CLOSEPATH) { serialize_byte(out, dv[i].op, fpos, fname); } else { serialize_byte(out, dv[i].op, fpos, fname); - serialize_long_long(out, dv[i].x, fpos, fname); - serialize_long_long(out, dv[i].y, fpos, fname); + serialize_long_long(out, dv[i].x - wx, fpos, fname); + serialize_long_long(out, dv[i].y - wy, fpos, fname); + wx = dv[i].x; + wy = dv[i].y; } } } -long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) { +long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, int op, const char *fname, int line, int *initialized, unsigned *initial_x, unsigned *initial_y) { long long g = 0; if (j == NULL || j->type != JSON_ARRAY) { @@ -95,7 +97,7 @@ long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, i } } - g += parse_geometry(within, j->array[i], bbox, out, op, fname, line, wx, wy, initialized, initial_x, initial_y); + g += parse_geometry(within, j->array[i], bbox, out, op, fname, line, initialized, initial_x, initial_y); } } else { if (j->length >= 2 && j->array[0]->type == JSON_NUMBER && j->array[1]->type == JSON_NUMBER) { @@ -130,22 +132,16 @@ long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, i if (x < 0 || x >= (1LL << 32) || y < 0 || y >= (1LL < 32)) { *initial_x = 1LL << 31; *initial_y = 1LL << 31; - *wx = 1LL << 31; - *wy = 1LL << 31; } else { *initial_x = (x >> geometry_scale) << geometry_scale; *initial_y = (y >> geometry_scale) << geometry_scale; - *wx = x; - *wy = y; } *initialized = 1; } - draw d(op, (x >> geometry_scale) - (*wx >> geometry_scale), (y >> geometry_scale) - (*wy >> geometry_scale)); + draw d(op, (x >> geometry_scale), (y >> geometry_scale)); out.push_back(d); - *wx = x; - *wy = y; g++; } else { fprintf(stderr, "%s:%d: malformed point\n", fname, line); @@ -324,10 +320,13 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje } serialize_int(geomfile, segment, geompos, fname); - long long wx = *initial_x, wy = *initial_y; + drawvec dv; - long long g = parse_geometry(t, coordinates, bbox, dv, VT_MOVETO, fname, line, &wx, &wy, initialized, initial_x, initial_y); - write_geometry(dv, geompos, geomfile, fname); + long long g = parse_geometry(t, coordinates, bbox, dv, VT_MOVETO, fname, line, initialized, initial_x, initial_y); + if (mb_geometry[t] == VT_POLYGON) { + dv = fix_polygon(dv); + } + write_geometry(dv, geompos, geomfile, fname, *initial_x >> geometry_scale, *initial_y >> geometry_scale); serialize_byte(geomfile, VT_END, geompos, fname); bool inline_meta = true; diff --git a/tile.cpp b/tile.cpp index 5fe69e2..9fd39c4 100644 --- a/tile.cpp +++ b/tile.cpp @@ -710,10 +710,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s original_features++; - if (z == 0 && t == VT_POLYGON) { - geom = fix_polygon(geom); - } - int quick = quick_check(bbox, z, line_detail, buffer); if (quick == 0) { continue; From 84370c59b8c3248937f50a27166cf5018304ee28 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 2 Aug 2016 14:07:56 -0700 Subject: [PATCH 3/8] Get rid of some more explicit memory management --- geojson.cpp | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index 5424e4a..2ce33d0 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -251,9 +251,9 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje long long metastart = *metapos; char *metakey[nprop]; - const char *metaval[nprop]; + std::vector metaval; + metaval.resize(nprop); int metatype[nprop]; - int mustfree[nprop]; int m = 0; int i; @@ -274,26 +274,26 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje tas.type = -1; metakey[m] = properties->keys[i]->string; - mustfree[m] = 0; if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) { tas.type = metatype[m] = VT_STRING; - metaval[m] = properties->values[i]->string; + metaval[m] = std::string(properties->values[i]->string); m++; } else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) { tas.type = metatype[m] = VT_NUMBER; - metaval[m] = properties->values[i]->string; + metaval[m] = std::string(properties->values[i]->string); m++; } else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) { tas.type = metatype[m] = VT_BOOLEAN; - metaval[m] = properties->values[i]->type == JSON_TRUE ? "true" : "false"; + metaval[m] = std::string(properties->values[i]->type == JSON_TRUE ? "true" : "false"); m++; } else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) { ; } else { tas.type = metatype[m] = VT_STRING; - metaval[m] = json_stringify(properties->values[i]); - mustfree[m] = 1; + const char *v = json_stringify(properties->values[i]); + metaval[m] = std::string(v); + free((void *) v); m++; } @@ -303,6 +303,12 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje } } + drawvec dv; + long long g = parse_geometry(t, coordinates, bbox, dv, VT_MOVETO, fname, line, initialized, initial_x, initial_y); + if (mb_geometry[t] == VT_POLYGON) { + dv = fix_polygon(dv); + } + long long geomstart = *geompos; serialize_byte(geomfile, mb_geometry[t], geompos, fname); @@ -321,11 +327,6 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje serialize_int(geomfile, segment, geompos, fname); - drawvec dv; - long long g = parse_geometry(t, coordinates, bbox, dv, VT_MOVETO, fname, line, initialized, initial_x, initial_y); - if (mb_geometry[t] == VT_POLYGON) { - dv = fix_polygon(dv); - } write_geometry(dv, geompos, geomfile, fname, *initial_x >> geometry_scale, *initial_y >> geometry_scale); serialize_byte(geomfile, VT_END, geompos, fname); @@ -358,20 +359,14 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje for (i = 0; i < m; i++) { serialize_long_long(geomfile, addpool(poolfile, treefile, metakey[i], VT_STRING), geompos, fname); - serialize_long_long(geomfile, addpool(poolfile, treefile, metaval[i], metatype[i]), geompos, fname); + serialize_long_long(geomfile, addpool(poolfile, treefile, metaval[i].c_str(), metatype[i]), geompos, fname); } } else { serialize_long_long(geomfile, metastart, geompos, fname); for (i = 0; i < m; i++) { serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname); - serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname); - } - } - - for (i = 0; i < m; i++) { - if (mustfree[i]) { - free((void *) metaval[i]); + serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i].c_str(), metatype[i]), metapos, fname); } } From 6de7920c9e147fe6eec2a922045398ebde71be01 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 2 Aug 2016 14:53:30 -0700 Subject: [PATCH 4/8] Separate data gathering from serialization a little better --- geojson.cpp | 53 ++++++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index 2ce33d0..dfa9189 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -256,8 +256,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje int metatype[nprop]; int m = 0; - int i; - for (i = 0; i < nprop; i++) { + for (size_t i = 0; i < nprop; i++) { if (properties->keys[i]->type == JSON_STRING) { std::string s(properties->keys[i]->string); @@ -309,27 +308,6 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje dv = fix_polygon(dv); } - long long geomstart = *geompos; - - serialize_byte(geomfile, mb_geometry[t], geompos, fname); - serialize_long_long(geomfile, *layer_seq, geompos, fname); - - serialize_long_long(geomfile, (layer << 3) | (has_id << 2) | ((tippecanoe_minzoom != -1) << 1) | (tippecanoe_maxzoom != -1), geompos, fname); - if (tippecanoe_minzoom != -1) { - serialize_int(geomfile, tippecanoe_minzoom, geompos, fname); - } - if (tippecanoe_maxzoom != -1) { - serialize_int(geomfile, tippecanoe_maxzoom, geompos, fname); - } - if (has_id) { - serialize_ulong_long(geomfile, id_value, geompos, fname); - } - - serialize_int(geomfile, segment, geompos, fname); - - write_geometry(dv, geompos, geomfile, fname, *initial_x >> geometry_scale, *initial_y >> geometry_scale); - serialize_byte(geomfile, VT_END, geompos, fname); - bool inline_meta = true; // Don't inline metadata for features that will span several tiles at maxzoom if (g > 0 && (bbox[2] < bbox[0] || bbox[3] < bbox[1])) { @@ -353,18 +331,39 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje } } + long long geomstart = *geompos; + + serialize_byte(geomfile, mb_geometry[t], geompos, fname); + serialize_long_long(geomfile, *layer_seq, geompos, fname); + + serialize_long_long(geomfile, (layer << 3) | (has_id << 2) | ((tippecanoe_minzoom != -1) << 1) | (tippecanoe_maxzoom != -1), geompos, fname); + if (tippecanoe_minzoom != -1) { + serialize_int(geomfile, tippecanoe_minzoom, geompos, fname); + } + if (tippecanoe_maxzoom != -1) { + serialize_int(geomfile, tippecanoe_maxzoom, geompos, fname); + } + if (has_id) { + serialize_ulong_long(geomfile, id_value, geompos, fname); + } + + serialize_int(geomfile, segment, geompos, fname); + + write_geometry(dv, geompos, geomfile, fname, *initial_x >> geometry_scale, *initial_y >> geometry_scale); + serialize_byte(geomfile, VT_END, geompos, fname); + serialize_int(geomfile, m, geompos, fname); if (inline_meta) { serialize_long_long(geomfile, -1, geompos, fname); - for (i = 0; i < m; i++) { + for (size_t i = 0; i < m; i++) { serialize_long_long(geomfile, addpool(poolfile, treefile, metakey[i], VT_STRING), geompos, fname); serialize_long_long(geomfile, addpool(poolfile, treefile, metaval[i].c_str(), metatype[i]), geompos, fname); } } else { serialize_long_long(geomfile, metastart, geompos, fname); - for (i = 0; i < m; i++) { + for (size_t i = 0; i < m; i++) { serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname); serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i].c_str(), metatype[i]), metapos, fname); } @@ -415,12 +414,12 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje fwrite_check(&index, sizeof(struct index), 1, indexfile, fname); *indexpos += sizeof(struct index); - for (i = 0; i < 2; i++) { + for (size_t i = 0; i < 2; i++) { if (bbox[i] < file_bbox[i]) { file_bbox[i] = bbox[i]; } } - for (i = 2; i < 4; i++) { + for (size_t i = 2; i < 4; i++) { if (bbox[i] > file_bbox[i]) { file_bbox[i] = bbox[i]; } From bf571571a9e97f10051e1ae5b2b9b3fcc901f457 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 8 Aug 2016 15:36:49 -0700 Subject: [PATCH 5/8] Factor out (initial) feature serialization --- geojson.cpp | 88 ++++++++++++++++++++--------------------------------- main.cpp | 2 +- serial.cpp | 48 +++++++++++++++++++++++++++++ serial.hpp | 27 ++++++++++++++++ 4 files changed, 109 insertions(+), 56 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index dfa9189..b99d8cf 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -31,12 +31,12 @@ extern "C" { #include "projection.hpp" #include "version.hpp" #include "memfile.hpp" -#include "serial.hpp" #include "main.hpp" #include "mbtiles.hpp" #include "geojson.hpp" #include "geometry.hpp" #include "options.hpp" +#include "serial.hpp" #define GEOM_POINT 0 /* array of positions */ #define GEOM_MULTIPOINT 1 /* array of arrays of positions */ @@ -63,20 +63,6 @@ static int mb_geometry[GEOM_TYPES] = { VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON, }; -static void write_geometry(drawvec const &dv, long long *fpos, FILE *out, const char *fname, long long wx, long long wy) { - for (size_t i = 0; i < dv.size(); i++) { - if (dv[i].op == VT_CLOSEPATH) { - serialize_byte(out, dv[i].op, fpos, fname); - } else { - serialize_byte(out, dv[i].op, fpos, fname); - serialize_long_long(out, dv[i].x - wx, fpos, fname); - serialize_long_long(out, dv[i].y - wy, fpos, fname); - wx = dv[i].x; - wy = dv[i].y; - } - } -} - long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, int op, const char *fname, int line, int *initialized, unsigned *initial_x, unsigned *initial_y) { long long g = 0; @@ -249,7 +235,6 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje nprop = properties->length; } - long long metastart = *metapos; char *metakey[nprop]; std::vector metaval; metaval.resize(nprop); @@ -331,44 +316,6 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje } } - long long geomstart = *geompos; - - serialize_byte(geomfile, mb_geometry[t], geompos, fname); - serialize_long_long(geomfile, *layer_seq, geompos, fname); - - serialize_long_long(geomfile, (layer << 3) | (has_id << 2) | ((tippecanoe_minzoom != -1) << 1) | (tippecanoe_maxzoom != -1), geompos, fname); - if (tippecanoe_minzoom != -1) { - serialize_int(geomfile, tippecanoe_minzoom, geompos, fname); - } - if (tippecanoe_maxzoom != -1) { - serialize_int(geomfile, tippecanoe_maxzoom, geompos, fname); - } - if (has_id) { - serialize_ulong_long(geomfile, id_value, geompos, fname); - } - - serialize_int(geomfile, segment, geompos, fname); - - write_geometry(dv, geompos, geomfile, fname, *initial_x >> geometry_scale, *initial_y >> geometry_scale); - serialize_byte(geomfile, VT_END, geompos, fname); - - serialize_int(geomfile, m, geompos, fname); - if (inline_meta) { - serialize_long_long(geomfile, -1, geompos, fname); - - for (size_t i = 0; i < m; i++) { - serialize_long_long(geomfile, addpool(poolfile, treefile, metakey[i], VT_STRING), geompos, fname); - serialize_long_long(geomfile, addpool(poolfile, treefile, metaval[i].c_str(), metatype[i]), geompos, fname); - } - } else { - serialize_long_long(geomfile, metastart, geompos, fname); - - for (size_t i = 0; i < m; i++) { - serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname); - serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i].c_str(), metatype[i]), metapos, fname); - } - } - /* * Note that feature_minzoom for lines is the dimension * of the geometry in world coordinates, but @@ -397,7 +344,38 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje feature_minzoom = basezoom - floor(log(r) / -log(droprate)); } - serialize_byte(geomfile, feature_minzoom, geompos, fname); + long long geomstart = *geompos; + + serial_feature sf; + sf.layer = layer; + sf.segment = segment; + sf.seq = *layer_seq; + sf.t = mb_geometry[t]; + sf.has_id = has_id; + sf.id = id_value; + sf.has_tippecanoe_minzoom = (tippecanoe_minzoom != -1); + sf.tippecanoe_minzoom = tippecanoe_minzoom; + sf.has_tippecanoe_maxzoom = (tippecanoe_maxzoom != -1); + sf.tippecanoe_maxzoom = tippecanoe_maxzoom; + sf.geometry = dv; + sf.m = m; + sf.feature_minzoom = feature_minzoom; + + if (inline_meta) { + sf.metapos = -1; + for (size_t i = 0; i < m; i++) { + sf.keys.push_back(addpool(poolfile, treefile, metakey[i], VT_STRING)); + sf.values.push_back(addpool(poolfile, treefile, metaval[i].c_str(), metatype[i])); + } + } else { + sf.metapos = *metapos; + for (size_t i = 0; i < m; i++) { + serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname); + serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i].c_str(), metatype[i]), metapos, fname); + } + } + + serialize_feature(geomfile, &sf, geompos, fname, *initial_x >> geometry_scale, *initial_y >> geometry_scale); struct index index; index.start = geomstart; diff --git a/main.cpp b/main.cpp index 6510f41..402215e 100644 --- a/main.cpp +++ b/main.cpp @@ -42,11 +42,11 @@ extern "C" { #include "projection.hpp" #include "version.hpp" #include "memfile.hpp" -#include "serial.hpp" #include "mbtiles.hpp" #include "main.hpp" #include "geojson.hpp" #include "geometry.hpp" +#include "serial.hpp" #include "options.hpp" static int low_detail = 12; diff --git a/serial.cpp b/serial.cpp index dc9d6c5..ae32eae 100644 --- a/serial.cpp +++ b/serial.cpp @@ -2,7 +2,11 @@ #include #include #include +#include +#include #include "protozero/varint.hpp" +#include "geometry.hpp" +#include "tile.hpp" #include "serial.hpp" size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) { @@ -151,3 +155,47 @@ int deserialize_byte_io(FILE *f, signed char *n, long long *geompos) { (*geompos)++; return 1; } + +static void write_geometry(drawvec const &dv, long long *fpos, FILE *out, const char *fname, long long wx, long long wy) { + for (size_t i = 0; i < dv.size(); i++) { + if (dv[i].op == VT_CLOSEPATH) { + serialize_byte(out, dv[i].op, fpos, fname); + } else { + serialize_byte(out, dv[i].op, fpos, fname); + serialize_long_long(out, dv[i].x - wx, fpos, fname); + serialize_long_long(out, dv[i].y - wy, fpos, fname); + wx = dv[i].x; + wy = dv[i].y; + } + } +} + +void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, const char *fname, long long wx, long long wy) { + serialize_byte(geomfile, sf->t, geompos, fname); + serialize_long_long(geomfile, sf->seq, geompos, fname); + + serialize_long_long(geomfile, (sf->layer << 3) | (sf->has_id ? 4 : 0) | (sf->has_tippecanoe_minzoom ? 2 : 0) | (sf->has_tippecanoe_maxzoom ? 1 : 0), geompos, fname); + if (sf->has_tippecanoe_minzoom) { + serialize_int(geomfile, sf->tippecanoe_minzoom, geompos, fname); + } + if (sf->has_tippecanoe_maxzoom) { + serialize_int(geomfile, sf->tippecanoe_maxzoom, geompos, fname); + } + if (sf->has_id) { + serialize_ulong_long(geomfile, sf->id, geompos, fname); + } + + serialize_int(geomfile, sf->segment, geompos, fname); + + write_geometry(sf->geometry, geompos, geomfile, fname, wx, wy); + serialize_byte(geomfile, VT_END, geompos, fname); + + serialize_int(geomfile, sf->m, geompos, fname); + serialize_long_long(geomfile, sf->metapos, geompos, fname); + for (size_t i = 0; i < sf->keys.size(); i++) { + serialize_long_long(geomfile, sf->keys[i], geompos, fname); + serialize_long_long(geomfile, sf->values[i], geompos, fname); + } + + serialize_byte(geomfile, sf->feature_minzoom, geompos, fname); +} diff --git a/serial.hpp b/serial.hpp index f1256a8..474af6f 100644 --- a/serial.hpp +++ b/serial.hpp @@ -18,3 +18,30 @@ int deserialize_long_long_io(FILE *f, long long *n, long long *geompos); int deserialize_ulong_long_io(FILE *f, unsigned long long *n, long long *geompos); int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos); int deserialize_byte_io(FILE *f, signed char *n, long long *geompos); + +struct serial_feature { + long long layer; + int segment; + long long seq; + + signed char t; + signed char feature_minzoom; + + bool has_id; + unsigned long long id; + + bool has_tippecanoe_minzoom; + int tippecanoe_minzoom; + + bool has_tippecanoe_maxzoom; + int tippecanoe_maxzoom; + + drawvec geometry; + + int m; + std::vector keys; + std::vector values; + long long metapos; +}; + +void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, const char *fname, long long wx, long long wy); From 8b339abd40a65faa3f0370bd70362639c4dcdb3a Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 8 Aug 2016 17:08:36 -0700 Subject: [PATCH 6/8] Use the same serialization in both places, and add sanity checks --- geometry.cpp | 5 +++- serial.cpp | 12 ++++++--- tile.cpp | 70 ++++++++++++++++++++++------------------------------ 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/geometry.cpp b/geometry.cpp index fdfd0e7..21b6f76 100644 --- a/geometry.cpp +++ b/geometry.cpp @@ -31,7 +31,10 @@ drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsi while (1) { draw d; - deserialize_byte_io(meta, &d.op, geompos); + if (!deserialize_byte_io(meta, &d.op, geompos)) { + fprintf(stderr, "Internal error: Unexpected end of file in geometry\n"); + exit(EXIT_FAILURE); + } if (d.op == VT_END) { break; } diff --git a/serial.cpp b/serial.cpp index ae32eae..ba7af1b 100644 --- a/serial.cpp +++ b/serial.cpp @@ -158,14 +158,14 @@ int deserialize_byte_io(FILE *f, signed char *n, long long *geompos) { static void write_geometry(drawvec const &dv, long long *fpos, FILE *out, const char *fname, long long wx, long long wy) { for (size_t i = 0; i < dv.size(); i++) { - if (dv[i].op == VT_CLOSEPATH) { - serialize_byte(out, dv[i].op, fpos, fname); - } else { + if (dv[i].op == VT_MOVETO || dv[i].op == VT_LINETO) { serialize_byte(out, dv[i].op, fpos, fname); serialize_long_long(out, dv[i].x - wx, fpos, fname); serialize_long_long(out, dv[i].y - wy, fpos, fname); wx = dv[i].x; wy = dv[i].y; + } else { + serialize_byte(out, dv[i].op, fpos, fname); } } } @@ -192,6 +192,12 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, c serialize_int(geomfile, sf->m, geompos, fname); serialize_long_long(geomfile, sf->metapos, geompos, fname); + + if (sf->metapos < 0 && sf->m != sf->keys.size()) { + fprintf(stderr, "Internal error: %d doesn't match %lld\n", sf->m, (long long) sf->keys.size()); + exit(EXIT_FAILURE); + } + for (size_t i = 0; i < sf->keys.size(); i++) { serialize_long_long(geomfile, sf->keys[i], geompos, fname); serialize_long_long(geomfile, sf->values[i], geompos, fname); diff --git a/tile.cpp b/tile.cpp index 9fd39c4..3293da8 100644 --- a/tile.cpp +++ b/tile.cpp @@ -293,6 +293,18 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u bbox2[k] /= 256; } + // Offset from tile coordinates back to world coordinates + unsigned sx = 0, sy = 0; + if (z != 0) { + sx = tx << (32 - z); + sy = ty << (32 - z); + } + + drawvec geom2; + for (size_t i = 0; i < geom.size(); i++) { + geom2.push_back(draw(geom[i].op, (geom[i].x + sx) >> geometry_scale, (geom[i].y + sy) >> geometry_scale)); + } + for (xo = bbox2[0]; xo <= bbox2[2]; xo++) { for (yo = bbox2[1]; yo <= bbox2[3]; yo++) { unsigned jx = tx * span + xo; @@ -326,52 +338,30 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u within[j] = 1; } - // Offset from tile coordinates back to world coordinates - unsigned sx = 0, sy = 0; - if (z != 0) { - sx = tx << (32 - z); - sy = ty << (32 - z); - } + serial_feature sf; + sf.layer = layer; + sf.segment = segment; + sf.seq = seq; + sf.t = t; + sf.has_id = has_id; + sf.id = id; + sf.has_tippecanoe_minzoom = tippecanoe_minzoom != -1; + sf.tippecanoe_minzoom = tippecanoe_minzoom; + sf.has_tippecanoe_maxzoom = tippecanoe_maxzoom != -1; + sf.tippecanoe_maxzoom = tippecanoe_maxzoom; + sf.metapos = metastart; + sf.geometry = geom2; + sf.m = m; + sf.feature_minzoom = feature_minzoom; - // printf("type %d, meta %lld\n", t, metastart); - serialize_byte(geomfile[j], t, &geompos[j], fname); - serialize_long_long(geomfile[j], seq, &geompos[j], fname); - serialize_long_long(geomfile[j], (layer << 3) | (has_id << 2) | ((tippecanoe_minzoom != -1) << 1) | (tippecanoe_maxzoom != -1), &geompos[j], fname); - if (tippecanoe_minzoom != -1) { - serialize_int(geomfile[j], tippecanoe_minzoom, geompos, fname); - } - if (tippecanoe_maxzoom != -1) { - serialize_int(geomfile[j], tippecanoe_maxzoom, geompos, fname); - } - if (has_id) { - serialize_ulong_long(geomfile[j], id, geompos, fname); - } - serialize_int(geomfile[j], segment, &geompos[j], fname); - long long wx = initial_x[segment], wy = initial_y[segment]; - - for (size_t u = 0; u < geom.size(); u++) { - serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname); - - if (geom[u].op != VT_CLOSEPATH) { - serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname); - serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname); - wx = geom[u].x + sx; - wy = geom[u].y + sy; - } - } - - serialize_byte(geomfile[j], VT_END, &geompos[j], fname); - - serialize_int(geomfile[j], m, &geompos[j], fname); - serialize_long_long(geomfile[j], metastart, &geompos[j], fname); if (metastart < 0) { for (int i = 0; i < m; i++) { - serialize_long_long(geomfile[j], metakeys[i], &geompos[j], fname); - serialize_long_long(geomfile[j], metavals[i], &geompos[j], fname); + sf.keys.push_back(metakeys[i]); + sf.values.push_back(metavals[i]); } } - serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname); + serialize_feature(geomfile[j], &sf, &geompos[j], fname, initial_x[segment] >> geometry_scale, initial_y[segment] >> geometry_scale); } } } From 475ce9dd23de34dd5f76106f74df43cbca69583e Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 8 Aug 2016 17:14:48 -0700 Subject: [PATCH 7/8] Fix g++ compiler warnings --- geojson.cpp | 6 +++--- serial.cpp | 3 ++- serial.hpp | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index b99d8cf..272873c 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -206,7 +206,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje } bool has_id = false; - unsigned long long id_value; + unsigned long long id_value = 0; if (id != NULL) { if (id->type == JSON_NUMBER) { if (id->number >= 0) { @@ -230,7 +230,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje long long bbox[] = {LLONG_MAX, LLONG_MAX, LLONG_MIN, LLONG_MIN}; - int nprop = 0; + size_t nprop = 0; if (properties != NULL && properties->type == JSON_HASH) { nprop = properties->length; } @@ -239,7 +239,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje std::vector metaval; metaval.resize(nprop); int metatype[nprop]; - int m = 0; + size_t m = 0; for (size_t i = 0; i < nprop; i++) { if (properties->keys[i]->type == JSON_STRING) { diff --git a/serial.cpp b/serial.cpp index ba7af1b..fd98343 100644 --- a/serial.cpp +++ b/serial.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include "protozero/varint.hpp" @@ -194,7 +195,7 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, c serialize_long_long(geomfile, sf->metapos, geompos, fname); if (sf->metapos < 0 && sf->m != sf->keys.size()) { - fprintf(stderr, "Internal error: %d doesn't match %lld\n", sf->m, (long long) sf->keys.size()); + fprintf(stderr, "Internal error: %lld doesn't match %lld\n", (long long) sf->m, (long long) sf->keys.size()); exit(EXIT_FAILURE); } diff --git a/serial.hpp b/serial.hpp index 474af6f..e8ca1e7 100644 --- a/serial.hpp +++ b/serial.hpp @@ -38,7 +38,7 @@ struct serial_feature { drawvec geometry; - int m; + size_t m; std::vector keys; std::vector values; long long metapos; From b2aa6de8983d408b70a49fbc4297184025bda98d Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 15 Aug 2016 13:11:35 -0700 Subject: [PATCH 8/8] Provide some JSON context when reporting parsing errors --- CHANGELOG.md | 4 ++++ geojson.cpp | 41 +++++++++++++++++++++++++++++++++-------- version.hpp | 2 +- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03c61d1..3d3647c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.12.9 + +* Clean up parsing and serialization. Provide some context with parsing errors. + ## 1.12.8 * Fix the spelling of the --preserve-input-order option diff --git a/geojson.cpp b/geojson.cpp index 272873c..3f071e2 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -63,11 +63,23 @@ static int mb_geometry[GEOM_TYPES] = { VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON, }; -long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, int op, const char *fname, int line, int *initialized, unsigned *initial_x, unsigned *initial_y) { +void json_context(json_object *j) { + char *s = json_stringify(j); + + if (strlen(s) >= 500) { + sprintf(s + 497, "..."); + } + + fprintf(stderr, "In JSON object %s\n", s); + free(s); +} + +long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, int op, const char *fname, int line, int *initialized, unsigned *initial_x, unsigned *initial_y, json_object *feature) { long long g = 0; if (j == NULL || j->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: expected array for type %d\n", fname, line, t); + json_context(feature); return g; } @@ -83,7 +95,7 @@ long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, i } } - g += parse_geometry(within, j->array[i], bbox, out, op, fname, line, initialized, initial_x, initial_y); + g += parse_geometry(within, j->array[i], bbox, out, op, fname, line, initialized, initial_x, initial_y, feature); } } else { if (j->length >= 2 && j->array[0]->type == JSON_NUMBER && j->array[1]->type == JSON_NUMBER) { @@ -97,6 +109,8 @@ long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, i if (!warned) { fprintf(stderr, "%s:%d: ignoring dimensions beyond two\n", fname, line); + json_context(j); + json_context(feature); warned = 1; } } @@ -131,6 +145,8 @@ long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, i g++; } else { fprintf(stderr, "%s:%d: malformed point\n", fname, line); + json_context(j); + json_context(feature); } } @@ -150,12 +166,13 @@ long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, i return g; } -int serialize_geometry(json_object *geometry, json_object *properties, json_object *id, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom) { +int serialize_geometry(json_object *geometry, json_object *properties, json_object *id, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, json_object *feature) { json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == NULL) { static int warned = 0; if (!warned) { fprintf(stderr, "%s:%d: null geometry (additional not reported)\n", reading, line); + json_context(feature); warned = 1; } @@ -163,13 +180,15 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje } if (geometry_type->type != JSON_STRING) { - fprintf(stderr, "%s:%d: geometry without type\n", reading, line); + fprintf(stderr, "%s:%d: geometry type is not a string\n", reading, line); + json_context(feature); return 0; } json_object *coordinates = json_hash_get(geometry, "coordinates"); if (coordinates == NULL || coordinates->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: feature without coordinates array\n", reading, line); + json_context(feature); return 0; } @@ -181,6 +200,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje } if (t >= GEOM_TYPES) { fprintf(stderr, "%s:%d: Can't handle geometry type %s\n", reading, line, geometry_type->string); + json_context(feature); return 0; } @@ -288,7 +308,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje } drawvec dv; - long long g = parse_geometry(t, coordinates, bbox, dv, VT_MOVETO, fname, line, initialized, initial_x, initial_y); + long long g = parse_geometry(t, coordinates, bbox, dv, VT_MOVETO, fname, line, initialized, initial_x, initial_y, feature); if (mb_geometry[t] == VT_POLYGON) { dv = fix_polygon(dv); } @@ -440,6 +460,9 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se if (j == NULL) { if (jp->error != NULL) { fprintf(stderr, "%s:%d: %s\n", reading, jp->line, jp->error); + if (jp->root != NULL) { + json_context(jp->root); + } } json_free(jp->root); @@ -495,7 +518,7 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se } found_geometries++; - serialize_geometry(j, NULL, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom); + serialize_geometry(j, NULL, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom, j); json_free(j); continue; } @@ -517,6 +540,7 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se json_object *geometry = json_hash_get(j, "geometry"); if (geometry == NULL) { fprintf(stderr, "%s:%d: feature with no geometry\n", reading, jp->line); + json_context(j); json_free(j); continue; } @@ -524,6 +548,7 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se json_object *properties = json_hash_get(j, "properties"); if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "%s:%d: feature without properties hash\n", reading, jp->line); + json_context(j); json_free(j); continue; } @@ -535,10 +560,10 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se if (geometries != NULL) { size_t g; for (g = 0; g < geometries->length; g++) { - serialize_geometry(geometries->array[g], properties, id, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom); + serialize_geometry(geometries->array[g], properties, id, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom, j); } } else { - serialize_geometry(geometry, properties, id, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom); + serialize_geometry(geometry, properties, id, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom, j); } json_free(j); diff --git a/version.hpp b/version.hpp index 932c0e7..6eae3fa 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.12.8\n" +#define VERSION "tippecanoe v1.12.9\n"