From 9fdd547582c15041ae85e55a57fa59718e725ed7 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 29 Oct 2019 15:37:35 -0700 Subject: [PATCH] Clean up naming around tilestats and JSON writing --- decode.cpp | 224 +++++++++++++++++----------------- main.cpp | 46 +++---- mbtiles.cpp | 288 +++++++++++++++++++++---------------------- mbtiles.hpp | 24 ++-- plugin.cpp | 74 +++++------ plugin.hpp | 4 +- serial.cpp | 14 +-- serial.hpp | 2 +- tile-join.cpp | 76 ++++++------ tile.cpp | 66 +++++----- tile.hpp | 2 +- write_json.cpp | 324 ++++++++++++++++++++++++------------------------- write_json.hpp | 36 +++--- 13 files changed, 590 insertions(+), 590 deletions(-) diff --git a/decode.cpp b/decode.cpp index e75248c..a6d5676 100644 --- a/decode.cpp +++ b/decode.cpp @@ -27,29 +27,29 @@ int minzoom = 0; int maxzoom = 32; bool force = false; -void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, unsigned y, json_writer &state) { - state.json_write_hash(); +void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, unsigned y, json_writer &jw) { + jw.begin_hash(); - state.json_write_string("zoom"); - state.json_write_signed(z); + jw.write_string("zoom"); + jw.write_signed(z); - state.json_write_string("x"); - state.json_write_unsigned(x); + jw.write_string("x"); + jw.write_unsigned(x); - state.json_write_string("y"); - state.json_write_unsigned(y); + jw.write_string("y"); + jw.write_unsigned(y); - state.json_write_string("bytes"); - state.json_write_unsigned(size); + jw.write_string("bytes"); + jw.write_unsigned(size); - state.json_write_string("compressed"); - state.json_write_bool(compressed); + jw.write_string("compressed"); + jw.write_bool(compressed); - state.json_write_string("layers"); - state.json_write_hash(); + jw.write_string("layers"); + jw.begin_hash(); for (size_t i = 0; i < tile.layers.size(); i++) { - state.json_write_string(tile.layers[i].name); + jw.write_string(tile.layers[i].name); size_t points = 0, lines = 0, polygons = 0; for (size_t j = 0; j < tile.layers[i].features.size(); j++) { @@ -62,30 +62,30 @@ void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, u } } - state.json_write_hash(); + jw.begin_hash(); - state.json_write_string("points"); - state.json_write_unsigned(points); + jw.write_string("points"); + jw.write_unsigned(points); - state.json_write_string("lines"); - state.json_write_unsigned(lines); + jw.write_string("lines"); + jw.write_unsigned(lines); - state.json_write_string("polygons"); - state.json_write_unsigned(polygons); + jw.write_string("polygons"); + jw.write_unsigned(polygons); - state.json_write_string("extent"); - state.json_write_signed(tile.layers[i].extent); + jw.write_string("extent"); + jw.write_signed(tile.layers[i].extent); - state.json_end_hash(); + jw.end_hash(); } - state.json_end_hash(); - state.json_end_hash(); + jw.end_hash(); + jw.end_hash(); - state.json_write_newline(); + jw.write_newline(); } -void handle(std::string message, int z, unsigned x, unsigned y, std::set const &to_decode, bool pipeline, bool stats, json_writer &state) { +void handle(std::string message, int z, unsigned x, unsigned y, std::set const &to_decode, bool pipeline, bool stats, json_writer &jw) { mvt_tile tile; bool was_compressed; @@ -100,57 +100,57 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::setalias); + jw.write_string("name"); + jw.write_string(projection->alias); - state.json_end_hash(); - state.json_end_hash(); + jw.end_hash(); + jw.end_hash(); } } - state.json_write_string("features"); - state.json_write_array(); - state.json_write_newline(); + jw.write_string("features"); + jw.begin_array(); + jw.write_newline(); } bool first_layer = true; @@ -169,32 +169,32 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::set co bool isdir = false; int oz = z; unsigned ox = x, oy = y; - json_writer state(stdout); + json_writer jw(stdout); int fd = open(fname, O_RDONLY | O_CLOEXEC); if (fd >= 0) { @@ -240,7 +240,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co if (strcmp(map, "SQLite format 3") != 0) { if (z >= 0) { std::string s = std::string(map, st.st_size); - handle(s, z, x, y, to_decode, pipeline, stats, state); + handle(s, z, x, y, to_decode, pipeline, stats, jw); munmap(map, st.st_size); return; } else { @@ -286,14 +286,14 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co int within = 0; if (!pipeline && !stats) { - state.json_write_hash(); + jw.begin_hash(); - state.json_write_string("type"); - state.json_write_string("FeatureCollection"); + jw.write_string("type"); + jw.write_string("FeatureCollection"); - state.json_write_string("properties"); - state.json_write_hash(); - state.json_write_newline(); + jw.write_string("properties"); + jw.begin_hash(); + jw.write_newline(); const char *sql2 = "SELECT name, value from metadata order by name;"; sqlite3_stmt *stmt2; @@ -308,7 +308,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co if (exclude_meta.count((char *) name) == 0) { if (within) { - state.json_comma_newline(); + jw.json_comma_newline(); } within = 1; @@ -317,28 +317,28 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co exit(EXIT_FAILURE); } - state.json_write_string((char *) name); - state.json_write_string((char *) value); + jw.write_string((char *) name); + jw.write_string((char *) value); } } - state.json_write_newline(); - state.wantnl = false; // XXX + jw.write_newline(); + jw.wantnl = false; // XXX sqlite3_finalize(stmt2); } if (stats) { - state.json_write_array(); - state.json_write_newline(); + jw.begin_array(); + jw.write_newline(); } if (!pipeline && !stats) { - state.json_end_hash(); + jw.end_hash(); - state.json_write_string("features"); - state.json_write_array(); - state.json_write_newline(); + jw.write_string("features"); + jw.begin_array(); + jw.write_newline(); } if (isdir) { @@ -346,13 +346,13 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co for (size_t i = 0; i < tiles.size(); i++) { if (!pipeline && !stats) { if (within) { - state.json_comma_newline(); + jw.json_comma_newline(); } within = 1; } if (stats) { if (within) { - state.json_comma_newline(); + jw.json_comma_newline(); } within = 1; } @@ -372,7 +372,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co } fclose(f); - handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, state); + handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, jw); } } else { const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;"; @@ -389,13 +389,13 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co while (sqlite3_step(stmt) == SQLITE_ROW) { if (!pipeline && !stats) { if (within) { - state.json_comma_newline(); + jw.json_comma_newline(); } within = 1; } if (stats) { if (within) { - state.json_comma_newline(); + jw.json_comma_newline(); } within = 1; } @@ -413,23 +413,23 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co ty = (1LL << tz) - 1 - ty; const char *s = (const char *) sqlite3_column_blob(stmt, 0); - handle(std::string(s, len), tz, tx, ty, to_decode, pipeline, stats, state); + handle(std::string(s, len), tz, tx, ty, to_decode, pipeline, stats, jw); } sqlite3_finalize(stmt); } if (!pipeline && !stats) { - state.json_end_array(); - state.json_end_hash(); - state.json_write_newline(); + jw.end_array(); + jw.end_hash(); + jw.write_newline(); } if (stats) { - state.json_end_array(); - state.json_write_newline(); + jw.end_array(); + jw.write_newline(); } if (pipeline) { - state.json_write_newline(); + jw.write_newline(); } } else { int handled = 0; @@ -453,7 +453,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy); } - handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state); + handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, jw); handled = 1; } diff --git a/main.cpp b/main.cpp index ae41d82..013baee 100644 --- a/main.cpp +++ b/main.cpp @@ -394,7 +394,7 @@ void *run_sort(void *v) { return NULL; } -void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector *readers, std::atomic *progress_seq, std::set *exclude, std::set *include, int exclude_all, int basezoom, int source, std::vector > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { +void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector *readers, std::atomic *progress_seq, std::set *exclude, std::set *include, int exclude_all, int basezoom, int source, std::vector > *tilestats, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { long long segs[CPUS + 1]; segs[0] = 0; segs[CPUS] = len; @@ -424,10 +424,10 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const sst.resize(CPUS); pthread_t pthreads[CPUS]; - std::vector > file_subkeys; + std::vector > file_subkeys; for (size_t i = 0; i < CPUS; i++) { - file_subkeys.push_back(std::set()); + file_subkeys.push_back(std::set()); } for (size_t i = 0; i < CPUS; i++) { @@ -446,7 +446,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const sst[i].maxzoom = maxzoom; sst[i].uses_gamma = uses_gamma; sst[i].filters = filters; - sst[i].layermap = &(*layermaps)[i]; + sst[i].tilestat = &(*tilestats)[i]; sst[i].exclude = exclude; sst[i].include = include; sst[i].exclude_all = exclude_all; @@ -585,7 +585,7 @@ struct read_parallel_arg { int maxzoom = 0; int basezoom = 0; int source = 0; - std::vector > *layermaps = NULL; + std::vector > *tilestats = NULL; int *initialized = NULL; unsigned *initial_x = NULL; unsigned *initial_y = NULL; @@ -617,7 +617,7 @@ void *run_read_parallel(void *v) { } madvise(map, rpa->len, MADV_RANDOM); // sequential, but from several pointers at once - do_read_parallel(map, rpa->len, rpa->offset, rpa->reading, rpa->readers, rpa->progress_seq, rpa->exclude, rpa->include, rpa->exclude_all, rpa->basezoom, rpa->source, rpa->layermaps, rpa->initialized, rpa->initial_x, rpa->initial_y, rpa->maxzoom, rpa->layername, rpa->uses_gamma, rpa->attribute_types, rpa->separator, rpa->dist_sum, rpa->dist_count, rpa->want_dist, rpa->filters); + do_read_parallel(map, rpa->len, rpa->offset, rpa->reading, rpa->readers, rpa->progress_seq, rpa->exclude, rpa->include, rpa->exclude_all, rpa->basezoom, rpa->source, rpa->tilestats, rpa->initialized, rpa->initial_x, rpa->initial_y, rpa->maxzoom, rpa->layername, rpa->uses_gamma, rpa->attribute_types, rpa->separator, rpa->dist_sum, rpa->dist_count, rpa->want_dist, rpa->filters); madvise(map, rpa->len, MADV_DONTNEED); if (munmap(map, rpa->len) != 0) { @@ -634,7 +634,7 @@ void *run_read_parallel(void *v) { return NULL; } -void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::atomic *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector *readers, std::atomic *progress_seq, std::set *exclude, std::set *include, int exclude_all, int basezoom, int source, std::vector > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { +void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::atomic *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector *readers, std::atomic *progress_seq, std::set *exclude, std::set *include, int exclude_all, int basezoom, int source, std::vector > &tilestats, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { // This has to kick off an intermediate thread to start the parser threads, // so the main thread can get back to reading the next input stage while // the intermediate thread waits for the completion of the parser threads. @@ -662,7 +662,7 @@ void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::ato rpa->exclude_all = exclude_all; rpa->basezoom = basezoom; rpa->source = source; - rpa->layermaps = &layermaps; + rpa->tilestats = &tilestats; rpa->initialized = initialized; rpa->initial_x = initial_x; rpa->initial_y = initial_y; @@ -1303,16 +1303,16 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } } - std::map layermap; + std::map tilestat; for (size_t l = 0; l < nlayers; l++) { - layermap_entry e = layermap_entry(l); + tilestats_entry e = tilestats_entry(l); e.description = sources[l].description; - layermap.insert(std::pair(sources[l].layer, e)); + tilestat.insert(std::pair(sources[l].layer, e)); } - std::vector > layermaps; + std::vector > tilestats; for (size_t l = 0; l < CPUS; l++) { - layermaps.push_back(layermap); + tilestats.push_back(tilestat); } long overall_offset = 0; @@ -1346,8 +1346,8 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } } - auto a = layermap.find(sources[source].layer); - if (a == layermap.end()) { + auto a = tilestat.find(sources[source].layer); + if (a == tilestat.end()) { fprintf(stderr, "Internal error: couldn't find layer %s", sources[source].layer.c_str()); exit(EXIT_FAILURE); } @@ -1393,7 +1393,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo sst[i].maxzoom = maxzoom; sst[i].filters = prefilter != NULL || postfilter != NULL; sst[i].uses_gamma = uses_gamma; - sst[i].layermap = &layermaps[i]; + sst[i].tilestat = &tilestats[i]; sst[i].exclude = exclude; sst[i].include = include; sst[i].exclude_all = exclude_all; @@ -1451,7 +1451,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo sst[i].maxzoom = maxzoom; sst[i].filters = prefilter != NULL || postfilter != NULL; sst[i].uses_gamma = uses_gamma; - sst[i].layermap = &layermaps[i]; + sst[i].tilestat = &tilestats[i]; sst[i].exclude = exclude; sst[i].include = include; sst[i].exclude_all = exclude_all; @@ -1508,7 +1508,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } if (map != NULL && map != MAP_FAILED && read_parallel_this) { - do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, basezoom, layer, &layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, uses_gamma, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); + do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, basezoom, layer, &tilestats, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, uses_gamma, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); overall_offset += st.st_size - off; checkdisk(&readers); @@ -1586,7 +1586,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } fflush(readfp); - start_parsing(readfd, streamfpopen(readfp), initial_offset, ahead, &is_parsing, ¶llel_parser, parser_created, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, basezoom, layer, layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); + start_parsing(readfd, streamfpopen(readfp), initial_offset, ahead, &is_parsing, ¶llel_parser, parser_created, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, basezoom, layer, tilestats, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); initial_offset += ahead; overall_offset += ahead; @@ -1623,7 +1623,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo fflush(readfp); if (ahead > 0) { - start_parsing(readfd, streamfpopen(readfp), initial_offset, ahead, &is_parsing, ¶llel_parser, parser_created, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, basezoom, layer, layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); + start_parsing(readfd, streamfpopen(readfp), initial_offset, ahead, &is_parsing, ¶llel_parser, parser_created, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, basezoom, layer, tilestats, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); if (parser_created) { if (pthread_join(parallel_parser, NULL) != 0) { @@ -1657,7 +1657,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo sst.maxzoom = maxzoom; sst.filters = prefilter != NULL || postfilter != NULL; sst.uses_gamma = uses_gamma; - sst.layermap = &layermaps[0]; + sst.tilestat = &tilestats[0]; sst.exclude = exclude; sst.include = include; sst.exclude_all = exclude_all; @@ -2267,7 +2267,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo std::atomic midx(0); std::atomic midy(0); - int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum, filter); + int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, tilestats, prefilter, postfilter, attribute_accum, filter); if (maxzoom != written) { if (written > minzoom) { @@ -2322,7 +2322,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo midlon = maxlon; } - std::map merged_lm = merge_layermaps(layermaps); + std::map merged_lm = merge_layermaps(tilestats); for (auto ai = merged_lm.begin(); ai != merged_lm.end(); ++ai) { ai->second.minzoom = minzoom; diff --git a/mbtiles.cpp b/mbtiles.cpp index 45eb388..8d0b53c 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -94,7 +94,7 @@ void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, } } -bool type_and_string::operator<(const type_and_string &o) const { +bool tilestats_attributes_entry::operator<(const tilestats_attributes_entry &o) const { if (string < o.string) { return true; } @@ -104,7 +104,7 @@ bool type_and_string::operator<(const type_and_string &o) const { return false; } -bool type_and_string::operator!=(const type_and_string &o) const { +bool tilestats_attributes_entry::operator!=(const tilestats_attributes_entry &o) const { if (type != o.type) { return true; } @@ -114,36 +114,36 @@ bool type_and_string::operator!=(const type_and_string &o) const { return false; } -void tilestats(std::map const &layermap1, size_t elements, json_writer &state) { +void tilestats(std::map const &layermap1, size_t elements, json_writer &jw) { // Consolidate layers/attributes whose names are truncated - std::vector> lmv; + std::vector> lmv; lmv.push_back(layermap1); - std::map layermap = merge_layermaps(lmv, true); + std::map tilestat = merge_layermaps(lmv, true); - state.json_write_hash(); + jw.begin_hash(); - state.nospace = true; - state.json_write_string("layerCount"); - state.json_write_unsigned(layermap.size()); + jw.nospace = true; + jw.write_string("layerCount"); + jw.write_unsigned(tilestat.size()); - state.nospace = true; - state.json_write_string("layers"); - state.json_write_array(); + jw.nospace = true; + jw.write_string("layers"); + jw.begin_array(); bool first = true; - for (auto layer : layermap) { + for (auto layer : tilestat) { first = false; - state.nospace = true; - state.json_write_hash(); + jw.nospace = true; + jw.begin_hash(); - state.nospace = true; - state.json_write_string("layer"); - state.json_write_string(layer.first); + jw.nospace = true; + jw.write_string("layer"); + jw.write_string(layer.first); - state.nospace = true; - state.json_write_string("count"); - state.json_write_unsigned(layer.second.points + layer.second.lines + layer.second.polygons); + jw.nospace = true; + jw.write_string("count"); + jw.write_unsigned(layer.second.points + layer.second.lines + layer.second.polygons); std::string geomtype = "Polygon"; if (layer.second.points >= layer.second.lines && layer.second.points >= layer.second.polygons) { @@ -152,46 +152,46 @@ void tilestats(std::map const &layermap1, size_t el geomtype = "LineString"; } - state.nospace = true; - state.json_write_string("geometry"); - state.json_write_string(geomtype); + jw.nospace = true; + jw.write_string("geometry"); + jw.write_string(geomtype); - size_t attrib_count = layer.second.file_keys.size(); + size_t attrib_count = layer.second.tilestats.size(); if (attrib_count > max_tilestats_attributes) { attrib_count = max_tilestats_attributes; } - state.nospace = true; - state.json_write_string("attributeCount"); - state.json_write_unsigned(attrib_count); + jw.nospace = true; + jw.write_string("attributeCount"); + jw.write_unsigned(attrib_count); - state.nospace = true; - state.json_write_string("attributes"); - state.nospace = true; - state.json_write_array(); + jw.nospace = true; + jw.write_string("attributes"); + jw.nospace = true; + jw.begin_array(); size_t attrs = 0; - for (auto attribute : layer.second.file_keys) { + for (auto attribute : layer.second.tilestats) { if (attrs == elements) { break; } attrs++; - state.nospace = true; - state.json_write_hash(); + jw.nospace = true; + jw.begin_hash(); - state.nospace = true; - state.json_write_string("attribute"); - state.json_write_string(attribute.first); + jw.nospace = true; + jw.write_string("attribute"); + jw.write_string(attribute.first); size_t val_count = attribute.second.sample_values.size(); if (val_count > max_tilestats_sample_values) { val_count = max_tilestats_sample_values; } - state.nospace = true; - state.json_write_string("count"); - state.json_write_unsigned(val_count); + jw.nospace = true; + jw.write_string("count"); + jw.write_unsigned(val_count); int type = 0; for (auto s : attribute.second.sample_values) { @@ -210,13 +210,13 @@ void tilestats(std::map const &layermap1, size_t el type_str = "mixed"; } - state.nospace = true; - state.json_write_string("type"); - state.json_write_string(type_str); + jw.nospace = true; + jw.write_string("type"); + jw.write_string(type_str); - state.nospace = true; - state.json_write_string("values"); - state.json_write_array(); + jw.nospace = true; + jw.write_string("values"); + jw.begin_array(); size_t vals = 0; for (auto value : attribute.second.sample_values) { @@ -224,53 +224,53 @@ void tilestats(std::map const &layermap1, size_t el break; } - state.nospace = true; + jw.nospace = true; if (value.type == mvt_double || value.type == mvt_bool) { vals++; - state.json_write_stringified(value.string); + jw.write_stringified(value.string); } else { std::string trunc = truncate16(value.string, 256); if (trunc.size() == value.string.size()) { vals++; - state.json_write_string(value.string); + jw.write_string(value.string); } } } - state.nospace = true; - state.json_end_array(); + jw.nospace = true; + jw.end_array(); if ((type & (1 << mvt_double)) != 0) { - state.nospace = true; - state.json_write_string("min"); - state.json_write_number(attribute.second.min); + jw.nospace = true; + jw.write_string("min"); + jw.write_number(attribute.second.min); - state.nospace = true; - state.json_write_string("max"); - state.json_write_number(attribute.second.max); + jw.nospace = true; + jw.write_string("max"); + jw.write_number(attribute.second.max); } - state.nospace = true; - state.json_end_hash(); + jw.nospace = true; + jw.end_hash(); } - state.nospace = true; - state.json_end_array(); - state.nospace = true; - state.json_end_hash(); + jw.nospace = true; + jw.end_array(); + jw.nospace = true; + jw.end_hash(); } - state.nospace = true; - state.json_end_array(); - state.nospace = true; - state.json_end_hash(); + jw.nospace = true; + jw.end_array(); + jw.nospace = true; + jw.end_hash(); } -void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description, bool do_tilestats, std::map const &attribute_descriptions, std::string const &program, std::string const &commandline) { +void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &tilestat, bool vector, const char *description, bool do_tilestats, std::map const &attribute_descriptions, std::string const &program, std::string const &commandline) { char *sql, *err; sqlite3 *db = outdb; @@ -401,46 +401,46 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam std::string buf; { - json_writer state(&buf); + json_writer jw(&buf); - state.json_write_hash(); - state.nospace = true; + jw.begin_hash(); + jw.nospace = true; - state.json_write_string("vector_layers"); - state.json_write_array(); + jw.write_string("vector_layers"); + jw.begin_array(); std::vector lnames; - for (auto ai = layermap.begin(); ai != layermap.end(); ++ai) { + for (auto ai = tilestat.begin(); ai != tilestat.end(); ++ai) { lnames.push_back(ai->first); } for (size_t i = 0; i < lnames.size(); i++) { - auto fk = layermap.find(lnames[i]); - state.json_write_hash(); + auto ts = tilestat.find(lnames[i]); + jw.begin_hash(); - state.json_write_string("id"); - state.json_write_string(lnames[i]); + jw.write_string("id"); + jw.write_string(lnames[i]); - state.json_write_string("description"); - state.json_write_string(fk->second.description); + jw.write_string("description"); + jw.write_string(ts->second.description); - state.json_write_string("minzoom"); - state.json_write_signed(fk->second.minzoom); + jw.write_string("minzoom"); + jw.write_signed(ts->second.minzoom); - state.json_write_string("maxzoom"); - state.json_write_signed(fk->second.maxzoom); + jw.write_string("maxzoom"); + jw.write_signed(ts->second.maxzoom); - state.json_write_string("fields"); - state.json_write_hash(); - state.nospace = true; + jw.write_string("fields"); + jw.begin_hash(); + jw.nospace = true; bool first = true; - for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) { + for (auto j = ts->second.tilestats.begin(); j != ts->second.tilestats.end(); ++j) { if (first) { first = false; } - state.json_write_string(j->first); + jw.write_string(j->first); auto f = attribute_descriptions.find(j->first); if (f == attribute_descriptions.end()) { @@ -450,34 +450,34 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } if (type == (1 << mvt_double)) { - state.json_write_string("Number"); + jw.write_string("Number"); } else if (type == (1 << mvt_bool)) { - state.json_write_string("Boolean"); + jw.write_string("Boolean"); } else if (type == (1 << mvt_string)) { - state.json_write_string("String"); + jw.write_string("String"); } else { - state.json_write_string("Mixed"); + jw.write_string("Mixed"); } } else { - state.json_write_string(f->second); + jw.write_string(f->second); } } - state.nospace = true; - state.json_end_hash(); - state.json_end_hash(); + jw.nospace = true; + jw.end_hash(); + jw.end_hash(); } - state.json_end_array(); + jw.end_array(); if (do_tilestats && elements > 0) { - state.nospace = true; - state.json_write_string("tilestats"); - tilestats(layermap, elements, state); + jw.nospace = true; + jw.write_string("tilestats"); + tilestats(tilestat, elements, jw); } - state.nospace = true; - state.json_end_hash(); + jw.nospace = true; + jw.end_hash(); } sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str()); @@ -503,10 +503,10 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam exit(EXIT_FAILURE); } - json_writer state(fp); + json_writer jw(fp); - state.json_write_hash(); - state.json_write_newline(); + jw.begin_hash(); + jw.write_newline(); sqlite3_stmt *stmt; bool first = true; @@ -521,17 +521,17 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam exit(EXIT_FAILURE); } - state.json_comma_newline(); - state.json_write_string(k); - state.json_write_string(v); + jw.json_comma_newline(); + jw.write_string(k); + jw.write_string(v); first = false; } sqlite3_finalize(stmt); } - state.json_write_newline(); - state.json_end_hash(); - state.json_write_newline(); + jw.write_newline(); + jw.end_hash(); + jw.write_newline(); fclose(fp); } } @@ -557,12 +557,12 @@ void mbtiles_close(sqlite3 *outdb, const char *pgm) { } } -std::map merge_layermaps(std::vector> const &maps) { +std::map merge_layermaps(std::vector> const &maps) { return merge_layermaps(maps, false); } -std::map merge_layermaps(std::vector> const &maps, bool trunc) { - std::map out; +std::map merge_layermaps(std::vector> const &maps, bool trunc) { + std::map out; for (size_t i = 0; i < maps.size(); i++) { for (auto map = maps[i].begin(); map != maps[i].end(); ++map) { @@ -576,7 +576,7 @@ std::map merge_layermaps(std::vector(layername, layermap_entry(out.size()))); + out.insert(std::pair(layername, tilestats_entry(out.size()))); auto out_entry = out.find(layername); out_entry->second.minzoom = map->second.minzoom; out_entry->second.maxzoom = map->second.maxzoom; @@ -589,18 +589,18 @@ std::map merge_layermaps(std::vectorsecond.file_keys.begin(); fk != map->second.file_keys.end(); ++fk) { - std::string attribname = fk->first; + for (auto ts = map->second.tilestats.begin(); ts != map->second.tilestats.end(); ++ts) { + std::string attribname = ts->first; if (trunc) { attribname = truncate16(attribname, 256); } - auto fk2 = out_entry->second.file_keys.find(attribname); + auto fk2 = out_entry->second.tilestats.find(attribname); - if (fk2 == out_entry->second.file_keys.end()) { - out_entry->second.file_keys.insert(std::pair(attribname, fk->second)); + if (fk2 == out_entry->second.tilestats.end()) { + out_entry->second.tilestats.insert(std::pair(attribname, ts->second)); } else { - for (auto val : fk->second.sample_values) { + for (auto val : ts->second.sample_values) { auto pt = std::lower_bound(fk2->second.sample_values.begin(), fk2->second.sample_values.end(), val); if (pt == fk2->second.sample_values.end() || *pt != val) { // not found fk2->second.sample_values.insert(pt, val); @@ -611,13 +611,13 @@ std::map merge_layermaps(std::vectorsecond.type |= fk->second.type; + fk2->second.type |= ts->second.type; - if (fk->second.min < fk2->second.min) { - fk2->second.min = fk->second.min; + if (ts->second.min < fk2->second.min) { + fk2->second.min = ts->second.min; } - if (fk->second.max > fk2->second.max) { - fk2->second.max = fk->second.max; + if (ts->second.max > fk2->second.max) { + fk2->second.max = ts->second.max; } } } @@ -638,18 +638,18 @@ std::map merge_layermaps(std::vector &file_keys, std::string const &attrib, type_and_string const &val) { +void add_to_tilestats(std::map &tilestats, std::string const &attrib, tilestats_attributes_entry const &val) { if (val.type == mvt_null) { return; } - auto fka = file_keys.find(attrib); - if (fka == file_keys.end()) { - file_keys.insert(std::pair(attrib, type_and_string_stats())); - fka = file_keys.find(attrib); + auto tsa = tilestats.find(attrib); + if (tsa == tilestats.end()) { + tilestats.insert(std::pair(attrib, tilestats_attributes())); + tsa = tilestats.find(attrib); } - if (fka == file_keys.end()) { + if (tsa == tilestats.end()) { fprintf(stderr, "Can't happen (tilestats)\n"); exit(EXIT_FAILURE); } @@ -657,22 +657,22 @@ void add_to_file_keys(std::map &file_keys, s if (val.type == mvt_double) { double d = atof(val.string.c_str()); - if (d < fka->second.min) { - fka->second.min = d; + if (d < tsa->second.min) { + tsa->second.min = d; } - if (d > fka->second.max) { - fka->second.max = d; + if (d > tsa->second.max) { + tsa->second.max = d; } } - auto pt = std::lower_bound(fka->second.sample_values.begin(), fka->second.sample_values.end(), val); - if (pt == fka->second.sample_values.end() || *pt != val) { // not found - fka->second.sample_values.insert(pt, val); + auto pt = std::lower_bound(tsa->second.sample_values.begin(), tsa->second.sample_values.end(), val); + if (pt == tsa->second.sample_values.end() || *pt != val) { // not found + tsa->second.sample_values.insert(pt, val); - if (fka->second.sample_values.size() > max_tilestats_sample_values) { - fka->second.sample_values.pop_back(); + if (tsa->second.sample_values.size() > max_tilestats_sample_values) { + tsa->second.sample_values.pop_back(); } } - fka->second.type |= (1 << val.type); + tsa->second.type |= (1 << val.type); } diff --git a/mbtiles.hpp b/mbtiles.hpp index 0868616..66c9725 100644 --- a/mbtiles.hpp +++ b/mbtiles.hpp @@ -9,24 +9,24 @@ extern size_t max_tilestats_attributes; extern size_t max_tilestats_sample_values; extern size_t max_tilestats_values; -struct type_and_string { +struct tilestats_attributes_entry { int type = 0; std::string string = ""; - bool operator<(const type_and_string &o) const; - bool operator!=(const type_and_string &o) const; + bool operator<(const tilestats_attributes_entry &o) const; + bool operator!=(const tilestats_attributes_entry &o) const; }; -struct type_and_string_stats { - std::vector sample_values = std::vector(); // sorted +struct tilestats_attributes { + std::vector sample_values = std::vector(); // sorted double min = INFINITY; double max = -INFINITY; int type = 0; }; -struct layermap_entry { +struct tilestats_entry { size_t id = 0; - std::map file_keys{}; + std::map tilestats{}; int minzoom = 0; int maxzoom = 0; std::string description = ""; @@ -36,7 +36,7 @@ struct layermap_entry { size_t polygons = 0; size_t retain = 0; // keep for tilestats, even if no features directly here - layermap_entry(size_t _id) { + tilestats_entry(size_t _id) { id = _id; } }; @@ -45,13 +45,13 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); -void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description, bool do_tilestats, std::map const &attribute_descriptions, std::string const &program, std::string const &commandline); +void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &tilestat, bool vector, const char *description, bool do_tilestats, std::map const &attribute_descriptions, std::string const &program, std::string const &commandline); void mbtiles_close(sqlite3 *outdb, const char *pgm); -std::map merge_layermaps(std::vector > const &maps); -std::map merge_layermaps(std::vector > const &maps, bool trunc); +std::map merge_layermaps(std::vector > const &maps); +std::map merge_layermaps(std::vector > const &maps, bool trunc); -void add_to_file_keys(std::map &file_keys, std::string const &layername, type_and_string const &val); +void add_to_tilestats(std::map &tilestats, std::string const &layername, tilestats_attributes_entry const &val); #endif diff --git a/plugin.cpp b/plugin.cpp index d416299..523235d 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -51,9 +51,9 @@ void *run_writer(void *a) { exit(EXIT_FAILURE); } - json_writer state(fp); + json_writer jw(fp); for (size_t i = 0; i < wa->layers->size(); i++) { - layer_to_geojson((*(wa->layers))[i], wa->z, wa->x, wa->y, false, true, false, true, 0, 0, 0, true, state); + layer_to_geojson((*(wa->layers))[i], wa->z, wa->x, wa->y, false, true, false, true, 0, 0, 0, true, jw); } if (fclose(fp) != 0) { @@ -84,7 +84,7 @@ static std::vector to_feature(drawvec &geom) { } // Reads from the postfilter -std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, int extent) { +std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std::vector> *tilestats, size_t tiling_seg, std::vector> *layer_unmaps, int extent) { std::map ret; FILE *f = fdopen(fd, "r"); @@ -220,13 +220,13 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: feature.has_id = true; } - std::map &layermap = (*layermaps)[tiling_seg]; - if (layermap.count(layername) == 0) { - layermap_entry lme = layermap_entry(layermap.size()); + std::map &tilestat = (*tilestats)[tiling_seg]; + if (tilestat.count(layername) == 0) { + tilestats_entry lme = tilestats_entry(tilestat.size()); lme.minzoom = z; lme.maxzoom = z; - layermap.insert(std::pair(layername, lme)); + tilestat.insert(std::pair(layername, lme)); if (lme.id >= (*layer_unmaps)[tiling_seg].size()) { (*layer_unmaps)[tiling_seg].resize(lme.id + 1); @@ -234,24 +234,24 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: } } - auto fk = layermap.find(layername); - if (fk == layermap.end()) { + auto ts = tilestat.find(layername); + if (ts == tilestat.end()) { fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str()); exit(EXIT_FAILURE); } - if (z < fk->second.minzoom) { - fk->second.minzoom = z; + if (z < ts->second.minzoom) { + ts->second.minzoom = z; } - if (z > fk->second.maxzoom) { - fk->second.maxzoom = z; + if (z > ts->second.maxzoom) { + ts->second.maxzoom = z; } if (feature.type == mvt_point) { - fk->second.points++; + ts->second.points++; } else if (feature.type == mvt_linestring) { - fk->second.lines++; + ts->second.lines++; } else if (feature.type == mvt_polygon) { - fk->second.polygons++; + ts->second.polygons++; } for (size_t i = 0; i < properties->length; i++) { @@ -267,11 +267,11 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: mvt_value v = stringified_to_mvt_value(tp, s.c_str()); l->second.tag(feature, std::string(properties->keys[i]->string), v); - type_and_string attrib; + tilestats_attributes_entry attrib; attrib.type = tp; attrib.string = s; - add_to_file_keys(fk->second.file_keys, std::string(properties->keys[i]->string), attrib); + add_to_tilestats(ts->second.tilestats, std::string(properties->keys[i]->string), attrib); } } @@ -295,7 +295,7 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: } // Reads from the prefilter -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter) { +serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *tilestats, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter) { serial_feature sf; while (1) { @@ -452,14 +452,14 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: sf.has_id = true; } - std::map &layermap = (*layermaps)[tiling_seg]; + std::map &tilestat = (*tilestats)[tiling_seg]; - if (layermap.count(layername) == 0) { - layermap_entry lme = layermap_entry(layermap.size()); + if (tilestat.count(layername) == 0) { + tilestats_entry lme = tilestats_entry(tilestat.size()); lme.minzoom = z; lme.maxzoom = z; - layermap.insert(std::pair(layername, lme)); + tilestat.insert(std::pair(layername, lme)); if (lme.id >= (*layer_unmaps)[tiling_seg].size()) { (*layer_unmaps)[tiling_seg].resize(lme.id + 1); @@ -467,27 +467,27 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } } - auto fk = layermap.find(layername); - if (fk == layermap.end()) { + auto ts = tilestat.find(layername); + if (ts == tilestat.end()) { fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str()); exit(EXIT_FAILURE); } - sf.layer = fk->second.id; + sf.layer = ts->second.id; - if (z < fk->second.minzoom) { - fk->second.minzoom = z; + if (z < ts->second.minzoom) { + ts->second.minzoom = z; } - if (z > fk->second.maxzoom) { - fk->second.maxzoom = z; + if (z > ts->second.maxzoom) { + ts->second.maxzoom = z; } if (!postfilter) { if (sf.t == mvt_point) { - fk->second.points++; + ts->second.points++; } else if (sf.t == mvt_linestring) { - fk->second.lines++; + ts->second.lines++; } else if (sf.t == mvt_polygon) { - fk->second.polygons++; + ts->second.polygons++; } } @@ -504,12 +504,12 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: sf.full_keys.push_back(std::string(properties->keys[i]->string)); sf.full_values.push_back(v); - type_and_string attrib; + tilestats_attributes_entry attrib; attrib.string = v.s; attrib.type = v.type; if (!postfilter) { - add_to_file_keys(fk->second.file_keys, std::string(properties->keys[i]->string), attrib); + add_to_tilestats(ts->second.tilestats, std::string(properties->keys[i]->string), attrib); } } } @@ -618,7 +618,7 @@ void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, } } -std::vector filter_layers(const char *filter, std::vector &layers, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, int extent) { +std::vector filter_layers(const char *filter, std::vector &layers, unsigned z, unsigned x, unsigned y, std::vector> *tilestats, size_t tiling_seg, std::vector> *layer_unmaps, int extent) { int write_to, read_from; pid_t pid; setup_filter(filter, &write_to, &read_from, &pid, z, x, y); @@ -637,7 +637,7 @@ std::vector filter_layers(const char *filter, std::vector exit(EXIT_FAILURE); } - std::vector nlayers = parse_layers(read_from, z, x, y, layermaps, tiling_seg, layer_unmaps, extent); + std::vector nlayers = parse_layers(read_from, z, x, y, tilestats, tiling_seg, layer_unmaps, extent); while (1) { int stat_loc; diff --git a/plugin.hpp b/plugin.hpp index 595e7f1..8be09dc 100644 --- a/plugin.hpp +++ b/plugin.hpp @@ -1,3 +1,3 @@ -std::vector filter_layers(const char *filter, std::vector &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, int extent); +std::vector filter_layers(const char *filter, std::vector &layer, unsigned z, unsigned x, unsigned y, std::vector> *tilestats, size_t tiling_seg, std::vector> *layer_unmaps, int extent); void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y); -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters); +serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *tilestats, size_t tiling_seg, std::vector> *layer_unmaps, bool filters); diff --git a/serial.cpp b/serial.cpp index 797e4b5..2d413a2 100644 --- a/serial.cpp +++ b/serial.cpp @@ -565,12 +565,12 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { sf.index = 0; } - if (sst->layermap->count(sf.layername) == 0) { - sst->layermap->insert(std::pair(sf.layername, layermap_entry(sst->layermap->size()))); + if (sst->tilestat->count(sf.layername) == 0) { + sst->tilestat->insert(std::pair(sf.layername, tilestats_entry(sst->tilestat->size()))); } - auto ai = sst->layermap->find(sf.layername); - if (ai != sst->layermap->end()) { + auto ai = sst->tilestat->find(sf.layername); + if (ai != sst->tilestat->end()) { sf.layer = ai->second.id; if (!sst->filters) { @@ -642,12 +642,12 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { if (!sst->filters) { for (size_t i = 0; i < sf.full_keys.size(); i++) { - type_and_string attrib; + tilestats_attributes_entry attrib; attrib.type = sf.full_values[i].type; attrib.string = sf.full_values[i].s; - auto fk = sst->layermap->find(sf.layername); - add_to_file_keys(fk->second.file_keys, sf.full_keys[i], attrib); + auto ts = sst->tilestat->find(sf.layername); + add_to_tilestats(ts->second.tilestats, sf.full_keys[i], attrib); } } diff --git a/serial.hpp b/serial.hpp index a662d49..01df77c 100644 --- a/serial.hpp +++ b/serial.hpp @@ -157,7 +157,7 @@ struct serialization_state { bool filters = false; bool uses_gamma = false; - std::map *layermap = NULL; + std::map *tilestat = NULL; std::map const *attribute_types = NULL; std::set *exclude = NULL; diff --git a/tile-join.cpp b/tile-join.cpp index 25e63ef..5a22661 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -70,7 +70,7 @@ void aprintf(std::string *buf, const char *format, ...) { free(tmp); } -void handle(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, std::set &exclude, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter) { +void handle(std::string message, int z, unsigned x, unsigned y, std::map &tilestat, std::vector &header, std::map> &mapping, std::set &exclude, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter) { mvt_tile tile; int features_added = 0; bool was_compressed; @@ -125,7 +125,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::map> attributes; + std::map> attributes; std::vector key_order; for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { @@ -220,11 +220,11 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::map>(key, std::pair(val, tas))); + attributes.insert(std::pair>(key, std::pair(val, tas))); key_order.push_back(key); } @@ -267,14 +267,14 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::map>(joinkey, std::pair(outval, tas))); + attributes.insert(std::pair>(joinkey, std::pair(outval, tas))); key_order.push_back(joinkey); } } @@ -283,11 +283,11 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::map(layer.name, layermap_entry(layermap.size()))); - file_keys = layermap.find(layer.name); - file_keys->second.minzoom = z; - file_keys->second.maxzoom = z; + if (tilestats == tilestat.end()) { + tilestat.insert(std::pair(layer.name, tilestats_entry(tilestat.size()))); + tilestats = tilestat.find(layer.name); + tilestats->second.minzoom = z; + tilestats->second.maxzoom = z; } // To keep attributes in their original order instead of alphabetical @@ -296,7 +296,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::mapsecond.first); - add_to_file_keys(file_keys->second.file_keys, k, fa->second.second); + add_to_tilestats(tilestats->second.tilestats, k, fa->second.second); attributes.erase(fa); } } @@ -314,19 +314,19 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::mapsecond.minzoom) { - file_keys->second.minzoom = z; + if (z < tilestats->second.minzoom) { + tilestats->second.minzoom = z; } - if (z > file_keys->second.maxzoom) { - file_keys->second.maxzoom = z; + if (z > tilestats->second.maxzoom) { + tilestats->second.maxzoom = z; } if (feat.type == mvt_point) { - file_keys->second.points++; + tilestats->second.points++; } else if (feat.type == mvt_linestring) { - file_keys->second.lines++; + tilestats->second.lines++; } else if (feat.type == mvt_polygon) { - file_keys->second.polygons++; + tilestats->second.polygons++; } } } @@ -472,7 +472,7 @@ struct arg { std::map> inputs{}; std::map outputs{}; - std::map *layermap = NULL; + std::map *tilestat = NULL; std::vector *header = NULL; std::map> *mapping = NULL; @@ -490,7 +490,7 @@ void *join_worker(void *v) { mvt_tile tile; for (size_t i = 0; i < ai->second.size(); i++) { - handle(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), *(a->exclude), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter); + handle(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->tilestat), *(a->header), *(a->mapping), *(a->exclude), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter); } ai->second.clear(); @@ -525,14 +525,14 @@ void *join_worker(void *v) { return NULL; } -void handle_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object *filter) { +void handle_tasks(std::map> &tasks, std::vector> &tilestats, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object *filter) { pthread_t pthreads[CPUS]; std::vector args; for (size_t i = 0; i < CPUS; i++) { args.push_back(arg()); - args[i].layermap = &layermaps[i]; + args[i].tilestat = &tilestats[i]; args[i].header = &header; args[i].mapping = &mapping; args[i].exclude = &exclude; @@ -581,7 +581,7 @@ void handle_tasks(std::map> &tasks, std::vector &layermap, std::map &attribute_descriptions) { +void handle_vector_layers(json_object *vector_layers, std::map &tilestat, std::map &attribute_descriptions) { if (vector_layers != NULL && vector_layers->type == JSON_ARRAY) { for (size_t i = 0; i < vector_layers->length; i++) { if (vector_layers->array[i]->type == JSON_HASH) { @@ -593,8 +593,8 @@ void handle_vector_layers(json_object *vector_layers, std::mapstring; if (sdesc.size() != 0) { - auto f = layermap.find(sid); - if (f != layermap.end()) { + auto f = tilestat.find(sid); + if (f != tilestat.end()) { f->second.description = sdesc; } } @@ -620,10 +620,10 @@ void handle_vector_layers(json_object *vector_layers, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object *filter, std::map &attribute_descriptions, std::string &generator_options) { - std::vector> layermaps; +void decode(struct reader *readers, std::map &tilestat, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object *filter, std::map &attribute_descriptions, std::string &generator_options) { + std::vector> tilestats; for (size_t i = 0; i < CPUS; i++) { - layermaps.push_back(std::map()); + tilestats.push_back(std::map()); } std::map> tasks; @@ -665,7 +665,7 @@ void decode(struct reader *readers, std::map &layer if (readers == NULL || readers->zoom != r->zoom || readers->x != r->x || readers->y != r->y) { if (tasks.size() > 100 * CPUS) { - handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers, filter); + handle_tasks(tasks, tilestats, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers, filter); tasks.clear(); } } @@ -714,8 +714,8 @@ void decode(struct reader *readers, std::map &layer st->minlat = min(minlat, st->minlat); st->maxlat = max(maxlat, st->maxlat); - handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers, filter); - layermap = merge_layermaps(layermaps); + handle_tasks(tasks, tilestats, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers, filter); + tilestat = merge_layermaps(tilestats); struct reader *next; for (struct reader *r = readers; r != NULL; r = next) { @@ -815,7 +815,7 @@ void decode(struct reader *readers, std::map &layer if (o != NULL && o->type == JSON_HASH) { json_object *vector_layers = json_hash_get(o, "vector_layers"); - handle_vector_layers(vector_layers, layermap, attribute_descriptions); + handle_vector_layers(vector_layers, tilestat, attribute_descriptions); json_free(o); } @@ -1074,7 +1074,7 @@ int main(int argc, char **argv) { st.minzoom = st.minlat = st.minlon = INT_MAX; st.maxzoom = st.maxlat = st.maxlon = INT_MIN; - std::map layermap; + std::map tilestat; std::string attribution; std::string description; std::string name; @@ -1098,7 +1098,7 @@ int main(int argc, char **argv) { std::map attribute_descriptions; std::string generator_options; - decode(readers, layermap, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options); + decode(readers, tilestat, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options); if (set_attribution.size() != 0) { attribution = set_attribution; @@ -1115,7 +1115,7 @@ int main(int argc, char **argv) { } generator_options.append(commandline); - for (auto &l : layermap) { + for (auto &l : tilestat) { if (l.second.minzoom < st.minzoom) { st.minzoom = l.second.minzoom; } @@ -1124,7 +1124,7 @@ int main(int argc, char **argv) { } } - mbtiles_write_metadata(outdb, out_dir, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options); + mbtiles_write_metadata(outdb, out_dir, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, tilestat, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options); if (outdb != NULL) { mbtiles_close(outdb, argv[0]); diff --git a/tile.cpp b/tile.cpp index 2cf9552..1f8ce64 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1198,7 +1198,7 @@ struct write_tile_args { unsigned *initial_y = NULL; std::atomic *running = NULL; int err = 0; - std::vector> *layermaps = NULL; + std::vector> *tilestats = NULL; std::vector> *layer_unmaps = NULL; size_t pass = 0; size_t passes = 0; @@ -1477,7 +1477,7 @@ struct run_prefilter_args { void *run_prefilter(void *v) { run_prefilter_args *rpa = (run_prefilter_args *) v; - json_writer state(rpa->prefilter_fp); + json_writer jw(rpa->prefilter_fp); while (1) { serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps); @@ -1514,7 +1514,7 @@ void *run_prefilter(void *v) { decode_meta(sf.keys, sf.values, rpa->stringpool + rpa->pool_off[sf.segment], tmp_layer, tmp_feature); tmp_layer.features.push_back(tmp_feature); - layer_to_geojson(tmp_layer, 0, 0, 0, false, true, false, true, sf.index, sf.seq, sf.extent, true, state); + layer_to_geojson(tmp_layer, 0, 0, 0, false, true, false, true, sf.index, sf.seq, sf.extent, true, jw); } if (fclose(rpa->prefilter_fp) != 0) { @@ -1532,32 +1532,32 @@ void *run_prefilter(void *v) { return NULL; } -void add_tilestats(std::string const &layername, int z, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, std::string const &key, serial_val const &val) { - std::map &layermap = (*layermaps)[tiling_seg]; - if (layermap.count(layername) == 0) { - layermap_entry lme = layermap_entry(layermap.size()); +void add_tilestats(std::string const &layername, int z, std::vector> *tilestats, size_t tiling_seg, std::vector> *layer_unmaps, std::string const &key, serial_val const &val) { + std::map &tilestat = (*tilestats)[tiling_seg]; + if (tilestat.count(layername) == 0) { + tilestats_entry lme = tilestats_entry(tilestat.size()); lme.minzoom = z; lme.maxzoom = z; lme.retain = 1; - layermap.insert(std::pair(layername, lme)); + tilestat.insert(std::pair(layername, lme)); if (lme.id >= (*layer_unmaps)[tiling_seg].size()) { (*layer_unmaps)[tiling_seg].resize(lme.id + 1); (*layer_unmaps)[tiling_seg][lme.id] = layername; } } - auto fk = layermap.find(layername); - if (fk == layermap.end()) { + auto ts = tilestat.find(layername); + if (ts == tilestat.end()) { fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str()); exit(EXIT_FAILURE); } - type_and_string attrib; + tilestats_attributes_entry attrib; attrib.type = val.type; attrib.string = val.s; - add_to_file_keys(fk->second.file_keys, key, attrib); + add_to_tilestats(ts->second.tilestats, key, attrib); } struct accum_state { @@ -1623,8 +1623,8 @@ void preserve_attribute(attribute_op op, std::map &att } case op_mean: { - auto state = attribute_accum_state.find(key); - if (state == attribute_accum_state.end()) { + auto jw = attribute_accum_state.find(key); + if (jw == attribute_accum_state.end()) { accum_state s; s.sum = atof(p.full_values[i].s.c_str()) + atof(val.s.c_str()); s.count = 2; @@ -1632,10 +1632,10 @@ void preserve_attribute(attribute_op op, std::map &att p.full_values[i].s = milo::dtoa_milo(s.sum / s.count); } else { - state->second.sum += atof(val.s.c_str()); - state->second.count += 1; + jw->second.sum += atof(val.s.c_str()); + jw->second.count += 1; - p.full_values[i].s = milo::dtoa_milo(state->second.sum / state->second.count); + p.full_values[i].s = milo::dtoa_milo(jw->second.sum / jw->second.count); } break; } @@ -1714,7 +1714,7 @@ static bool line_is_too_small(drawvec const &geometry, int z, int detail) { return true; } -long long write_tile(FILE *geoms, std::atomic *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg) { +long long write_tile(FILE *geoms, std::atomic *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *tilestats, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg) { int line_detail; double merge_fraction = 1; double mingap_fraction = 1; @@ -1860,7 +1860,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta if (prefilter == NULL) { sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps); } else { - sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); + sf = parse_feature(prefilter_jp, z, tx, ty, tilestats, tiling_seg, layer_unmaps, postfilter != NULL); } if (sf.t < 0) { @@ -2034,21 +2034,21 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta sv.s = "true"; p.full_values.push_back(sv); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "clustered", sv); + add_tilestats(layername, z, tilestats, tiling_seg, layer_unmaps, "clustered", sv); p.full_keys.push_back("point_count"); sv2.type = mvt_double; sv2.s = std::to_string(p.clustered + 1); p.full_values.push_back(sv2); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "point_count", sv2); + add_tilestats(layername, z, tilestats, tiling_seg, layer_unmaps, "point_count", sv2); p.full_keys.push_back("sqrt_point_count"); sv3.type = mvt_double; sv3.s = std::to_string(round(100 * sqrt(p.clustered + 1)) / 100.0); p.full_values.push_back(sv3); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "sqrt_point_count", sv3); + add_tilestats(layername, z, tilestats, tiling_seg, layer_unmaps, "sqrt_point_count", sv3); } if (p.need_tilestats.size() > 0) { @@ -2056,7 +2056,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta for (size_t j = 0; j < p.full_keys.size(); j++) { if (p.need_tilestats.count(p.full_keys[j]) > 0) { - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, p.full_keys[j], p.full_values[j]); + add_tilestats(layername, z, tilestats, tiling_seg, layer_unmaps, p.full_keys[j], p.full_values[j]); } } } @@ -2294,7 +2294,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta sv.type = mvt_double; sv.s = std::to_string(glow); - add_tilestats(layer.name, z, layermaps, tiling_seg, layer_unmaps, "tippecanoe_feature_density", sv); + add_tilestats(layer.name, z, tilestats, tiling_seg, layer_unmaps, "tippecanoe_feature_density", sv); } layer.features.push_back(feature); @@ -2306,7 +2306,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta } if (postfilter != NULL) { - tile.layers = filter_layers(postfilter, tile.layers, z, tx, ty, layermaps, tiling_seg, layer_unmaps, 1 << line_detail); + tile.layers = filter_layers(postfilter, tile.layers, z, tx, ty, tilestats, tiling_seg, layer_unmaps, 1 << line_detail); } if (z == 0 && unclipped_features < original_features / 2 && clipbboxes.size() == 0) { @@ -2582,7 +2582,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg); + long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->tilestats, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg); if (len < 0) { int *err = &arg->err; @@ -2647,23 +2647,23 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &tilestats, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter) { last_progress = 0; - // The existing layermaps are one table per input thread. + // The existing tilestats are one table per input thread. // We need to add another one per *tiling* thread so that it can be // safely changed during tiling. - size_t layermaps_off = layermaps.size(); + size_t layermaps_off = tilestats.size(); for (size_t i = 0; i < CPUS; i++) { - layermaps.push_back(std::map()); + tilestats.push_back(std::map()); } // Table to map segment and layer number back to layer name std::vector> layer_unmaps; - for (size_t seg = 0; seg < layermaps.size(); seg++) { + for (size_t seg = 0; seg < tilestats.size(); seg++) { layer_unmaps.push_back(std::vector()); - for (auto a = layermaps[seg].begin(); a != layermaps[seg].end(); ++a) { + for (auto a = tilestats[seg].begin(); a != tilestats[seg].end(); ++a) { if (a->second.id >= layer_unmaps[seg].size()) { layer_unmaps[seg].resize(a->second.id + 1); } @@ -2830,7 +2830,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].pool_off = pool_off; args[thread].initial_x = initial_x; args[thread].initial_y = initial_y; - args[thread].layermaps = &layermaps; + args[thread].tilestats = &tilestats; args[thread].layer_unmaps = &layer_unmaps; args[thread].tiling_seg = thread + layermaps_off; args[thread].prefilter = prefilter; diff --git a/tile.hpp b/tile.hpp index 32fb227..6104fab 100644 --- a/tile.hpp +++ b/tile.hpp @@ -21,7 +21,7 @@ enum attribute_op { long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &tilestat, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/write_json.cpp b/write_json.cpp index cee0e47..f7aa756 100644 --- a/write_json.cpp +++ b/write_json.cpp @@ -16,21 +16,21 @@ #include "milo/dtoa_milo.h" void json_writer::json_adjust() { - if (state.size() == 0) { - state.push_back(JSON_WRITE_TOP); - } else if (state[state.size() - 1] == JSON_WRITE_TOP) { + if (jw.size() == 0) { + jw.push_back(JSON_WRITE_TOP); + } else if (jw[jw.size() - 1] == JSON_WRITE_TOP) { addc('\n'); - state[state.size() - 1] = JSON_WRITE_TOP; - } else if (state[state.size() - 1] == JSON_WRITE_HASH) { + jw[jw.size() - 1] = JSON_WRITE_TOP; + } else if (jw[jw.size() - 1] == JSON_WRITE_HASH) { if (!nospace) { addc(' '); } nospace = false; - state[state.size() - 1] = JSON_WRITE_HASH_KEY; - } else if (state[state.size() - 1] == JSON_WRITE_HASH_KEY) { + jw[jw.size() - 1] = JSON_WRITE_HASH_KEY; + } else if (jw[jw.size() - 1] == JSON_WRITE_HASH_KEY) { adds(": "); - state[state.size() - 1] = JSON_WRITE_HASH_VALUE; - } else if (state[state.size() - 1] == JSON_WRITE_HASH_VALUE) { + jw[jw.size() - 1] = JSON_WRITE_HASH_VALUE; + } else if (jw[jw.size() - 1] == JSON_WRITE_HASH_VALUE) { if (wantnl) { adds(",\n"); nospace = false; @@ -41,14 +41,14 @@ void json_writer::json_adjust() { adds(", "); } wantnl = false; - state[state.size() - 1] = JSON_WRITE_HASH_KEY; - } else if (state[state.size() - 1] == JSON_WRITE_ARRAY) { + jw[jw.size() - 1] = JSON_WRITE_HASH_KEY; + } else if (jw[jw.size() - 1] == JSON_WRITE_ARRAY) { if (!nospace) { addc(' '); } nospace = false; - state[state.size() - 1] = JSON_WRITE_ARRAY_ELEMENT; - } else if (state[state.size() - 1] == JSON_WRITE_ARRAY_ELEMENT) { + jw[jw.size() - 1] = JSON_WRITE_ARRAY_ELEMENT; + } else if (jw[jw.size() - 1] == JSON_WRITE_ARRAY_ELEMENT) { if (wantnl) { adds(",\n"); nospace = false; @@ -59,28 +59,28 @@ void json_writer::json_adjust() { adds(", "); } wantnl = false; - state[state.size() - 1] = JSON_WRITE_ARRAY_ELEMENT; + jw[jw.size() - 1] = JSON_WRITE_ARRAY_ELEMENT; } else { - fprintf(stderr, "Impossible JSON state\n"); + fprintf(stderr, "Impossible JSON jw\n"); exit(EXIT_FAILURE); } } -void json_writer::json_write_array() { +void json_writer::begin_array() { json_adjust(); addc('['); - state.push_back(JSON_WRITE_ARRAY); + jw.push_back(JSON_WRITE_ARRAY); } -void json_writer::json_end_array() { - if (state.size() == 0) { +void json_writer::end_array() { + if (jw.size() == 0) { fprintf(stderr, "End JSON array at top level\n"); exit(EXIT_FAILURE); } - json_write_tok tok = state[state.size() - 1]; - state.pop_back(); + write_tok tok = jw[jw.size() - 1]; + jw.pop_back(); if (tok == JSON_WRITE_ARRAY || tok == JSON_WRITE_ARRAY_ELEMENT) { if (!nospace) { @@ -89,26 +89,26 @@ void json_writer::json_end_array() { nospace = false; addc(']'); } else { - fprintf(stderr, "End JSON array with unexpected state\n"); + fprintf(stderr, "End JSON array with unexpected jw\n"); exit(EXIT_FAILURE); } } -void json_writer::json_write_hash() { +void json_writer::begin_hash() { json_adjust(); addc('{'); - state.push_back(JSON_WRITE_HASH); + jw.push_back(JSON_WRITE_HASH); } -void json_writer::json_end_hash() { - if (state.size() == 0) { +void json_writer::end_hash() { + if (jw.size() == 0) { fprintf(stderr, "End JSON hash at top level\n"); exit(EXIT_FAILURE); } - json_write_tok tok = state[state.size() - 1]; - state.pop_back(); + write_tok tok = jw[jw.size() - 1]; + jw.pop_back(); if (tok == JSON_WRITE_HASH) { if (!nospace) { @@ -123,12 +123,12 @@ void json_writer::json_end_hash() { nospace = false; addc('}'); } else { - fprintf(stderr, "End JSON hash with unexpected state\n"); + fprintf(stderr, "End JSON hash with unexpected jw\n"); exit(EXIT_FAILURE); } } -void json_writer::json_write_string(std::string const &str) { +void json_writer::write_string(std::string const &str) { json_adjust(); addc('"'); @@ -144,38 +144,38 @@ void json_writer::json_write_string(std::string const &str) { addc('"'); } -void json_writer::json_write_number(double d) { +void json_writer::write_number(double d) { json_adjust(); adds(milo::dtoa_milo(d).c_str()); } // Just to avoid json_writer:: changing expected output format -void json_writer::json_write_float(double d) { +void json_writer::write_float(double d) { json_adjust(); aprintf("%f", d); } -void json_writer::json_write_unsigned(unsigned long long v) { +void json_writer::write_unsigned(unsigned long long v) { json_adjust(); aprintf("%llu", v); } -void json_writer::json_write_signed(long long v) { +void json_writer::write_signed(long long v) { json_adjust(); aprintf("%lld", v); } -void json_writer::json_write_stringified(std::string const &str) { +void json_writer::write_stringified(std::string const &str) { json_adjust(); adds(str); } -void json_writer::json_write_bool(bool b) { +void json_writer::write_bool(bool b) { json_adjust(); if (b) { @@ -185,13 +185,13 @@ void json_writer::json_write_bool(bool b) { } } -void json_writer::json_write_null() { +void json_writer::write_null() { json_adjust(); adds("null"); } -void json_writer::json_write_newline() { +void json_writer::write_newline() { addc('\n'); nospace = true; } @@ -247,61 +247,61 @@ struct lonlat { } }; -void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool dropped, unsigned long long index, long long sequence, long long extent, bool complain, json_writer &state) { +void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool dropped, unsigned long long index, long long sequence, long long extent, bool complain, json_writer &jw) { for (size_t f = 0; f < layer.features.size(); f++) { mvt_feature const &feat = layer.features[f]; - state.json_write_hash(); - state.json_write_string("type"); - state.json_write_string("Feature"); + jw.begin_hash(); + jw.write_string("type"); + jw.write_string("Feature"); if (feat.has_id) { - state.json_write_string("id"); - state.json_write_unsigned(feat.id); + jw.write_string("id"); + jw.write_unsigned(feat.id); } if (name || zoom || index != 0 || sequence != 0 || extent != 0) { - state.json_write_string("tippecanoe"); - state.json_write_hash(); + jw.write_string("tippecanoe"); + jw.begin_hash(); if (name) { - state.json_write_string("layer"); - state.json_write_string(layer.name); + jw.write_string("layer"); + jw.write_string(layer.name); } if (zoom) { - state.json_write_string("minzoom"); - state.json_write_unsigned(z); + jw.write_string("minzoom"); + jw.write_unsigned(z); - state.json_write_string("maxzoom"); - state.json_write_unsigned(z); + jw.write_string("maxzoom"); + jw.write_unsigned(z); } if (dropped) { - state.json_write_string("dropped"); - state.json_write_bool(feat.dropped); + jw.write_string("dropped"); + jw.write_bool(feat.dropped); } if (index != 0) { - state.json_write_string("index"); - state.json_write_unsigned(index); + jw.write_string("index"); + jw.write_unsigned(index); } if (sequence != 0) { - state.json_write_string("sequence"); - state.json_write_signed(sequence); + jw.write_string("sequence"); + jw.write_signed(sequence); } if (extent != 0) { - state.json_write_string("extent"); - state.json_write_signed(extent); + jw.write_string("extent"); + jw.write_signed(extent); } - state.json_end_hash(); + jw.end_hash(); } - state.json_write_string("properties"); - state.json_write_hash(); + jw.write_string("properties"); + jw.begin_hash(); for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { if (feat.tags[t] >= layer.keys.size()) { @@ -317,39 +317,39 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y mvt_value const &val = layer.values[feat.tags[t + 1]]; if (val.type == mvt_string) { - state.json_write_string(key); - state.json_write_string(val.string_value); + jw.write_string(key); + jw.write_string(val.string_value); } else if (val.type == mvt_int) { - state.json_write_string(key); - state.json_write_signed(val.numeric_value.int_value); + jw.write_string(key); + jw.write_signed(val.numeric_value.int_value); } else if (val.type == mvt_double) { - state.json_write_string(key); - state.json_write_number(val.numeric_value.double_value); + jw.write_string(key); + jw.write_number(val.numeric_value.double_value); } else if (val.type == mvt_float) { - state.json_write_string(key); - state.json_write_number(val.numeric_value.float_value); + jw.write_string(key); + jw.write_number(val.numeric_value.float_value); } else if (val.type == mvt_sint) { - state.json_write_string(key); - state.json_write_signed(val.numeric_value.sint_value); + jw.write_string(key); + jw.write_signed(val.numeric_value.sint_value); } else if (val.type == mvt_uint) { - state.json_write_string(key); - state.json_write_unsigned(val.numeric_value.uint_value); + jw.write_string(key); + jw.write_unsigned(val.numeric_value.uint_value); } else if (val.type == mvt_bool) { - state.json_write_string(key); - state.json_write_bool(val.numeric_value.bool_value); + jw.write_string(key); + jw.write_bool(val.numeric_value.bool_value); } else if (val.type == mvt_null) { - state.json_write_string(key); - state.json_write_null(); + jw.write_string(key); + jw.write_null(); } else { fprintf(stderr, "Internal error: property with unknown type\n"); exit(EXIT_FAILURE); } } - state.json_end_hash(); + jw.end_hash(); - state.json_write_string("geometry"); - state.json_write_hash(); + jw.write_string("geometry"); + jw.begin_hash(); std::vector ops; @@ -374,30 +374,30 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y if (feat.type == VT_POINT) { if (ops.size() == 1) { - state.json_write_string("type"); - state.json_write_string("Point"); + jw.write_string("type"); + jw.write_string("Point"); - state.json_write_string("coordinates"); + jw.write_string("coordinates"); - state.json_write_array(); - state.json_write_float(ops[0].lon); - state.json_write_float(ops[0].lat); - state.json_end_array(); + jw.begin_array(); + jw.write_float(ops[0].lon); + jw.write_float(ops[0].lat); + jw.end_array(); } else { - state.json_write_string("type"); - state.json_write_string("MultiPoint"); + jw.write_string("type"); + jw.write_string("MultiPoint"); - state.json_write_string("coordinates"); - state.json_write_array(); + jw.write_string("coordinates"); + jw.begin_array(); for (size_t i = 0; i < ops.size(); i++) { - state.json_write_array(); - state.json_write_float(ops[i].lon); - state.json_write_float(ops[i].lat); - state.json_end_array(); + jw.begin_array(); + jw.write_float(ops[i].lon); + jw.write_float(ops[i].lat); + jw.end_array(); } - state.json_end_array(); + jw.end_array(); } } else if (feat.type == VT_LINE) { int movetos = 0; @@ -408,59 +408,59 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y } if (movetos < 2) { - state.json_write_string("type"); - state.json_write_string("LineString"); + jw.write_string("type"); + jw.write_string("LineString"); - state.json_write_string("coordinates"); - state.json_write_array(); + jw.write_string("coordinates"); + jw.begin_array(); for (size_t i = 0; i < ops.size(); i++) { - state.json_write_array(); - state.json_write_float(ops[i].lon); - state.json_write_float(ops[i].lat); - state.json_end_array(); + jw.begin_array(); + jw.write_float(ops[i].lon); + jw.write_float(ops[i].lat); + jw.end_array(); } - state.json_end_array(); + jw.end_array(); } else { - state.json_write_string("type"); - state.json_write_string("MultiLineString"); + jw.write_string("type"); + jw.write_string("MultiLineString"); - state.json_write_string("coordinates"); - state.json_write_array(); - state.json_write_array(); + jw.write_string("coordinates"); + jw.begin_array(); + jw.begin_array(); int sstate = 0; for (size_t i = 0; i < ops.size(); i++) { if (ops[i].op == VT_MOVETO) { if (sstate == 0) { - state.json_write_array(); - state.json_write_float(ops[i].lon); - state.json_write_float(ops[i].lat); - state.json_end_array(); + jw.begin_array(); + jw.write_float(ops[i].lon); + jw.write_float(ops[i].lat); + jw.end_array(); sstate = 1; } else { - state.json_end_array(); - state.json_write_array(); + jw.end_array(); + jw.begin_array(); - state.json_write_array(); - state.json_write_float(ops[i].lon); - state.json_write_float(ops[i].lat); - state.json_end_array(); + jw.begin_array(); + jw.write_float(ops[i].lon); + jw.write_float(ops[i].lat); + jw.end_array(); sstate = 1; } } else { - state.json_write_array(); - state.json_write_float(ops[i].lon); - state.json_write_float(ops[i].lat); - state.json_end_array(); + jw.begin_array(); + jw.write_float(ops[i].lon); + jw.write_float(ops[i].lat); + jw.end_array(); } } - state.json_end_array(); - state.json_end_array(); + jw.end_array(); + jw.end_array(); } } else if (feat.type == VT_POLYGON) { std::vector > rings; @@ -518,20 +518,20 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y } if (outer > 1) { - state.json_write_string("type"); - state.json_write_string("MultiPolygon"); + jw.write_string("type"); + jw.write_string("MultiPolygon"); - state.json_write_string("coordinates"); - state.json_write_array(); - state.json_write_array(); - state.json_write_array(); + jw.write_string("coordinates"); + jw.begin_array(); + jw.begin_array(); + jw.begin_array(); } else { - state.json_write_string("type"); - state.json_write_string("Polygon"); + jw.write_string("type"); + jw.write_string("Polygon"); - state.json_write_string("coordinates"); - state.json_write_array(); - state.json_write_array(); + jw.write_string("coordinates"); + jw.begin_array(); + jw.begin_array(); } int sstate = 0; @@ -552,32 +552,32 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y if (areas[i] >= 0) { if (sstate != 0) { // new multipolygon - state.json_end_array(); - state.json_end_array(); + jw.end_array(); + jw.end_array(); - state.json_write_array(); - state.json_write_array(); + jw.begin_array(); + jw.begin_array(); } sstate = 1; } if (sstate == 2) { // new ring in the same polygon - state.json_end_array(); - state.json_write_array(); + jw.end_array(); + jw.begin_array(); } for (size_t j = 0; j < rings[i].size(); j++) { if (rings[i][j].op != VT_CLOSEPATH) { - state.json_write_array(); - state.json_write_float(rings[i][j].lon); - state.json_write_float(rings[i][j].lat); - state.json_end_array(); + jw.begin_array(); + jw.write_float(rings[i][j].lon); + jw.write_float(rings[i][j].lat); + jw.end_array(); } else { - state.json_write_array(); - state.json_write_float(rings[i][0].lon); - state.json_write_float(rings[i][0].lat); - state.json_end_array(); + jw.begin_array(); + jw.write_float(rings[i][0].lon); + jw.write_float(rings[i][0].lat); + jw.end_array(); } } @@ -585,21 +585,21 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y } if (outer > 1) { - state.json_end_array(); - state.json_end_array(); - state.json_end_array(); + jw.end_array(); + jw.end_array(); + jw.end_array(); } else { - state.json_end_array(); - state.json_end_array(); + jw.end_array(); + jw.end_array(); } } - state.json_end_hash(); - state.json_end_hash(); + jw.end_hash(); + jw.end_hash(); if (comma) { - state.json_write_newline(); - state.json_comma_newline(); + jw.write_newline(); + jw.json_comma_newline(); } } } diff --git a/write_json.hpp b/write_json.hpp index d0b9d18..180382d 100644 --- a/write_json.hpp +++ b/write_json.hpp @@ -5,7 +5,7 @@ #include #include -enum json_write_tok { +enum write_tok { JSON_WRITE_HASH, JSON_WRITE_HASH_KEY, JSON_WRITE_HASH_VALUE, @@ -15,15 +15,15 @@ enum json_write_tok { }; struct json_writer { - std::vector state; + std::vector jw; bool nospace = false; bool wantnl = false; FILE *f = NULL; std::string *s = NULL; ~json_writer() { - if (state.size() > 0) { - if (state.size() != 1 || state[0] != JSON_WRITE_TOP) { + if (jw.size() > 0) { + if (jw.size() != 1 || jw[0] != JSON_WRITE_TOP) { fprintf(stderr, "JSON not closed at end\n"); exit(EXIT_FAILURE); } @@ -38,19 +38,19 @@ struct json_writer { s = out; } - void json_write_array(); - void json_end_array(); - void json_write_hash(); - void json_end_hash(); - void json_write_string(std::string const &s); - void json_write_number(double d); - void json_write_float(double d); - void json_write_unsigned(unsigned long long v); - void json_write_signed(long long v); - void json_write_stringified(std::string const &s); - void json_write_bool(bool b); - void json_write_null(); - void json_write_newline(); + void begin_array(); + void end_array(); + void begin_hash(); + void end_hash(); + void write_string(std::string const &s); + void write_number(double d); + void write_float(double d); + void write_unsigned(unsigned long long v); + void write_signed(long long v); + void write_stringified(std::string const &s); + void write_bool(bool b); + void write_null(); + void write_newline(); void json_comma_newline(); private: @@ -60,7 +60,7 @@ struct json_writer { void adds(std::string const &s); }; -void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool dropped, unsigned long long index, long long sequence, long long extent, bool complain, json_writer &state); +void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool dropped, unsigned long long index, long long sequence, long long extent, bool complain, json_writer &jw); void fprintq(FILE *f, const char *s); #endif