From d1dc310bbcb8c3aa210c722bb1969cd09556839a Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 8 Dec 2016 15:13:38 -0800 Subject: [PATCH] The GeoJSON-producing part of prefiltering --- decode.cpp | 4 ++-- main.cpp | 20 +++++++++++++------- mvt.hpp | 2 +- plugin.cpp | 2 +- tile.cpp | 47 +++++++++++++++++++++++++++++++++++++++-------- tile.hpp | 2 +- write_json.cpp | 8 +++++++- write_json.hpp | 2 +- 8 files changed, 65 insertions(+), 22 deletions(-) diff --git a/decode.cpp b/decode.cpp index 8a9f22d..49a66dc 100644 --- a/decode.cpp +++ b/decode.cpp @@ -56,14 +56,14 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) { printf("{ \"type\": \"FeatureCollection\""); printf(", \"properties\": { \"layer\": "); fprintq(stdout, layer.name.c_str()); - printf(", \"version\": %d, \"extent\": %d", layer.version, layer.extent); + printf(", \"version\": %d, \"extent\": %lld", layer.version, layer.extent); printf(" }"); printf(", \"features\": [\n"); within = 0; } - layer_to_geojson(stdout, layer, z, x, y, true); + layer_to_geojson(stdout, layer, z, x, y, true, false); if (describe) { printf("] }\n"); diff --git a/main.cpp b/main.cpp index 16365d5..a431b7e 100644 --- a/main.cpp +++ b/main.cpp @@ -947,7 +947,7 @@ void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE } } -int read_input(std::vector &sources, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set *exclude, std::set *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, const char *filter) { +int read_input(std::vector &sources, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set *exclude, std::set *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, const char *prefilter, const char *postfilter) { int ret = EXIT_SUCCESS; struct reader reader[CPUS]; @@ -1729,7 +1729,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, } unsigned midx = 0, midy = 0; - int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, filter); + int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); @@ -1868,7 +1868,8 @@ int main(int argc, char **argv) { const char *tmpdir = "/tmp"; const char *attribution = NULL; std::vector sources; - const char *filter = NULL; + const char *prefilter = NULL; + const char *postfilter = NULL; std::set exclude, include; int exclude_all = 0; @@ -1904,7 +1905,8 @@ int main(int argc, char **argv) { {"projection", required_argument, 0, 's'}, {"simplification", required_argument, 0, 'S'}, {"maximum-tile-bytes", required_argument, 0, 'M'}, - {"filter", required_argument, 0, 'c'}, + {"prefilter", required_argument, 0, 'C'}, + {"postfilter", required_argument, 0, 'c'}, {"exclude-all", no_argument, 0, 'X'}, {"force", no_argument, 0, 'f'}, @@ -1960,7 +1962,7 @@ int main(int argc, char **argv) { } } - while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:c:", long_options, NULL)) != -1) { + while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:c:C:", long_options, NULL)) != -1) { switch (i) { case 0: break; @@ -2144,7 +2146,11 @@ int main(int argc, char **argv) { break; case 'c': - filter = optarg; + postfilter = optarg; + break; + + case 'C': + prefilter = optarg; break; default: { @@ -2247,7 +2253,7 @@ int main(int argc, char **argv) { sources.push_back(src); } - ret = read_input(sources, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, filter); + ret = read_input(sources, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, prefilter, postfilter); mbtiles_close(outdb, argv); diff --git a/mvt.hpp b/mvt.hpp index d1b641a..55c0ff0 100644 --- a/mvt.hpp +++ b/mvt.hpp @@ -65,7 +65,7 @@ struct mvt_layer { std::vector features; std::vector keys; std::vector values; - int extent; + long long extent; // Add a key-value pair to a feature, using this layer's constant pool void tag(mvt_feature &feature, std::string key, mvt_value value); diff --git a/plugin.cpp b/plugin.cpp index b70f49a..49bdaa2 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -45,7 +45,7 @@ void *run_writer(void *a) { exit(EXIT_FAILURE); } - layer_to_geojson(fp, *(wa->layer), wa->z, wa->x, wa->y, false); + layer_to_geojson(fp, *(wa->layer), wa->z, wa->x, wa->y, false, false); if (fclose(fp) != 0) { perror("fclose output to filter"); diff --git a/tile.cpp b/tile.cpp index 62bfe70..5844505 100644 --- a/tile.cpp +++ b/tile.cpp @@ -35,6 +35,7 @@ #include "options.hpp" #include "main.hpp" #include "plugin.hpp" +#include "write_json.hpp" #define CMD_BITS 3 @@ -142,7 +143,7 @@ mvt_value retrieve_string(long long off, char *stringpool, int *otype) { return stringified_to_mvt_value(type, s); } -void decode_meta(int m, std::vector &metakeys, std::vector &metavals, char *stringpool, mvt_layer &layer, mvt_feature &feature) { +void decode_meta(int m, std::vector const &metakeys, std::vector const &metavals, char *stringpool, mvt_layer &layer, mvt_feature &feature) { int i; for (i = 0; i < m; i++) { int otype; @@ -1167,10 +1168,11 @@ struct write_tile_args { long long minextent_out; double fraction; double fraction_out; - const char *filter; + const char *prefilter; + const char *postfilter; }; -long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *filter, write_tile_args *arg) { +long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) { int line_detail; double merge_fraction = 1; double mingap_fraction = 1; @@ -1422,6 +1424,34 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s continue; } + if (prefilter != NULL) { + mvt_layer tmp_layer; + tmp_layer.extent = 1LL << 32; + tmp_layer.name = (*layer_unmaps)[segment][layer]; + + mvt_feature tmp_feature; + tmp_feature.type = t; + tmp_feature.geometry = to_feature(geom); + tmp_feature.id = id; + tmp_feature.has_id = id; + + // Offset from tile coordinates back to world coordinates + unsigned sx = 0, sy = 0; + if (z != 0) { + sx = tx << (32 - z); + sy = ty << (32 - z); + } + for (size_t i = 0; i < tmp_feature.geometry.size(); i++) { + tmp_feature.geometry[i].x += sx; + tmp_feature.geometry[i].y += sy; + } + + decode_meta(m, metakeys, metavals, stringpool + pool_off[segment], tmp_layer, tmp_feature); + tmp_layer.features.push_back(tmp_feature); + + layer_to_geojson(stdout, tmp_layer, 0, 0, 0, false, true); + } + if (gamma > 0) { if (manage_gap(index, &previndex, scale, gamma, &gap)) { continue; @@ -1691,8 +1721,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s layer.features.push_back(feature); } - if (filter != NULL) { - layer = filter_layer(filter, layer, z, tx, ty); + if (postfilter != NULL) { + layer = filter_layer(postfilter, layer, z, tx, ty); } if (layer.features.size() > 0) { @@ -1921,7 +1951,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->filter, arg); + long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg); if (len < 0) { int *err = &arg->err; @@ -1986,7 +2016,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps, const char *filter) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter) { // Table to map segment and layer number back to layer name std::vector> layer_unmaps; for (size_t seg = 0; seg < layermaps.size(); seg++) { @@ -2153,7 +2183,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].initial_y = initial_y; args[thread].layermaps = &layermaps; args[thread].layer_unmaps = &layer_unmaps; - args[thread].filter = filter; + args[thread].prefilter = prefilter; + args[thread].postfilter = postfilter; args[thread].tasks = dispatches[thread].tasks; args[thread].running = &running; diff --git a/tile.hpp b/tile.hpp index a4835b4..6d3f0ca 100644 --- a/tile.hpp +++ b/tile.hpp @@ -1,3 +1,3 @@ -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap, const char *filter); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap, const char *prefilter, const char *postfilter); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/write_json.cpp b/write_json.cpp index 3097135..f5c3173 100644 --- a/write_json.cpp +++ b/write_json.cpp @@ -24,7 +24,7 @@ struct lonlat { } }; -void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsigned y, bool comma) { +void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name) { for (size_t f = 0; f < layer.features.size(); f++) { mvt_feature &feat = layer.features[f]; @@ -38,6 +38,12 @@ void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsign fprintf(fp, ", \"id\": %llu", feat.id); } + if (name) { + fprintf(fp, ", \"tippecanoe\": { \"layer\": "); + fprintq(fp, layer.name.c_str()); + fprintf(fp, " }"); + } + fprintf(fp, ", \"properties\": { "); for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { diff --git a/write_json.hpp b/write_json.hpp index 3799fce..685a853 100644 --- a/write_json.hpp +++ b/write_json.hpp @@ -1,2 +1,2 @@ -void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsigned y, bool comma); +void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name); void fprintq(FILE *f, const char *s);