From c2fa8e36333626974e022549d0d01460f9ed4919 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 12 Dec 2016 17:00:45 -0800 Subject: [PATCH] Switch the segment and layer ID based on prefilter output --- main.cpp | 15 ++++++++++----- plugin.cpp | 23 +++++++++++++++++------ plugin.hpp | 4 ++-- tile.cpp | 20 ++++++++++---------- 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/main.cpp b/main.cpp index a431b7e..1c34da0 100644 --- a/main.cpp +++ b/main.cpp @@ -1048,9 +1048,10 @@ int read_input(std::vector &sources, char *fname, const char *layername, volatile long long progress_seq = 0; - int initialized[CPUS]; - unsigned initial_x[CPUS], initial_y[CPUS]; - for (size_t i = 0; i < CPUS; i++) { + // 2 * CPUS: One per reader thread, one per tiling thread + int initialized[2 * CPUS]; + unsigned initial_x[2 * CPUS], initial_y[2 * CPUS]; + for (size_t i = 0; i < 2 * CPUS; i++) { initialized[i] = initial_x[i] = initial_y[i] = 0; } @@ -1332,8 +1333,12 @@ int read_input(std::vector &sources, char *fname, const char *layername, // but keep track of the offsets into it since we still need // segment+offset to find the data. - long long pool_off[CPUS]; - long long meta_off[CPUS]; + // 2 * CPUS: One per input thread, one per tiling thread + long long pool_off[2 * CPUS]; + long long meta_off[2 * CPUS]; + for (size_t i = 0; i < 2 * CPUS; i++) { + pool_off[i] = meta_off[i] = 0; + } char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX"); diff --git a/plugin.cpp b/plugin.cpp index 1f7d820..c5bcdd3 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -71,7 +71,7 @@ static std::vector to_feature(drawvec &geom) { return out; } -mvt_layer parse_layer(int fd, unsigned z, unsigned x, unsigned y, mvt_layer const &olayer, std::vector> *layermaps, size_t tiling_seg) { +mvt_layer parse_layer(int fd, int z, unsigned x, unsigned y, mvt_layer const &olayer, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps) { mvt_layer ret; ret.name = olayer.name; ret.version = olayer.version; @@ -209,6 +209,11 @@ mvt_layer parse_layer(int fd, unsigned z, unsigned x, unsigned y, mvt_layer cons lme.maxzoom = z; layermap.insert(std::pair(layername, lme)); + + if (lme.id >= (*layer_unmaps)[tiling_seg].size()) { + (*layer_unmaps)[tiling_seg].resize(lme.id + 1); + (*layer_unmaps)[tiling_seg][lme.id] = layername; + } } type_and_string tas; @@ -240,7 +245,7 @@ mvt_layer parse_layer(int fd, unsigned z, unsigned x, unsigned y, mvt_layer cons return ret; } -serial_feature parse_feature(json_pull *jp, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg) { +serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps) { serial_feature sf; while (1) { @@ -345,9 +350,8 @@ serial_feature parse_feature(json_pull *jp, unsigned z, unsigned x, unsigned y, } sf.t = mb_geometry[t]; + sf.segment = tiling_seg; sf.geometry = dv; - sf.segment = 0; - sf.layer = 0; // XXX sf.seq = 0; // XXX sf.index = 0; // XXX sf.bbox[0] = sf.bbox[1] = sf.bbox[2] = sf.bbox[3] = 0; // XXX @@ -379,6 +383,11 @@ serial_feature parse_feature(json_pull *jp, unsigned z, unsigned x, unsigned y, lme.maxzoom = z; layermap.insert(std::pair(layername, lme)); + + if (lme.id >= (*layer_unmaps)[tiling_seg].size()) { + (*layer_unmaps)[tiling_seg].resize(lme.id + 1); + (*layer_unmaps)[tiling_seg][lme.id] = layername; + } } type_and_string tas; @@ -393,6 +402,8 @@ serial_feature parse_feature(json_pull *jp, unsigned z, unsigned x, unsigned y, fk->second.maxzoom = z; } fk->second.file_keys.insert(tas); + + sf.layer = fk->second.id; } } @@ -500,7 +511,7 @@ void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, } } -mvt_layer filter_layer(const char *filter, mvt_layer &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg) { +mvt_layer filter_layer(const char *filter, mvt_layer &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps) { int write_to, read_from; pid_t pid; setup_filter(filter, &write_to, &read_from, &pid, z, x, y); @@ -518,7 +529,7 @@ mvt_layer filter_layer(const char *filter, mvt_layer &layer, unsigned z, unsigne exit(EXIT_FAILURE); } - layer = parse_layer(read_from, z, x, y, layer, layermaps, tiling_seg); + layer = parse_layer(read_from, z, x, y, layer, layermaps, tiling_seg, layer_unmaps); while (1) { int stat_loc; diff --git a/plugin.hpp b/plugin.hpp index 197983e..07f848f 100644 --- a/plugin.hpp +++ b/plugin.hpp @@ -1,3 +1,3 @@ -mvt_layer filter_layer(const char *filter, mvt_layer &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg); +mvt_layer filter_layer(const char *filter, mvt_layer &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps); void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y); -serial_feature parse_feature(json_pull *jp, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg); +serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps); diff --git a/tile.cpp b/tile.cpp index 11a0f4b..8b09f3e 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1522,7 +1522,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s if (prefilter == NULL) { sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, line_detail, geomfile, geompos, &oprogress, todo, fname, child_shards); } else { - sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg); + sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps); } if (sf.t < 0) { @@ -1826,7 +1826,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s } if (postfilter != NULL) { - layer = filter_layer(postfilter, layer, z, tx, ty, layermaps, tiling_seg); + layer = filter_layer(postfilter, layer, z, tx, ty, layermaps, tiling_seg, layer_unmaps); } if (layer.features.size() > 0) { @@ -2121,6 +2121,14 @@ void *run_thread(void *vargs) { } int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter) { + // The existing layermaps are one table per input thread. + // We need to add another one per *tiling* thread so that it can be + // safely changed during tiling. + size_t layermaps_off = layermaps.size(); + for (size_t i = 0; i < CPUS; i++) { + layermaps.push_back(std::map()); + } + // Table to map segment and layer number back to layer name std::vector> layer_unmaps; for (size_t seg = 0; seg < layermaps.size(); seg++) { @@ -2134,14 +2142,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo } } - // The existing layermaps are one table per input thread. - // We need to add another one per *tiling* thread so that it can be - // safely changed during tiling. - size_t layermaps_off = layermaps.size(); - for (size_t i = 0; i < CPUS; i++) { - layermaps.push_back(std::map()); - } - int i; for (i = 0; i <= maxzoom; i++) { long long most = 0;