From 94967369a76768c2fba5c3c3d6de9d69cdcd2601 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 8 Mar 2018 15:30:11 -0800 Subject: [PATCH] Allocate an ID when each feature is clipped for the first time --- tile.cpp | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/tile.cpp b/tile.cpp index ee41ea4..b6b535b 100644 --- a/tile.cpp +++ b/tile.cpp @@ -252,7 +252,7 @@ static int metacmp(int m1, const std::vector &keys1, const std::vecto } } -void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m, std::vector &metakeys, std::vector &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) { +void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m, std::vector &metakeys, std::vector &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent, long long clipid, size_t tiling_seg, std::vector *clipids) { if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) { int xo, yo; int span = 1 << (nextzoom - z); @@ -295,6 +295,17 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u geom2.push_back(draw(geom[i].op, (geom[i].x + sx) >> geometry_scale, (geom[i].y + sy) >> geometry_scale, geom[i].id)); } + if (bbox2[0] != bbox2[2] || bbox2[1] != bbox2[3]) { + // Feature is being split across multiple child tiles, + // so we must give the feature an ID if it doesn't + // already have one, and must give IDs to all the nodes + // where it crosses a tile boundary. + + if (clipid == 0) { + clipid = (*clipids)[tiling_seg]++ * CPUS + tiling_seg; + } + } + for (xo = bbox2[0]; xo <= bbox2[2]; xo++) { for (yo = bbox2[1]; yo <= bbox2[3]; yo++) { unsigned jx = tx * span + xo; @@ -1220,6 +1231,7 @@ struct write_tile_args { bool still_dropping = false; int wrote_zoom = 0; size_t tiling_seg = 0; + std::vector *clipids = NULL; }; bool clip_to_tile(serial_feature &sf, int z, long long buffer) { @@ -1304,7 +1316,7 @@ bool clip_to_tile(serial_feature &sf, int z, long long buffer) { return false; } -serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, volatile long long *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, long long *geompos, volatile double *oprogress, double todo, const char *fname, int child_shards) { +serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, volatile long long *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, long long *geompos, volatile double *oprogress, double todo, const char *fname, int child_shards, size_t tiling_seg, std::vector *clipids) { while (1) { serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y); if (sf.t < 0) { @@ -1332,7 +1344,7 @@ serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, if (*first_time && pass == 1) { /* only write out the next zoom once, even if we retry */ if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) { - rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.m, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent); + rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.m, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent, sf.clipid, tiling_seg, clipids); } } @@ -1388,13 +1400,15 @@ struct run_prefilter_args { char *stringpool = NULL; long long *pool_off = NULL; FILE *prefilter_fp = NULL; + std::vector *clipids; + size_t tiling_seg; }; void *run_prefilter(void *v) { run_prefilter_args *rpa = (run_prefilter_args *) v; while (1) { - serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards); + serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->tiling_seg, rpa->clipids); if (sf.t < 0) { break; } @@ -1725,6 +1739,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s rpa.layer_unmaps = layer_unmaps; rpa.stringpool = stringpool; rpa.pool_off = pool_off; + rpa.clipids = arg->clipids; + rpa.tiling_seg = tiling_seg; if (pthread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) { perror("pthread_create (prefilter writer)"); @@ -1744,7 +1760,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s ssize_t which_partial = -1; if (prefilter == NULL) { - sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards); + sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards, tiling_seg, arg->clipids); } else { sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); } @@ -2519,6 +2535,13 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo } } + // Allocation pool for feature clip IDs. + // This is sized to the number of (reading + tiling) layermaps + // instead of just to the number of CPUs because tiling_seg + // includes the offset. + std::vector clipids; + clipids.resize(layermaps.size()); + int i; for (i = 0; i <= maxzoom; i++) { long long most = 0; @@ -2691,6 +2714,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].passes = 2 - start; args[thread].wrote_zoom = -1; args[thread].still_dropping = false; + args[thread].clipids = &clipids; if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) { perror("pthread_create");