From b2277d76e58c14d7a672c6a8eef824d97c5a77fa Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 17 Feb 2020 14:56:11 -0800 Subject: [PATCH] Eliminate the separate metadata file --- main.cpp | 95 +++--------------------------------------------------- plugin.cpp | 1 - serial.cpp | 89 +++++++++----------------------------------------- serial.hpp | 16 ++------- tile.cpp | 35 +++++++------------- tile.hpp | 2 +- 6 files changed, 36 insertions(+), 202 deletions(-) diff --git a/main.cpp b/main.cpp index ae41d82..a945762 100644 --- a/main.cpp +++ b/main.cpp @@ -101,7 +101,7 @@ void checkdisk(std::vector *r) { for (size_t i = 0; i < r->size(); i++) { // Meta, pool, and tree are used once. // Geometry and index will be duplicated during sorting and tiling. - used += (*r)[i].metapos + 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len; + used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len; } static int warned = 0; @@ -1143,23 +1143,16 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo for (size_t i = 0; i < CPUS; i++) { struct reader *r = &readers[i]; - char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; char treename[strlen(tmpdir) + strlen("/tree.XXXXXXXX") + 1]; char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; - sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX"); sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX"); sprintf(treename, "%s%s", tmpdir, "/tree.XXXXXXXX"); sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX"); sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX"); - r->metafd = mkstemp_cloexec(metaname); - if (r->metafd < 0) { - perror(metaname); - exit(EXIT_FAILURE); - } r->poolfd = mkstemp_cloexec(poolname); if (r->poolfd < 0) { perror(poolname); @@ -1181,11 +1174,6 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo exit(EXIT_FAILURE); } - r->metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC); - if (r->metafile == NULL) { - perror(metaname); - exit(EXIT_FAILURE); - } r->poolfile = memfile_open(r->poolfd); if (r->poolfile == NULL) { perror(poolname); @@ -1206,11 +1194,9 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo perror(indexname); exit(EXIT_FAILURE); } - r->metapos = 0; r->geompos = 0; r->indexpos = 0; - unlink(metaname); unlink(poolname); unlink(treename); unlink(geomname); @@ -1221,8 +1207,6 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo struct stringpool p; memfile_write(r->treefile, &p, sizeof(struct stringpool)); } - // Keep metadata file from being completely empty if no attributes - serialize_int(r->metafile, 0, &r->metapos, "meta"); r->file_bbox[0] = r->file_bbox[1] = UINT_MAX; r->file_bbox[2] = r->file_bbox[3] = 0; @@ -1699,10 +1683,6 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } for (size_t i = 0; i < CPUS; i++) { - if (fclose(readers[i].metafile) != 0) { - perror("fclose meta"); - exit(EXIT_FAILURE); - } if (fclose(readers[i].geomfile) != 0) { perror("fclose geom"); exit(EXIT_FAILURE); @@ -1717,10 +1697,6 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo perror("stat geom\n"); exit(EXIT_FAILURE); } - if (fstat(readers[i].metafd, &readers[i].metast) != 0) { - perror("stat meta\n"); - exit(EXIT_FAILURE); - } } // Create a combined string pool and a combined metadata file @@ -1729,9 +1705,8 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo // 2 * CPUS: One per input thread, one per tiling thread long long pool_off[2 * CPUS]; - long long meta_off[2 * CPUS]; for (size_t i = 0; i < 2 * CPUS; i++) { - pool_off[i] = meta_off[i] = 0; + pool_off[i] = 0; } char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; @@ -1751,51 +1726,9 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo unlink(poolname); - char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; - sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX"); - - int metafd = mkstemp_cloexec(metaname); - if (metafd < 0) { - perror(metaname); - exit(EXIT_FAILURE); - } - - FILE *metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC); - if (metafile == NULL) { - perror(metaname); - exit(EXIT_FAILURE); - } - - unlink(metaname); - - std::atomic metapos(0); std::atomic poolpos(0); for (size_t i = 0; i < CPUS; i++) { - if (readers[i].metapos > 0) { - void *map = mmap(NULL, readers[i].metapos, PROT_READ, MAP_PRIVATE, readers[i].metafd, 0); - if (map == MAP_FAILED) { - perror("mmap unmerged meta"); - exit(EXIT_FAILURE); - } - madvise(map, readers[i].metapos, MADV_SEQUENTIAL); - madvise(map, readers[i].metapos, MADV_WILLNEED); - if (fwrite(map, readers[i].metapos, 1, metafile) != 1) { - perror("Reunify meta"); - exit(EXIT_FAILURE); - } - madvise(map, readers[i].metapos, MADV_DONTNEED); - if (munmap(map, readers[i].metapos) != 0) { - perror("unmap unmerged meta"); - } - } - - meta_off[i] = metapos; - metapos += readers[i].metapos; - if (close(readers[i].metafd) != 0) { - perror("close unmerged meta"); - } - if (readers[i].poolfile->off > 0) { if (fwrite(readers[i].poolfile->map, readers[i].poolfile->off, 1, poolfile) != 1) { perror("Reunify string pool"); @@ -1812,17 +1745,6 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo perror("fclose pool"); exit(EXIT_FAILURE); } - if (fclose(metafile) != 0) { - perror("fclose meta"); - exit(EXIT_FAILURE); - } - - char *meta = (char *) mmap(NULL, metapos, PROT_READ, MAP_PRIVATE, metafd, 0); - if (meta == MAP_FAILED) { - perror("mmap meta"); - exit(EXIT_FAILURE); - } - madvise(meta, metapos, MADV_RANDOM); char *stringpool = NULL; if (poolpos > 0) { // Will be 0 if -X was specified @@ -1907,9 +1829,8 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo if (!quiet) { long long s = progress_seq; long long geompos_print = geompos; - long long metapos_print = metapos; long long poolpos_print = poolpos; - fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", s, geompos_print, metapos_print, poolpos_print); + fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of string pool\n", s, geompos_print, poolpos_print); } if (indexpos == 0) { @@ -2267,7 +2188,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo std::atomic midx(0); std::atomic midy(0); - int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum, filter); + int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum, filter); if (maxzoom != written) { if (written > minzoom) { @@ -2280,14 +2201,6 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } } - madvise(meta, metapos, MADV_DONTNEED); - if (munmap(meta, metapos) != 0) { - perror("munmap meta"); - } - if (close(metafd) < 0) { - perror("close meta"); - } - if (poolpos > 0) { madvise((void *) stringpool, poolpos, MADV_DONTNEED); if (munmap(stringpool, poolpos) != 0) { diff --git a/plugin.cpp b/plugin.cpp index d416299..fcc4cae 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -397,7 +397,6 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: sf.bbox[0] = sf.bbox[1] = LLONG_MAX; sf.bbox[2] = sf.bbox[3] = LLONG_MIN; sf.extent = 0; - sf.metapos = 0; sf.has_id = false; std::string layername = "unknown"; diff --git a/serial.cpp b/serial.cpp index 797e4b5..7dca00c 100644 --- a/serial.cpp +++ b/serial.cpp @@ -225,15 +225,11 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomicextent, geompos, fname); } - serialize_long_long(geomfile, sf->metapos, geompos, fname); + serialize_long_long(geomfile, sf->keys.size(), geompos, fname); - if (sf->metapos < 0) { - serialize_long_long(geomfile, sf->keys.size(), geompos, fname); - - for (size_t i = 0; i < sf->keys.size(); i++) { - serialize_long_long(geomfile, sf->keys[i], geompos, fname); - serialize_long_long(geomfile, sf->values[i], geompos, fname); - } + for (size_t i = 0; i < sf->keys.size(); i++) { + serialize_long_long(geomfile, sf->keys[i], geompos, fname); + serialize_long_long(geomfile, sf->values[i], geompos, fname); } if (include_minzoom) { @@ -241,7 +237,7 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) { +serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) { serial_feature sf; deserialize_byte_io(geoms, &sf.t, geompos_in); @@ -286,32 +282,15 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_ sf.layer >>= 6; - sf.metapos = 0; - deserialize_long_long_io(geoms, &sf.metapos, geompos_in); + long long count; + deserialize_long_long_io(geoms, &count, geompos_in); - if (sf.metapos >= 0) { - char *meta = metabase + sf.metapos + meta_off[sf.segment]; - long long count; - deserialize_long_long(&meta, &count); - - for (long long i = 0; i < count; i++) { - long long k, v; - deserialize_long_long(&meta, &k); - deserialize_long_long(&meta, &v); - sf.keys.push_back(k); - sf.values.push_back(v); - } - } else { - long long count; - deserialize_long_long_io(geoms, &count, geompos_in); - - for (long long i = 0; i < count; i++) { - long long k, v; - deserialize_long_long_io(geoms, &k, geompos_in); - deserialize_long_long_io(geoms, &v, geompos_in); - sf.keys.push_back(k); - sf.values.push_back(v); - } + for (long long i = 0; i < count; i++) { + long long k, v; + deserialize_long_long_io(geoms, &k, geompos_in); + deserialize_long_long_io(geoms, &v, geompos_in); + sf.keys.push_back(k); + sf.values.push_back(v); } deserialize_byte_io(geoms, &sf.feature_minzoom, geompos_in); @@ -488,32 +467,6 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { locs.clear(); } - bool inline_meta = true; - // Don't inline metadata for features that will span several tiles at maxzoom - if (sf.geometry.size() > 0 && (sf.bbox[2] < sf.bbox[0] || sf.bbox[3] < sf.bbox[1])) { - fprintf(stderr, "Internal error: impossible feature bounding box %llx,%llx,%llx,%llx\n", sf.bbox[0], sf.bbox[1], sf.bbox[2], sf.bbox[3]); - } - if (sf.bbox[0] == LLONG_MAX) { - // No bounding box (empty geometry) - // Shouldn't happen, but avoid arithmetic overflow below - } else if (sf.bbox[2] - sf.bbox[0] > (2LL << (32 - sst->maxzoom)) || sf.bbox[3] - sf.bbox[1] > (2LL << (32 - sst->maxzoom))) { - inline_meta = false; - - if (prevent[P_CLIPPING]) { - static std::atomic warned(0); - long long extent = ((sf.bbox[2] - sf.bbox[0]) / ((1LL << (32 - sst->maxzoom)) + 1)) * ((sf.bbox[3] - sf.bbox[1]) / ((1LL << (32 - sst->maxzoom)) + 1)); - if (extent > warned) { - fprintf(stderr, "Warning: %s:%d: Large unclipped (-pc) feature may be duplicated across %lld tiles\n", sst->fname, sst->line, extent); - warned = extent; - - if (extent > 10000) { - fprintf(stderr, "Exiting because this can't be right.\n"); - exit(EXIT_FAILURE); - } - } - } - } - double extent = 0; if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) { if (sf.t == VT_POLYGON) { @@ -651,19 +604,9 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { } } - if (inline_meta) { - sf.metapos = -1; - for (size_t i = 0; i < sf.full_keys.size(); i++) { - sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string)); - sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type)); - } - } else { - sf.metapos = r->metapos; - serialize_long_long(r->metafile, sf.full_keys.size(), &r->metapos, sst->fname); - for (size_t i = 0; i < sf.full_keys.size(); i++) { - serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string), &r->metapos, sst->fname); - serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type), &r->metapos, sst->fname); - } + for (size_t i = 0; i < sf.full_keys.size(); i++) { + sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string)); + sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type)); } long long geomstart = r->geompos; diff --git a/serial.hpp b/serial.hpp index a662d49..52bf1f8 100644 --- a/serial.hpp +++ b/serial.hpp @@ -60,8 +60,6 @@ struct serial_feature { std::vector keys{}; std::vector values{}; - // If >= 0, metadata is external - long long metapos = 0; // XXX This isn't serialized. Should it be here? long long bbox[4] = {0, 0, 0, 0}; @@ -72,22 +70,19 @@ struct serial_feature { }; void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic *geompos, const char *fname, long long wx, long long wy, bool include_minzoom); -serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y); +serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y); struct reader { - int metafd = -1; int poolfd = -1; int treefd = -1; int geomfd = -1; int indexfd = -1; - FILE *metafile = NULL; struct memfile *poolfile = NULL; struct memfile *treefile = NULL; FILE *geomfile = NULL; FILE *indexfile = NULL; - std::atomic metapos; std::atomic geompos; std::atomic indexpos; @@ -99,26 +94,21 @@ struct reader { char *geom_map = NULL; reader() - : metapos(0), geompos(0), indexpos(0) { + : geompos(0), indexpos(0) { } reader(reader const &r) { - metafd = r.metafd; poolfd = r.poolfd; treefd = r.treefd; geomfd = r.geomfd; indexfd = r.indexfd; - metafile = r.metafile; poolfile = r.poolfile; treefile = r.treefile; geomfile = r.geomfile; indexfile = r.indexfile; - long long p = r.metapos; - metapos = p; - - p = r.geompos; + long long p = r.geompos; geompos = p; p = r.indexpos; diff --git a/tile.cpp b/tile.cpp index cf41982..2fd876b 100644 --- a/tile.cpp +++ b/tile.cpp @@ -248,7 +248,7 @@ static int metacmp(const std::vector &keys1, const std::vector *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector &metakeys, std::vector &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) { +void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic *geompos, FILE **geomfile, const char *fname, signed char t, int layer, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector &metakeys, std::vector &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) { if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) { int xo, yo; int span = 1 << (nextzoom - z); @@ -335,17 +335,14 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u sf.tippecanoe_minzoom = tippecanoe_minzoom; sf.has_tippecanoe_maxzoom = tippecanoe_maxzoom != -1; sf.tippecanoe_maxzoom = tippecanoe_maxzoom; - sf.metapos = metastart; sf.geometry = geom2; sf.index = index; sf.extent = extent; sf.feature_minzoom = feature_minzoom; - if (metastart < 0) { - for (size_t i = 0; i < metakeys.size(); i++) { - sf.keys.push_back(metakeys[i]); - sf.values.push_back(metavals[i]); - } + for (size_t i = 0; i < metakeys.size(); i++) { + sf.keys.push_back(metakeys[i]); + sf.values.push_back(metavals[i]); } serialize_feature(geomfile[j], &sf, &geompos[j], fname, SHIFT_RIGHT(initial_x[segment]), SHIFT_RIGHT(initial_y[segment]), true); @@ -1175,7 +1172,6 @@ long long choose_minextent(std::vector &extents, double f) { struct write_tile_args { struct task *tasks = NULL; - char *metabase = NULL; char *stringpool = NULL; int min_detail = 0; sqlite3 *outdb = NULL; @@ -1198,7 +1194,6 @@ struct write_tile_args { int low_detail = 0; double simplification = 0; std::atomic *most = NULL; - long long *meta_off = NULL; long long *pool_off = NULL; unsigned *initial_x = NULL; unsigned *initial_y = NULL; @@ -1318,9 +1313,9 @@ void remove_attributes(serial_feature &sf, std::set const &exclude_ } } -serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, std::atomic *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, std::atomic *geompos, std::atomic *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector> *layer_unmaps) { +serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, std::atomic *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, std::atomic *geompos, std::atomic *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector> *layer_unmaps) { while (1) { - serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y); + serial_feature sf = deserialize_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y); if (sf.t < 0) { return sf; } @@ -1345,7 +1340,7 @@ serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, cha if (*first_time && pass == 1) { /* only write out the next zoom once, even if we retry */ if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) { - rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent); + rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent); } } @@ -1448,8 +1443,6 @@ serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, cha struct run_prefilter_args { FILE *geoms = NULL; std::atomic *geompos_in = NULL; - char *metabase = NULL; - long long *meta_off = NULL; int z = 0; unsigned tx = 0; unsigned ty = 0; @@ -1486,7 +1479,7 @@ void *run_prefilter(void *v) { json_writer state(rpa->prefilter_fp); while (1) { - serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps); + serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps); if (sf.t < 0) { break; } @@ -1715,7 +1708,7 @@ static bool line_is_too_small(drawvec const &geometry, int z, int detail) { return true; } -long long write_tile(FILE *geoms, std::atomic *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg) { +long long write_tile(FILE *geoms, std::atomic *geompos_in, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg) { int line_detail; double merge_fraction = 1; double mingap_fraction = 1; @@ -1808,8 +1801,6 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta rpa.geoms = geoms; rpa.geompos_in = geompos_in; - rpa.metabase = metabase; - rpa.meta_off = meta_off; rpa.z = z; rpa.tx = tx; rpa.ty = ty; @@ -1858,7 +1849,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta ssize_t which_partial = -1; if (prefilter == NULL) { - sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps); + sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps); } else { sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); } @@ -2582,7 +2573,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg); + long long len = write_tile(geom, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg); if (len < 0) { int *err = &arg->err; @@ -2647,7 +2638,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter) { last_progress = 0; // The existing layermaps are one table per input thread. @@ -2796,7 +2787,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo std::atomic along(0); for (size_t thread = 0; thread < threads; thread++) { - args[thread].metabase = metabase; args[thread].stringpool = stringpool; args[thread].min_detail = min_detail; args[thread].outdb = outdb; // locked with db_lock @@ -2826,7 +2816,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].full_detail = full_detail; args[thread].low_detail = low_detail; args[thread].most = &most; // locked with var_lock - args[thread].meta_off = meta_off; args[thread].pool_off = pool_off; args[thread].initial_x = initial_x; args[thread].initial_y = initial_y; diff --git a/tile.hpp b/tile.hpp index 32fb227..f970281 100644 --- a/tile.hpp +++ b/tile.hpp @@ -21,7 +21,7 @@ enum attribute_op { long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);