From 5c647cdb83e07c843c09ba837a9676dc257bdaab Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Tue, 29 Nov 2022 12:42:30 -0800 Subject: [PATCH] Don't preflight zoom levels for potential as-needed dropping (#40) * Working on eliminating preflighting * Adjust for the map/images schema change * Avoid generating duplicate tiles with the detail reduction strategy * Do error checking if tiles in a directory can't be written * Update changelog and version * Revert unintentional code reordering * Neglected to add the exit on error here --- CHANGELOG.md | 4 +++ dirtiles.cpp | 72 +++++++++++++++++++++++++++++++++++++-- dirtiles.hpp | 1 + mbtiles.cpp | 57 ++++++++++++++++++++++++++----- mbtiles.hpp | 1 + tile.cpp | 95 ++++++++++++++++++++++++++++------------------------ version.hpp | 2 +- 7 files changed, 176 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c59f94b..8a2753a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.14.0 + +* Don't preflight each zoom level if one of the as-needed options is specified. Instead, go ahead and write out the tiles, and then clear out the zoom level for another try if necessary. + ## 2.13.1 * Simplify geometry earlier when the in-memory representation of a tile gets large, to reduce peak memory usage diff --git a/dirtiles.cpp b/dirtiles.cpp index 711e469..7e293b9 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -24,13 +24,19 @@ std::string dir_read_tile(std::string base, struct zxy tile) { } void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) { + // Don't check mkdir error returns, since most of these calls to + // mkdir will be creating directories that already exist. mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO); + std::string curdir(outdir); std::string slash("/"); + std::string newdir = curdir + slash + std::to_string(z); mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(tx); mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(ty) + ".pbf"; struct stat st; @@ -39,9 +45,21 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const exit(EXIT_EXISTS); } - std::ofstream pbfFile(newdir, std::ios::out | std::ios::binary); - pbfFile.write(pbf.data(), pbf.size()); - pbfFile.close(); + FILE *fp = fopen(newdir.c_str(), "wb"); + if (fp == NULL) { + fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno)); + exit(EXIT_WRITE); + } + + if (fwrite(pbf.c_str(), sizeof(char), pbf.size(), fp) != pbf.size()) { + fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno)); + exit(EXIT_WRITE); + } + + if (fclose(fp) != 0) { + fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno)); + exit(EXIT_CLOSE); + } } static bool numeric(const char *s) { @@ -160,6 +178,54 @@ std::vector enumerate_dirtiles(const char *fname, int minzoom, int maxzoom) return tiles; } +void dir_erase_zoom(const char *fname, int zoom) { + DIR *d1 = opendir(fname); + if (d1 != NULL) { + struct dirent *dp; + while ((dp = readdir(d1)) != NULL) { + if (numeric(dp->d_name) && atoi(dp->d_name) == zoom) { + std::string z = std::string(fname) + "/" + dp->d_name; + + DIR *d2 = opendir(z.c_str()); + if (d2 == NULL) { + perror(z.c_str()); + exit(EXIT_OPEN); + } + + struct dirent *dp2; + while ((dp2 = readdir(d2)) != NULL) { + if (numeric(dp2->d_name)) { + std::string x = z + "/" + dp2->d_name; + + DIR *d3 = opendir(x.c_str()); + if (d3 == NULL) { + perror(x.c_str()); + exit(EXIT_OPEN); + } + + struct dirent *dp3; + while ((dp3 = readdir(d3)) != NULL) { + if (pbfname(dp3->d_name)) { + std::string y = x + "/" + dp3->d_name; + if (unlink(y.c_str()) != 0) { + perror(y.c_str()); + exit(EXIT_UNLINK); + } + } + } + + closedir(d3); + } + } + + closedir(d2); + } + } + + closedir(d1); + } +} + sqlite3 *dirmeta2tmp(const char *fname) { sqlite3 *db; char *err = NULL; diff --git a/dirtiles.hpp b/dirtiles.hpp index 6855f9a..83149cb 100644 --- a/dirtiles.hpp +++ b/dirtiles.hpp @@ -6,6 +6,7 @@ #define DIRTILES_HPP void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf); +void dir_erase_zoom(const char *outdir, int z); void check_dir(const char *d, char **argv, bool force, bool forcetable); diff --git a/mbtiles.cpp b/mbtiles.cpp index 260d366..258b763 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -73,14 +73,14 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) { } } - // "images" maps a content hash to tile contents - if (sqlite3_exec(outdb, "CREATE TABLE images (tile_data blob, tile_id text);", NULL, NULL, &err) != SQLITE_OK) { + // "images" maps a content hash to tile contents, per zoom level + if (sqlite3_exec(outdb, "CREATE TABLE images (zoom_level integer, tile_data blob, tile_id text);", NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "%s: create images table: %s\n", argv[0], err); if (!forcetable) { exit(EXIT_EXISTS); } } - if (sqlite3_exec(outdb, "CREATE UNIQUE INDEX images_id ON images (tile_id);", NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(outdb, "CREATE UNIQUE INDEX images_id ON images (zoom_level, tile_id);", NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "%s: create images index: %s\n", argv[0], err); if (!forcetable) { exit(EXIT_EXISTS); @@ -89,7 +89,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) { // "tiles" is a view that retrieves content from "images" // via the content hash looked up from "map". - if (sqlite3_exec(outdb, "CREATE VIEW tiles AS SELECT map.zoom_level AS zoom_level, map.tile_column AS tile_column, map.tile_row AS tile_row, images.tile_data AS tile_data FROM map JOIN images ON images.tile_id = map.tile_id;", NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(outdb, "CREATE VIEW tiles AS SELECT map.zoom_level AS zoom_level, map.tile_column AS tile_column, map.tile_row AS tile_row, images.tile_data AS tile_data FROM map JOIN images ON images.tile_id = map.tile_id and images.zoom_level = map.zoom_level;", NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "%s: create tiles view: %s\n", argv[0], err); if (!forcetable) { exit(EXIT_EXISTS); @@ -113,20 +113,23 @@ void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, // following https://github.com/mapbox/node-mbtiles/blob/master/lib/mbtiles.js sqlite3_stmt *stmt; - const char *images = "replace into images (tile_id, tile_data) values (?, ?)"; + const char *images = "replace into images (zoom_level, tile_id, tile_data) values (?, ?, ?)"; if (sqlite3_prepare_v2(outdb, images, -1, &stmt, NULL) != SQLITE_OK) { fprintf(stderr, "sqlite3 images prep failed\n"); exit(EXIT_SQLITE); } - sqlite3_bind_blob(stmt, 1, hash.c_str(), hash.size(), NULL); - sqlite3_bind_blob(stmt, 2, data, size, NULL); + sqlite3_bind_int(stmt, 1, z); + sqlite3_bind_blob(stmt, 2, hash.c_str(), hash.size(), NULL); + sqlite3_bind_blob(stmt, 3, data, size, NULL); if (sqlite3_step(stmt) != SQLITE_DONE) { fprintf(stderr, "sqlite3 images insert failed: %s\n", sqlite3_errmsg(outdb)); + exit(EXIT_SQLITE); } if (sqlite3_finalize(stmt) != SQLITE_OK) { fprintf(stderr, "sqlite3 images finalize failed: %s\n", sqlite3_errmsg(outdb)); + exit(EXIT_SQLITE); } const char *map = "insert into map (zoom_level, tile_column, tile_row, tile_id) values (?, ?, ?, ?)"; @@ -142,9 +145,47 @@ void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, if (sqlite3_step(stmt) != SQLITE_DONE) { fprintf(stderr, "sqlite3 map insert failed: %s\n", sqlite3_errmsg(outdb)); + exit(EXIT_SQLITE); } if (sqlite3_finalize(stmt) != SQLITE_OK) { - fprintf(stderr, "sqlite3 map finalize failed: %s\n", sqlite3_errmsg(outdb)); + fprintf(stderr, "sqlite3 finalize failed: %s\n", sqlite3_errmsg(outdb)); + exit(EXIT_SQLITE); + } +} + +void mbtiles_erase_zoom(sqlite3 *outdb, int z) { + sqlite3_stmt *stmt; + + const char *query = "delete from map where zoom_level = ?"; + if (sqlite3_prepare_v2(outdb, query, -1, &stmt, NULL) != SQLITE_OK) { + fprintf(stderr, "sqlite3 delete map prep failed\n"); + exit(EXIT_SQLITE); + } + + sqlite3_bind_int(stmt, 1, z); + if (sqlite3_step(stmt) != SQLITE_DONE) { + fprintf(stderr, "sqlite3 delete map failed: %s\n", sqlite3_errmsg(outdb)); + exit(EXIT_SQLITE); + } + if (sqlite3_finalize(stmt) != SQLITE_OK) { + fprintf(stderr, "sqlite3 delete map finalize failed: %s\n", sqlite3_errmsg(outdb)); + exit(EXIT_SQLITE); + } + + query = "delete from images where zoom_level = ?"; + if (sqlite3_prepare_v2(outdb, query, -1, &stmt, NULL) != SQLITE_OK) { + fprintf(stderr, "sqlite3 delete images prep failed\n"); + exit(EXIT_SQLITE); + } + + sqlite3_bind_int(stmt, 1, z); + if (sqlite3_step(stmt) != SQLITE_DONE) { + fprintf(stderr, "sqlite3 delete images failed: %s\n", sqlite3_errmsg(outdb)); + exit(EXIT_SQLITE); + } + if (sqlite3_finalize(stmt) != SQLITE_OK) { + fprintf(stderr, "sqlite3 delete images finalize failed: %s\n", sqlite3_errmsg(outdb)); + exit(EXIT_SQLITE); } } diff --git a/mbtiles.hpp b/mbtiles.hpp index b8793cb..8fcd17d 100644 --- a/mbtiles.hpp +++ b/mbtiles.hpp @@ -46,6 +46,7 @@ struct layermap_entry { sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); +void mbtiles_erase_zoom(sqlite3 *outdb, int z); void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description, bool do_tilestats, std::map const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector const &strategies); diff --git a/tile.cpp b/tile.cpp index f617702..49de135 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1315,7 +1315,6 @@ struct write_tile_args { std::vector> *layermaps = NULL; std::vector> *layer_unmaps = NULL; size_t pass = 0; - size_t passes = 0; unsigned long long mingap = 0; unsigned long long mingap_out = 0; long long minextent = 0; @@ -1430,14 +1429,15 @@ void remove_attributes(serial_feature &sf, std::set const &exclude_ } } -serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, std::atomic *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, std::atomic *geompos, std::atomic *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector> *layer_unmaps) { +serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, int *within, FILE **geomfile, std::atomic *geompos, std::atomic *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time) { while (1) { serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y); if (sf.t < 0) { return sf; } - double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10; + size_t passes = pass + 1; + double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + pass) / passes + z) / (maxzoom + 1) * 1000) / 10; if (progress >= *oprogress + 0.1) { if (!quiet && !quiet_progress && progress_time()) { fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); @@ -1458,7 +1458,7 @@ serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, cha (*unclipped_features)++; } - if (*first_time && pass == 1) { /* only write out the next zoom once, even if we retry */ + if (first_time && pass == 0) { /* only write out the next zoom once, even if we retry */ if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) { rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.label_point, sf.extent); } @@ -1577,12 +1577,10 @@ struct run_prefilter_args { int minzoom = 0; int max_zoom_increment = 0; size_t pass = 0; - size_t passes = 0; std::atomic *along = 0; long long alongminus = 0; int buffer = 0; int *within = NULL; - bool *first_time = NULL; FILE **geomfile = NULL; std::atomic *geompos = NULL; std::atomic *oprogress = NULL; @@ -1594,6 +1592,7 @@ struct run_prefilter_args { long long *pool_off = NULL; FILE *prefilter_fp = NULL; struct json_object *filter = NULL; + bool first_time = false; }; void *run_prefilter(void *v) { @@ -1601,7 +1600,7 @@ void *run_prefilter(void *v) { json_writer state(rpa->prefilter_fp); while (1) { - serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps); + serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time); if (sf.t < 0) { break; } @@ -1830,7 +1829,7 @@ static bool line_is_too_small(drawvec const &geometry, int z, int detail) { return true; } -long long write_tile(FILE *geoms, std::atomic *geompos_in, char *metabase, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) { +long long write_tile(FILE *geoms, std::atomic *geompos_in, char *metabase, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) { double merge_fraction = 1; double mingap_fraction = 1; double minextent_fraction = 1; @@ -1945,12 +1944,10 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta rpa.minzoom = minzoom; rpa.max_zoom_increment = max_zoom_increment; rpa.pass = pass; - rpa.passes = passes; rpa.along = along; rpa.alongminus = alongminus; rpa.buffer = buffer; rpa.within = within; - rpa.first_time = &first_time; rpa.geomfile = geomfile; rpa.geompos = geompos; rpa.oprogress = &oprogress; @@ -1962,6 +1959,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta rpa.stringpool = stringpool; rpa.pool_off = pool_off; rpa.filter = filter; + rpa.first_time = first_time; if (pthread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) { perror("pthread_create (prefilter writer)"); @@ -1981,7 +1979,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta ssize_t which_partial = -1; if (prefilter == NULL) { - sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps); + sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time); } else { sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); } @@ -2527,7 +2525,8 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta totalsize += layer_features.size(); } - double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10; + size_t passes = pass + 1; + double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + pass) / passes + z) / (maxzoom + 1) * 1000) / 10; if (progress >= oprogress + 0.1) { if (!quiet && !quiet_progress && progress_time()) { fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); @@ -2725,22 +2724,20 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta strategy->detail_reduced++; } } else { - if (pass == 1) { - if (pthread_mutex_lock(&db_lock) != 0) { - perror("pthread_mutex_lock"); - exit(EXIT_PTHREAD); - } + if (pthread_mutex_lock(&db_lock) != 0) { + perror("pthread_mutex_lock"); + exit(EXIT_PTHREAD); + } - if (outdb != NULL) { - mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); - } else if (outdir != NULL) { - dir_write_tile(outdir, z, tx, ty, compressed); - } + if (outdb != NULL) { + mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); + } else if (outdir != NULL) { + dir_write_tile(outdir, z, tx, ty, compressed); + } - if (pthread_mutex_unlock(&db_lock) != 0) { - perror("pthread_mutex_unlock"); - exit(EXIT_PTHREAD); - } + if (pthread_mutex_unlock(&db_lock) != 0) { + perror("pthread_mutex_unlock"); + exit(EXIT_PTHREAD); } return count; @@ -2799,7 +2796,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy); + long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy); if (len < 0) { int *err = &arg->err; @@ -2888,8 +2885,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo } } - int i; - for (i = 0; i <= maxzoom; i++) { + int z; + for (z = 0; z <= maxzoom; z++) { std::atomic most(0); FILE *sub[TEMP_FILES]; @@ -2995,19 +2992,14 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo int err = INT_MAX; - size_t start = 1; - if (additional[A_INCREASE_GAMMA_AS_NEEDED] || additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED] || additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) { - start = 0; - } - double zoom_gamma = gamma; - unsigned long long zoom_mingap = ((1LL << (32 - i)) / 256 * cluster_distance) * ((1LL << (32 - i)) / 256 * cluster_distance); + unsigned long long zoom_mingap = ((1LL << (32 - z)) / 256 * cluster_distance) * ((1LL << (32 - z)) / 256 * cluster_distance); long long zoom_minextent = 0; double zoom_fraction = 1; size_t zoom_tile_size = 0; size_t zoom_feature_count = 0; - for (size_t pass = start; pass < 2; pass++) { + for (size_t pass = 0; ; pass++) { pthread_t pthreads[threads]; std::vector args; args.resize(threads); @@ -3038,7 +3030,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].feature_count_out = 0; args[thread].child_shards = TEMP_FILES / threads; - if (i == maxzoom && maxzoom_simplification > 0) { + if (z == maxzoom && maxzoom_simplification > 0) { args[thread].simplification = maxzoom_simplification; } else { args[thread].simplification = simplification; @@ -3068,7 +3060,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].tasks = dispatches[thread].tasks; args[thread].running = &running; args[thread].pass = pass; - args[thread].passes = 2 - start; args[thread].wrote_zoom = -1; args[thread].still_dropping = false; args[thread].strategy = &strategy; @@ -3079,6 +3070,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo } } + bool again = false; for (size_t thread = 0; thread < threads; thread++) { void *retval; @@ -3092,15 +3084,19 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo if (args[thread].gamma_out > zoom_gamma) { zoom_gamma = args[thread].gamma_out; + again = true; } if (args[thread].mingap_out > zoom_mingap) { zoom_mingap = args[thread].mingap_out; + again = true; } if (args[thread].minextent_out > zoom_minextent) { zoom_minextent = args[thread].minextent_out; + again = true; } if (args[thread].fraction_out < zoom_fraction) { zoom_fraction = args[thread].fraction_out; + again = true; } if (args[thread].tile_size_out > zoom_tile_size) { zoom_tile_size = args[thread].tile_size_out; @@ -3110,20 +3106,31 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo } // Zoom counter might be lower than reality if zooms are being skipped - if (args[thread].wrote_zoom > i) { - i = args[thread].wrote_zoom; + if (args[thread].wrote_zoom > z) { + z = args[thread].wrote_zoom; } - if (additional[A_EXTEND_ZOOMS] && i == maxzoom && args[thread].still_dropping && maxzoom < MAX_ZOOM) { + if (additional[A_EXTEND_ZOOMS] && z == maxzoom && args[thread].still_dropping && maxzoom < MAX_ZOOM) { maxzoom++; } } - if ((size_t) i >= strategies.size()) { - strategies.resize(i + 1); + if ((size_t) z >= strategies.size()) { + strategies.resize(z + 1); } + struct strategy s(strategy, zoom_tile_size, zoom_feature_count); - strategies[i] = s; + strategies[z] = s; + + if (again) { + if (outdb != NULL) { + mbtiles_erase_zoom(outdb, z); + } else if (outdir != NULL) { + dir_erase_zoom(outdir, z); + } + } else { + break; + } } for (size_t j = 0; j < TEMP_FILES; j++) { diff --git a/version.hpp b/version.hpp index 5fc8270..3988b5c 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v2.13.1" +#define VERSION "v2.14.0" #endif