From 555ababd2e0d792500707846c8028181b1907fcc Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 20 May 2015 14:57:00 -0700 Subject: [PATCH 1/3] Add a flag to dynamically drop a fraction of features from large tiles --- README.md | 1 + tile.cc | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fa6382c..f796951 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Options * -pk: Don't limit tiles to 500K bytes * -po: Don't reorder features to put the same properties in sequence * -pl: Let "dot" simplification apply to lines too + * -pd: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. Example ------- diff --git a/tile.cc b/tile.cc index b1ffff9..887f50e 100644 --- a/tile.cc +++ b/tile.cc @@ -354,6 +354,7 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u int line_detail; static bool evaluated = false; double oprogress = 0; + double fraction = 1; char *og = *geoms; @@ -381,6 +382,8 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u interval = exp(log(droprate) * (basezoom - z)); } + double fraction_accum = 0; + unsigned long long previndex = 0; double scale = (double) (1LL << (64 - 2 * (z + 8))); double gap = 0; @@ -552,6 +555,12 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u } } + fraction_accum += fraction; + if (fraction_accum < 1) { + continue; + } + fraction_accum -= 1; + bool reduced = false; if (t == VT_POLYGON) { geom = reduce_tiny_poly(geom, z, line_detail, &reduced, &accum_area); @@ -611,7 +620,7 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u } for (j = 0; j < nlayers; j++) { - if (!prevent['o'] & 0xFF) { + if (!prevent['o' & 0xFF]) { std::sort(features[j].begin(), features[j].end()); } @@ -681,6 +690,11 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u evaluate(features[0], metabase, file_keys[0], layername, line_detail, compressed.size()); // XXX layer #endif } + + if (prevent['d' & 0xFF]) { + fraction = fraction * 500000 / compressed.size(); + fprintf(stderr, "Going to try keeping %0.2f%% of the features to make it fit\n", fraction * 100); + } } else { mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); return count; From 901f6a76b64be339b1dbe5d56e2b6f025fa487ee Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 20 May 2015 15:15:45 -0700 Subject: [PATCH 2/3] Keep the resolution constant as we drop features to make the tile fit --- tile.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tile.cc b/tile.cc index 887f50e..3dc8777 100644 --- a/tile.cc +++ b/tile.cc @@ -443,7 +443,7 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u geom = remove_noop(geom, t); } - if (line_detail == detail) { /* only write out the next zoom once, even if we retry */ + if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */ if (geom.size() > 0 && z + 1 <= file_maxzoom) { int j; for (j = 0; j < 4; j++) { @@ -692,8 +692,12 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u } if (prevent['d' & 0xFF]) { - fraction = fraction * 500000 / compressed.size(); + // The 95% is a guess to avoid too many retries + // and probably actually varies based on how much duplicated metadata there is + + fraction = fraction * 500000 / compressed.size() * 0.95; fprintf(stderr, "Going to try keeping %0.2f%% of the features to make it fit\n", fraction * 100); + line_detail++; // to keep it the same when the loop decrements it } } else { mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); From 38dc80ec6816bafe0ebfcca42473ea561fc2d3ad Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 20 May 2015 18:04:34 -0700 Subject: [PATCH 3/3] Fix a place that wasn't checking for disk write errors --- geojson.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geojson.c b/geojson.c index 58d6f84..9319dde 100644 --- a/geojson.c +++ b/geojson.c @@ -341,7 +341,7 @@ static void merge(struct merge *merges, int nmerges, unsigned char *map, FILE *f } while (head != NULL) { - fwrite(map + head->start, bytes, 1, f); + fwrite_check(map + head->start, bytes, 1, f, "merge temporary"); head->start += bytes; struct merge *m = head;