Merge pull request #471 from mapbox/coalesce-smallest

Experiment with coalescing features to reduce tile size

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.25.0
+
+* Add --coalesce-smallest-as-needed strategy for reducing tile sizes
+* Add --stats option to tipppecanoe-decode
+
 ## 1.24.1
 
 * Limit the size and depth of the string pool for better performance

Makefile

@@ -141,9 +141,11 @@ decode-test:
 	./tippecanoe -z11 -Z11 -f -o tests/muni/decode/multi.mbtiles tests/muni/*.json
 	./tippecanoe-decode -l subway tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.json.check
 	./tippecanoe-decode -c tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.pipeline.json.check
+	./tippecanoe-decode --stats tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.stats.json.check
 	cmp tests/muni/decode/multi.mbtiles.json.check tests/muni/decode/multi.mbtiles.json
 	cmp tests/muni/decode/multi.mbtiles.pipeline.json.check tests/muni/decode/multi.mbtiles.pipeline.json
-	rm -f tests/muni/decode/multi.mbtiles.json.check tests/muni/decode/multi.mbtiles tests/muni/decode/multi.mbtiles.pipeline.json.check
+	cmp tests/muni/decode/multi.mbtiles.stats.json.check tests/muni/decode/multi.mbtiles.stats.json
+	rm -f tests/muni/decode/multi.mbtiles.json.check tests/muni/decode/multi.mbtiles tests/muni/decode/multi.mbtiles.pipeline.json.check tests/muni/decode/multi.mbtiles.stats.json.check
 
 pbf-test:
 	./tippecanoe-decode tests/pbf/11-328-791.vector.pbf 11 328 791 > tests/pbf/11-328-791.vector.pbf.out

README.md

@@ -205,6 +205,7 @@ tippecanoe -z5 -o filtered.mbtiles -j '{ "ne_10m_admin_0_countries": [ "all", [
  * `-as` or `--drop-densest-as-needed`: If a tile is too large, try to reduce it to under 500K by increasing the minimum spacing between features. The discovered spacing applies to the entire zoom level.
  * `-ad` or `--drop-fraction-as-needed`: Dynamically drop some fraction of features from each zoom level to keep large tiles under the 500K size limit. (This is like `-pd` but applies to the entire zoom level, not to each tile.)
  * `-an` or `--drop-smallest-as-needed`: Dynamically drop the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level to keep large tiles under the 500K size limit. This option will not work for point features.
+ * `-aN` or `--coalesce-smallest-as-needed`: Dynamically combine the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level into other nearby features to keep large tiles under the 500K size limit. This option will not work for point features, and will probably not help very much with LineStrings. It is mostly intended for polygons, to maintain the full original area covered by polygons while still reducing the feature count somehow. The attributes of the small polygons are *not* preserved into the combined features, only their geometry.
  * `-pd` or `--force-feature-limit`: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. (This is like `-ad` but applies to each tile individually, not to the entire zoom level.) You probably don't want to use this.
 
 ### Dropping tightly overlapping features
@@ -612,4 +613,5 @@ resolutions.
  * `-Z` _minzoom_ or `--minimum-zoom=`*minzoom*: Specify the lowest zoom level to decode from the tileset
  * `-l` _layer_ or `--layer=`*layer*: Decode only layers with the specified names. (Multiple `-l` options can be specified.)
  * `-c` or `--tag-layer-and-zoom`: Include each feature's layer and zoom level as part of its `tippecanoe` object rather than as a FeatureCollection wrapper
+ * `-S` or `--stats`: Just report statistics about each tile's size and the number of features in it, as a JSON structure.
  * `-f` or `--force`: Decode tiles even if polygon ring order or closure problems are detected

decode.cpp

@@ -23,7 +23,34 @@ int minzoom = 0;
 int maxzoom = 32;
 bool force = false;
 
-void handle(std::string message, int z, unsigned x, unsigned y, int describe, std::set<std::string> const &to_decode, bool pipeline) {
+void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, unsigned y) {
+	printf("{ \"zoom\": %d, \"x\": %u, \"y\": %u, \"bytes\": %zu, \"compressed\": %s", z, x, y, size, compressed ? "true" : "false");
+
+	printf(", \"layers\": { ");
+	for (size_t i = 0; i < tile.layers.size(); i++) {
+		if (i != 0) {
+			printf(", ");
+		}
+		fprintq(stdout, tile.layers[i].name.c_str());
+
+		int points = 0, lines = 0, polygons = 0;
+		for (size_t j = 0; j < tile.layers[i].features.size(); j++) {
+			if (tile.layers[i].features[j].type == mvt_point) {
+				points++;
+			} else if (tile.layers[i].features[j].type == mvt_linestring) {
+				lines++;
+			} else if (tile.layers[i].features[j].type == mvt_polygon) {
+				polygons++;
+			}
+		}
+
+		printf(": { \"points\": %d, \"lines\": %d, \"polygons\": %d, \"extent\": %lld }", points, lines, polygons, tile.layers[i].extent);
+	}
+
+	printf(" } }\n");
+}
+
+void handle(std::string message, int z, unsigned x, unsigned y, int describe, std::set<std::string> const &to_decode, bool pipeline, bool stats) {
 	mvt_tile tile;
 	bool was_compressed;
 
@@ -37,6 +64,11 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st
 		exit(EXIT_FAILURE);
 	}
 
+	if (stats) {
+		do_stats(tile, message.size(), was_compressed, z, x, y);
+		return;
+	}
+
 	if (!pipeline) {
 		printf("{ \"type\": \"FeatureCollection\"");
 
@@ -96,7 +128,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st
 	}
 }
 
-void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline) {
+void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats) {
 	sqlite3 *db;
 	int oz = z;
 	unsigned ox = x, oy = y;
@@ -111,7 +143,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 					if (strcmp(map, "SQLite format 3") != 0) {
 						if (z >= 0) {
 							std::string s = std::string(map, st.st_size);
-							handle(s, z, x, y, 1, to_decode, pipeline);
+							handle(s, z, x, y, 1, to_decode, pipeline, stats);
 							munmap(map, st.st_size);
 							return;
 						} else {
@@ -141,7 +173,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 	if (z < 0) {
 		int within = 0;
 
-		if (!pipeline) {
+		if (!pipeline && !stats) {
 			printf("{ \"type\": \"FeatureCollection\", \"properties\": {\n");
 
 			const char *sql2 = "SELECT name, value from metadata order by name;";
@@ -168,6 +200,10 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 			sqlite3_finalize(stmt2);
 		}
 
+		if (stats) {
+			printf("[\n");
+		}
+
 		const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;";
 		sqlite3_stmt *stmt;
 		if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
@@ -178,13 +214,19 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 		sqlite3_bind_int(stmt, 1, minzoom);
 		sqlite3_bind_int(stmt, 2, maxzoom);
 
-		if (!pipeline) {
+		if (!pipeline && !stats) {
 			printf("\n}, \"features\": [\n");
 		}
 
 		within = 0;
 		while (sqlite3_step(stmt) == SQLITE_ROW) {
-			if (!pipeline) {
+			if (!pipeline && !stats) {
+				if (within) {
+					printf(",\n");
+				}
+				within = 1;
+			}
+			if (stats) {
 				if (within) {
 					printf(",\n");
 				}
@@ -198,12 +240,15 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 			ty = (1LL << tz) - 1 - ty;
 			const char *s = (const char *) sqlite3_column_blob(stmt, 0);
 
-			handle(std::string(s, len), tz, tx, ty, 1, to_decode, pipeline);
+			handle(std::string(s, len), tz, tx, ty, 1, to_decode, pipeline, stats);
 		}
 
-		if (!pipeline) {
+		if (!pipeline && !stats) {
 			printf("] }\n");
 		}
+		if (stats) {
+			printf("]\n");
+		}
 
 		sqlite3_finalize(stmt);
 	} else {
@@ -228,7 +273,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 					fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy);
 				}
 
-				handle(std::string(s, len), z, x, y, 0, to_decode, pipeline);
+				handle(std::string(s, len), z, x, y, 0, to_decode, pipeline, stats);
 				handled = 1;
 			}
 
@@ -257,6 +302,7 @@ int main(int argc, char **argv) {
 	int i;
 	std::set<std::string> to_decode;
 	bool pipeline = false;
+	bool stats = false;
 
 	struct option long_options[] = {
 		{"projection", required_argument, 0, 's'},
@@ -264,6 +310,7 @@ int main(int argc, char **argv) {
 		{"minimum-zoom", required_argument, 0, 'Z'},
 		{"layer", required_argument, 0, 'l'},
 		{"tag-layer-and-zoom", no_argument, 0, 'c'},
+		{"stats", no_argument, 0, 'S'},
 		{"force", no_argument, 0, 'f'},
 		{0, 0, 0, 0},
 	};
@@ -304,6 +351,10 @@ int main(int argc, char **argv) {
 			pipeline = true;
 			break;
 
+		case 'S':
+			stats = true;
+			break;
+
 		case 'f':
 			force = true;
 			break;
@@ -314,9 +365,9 @@ int main(int argc, char **argv) {
 	}
 
 	if (argc == optind + 4) {
-		decode(argv[optind], atoi(argv[optind + 1]), atoi(argv[optind + 2]), atoi(argv[optind + 3]), to_decode, pipeline);
+		decode(argv[optind], atoi(argv[optind + 1]), atoi(argv[optind + 2]), atoi(argv[optind + 3]), to_decode, pipeline, stats);
 	} else if (argc == optind + 1) {
-		decode(argv[optind], -1, -1, -1, to_decode, pipeline);
+		decode(argv[optind], -1, -1, -1, to_decode, pipeline, stats);
 	} else {
 		usage(argv);
 	}

main.cpp

@@ -2206,6 +2206,7 @@ int main(int argc, char **argv) {
 		{"drop-densest-as-needed", no_argument, &additional[A_DROP_DENSEST_AS_NEEDED], 1},
 		{"drop-fraction-as-needed", no_argument, &additional[A_DROP_FRACTION_AS_NEEDED], 1},
 		{"drop-smallest-as-needed", no_argument, &additional[A_DROP_SMALLEST_AS_NEEDED], 1},
+		{"coalesce-smallest-as-needed", no_argument, &additional[A_COALESCE_SMALLEST_AS_NEEDED], 1},
 		{"force-feature-limit", no_argument, &prevent[P_DYNAMIC_DROP], 1},
 
 		{"Dropping tightly overlapping features", 0, 0, 0},

man/tippecanoe.1

@@ -237,6 +237,8 @@ compensate for the larger marker, or \fB\fC\-Bf\fR\fInumber\fP to allow at most
 .IP \(bu 2
 \fB\fC\-an\fR or \fB\fC\-\-drop\-smallest\-as\-needed\fR: Dynamically drop the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level to keep large tiles under the 500K size limit. This option will not work for point features.
 .IP \(bu 2
+\fB\fC\-aN\fR or \fB\fC\-\-coalesce\-smallest\-as\-needed\fR: Dynamically combine the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level into other nearby features to keep large tiles under the 500K size limit. This option will not work for point features, and will probably not help very much with LineStrings. It is mostly intended for polygons, to maintain the full original area covered by polygons while still reducing the feature count somehow. The attributes of the small polygons are \fInot\fP preserved into the combined features, only their geometry.
+.IP \(bu 2
 \fB\fC\-pd\fR or \fB\fC\-\-force\-feature\-limit\fR: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. (This is like \fB\fC\-ad\fR but applies to each tile individually, not to the entire zoom level.) You probably don't want to use this.
 .RE
 .SS Dropping tightly overlapping features
@@ -726,5 +728,7 @@ resolutions.
 .IP \(bu 2
 \fB\fC\-c\fR or \fB\fC\-\-tag\-layer\-and\-zoom\fR: Include each feature's layer and zoom level as part of its \fB\fCtippecanoe\fR object rather than as a FeatureCollection wrapper
 .IP \(bu 2
+\fB\fC\-S\fR or \fB\fC\-\-stats\fR: Just report statistics about each tile's size and the number of features in it, as a JSON structure.
+.IP \(bu 2
 \fB\fC\-f\fR or \fB\fC\-\-force\fR: Decode tiles even if polygon ring order or closure problems are detected
 .RE

options.hpp

@@ -15,6 +15,7 @@
 #define A_DROP_DENSEST_AS_NEEDED ((int) 's')
 #define A_DROP_FRACTION_AS_NEEDED ((int) 'd')
 #define A_DROP_SMALLEST_AS_NEEDED ((int) 'n')
+#define A_COALESCE_SMALLEST_AS_NEEDED ((int) 'N')
 #define A_GRID_LOW_ZOOMS ((int) 'L')
 #define A_DETECT_WRAPAROUND ((int) 'w')
 #define A_EXTEND_ZOOMS ((int) 'e')

serial.cpp

@@ -447,7 +447,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
 	}
 
 	double extent = 0;
-	if (additional[A_DROP_SMALLEST_AS_NEEDED]) {
+	if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
 		if (sf.t == VT_POLYGON) {
 			for (size_t i = 0; i < sf.geometry.size(); i++) {
 				if (sf.geometry[i].op == VT_MOVETO) {

tests/muni/decode/multi.mbtiles.stats.json

@@ -0,0 +1,9 @@
+[
+{ "zoom": 11, "x": 326, "y": 791, "bytes": 372, "compressed": true, "layers": { "muni": { "points": 14, "lines": 0, "polygons": 0, "extent": 4096 } } }
+,
+{ "zoom": 11, "x": 327, "y": 792, "bytes": 6481, "compressed": true, "layers": { "muni": { "points": 528, "lines": 0, "polygons": 0, "extent": 4096 } } }
+,
+{ "zoom": 11, "x": 327, "y": 791, "bytes": 44376, "compressed": true, "layers": { "muni": { "points": 4285, "lines": 0, "polygons": 0, "extent": 4096 }, "subway": { "points": 19, "lines": 0, "polygons": 0, "extent": 4096 } } }
+,
+{ "zoom": 11, "x": 954, "y": 791, "bytes": 75, "compressed": true, "layers": { "muni": { "points": 12, "lines": 0, "polygons": 0, "extent": 4096 } } }
+]

tile.cpp

@@ -1449,6 +1449,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 		std::map<std::string, std::vector<coalesce>> layers;
 		std::vector<unsigned long long> indices;
 		std::vector<long long> extents;
+		std::vector<serial_feature> coalesced_geometry;
 
 		int within[child_shards];
 		long long geompos[child_shards];
@@ -1545,6 +1546,13 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				}
 			}
 
+			double coalesced_area = 0;
+			for (size_t i = 0; i < coalesced_geometry.size(); i++) {
+				if (coalesced_geometry[i].t == sf.t) {
+					coalesced_area += coalesced_geometry[i].extent;
+				}
+			}
+
 			if (additional[A_DROP_DENSEST_AS_NEEDED]) {
 				indices.push_back(sf.index);
 				if (sf.index - merge_previndex < mingap) {
@@ -1553,10 +1561,28 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 			}
 			if (additional[A_DROP_SMALLEST_AS_NEEDED]) {
 				extents.push_back(sf.extent);
-				if (sf.extent <= minextent && sf.t != VT_POINT) {
+				if (sf.extent + coalesced_area <= minextent && sf.t != VT_POINT) {
 					continue;
 				}
 			}
+			if (additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
+				extents.push_back(sf.extent);
+				if (sf.extent + coalesced_area <= minextent) {
+					coalesced_geometry.push_back(sf);
+					continue;
+				}
+			}
+
+			if (coalesced_geometry.size() != 0) {
+				for (ssize_t i = coalesced_geometry.size() - 1; i >= 0; i--) {
+					if (coalesced_geometry[i].t == sf.t && coalesced_geometry[i].layer == sf.layer) {
+						for (size_t j = 0; j < coalesced_geometry[i].geometry.size(); j++) {
+							sf.geometry.push_back(coalesced_geometry[i].geometry[j]);
+						}
+						coalesced_geometry.erase(coalesced_geometry.begin() + i);
+					}
+				}
+			}
 
 			if (additional[A_CALCULATE_FEATURE_DENSITY]) {
 				// Gamma is always 1 for this calculation so there is a reasonable
@@ -1613,6 +1639,20 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 			merge_previndex = sf.index;
 		}
 
+		// Attach any pieces that were waiting to be coalesced onto some features that did make it.
+		for (ssize_t i = coalesced_geometry.size() - 1; i >= 0; i--) {
+			for (size_t j = 0; j < partials.size(); j++) {
+				if (partials[j].layer == coalesced_geometry[i].layer && partials[j].t == coalesced_geometry[i].t) {
+					for (size_t k = 0; k < coalesced_geometry[i].geometry.size(); k++) {
+						partials[j].geoms[0].push_back(coalesced_geometry[i].geometry[k]);
+					}
+
+					coalesced_geometry.erase(coalesced_geometry.begin() + i);
+					break;
+				}
+			}
+		}
+
 		if (prefilter != NULL) {
 			json_end(prefilter_jp);
 			if (fclose(prefilter_read_fp) != 0) {
@@ -1916,7 +1956,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 					}
 					line_detail++;
 					continue;
-				} else if (additional[A_DROP_SMALLEST_AS_NEEDED]) {
+				} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
 					minextent_fraction = minextent_fraction * 200000.0 / totalsize * 0.90;
 					long long m = choose_minextent(extents, minextent_fraction);
 					if (m != minextent) {
@@ -1999,7 +2039,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 						fprintf(stderr, "Going to try keeping the sparsest %0.2f%% of the features to make it fit\n", mingap_fraction * 100.0);
 					}
 					line_detail++;
-				} else if (additional[A_DROP_SMALLEST_AS_NEEDED]) {
+				} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
 					minextent_fraction = minextent_fraction * max_tile_size / compressed.size() * 0.90;
 					long long m = choose_minextent(extents, minextent_fraction);
 					if (m != minextent) {
@@ -2290,7 +2330,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
 		int err = INT_MAX;
 
 		size_t start = 1;
-		if (additional[A_INCREASE_GAMMA_AS_NEEDED] || additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_DROP_SMALLEST_AS_NEEDED]) {
+		if (additional[A_INCREASE_GAMMA_AS_NEEDED] || additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
 			start = 0;
 		}
 

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "tippecanoe v1.24.1\n"
+#define VERSION "tippecanoe v1.25.0\n"
 
 #endif