Merge pull request #397 from mapbox/guess-maxzoom

Add -zg option to automatically choose an appropriate maxzoom

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.17.0
+
+* Add -zg option to guess an appropriate maxzoom
+
 ## 1.16.17
 
 * Clean up JSON parsing at the end of each FeatureCollection

README.md

@@ -59,6 +59,9 @@ There are a lot of options. A lot of the time you won't want to use any of them
 other than `-o` _output_`.mbtiles` to name the output file, and probably `-f` to
 delete the file that already exists with that name.
 
+If you aren't sure what the right maxzoom is for your data, `-zg` will guess one for you
+based on the density of features.
+
 If you are mapping point features, you will often want to use `-Bg` to automatically choose
 a base zoom level for dot dropping. If that doesn't work out for you, try
 `-r1 --drop-fraction-as-needed` to turn off the normal dot dropping and instead
@@ -108,6 +111,7 @@ If your input is formatted as newline-delimited GeoJSON, use `-P` to make input
 ### Zoom levels
 
  * `-z` _zoom_ or `--maximum-zoom=`_zoom_: Maxzoom: the highest zoom level for which tiles are generated (default 14)
+ * `-zg` or `--maximum-zoom=g`: Guess what is probably a reasonable maxzoom based on the spacing of features.
  * `-Z` _zoom_ or `--minimum-zoom=`_zoom_: Minzoom: the lowest zoom level for which tiles are generated (default 0)
 
 ### Tile resolution

main.cpp

@@ -1006,7 +1006,7 @@ void choose_first_zoom(long long *file_bbox, struct reader *reader, unsigned *iz
 	}
 }
 
-int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description) {
+int read_input(std::vector<source> &sources, char *fname, int &maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description, bool guess_maxzoom) {
 	int ret = EXIT_SUCCESS;
 
 	struct reader reader[CPUS];
@@ -1555,15 +1555,75 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 		exit(EXIT_FAILURE);
 	}
 
-	if (basezoom < 0 || droprate < 0) {
-		struct index *map = (struct index *) mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0);
-		if (map == MAP_FAILED) {
-			perror("mmap index for basezoom");
+	struct index *map = (struct index *) mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0);
+	if (map == MAP_FAILED) {
+		perror("mmap index for basezoom");
+		exit(EXIT_FAILURE);
+	}
+	madvise(map, indexpos, MADV_SEQUENTIAL);
+	madvise(map, indexpos, MADV_WILLNEED);
+	long long indices = indexpos / sizeof(struct index);
+	bool fix_dropping = false;
+
+	if (guess_maxzoom) {
+		double sum = 0;
+		size_t count = 0;
+
+		long long progress = -1;
+		long long ip;
+		for (ip = 1; ip < indices; ip++) {
+			if (map[ip].index != map[ip - 1].index) {
+				count++;
+				sum += log(map[ip].index - map[ip - 1].index);
+			}
+
+			long long nprogress = 100 * ip / indices;
+			if (nprogress != progress) {
+				progress = nprogress;
+				if (!quiet) {
+					fprintf(stderr, "Maxzoom: %lld%% \r", progress);
+				}
+			}
+		}
+
+		if (count > 0) {
+			// Geometric mean is appropriate because distances between features
+			// are typically lognormally distributed
+			double avg = exp(sum / count);
+
+			// Convert approximately from tile units to feet
+			double dist_ft = sqrt(avg) / 33;
+			double want = dist_ft / 250;
+
+			maxzoom = ceil(log(360 / (.00000274 * want)) / log(2) - full_detail);
+			if (maxzoom < 0) {
+				maxzoom = 0;
+			}
+			if (maxzoom > MAX_ZOOM) {
+				maxzoom = MAX_ZOOM;
+			}
+
+			if (!quiet) {
+				fprintf(stderr, "Choosing a maxzoom of -z%d for features about %d feet apart\n", maxzoom, (int) ceil(dist_ft));
+			}
+		} else {
+			fprintf(stderr, "Can't guess maxzoom (-zg) without at least two distinct feature locations\n");
 			exit(EXIT_FAILURE);
 		}
-		madvise(map, indexpos, MADV_SEQUENTIAL);
-		madvise(map, indexpos, MADV_WILLNEED);
 
+		if (maxzoom < minzoom) {
+			fprintf(stderr, "Can't use %d for maxzoom because minzoom is %d\n", maxzoom, minzoom);
+			maxzoom = minzoom;
+		}
+
+		fix_dropping = true;
+
+		if (basezoom == -1) {
+			basezoom = maxzoom;
+		}
+	}
+
+	if (basezoom < 0 || droprate < 0) {
 		struct tile {
 			unsigned x;
 			unsigned y;
@@ -1583,7 +1643,6 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 
 		long long progress = -1;
 
-		long long indices = indexpos / sizeof(struct index);
 		long long ip;
 		for (ip = 0; ip < indices; ip++) {
 			unsigned xx, yy;
@@ -1724,6 +1783,10 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 			}
 		}
 
+		fix_dropping = true;
+	}
+
+	if (fix_dropping) {
 		// Fix up the minzooms for features, now that we really know the base zoom
 		// and drop rate.
 
@@ -1743,7 +1806,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 		struct drop_state ds[maxzoom + 1];
 		prep_drop_states(ds, maxzoom, basezoom, droprate);
 
-		for (ip = 0; ip < indices; ip++) {
+		for (long long ip = 0; ip < indices; ip++) {
 			if (ip > 0 && map[ip].start != map[ip - 1].end) {
 				fprintf(stderr, "Mismatched index at %lld: %lld vs %lld\n", ip, map[ip].start, map[ip].end);
 			}
@@ -1752,10 +1815,11 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 		}
 
 		munmap(geom, geomst.st_size);
-		madvise(map, indexpos, MADV_DONTNEED);
-		munmap(map, indexpos);
 	}
 
+	madvise(map, indexpos, MADV_DONTNEED);
+	munmap(map, indexpos);
+
 	if (close(indexfd) != 0) {
 		perror("close sorted index");
 	}
@@ -1889,6 +1953,7 @@ int main(int argc, char **argv) {
 	const char *tmpdir = "/tmp";
 	const char *attribution = NULL;
 	std::vector<source> sources;
+	bool guess_maxzoom = false;
 
 	std::set<std::string> exclude, include;
 	int exclude_all = 0;
@@ -2017,7 +2082,12 @@ int main(int argc, char **argv) {
 		} break;
 
 		case 'z':
-			maxzoom = atoi(optarg);
+			if (strcmp(optarg, "g") == 0) {
+				maxzoom = MAX_ZOOM;
+				guess_maxzoom = true;
+			} else {
+				maxzoom = atoi(optarg);
+			}
 			break;
 
 		case 'Z':
@@ -2217,9 +2287,11 @@ int main(int argc, char **argv) {
 
 	// Need two checks: one for geometry representation, the other for
 	// index traversal when guessing base zoom and drop rate
-	if (maxzoom > 32 - full_detail) {
-		maxzoom = 32 - full_detail;
-		fprintf(stderr, "Highest supported zoom with detail %d is %d\n", full_detail, maxzoom);
+	if (!guess_maxzoom) {
+		if (maxzoom > 32 - full_detail) {
+			maxzoom = 32 - full_detail;
+			fprintf(stderr, "Highest supported zoom with detail %d is %d\n", full_detail, maxzoom);
+		}
 	}
 	if (maxzoom > MAX_ZOOM) {
 		maxzoom = MAX_ZOOM;
@@ -2232,13 +2304,17 @@ int main(int argc, char **argv) {
 	}
 
 	if (basezoom == -1) {
-		basezoom = maxzoom;
+		if (!guess_maxzoom) {
+			basezoom = maxzoom;
+		}
 	}
 
 	geometry_scale = 32 - (full_detail + maxzoom);
 	if (geometry_scale < 0) {
 		geometry_scale = 0;
-		fprintf(stderr, "Full detail + maxzoom > 32, so you are asking for more detail than is available.\n");
+		if (!guess_maxzoom) {
+			fprintf(stderr, "Full detail + maxzoom > 32, so you are asking for more detail than is available.\n");
+		}
 	}
 
 	if ((basezoom < 0 || droprate < 0) && (gamma < 0)) {
@@ -2290,7 +2366,7 @@ int main(int argc, char **argv) {
 
 	long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0};
 
-	ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_directory, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_directory, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description);
+	ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_directory, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_directory, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description, guess_maxzoom);
 
 	if (outdb != NULL) {
 		mbtiles_close(outdb, argv);

man/tippecanoe.1

@@ -54,6 +54,9 @@ There are a lot of options. A lot of the time you won't want to use any of them
 other than \fB\fC\-o\fR \fIoutput\fP\fB\fC\&.mbtiles\fR to name the output file, and probably \fB\fC\-f\fR to
 delete the file that already exists with that name.
 .PP
+If you aren't sure what the right maxzoom is for your data, \fB\fC\-zg\fR will guess one for you
+based on the density of features.
+.PP
 If you are mapping point features, you will often want to use \fB\fC\-Bg\fR to automatically choose
 a base zoom level for dot dropping. If that doesn't work out for you, try
 \fB\fC\-r1 \-\-drop\-fraction\-as\-needed\fR to turn off the normal dot dropping and instead
@@ -116,6 +119,8 @@ rather than a stream that can only be read sequentially.
 .IP \(bu 2
 \fB\fC\-z\fR \fIzoom\fP or \fB\fC\-\-maximum\-zoom=\fR\fIzoom\fP: Maxzoom: the highest zoom level for which tiles are generated (default 14)
 .IP \(bu 2
+\fB\fC\-zg\fR or \fB\fC\-\-maximum\-zoom=g\fR: Guess what is probably a reasonable maxzoom based on the spacing of features.
+.IP \(bu 2
 \fB\fC\-Z\fR \fIzoom\fP or \fB\fC\-\-minimum\-zoom=\fR\fIzoom\fP: Minzoom: the lowest zoom level for which tiles are generated (default 0)
 .RE
 .SS Tile resolution

version.hpp

@@ -1 +1 @@
-#define VERSION "tippecanoe v1.16.17\n"
+#define VERSION "tippecanoe v1.17.0\n"