Add --progress-interval setting to reduce progress indicator frequency

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.27.10
+
+* Add --progress-interval setting to reduce progress indicator frequency
+
 ## 1.27.9
 
 * Make clusters look better by averaging locations of clustered points

README.md

@@ -288,6 +288,7 @@ tippecanoe -z5 -o filtered.mbtiles -j '{ "ne_10m_admin_0_countries": [ "all", [
 
  * `-q` or `--quiet`: Work quietly instead of reporting progress or warning messages
  * `-Q` or `--no-progress-indicator`: Don't report progress, but still give warnings
+ * `-U` _seconds_ or `--progress-interval=`_seconds_: Don't report progress more often than the specified number of _seconds_.
  * `-v` or `--version`: Report Tippecanoe's version number
 
 ### Filters

main.cpp

@@ -25,6 +25,7 @@
 #include <pthread.h>
 #include <getopt.h>
 #include <signal.h>
+#include <sys/time.h>
 #include <algorithm>
 #include <vector>
 #include <string>
@@ -66,6 +67,8 @@ static int min_detail = 7;
 
 int quiet = 0;
 int quiet_progress = 0;
+double progress_interval = 0;
+volatile double last_progress = 0;
 int geometry_scale = 0;
 double simplification = 1;
 size_t max_tile_size = 500000;
@@ -293,6 +296,8 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map,
 		}
 	}
 
+	last_progress = 0;
+
 	while (head != NULL) {
 		struct index ix = *((struct index *) (map + head->start));
 		long long pos = *geompos;
@@ -303,7 +308,7 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map,
 
 		// Count this as an 75%-accomplishment, since we already 25%-counted it
 		*progress += (ix.end - ix.start) * 3 / 4;
-		if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) {
+		if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) {
 			fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
 			*progress_reported = 100 * *progress / *progress_max;
 		}
@@ -673,7 +678,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
 
 				// Count this as a 25%-accomplishment, since we will copy again
 				*progress += (ix.end - ix.start) / 4;
-				if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) {
+				if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) {
 					fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
 					*progress_reported = 100 * *progress / *progress_max;
 				}
@@ -845,7 +850,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
 
 					// Count this as an 75%-accomplishment, since we already 25%-counted it
 					*progress += (ix.end - ix.start) * 3 / 4;
-					if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) {
+					if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) {
 						fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
 						*progress_reported = 100 * *progress / *progress_max;
 					}
@@ -1819,6 +1824,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 	long long indexpos = indexst.st_size;
 	progress_seq = indexpos / sizeof(struct index);
 
+	last_progress = 0;
 	if (!quiet) {
 		fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", progress_seq, geompos, metapos, poolpos);
 	}
@@ -1856,7 +1862,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 			long long nprogress = 100 * ip / indices;
 			if (nprogress != progress) {
 				progress = nprogress;
-				if (!quiet && !quiet_progress) {
+				if (!quiet && !quiet_progress && progress_time()) {
 					fprintf(stderr, "Maxzoom: %lld%% \r", progress);
 				}
 			}
@@ -1952,7 +1958,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 			long long nprogress = 100 * ip / indices;
 			if (nprogress != progress) {
 				progress = nprogress;
-				if (!quiet && !quiet_progress) {
+				if (!quiet && !quiet_progress && progress_time()) {
 					fprintf(stderr, "Base zoom/drop rate: %lld%% \r", progress);
 				}
 			}
@@ -2436,6 +2442,7 @@ int main(int argc, char **argv) {
 		{"Progress indicator", 0, 0, 0},
 		{"quiet", no_argument, 0, 'q'},
 		{"no-progress-indicator", no_argument, 0, 'Q'},
+		{"progress-interval", required_argument, 0, 'U'},
 		{"version", no_argument, 0, 'v'},
 
 		{"", 0, 0, 0},
@@ -2686,6 +2693,10 @@ int main(int argc, char **argv) {
 			quiet_progress = 1;
 			break;
 
+		case 'U':
+			progress_interval = atof_require(optarg, "Progress interval");
+			break;
+
 		case 'p': {
 			char *cp;
 			for (cp = optarg; *cp != '\0'; cp++) {
@@ -2937,3 +2948,25 @@ FILE *fopen_oflag(const char *name, const char *mode, int oflag) {
 	}
 	return fdopen(fd, mode);
 }
+
+bool progress_time() {
+	if (progress_interval == 0.0) {
+		return true;
+	}
+
+	struct timeval tv;
+	double now;
+	if (gettimeofday(&tv, NULL) != 0) {
+		fprintf(stderr, "%s: Can't get the time of day: %s\n", *av, strerror(errno));
+		now = 0;
+	} else {
+		now = tv.tv_sec + tv.tv_usec / 1000000.0;
+	}
+
+	if (now - last_progress >= progress_interval) {
+		last_progress = now;
+		return true;
+	} else {
+		return false;
+	}
+}

main.hpp

@@ -22,6 +22,8 @@ void checkdisk(std::vector<struct reader> *r);
 extern int geometry_scale;
 extern int quiet;
 extern int quiet_progress;
+extern double progress_interval;
+extern volatile double last_progress;
 
 extern size_t CPUS;
 extern size_t TEMP_FILES;
@@ -32,6 +34,7 @@ extern int cluster_distance;
 
 int mkstemp_cloexec(char *name);
 FILE *fopen_oflag(const char *name, const char *mode, int oflag);
+bool progress_time();
 
 #define MAX_ZOOM 24
 

man/tippecanoe.1

@@ -355,6 +355,8 @@ If you don't specify, it will use \fB\fC/tmp\fR\&.
 .IP \(bu 2
 \fB\fC\-Q\fR or \fB\fC\-\-no\-progress\-indicator\fR: Don't report progress, but still give warnings
 .IP \(bu 2
+\fB\fC\-U\fR \fIseconds\fP or \fB\fC\-\-progress\-interval=\fR\fIseconds\fP: Don't report progress more often than the specified number of \fIseconds\fP\&.
+.IP \(bu 2
 \fB\fC\-v\fR or \fB\fC\-\-version\fR: Report Tippecanoe's version number
 .RE
 .SS Filters

serial.cpp

@@ -633,7 +633,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
 
 	if (*(sst->progress_seq) % 10000 == 0) {
 		checkdisk(sst->readers);
-		if (!quiet && !quiet_progress) {
+		if (!quiet && !quiet_progress && progress_time()) {
 			fprintf(stderr, "Read %.2f million features\r", *sst->progress_seq / 1000000.0);
 		}
 	}

tile.cpp

@@ -1298,7 +1298,7 @@ serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase,
 
 		double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10;
 		if (progress >= *oprogress + 0.1) {
-			if (!quiet && !quiet_progress) {
+			if (!quiet && !quiet_progress && progress_time()) {
 				fprintf(stderr, "  %3.1f%%  %d/%u/%u  \r", progress, z, tx, ty);
 			}
 			*oprogress = progress;
@@ -2163,7 +2163,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 
 		double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10;
 		if (progress >= oprogress + 0.1) {
-			if (!quiet && !quiet_progress) {
+			if (!quiet && !quiet_progress && progress_time()) {
 				fprintf(stderr, "  %3.1f%%  %d/%u/%u  \r", progress, z, tx, ty);
 			}
 			oprogress = progress;
@@ -2477,6 +2477,8 @@ void *run_thread(void *vargs) {
 }
 
 int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum) {
+	last_progress = 0;
+
 	// The existing layermaps are one table per input thread.
 	// We need to add another one per *tiling* thread so that it can be
 	// safely changed during tiling.

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "tippecanoe v1.27.9\n"
+#define VERSION "tippecanoe v1.27.10\n"
 
 #endif