Merge pull request #210 from mapbox/block-input

Don't let the input buffer for parallel streaming input get too big.

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.9.12
+
+* Limit the size of the parallel parsing streaming input buffer
+* Add an option to set the tileset's attribution
+
 ## 1.9.11
 
 * Fix a line simplification crash when a segment degenerates to a single point

README.md

@@ -61,6 +61,7 @@ Options
    specified, the files are all merged into the single named layer, even if they try to specify individual names with -L.
  * -L _name_:_file.json_ or --named-layer=_name_:_file.json_: Specify layer names for individual files. If your shell supports it, you can use a subshell redirect like -L _name_:<(cat dir/*.json) to specify a layer name for the output of streamed input.
  * -n _name_ or --name=_name_: Human-readable name for the tileset (default file.json)
+ * -A _text_ or --attribution=_text_: Attribution (HTML) to be shown with maps that use data from this tileset.
 
 ### File control
 

geojson.c

@@ -332,6 +332,56 @@ void deserialize_byte(char **f, signed char *n) {
 	*f += sizeof(signed char);
 }
 
+int deserialize_long_long_io(FILE *f, long long *n, long long *geompos) {
+	unsigned long long zigzag = 0;
+	int shift = 0;
+
+	while (1) {
+		int c = getc(f);
+		if (c == EOF) {
+			return 0;
+		}
+		(*geompos)++;
+
+		if ((c & 0x80) == 0) {
+			zigzag |= ((unsigned long long) c) << shift;
+			shift += 7;
+			break;
+		} else {
+			zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
+			shift += 7;
+		}
+	}
+
+	*n = (zigzag >> 1) ^ (-(zigzag & 1));
+	return 1;
+}
+
+int deserialize_int_io(FILE *f, int *n, long long *geompos) {
+	long long ll = 0;
+	int ret = deserialize_long_long_io(f, &ll, geompos);
+	*n = ll;
+	return ret;
+}
+
+int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos) {
+	if (fread(n, sizeof(unsigned), 1, f) != 1) {
+		return 0;
+	}
+	*geompos += sizeof(unsigned);
+	return 1;
+}
+
+int deserialize_byte_io(FILE *f, signed char *n, long long *geompos) {
+	int c = getc(f);
+	if (c == EOF) {
+		return 0;
+	}
+	*n = c;
+	(*geompos)++;
+	return 1;
+}
+
 struct index {
 	long long start;
 	long long end;
@@ -1498,7 +1548,7 @@ void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE
 	radix1(geomfds, indexfds, nreaders, 0, splits, mem, tmpdir, availfiles, geomfile, indexfile, geompos, &progress, &progress_max, &progress_reported);
 }
 
-int read_json(int argc, struct source **sourcelist, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int *prevent, int *additional, int read_parallel, int forcetable) {
+int read_json(int argc, struct source **sourcelist, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int *prevent, int *additional, int read_parallel, int forcetable, const char *attribution) {
 	int ret = EXIT_SUCCESS;
 
 	struct reader reader[CPUS];
@@ -1711,6 +1761,8 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay
 
 #define READ_BUF 2000
 #define PARSE_MIN 10000000
+#define PARSE_MAX (1LL * 1024 * 1024 * 1024)
+
 				char buf[READ_BUF];
 				int n;
 
@@ -1718,34 +1770,40 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay
 					fwrite_check(buf, sizeof(char), n, readfp, reading);
 					ahead += n;
 
-					if (buf[n - 1] == '\n' && ahead > PARSE_MIN && is_parsing == 0) {
+					if (buf[n - 1] == '\n' && ahead > PARSE_MIN) {
-						if (initial_offset != 0) {
+						// Don't let the streaming reader get too far ahead of the parsers.
-							if (pthread_join(parallel_parser, NULL) != 0) {
+						// If the buffered input gets huge, even if the parsers are still running,
-								perror("pthread_join");
+						// wait for the parser thread instead of continuing to stream input.
+
+						if (is_parsing == 0 || ahead >= PARSE_MAX) {
+							if (initial_offset != 0) {
+								if (pthread_join(parallel_parser, NULL) != 0) {
+									perror("pthread_join");
+									exit(EXIT_FAILURE);
+								}
+							}
+
+							fflush(readfp);
+							start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y);
+
+							initial_offset += ahead;
+							overall_offset += ahead;
+							checkdisk(reader, CPUS);
+							ahead = 0;
+
+							sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX");
+							readfd = mkstemp(readname);
+							if (readfd < 0) {
+								perror(readname);
 								exit(EXIT_FAILURE);
 							}
+							readfp = fdopen(readfd, "w");
+							if (readfp == NULL) {
+								perror(readname);
+								exit(EXIT_FAILURE);
+							}
+							unlink(readname);
 						}
-
-						fflush(readfp);
-						start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y);
-
-						initial_offset += ahead;
-						overall_offset += ahead;
-						checkdisk(reader, CPUS);
-						ahead = 0;
-
-						sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX");
-						readfd = mkstemp(readname);
-						if (readfd < 0) {
-							perror(readname);
-							exit(EXIT_FAILURE);
-						}
-						readfp = fdopen(readfd, "w");
-						if (readfp == NULL) {
-							perror(readname);
-							exit(EXIT_FAILURE);
-						}
-						unlink(readname);
 					}
 				}
 				if (n < 0) {
@@ -2339,7 +2397,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay
 		midlon = maxlon;
 	}
 
-	mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable);
+	mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable, attribution);
 
 	for (i = 0; i < nlayers; i++) {
 		pool_free_strings(&file_keys1[i]);
@@ -2385,6 +2443,7 @@ int main(int argc, char **argv) {
 	double gamma = 0;
 	int buffer = 5;
 	const char *tmpdir = "/tmp";
+	const char *attribution = NULL;
 
 	int nsources = 0;
 	struct source *sources = NULL;
@@ -2403,6 +2462,7 @@ int main(int argc, char **argv) {
 	static struct option long_options[] = {
 		{"name", required_argument, 0, 'n'},
 		{"layer", required_argument, 0, 'l'},
+		{"attribution", required_argument, 0, 'A'},
 		{"named-layer", required_argument, 0, 'L'},
 		{"maximum-zoom", required_argument, 0, 'z'},
 		{"minimum-zoom", required_argument, 0, 'Z'},
@@ -2445,7 +2505,7 @@ int main(int argc, char **argv) {
 		{0, 0, 0, 0},
 	};
 
-	while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:", long_options, NULL)) != -1) {
+	while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:", long_options, NULL)) != -1) {
 		switch (i) {
 		case 0:
 			break;
@@ -2458,6 +2518,10 @@ int main(int argc, char **argv) {
 			layer = optarg;
 			break;
 
+		case 'A':
+			attribution = optarg;
+			break;
+
 		case 'L': {
 			char *cp = strchr(optarg, ':');
 			if (cp == NULL || cp == optarg) {
@@ -2694,7 +2758,7 @@ int main(int argc, char **argv) {
 		sourcelist[i--] = sources;
 	}
 
-	ret = read_json(nsources, sourcelist, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel, forcetable);
+	ret = read_json(nsources, sourcelist, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel, forcetable, attribution);
 
 	mbtiles_close(outdb, argv);
 

geometry.cc

@@ -18,7 +18,7 @@ extern "C" {
 #include "projection.h"
 }
 
-drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y) {
+drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y) {
 	drawvec out;
 
 	bbox[0] = LLONG_MAX;
@@ -31,7 +31,7 @@ drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail
 	while (1) {
 		draw d;
 
-		deserialize_byte(meta, &d.op);
+		deserialize_byte_io(meta, &d.op, geompos);
 		if (d.op == VT_END) {
 			break;
 		}
@@ -39,8 +39,8 @@ drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail
 		if (d.op == VT_MOVETO || d.op == VT_LINETO) {
 			long long dx, dy;
 
-			deserialize_long_long(meta, &dx);
+			deserialize_long_long_io(meta, &dx, geompos);
-			deserialize_long_long(meta, &dy);
+			deserialize_long_long_io(meta, &dy, geompos);
 
 			wx += dx << geometry_scale;
 			wy += dy << geometry_scale;

geometry.hh

@@ -16,7 +16,7 @@ struct draw {
 
 typedef std::vector<draw> drawvec;
 
-drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y);
+drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y);
 void to_tile_scale(drawvec &geom, int z, int detail);
 drawvec remove_noop(drawvec geom, int type, int shift);
 drawvec clip_point(drawvec &geom, int z, int detail, long long buffer);

man/tippecanoe.1

@@ -65,6 +65,8 @@ specified, the files are all merged into the single named layer, even if they tr
 \-L \fIname\fP:\fIfile.json\fP or \-\-named\-layer=\fIname\fP:\fIfile.json\fP: Specify layer names for individual files. If your shell supports it, you can use a subshell redirect like \-L \fIname\fP:<(cat dir/*.json) to specify a layer name for the output of streamed input.
 .IP \(bu 2
 \-n \fIname\fP or \-\-name=\fIname\fP: Human\-readable name for the tileset (default file.json)
+.IP \(bu 2
+\-A \fItext\fP or \-\-attribution=\fItext\fP: Attribution (HTML) to be shown with maps that use data from this tileset.
 .RE
 .SS File control
 .RS

mbtiles.c

@@ -138,7 +138,7 @@ static int pvcmp(const void *v1, const void *v2) {
 	return (*pv1)->type - (*pv2)->type;
 }
 
-void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable) {
+void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable, const char *attribution) {
 	char *sql, *err;
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname);
@@ -213,6 +213,17 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername,
 	}
 	sqlite3_free(sql);
 
+	if (attribution != NULL) {
+		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", attribution);
+		if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+			fprintf(stderr, "set type: %s\n", err);
+			if (!forcetable) {
+				exit(EXIT_FAILURE);
+			}
+		}
+		sqlite3_free(sql);
+	}
+
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", "pbf");
 	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set format: %s\n", err);

mbtiles.h

@@ -2,6 +2,6 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
 
 void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
 
-void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable);
+void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable, const char *attribution);
 
 void mbtiles_close(sqlite3 *outdb, char **argv);

tests/nullisland/out/-b0_-z4_-ANullIsland.json

@@ -0,0 +1,251 @@
+{ "type": "FeatureCollection", "properties": {
+"attribution": "NullIsland",
+"bounds": "-1.000000,-1.000000,1.000000,1.000000",
+"center": "-1.000000,1.000000,4",
+"description": "tests/nullisland/out/-b0_-z4_-ANullIsland.json.check.mbtiles",
+"format": "pbf",
+"json": "{\"vector_layers\": [ { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 4, \"fields\": {} } ] }",
+"maxzoom": "4",
+"minzoom": "0",
+"name": "tests/nullisland/out/-b0_-z4_-ANullIsland.json.check.mbtiles",
+"type": "overlay",
+"version": "2"
+}, "features": [
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.054688, 0.000000 ], [ -1.054688, 1.054628 ], [ 0.000000, 1.054628 ], [ 0.000000, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ -1.054688, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.054688, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.966751 ], [ -1.054688, -0.966751 ], [ -1.054688, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.054628 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.966797, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.054628 ], [ 0.966797, 1.054628 ], [ 0.966797, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.966797, -0.966751 ], [ 0.000000, -0.966751 ], [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ], [ 0.966797, -0.966751 ] ] ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 0, "y": 1 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.966751 ], [ -1.010742, -0.966751 ], [ -1.010742, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ], [ -1.010742, 1.010690 ], [ 0.000000, 1.010690 ], [ 0.000000, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 1 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.966797, -0.966751 ], [ 0.000000, -0.966751 ], [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ], [ 0.966797, -0.966751 ] ] ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.010690 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.966797, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ], [ 0.966797, 1.010690 ], [ 0.966797, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ] ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 1, "y": 2 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.988720 ], [ -1.010742, -0.988720 ], [ -1.010742, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.988720 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.988720 ] ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 1, "y": 1 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ], [ -1.010742, 1.010690 ], [ 0.000000, 1.010690 ], [ 0.000000, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 2, "y": 2 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.988720 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.988770, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.988770, -0.988720 ], [ 0.000000, -0.988720 ], [ 0.000000, 0.000000 ], [ 0.988770, 0.000000 ], [ 0.988770, -0.988720 ] ] ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 2, "y": 1 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.010690 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.988770, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ], [ 0.988770, 1.010690 ], [ 0.988770, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.988770, 0.000000 ] ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 3, "y": 4 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.999705 ], [ -1.010742, -0.999705 ], [ -1.010742, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, -0.999705 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 3, "y": 3 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ], [ -1.010742, 1.010690 ], [ 0.000000, 1.010690 ], [ 0.000000, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 4, "y": 4 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.999756, -0.999705 ], [ 0.000000, -0.999705 ], [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ], [ 0.999756, -0.999705 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 0.000000 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 4, "y": 3 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.010690 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.999756, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ], [ 0.999756, 1.010690 ], [ 0.999756, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 0.000000 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 7, "y": 8 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ -1.005249, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.005249, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.999705 ], [ -1.005249, -0.999705 ], [ -1.005249, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, -0.999705 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 7, "y": 7 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.005249, 0.000000 ], [ -1.005249, 1.005197 ], [ 0.000000, 1.005197 ], [ 0.000000, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ -1.005249, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.005249, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.005197 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.005197 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 8, "y": 8 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.999756, -0.999705 ], [ 0.000000, -0.999705 ], [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ], [ 0.999756, -0.999705 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, -0.999705 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 0.000000 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 8, "y": 7 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.005197 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.005197 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.999756, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.005197 ], [ 0.999756, 1.005197 ], [ 0.999756, 0.000000 ] ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ] ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 0.000000 ] } }
+] }
+] }
+] }

tile-join.cc

@@ -377,7 +377,7 @@ double max(double a, double b) {
 	}
 }
 
-void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string> > &mapping, struct pool *exclude, int ifmatched) {
+void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string> > &mapping, struct pool *exclude, int ifmatched, char **attribution) {
 	sqlite3 *db;
 
 	if (sqlite3_open(fname, &db) != SQLITE_OK) {
@@ -429,6 +429,12 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames,
 		}
 		sqlite3_finalize(stmt);
 	}
+	if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'attribution'", -1, &stmt, NULL) == SQLITE_OK) {
+		if (sqlite3_step(stmt) == SQLITE_ROW) {
+			*attribution = strdup((char *) sqlite3_column_text(stmt, 0));
+		}
+		sqlite3_finalize(stmt);
+	}
 	if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'bounds'", -1, &stmt, NULL) == SQLITE_OK) {
 		if (sqlite3_step(stmt) == SQLITE_ROW) {
 			const unsigned char *s = sqlite3_column_text(stmt, 0);
@@ -593,9 +599,10 @@ int main(int argc, char **argv) {
 	struct pool *file_keys = NULL;
 	char **layernames = NULL;
 	int nlayers = 0;
+	char *attribution = NULL;
 
 	for (i = optind; i < argc; i++) {
-		decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping, &exclude, ifmatched);
+		decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping, &exclude, ifmatched, &attribution);
 	}
 
 	struct pool *fk[nlayers];
@@ -603,7 +610,7 @@ int main(int argc, char **argv) {
 		fk[i] = &(file_keys[i]);
 	}
 
-	mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, fk, nlayers, 0);
+	mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, fk, nlayers, 0, attribution);
 	mbtiles_close(outdb, argv);
 
 	return 0;

tile.cc

@@ -596,11 +596,11 @@ int manage_gap(unsigned long long index, unsigned long long *previndex, double s
 	return 0;
 }
 
-long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, int *prevent, int *additional, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) {
+long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, double gamma, int nlayers, int *prevent, int *additional, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) {
 	int line_detail;
 	double fraction = 1;
 
-	char *og = *geoms;
+	long long og = *geompos_in;
 
 	// XXX is there a way to do this without floating point?
 	int max_zoom_increment = log(child_shards) / log(4);
@@ -669,45 +669,51 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsi
 		memset(within, '\0', sizeof(within));
 		memset(geompos, '\0', sizeof(geompos));
 
-		*geoms = og;
+		if (*geompos_in != og) {
+			if (fseek(geoms, og, SEEK_SET) != 0) {
+				perror("fseek geom");
+				exit(EXIT_FAILURE);
+			}
+			*geompos_in = og;
+		}
 
 		while (1) {
 			signed char t;
-			deserialize_byte(geoms, &t);
+			deserialize_byte_io(geoms, &t, geompos_in);
 			if (t < 0) {
 				break;
 			}
 
 			long long original_seq;
-			deserialize_long_long(geoms, &original_seq);
+			deserialize_long_long_io(geoms, &original_seq, geompos_in);
 
 			long long layer;
-			deserialize_long_long(geoms, &layer);
+			deserialize_long_long_io(geoms, &layer, geompos_in);
 			int tippecanoe_minzoom = -1, tippecanoe_maxzoom = -1;
 			if (layer & 2) {
-				deserialize_int(geoms, &tippecanoe_minzoom);
+				deserialize_int_io(geoms, &tippecanoe_minzoom, geompos_in);
 			}
 			if (layer & 1) {
-				deserialize_int(geoms, &tippecanoe_maxzoom);
+				deserialize_int_io(geoms, &tippecanoe_maxzoom, geompos_in);
 			}
 			layer >>= 2;
 
 			int segment;
-			deserialize_int(geoms, &segment);
+			deserialize_int_io(geoms, &segment, geompos_in);
 
 			long long metastart;
 			int m;
-			deserialize_long_long(geoms, &metastart);
+			deserialize_long_long_io(geoms, &metastart, geompos_in);
-			deserialize_int(geoms, &m);
+			deserialize_int_io(geoms, &m, geompos_in);
 			char *meta = metabase + metastart + meta_off[segment];
 			long long bbox[4];
 
-			drawvec geom = decode_geometry(geoms, z, tx, ty, line_detail, bbox, initial_x[segment], initial_y[segment]);
+			drawvec geom = decode_geometry(geoms, geompos_in, z, tx, ty, line_detail, bbox, initial_x[segment], initial_y[segment]);
 
 			signed char feature_minzoom;
-			deserialize_byte(geoms, &feature_minzoom);
+			deserialize_byte_io(geoms, &feature_minzoom, geompos_in);
 
-			double progress = floor((((*geoms - geomstart + *along) / (double) todo) + z) / (maxzoom + 1) * 1000) / 10;
+			double progress = floor((((*geompos_in + *along) / (double) todo) + z) / (maxzoom + 1) * 1000) / 10;
 			if (progress >= oprogress + 0.1) {
 				if (!quiet) {
 					fprintf(stderr, "  %3.1f%%  %d/%u/%u  \r", progress, z, tx, ty);
@@ -1089,29 +1095,28 @@ void *run_thread(void *vargs) {
 
 		// printf("%lld of geom_size\n", (long long) geom_size[j]);
 
-		char *geom = (char *) mmap(NULL, arg->geom_size[j], PROT_READ, MAP_PRIVATE, arg->geomfd[j], 0);
+		FILE *geom = fdopen(arg->geomfd[j], "rb");
-		if (geom == MAP_FAILED) {
+		if (geom == NULL) {
 			perror("mmap geom");
 			exit(EXIT_FAILURE);
 		}
-		madvise(geom, arg->geom_size[j], MADV_SEQUENTIAL);
-		madvise(geom, arg->geom_size[j], MADV_WILLNEED);
 
-		char *geomstart = geom;
+		long long geompos = 0;
-		char *end = geom + arg->geom_size[j];
+		long long prevgeom = 0;
-		char *prevgeom = geom;
 
-		while (geom < end) {
+		while (1) {
 			int z;
 			unsigned x, y;
 
-			deserialize_int(&geom, &z);
+			if (!deserialize_int_io(geom, &z, &geompos)) {
-			deserialize_uint(&geom, &x);
+				break;
-			deserialize_uint(&geom, &y);
+			}
+			deserialize_uint_io(geom, &x, &geompos);
+			deserialize_uint_io(geom, &y, &geompos);
 
 			// fprintf(stderr, "%d/%u/%u\n", z, x, y);
 
-			long long len = write_tile(&geom, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running);
+			long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running);
 
 			if (len < 0) {
 				int *err = (int *) malloc(sizeof(int));
@@ -1145,8 +1150,8 @@ void *run_thread(void *vargs) {
 				}
 			}
 
-			*arg->along += geom - prevgeom;
+			*arg->along += geompos - prevgeom;
-			prevgeom = geom;
+			prevgeom = geompos;
 
 			if (pthread_mutex_unlock(&var_lock) != 0) {
 				perror("pthread_mutex_unlock");
@@ -1154,10 +1159,12 @@ void *run_thread(void *vargs) {
 			}
 		}
 
-		madvise(geomstart, arg->geom_size[j], MADV_DONTNEED);
+		if (fclose(geom) != 0) {
-		if (munmap(geomstart, arg->geom_size[j]) != 0) {
+			perror("close geom");
-			perror("munmap geom");
+			exit(EXIT_FAILURE);
 		}
+		// Since the fclose() has closed the underlying file descriptor
+		arg->geomfd[j] = -1;
 	}
 
 	arg->running--;

tile.h

@@ -23,7 +23,11 @@ void deserialize_int(char **f, int *n);
 void deserialize_long_long(char **f, long long *n);
 void deserialize_uint(char **f, unsigned *n);
 void deserialize_byte(char **f, signed char *n);
-struct pool_val *deserialize_string(char **f, struct pool *p, int type);
+
+int deserialize_int_io(FILE *f, int *n, long long *geompos);
+int deserialize_long_long_io(FILE *f, long long *n, long long *geompos);
+int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos);
+int deserialize_byte_io(FILE *f, signed char *n, long long *geompos);
 
 long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, int *prevent, int *additional);
 

version.h

@@ -1 +1 @@
-#define VERSION "tippecanoe v1.9.11\n"
+#define VERSION "tippecanoe v1.9.12\n"