Merge pull request #389 from mapbox/joykuotw-master

Add --output-to-directory and --no-tile-compression options

.gitignore

@@ -27,3 +27,6 @@
 *.exe
 *.out
 *.app
+
+# Mac 
+.DS_Store

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.16.15
+
+* Add --output-to-directory and --no-tile-compression options
+
 ## 1.16.14
 
 * Add --description option for mbtiles metadata

Makefile

@@ -46,7 +46,7 @@ C = $(wildcard *.c) $(wildcard *.cpp)
 INCLUDES = -I/usr/local/include -I.
 LIBS = -L/usr/local/lib
 
-tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o
+tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o rawtiles.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 tippecanoe-enumerate: enumerate.o
@@ -78,7 +78,7 @@ indent:
 TESTS = $(wildcard tests/*/out/*.json)
 SPACE = $(NULL) $(NULL)
 
-test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) parallel-test pbf-test join-test enumerate-test decode-test unit
+test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test unit
 	./unit
 
 # Work around Makefile and filename punctuation limits: _ for space, @ for :, % for /
@@ -111,6 +111,11 @@ parallel-test:
 	cmp tests/parallel/linear-file.json tests/parallel/parallel-pipes.json
 	rm tests/parallel/*.mbtiles tests/parallel/*.json
 
+raw-tiles-test:	
+	./tippecanoe -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC
+	diff -x '.*' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare
+	rm -rf tests/raw-tiles/raw-tiles
+
 decode-test:
 	mkdir -p tests/muni/decode
 	./tippecanoe -z11 -Z11 -f -o tests/muni/decode/multi.mbtiles tests/muni/*.json

README.md

@@ -67,6 +67,7 @@ Options
 ### File control
 
  * -o _file_.mbtiles or --output=_file_.mbtiles: Name the output file.
+ * -e _directory_ or --output-directory=_directory_: Write tiles to the specified *directory* instead of to an mbtiles file.
  * -f or --force: Delete the mbtiles file if it already exists instead of giving an error
  * -F or --allow-existing: Proceed (without deleting existing data) if the metadata or tiles table already exists
    or if metadata fields can't be set
@@ -147,6 +148,7 @@ resolution is obtained than by using a smaller _maxzoom_ or _detail_.
  * -pc or --no-clipping: Don't clip features to the size of the tile. If a feature overlaps the tile's bounds or buffer at all, it is included completely. Be careful: this can produce very large tilesets, especially with large polygons.
  * -pD or --no-duplication: As with --no-clipping, each feature is included intact instead of cut to tile boundaries. In addition, it is included only in a single tile per zoom level rather than potentially in multiple copies. Clients of the tileset must check adjacent tiles (possibly some distance away) to ensure they have all features.
  * -pt or --no-tiny-polygon-reduction: Don't combine the area of very small polygons into small squares that represent their combined area.
+ * -pC or --no-tile-compression: Don't compress the PBF vector tile data.
  * -q or --quiet: Work quietly instead of reporting progress
 
 Environment

main.cpp

@@ -20,6 +20,7 @@
 #include <sys/resource.h>
 #include <pthread.h>
 #include <getopt.h>
+#include <algorithm>
 #include <vector>
 #include <string>
 #include <set>
@@ -1005,7 +1006,7 @@ void choose_first_zoom(long long *file_bbox, struct reader *reader, unsigned *iz
 	}
 }
 
-int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description) {
+int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description) {
 	int ret = EXIT_SUCCESS;
 
 	struct reader reader[CPUS];
@@ -1779,7 +1780,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 	}
 
 	unsigned midx = 0, midy = 0;
-	int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps);
+	int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, outdir, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps);
 
 	if (maxzoom != written) {
 		fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written);
@@ -1843,7 +1844,8 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 		ai->second.minzoom = minzoom;
 		ai->second.maxzoom = maxzoom;
 	}
-	mbtiles_write_metadata(outdb, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description);
+
+	mbtiles_write_metadata(outdb, outdir, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description);
 
 	return ret;
 }
@@ -1872,7 +1874,9 @@ int main(int argc, char **argv) {
 	char *name = NULL;
 	char *description = NULL;
 	char *layername = NULL;
-	char *outdir = NULL;
+	char *out_mbtiles = NULL;
+	char *out_directory = NULL;
+	sqlite3 *outdb = NULL;
 	int maxzoom = 14;
 	int minzoom = 0;
 	int basezoom = -1;
@@ -1898,6 +1902,7 @@ int main(int argc, char **argv) {
 
 	static struct option long_options[] = {
 		{"output", required_argument, 0, 'o'},
+		{"output-to-directory", required_argument, 0, 'e'},
 
 		{"name", required_argument, 0, 'n'},
 		{"description", required_argument, 0, 'N'},
@@ -1955,6 +1960,7 @@ int main(int argc, char **argv) {
 		{"no-clipping", no_argument, &prevent[P_CLIPPING], 1},
 		{"no-duplication", no_argument, &prevent[P_DUPLICATION], 1},
 		{"no-tiny-polygon-reduction", no_argument, &prevent[P_TINY_POLYGON_REDUCTION], 1},
+		{"no-tile-compression", no_argument, &prevent[P_TILE_COMPRESSION], 1},
 
 		{0, 0, 0, 0},
 	};
@@ -1977,7 +1983,7 @@ int main(int argc, char **argv) {
 		}
 	}
 
-	while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:", long_options, NULL)) != -1) {
+	while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:e:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:", long_options, NULL)) != -1) {
 		switch (i) {
 		case 0:
 			break;
@@ -2054,7 +2060,11 @@ int main(int argc, char **argv) {
 			break;
 
 		case 'o':
-			outdir = optarg;
+			out_mbtiles = optarg;
+			break;
+
+		case 'e':
+			out_directory = optarg;
 			break;
 
 		case 'x':
@@ -2238,16 +2248,24 @@ int main(int argc, char **argv) {
 		fprintf(stderr, "Forcing -g0 since -B or -r is not known\n");
 	}
 
-	if (outdir == NULL) {
-		fprintf(stderr, "%s: must specify -o out.mbtiles\n", argv[0]);
+	if (out_mbtiles == NULL && out_directory == NULL) {
+		fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory\n", argv[0]);
 		exit(EXIT_FAILURE);
 	}
 
-	if (force) {
-		unlink(outdir);
+	if (out_mbtiles != NULL && out_directory != NULL) {
+		fprintf(stderr, "%s: Options -o and -e cannot be used together\n", argv[0]);
+		exit(EXIT_FAILURE);
+	}
+
+	if (out_mbtiles != NULL) {
+		if (force) {
+			unlink(out_mbtiles);
+		}
+
+		outdb = mbtiles_open(out_mbtiles, argv, forcetable);
 	}
 
-	sqlite3 *outdb = mbtiles_open(outdir, argv, forcetable);
 	int ret = EXIT_SUCCESS;
 
 	for (i = optind; i < argc; i++) {
@@ -2272,9 +2290,11 @@ int main(int argc, char **argv) {
 
 	long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0};
 
-	ret = read_input(sources, name ? name : outdir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description);
+	ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_directory, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_directory, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description);
 
+	if (outdb != NULL) {
 		mbtiles_close(outdb, argv);
+	}
 
 #ifdef MTRACE
 	muntrace();

man/tippecanoe.1

@@ -60,12 +60,16 @@ specified, the files are all merged into the single named layer, even if they tr
 \-n \fIname\fP or \-\-name=\fIname\fP: Human\-readable name for the tileset (default file.json)
 .IP \(bu 2
 \-A \fItext\fP or \-\-attribution=\fItext\fP: Attribution (HTML) to be shown with maps that use data from this tileset.
+.IP \(bu 2
+\-N \fIdescription\fP or \-\-description=\fIdescription\fP: Description for the tileset (default file.mbtiles)
 .RE
 .SS File control
 .RS
 .IP \(bu 2
 \-o \fIfile\fP\&.mbtiles or \-\-output=\fIfile\fP\&.mbtiles: Name the output file.
 .IP \(bu 2
+\-e \fIdirectory\fP or \-\-output\-directory=\fIdirectory\fP: Write tiles to the specified \fIdirectory\fP instead of to an mbtiles file.
+.IP \(bu 2
 \-f or \-\-force: Delete the mbtiles file if it already exists instead of giving an error
 .IP \(bu 2
 \-F or \-\-allow\-existing: Proceed (without deleting existing data) if the metadata or tiles table already exists
@@ -162,6 +166,8 @@ which may not be what you want.
 \-an or \-\-drop\-smallest\-as\-needed: Dynamically drop the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level to keep large tiles under the 500K size limit. This option will not work for point features.
 .IP \(bu 2
 \-aL or \-\-grid\-low\-zooms: At all zoom levels below \fImaxzoom\fP, snap all lines and polygons to a stairstep grid instead of allowing diagonals. You will also want to specify a tile resolution, probably \fB\fC\-D8\fR\&. This option provides a way to display continuous parcel, gridded, or binned data at low zooms without overwhelming the tiles with tiny polygons, since features will either get stretched out to the grid unit or lost entirely, depending on how they happened to be aligned in the original data.
+.IP \(bu 2
+\-aw or \-\-detect\-longitude\-wraparound: Detect when adjacent points within a feature jump to the other side of the world, and try to fix the geometry.
 .RE
 .SS Doing less
 .RS
@@ -178,7 +184,7 @@ which may not be what you want.
 .IP \(bu 2
 \-pi or \-\-preserve\-input\-order: Preserve the original input order of features as the drawing order instead of ordering geographically. (This is implemented as a restoration of the original order at the end, so that dot\-dropping is still geographic, which means it also undoes \-ao).
 .IP \(bu 2
-\-pp or \-\-no\-polygon\-splitting: Don't split complex polygons (over 700 vertices after simplification) into multiple features.
+\-pp or \-\-no\-polygon\-splitting: This no longer has any effect.
 .IP \(bu 2
 \-pc or \-\-no\-clipping: Don't clip features to the size of the tile. If a feature overlaps the tile's bounds or buffer at all, it is included completely. Be careful: this can produce very large tilesets, especially with large polygons.
 .IP \(bu 2
@@ -186,6 +192,8 @@ which may not be what you want.
 .IP \(bu 2
 \-pt or \-\-no\-tiny\-polygon\-reduction: Don't combine the area of very small polygons into small squares that represent their combined area.
 .IP \(bu 2
+\-pC or \-\-no\-tile\-compression: Don't compress the PBF vector tile data.
+.IP \(bu 2
 \-q or \-\-quiet: Work quietly instead of reporting progress
 .RE
 .SH Environment
@@ -301,9 +309,6 @@ have their probability diffused, so that some of them will be drawn as a square
 this minimum size and others will not be drawn at all, preserving the total area that
 all of them should have had together.
 .PP
-Any polygons that have over 700 vertices after line simplification will be split into
-multiple features so they can be rendered efficiently, unless you use \-pp to prevent this.
-.PP
 Features in the same tile that share the same type and attributes are coalesced
 together into a single geometry if you use \fB\fC\-\-coalesce\fR\&. You are strongly encouraged to use \-x to exclude
 any unnecessary properties to reduce wasted file size.

mbtiles.cpp

@@ -131,11 +131,23 @@ bool type_and_string::operator<(const type_and_string &o) const {
 	return false;
 }
 
-void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description) {
+void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description) {
 	char *sql, *err;
 
+	sqlite3 *db = outdb;
+	if (outdb == NULL) {
+		if (sqlite3_open("", &db) != SQLITE_OK) {
+			fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db));
+			exit(EXIT_FAILURE);
+		}
+		if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) {
+			fprintf(stderr, "Create metadata table: %s\n", err);
+			exit(EXIT_FAILURE);
+		}
+	}
+
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname);
-	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set name in metadata: %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_FAILURE);
@@ -144,7 +156,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 	sqlite3_free(sql);
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", description != NULL ? description : fname);
-	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set description in metadata: %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_FAILURE);
@@ -153,7 +165,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 	sqlite3_free(sql);
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", 2);
-	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set version : %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_FAILURE);
@@ -162,7 +174,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 	sqlite3_free(sql);
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", minzoom);
-	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set minzoom: %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_FAILURE);
@@ -171,7 +183,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 	sqlite3_free(sql);
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", maxzoom);
-	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set maxzoom: %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_FAILURE);
@@ -180,7 +192,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 	sqlite3_free(sql);
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", midlon, midlat, maxzoom);
-	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set center: %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_FAILURE);
@@ -189,7 +201,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 	sqlite3_free(sql);
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", minlon, minlat, maxlon, maxlat);
-	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set bounds: %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_FAILURE);
@@ -198,7 +210,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 	sqlite3_free(sql);
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", "overlay");
-	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set type: %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_FAILURE);
@@ -208,7 +220,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 
 	if (attribution != NULL) {
 		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", attribution);
-		if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+		if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 			fprintf(stderr, "set type: %s\n", err);
 			if (!forcetable) {
 				exit(EXIT_FAILURE);
@@ -218,7 +230,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 	}
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", vector ? "pbf" : "png");
-	if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set format: %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_FAILURE);
@@ -276,7 +288,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 		aprintf(&buf, " ] }");
 
 		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str());
-		if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
+		if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 			fprintf(stderr, "set json: %s\n", err);
 			if (!forcetable) {
 				exit(EXIT_FAILURE);
@@ -284,6 +296,45 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 		}
 		sqlite3_free(sql);
 	}
+
+	if (outdir != NULL) {
+		std::string metadata = std::string(outdir) + "/metadata.json";
+		FILE *fp = fopen(metadata.c_str(), "w");
+		if (fp == NULL) {
+			perror(metadata.c_str());
+			exit(EXIT_FAILURE);
+		}
+
+		fprintf(fp, "{\n");
+
+		sqlite3_stmt *stmt;
+		bool first = true;
+		if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) {
+			while (sqlite3_step(stmt) == SQLITE_ROW) {
+				std::string key, value;
+
+				quote(&key, (const char *) sqlite3_column_text(stmt, 0));
+				quote(&value, (const char *) sqlite3_column_text(stmt, 1));
+
+				if (!first) {
+					fprintf(fp, ",\n");
+				}
+				fprintf(fp, "    \"%s\": \"%s\"", key.c_str(), value.c_str());
+				first = false;
+			}
+			sqlite3_finalize(stmt);
+		}
+
+		fprintf(fp, "\n}\n");
+		fclose(fp);
+	}
+
+	if (outdb == NULL) {
+		if (sqlite3_close(db) != SQLITE_OK) {
+			fprintf(stderr, "Could not close temp database: %s\n", sqlite3_errmsg(db));
+			exit(EXIT_FAILURE);
+		}
+	}
 }
 
 void mbtiles_close(sqlite3 *outdb, char **argv) {

mbtiles.hpp

@@ -20,7 +20,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
 
 void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
 
-void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description);
+void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description);
 
 void mbtiles_close(sqlite3 *outdb, char **argv);
 

mvt.cpp

@@ -367,10 +367,7 @@ std::string mvt_tile::encode() {
 		writer.add_message(3, layer_string);
 	}
 
-	std::string compressed;
-	compress(data, compressed);
-
-	return compressed;
+	return data;
 }
 
 bool mvt_value::operator<(const mvt_value &o) const {

options.hpp

@@ -25,6 +25,7 @@
 #define P_CLIPPING ((int) 'c')
 #define P_DUPLICATION ((int) 'D')
 #define P_TINY_POLYGON_REDUCTION ((int) 't')
+#define P_TILE_COMPRESSION ((int) 'C')
 
 extern int prevent[256];
 extern int additional[256];

rawtiles.cpp

@@ -0,0 +1,20 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <sys/stat.h>
+#include "rawtiles.hpp"
+
+void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) {
+	mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO);
+	std::string curdir(outdir);
+	std::string slash("/");
+	std::string newdir = curdir + slash + std::to_string(z);
+	mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
+	newdir = newdir + "/" + std::to_string(tx);
+	mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
+	newdir = newdir + "/" + std::to_string(ty) + ".pbf";
+
+	std::ofstream pbfFile(newdir, std::ios::out | std::ios::binary);
+	pbfFile.write(pbf.data(), pbf.size());
+	pbfFile.close();
+}

rawtiles.hpp

@@ -0,0 +1 @@
+void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);

tests/raw-tiles/compare/metadata.json

@@ -0,0 +1,12 @@
+{
+    "name": "tests/raw-tiles/raw-tiles",
+    "description": "tests/raw-tiles/raw-tiles",
+    "version": "2",
+    "minzoom": "0",
+    "maxzoom": "14",
+    "center": "-122.662354,45.514045,14",
+    "bounds": "-122.682427,45.512332,-122.654961,45.569975",
+    "type": "overlay",
+    "format": "pbf",
+    "json": "{\"vector_layers\": [ { \"id\": \"hackspotsgeojson\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 14, \"fields\": {\"Address\": \"String\", \"Name\": \"String\", \"Notes\": \"String\"} } ] }"
+}

tests/raw-tiles/hackspots.geojson

@@ -0,0 +1,65 @@
+{
+  "type": "FeatureCollection",
+  "features": [
+    {
+      "type": "Feature",
+      "properties": {
+        "Name": "Albina Press",
+        "Address": "4637 N Albina Ave  Portland, OR 97217",
+        "Notes": "usually busy, outlets on side wall only"
+      },
+      "geometry": {
+        "type": "Point",
+        "coordinates": [
+          -122.67516911029816,
+          45.55673233031101
+        ]
+      }
+    },
+    {
+      "type": "Feature",
+      "properties": {
+        "Name": "Arbor Lodge",
+        "Address": "1507 N Rosa Parks Way  Portland, OR 97217",
+        "Notes": ""
+      },
+      "geometry": {
+        "type": "Point",
+        "coordinates": [
+          -122.68242716789246,
+          45.56997505986905
+        ]
+      }
+    },
+    {
+      "type": "Feature",
+      "properties": {
+        "Name": "Three Friends Coffeehouse",
+        "Address": "201 SE 12th Ave, Portland, OR 97214",
+        "Notes": ""
+      },
+      "geometry": {
+        "type": "Point",
+        "coordinates": [
+          -122.65496134757996,
+          45.5212590460849
+        ]
+      }
+    },
+    {
+      "type": "Feature",
+      "properties": {
+        "Name": "Lucky Labrador Brew Pub",
+        "Address": "915 SE Hawthorne Blvd. Portland, OR 97214",
+        "Notes": "Dog friendly"
+      },
+      "geometry": {
+        "type": "Point",
+        "coordinates": [
+          -122.65625953674315,
+          45.512331780814186
+        ]
+      }
+    }
+  ]
+}

tile-join.cpp

@@ -750,7 +750,7 @@ int main(int argc, char **argv) {
 
 	decode(readers, csv, layermap, outdb, &st, header, mapping, exclude, ifmatched, attribution, description);
 
-	mbtiles_write_metadata(outdb, outfile, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str());
+	mbtiles_write_metadata(outdb, NULL, outfile, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str());
 	mbtiles_close(outdb, argv);
 
 	return 0;

tile.cpp

@@ -22,6 +22,7 @@
 #include <time.h>
 #include "mvt.hpp"
 #include "mbtiles.hpp"
+#include "rawtiles.hpp"
 #include "geometry.hpp"
 #include "tile.hpp"
 #include "pool.hpp"
@@ -1184,6 +1185,7 @@ struct write_tile_args {
 	int min_detail;
 	int basezoom;
 	sqlite3 *outdb;
+	const char *outdir;
 	double droprate;
 	int buffer;
 	const char *fname;
@@ -1221,7 +1223,7 @@ struct write_tile_args {
 	double fraction_out;
 };
 
-long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, write_tile_args *arg) {
+long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, write_tile_args *arg) {
 	int line_detail;
 	double merge_fraction = 1;
 	double mingap_fraction = 1;
@@ -1837,7 +1839,14 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				}
 			}
 
-			std::string compressed = tile.encode();
+			std::string compressed;
+			std::string pbf = tile.encode();
+
+			if (!prevent[P_TILE_COMPRESSION]) {
+				compress(pbf, compressed);
+			} else {
+				compressed = pbf;
+			}
 
 			if (compressed.size() > max_tile_size && !prevent[P_KILOBYTE_LIMIT]) {
 				if (!quiet) {
@@ -1909,7 +1918,11 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 						exit(EXIT_FAILURE);
 					}
 
+					if (outdb != NULL) {
 						mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
+					} else if (outdir != NULL) {
+						write_raw_tile(outdir, z, tx, ty, compressed);
+					}
 
 					if (pthread_mutex_unlock(&db_lock) != 0) {
 						perror("pthread_mutex_unlock");
@@ -1971,7 +1984,7 @@ void *run_thread(void *vargs) {
 
 			// fprintf(stderr, "%d/%u/%u\n", z, x, y);
 
-			long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg);
+			long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->outdir, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg);
 
 			if (len < 0) {
 				int *err = &arg->err;
@@ -2036,7 +2049,7 @@ void *run_thread(void *vargs) {
 	return NULL;
 }
 
-int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps) {
+int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps) {
 	// Table to map segment and layer number back to layer name
 	std::vector<std::vector<std::string>> layer_unmaps;
 	for (size_t seg = 0; seg < layermaps.size(); seg++) {
@@ -2171,6 +2184,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
 				args[thread].min_detail = min_detail;
 				args[thread].basezoom = basezoom;
 				args[thread].outdb = outdb;  // locked with db_lock
+				args[thread].outdir = outdir;
 				args[thread].droprate = droprate;
 				args[thread].buffer = buffer;
 				args[thread].fname = fname;

tile.hpp

@@ -1,5 +1,5 @@
-long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers);
+long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers);
 
-int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap);
+int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap);
 
 int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);

version.hpp

@@ -1 +1 @@
-#define VERSION "tippecanoe v1.16.13\n"
+#define VERSION "tippecanoe v1.16.14\n"