Add tile-join options to no tile compression and output to directory

Makefile

@@ -55,7 +55,7 @@ tippecanoe-enumerate: enumerate.o
 tippecanoe-decode: decode.o projection.o mvt.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
-tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o
+tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 unit: unit.o text.o
@@ -116,7 +116,7 @@ parallel-test:
 
 raw-tiles-test:	
 	./tippecanoe -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC
-	diff -x '.*' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare
+	diff -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare
 	rm -rf tests/raw-tiles/raw-tiles
 
 decode-test:
@@ -162,8 +162,11 @@ join-test:
 	./tippecanoe-decode tests/join-population/just-macarthur.mbtiles > tests/join-population/just-macarthur.mbtiles.json.check
 	./tippecanoe-decode tests/join-population/no-macarthur.mbtiles > tests/join-population/no-macarthur.mbtiles.json.check
 	cmp tests/join-population/just-macarthur.mbtiles.json.check tests/join-population/just-macarthur.mbtiles.json
-	cmp tests/join-population/no-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json
+	cmp tests/join-population/no-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json	
+	./tile-join -pC -e tests/join-population/raw-merged-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
+	diff -rq tests/join-population/raw-merged-folder tests/join-population/raw-merged-folder-compare
 	rm tests/join-population/tabblock_06001420.mbtiles tests/join-population/joined.mbtiles tests/join-population/joined-i.mbtiles tests/join-population/joined.mbtiles.json.check tests/join-population/joined-i.mbtiles.json.check tests/join-population/macarthur.mbtiles tests/join-population/merged.mbtiles tests/join-population/merged.mbtiles.json.check tests/join-population/macarthur2.mbtiles tests/join-population/windows.mbtiles tests/join-population/windows.mbtiles.json.check tests/join-population/just-macarthur.mbtiles tests/join-population/no-macarthur.mbtiles tests/join-population/just-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json.check
+	rm -rf tests/join-population/raw-merged-folder
 
 # Use this target to regenerate the standards that the tests are compared against
 # after making a change that legitimately changes their output

README.md

@@ -402,12 +402,14 @@ the same tiles, the layers or tiles are merged.
 
 The options are:
 
- * `-o` *out.mbtiles*: Write the new tiles to the specified .mbtiles file
- * `-f`: Remove *out.mbtiles* if it already exists
+ * `-o` *out.mbtiles*: Write the new tiles to the specified .mbtiles file.
+ * `-e` *directory*: Write the new tiles to the specified directory instead of to an mbtiles file.
+ * `-f`: Remove *out.mbtiles* if it already exists.
  * `-c` *match*`.csv`: Use *match*`.csv` as the source for new attributes to join to the features. The first line of the file should be the key names; the other lines are values. The first column is the one to match against the existing features; the other columns are the new data to add.
  * `-x` *key*: Remove attributes of type *key* from the output. You can use this to remove the field you are matching against if you no longer need it after joining, or to remove any other attributes you don't want.
  * `-i`: Only include features that matched the CSV.
  * `-pk`: Don't skip tiles larger than 500K.
+ * `-pC`: Don't compress the PBF vector tile data.
  * `-l` *layer*: Include the named layer in the output. You can specify multiple `-l` options to keep multiple layers. If you don't specify, they will all be retained.
  * `-L` *layer*: Remove the named layer from the output. You can specify multiple `-L` options to remove multiple layers.
  * `-A` *attribution*: Set the attribution string.

tests/join-population/raw-merged-folder-compare/metadata.json

@@ -0,0 +1,12 @@
+{
+    "name": "tests/join-population/macarthur.mbtiles + tests/join-population/macarthur2.mbtiles + tests/join-population/tabblock_06001420.mbtiles",
+    "description": "tests/join-population/tabblock_06001420.mbtiles",
+    "version": "2",
+    "minzoom": "0",
+    "maxzoom": "12",
+    "center": "-122.299805,37.892187,12",
+    "bounds": "-122.343750,37.695438,-122.104097,37.926868",
+    "type": "overlay",
+    "format": "pbf",
+    "json": "{\"vector_layers\": [ { \"id\": \"macarthur\", \"description\": \"\", \"minzoom\": 5, \"maxzoom\": 11, \"fields\": {\"FULLNAME\": \"String\", \"LINEARID\": \"String\", \"MTFCC\": \"String\", \"RTTYP\": \"String\"} }, { \"id\": \"tabblock_06001420\", \"description\": \"\", \"minzoom\": 3, \"maxzoom\": 12, \"fields\": {\"ALAND10\": \"Number\", \"AWATER10\": \"Number\", \"BLOCKCE10\": \"String\", \"COUNTYFP10\": \"String\", \"FUNCSTAT10\": \"String\", \"GEOID10\": \"String\", \"INTPTLAT10\": \"String\", \"INTPTLON10\": \"String\", \"MTFCC10\": \"String\", \"NAME10\": \"String\", \"STATEFP10\": \"String\", \"TRACTCE10\": \"String\", \"UACE10\": \"String\", \"UATYP10\": \"String\", \"UR10\": \"String\"} } ] }"
+}

tile-join.cpp

@@ -16,10 +16,12 @@
 #include "pool.hpp"
 #include "mbtiles.hpp"
 #include "geometry.hpp"
+#include "dirtiles.hpp"
 
 std::string dequote(std::string s);
 
 bool pk = false;
+bool pC = false;
 size_t CPUS;
 
 struct stats {
@@ -405,7 +407,12 @@ void *join_worker(void *v) {
 		if (anything) {
 			std::string pbf = tile.encode();
 			std::string compressed;
-			compress(pbf, compressed);
+
+			if (!pC) {
+				compress(pbf, compressed);
+			} else {
+				compressed = pbf;
+			}
 
 			if (!pk && compressed.size() > 500000) {
 				fprintf(stderr, "Tile %lld/%lld/%lld size is %lld, >500000. Skipping this tile\n.", ai->first.z, ai->first.x, ai->first.y, (long long) compressed.size());
@@ -418,7 +425,7 @@ void *join_worker(void *v) {
 	return NULL;
 }
 
-void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers) {
+void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, const char *outdir, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers) {
 	pthread_t pthreads[CPUS];
 	std::vector<arg> args;
 
@@ -462,12 +469,16 @@ void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<st
 		}
 
 		for (auto ai = args[i].outputs.begin(); ai != args[i].outputs.end(); ++ai) {
-			mbtiles_write_tile(outdb, ai->first.z, ai->first.x, ai->first.y, ai->second.data(), ai->second.size());
+			if (outdb != NULL) {
+				mbtiles_write_tile(outdb, ai->first.z, ai->first.x, ai->first.y, ai->second.data(), ai->second.size());
+			} else if (outdir != NULL) {
+				dir_write_tile(outdir, ai->first.z, ai->first.x, ai->first.y, ai->second);
+			}
 		}
 	}
 }
 
-void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name) {
+void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name) {
 	std::vector<std::map<std::string, layermap_entry>> layermaps;
 	for (size_t i = 0; i < CPUS; i++) {
 		layermaps.push_back(std::map<std::string, layermap_entry>());
@@ -512,7 +523,7 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 
 		if (readers == NULL || readers->zoom != r->zoom || readers->x != r->x || readers->y != r->y) {
 			if (tasks.size() > 100 * CPUS) {
-				handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
+				handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
 				tasks.clear();
 			}
 		}
@@ -548,7 +559,7 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 	st->minlat = min(minlat, st->minlat);
 	st->maxlat = max(maxlat, st->maxlat);
 
-	handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
+	handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
 	layermap = merge_layermaps(layermaps);
 
 	struct reader *next;
@@ -622,7 +633,7 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 }
 
 void usage(char **argv) {
-	fprintf(stderr, "Usage: %s [-f] [-i] [-pk] [-c joins.csv] [-x exclude ...] -o new.mbtiles source.mbtiles ...\n", argv[0]);
+	fprintf(stderr, "Usage: %s [-f] [-i] [-pk] [-pC] [-c joins.csv] [-x exclude ...] -o new.mbtiles source.mbtiles ...\n", argv[0]);
 	exit(EXIT_FAILURE);
 }
 
@@ -705,7 +716,9 @@ void readcsv(char *fn, std::vector<std::string> &header, std::map<std::string, s
 }
 
 int main(int argc, char **argv) {
-	char *outfile = NULL;
+	char *out_mbtiles = NULL;
+	char *out_dir = NULL;
+	sqlite3 *outdb = NULL;
 	char *csv = NULL;
 	int force = 0;
 	int ifmatched = 0;
@@ -728,10 +741,14 @@ int main(int argc, char **argv) {
 	extern char *optarg;
 	int i;
 
-	while ((i = getopt(argc, argv, "fo:c:x:ip:l:L:A:N:n:")) != -1) {
+	while ((i = getopt(argc, argv, "fo:e:c:x:ip:l:L:A:N:n:")) != -1) {
 		switch (i) {
 		case 'o':
-			outfile = optarg;
+			out_mbtiles = optarg;
+			break;
+
+		case 'e':
+			out_dir = optarg;
 			break;
 
 		case 'f':
@@ -757,6 +774,8 @@ int main(int argc, char **argv) {
 		case 'p':
 			if (strcmp(optarg, "k") == 0) {
 				pk = true;
+			} else if (strcmp(optarg, "C") == 0) {
+				pC = true;
 			} else {
 				fprintf(stderr, "%s: Unknown option for -p%s\n", argv[0], optarg);
 				exit(EXIT_FAILURE);
@@ -790,15 +809,27 @@ int main(int argc, char **argv) {
 		}
 	}
 
-	if (argc - optind < 1 || outfile == NULL) {
+	if (argc - optind < 1) {
 		usage(argv);
 	}
 
-	if (force) {
-		unlink(outfile);
+	if (out_mbtiles == NULL && out_dir == NULL) {
+		fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory\n", argv[0]);
+		usage(argv);
+	}
+
+	if (out_mbtiles != NULL && out_dir != NULL) {
+		fprintf(stderr, "%s: Options -o and -e cannot be used together\n", argv[0]);
+		usage(argv);
+	}
+
+	if (out_mbtiles != NULL) {
+		if (force) {
+			unlink(out_mbtiles);
+		}
+		outdb = mbtiles_open(out_mbtiles, argv, 0);
 	}
 
-	sqlite3 *outdb = mbtiles_open(outfile, argv, 0);
 	struct stats st;
 	memset(&st, 0, sizeof(st));
 	st.minzoom = st.minlat = st.minlon = INT_MAX;
@@ -825,7 +856,7 @@ int main(int argc, char **argv) {
 		*rr = r;
 	}
 
-	decode(readers, csv, layermap, outdb, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name);
+	decode(readers, csv, layermap, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name);
 
 	if (set_attribution.size() != 0) {
 		attribution = set_attribution;
@@ -837,8 +868,11 @@ int main(int argc, char **argv) {
 		name = set_name;
 	}
 
-	mbtiles_write_metadata(outdb, NULL, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str());
-	mbtiles_close(outdb, argv);
+	mbtiles_write_metadata(outdb, out_dir, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str());
+
+	if (outdb != NULL) {
+		mbtiles_close(outdb, argv);
+	}
 
 	return 0;
 }