Make --allow-existing work on dirs; trim .geojson from layer names

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.27.2
+
+* Tippecanoe-decode can decode directories of tiles, not just mbtiles
+* The --allow-existing option works on directories of tiles
+* Trim .geojson, not just .json, when making layer names from filenames
+
 ## 1.27.1
 
 * Fix a potential null pointer when parsing GeoJSON with bare geometries

Makefile

@@ -53,7 +53,7 @@ tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o pro
 tippecanoe-enumerate: enumerate.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3
 
-tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o
+tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
 tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o
@@ -82,7 +82,7 @@ indent:
 TESTS = $(wildcard tests/*/out/*.json)
 SPACE = $(NULL) $(NULL)
 
-test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test
+test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test allow-existing-test
 	./unit
 
 # Work around Makefile and filename punctuation limits: _ for space, @ for :, % for /
@@ -239,6 +239,17 @@ json-tool-test: tippecanoe-json-tool
 	cmp tests/join-population/tabblock_06001420.json.sort.joined tests/join-population/tabblock_06001420.json.sort.joined.standard
 	rm -f tests/join-population/tabblock_06001420.json.sort tests/join-population/tabblock_06001420.json.sort.joined
 
+allow-existing-test:
+	./tippecanoe -Z8 -z9 -f -o tests/allow-existing/both.mbtiles tests/coalesce-tract/tl_2010_06001_tract10.json
+	./tippecanoe -Z10 -z11 -F -o tests/allow-existing/both.mbtiles tests/coalesce-tract/tl_2010_06001_tract10.json
+	./tippecanoe-decode tests/allow-existing/both.mbtiles > tests/allow-existing/both.mbtiles.json.check
+	cmp tests/allow-existing/both.mbtiles.json.check tests/allow-existing/both.mbtiles.json
+	./tippecanoe -Z8 -z9 -f -e tests/allow-existing/both.dir tests/coalesce-tract/tl_2010_06001_tract10.json
+	./tippecanoe -Z10 -z11 -F -e tests/allow-existing/both.dir tests/coalesce-tract/tl_2010_06001_tract10.json
+	./tippecanoe-decode tests/allow-existing/both.dir | sed 's/both\.dir/both.mbtiles/g' > tests/allow-existing/both.dir.json.check
+	cmp tests/allow-existing/both.dir.json.check tests/allow-existing/both.mbtiles.json
+	rm -r tests/allow-existing/both.dir.json.check tests/allow-existing/both.dir tests/allow-existing/both.mbtiles.json.check tests/allow-existing/both.mbtiles
+
 # Use this target to regenerate the standards that the tests are compared against
 # after making a change that legitimately changes their output
 

decode.cpp

@@ -11,13 +11,16 @@
 #include <zlib.h>
 #include <math.h>
 #include <fcntl.h>
+#include <dirent.h>
 #include <sys/stat.h>
 #include <sys/mman.h>
 #include <protozero/pbf_reader.hpp>
+#include <sys/stat.h>
 #include "mvt.hpp"
 #include "projection.hpp"
 #include "geometry.hpp"
 #include "write_json.hpp"
+#include "jsonpull/jsonpull.h"
 
 int minzoom = 0;
 int maxzoom = 32;
@@ -139,8 +142,154 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st
 	}
 }
 
-void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats) {
+struct zxy {
+	int z;
+	int x;
+	int y;
+
+	zxy(int _z, int _x, int _y)
+	    : z(_z), x(_x), y(_y) {
+	}
+
+	bool operator<(const zxy &other) const {
+		if (z < other.z) {
+			return true;
+		}
+		if (z == other.z) {
+			if (x < other.x) {
+				return true;
+			}
+			if (x == other.x) {
+				if (y > other.y) {
+					return true;  // reversed for TMS
+				}
+			}
+		}
+
+		return false;
+	}
+};
+
+// XXX deduplicate from dirtiles
+bool numeric(const char *s) {
+	if (*s == '\0') {
+		return false;
+	}
+	for (; *s != 0; s++) {
+		if (*s < '0' || *s > '9') {
+			return false;
+		}
+	}
+	return true;
+}
+
+// XXX deduplicate from dirtiles
+bool pbfname(const char *s) {
+	while (*s >= '0' && *s <= '9') {
+		s++;
+	}
+
+	return strcmp(s, ".pbf") == 0;
+}
+
+sqlite3 *meta2tmp(const char *fname, std::vector<zxy> &tiles) {
 	sqlite3 *db;
+	char *err = NULL;
+
+	if (sqlite3_open("", &db) != SQLITE_OK) {
+		fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db));
+		exit(EXIT_FAILURE);
+	}
+	if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) {
+		fprintf(stderr, "Create metadata table: %s\n", err);
+		exit(EXIT_FAILURE);
+	}
+
+	std::string name = fname;
+	name += "/metadata.json";
+
+	FILE *f = fopen(name.c_str(), "r");
+	if (f == NULL) {
+		perror(name.c_str());
+		exit(EXIT_FAILURE);
+	}
+
+	json_pull *jp = json_begin_file(f);
+	json_object *o = json_read_tree(jp);
+
+	if (o->type != JSON_HASH) {
+		fprintf(stderr, "%s: bad metadata format\n", name.c_str());
+		exit(EXIT_FAILURE);
+	}
+
+	for (size_t i = 0; i < o->length; i++) {
+		if (o->keys[i]->type != JSON_STRING || o->values[i]->type != JSON_STRING) {
+			fprintf(stderr, "%s: non-string in metadata\n", name.c_str());
+		}
+
+		char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->keys[i]->string, o->values[i]->string);
+		if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
+			fprintf(stderr, "set %s in metadata: %s\n", o->keys[i]->string, err);
+		}
+		sqlite3_free(sql);
+	}
+
+	json_end(jp);
+	fclose(f);
+
+	// XXX deduplicate from dirtiles
+	DIR *d1 = opendir(fname);
+	if (d1 != NULL) {
+		struct dirent *dp;
+		while ((dp = readdir(d1)) != NULL) {
+			if (numeric(dp->d_name)) {
+				std::string z = std::string(fname) + "/" + dp->d_name;
+				int tz = atoi(dp->d_name);
+
+				DIR *d2 = opendir(z.c_str());
+				if (d2 == NULL) {
+					perror(z.c_str());
+					exit(EXIT_FAILURE);
+				}
+
+				struct dirent *dp2;
+				while ((dp2 = readdir(d2)) != NULL) {
+					if (numeric(dp2->d_name)) {
+						std::string x = z + "/" + dp2->d_name;
+						int tx = atoi(dp2->d_name);
+
+						DIR *d3 = opendir(x.c_str());
+						if (d3 == NULL) {
+							perror(x.c_str());
+							exit(EXIT_FAILURE);
+						}
+
+						struct dirent *dp3;
+						while ((dp3 = readdir(d3)) != NULL) {
+							if (pbfname(dp3->d_name)) {
+								int ty = atoi(dp3->d_name);
+								tiles.push_back(zxy(tz, tx, ty));
+							}
+						}
+
+						closedir(d3);
+					}
+				}
+
+				closedir(d2);
+			}
+		}
+
+		closedir(d1);
+	}
+
+	std::sort(tiles.begin(), tiles.end());
+	return db;
+}
+
+void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats) {
+	sqlite3 *db = NULL;
+	bool isdir = false;
 	int oz = z;
 	unsigned ox = x, oy = y;
 
@@ -176,9 +325,17 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 		perror(fname);
 	}
 
-	if (sqlite3_open(fname, &db) != SQLITE_OK) {
-		fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db));
-		exit(EXIT_FAILURE);
+	struct stat st;
+	std::vector<zxy> tiles;
+	if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) {
+		isdir = true;
+
+		db = meta2tmp(fname, tiles);
+	} else {
+		if (sqlite3_open(fname, &db) != SQLITE_OK) {
+			fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db));
+			exit(EXIT_FAILURE);
+		}
 	}
 
 	if (z < 0) {
@@ -220,49 +377,86 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 			printf("[\n");
 		}
 
-		const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;";
-		sqlite3_stmt *stmt;
-		if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
-			fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
-			exit(EXIT_FAILURE);
-		}
-
-		sqlite3_bind_int(stmt, 1, minzoom);
-		sqlite3_bind_int(stmt, 2, maxzoom);
-
 		if (!pipeline && !stats) {
 			printf("\n}, \"features\": [\n");
 		}
 
-		within = 0;
-		while (sqlite3_step(stmt) == SQLITE_ROW) {
-			if (!pipeline && !stats) {
-				if (within) {
-					printf(",\n");
+		if (isdir) {
+			within = 0;
+			for (size_t i = 0; i < tiles.size(); i++) {
+				if (!pipeline && !stats) {
+					if (within) {
+						printf(",\n");
+					}
+					within = 1;
 				}
-				within = 1;
-			}
-			if (stats) {
-				if (within) {
-					printf(",\n");
+				if (stats) {
+					if (within) {
+						printf(",\n");
+					}
+					within = 1;
 				}
-				within = 1;
+
+				std::string fn = std::string(fname) + "/" + std::to_string(tiles[i].z) + "/" + std::to_string(tiles[i].x) + "/" + std::to_string(tiles[i].y) + ".pbf";
+				FILE *f = fopen(fn.c_str(), "rb");
+				if (f == NULL) {
+					perror(fn.c_str());
+					exit(EXIT_FAILURE);
+				}
+
+				std::string s;
+				char buf[2000];
+				ssize_t n;
+				while ((n = fread(buf, 1, 2000, f)) > 0) {
+					s.append(std::string(buf, n));
+				}
+				fclose(f);
+
+				handle(s, tiles[i].z, tiles[i].x, tiles[i].y, 1, to_decode, pipeline, stats);
 			}
-
-			int len = sqlite3_column_bytes(stmt, 0);
-			int tz = sqlite3_column_int(stmt, 1);
-			int tx = sqlite3_column_int(stmt, 2);
-			int ty = sqlite3_column_int(stmt, 3);
-
-			if (tz < 0 || tz >= 32) {
-				fprintf(stderr, "Impossible zoom level %d in mbtiles\n", tz);
+		} else {
+			const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;";
+			sqlite3_stmt *stmt;
+			if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
+				fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
 				exit(EXIT_FAILURE);
 			}
 
-			ty = (1LL << tz) - 1 - ty;
-			const char *s = (const char *) sqlite3_column_blob(stmt, 0);
+			sqlite3_bind_int(stmt, 1, minzoom);
+			sqlite3_bind_int(stmt, 2, maxzoom);
 
-			handle(std::string(s, len), tz, tx, ty, 1, to_decode, pipeline, stats);
+			within = 0;
+			while (sqlite3_step(stmt) == SQLITE_ROW) {
+				if (!pipeline && !stats) {
+					if (within) {
+						printf(",\n");
+					}
+					within = 1;
+				}
+				if (stats) {
+					if (within) {
+						printf(",\n");
+					}
+					within = 1;
+				}
+
+				int len = sqlite3_column_bytes(stmt, 0);
+				int tz = sqlite3_column_int(stmt, 1);
+				int tx = sqlite3_column_int(stmt, 2);
+				int ty = sqlite3_column_int(stmt, 3);
+
+				if (tz < 0 || tz >= 32) {
+					fprintf(stderr, "Impossible zoom level %d in mbtiles\n", tz);
+					exit(EXIT_FAILURE);
+				}
+
+				ty = (1LL << tz) - 1 - ty;
+				const char *s = (const char *) sqlite3_column_blob(stmt, 0);
+
+				handle(std::string(s, len), tz, tx, ty, 1, to_decode, pipeline, stats);
+			}
+
+			sqlite3_finalize(stmt);
 		}
 
 		if (!pipeline && !stats) {
@@ -271,8 +465,6 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 		if (stats) {
 			printf("]\n");
 		}
-
-		sqlite3_finalize(stmt);
 	} else {
 		int handled = 0;
 		while (z >= 0 && !handled) {

dirtiles.cpp

@@ -59,19 +59,26 @@ bool pbfname(const char *s) {
 	return strcmp(s, ".pbf") == 0;
 }
 
-void check_dir(const char *dir, bool rm) {
+void check_dir(const char *dir, bool force, bool forcetable) {
 	struct stat st;
 
 	std::string meta = std::string(dir) + "/" + "metadata.json";
-	if (rm) {
+	if (force) {
 		unlink(meta.c_str());  // error OK since it may not exist;
 	} else {
 		if (stat(meta.c_str(), &st) == 0) {
 			fprintf(stderr, "%s: file exists\n", meta.c_str());
-			exit(EXIT_FAILURE);
+			if (!forcetable) {
+				exit(EXIT_FAILURE);
+			}
 		}
 	}
 
+	if (forcetable) {
+		// Don't clear existing tiles
+		return;
+	}
+
 	DIR *d1 = opendir(dir);
 	if (d1 != NULL) {
 		struct dirent *dp;
@@ -101,7 +108,7 @@ void check_dir(const char *dir, bool rm) {
 							if (pbfname(dp3->d_name)) {
 								std::string y = x + "/" + dp3->d_name;
 
-								if (rm) {
+								if (force) {
 									if (unlink(y.c_str()) != 0) {
 										perror(y.c_str());
 										exit(EXIT_FAILURE);

dirtiles.hpp

@@ -7,6 +7,6 @@ std::string dir_read_tile(std::string pbfPath);
 
 void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);
 
-void check_dir(const char *d, bool rm);
+void check_dir(const char *d, bool force, bool forcetable);
 
 #endif

main.cpp

@@ -1142,11 +1142,15 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 			// Trim .json or .mbtiles from the name
 			ssize_t cp;
 			cp = trunc.find(".json");
-			if (cp >= 0) {
+			if (cp >= 0 && (size_t) cp + 5 == trunc.size()) {
+				trunc = trunc.substr(0, cp);
+			}
+			cp = trunc.find(".geojson");
+			if (cp >= 0 && (size_t) cp + 8 == trunc.size()) {
 				trunc = trunc.substr(0, cp);
 			}
 			cp = trunc.find(".mbtiles");
-			if (cp >= 0) {
+			if (cp >= 0 && (size_t) cp + 8 == trunc.size()) {
 				trunc = trunc.substr(0, cp);
 			}
 
@@ -2664,10 +2668,7 @@ int main(int argc, char **argv) {
 		outdb = mbtiles_open(out_mbtiles, argv, forcetable);
 	}
 	if (out_dir != NULL) {
-		if (force) {
-			check_dir(out_dir, true);
-		}
-		check_dir(out_dir, false);
+		check_dir(out_dir, force, forcetable);
 	}
 
 	int ret = EXIT_SUCCESS;

mbtiles.cpp

@@ -11,6 +11,7 @@
 #include <string>
 #include <set>
 #include <map>
+#include <sys/stat.h>
 #include "mvt.hpp"
 #include "mbtiles.hpp"
 #include "text.hpp"
@@ -487,41 +488,47 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 
 	if (outdir != NULL) {
 		std::string metadata = std::string(outdir) + "/metadata.json";
-		FILE *fp = fopen(metadata.c_str(), "w");
-		if (fp == NULL) {
-			perror(metadata.c_str());
-			exit(EXIT_FAILURE);
-		}
 
-		fprintf(fp, "{\n");
-
-		sqlite3_stmt *stmt;
-		bool first = true;
-		if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) {
-			while (sqlite3_step(stmt) == SQLITE_ROW) {
-				std::string key, value;
-
-				const char *k = (const char *) sqlite3_column_text(stmt, 0);
-				const char *v = (const char *) sqlite3_column_text(stmt, 1);
-				if (k == NULL || v == NULL) {
-					fprintf(stderr, "Corrupt mbtiles file: null metadata\n");
-					exit(EXIT_FAILURE);
-				}
-
-				quote(key, k);
-				quote(value, v);
-
-				if (!first) {
-					fprintf(fp, ",\n");
-				}
-				fprintf(fp, "    \"%s\": \"%s\"", key.c_str(), value.c_str());
-				first = false;
+		struct stat st;
+		if (stat(metadata.c_str(), &st) == 0) {
+			// Leave existing metadata in place with --allow-existing
+		} else {
+			FILE *fp = fopen(metadata.c_str(), "w");
+			if (fp == NULL) {
+				perror(metadata.c_str());
+				exit(EXIT_FAILURE);
 			}
-			sqlite3_finalize(stmt);
-		}
 
-		fprintf(fp, "\n}\n");
-		fclose(fp);
+			fprintf(fp, "{\n");
+
+			sqlite3_stmt *stmt;
+			bool first = true;
+			if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) {
+				while (sqlite3_step(stmt) == SQLITE_ROW) {
+					std::string key, value;
+
+					const char *k = (const char *) sqlite3_column_text(stmt, 0);
+					const char *v = (const char *) sqlite3_column_text(stmt, 1);
+					if (k == NULL || v == NULL) {
+						fprintf(stderr, "Corrupt mbtiles file: null metadata\n");
+						exit(EXIT_FAILURE);
+					}
+
+					quote(key, k);
+					quote(value, v);
+
+					if (!first) {
+						fprintf(fp, ",\n");
+					}
+					fprintf(fp, "    \"%s\": \"%s\"", key.c_str(), value.c_str());
+					first = false;
+				}
+				sqlite3_finalize(stmt);
+			}
+
+			fprintf(fp, "\n}\n");
+			fclose(fp);
+		}
 	}
 
 	if (outdb == NULL) {

tests/raw-tiles/compare/metadata.json

@@ -8,5 +8,5 @@
     "bounds": "-122.682427,45.512331,-122.654961,45.569975",
     "type": "overlay",
     "format": "pbf",
-    "json": "{\"vector_layers\": [ { \"id\": \"hackspotsgeojson\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 14, \"fields\": {\"Address\": \"String\", \"Name\": \"String\", \"Notes\": \"String\"} } ],\"tilestats\": {\"layerCount\": 1,\"layers\": [{\"layer\": \"hackspotsgeojson\",\"count\": 4,\"geometry\": \"Point\",\"attributeCount\": 3,\"attributes\": [{\"attribute\": \"Address\",\"count\": 4,\"type\": \"string\",\"values\": [\"1507 N Rosa Parks Way  Portland, OR 97217\",\"201 SE 12th Ave, Portland, OR 97214\",\"4637 N Albina Ave  Portland, OR 97217\",\"915 SE Hawthorne Blvd. Portland, OR 97214\"]},{\"attribute\": \"Name\",\"count\": 4,\"type\": \"string\",\"values\": [\"Albina Press\",\"Arbor Lodge\",\"Lucky Labrador Brew Pub\",\"Three Friends Coffeehouse\"]},{\"attribute\": \"Notes\",\"count\": 3,\"type\": \"string\",\"values\": [\"\",\"Dog friendly\",\"usually busy, outlets on side wall only\"]}]}]}}"
+    "json": "{\"vector_layers\": [ { \"id\": \"hackspots\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 14, \"fields\": {\"Address\": \"String\", \"Name\": \"String\", \"Notes\": \"String\"} } ],\"tilestats\": {\"layerCount\": 1,\"layers\": [{\"layer\": \"hackspots\",\"count\": 4,\"geometry\": \"Point\",\"attributeCount\": 3,\"attributes\": [{\"attribute\": \"Address\",\"count\": 4,\"type\": \"string\",\"values\": [\"1507 N Rosa Parks Way  Portland, OR 97217\",\"201 SE 12th Ave, Portland, OR 97214\",\"4637 N Albina Ave  Portland, OR 97217\",\"915 SE Hawthorne Blvd. Portland, OR 97214\"]},{\"attribute\": \"Name\",\"count\": 4,\"type\": \"string\",\"values\": [\"Albina Press\",\"Arbor Lodge\",\"Lucky Labrador Brew Pub\",\"Three Friends Coffeehouse\"]},{\"attribute\": \"Notes\",\"count\": 3,\"type\": \"string\",\"values\": [\"\",\"Dog friendly\",\"usually busy, outlets on side wall only\"]}]}]}}"
 }

tile-join.cpp

@@ -1173,10 +1173,7 @@ int main(int argc, char **argv) {
 		outdb = mbtiles_open(out_mbtiles, argv, 0);
 	}
 	if (out_dir != NULL) {
-		if (force) {
-			check_dir(out_dir, true);
-		}
-		check_dir(out_dir, false);
+		check_dir(out_dir, force, false);
 	}
 
 	struct stats st;

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "tippecanoe v1.27.1\n"
+#define VERSION "tippecanoe v1.27.2\n"
 
 #endif