From 54cbc570f1b0890d28e2c5dc2744f603815ff36d Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Jul 2017 14:17:09 -0700 Subject: [PATCH] Don't write to a directory that contains an existing tileset. The -f option will clear the directory now, as with mbtiles. --- Makefile | 18 ++--- dirtiles.cpp | 94 ++++++++++++++++++++++ dirtiles.hpp | 2 + main.cpp | 25 +++--- mbtiles.cpp | 5 ++ tests/join-population/windows.mbtiles.json | 2 +- tile-join.cpp | 6 ++ 7 files changed, 133 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index f883f06..8c42903 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,7 @@ parallel-test: rm tests/parallel/*.mbtiles tests/parallel/*.json raw-tiles-test: - ./tippecanoe -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC + ./tippecanoe -f -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC diff -x '*.DS_Store' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare rm -rf tests/raw-tiles/raw-tiles @@ -163,25 +163,25 @@ join-test: ./tippecanoe-decode tests/join-population/no-macarthur.mbtiles > tests/join-population/no-macarthur.mbtiles.json.check cmp tests/join-population/just-macarthur.mbtiles.json.check tests/join-population/just-macarthur.mbtiles.json cmp tests/join-population/no-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json - ./tile-join --no-tile-compression -e tests/join-population/raw-merged-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles + ./tile-join --no-tile-compression -f -e tests/join-population/raw-merged-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles diff -x '*.DS_Store' -rq tests/join-population/raw-merged-folder tests/join-population/raw-merged-folder-compare - ./tippecanoe -z12 -e tests/join-population/tabblock_06001420-folder tests/join-population/tabblock_06001420.json - ./tippecanoe -Z5 -z10 -e tests/join-population/macarthur-folder -l macarthur tests/join-population/macarthur.json - ./tippecanoe -d10 -D10 -Z9 -z11 -e tests/join-population/macarthur2-folder -l macarthur tests/join-population/macarthur2.json + ./tippecanoe -z12 -f -e tests/join-population/tabblock_06001420-folder tests/join-population/tabblock_06001420.json + ./tippecanoe -Z5 -z10 -f -e tests/join-population/macarthur-folder -l macarthur tests/join-population/macarthur.json + ./tippecanoe -d10 -D10 -Z9 -z11 -f -e tests/join-population/macarthur2-folder -l macarthur tests/join-population/macarthur2.json ./tile-join -f -o tests/join-population/merged-folder.mbtiles tests/join-population/tabblock_06001420-folder tests/join-population/macarthur-folder tests/join-population/macarthur2-folder ./tippecanoe-decode tests/join-population/merged-folder.mbtiles > tests/join-population/merged-folder.mbtiles.json.check cmp tests/join-population/merged-folder.mbtiles.json.check tests/join-population/merged-folder.mbtiles.json - ./tile-join -n "merged name" -N "merged description" -e tests/join-population/merged-mbtiles-to-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles - ./tile-join -n "merged name" -N "merged description" -e tests/join-population/merged-folders-to-folder tests/join-population/tabblock_06001420-folder tests/join-population/macarthur-folder tests/join-population/macarthur2-folder + ./tile-join -n "merged name" -N "merged description" -f -e tests/join-population/merged-mbtiles-to-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles + ./tile-join -n "merged name" -N "merged description" -f -e tests/join-population/merged-folders-to-folder tests/join-population/tabblock_06001420-folder tests/join-population/macarthur-folder tests/join-population/macarthur2-folder diff -x '*.DS_Store' -rq tests/join-population/merged-mbtiles-to-folder tests/join-population/merged-folders-to-folder ./tile-join -f -c tests/join-population/windows.csv -o tests/join-population/windows-merged.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder - ./tile-join -c tests/join-population/windows.csv -e tests/join-population/windows-merged-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder + ./tile-join -c tests/join-population/windows.csv -f -e tests/join-population/windows-merged-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder ./tile-join -f -o tests/join-population/windows-merged2.mbtiles tests/join-population/windows-merged-folder ./tippecanoe-decode tests/join-population/windows-merged.mbtiles > tests/join-population/windows-merged.mbtiles.json.check ./tippecanoe-decode tests/join-population/windows-merged2.mbtiles > tests/join-population/windows-merged2.mbtiles.json.check cmp tests/join-population/windows-merged.mbtiles.json.check tests/join-population/windows-merged2.mbtiles.json.check ./tile-join -f -o tests/join-population/macarthur-and-macarthur2-merged.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder - ./tile-join -e tests/join-population/macarthur-and-macarthur2-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder + ./tile-join -f -e tests/join-population/macarthur-and-macarthur2-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder ./tile-join -f -o tests/join-population/macarthur-and-macarthur2-merged2.mbtiles tests/join-population/macarthur-and-macarthur2-folder ./tippecanoe-decode tests/join-population/macarthur-and-macarthur2-merged.mbtiles > tests/join-population/macarthur-and-macarthur2-merged.mbtiles.json.check ./tippecanoe-decode tests/join-population/macarthur-and-macarthur2-merged2.mbtiles > tests/join-population/macarthur-and-macarthur2-merged2.mbtiles.json.check diff --git a/dirtiles.cpp b/dirtiles.cpp index 5e48907..8497b09 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include "dirtiles.hpp" std::string dir_read_tile(std::string pbfPath) { @@ -24,7 +26,99 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); newdir = newdir + "/" + std::to_string(ty) + ".pbf"; + struct stat st; + if (stat(newdir.c_str(), &st) == 0) { + fprintf(stderr, "Can't write tile to already existing %s\n", newdir.c_str()); + exit(EXIT_FAILURE); + } + std::ofstream pbfFile(newdir, std::ios::out | std::ios::binary); pbfFile.write(pbf.data(), pbf.size()); pbfFile.close(); } + +bool numeric(const char *s) { + if (*s == '\0') { + return false; + } + for (; *s != 0; s++) { + if (*s < '0' || *s > '9') { + return false; + } + } + return true; +} + +bool pbfname(const char *s) { + while (*s >= '0' && *s <= '9') { + s++; + } + + return strcmp(s, ".pbf") == 0; +} + +void check_dir(const char *dir, bool rm) { + struct stat st; + + std::string meta = std::string(dir) + "/" + "metadata.json"; + if (rm) { + unlink(meta.c_str()); // error OK since it may not exist; + } else { + if (stat(meta.c_str(), &st) == 0) { + fprintf(stderr, "%s: file exists\n", meta.c_str()); + exit(EXIT_FAILURE); + } + } + + DIR *d1 = opendir(dir); + if (d1 != NULL) { + struct dirent *dp; + while ((dp = readdir(d1)) != NULL) { + if (numeric(dp->d_name)) { + std::string z = std::string(dir) + "/" + dp->d_name; + + DIR *d2 = opendir(z.c_str()); + if (d2 == NULL) { + perror(z.c_str()); + exit(EXIT_FAILURE); + } + + struct dirent *dp2; + while ((dp2 = readdir(d2)) != NULL) { + if (numeric(dp2->d_name)) { + std::string x = z + "/" + dp2->d_name; + + DIR *d3 = opendir(x.c_str()); + if (d3 == NULL) { + perror(x.c_str()); + exit(EXIT_FAILURE); + } + + struct dirent *dp3; + while ((dp3 = readdir(d3)) != NULL) { + if (pbfname(dp3->d_name)) { + std::string y = x + "/" + dp3->d_name; + + if (rm) { + if (unlink(y.c_str()) != 0) { + perror(y.c_str()); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "%s: file exists\n", y.c_str()); + exit(EXIT_FAILURE); + } + } + } + + closedir(d3); + } + } + + closedir(d2); + } + } + + closedir(d1); + } +} diff --git a/dirtiles.hpp b/dirtiles.hpp index 8447484..a3de88c 100644 --- a/dirtiles.hpp +++ b/dirtiles.hpp @@ -7,4 +7,6 @@ std::string dir_read_tile(std::string pbfPath); void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf); +void check_dir(const char *d, bool rm); + #endif diff --git a/main.cpp b/main.cpp index b09945b..fb5fbe7 100644 --- a/main.cpp +++ b/main.cpp @@ -49,6 +49,7 @@ #include "serial.hpp" #include "options.hpp" #include "mvt.hpp" +#include "dirtiles.hpp" static int low_detail = 12; static int full_detail = -1; @@ -2037,7 +2038,7 @@ int main(int argc, char **argv) { char *description = NULL; char *layername = NULL; char *out_mbtiles = NULL; - char *out_directory = NULL; + char *out_dir = NULL; sqlite3 *outdb = NULL; int maxzoom = 14; int minzoom = 0; @@ -2293,8 +2294,8 @@ int main(int argc, char **argv) { fprintf(stderr, "%s: Can't specify both %s and %s as output\n", argv[0], out_mbtiles, optarg); exit(EXIT_FAILURE); } - if (out_directory != NULL) { - fprintf(stderr, "%s: Can't specify both %s and %s as output\n", argv[0], out_directory, optarg); + if (out_dir != NULL) { + fprintf(stderr, "%s: Can't specify both %s and %s as output\n", argv[0], out_dir, optarg); exit(EXIT_FAILURE); } out_mbtiles = optarg; @@ -2305,11 +2306,11 @@ int main(int argc, char **argv) { fprintf(stderr, "%s: Can't specify both %s and %s as output\n", argv[0], out_mbtiles, optarg); exit(EXIT_FAILURE); } - if (out_directory != NULL) { - fprintf(stderr, "%s: Can't specify both %s and %s as output\n", argv[0], out_directory, optarg); + if (out_dir != NULL) { + fprintf(stderr, "%s: Can't specify both %s and %s as output\n", argv[0], out_dir, optarg); exit(EXIT_FAILURE); } - out_directory = optarg; + out_dir = optarg; break; case 'x': @@ -2508,12 +2509,12 @@ int main(int argc, char **argv) { fprintf(stderr, "Forcing -g0 since -B or -r is not known\n"); } - if (out_mbtiles == NULL && out_directory == NULL) { + if (out_mbtiles == NULL && out_dir == NULL) { fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory\n", argv[0]); exit(EXIT_FAILURE); } - if (out_mbtiles != NULL && out_directory != NULL) { + if (out_mbtiles != NULL && out_dir != NULL) { fprintf(stderr, "%s: Options -o and -e cannot be used together\n", argv[0]); exit(EXIT_FAILURE); } @@ -2525,6 +2526,12 @@ int main(int argc, char **argv) { outdb = mbtiles_open(out_mbtiles, argv, forcetable); } + if (out_dir != NULL) { + if (force) { + check_dir(out_dir, true); + } + check_dir(out_dir, false); + } int ret = EXIT_SUCCESS; @@ -2550,7 +2557,7 @@ int main(int argc, char **argv) { long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0}; - ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_directory, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_directory, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description, guess_maxzoom, &attribute_types); + ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_dir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description, guess_maxzoom, &attribute_types); if (outdb != NULL) { mbtiles_close(outdb, argv); diff --git a/mbtiles.cpp b/mbtiles.cpp index 47246ce..f34b819 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -461,6 +461,11 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); + FILE *f = fopen("/tmp/stats.json", "w"); + std::string st = tilestats(layermap); + fprintf(f, "%s\n", st.c_str()); + fclose(f); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tilestats', %Q);", tilestats(layermap).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set tilestats: %s\n", err); diff --git a/tests/join-population/windows.mbtiles.json b/tests/join-population/windows.mbtiles.json index 88ed582..88fbe93 100644 --- a/tests/join-population/windows.mbtiles.json +++ b/tests/join-population/windows.mbtiles.json @@ -7,7 +7,7 @@ "maxzoom": "10", "minzoom": "5", "name": "tests/join-population/macarthur.mbtiles", -"tilestats": "{\"layerCount\": 1,\"layers\": [{\"layer\": \"macarthur\",\"count\": 0,\"geometry\": \"Point\",\"attributeCount\": 4,\"attributes\": [{\"attribute\": \"FULLNAME\",\"count\": 3,\"type\": \"string\",\"values\": [\"Macarthur\",\"Macarthur Fwy\",\"W Macarthur\"]},{\"attribute\": \"LINEARID\",\"count\": 19,\"type\": \"string\",\"values\": [\"1102155930810\",\"1102156241736\",\"1102156510290\",\"1102157651658\",\"1102638069562\",\"1102654601627\",\"1102654601663\",\"1102654602215\",\"1102954918511\",\"1103690383700\",\"1103690474249\",\"1103690474250\",\"1103690483026\",\"1103690483032\",\"1104474748623\",\"1104486090991\",\"1104486392881\",\"first\",\"second\"]},{\"attribute\": \"MTFCC\",\"count\": 2,\"type\": \"string\",\"values\": [\"S1100\",\"S1400\"]},{\"attribute\": \"RTTYP\",\"count\": 1,\"type\": \"string\",\"values\": [\"M\"]}]}]}", +"tilestats": "{\"layerCount\": 1,\"layers\": [{\"layer\": \"macarthur\",\"count\": 0,\"geometry\": \"Point\",\"attributeCount\": 4,\"attributes\": [{\"attribute\": \"FULLNAME\",\"count\": 3,\"type\": \"string\",\"values\": [\"Macarthur\",\"Macarthur Fwy\",\"W Macarthur\"]},{\"attribute\": \"LINEARID\",\"count\": 17,\"type\": \"string\",\"values\": [\"1102155930810\",\"1102156241736\",\"1102156510290\",\"1102157651658\",\"1102638069562\",\"1102654601627\",\"1102654601663\",\"1102654602215\",\"1102954918511\",\"1103690383700\",\"1103690474249\",\"1103690474250\",\"1103690483026\",\"1103690483032\",\"1104474748623\",\"1104486090991\",\"1104486392881\"]},{\"attribute\": \"MTFCC\",\"count\": 2,\"type\": \"string\",\"values\": [\"S1100\",\"S1400\"]},{\"attribute\": \"RTTYP\",\"count\": 1,\"type\": \"string\",\"values\": [\"M\"]}]}]}", "type": "overlay", "version": "2" }, "features": [ diff --git a/tile-join.cpp b/tile-join.cpp index d3cfa70..361cf06 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -1148,6 +1148,12 @@ int main(int argc, char **argv) { } outdb = mbtiles_open(out_mbtiles, argv, 0); } + if (out_dir != NULL) { + if (force) { + check_dir(out_dir, true); + } + check_dir(out_dir, false); + } struct stats st; memset(&st, 0, sizeof(st));