From 1acd7717438a32696f0d712df43e69b64bfbc775 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 30 Nov 2017 15:37:46 -0800 Subject: [PATCH 1/3] Factor out code for enumerating the tiles in a directory --- Makefile | 2 +- decode.cpp | 101 +-------------------- dirtiles.cpp | 51 +++++++---- dirtiles.hpp | 38 +++++++- tile-join.cpp | 246 +++++++------------------------------------------- 5 files changed, 106 insertions(+), 332 deletions(-) diff --git a/Makefile b/Makefile index 7d5c5ea..c5c9c84 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o pro tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 -tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o +tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o diff --git a/decode.cpp b/decode.cpp index 6a265c1..bf8a99c 100644 --- a/decode.cpp +++ b/decode.cpp @@ -21,6 +21,7 @@ #include "geometry.hpp" #include "write_json.hpp" #include "jsonpull/jsonpull.h" +#include "dirtiles.hpp" int minzoom = 0; int maxzoom = 32; @@ -142,56 +143,6 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st } } -struct zxy { - int z; - int x; - int y; - - zxy(int _z, int _x, int _y) - : z(_z), x(_x), y(_y) { - } - - bool operator<(const zxy &other) const { - if (z < other.z) { - return true; - } - if (z == other.z) { - if (x < other.x) { - return true; - } - if (x == other.x) { - if (y > other.y) { - return true; // reversed for TMS - } - } - } - - return false; - } -}; - -// XXX deduplicate from dirtiles -bool numeric(const char *s) { - if (*s == '\0') { - return false; - } - for (; *s != 0; s++) { - if (*s < '0' || *s > '9') { - return false; - } - } - return true; -} - -// XXX deduplicate from dirtiles -bool pbfname(const char *s) { - while (*s >= '0' && *s <= '9') { - s++; - } - - return strcmp(s, ".pbf") == 0; -} - sqlite3 *meta2tmp(const char *fname, std::vector &tiles) { sqlite3 *db; char *err = NULL; @@ -237,53 +188,7 @@ sqlite3 *meta2tmp(const char *fname, std::vector &tiles) { json_end(jp); fclose(f); - // XXX deduplicate from dirtiles - DIR *d1 = opendir(fname); - if (d1 != NULL) { - struct dirent *dp; - while ((dp = readdir(d1)) != NULL) { - if (numeric(dp->d_name)) { - std::string z = std::string(fname) + "/" + dp->d_name; - int tz = atoi(dp->d_name); - - DIR *d2 = opendir(z.c_str()); - if (d2 == NULL) { - perror(z.c_str()); - exit(EXIT_FAILURE); - } - - struct dirent *dp2; - while ((dp2 = readdir(d2)) != NULL) { - if (numeric(dp2->d_name)) { - std::string x = z + "/" + dp2->d_name; - int tx = atoi(dp2->d_name); - - DIR *d3 = opendir(x.c_str()); - if (d3 == NULL) { - perror(x.c_str()); - exit(EXIT_FAILURE); - } - - struct dirent *dp3; - while ((dp3 = readdir(d3)) != NULL) { - if (pbfname(dp3->d_name)) { - int ty = atoi(dp3->d_name); - tiles.push_back(zxy(tz, tx, ty)); - } - } - - closedir(d3); - } - } - - closedir(d2); - } - } - - closedir(d1); - } - - std::sort(tiles.begin(), tiles.end()); + tiles = enumerate_dirtiles(fname); return db; } @@ -397,7 +302,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co within = 1; } - std::string fn = std::string(fname) + "/" + std::to_string(tiles[i].z) + "/" + std::to_string(tiles[i].x) + "/" + std::to_string(tiles[i].y) + ".pbf"; + std::string fn = std::string(fname) + "/" + tiles[i].path(); FILE *f = fopen(fn.c_str(), "rb"); if (f == NULL) { perror(fn.c_str()); diff --git a/dirtiles.cpp b/dirtiles.cpp index 778df6c..5a0f7c6 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -9,8 +10,8 @@ #include #include "dirtiles.hpp" -std::string dir_read_tile(std::string pbfPath) { - std::ifstream pbfFile(pbfPath, std::ios::in | std::ios::binary); +std::string dir_read_tile(std::string base, struct zxy tile) { + std::ifstream pbfFile(base + "/" + tile.path(), std::ios::in | std::ios::binary); std::ostringstream contents; contents << pbfFile.rdbuf(); pbfFile.close(); @@ -39,7 +40,7 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const pbfFile.close(); } -bool numeric(const char *s) { +static bool numeric(const char *s) { if (*s == '\0') { return false; } @@ -51,7 +52,7 @@ bool numeric(const char *s) { return true; } -bool pbfname(const char *s) { +static bool pbfname(const char *s) { while (*s >= '0' && *s <= '9') { s++; } @@ -79,12 +80,33 @@ void check_dir(const char *dir, bool force, bool forcetable) { return; } - DIR *d1 = opendir(dir); + std::vector tiles = enumerate_dirtiles(dir); + + for (size_t i = 0; i < tiles.size(); i++) { + std::string fn = std::string(dir) + "/" + tiles[i].path(); + + if (force) { + if (unlink(fn.c_str()) != 0) { + perror(fn.c_str()); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "%s: file exists\n", fn.c_str()); + exit(EXIT_FAILURE); + } + } +} + +std::vector enumerate_dirtiles(const char *fname) { + std::vector tiles; + + DIR *d1 = opendir(fname); if (d1 != NULL) { struct dirent *dp; while ((dp = readdir(d1)) != NULL) { if (numeric(dp->d_name)) { - std::string z = std::string(dir) + "/" + dp->d_name; + std::string z = std::string(fname) + "/" + dp->d_name; + int tz = atoi(dp->d_name); DIR *d2 = opendir(z.c_str()); if (d2 == NULL) { @@ -96,6 +118,7 @@ void check_dir(const char *dir, bool force, bool forcetable) { while ((dp2 = readdir(d2)) != NULL) { if (numeric(dp2->d_name)) { std::string x = z + "/" + dp2->d_name; + int tx = atoi(dp2->d_name); DIR *d3 = opendir(x.c_str()); if (d3 == NULL) { @@ -106,17 +129,8 @@ void check_dir(const char *dir, bool force, bool forcetable) { struct dirent *dp3; while ((dp3 = readdir(d3)) != NULL) { if (pbfname(dp3->d_name)) { - std::string y = x + "/" + dp3->d_name; - - if (force) { - if (unlink(y.c_str()) != 0) { - perror(y.c_str()); - exit(EXIT_FAILURE); - } - } else { - fprintf(stderr, "%s: file exists\n", y.c_str()); - exit(EXIT_FAILURE); - } + int ty = atoi(dp3->d_name); + tiles.push_back(zxy(tz, tx, ty)); } } @@ -130,4 +144,7 @@ void check_dir(const char *dir, bool force, bool forcetable) { closedir(d1); } + + std::sort(tiles.begin(), tiles.end()); + return tiles; } diff --git a/dirtiles.hpp b/dirtiles.hpp index 31b3633..8ce5bda 100644 --- a/dirtiles.hpp +++ b/dirtiles.hpp @@ -1,12 +1,46 @@ #include +#include #ifndef DIRTILES_HPP #define DIRTILES_HPP -std::string dir_read_tile(std::string pbfPath); - void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf); void check_dir(const char *d, bool force, bool forcetable); +struct zxy { + long long z; + long long x; + long long y; + + zxy(int _z, int _x, int _y) + : z(_z), x(_x), y(_y) { + } + + bool operator<(const zxy &other) const { + if (z < other.z) { + return true; + } + if (z == other.z) { + if (x < other.x) { + return true; + } + if (x == other.x) { + if (y > other.y) { + return true; // reversed for TMS + } + } + } + + return false; + } + + std::string path() { + return std::to_string(z) + "/" + std::to_string(x) + "/" + std::to_string(y) + ".pbf"; + } +}; + +std::vector enumerate_dirtiles(const char *fname); +std::string dir_read_tile(std::string pbfPath, struct zxy tile); + #endif diff --git a/tile-join.cpp b/tile-join.cpp index 0c4b047..762c327 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -326,12 +326,12 @@ struct reader { long long x = 0; long long sorty = 0; long long y = 0; - int pbf_count = 0; int z_flag = 0; std::string data = ""; - std::vector pbf_path{}; - std::vector large_zoom{}; + + std::vector dirtiles; + std::string dirbase; sqlite3 *db = NULL; sqlite3_stmt *stmt = NULL; @@ -367,165 +367,29 @@ struct reader { } }; -std::vector split_slash(std::string pbf_path) { - std::vector path_parts; - std::string path(pbf_path); - std::istringstream iss(path); - std::string token; - - while (std::getline(iss, token, '/')) { - path_parts.push_back(token); - } - - return path_parts; -} - -int filter(const struct dirent *dir) { - if (strcmp(dir->d_name, ".") == 0 || strcmp(dir->d_name, "..") == 0 || strcmp(dir->d_name, ".DS_Store") == 0 || strcmp(dir->d_name, "metadata.json") == 0) { - return 0; - } else { - return 1; - } -} - -// Recursively walk through a specified directory and its subdirectories, -// using alphasort function and integer variable zoom_range to handle input in numerical order. -// Store the path of all pbf files in a pbf_path vector member of reader struct, -// with the help of a large_zoom vector and two integer members pbf_count and z_flag -// to ensure the tiles order in pbf_path to be the same as in mbtiles. -struct reader *read_dir(struct reader *readers, const char *name, int level, int zoom_range) { - struct dirent **namelist; - struct stat buf; - std::string path; - int i = 0; - int n = scandir(name, &namelist, filter, alphasort); - std::vector path_parts1, path_parts2; - readers->pbf_count = 0; - - if (n > 0) { - while (i < n) { - path = std::string(name) + "/" + std::string(namelist[i]->d_name); - - if (stat(path.c_str(), &buf) == 0 && S_ISDIR(buf.st_mode)) { - if (level == 0) { - if (std::stoi(namelist[i]->d_name) <= 9) { - zoom_range = 0; - } else { - zoom_range = 1; - } - - if (readers->pbf_count > 0) { - if (readers->z_flag == 0) { - std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater()); - } else { - std::sort(readers->large_zoom.end() - (readers->pbf_count + 1), readers->large_zoom.end(), std::greater()); - } - readers->pbf_count = 0; - } - } - - if (level == 1 && readers->pbf_count > 0) { - if (zoom_range == 0) { - std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater()); - } else { - std::sort(readers->large_zoom.end() - (readers->pbf_count + 1), readers->large_zoom.end(), std::greater()); - } - readers->pbf_count = 0; - } - - read_dir(readers, path.c_str(), level + 1, zoom_range); - } else { - if (level == 0) { - fprintf(stderr, "ERROR: Directory structure in '%s' should be zoom/x/y\n", name); - exit(EXIT_FAILURE); - } - - if (level == 1) { - fprintf(stderr, "ERROR: Directory structure in '%s' should be zoom/x/y\n", split_slash(name)[0].c_str()); - exit(EXIT_FAILURE); - } - - if (zoom_range == 0) { - readers->pbf_path.push_back(path); - - if (readers->pbf_path.size() > 1) { - path_parts1 = split_slash(readers->pbf_path[readers->pbf_path.size() - 1]); - path_parts2 = split_slash(readers->pbf_path[readers->pbf_path.size() - 2]); - int p1 = path_parts1.size(); - int p2 = path_parts2.size(); - - if (std::stoll(path_parts1[p1 - 3]) == std::stoll(path_parts2[p2 - 3]) && std::stoll(path_parts1[p1 - 2]) == std::stoll(path_parts2[p2 - 2])) { - readers->z_flag = 0; - readers->pbf_count++; - } - - path_parts1.clear(); - path_parts2.clear(); - } - } else { - readers->large_zoom.push_back(path); - - if (readers->large_zoom.size() > 1) { - path_parts1 = split_slash(readers->large_zoom[readers->large_zoom.size() - 1]); - path_parts2 = split_slash(readers->large_zoom[readers->large_zoom.size() - 2]); - int p1 = path_parts1.size(); - int p2 = path_parts2.size(); - - if (std::stoll(path_parts1[p1 - 3]) == std::stoll(path_parts2[p2 - 3]) && std::stoll(path_parts1[p1 - 2]) == std::stoll(path_parts2[p2 - 2])) { - readers->z_flag = 1; - readers->pbf_count++; - } - - path_parts1.clear(); - path_parts2.clear(); - } - } - } - - free(namelist[i]); - i++; - } - - if (level == 0) { - if (readers->pbf_count > 0) { - std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater()); - } - - readers->pbf_path.insert(std::end(readers->pbf_path), std::begin(readers->large_zoom), std::end(readers->large_zoom)); - } - - free(namelist); - } else if (n == 0) { - fprintf(stderr, "ERROR: Empty directory '%s'\n", name); - exit(EXIT_FAILURE); - } else { - perror("scandir"); - } - - return readers; -} - struct reader *begin_reading(char *fname) { - DIR *dir; struct reader *r = new reader; - if ((dir = opendir(fname)) != NULL) { - r = read_dir(r, fname, 0, 0); - - std::vector path_parts; - path_parts = split_slash(r->pbf_path[0]); - int p = path_parts.size(); + struct stat st; + if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) { r->db = NULL; r->stmt = NULL; r->next = NULL; - r->pbf_count = 0; - r->zoom = std::stoll(path_parts[p - 3]); - r->x = std::stoll(path_parts[p - 2]); - r->y = std::stoll(path_parts[p - 1].substr(0, path_parts[p - 1].find_last_of("."))); - r->sorty = (1LL << r->zoom) - 1 - r->y; - r->data = dir_read_tile(r->pbf_path[0]); - path_parts.clear(); - closedir(dir); + + r->dirtiles = enumerate_dirtiles(fname); + r->dirbase = fname; + + if (r->dirtiles.size() == 0) { + r->zoom = 32; + } else { + r->zoom = r->dirtiles[0].z; + r->x = r->dirtiles[0].x; + r->y = r->dirtiles[0].y; + r->sorty = (1LL << r->zoom) - 1 - r->y; + r->data = dir_read_tile(r->dirbase, r->dirtiles[0]); + + r->dirtiles.erase(r->dirtiles.begin()); + } } else { sqlite3 *db; @@ -564,40 +428,6 @@ struct reader *begin_reading(char *fname) { return r; } -struct zxy { - long long z; - long long x; - long long y; - - zxy(long long _z, long long _x, long long _y) - : z(_z), - x(_x), - y(_y) { - } - - bool operator<(zxy const &other) const { - if (z < other.z) { - return true; - } - if (z > other.z) { - return false; - } - - if (x < other.x) { - return true; - } - if (x > other.x) { - return false; - } - - if (y < other.y) { - return true; - } - - return false; - } -}; - struct arg { std::map> inputs{}; std::map outputs{}; @@ -775,20 +605,16 @@ void decode(struct reader *readers, std::map &layer r->zoom = 32; } } else { - r->pbf_count++; - - if (r->pbf_count != static_cast(r->pbf_path.size())) { - std::vector path_parts; - path_parts = split_slash(r->pbf_path[r->pbf_count]); - int p = path_parts.size(); - r->zoom = std::stoll(path_parts[p - 3]); - r->x = std::stoll(path_parts[p - 2]); - r->y = std::stoll(path_parts[p - 1].substr(0, path_parts[p - 1].find_last_of("."))); - r->sorty = (1LL << r->zoom) - 1 - r->y; - r->data = dir_read_tile(r->pbf_path[r->pbf_count]); - path_parts.clear(); - } else { + if (r->dirtiles.size() == 0) { r->zoom = 32; + } else { + r->zoom = r->dirtiles[0].z; + r->x = r->dirtiles[0].x; + r->y = r->dirtiles[0].y; + r->sorty = (1LL << r->zoom) - 1 - r->y; + r->data = dir_read_tile(r->dirbase, r->dirtiles[0]); + + r->dirtiles.erase(r->dirtiles.begin()); } } @@ -894,24 +720,16 @@ void decode(struct reader *readers, std::map &layer } } else { - std::vector path_parts; - path_parts = split_slash(r->pbf_path[0]); - std::string metadata_path = path_parts[0]; + std::string metadata_path = r->dirbase + "/metadata.json"; - for (int i = 1; i < static_cast(path_parts.size()) - 3; i++) { - metadata_path = metadata_path + "/" + path_parts[i]; - } - - metadata_path += "/metadata.json"; - - path_parts.clear(); FILE *f = fopen(metadata_path.c_str(), "r"); - if (f == NULL) { perror(metadata_path.c_str()); exit(EXIT_FAILURE); } + // XXX unify metadata reading + json_pull *jp = json_begin_file(f); json_object *j, *k; From d7f44ab0825fb1777e771db5fc9cf715548e93c8 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 30 Nov 2017 15:49:14 -0800 Subject: [PATCH 2/3] Factor out code for reading metadata.json --- CHANGELOG.md | 4 + decode.cpp | 52 +------------ dirtiles.cpp | 49 ++++++++++++ dirtiles.hpp | 1 + tile-join.cpp | 211 ++++++++++++++++++-------------------------------- version.hpp | 2 +- 6 files changed, 131 insertions(+), 188 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0960376..cb30ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.27.3 + +* Clean up duplicated code for reading tiles from a directory + ## 1.27.2 * Tippecanoe-decode can decode directories of tiles, not just mbtiles diff --git a/decode.cpp b/decode.cpp index bf8a99c..3959739 100644 --- a/decode.cpp +++ b/decode.cpp @@ -143,55 +143,6 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st } } -sqlite3 *meta2tmp(const char *fname, std::vector &tiles) { - sqlite3 *db; - char *err = NULL; - - if (sqlite3_open("", &db) != SQLITE_OK) { - fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db)); - exit(EXIT_FAILURE); - } - if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "Create metadata table: %s\n", err); - exit(EXIT_FAILURE); - } - - std::string name = fname; - name += "/metadata.json"; - - FILE *f = fopen(name.c_str(), "r"); - if (f == NULL) { - perror(name.c_str()); - exit(EXIT_FAILURE); - } - - json_pull *jp = json_begin_file(f); - json_object *o = json_read_tree(jp); - - if (o->type != JSON_HASH) { - fprintf(stderr, "%s: bad metadata format\n", name.c_str()); - exit(EXIT_FAILURE); - } - - for (size_t i = 0; i < o->length; i++) { - if (o->keys[i]->type != JSON_STRING || o->values[i]->type != JSON_STRING) { - fprintf(stderr, "%s: non-string in metadata\n", name.c_str()); - } - - char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->keys[i]->string, o->values[i]->string); - if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", o->keys[i]->string, err); - } - sqlite3_free(sql); - } - - json_end(jp); - fclose(f); - - tiles = enumerate_dirtiles(fname); - return db; -} - void decode(char *fname, int z, unsigned x, unsigned y, std::set const &to_decode, bool pipeline, bool stats) { sqlite3 *db = NULL; bool isdir = false; @@ -235,7 +186,8 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) { isdir = true; - db = meta2tmp(fname, tiles); + db = dirmeta2tmp(fname); + tiles = enumerate_dirtiles(fname); } else { if (sqlite3_open(fname, &db) != SQLITE_OK) { fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db)); diff --git a/dirtiles.cpp b/dirtiles.cpp index 5a0f7c6..757b1c6 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include "jsonpull/jsonpull.h" #include "dirtiles.hpp" std::string dir_read_tile(std::string base, struct zxy tile) { @@ -148,3 +150,50 @@ std::vector enumerate_dirtiles(const char *fname) { std::sort(tiles.begin(), tiles.end()); return tiles; } + +sqlite3 *dirmeta2tmp(const char *fname) { + sqlite3 *db; + char *err = NULL; + + if (sqlite3_open("", &db) != SQLITE_OK) { + fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "Create metadata table: %s\n", err); + exit(EXIT_FAILURE); + } + + std::string name = fname; + name += "/metadata.json"; + + FILE *f = fopen(name.c_str(), "r"); + if (f == NULL) { + perror(name.c_str()); + exit(EXIT_FAILURE); + } + + json_pull *jp = json_begin_file(f); + json_object *o = json_read_tree(jp); + + if (o->type != JSON_HASH) { + fprintf(stderr, "%s: bad metadata format\n", name.c_str()); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; i < o->length; i++) { + if (o->keys[i]->type != JSON_STRING || o->values[i]->type != JSON_STRING) { + fprintf(stderr, "%s: non-string in metadata\n", name.c_str()); + } + + char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->keys[i]->string, o->values[i]->string); + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "set %s in metadata: %s\n", o->keys[i]->string, err); + } + sqlite3_free(sql); + } + + json_end(jp); + fclose(f); + return db; +} diff --git a/dirtiles.hpp b/dirtiles.hpp index 8ce5bda..2a39e08 100644 --- a/dirtiles.hpp +++ b/dirtiles.hpp @@ -41,6 +41,7 @@ struct zxy { }; std::vector enumerate_dirtiles(const char *fname); +sqlite3 *dirmeta2tmp(const char *fname); std::string dir_read_tile(std::string pbfPath, struct zxy tile); #endif diff --git a/tile-join.cpp b/tile-join.cpp index 762c327..1f1762b 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -642,149 +642,86 @@ void decode(struct reader *readers, std::map &layer for (struct reader *r = readers; r != NULL; r = next) { next = r->next; - if (r->db != NULL) { - sqlite3_finalize(r->stmt); - - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'minzoom'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - int minz = max(sqlite3_column_int(r->stmt, 0), minzoom); - st->minzoom = min(st->minzoom, minz); - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'maxzoom'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - int maxz = min(sqlite3_column_int(r->stmt, 0), maxzoom); - st->maxzoom = max(st->maxzoom, maxz); - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'center'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat); - } - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'attribution'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - attribution = std::string((char *) s); - } - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'description'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - description = std::string((char *) s); - } - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'name'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - if (name.size() == 0) { - name = std::string((char *) s); - } else { - name += " + " + std::string((char *) s); - } - } - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'bounds'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) { - st->minlon = min(minlon, st->minlon); - st->maxlon = max(maxlon, st->maxlon); - st->minlat = min(minlat, st->minlat); - st->maxlat = max(maxlat, st->maxlat); - } - } - } - sqlite3_finalize(r->stmt); - } - - if (sqlite3_close(r->db) != SQLITE_OK) { - fprintf(stderr, "Could not close database: %s\n", sqlite3_errmsg(r->db)); - exit(EXIT_FAILURE); - } - + sqlite3 *db = r->db; + if (db == NULL) { + db = dirmeta2tmp(r->dirbase.c_str()); } else { - std::string metadata_path = r->dirbase + "/metadata.json"; + sqlite3_finalize(r->stmt); + } - FILE *f = fopen(metadata_path.c_str(), "r"); - if (f == NULL) { - perror(metadata_path.c_str()); - exit(EXIT_FAILURE); + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'minzoom'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + int minz = max(sqlite3_column_int(r->stmt, 0), minzoom); + st->minzoom = min(st->minzoom, minz); } - - // XXX unify metadata reading - - json_pull *jp = json_begin_file(f); - json_object *j, *k; - - while ((j = json_read(jp)) != NULL) { - if (j->type == JSON_HASH) { - if ((k = json_hash_get(j, "minzoom")) != NULL) { - const std::string minzoom_tmp = k->string; - int minz = max(std::stoi(minzoom_tmp), minzoom); - st->minzoom = min(st->minzoom, minz); - } - - if ((k = json_hash_get(j, "maxzoom")) != NULL) { - const std::string maxzoom_tmp = k->string; - int maxz = min(std::stoi(maxzoom_tmp), maxzoom); - st->maxzoom = max(st->maxzoom, maxz); - } - - if ((k = json_hash_get(j, "center")) != NULL) { - const std::string center = k->string; - const unsigned char *s = (const unsigned char *) center.c_str(); - sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat); - } - - if ((k = json_hash_get(j, "attribution")) != NULL) { - attribution = k->string; - } - - if ((k = json_hash_get(j, "description")) != NULL) { - description = k->string; - } - - if ((k = json_hash_get(j, "name")) != NULL) { - const std::string name_tmp = k->string; - if (name.size() == 0) { - name = name_tmp; - } else { - name += " + " + name_tmp; - } - } - - if ((k = json_hash_get(j, "bounds")) != NULL) { - const std::string bounds = k->string; - const unsigned char *s = (const unsigned char *) bounds.c_str(); - if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) { - st->minlon = min(minlon, st->minlon); - st->maxlon = max(maxlon, st->maxlon); - st->minlat = min(minlat, st->minlat); - st->maxlat = max(maxlat, st->maxlat); - } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'maxzoom'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + int maxz = min(sqlite3_column_int(r->stmt, 0), maxzoom); + st->maxzoom = max(st->maxzoom, maxz); + } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'center'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat); + } + } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'attribution'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + attribution = std::string((char *) s); + } + } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'description'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + description = std::string((char *) s); + } + } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'name'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + if (name.size() == 0) { + name = std::string((char *) s); + } else { + name += " + " + std::string((char *) s); } } } - json_free(j); - json_end(jp); - fclose(f); + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'bounds'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) { + st->minlon = min(minlon, st->minlon); + st->maxlon = max(maxlon, st->maxlon); + st->minlat = min(minlat, st->minlat); + st->maxlat = max(maxlat, st->maxlat); + } + } + } + sqlite3_finalize(r->stmt); + } + + // Closes either real db or temp mirror of metadata.json + if (sqlite3_close(db) != SQLITE_OK) { + fprintf(stderr, "Could not close database: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); } delete r; diff --git a/version.hpp b/version.hpp index 70895f9..f931915 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "tippecanoe v1.27.2\n" +#define VERSION "tippecanoe v1.27.3\n" #endif From e4743d95278e3d7a4fa9a9c2ca6396ee047ed0e8 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 30 Nov 2017 15:56:35 -0800 Subject: [PATCH 3/3] Test successful and failed attempts to write to existing tilesets --- Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Makefile b/Makefile index c5c9c84..6c87a13 100644 --- a/Makefile +++ b/Makefile @@ -240,10 +240,20 @@ json-tool-test: tippecanoe-json-tool rm -f tests/join-population/tabblock_06001420.json.sort tests/join-population/tabblock_06001420.json.sort.joined allow-existing-test: + # Make a tileset + ./tippecanoe -Z0 -z0 -f -o tests/allow-existing/both.mbtiles tests/coalesce-tract/tl_2010_06001_tract10.json + # Writing to existing should fail + if ./tippecanoe -Z1 -z1 -o tests/allow-existing/both.mbtiles tests/coalesce-tract/tl_2010_06001_tract10.json; then exit 1; else exit 0; fi + # Replace existing ./tippecanoe -Z8 -z9 -f -o tests/allow-existing/both.mbtiles tests/coalesce-tract/tl_2010_06001_tract10.json ./tippecanoe -Z10 -z11 -F -o tests/allow-existing/both.mbtiles tests/coalesce-tract/tl_2010_06001_tract10.json ./tippecanoe-decode tests/allow-existing/both.mbtiles > tests/allow-existing/both.mbtiles.json.check cmp tests/allow-existing/both.mbtiles.json.check tests/allow-existing/both.mbtiles.json + # Make a tileset + ./tippecanoe -Z0 -z0 -f -e tests/allow-existing/both.dir tests/coalesce-tract/tl_2010_06001_tract10.json + # Writing to existing should fail + if ./tippecanoe -Z1 -z1 -e tests/allow-existing/both.dir tests/coalesce-tract/tl_2010_06001_tract10.json; then exit 1; else exit 0; fi + # Replace existing ./tippecanoe -Z8 -z9 -f -e tests/allow-existing/both.dir tests/coalesce-tract/tl_2010_06001_tract10.json ./tippecanoe -Z10 -z11 -F -e tests/allow-existing/both.dir tests/coalesce-tract/tl_2010_06001_tract10.json ./tippecanoe-decode tests/allow-existing/both.dir | sed 's/both\.dir/both.mbtiles/g' > tests/allow-existing/both.dir.json.check