From 1acd7717438a32696f0d712df43e69b64bfbc775 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 30 Nov 2017 15:37:46 -0800 Subject: [PATCH] Factor out code for enumerating the tiles in a directory --- Makefile | 2 +- decode.cpp | 101 +-------------------- dirtiles.cpp | 51 +++++++---- dirtiles.hpp | 38 +++++++- tile-join.cpp | 246 +++++++------------------------------------------- 5 files changed, 106 insertions(+), 332 deletions(-) diff --git a/Makefile b/Makefile index 7d5c5ea..c5c9c84 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o pro tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 -tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o +tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o diff --git a/decode.cpp b/decode.cpp index 6a265c1..bf8a99c 100644 --- a/decode.cpp +++ b/decode.cpp @@ -21,6 +21,7 @@ #include "geometry.hpp" #include "write_json.hpp" #include "jsonpull/jsonpull.h" +#include "dirtiles.hpp" int minzoom = 0; int maxzoom = 32; @@ -142,56 +143,6 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st } } -struct zxy { - int z; - int x; - int y; - - zxy(int _z, int _x, int _y) - : z(_z), x(_x), y(_y) { - } - - bool operator<(const zxy &other) const { - if (z < other.z) { - return true; - } - if (z == other.z) { - if (x < other.x) { - return true; - } - if (x == other.x) { - if (y > other.y) { - return true; // reversed for TMS - } - } - } - - return false; - } -}; - -// XXX deduplicate from dirtiles -bool numeric(const char *s) { - if (*s == '\0') { - return false; - } - for (; *s != 0; s++) { - if (*s < '0' || *s > '9') { - return false; - } - } - return true; -} - -// XXX deduplicate from dirtiles -bool pbfname(const char *s) { - while (*s >= '0' && *s <= '9') { - s++; - } - - return strcmp(s, ".pbf") == 0; -} - sqlite3 *meta2tmp(const char *fname, std::vector &tiles) { sqlite3 *db; char *err = NULL; @@ -237,53 +188,7 @@ sqlite3 *meta2tmp(const char *fname, std::vector &tiles) { json_end(jp); fclose(f); - // XXX deduplicate from dirtiles - DIR *d1 = opendir(fname); - if (d1 != NULL) { - struct dirent *dp; - while ((dp = readdir(d1)) != NULL) { - if (numeric(dp->d_name)) { - std::string z = std::string(fname) + "/" + dp->d_name; - int tz = atoi(dp->d_name); - - DIR *d2 = opendir(z.c_str()); - if (d2 == NULL) { - perror(z.c_str()); - exit(EXIT_FAILURE); - } - - struct dirent *dp2; - while ((dp2 = readdir(d2)) != NULL) { - if (numeric(dp2->d_name)) { - std::string x = z + "/" + dp2->d_name; - int tx = atoi(dp2->d_name); - - DIR *d3 = opendir(x.c_str()); - if (d3 == NULL) { - perror(x.c_str()); - exit(EXIT_FAILURE); - } - - struct dirent *dp3; - while ((dp3 = readdir(d3)) != NULL) { - if (pbfname(dp3->d_name)) { - int ty = atoi(dp3->d_name); - tiles.push_back(zxy(tz, tx, ty)); - } - } - - closedir(d3); - } - } - - closedir(d2); - } - } - - closedir(d1); - } - - std::sort(tiles.begin(), tiles.end()); + tiles = enumerate_dirtiles(fname); return db; } @@ -397,7 +302,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co within = 1; } - std::string fn = std::string(fname) + "/" + std::to_string(tiles[i].z) + "/" + std::to_string(tiles[i].x) + "/" + std::to_string(tiles[i].y) + ".pbf"; + std::string fn = std::string(fname) + "/" + tiles[i].path(); FILE *f = fopen(fn.c_str(), "rb"); if (f == NULL) { perror(fn.c_str()); diff --git a/dirtiles.cpp b/dirtiles.cpp index 778df6c..5a0f7c6 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -9,8 +10,8 @@ #include #include "dirtiles.hpp" -std::string dir_read_tile(std::string pbfPath) { - std::ifstream pbfFile(pbfPath, std::ios::in | std::ios::binary); +std::string dir_read_tile(std::string base, struct zxy tile) { + std::ifstream pbfFile(base + "/" + tile.path(), std::ios::in | std::ios::binary); std::ostringstream contents; contents << pbfFile.rdbuf(); pbfFile.close(); @@ -39,7 +40,7 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const pbfFile.close(); } -bool numeric(const char *s) { +static bool numeric(const char *s) { if (*s == '\0') { return false; } @@ -51,7 +52,7 @@ bool numeric(const char *s) { return true; } -bool pbfname(const char *s) { +static bool pbfname(const char *s) { while (*s >= '0' && *s <= '9') { s++; } @@ -79,12 +80,33 @@ void check_dir(const char *dir, bool force, bool forcetable) { return; } - DIR *d1 = opendir(dir); + std::vector tiles = enumerate_dirtiles(dir); + + for (size_t i = 0; i < tiles.size(); i++) { + std::string fn = std::string(dir) + "/" + tiles[i].path(); + + if (force) { + if (unlink(fn.c_str()) != 0) { + perror(fn.c_str()); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "%s: file exists\n", fn.c_str()); + exit(EXIT_FAILURE); + } + } +} + +std::vector enumerate_dirtiles(const char *fname) { + std::vector tiles; + + DIR *d1 = opendir(fname); if (d1 != NULL) { struct dirent *dp; while ((dp = readdir(d1)) != NULL) { if (numeric(dp->d_name)) { - std::string z = std::string(dir) + "/" + dp->d_name; + std::string z = std::string(fname) + "/" + dp->d_name; + int tz = atoi(dp->d_name); DIR *d2 = opendir(z.c_str()); if (d2 == NULL) { @@ -96,6 +118,7 @@ void check_dir(const char *dir, bool force, bool forcetable) { while ((dp2 = readdir(d2)) != NULL) { if (numeric(dp2->d_name)) { std::string x = z + "/" + dp2->d_name; + int tx = atoi(dp2->d_name); DIR *d3 = opendir(x.c_str()); if (d3 == NULL) { @@ -106,17 +129,8 @@ void check_dir(const char *dir, bool force, bool forcetable) { struct dirent *dp3; while ((dp3 = readdir(d3)) != NULL) { if (pbfname(dp3->d_name)) { - std::string y = x + "/" + dp3->d_name; - - if (force) { - if (unlink(y.c_str()) != 0) { - perror(y.c_str()); - exit(EXIT_FAILURE); - } - } else { - fprintf(stderr, "%s: file exists\n", y.c_str()); - exit(EXIT_FAILURE); - } + int ty = atoi(dp3->d_name); + tiles.push_back(zxy(tz, tx, ty)); } } @@ -130,4 +144,7 @@ void check_dir(const char *dir, bool force, bool forcetable) { closedir(d1); } + + std::sort(tiles.begin(), tiles.end()); + return tiles; } diff --git a/dirtiles.hpp b/dirtiles.hpp index 31b3633..8ce5bda 100644 --- a/dirtiles.hpp +++ b/dirtiles.hpp @@ -1,12 +1,46 @@ #include +#include #ifndef DIRTILES_HPP #define DIRTILES_HPP -std::string dir_read_tile(std::string pbfPath); - void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf); void check_dir(const char *d, bool force, bool forcetable); +struct zxy { + long long z; + long long x; + long long y; + + zxy(int _z, int _x, int _y) + : z(_z), x(_x), y(_y) { + } + + bool operator<(const zxy &other) const { + if (z < other.z) { + return true; + } + if (z == other.z) { + if (x < other.x) { + return true; + } + if (x == other.x) { + if (y > other.y) { + return true; // reversed for TMS + } + } + } + + return false; + } + + std::string path() { + return std::to_string(z) + "/" + std::to_string(x) + "/" + std::to_string(y) + ".pbf"; + } +}; + +std::vector enumerate_dirtiles(const char *fname); +std::string dir_read_tile(std::string pbfPath, struct zxy tile); + #endif diff --git a/tile-join.cpp b/tile-join.cpp index 0c4b047..762c327 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -326,12 +326,12 @@ struct reader { long long x = 0; long long sorty = 0; long long y = 0; - int pbf_count = 0; int z_flag = 0; std::string data = ""; - std::vector pbf_path{}; - std::vector large_zoom{}; + + std::vector dirtiles; + std::string dirbase; sqlite3 *db = NULL; sqlite3_stmt *stmt = NULL; @@ -367,165 +367,29 @@ struct reader { } }; -std::vector split_slash(std::string pbf_path) { - std::vector path_parts; - std::string path(pbf_path); - std::istringstream iss(path); - std::string token; - - while (std::getline(iss, token, '/')) { - path_parts.push_back(token); - } - - return path_parts; -} - -int filter(const struct dirent *dir) { - if (strcmp(dir->d_name, ".") == 0 || strcmp(dir->d_name, "..") == 0 || strcmp(dir->d_name, ".DS_Store") == 0 || strcmp(dir->d_name, "metadata.json") == 0) { - return 0; - } else { - return 1; - } -} - -// Recursively walk through a specified directory and its subdirectories, -// using alphasort function and integer variable zoom_range to handle input in numerical order. -// Store the path of all pbf files in a pbf_path vector member of reader struct, -// with the help of a large_zoom vector and two integer members pbf_count and z_flag -// to ensure the tiles order in pbf_path to be the same as in mbtiles. -struct reader *read_dir(struct reader *readers, const char *name, int level, int zoom_range) { - struct dirent **namelist; - struct stat buf; - std::string path; - int i = 0; - int n = scandir(name, &namelist, filter, alphasort); - std::vector path_parts1, path_parts2; - readers->pbf_count = 0; - - if (n > 0) { - while (i < n) { - path = std::string(name) + "/" + std::string(namelist[i]->d_name); - - if (stat(path.c_str(), &buf) == 0 && S_ISDIR(buf.st_mode)) { - if (level == 0) { - if (std::stoi(namelist[i]->d_name) <= 9) { - zoom_range = 0; - } else { - zoom_range = 1; - } - - if (readers->pbf_count > 0) { - if (readers->z_flag == 0) { - std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater()); - } else { - std::sort(readers->large_zoom.end() - (readers->pbf_count + 1), readers->large_zoom.end(), std::greater()); - } - readers->pbf_count = 0; - } - } - - if (level == 1 && readers->pbf_count > 0) { - if (zoom_range == 0) { - std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater()); - } else { - std::sort(readers->large_zoom.end() - (readers->pbf_count + 1), readers->large_zoom.end(), std::greater()); - } - readers->pbf_count = 0; - } - - read_dir(readers, path.c_str(), level + 1, zoom_range); - } else { - if (level == 0) { - fprintf(stderr, "ERROR: Directory structure in '%s' should be zoom/x/y\n", name); - exit(EXIT_FAILURE); - } - - if (level == 1) { - fprintf(stderr, "ERROR: Directory structure in '%s' should be zoom/x/y\n", split_slash(name)[0].c_str()); - exit(EXIT_FAILURE); - } - - if (zoom_range == 0) { - readers->pbf_path.push_back(path); - - if (readers->pbf_path.size() > 1) { - path_parts1 = split_slash(readers->pbf_path[readers->pbf_path.size() - 1]); - path_parts2 = split_slash(readers->pbf_path[readers->pbf_path.size() - 2]); - int p1 = path_parts1.size(); - int p2 = path_parts2.size(); - - if (std::stoll(path_parts1[p1 - 3]) == std::stoll(path_parts2[p2 - 3]) && std::stoll(path_parts1[p1 - 2]) == std::stoll(path_parts2[p2 - 2])) { - readers->z_flag = 0; - readers->pbf_count++; - } - - path_parts1.clear(); - path_parts2.clear(); - } - } else { - readers->large_zoom.push_back(path); - - if (readers->large_zoom.size() > 1) { - path_parts1 = split_slash(readers->large_zoom[readers->large_zoom.size() - 1]); - path_parts2 = split_slash(readers->large_zoom[readers->large_zoom.size() - 2]); - int p1 = path_parts1.size(); - int p2 = path_parts2.size(); - - if (std::stoll(path_parts1[p1 - 3]) == std::stoll(path_parts2[p2 - 3]) && std::stoll(path_parts1[p1 - 2]) == std::stoll(path_parts2[p2 - 2])) { - readers->z_flag = 1; - readers->pbf_count++; - } - - path_parts1.clear(); - path_parts2.clear(); - } - } - } - - free(namelist[i]); - i++; - } - - if (level == 0) { - if (readers->pbf_count > 0) { - std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater()); - } - - readers->pbf_path.insert(std::end(readers->pbf_path), std::begin(readers->large_zoom), std::end(readers->large_zoom)); - } - - free(namelist); - } else if (n == 0) { - fprintf(stderr, "ERROR: Empty directory '%s'\n", name); - exit(EXIT_FAILURE); - } else { - perror("scandir"); - } - - return readers; -} - struct reader *begin_reading(char *fname) { - DIR *dir; struct reader *r = new reader; - if ((dir = opendir(fname)) != NULL) { - r = read_dir(r, fname, 0, 0); - - std::vector path_parts; - path_parts = split_slash(r->pbf_path[0]); - int p = path_parts.size(); + struct stat st; + if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) { r->db = NULL; r->stmt = NULL; r->next = NULL; - r->pbf_count = 0; - r->zoom = std::stoll(path_parts[p - 3]); - r->x = std::stoll(path_parts[p - 2]); - r->y = std::stoll(path_parts[p - 1].substr(0, path_parts[p - 1].find_last_of("."))); - r->sorty = (1LL << r->zoom) - 1 - r->y; - r->data = dir_read_tile(r->pbf_path[0]); - path_parts.clear(); - closedir(dir); + + r->dirtiles = enumerate_dirtiles(fname); + r->dirbase = fname; + + if (r->dirtiles.size() == 0) { + r->zoom = 32; + } else { + r->zoom = r->dirtiles[0].z; + r->x = r->dirtiles[0].x; + r->y = r->dirtiles[0].y; + r->sorty = (1LL << r->zoom) - 1 - r->y; + r->data = dir_read_tile(r->dirbase, r->dirtiles[0]); + + r->dirtiles.erase(r->dirtiles.begin()); + } } else { sqlite3 *db; @@ -564,40 +428,6 @@ struct reader *begin_reading(char *fname) { return r; } -struct zxy { - long long z; - long long x; - long long y; - - zxy(long long _z, long long _x, long long _y) - : z(_z), - x(_x), - y(_y) { - } - - bool operator<(zxy const &other) const { - if (z < other.z) { - return true; - } - if (z > other.z) { - return false; - } - - if (x < other.x) { - return true; - } - if (x > other.x) { - return false; - } - - if (y < other.y) { - return true; - } - - return false; - } -}; - struct arg { std::map> inputs{}; std::map outputs{}; @@ -775,20 +605,16 @@ void decode(struct reader *readers, std::map &layer r->zoom = 32; } } else { - r->pbf_count++; - - if (r->pbf_count != static_cast(r->pbf_path.size())) { - std::vector path_parts; - path_parts = split_slash(r->pbf_path[r->pbf_count]); - int p = path_parts.size(); - r->zoom = std::stoll(path_parts[p - 3]); - r->x = std::stoll(path_parts[p - 2]); - r->y = std::stoll(path_parts[p - 1].substr(0, path_parts[p - 1].find_last_of("."))); - r->sorty = (1LL << r->zoom) - 1 - r->y; - r->data = dir_read_tile(r->pbf_path[r->pbf_count]); - path_parts.clear(); - } else { + if (r->dirtiles.size() == 0) { r->zoom = 32; + } else { + r->zoom = r->dirtiles[0].z; + r->x = r->dirtiles[0].x; + r->y = r->dirtiles[0].y; + r->sorty = (1LL << r->zoom) - 1 - r->y; + r->data = dir_read_tile(r->dirbase, r->dirtiles[0]); + + r->dirtiles.erase(r->dirtiles.begin()); } } @@ -894,24 +720,16 @@ void decode(struct reader *readers, std::map &layer } } else { - std::vector path_parts; - path_parts = split_slash(r->pbf_path[0]); - std::string metadata_path = path_parts[0]; + std::string metadata_path = r->dirbase + "/metadata.json"; - for (int i = 1; i < static_cast(path_parts.size()) - 3; i++) { - metadata_path = metadata_path + "/" + path_parts[i]; - } - - metadata_path += "/metadata.json"; - - path_parts.clear(); FILE *f = fopen(metadata_path.c_str(), "r"); - if (f == NULL) { perror(metadata_path.c_str()); exit(EXIT_FAILURE); } + // XXX unify metadata reading + json_pull *jp = json_begin_file(f); json_object *j, *k;