From d7f44ab0825fb1777e771db5fc9cf715548e93c8 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 30 Nov 2017 15:49:14 -0800 Subject: [PATCH] Factor out code for reading metadata.json --- CHANGELOG.md | 4 + decode.cpp | 52 +------------ dirtiles.cpp | 49 ++++++++++++ dirtiles.hpp | 1 + tile-join.cpp | 211 ++++++++++++++++++-------------------------------- version.hpp | 2 +- 6 files changed, 131 insertions(+), 188 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0960376..cb30ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.27.3 + +* Clean up duplicated code for reading tiles from a directory + ## 1.27.2 * Tippecanoe-decode can decode directories of tiles, not just mbtiles diff --git a/decode.cpp b/decode.cpp index bf8a99c..3959739 100644 --- a/decode.cpp +++ b/decode.cpp @@ -143,55 +143,6 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st } } -sqlite3 *meta2tmp(const char *fname, std::vector &tiles) { - sqlite3 *db; - char *err = NULL; - - if (sqlite3_open("", &db) != SQLITE_OK) { - fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db)); - exit(EXIT_FAILURE); - } - if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "Create metadata table: %s\n", err); - exit(EXIT_FAILURE); - } - - std::string name = fname; - name += "/metadata.json"; - - FILE *f = fopen(name.c_str(), "r"); - if (f == NULL) { - perror(name.c_str()); - exit(EXIT_FAILURE); - } - - json_pull *jp = json_begin_file(f); - json_object *o = json_read_tree(jp); - - if (o->type != JSON_HASH) { - fprintf(stderr, "%s: bad metadata format\n", name.c_str()); - exit(EXIT_FAILURE); - } - - for (size_t i = 0; i < o->length; i++) { - if (o->keys[i]->type != JSON_STRING || o->values[i]->type != JSON_STRING) { - fprintf(stderr, "%s: non-string in metadata\n", name.c_str()); - } - - char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->keys[i]->string, o->values[i]->string); - if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", o->keys[i]->string, err); - } - sqlite3_free(sql); - } - - json_end(jp); - fclose(f); - - tiles = enumerate_dirtiles(fname); - return db; -} - void decode(char *fname, int z, unsigned x, unsigned y, std::set const &to_decode, bool pipeline, bool stats) { sqlite3 *db = NULL; bool isdir = false; @@ -235,7 +186,8 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) { isdir = true; - db = meta2tmp(fname, tiles); + db = dirmeta2tmp(fname); + tiles = enumerate_dirtiles(fname); } else { if (sqlite3_open(fname, &db) != SQLITE_OK) { fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db)); diff --git a/dirtiles.cpp b/dirtiles.cpp index 5a0f7c6..757b1c6 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include "jsonpull/jsonpull.h" #include "dirtiles.hpp" std::string dir_read_tile(std::string base, struct zxy tile) { @@ -148,3 +150,50 @@ std::vector enumerate_dirtiles(const char *fname) { std::sort(tiles.begin(), tiles.end()); return tiles; } + +sqlite3 *dirmeta2tmp(const char *fname) { + sqlite3 *db; + char *err = NULL; + + if (sqlite3_open("", &db) != SQLITE_OK) { + fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "Create metadata table: %s\n", err); + exit(EXIT_FAILURE); + } + + std::string name = fname; + name += "/metadata.json"; + + FILE *f = fopen(name.c_str(), "r"); + if (f == NULL) { + perror(name.c_str()); + exit(EXIT_FAILURE); + } + + json_pull *jp = json_begin_file(f); + json_object *o = json_read_tree(jp); + + if (o->type != JSON_HASH) { + fprintf(stderr, "%s: bad metadata format\n", name.c_str()); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; i < o->length; i++) { + if (o->keys[i]->type != JSON_STRING || o->values[i]->type != JSON_STRING) { + fprintf(stderr, "%s: non-string in metadata\n", name.c_str()); + } + + char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->keys[i]->string, o->values[i]->string); + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "set %s in metadata: %s\n", o->keys[i]->string, err); + } + sqlite3_free(sql); + } + + json_end(jp); + fclose(f); + return db; +} diff --git a/dirtiles.hpp b/dirtiles.hpp index 8ce5bda..2a39e08 100644 --- a/dirtiles.hpp +++ b/dirtiles.hpp @@ -41,6 +41,7 @@ struct zxy { }; std::vector enumerate_dirtiles(const char *fname); +sqlite3 *dirmeta2tmp(const char *fname); std::string dir_read_tile(std::string pbfPath, struct zxy tile); #endif diff --git a/tile-join.cpp b/tile-join.cpp index 762c327..1f1762b 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -642,149 +642,86 @@ void decode(struct reader *readers, std::map &layer for (struct reader *r = readers; r != NULL; r = next) { next = r->next; - if (r->db != NULL) { - sqlite3_finalize(r->stmt); - - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'minzoom'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - int minz = max(sqlite3_column_int(r->stmt, 0), minzoom); - st->minzoom = min(st->minzoom, minz); - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'maxzoom'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - int maxz = min(sqlite3_column_int(r->stmt, 0), maxzoom); - st->maxzoom = max(st->maxzoom, maxz); - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'center'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat); - } - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'attribution'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - attribution = std::string((char *) s); - } - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'description'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - description = std::string((char *) s); - } - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'name'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - if (name.size() == 0) { - name = std::string((char *) s); - } else { - name += " + " + std::string((char *) s); - } - } - } - sqlite3_finalize(r->stmt); - } - if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'bounds'", -1, &r->stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(r->stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(r->stmt, 0); - if (s != NULL) { - if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) { - st->minlon = min(minlon, st->minlon); - st->maxlon = max(maxlon, st->maxlon); - st->minlat = min(minlat, st->minlat); - st->maxlat = max(maxlat, st->maxlat); - } - } - } - sqlite3_finalize(r->stmt); - } - - if (sqlite3_close(r->db) != SQLITE_OK) { - fprintf(stderr, "Could not close database: %s\n", sqlite3_errmsg(r->db)); - exit(EXIT_FAILURE); - } - + sqlite3 *db = r->db; + if (db == NULL) { + db = dirmeta2tmp(r->dirbase.c_str()); } else { - std::string metadata_path = r->dirbase + "/metadata.json"; + sqlite3_finalize(r->stmt); + } - FILE *f = fopen(metadata_path.c_str(), "r"); - if (f == NULL) { - perror(metadata_path.c_str()); - exit(EXIT_FAILURE); + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'minzoom'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + int minz = max(sqlite3_column_int(r->stmt, 0), minzoom); + st->minzoom = min(st->minzoom, minz); } - - // XXX unify metadata reading - - json_pull *jp = json_begin_file(f); - json_object *j, *k; - - while ((j = json_read(jp)) != NULL) { - if (j->type == JSON_HASH) { - if ((k = json_hash_get(j, "minzoom")) != NULL) { - const std::string minzoom_tmp = k->string; - int minz = max(std::stoi(minzoom_tmp), minzoom); - st->minzoom = min(st->minzoom, minz); - } - - if ((k = json_hash_get(j, "maxzoom")) != NULL) { - const std::string maxzoom_tmp = k->string; - int maxz = min(std::stoi(maxzoom_tmp), maxzoom); - st->maxzoom = max(st->maxzoom, maxz); - } - - if ((k = json_hash_get(j, "center")) != NULL) { - const std::string center = k->string; - const unsigned char *s = (const unsigned char *) center.c_str(); - sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat); - } - - if ((k = json_hash_get(j, "attribution")) != NULL) { - attribution = k->string; - } - - if ((k = json_hash_get(j, "description")) != NULL) { - description = k->string; - } - - if ((k = json_hash_get(j, "name")) != NULL) { - const std::string name_tmp = k->string; - if (name.size() == 0) { - name = name_tmp; - } else { - name += " + " + name_tmp; - } - } - - if ((k = json_hash_get(j, "bounds")) != NULL) { - const std::string bounds = k->string; - const unsigned char *s = (const unsigned char *) bounds.c_str(); - if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) { - st->minlon = min(minlon, st->minlon); - st->maxlon = max(maxlon, st->maxlon); - st->minlat = min(minlat, st->minlat); - st->maxlat = max(maxlat, st->maxlat); - } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'maxzoom'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + int maxz = min(sqlite3_column_int(r->stmt, 0), maxzoom); + st->maxzoom = max(st->maxzoom, maxz); + } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'center'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat); + } + } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'attribution'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + attribution = std::string((char *) s); + } + } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'description'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + description = std::string((char *) s); + } + } + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'name'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + if (name.size() == 0) { + name = std::string((char *) s); + } else { + name += " + " + std::string((char *) s); } } } - json_free(j); - json_end(jp); - fclose(f); + sqlite3_finalize(r->stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'bounds'", -1, &r->stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(r->stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(r->stmt, 0); + if (s != NULL) { + if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) { + st->minlon = min(minlon, st->minlon); + st->maxlon = max(maxlon, st->maxlon); + st->minlat = min(minlat, st->minlat); + st->maxlat = max(maxlat, st->maxlat); + } + } + } + sqlite3_finalize(r->stmt); + } + + // Closes either real db or temp mirror of metadata.json + if (sqlite3_close(db) != SQLITE_OK) { + fprintf(stderr, "Could not close database: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); } delete r; diff --git a/version.hpp b/version.hpp index 70895f9..f931915 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "tippecanoe v1.27.2\n" +#define VERSION "tippecanoe v1.27.3\n" #endif