ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-03-23 12:25:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Factor out code for enumerating the tiles in a directory

Browse Source
This commit is contained in:
Eric Fischer 2017-11-30 15:37:46 -08:00
parent ad86d06cce
commit 1acd771743
5 changed files with 106 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -53,7 +53,7 @@ tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o pro
tippecanoe-enumerate: enumerate.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3
tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o
tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o

101
decode.cpp
View File

@ -21,6 +21,7 @@
#include "geometry.hpp"
#include "write_json.hpp"
#include "jsonpull/jsonpull.h"
#include "dirtiles.hpp"
int minzoom = 0;
int maxzoom = 32;
@ -142,56 +143,6 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st
}
}
struct zxy {
int z;
int x;
int y;
zxy(int _z, int _x, int _y)
: z(_z), x(_x), y(_y) {
}
bool operator<(const zxy &other) const {
if (z < other.z) {
return true;
}
if (z == other.z) {
if (x < other.x) {
return true;
}
if (x == other.x) {
if (y > other.y) {
return true; // reversed for TMS
}
}
}
return false;
}
};
// XXX deduplicate from dirtiles
bool numeric(const char *s) {
if (*s == '\0') {
return false;
}
for (; *s != 0; s++) {
if (*s < '0' || *s > '9') {
return false;
}
}
return true;
}
// XXX deduplicate from dirtiles
bool pbfname(const char *s) {
while (*s >= '0' && *s <= '9') {
s++;
}
return strcmp(s, ".pbf") == 0;
}
sqlite3 *meta2tmp(const char *fname, std::vector<zxy> &tiles) {
sqlite3 *db;
char *err = NULL;
@ -237,53 +188,7 @@ sqlite3 *meta2tmp(const char *fname, std::vector<zxy> &tiles) {
json_end(jp);
fclose(f);
// XXX deduplicate from dirtiles
DIR *d1 = opendir(fname);
if (d1 != NULL) {
struct dirent *dp;
while ((dp = readdir(d1)) != NULL) {
if (numeric(dp->d_name)) {
std::string z = std::string(fname) + "/" + dp->d_name;
int tz = atoi(dp->d_name);
DIR *d2 = opendir(z.c_str());
if (d2 == NULL) {
perror(z.c_str());
exit(EXIT_FAILURE);
}
struct dirent *dp2;
while ((dp2 = readdir(d2)) != NULL) {
if (numeric(dp2->d_name)) {
std::string x = z + "/" + dp2->d_name;
int tx = atoi(dp2->d_name);
DIR *d3 = opendir(x.c_str());
if (d3 == NULL) {
perror(x.c_str());
exit(EXIT_FAILURE);
}
struct dirent *dp3;
while ((dp3 = readdir(d3)) != NULL) {
if (pbfname(dp3->d_name)) {
int ty = atoi(dp3->d_name);
tiles.push_back(zxy(tz, tx, ty));
}
}
closedir(d3);
}
}
closedir(d2);
}
}
closedir(d1);
}
std::sort(tiles.begin(), tiles.end());
tiles = enumerate_dirtiles(fname);
return db;
}
@ -397,7 +302,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
within = 1;
}
std::string fn = std::string(fname) + "/" + std::to_string(tiles[i].z) + "/" + std::to_string(tiles[i].x) + "/" + std::to_string(tiles[i].y) + ".pbf";
std::string fn = std::string(fname) + "/" + tiles[i].path();
FILE *f = fopen(fn.c_str(), "rb");
if (f == NULL) {
perror(fn.c_str());

51
dirtiles.cpp
View File

@ -2,6 +2,7 @@
#include <fstream>
#include <sstream>
#include <string>
#include <algorithm>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
@ -9,8 +10,8 @@
#include <sys/stat.h>
#include "dirtiles.hpp"
std::string dir_read_tile(std::string pbfPath) {
std::ifstream pbfFile(pbfPath, std::ios::in | std::ios::binary);
std::string dir_read_tile(std::string base, struct zxy tile) {
std::ifstream pbfFile(base + "/" + tile.path(), std::ios::in | std::ios::binary);
std::ostringstream contents;
contents << pbfFile.rdbuf();
pbfFile.close();
@ -39,7 +40,7 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const
pbfFile.close();
}
bool numeric(const char *s) {
static bool numeric(const char *s) {
if (*s == '\0') {
return false;
}
@ -51,7 +52,7 @@ bool numeric(const char *s) {
return true;
}
bool pbfname(const char *s) {
static bool pbfname(const char *s) {
while (*s >= '0' && *s <= '9') {
s++;
}
@ -79,12 +80,33 @@ void check_dir(const char *dir, bool force, bool forcetable) {
return;
}
DIR *d1 = opendir(dir);
std::vector<zxy> tiles = enumerate_dirtiles(dir);
for (size_t i = 0; i < tiles.size(); i++) {
std::string fn = std::string(dir) + "/" + tiles[i].path();
if (force) {
if (unlink(fn.c_str()) != 0) {
perror(fn.c_str());
exit(EXIT_FAILURE);
}
} else {
fprintf(stderr, "%s: file exists\n", fn.c_str());
exit(EXIT_FAILURE);
}
}
}
std::vector<zxy> enumerate_dirtiles(const char *fname) {
std::vector<zxy> tiles;
DIR *d1 = opendir(fname);
if (d1 != NULL) {
struct dirent *dp;
while ((dp = readdir(d1)) != NULL) {
if (numeric(dp->d_name)) {
std::string z = std::string(dir) + "/" + dp->d_name;
std::string z = std::string(fname) + "/" + dp->d_name;
int tz = atoi(dp->d_name);
DIR *d2 = opendir(z.c_str());
if (d2 == NULL) {
@ -96,6 +118,7 @@ void check_dir(const char *dir, bool force, bool forcetable) {
while ((dp2 = readdir(d2)) != NULL) {
if (numeric(dp2->d_name)) {
std::string x = z + "/" + dp2->d_name;
int tx = atoi(dp2->d_name);
DIR *d3 = opendir(x.c_str());
if (d3 == NULL) {
@ -106,17 +129,8 @@ void check_dir(const char *dir, bool force, bool forcetable) {
struct dirent *dp3;
while ((dp3 = readdir(d3)) != NULL) {
if (pbfname(dp3->d_name)) {
std::string y = x + "/" + dp3->d_name;
if (force) {
if (unlink(y.c_str()) != 0) {
perror(y.c_str());
exit(EXIT_FAILURE);
}
} else {
fprintf(stderr, "%s: file exists\n", y.c_str());
exit(EXIT_FAILURE);
}
int ty = atoi(dp3->d_name);
tiles.push_back(zxy(tz, tx, ty));
}
}
@ -130,4 +144,7 @@ void check_dir(const char *dir, bool force, bool forcetable) {
closedir(d1);
}
std::sort(tiles.begin(), tiles.end());
return tiles;
}

38
dirtiles.hpp
View File

@ -1,12 +1,46 @@
#include <string>
#include <vector>
#ifndef DIRTILES_HPP
#define DIRTILES_HPP
std::string dir_read_tile(std::string pbfPath);
void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);
void check_dir(const char *d, bool force, bool forcetable);
struct zxy {
long long z;
long long x;
long long y;
zxy(int _z, int _x, int _y)
: z(_z), x(_x), y(_y) {
}
bool operator<(const zxy &other) const {
if (z < other.z) {
return true;
}
if (z == other.z) {
if (x < other.x) {
return true;
}
if (x == other.x) {
if (y > other.y) {
return true; // reversed for TMS
}
}
}
return false;
}
std::string path() {
return std::to_string(z) + "/" + std::to_string(x) + "/" + std::to_string(y) + ".pbf";
}
};
std::vector<zxy> enumerate_dirtiles(const char *fname);
std::string dir_read_tile(std::string pbfPath, struct zxy tile);
#endif

246
tile-join.cpp
View File

@ -326,12 +326,12 @@ struct reader {
long long x = 0;
long long sorty = 0;
long long y = 0;
int pbf_count = 0;
int z_flag = 0;
std::string data = "";
std::vector<std::string> pbf_path{};
std::vector<std::string> large_zoom{};
std::vector<zxy> dirtiles;
std::string dirbase;
sqlite3 *db = NULL;
sqlite3_stmt *stmt = NULL;
@ -367,165 +367,29 @@ struct reader {
}
};
std::vector<std::string> split_slash(std::string pbf_path) {
std::vector<std::string> path_parts;
std::string path(pbf_path);
std::istringstream iss(path);
std::string token;
while (std::getline(iss, token, '/')) {
path_parts.push_back(token);
}
return path_parts;
}
int filter(const struct dirent *dir) {
if (strcmp(dir->d_name, ".") == 0 || strcmp(dir->d_name, "..") == 0 || strcmp(dir->d_name, ".DS_Store") == 0 || strcmp(dir->d_name, "metadata.json") == 0) {
return 0;
} else {
return 1;
}
}
// Recursively walk through a specified directory and its subdirectories,
// using alphasort function and integer variable zoom_range to handle input in numerical order.
// Store the path of all pbf files in a pbf_path vector member of reader struct,
// with the help of a large_zoom vector and two integer members pbf_count and z_flag
// to ensure the tiles order in pbf_path to be the same as in mbtiles.
struct reader *read_dir(struct reader *readers, const char *name, int level, int zoom_range) {
struct dirent **namelist;
struct stat buf;
std::string path;
int i = 0;
int n = scandir(name, &namelist, filter, alphasort);
std::vector<std::string> path_parts1, path_parts2;
readers->pbf_count = 0;
if (n > 0) {
while (i < n) {
path = std::string(name) + "/" + std::string(namelist[i]->d_name);
if (stat(path.c_str(), &buf) == 0 && S_ISDIR(buf.st_mode)) {
if (level == 0) {
if (std::stoi(namelist[i]->d_name) <= 9) {
zoom_range = 0;
} else {
zoom_range = 1;
}
if (readers->pbf_count > 0) {
if (readers->z_flag == 0) {
std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater<std::string>());
} else {
std::sort(readers->large_zoom.end() - (readers->pbf_count + 1), readers->large_zoom.end(), std::greater<std::string>());
}
readers->pbf_count = 0;
}
}
if (level == 1 && readers->pbf_count > 0) {
if (zoom_range == 0) {
std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater<std::string>());
} else {
std::sort(readers->large_zoom.end() - (readers->pbf_count + 1), readers->large_zoom.end(), std::greater<std::string>());
}
readers->pbf_count = 0;
}
read_dir(readers, path.c_str(), level + 1, zoom_range);
} else {
if (level == 0) {
fprintf(stderr, "ERROR: Directory structure in '%s' should be zoom/x/y\n", name);
exit(EXIT_FAILURE);
}
if (level == 1) {
fprintf(stderr, "ERROR: Directory structure in '%s' should be zoom/x/y\n", split_slash(name)[0].c_str());
exit(EXIT_FAILURE);
}
if (zoom_range == 0) {
readers->pbf_path.push_back(path);
if (readers->pbf_path.size() > 1) {
path_parts1 = split_slash(readers->pbf_path[readers->pbf_path.size() - 1]);
path_parts2 = split_slash(readers->pbf_path[readers->pbf_path.size() - 2]);
int p1 = path_parts1.size();
int p2 = path_parts2.size();
if (std::stoll(path_parts1[p1 - 3]) == std::stoll(path_parts2[p2 - 3]) && std::stoll(path_parts1[p1 - 2]) == std::stoll(path_parts2[p2 - 2])) {
readers->z_flag = 0;
readers->pbf_count++;
}
path_parts1.clear();
path_parts2.clear();
}
} else {
readers->large_zoom.push_back(path);
if (readers->large_zoom.size() > 1) {
path_parts1 = split_slash(readers->large_zoom[readers->large_zoom.size() - 1]);
path_parts2 = split_slash(readers->large_zoom[readers->large_zoom.size() - 2]);
int p1 = path_parts1.size();
int p2 = path_parts2.size();
if (std::stoll(path_parts1[p1 - 3]) == std::stoll(path_parts2[p2 - 3]) && std::stoll(path_parts1[p1 - 2]) == std::stoll(path_parts2[p2 - 2])) {
readers->z_flag = 1;
readers->pbf_count++;
}
path_parts1.clear();
path_parts2.clear();
}
}
}
free(namelist[i]);
i++;
}
if (level == 0) {
if (readers->pbf_count > 0) {
std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater<std::string>());
}
readers->pbf_path.insert(std::end(readers->pbf_path), std::begin(readers->large_zoom), std::end(readers->large_zoom));
}
free(namelist);
} else if (n == 0) {
fprintf(stderr, "ERROR: Empty directory '%s'\n", name);
exit(EXIT_FAILURE);
} else {
perror("scandir");
}
return readers;
}
struct reader *begin_reading(char *fname) {
DIR *dir;
struct reader *r = new reader;
if ((dir = opendir(fname)) != NULL) {
r = read_dir(r, fname, 0, 0);
std::vector<std::string> path_parts;
path_parts = split_slash(r->pbf_path[0]);
int p = path_parts.size();
struct stat st;
if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) {
r->db = NULL;
r->stmt = NULL;
r->next = NULL;
r->pbf_count = 0;
r->zoom = std::stoll(path_parts[p - 3]);
r->x = std::stoll(path_parts[p - 2]);
r->y = std::stoll(path_parts[p - 1].substr(0, path_parts[p - 1].find_last_of(".")));
r->sorty = (1LL << r->zoom) - 1 - r->y;
r->data = dir_read_tile(r->pbf_path[0]);
path_parts.clear();
closedir(dir);
r->dirtiles = enumerate_dirtiles(fname);
r->dirbase = fname;
if (r->dirtiles.size() == 0) {
r->zoom = 32;
} else {
r->zoom = r->dirtiles[0].z;
r->x = r->dirtiles[0].x;
r->y = r->dirtiles[0].y;
r->sorty = (1LL << r->zoom) - 1 - r->y;
r->data = dir_read_tile(r->dirbase, r->dirtiles[0]);
r->dirtiles.erase(r->dirtiles.begin());
}
} else {
sqlite3 *db;
@ -564,40 +428,6 @@ struct reader *begin_reading(char *fname) {
return r;
}
struct zxy {
long long z;
long long x;
long long y;
zxy(long long _z, long long _x, long long _y)
: z(_z),
x(_x),
y(_y) {
}
bool operator<(zxy const &other) const {
if (z < other.z) {
return true;
}
if (z > other.z) {
return false;
}
if (x < other.x) {
return true;
}
if (x > other.x) {
return false;
}
if (y < other.y) {
return true;
}
return false;
}
};
struct arg {
std::map<zxy, std::vector<std::string>> inputs{};
std::map<zxy, std::string> outputs{};
@ -775,20 +605,16 @@ void decode(struct reader *readers, std::map<std::string, layermap_entry> &layer
r->zoom = 32;
}
} else {
r->pbf_count++;
if (r->pbf_count != static_cast<int>(r->pbf_path.size())) {
std::vector<std::string> path_parts;
path_parts = split_slash(r->pbf_path[r->pbf_count]);
int p = path_parts.size();
r->zoom = std::stoll(path_parts[p - 3]);
r->x = std::stoll(path_parts[p - 2]);
r->y = std::stoll(path_parts[p - 1].substr(0, path_parts[p - 1].find_last_of(".")));
r->sorty = (1LL << r->zoom) - 1 - r->y;
r->data = dir_read_tile(r->pbf_path[r->pbf_count]);
path_parts.clear();
} else {
if (r->dirtiles.size() == 0) {
r->zoom = 32;
} else {
r->zoom = r->dirtiles[0].z;
r->x = r->dirtiles[0].x;
r->y = r->dirtiles[0].y;
r->sorty = (1LL << r->zoom) - 1 - r->y;
r->data = dir_read_tile(r->dirbase, r->dirtiles[0]);
r->dirtiles.erase(r->dirtiles.begin());
}
}
@ -894,24 +720,16 @@ void decode(struct reader *readers, std::map<std::string, layermap_entry> &layer
}
} else {
std::vector<std::string> path_parts;
path_parts = split_slash(r->pbf_path[0]);
std::string metadata_path = path_parts[0];
std::string metadata_path = r->dirbase + "/metadata.json";
for (int i = 1; i < static_cast<int>(path_parts.size()) - 3; i++) {
metadata_path = metadata_path + "/" + path_parts[i];
}
metadata_path += "/metadata.json";
path_parts.clear();
FILE *f = fopen(metadata_path.c_str(), "r");
if (f == NULL) {
perror(metadata_path.c_str());
exit(EXIT_FAILURE);
}
// XXX unify metadata reading
json_pull *jp = json_begin_file(f);
json_object *j, *k;