mirror of
https://github.com/mapbox/tippecanoe.git
synced 2025-04-06 02:26:39 +00:00
Eliminate the separate metadata file
This commit is contained in:
Eric Fischer
parent
ddb79937d9
commit
b2277d76e5
95
main.cpp
95
main.cpp
@ -101,7 +101,7 @@ void checkdisk(std::vector<struct reader> *r) {
|
||||
for (size_t i = 0; i < r->size(); i++) {
|
||||
// Meta, pool, and tree are used once.
|
||||
// Geometry and index will be duplicated during sorting and tiling.
|
||||
used += (*r)[i].metapos + 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len;
|
||||
used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len;
|
||||
}
|
||||
|
||||
static int warned = 0;
|
||||
@ -1143,23 +1143,16 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
struct reader *r = &readers[i];
|
||||
|
||||
char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1];
|
||||
char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1];
|
||||
char treename[strlen(tmpdir) + strlen("/tree.XXXXXXXX") + 1];
|
||||
char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1];
|
||||
char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1];
|
||||
|
||||
sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX");
|
||||
sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX");
|
||||
sprintf(treename, "%s%s", tmpdir, "/tree.XXXXXXXX");
|
||||
sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX");
|
||||
sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX");
|
||||
|
||||
r->metafd = mkstemp_cloexec(metaname);
|
||||
if (r->metafd < 0) {
|
||||
perror(metaname);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
r->poolfd = mkstemp_cloexec(poolname);
|
||||
if (r->poolfd < 0) {
|
||||
perror(poolname);
|
||||
@ -1181,11 +1174,6 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
r->metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC);
|
||||
if (r->metafile == NULL) {
|
||||
perror(metaname);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
r->poolfile = memfile_open(r->poolfd);
|
||||
if (r->poolfile == NULL) {
|
||||
perror(poolname);
|
||||
@ -1206,11 +1194,9 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
perror(indexname);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
r->metapos = 0;
|
||||
r->geompos = 0;
|
||||
r->indexpos = 0;
|
||||
|
||||
unlink(metaname);
|
||||
unlink(poolname);
|
||||
unlink(treename);
|
||||
unlink(geomname);
|
||||
@ -1221,8 +1207,6 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
struct stringpool p;
|
||||
memfile_write(r->treefile, &p, sizeof(struct stringpool));
|
||||
}
|
||||
// Keep metadata file from being completely empty if no attributes
|
||||
serialize_int(r->metafile, 0, &r->metapos, "meta");
|
||||
|
||||
r->file_bbox[0] = r->file_bbox[1] = UINT_MAX;
|
||||
r->file_bbox[2] = r->file_bbox[3] = 0;
|
||||
@ -1699,10 +1683,6 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
if (fclose(readers[i].metafile) != 0) {
|
||||
perror("fclose meta");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (fclose(readers[i].geomfile) != 0) {
|
||||
perror("fclose geom");
|
||||
exit(EXIT_FAILURE);
|
||||
@ -1717,10 +1697,6 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
perror("stat geom\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (fstat(readers[i].metafd, &readers[i].metast) != 0) {
|
||||
perror("stat meta\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
// Create a combined string pool and a combined metadata file
|
||||
@ -1729,9 +1705,8 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
|
||||
// 2 * CPUS: One per input thread, one per tiling thread
|
||||
long long pool_off[2 * CPUS];
|
||||
long long meta_off[2 * CPUS];
|
||||
for (size_t i = 0; i < 2 * CPUS; i++) {
|
||||
pool_off[i] = meta_off[i] = 0;
|
||||
pool_off[i] = 0;
|
||||
}
|
||||
|
||||
char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1];
|
||||
@ -1751,51 +1726,9 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
|
||||
unlink(poolname);
|
||||
|
||||
char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1];
|
||||
sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX");
|
||||
|
||||
int metafd = mkstemp_cloexec(metaname);
|
||||
if (metafd < 0) {
|
||||
perror(metaname);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
FILE *metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC);
|
||||
if (metafile == NULL) {
|
||||
perror(metaname);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
unlink(metaname);
|
||||
|
||||
std::atomic<long long> metapos(0);
|
||||
std::atomic<long long> poolpos(0);
|
||||
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
if (readers[i].metapos > 0) {
|
||||
void *map = mmap(NULL, readers[i].metapos, PROT_READ, MAP_PRIVATE, readers[i].metafd, 0);
|
||||
if (map == MAP_FAILED) {
|
||||
perror("mmap unmerged meta");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
madvise(map, readers[i].metapos, MADV_SEQUENTIAL);
|
||||
madvise(map, readers[i].metapos, MADV_WILLNEED);
|
||||
if (fwrite(map, readers[i].metapos, 1, metafile) != 1) {
|
||||
perror("Reunify meta");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
madvise(map, readers[i].metapos, MADV_DONTNEED);
|
||||
if (munmap(map, readers[i].metapos) != 0) {
|
||||
perror("unmap unmerged meta");
|
||||
}
|
||||
}
|
||||
|
||||
meta_off[i] = metapos;
|
||||
metapos += readers[i].metapos;
|
||||
if (close(readers[i].metafd) != 0) {
|
||||
perror("close unmerged meta");
|
||||
}
|
||||
|
||||
if (readers[i].poolfile->off > 0) {
|
||||
if (fwrite(readers[i].poolfile->map, readers[i].poolfile->off, 1, poolfile) != 1) {
|
||||
perror("Reunify string pool");
|
||||
@ -1812,17 +1745,6 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
perror("fclose pool");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (fclose(metafile) != 0) {
|
||||
perror("fclose meta");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
char *meta = (char *) mmap(NULL, metapos, PROT_READ, MAP_PRIVATE, metafd, 0);
|
||||
if (meta == MAP_FAILED) {
|
||||
perror("mmap meta");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
madvise(meta, metapos, MADV_RANDOM);
|
||||
|
||||
char *stringpool = NULL;
|
||||
if (poolpos > 0) { // Will be 0 if -X was specified
|
||||
@ -1907,9 +1829,8 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
if (!quiet) {
|
||||
long long s = progress_seq;
|
||||
long long geompos_print = geompos;
|
||||
long long metapos_print = metapos;
|
||||
long long poolpos_print = poolpos;
|
||||
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", s, geompos_print, metapos_print, poolpos_print);
|
||||
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of string pool\n", s, geompos_print, poolpos_print);
|
||||
}
|
||||
|
||||
if (indexpos == 0) {
|
||||
@ -2267,7 +2188,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
|
||||
std::atomic<unsigned> midx(0);
|
||||
std::atomic<unsigned> midy(0);
|
||||
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum, filter);
|
||||
int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum, filter);
|
||||
|
||||
if (maxzoom != written) {
|
||||
if (written > minzoom) {
|
||||
@ -2280,14 +2201,6 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
|
||||
}
|
||||
}
|
||||
|
||||
madvise(meta, metapos, MADV_DONTNEED);
|
||||
if (munmap(meta, metapos) != 0) {
|
||||
perror("munmap meta");
|
||||
}
|
||||
if (close(metafd) < 0) {
|
||||
perror("close meta");
|
||||
}
|
||||
|
||||
if (poolpos > 0) {
|
||||
madvise((void *) stringpool, poolpos, MADV_DONTNEED);
|
||||
if (munmap(stringpool, poolpos) != 0) {
|
||||
|
||||
@ -397,7 +397,6 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
|
||||
sf.bbox[0] = sf.bbox[1] = LLONG_MAX;
|
||||
sf.bbox[2] = sf.bbox[3] = LLONG_MIN;
|
||||
sf.extent = 0;
|
||||
sf.metapos = 0;
|
||||
sf.has_id = false;
|
||||
|
||||
std::string layername = "unknown";
|
||||
|
||||
89
serial.cpp
89
serial.cpp
@ -225,15 +225,11 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long
|
||||
serialize_long_long(geomfile, sf->extent, geompos, fname);
|
||||
}
|
||||
|
||||
serialize_long_long(geomfile, sf->metapos, geompos, fname);
|
||||
serialize_long_long(geomfile, sf->keys.size(), geompos, fname);
|
||||
|
||||
if (sf->metapos < 0) {
|
||||
serialize_long_long(geomfile, sf->keys.size(), geompos, fname);
|
||||
|
||||
for (size_t i = 0; i < sf->keys.size(); i++) {
|
||||
serialize_long_long(geomfile, sf->keys[i], geompos, fname);
|
||||
serialize_long_long(geomfile, sf->values[i], geompos, fname);
|
||||
}
|
||||
for (size_t i = 0; i < sf->keys.size(); i++) {
|
||||
serialize_long_long(geomfile, sf->keys[i], geompos, fname);
|
||||
serialize_long_long(geomfile, sf->values[i], geompos, fname);
|
||||
}
|
||||
|
||||
if (include_minzoom) {
|
||||
@ -241,7 +237,7 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long
|
||||
}
|
||||
}
|
||||
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
|
||||
serial_feature sf;
|
||||
|
||||
deserialize_byte_io(geoms, &sf.t, geompos_in);
|
||||
@ -286,32 +282,15 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_
|
||||
|
||||
sf.layer >>= 6;
|
||||
|
||||
sf.metapos = 0;
|
||||
deserialize_long_long_io(geoms, &sf.metapos, geompos_in);
|
||||
long long count;
|
||||
deserialize_long_long_io(geoms, &count, geompos_in);
|
||||
|
||||
if (sf.metapos >= 0) {
|
||||
char *meta = metabase + sf.metapos + meta_off[sf.segment];
|
||||
long long count;
|
||||
deserialize_long_long(&meta, &count);
|
||||
|
||||
for (long long i = 0; i < count; i++) {
|
||||
long long k, v;
|
||||
deserialize_long_long(&meta, &k);
|
||||
deserialize_long_long(&meta, &v);
|
||||
sf.keys.push_back(k);
|
||||
sf.values.push_back(v);
|
||||
}
|
||||
} else {
|
||||
long long count;
|
||||
deserialize_long_long_io(geoms, &count, geompos_in);
|
||||
|
||||
for (long long i = 0; i < count; i++) {
|
||||
long long k, v;
|
||||
deserialize_long_long_io(geoms, &k, geompos_in);
|
||||
deserialize_long_long_io(geoms, &v, geompos_in);
|
||||
sf.keys.push_back(k);
|
||||
sf.values.push_back(v);
|
||||
}
|
||||
for (long long i = 0; i < count; i++) {
|
||||
long long k, v;
|
||||
deserialize_long_long_io(geoms, &k, geompos_in);
|
||||
deserialize_long_long_io(geoms, &v, geompos_in);
|
||||
sf.keys.push_back(k);
|
||||
sf.values.push_back(v);
|
||||
}
|
||||
|
||||
deserialize_byte_io(geoms, &sf.feature_minzoom, geompos_in);
|
||||
@ -488,32 +467,6 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
locs.clear();
|
||||
}
|
||||
|
||||
bool inline_meta = true;
|
||||
// Don't inline metadata for features that will span several tiles at maxzoom
|
||||
if (sf.geometry.size() > 0 && (sf.bbox[2] < sf.bbox[0] || sf.bbox[3] < sf.bbox[1])) {
|
||||
fprintf(stderr, "Internal error: impossible feature bounding box %llx,%llx,%llx,%llx\n", sf.bbox[0], sf.bbox[1], sf.bbox[2], sf.bbox[3]);
|
||||
}
|
||||
if (sf.bbox[0] == LLONG_MAX) {
|
||||
// No bounding box (empty geometry)
|
||||
// Shouldn't happen, but avoid arithmetic overflow below
|
||||
} else if (sf.bbox[2] - sf.bbox[0] > (2LL << (32 - sst->maxzoom)) || sf.bbox[3] - sf.bbox[1] > (2LL << (32 - sst->maxzoom))) {
|
||||
inline_meta = false;
|
||||
|
||||
if (prevent[P_CLIPPING]) {
|
||||
static std::atomic<long long> warned(0);
|
||||
long long extent = ((sf.bbox[2] - sf.bbox[0]) / ((1LL << (32 - sst->maxzoom)) + 1)) * ((sf.bbox[3] - sf.bbox[1]) / ((1LL << (32 - sst->maxzoom)) + 1));
|
||||
if (extent > warned) {
|
||||
fprintf(stderr, "Warning: %s:%d: Large unclipped (-pc) feature may be duplicated across %lld tiles\n", sst->fname, sst->line, extent);
|
||||
warned = extent;
|
||||
|
||||
if (extent > 10000) {
|
||||
fprintf(stderr, "Exiting because this can't be right.\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
double extent = 0;
|
||||
if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
|
||||
if (sf.t == VT_POLYGON) {
|
||||
@ -651,19 +604,9 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
}
|
||||
}
|
||||
|
||||
if (inline_meta) {
|
||||
sf.metapos = -1;
|
||||
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
||||
sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string));
|
||||
sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type));
|
||||
}
|
||||
} else {
|
||||
sf.metapos = r->metapos;
|
||||
serialize_long_long(r->metafile, sf.full_keys.size(), &r->metapos, sst->fname);
|
||||
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
||||
serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string), &r->metapos, sst->fname);
|
||||
serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type), &r->metapos, sst->fname);
|
||||
}
|
||||
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
||||
sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string));
|
||||
sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type));
|
||||
}
|
||||
|
||||
long long geomstart = r->geompos;
|
||||
|
||||
16
serial.hpp
16
serial.hpp
@ -60,8 +60,6 @@ struct serial_feature {
|
||||
|
||||
std::vector<long long> keys{};
|
||||
std::vector<long long> values{};
|
||||
// If >= 0, metadata is external
|
||||
long long metapos = 0;
|
||||
|
||||
// XXX This isn't serialized. Should it be here?
|
||||
long long bbox[4] = {0, 0, 0, 0};
|
||||
@ -72,22 +70,19 @@ struct serial_feature {
|
||||
};
|
||||
|
||||
void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long> *geompos, const char *fname, long long wx, long long wy, bool include_minzoom);
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
|
||||
|
||||
struct reader {
|
||||
int metafd = -1;
|
||||
int poolfd = -1;
|
||||
int treefd = -1;
|
||||
int geomfd = -1;
|
||||
int indexfd = -1;
|
||||
|
||||
FILE *metafile = NULL;
|
||||
struct memfile *poolfile = NULL;
|
||||
struct memfile *treefile = NULL;
|
||||
FILE *geomfile = NULL;
|
||||
FILE *indexfile = NULL;
|
||||
|
||||
std::atomic<long long> metapos;
|
||||
std::atomic<long long> geompos;
|
||||
std::atomic<long long> indexpos;
|
||||
|
||||
@ -99,26 +94,21 @@ struct reader {
|
||||
char *geom_map = NULL;
|
||||
|
||||
reader()
|
||||
: metapos(0), geompos(0), indexpos(0) {
|
||||
: geompos(0), indexpos(0) {
|
||||
}
|
||||
|
||||
reader(reader const &r) {
|
||||
metafd = r.metafd;
|
||||
poolfd = r.poolfd;
|
||||
treefd = r.treefd;
|
||||
geomfd = r.geomfd;
|
||||
indexfd = r.indexfd;
|
||||
|
||||
metafile = r.metafile;
|
||||
poolfile = r.poolfile;
|
||||
treefile = r.treefile;
|
||||
geomfile = r.geomfile;
|
||||
indexfile = r.indexfile;
|
||||
|
||||
long long p = r.metapos;
|
||||
metapos = p;
|
||||
|
||||
p = r.geompos;
|
||||
long long p = r.geompos;
|
||||
geompos = p;
|
||||
|
||||
p = r.indexpos;
|
||||
|
||||
35
tile.cpp
35
tile.cpp
@ -248,7 +248,7 @@ static int metacmp(const std::vector<long long> &keys1, const std::vector<long l
|
||||
}
|
||||
}
|
||||
|
||||
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic<long long> *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) {
|
||||
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic<long long> *geompos, FILE **geomfile, const char *fname, signed char t, int layer, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) {
|
||||
if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) {
|
||||
int xo, yo;
|
||||
int span = 1 << (nextzoom - z);
|
||||
@ -335,17 +335,14 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
|
||||
sf.tippecanoe_minzoom = tippecanoe_minzoom;
|
||||
sf.has_tippecanoe_maxzoom = tippecanoe_maxzoom != -1;
|
||||
sf.tippecanoe_maxzoom = tippecanoe_maxzoom;
|
||||
sf.metapos = metastart;
|
||||
sf.geometry = geom2;
|
||||
sf.index = index;
|
||||
sf.extent = extent;
|
||||
sf.feature_minzoom = feature_minzoom;
|
||||
|
||||
if (metastart < 0) {
|
||||
for (size_t i = 0; i < metakeys.size(); i++) {
|
||||
sf.keys.push_back(metakeys[i]);
|
||||
sf.values.push_back(metavals[i]);
|
||||
}
|
||||
for (size_t i = 0; i < metakeys.size(); i++) {
|
||||
sf.keys.push_back(metakeys[i]);
|
||||
sf.values.push_back(metavals[i]);
|
||||
}
|
||||
|
||||
serialize_feature(geomfile[j], &sf, &geompos[j], fname, SHIFT_RIGHT(initial_x[segment]), SHIFT_RIGHT(initial_y[segment]), true);
|
||||
@ -1175,7 +1172,6 @@ long long choose_minextent(std::vector<long long> &extents, double f) {
|
||||
|
||||
struct write_tile_args {
|
||||
struct task *tasks = NULL;
|
||||
char *metabase = NULL;
|
||||
char *stringpool = NULL;
|
||||
int min_detail = 0;
|
||||
sqlite3 *outdb = NULL;
|
||||
@ -1198,7 +1194,6 @@ struct write_tile_args {
|
||||
int low_detail = 0;
|
||||
double simplification = 0;
|
||||
std::atomic<long long> *most = NULL;
|
||||
long long *meta_off = NULL;
|
||||
long long *pool_off = NULL;
|
||||
unsigned *initial_x = NULL;
|
||||
unsigned *initial_y = NULL;
|
||||
@ -1318,9 +1313,9 @@ void remove_attributes(serial_feature &sf, std::set<std::string> const &exclude_
|
||||
}
|
||||
}
|
||||
|
||||
serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, std::atomic<long long> *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps) {
|
||||
serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, std::atomic<long long> *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps) {
|
||||
while (1) {
|
||||
serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y);
|
||||
serial_feature sf = deserialize_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y);
|
||||
if (sf.t < 0) {
|
||||
return sf;
|
||||
}
|
||||
@ -1345,7 +1340,7 @@ serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, cha
|
||||
|
||||
if (*first_time && pass == 1) { /* only write out the next zoom once, even if we retry */
|
||||
if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) {
|
||||
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent);
|
||||
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1448,8 +1443,6 @@ serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, cha
|
||||
struct run_prefilter_args {
|
||||
FILE *geoms = NULL;
|
||||
std::atomic<long long> *geompos_in = NULL;
|
||||
char *metabase = NULL;
|
||||
long long *meta_off = NULL;
|
||||
int z = 0;
|
||||
unsigned tx = 0;
|
||||
unsigned ty = 0;
|
||||
@ -1486,7 +1479,7 @@ void *run_prefilter(void *v) {
|
||||
json_writer state(rpa->prefilter_fp);
|
||||
|
||||
while (1) {
|
||||
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps);
|
||||
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps);
|
||||
if (sf.t < 0) {
|
||||
break;
|
||||
}
|
||||
@ -1715,7 +1708,7 @@ static bool line_is_too_small(drawvec const &geometry, int z, int detail) {
|
||||
return true;
|
||||
}
|
||||
|
||||
long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg) {
|
||||
long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg) {
|
||||
int line_detail;
|
||||
double merge_fraction = 1;
|
||||
double mingap_fraction = 1;
|
||||
@ -1808,8 +1801,6 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
|
||||
rpa.geoms = geoms;
|
||||
rpa.geompos_in = geompos_in;
|
||||
rpa.metabase = metabase;
|
||||
rpa.meta_off = meta_off;
|
||||
rpa.z = z;
|
||||
rpa.tx = tx;
|
||||
rpa.ty = ty;
|
||||
@ -1858,7 +1849,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
ssize_t which_partial = -1;
|
||||
|
||||
if (prefilter == NULL) {
|
||||
sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps);
|
||||
sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps);
|
||||
} else {
|
||||
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
|
||||
}
|
||||
@ -2582,7 +2573,7 @@ void *run_thread(void *vargs) {
|
||||
|
||||
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
|
||||
|
||||
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg);
|
||||
long long len = write_tile(geom, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg);
|
||||
|
||||
if (len < 0) {
|
||||
int *err = &arg->err;
|
||||
@ -2647,7 +2638,7 @@ void *run_thread(void *vargs) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter) {
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter) {
|
||||
last_progress = 0;
|
||||
|
||||
// The existing layermaps are one table per input thread.
|
||||
@ -2796,7 +2787,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
std::atomic<long long> along(0);
|
||||
|
||||
for (size_t thread = 0; thread < threads; thread++) {
|
||||
args[thread].metabase = metabase;
|
||||
args[thread].stringpool = stringpool;
|
||||
args[thread].min_detail = min_detail;
|
||||
args[thread].outdb = outdb; // locked with db_lock
|
||||
@ -2826,7 +2816,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
args[thread].full_detail = full_detail;
|
||||
args[thread].low_detail = low_detail;
|
||||
args[thread].most = &most; // locked with var_lock
|
||||
args[thread].meta_off = meta_off;
|
||||
args[thread].pool_off = pool_off;
|
||||
args[thread].initial_x = initial_x;
|
||||
args[thread].initial_y = initial_y;
|
||||
|
||||
2
tile.hpp
2
tile.hpp
@ -21,7 +21,7 @@ enum attribute_op {
|
||||
|
||||
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers);
|
||||
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter);
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter);
|
||||
|
||||
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user