mirror of
https://github.com/mapbox/tippecanoe.git
synced 2025-04-09 03:54:13 +00:00
Merge pull request #245 from mapbox/inline-meta2
Add the ability to inline metadata with geometry
This commit is contained in:
Eric Fischer
commit
4030cc7c58
@ -1,3 +1,8 @@
|
||||
## 1.11.7
|
||||
|
||||
* Keep metadata together with geometry for features that don't span many tiles,
|
||||
to avoid extra memory load from indexing into a separate metadata file
|
||||
|
||||
## 1.11.6
|
||||
|
||||
* Reduce the size of critical data structures to reduce dynamic memory use
|
||||
|
||||
109
geojson.cpp
109
geojson.cpp
@ -36,6 +36,7 @@ extern "C" {
|
||||
#include "mbtiles.hpp"
|
||||
#include "geojson.hpp"
|
||||
#include "geometry.hpp"
|
||||
#include "options.hpp"
|
||||
|
||||
#define GEOM_POINT 0 /* array of positions */
|
||||
#define GEOM_MULTIPOINT 1 /* array of arrays of positions */
|
||||
@ -62,10 +63,12 @@ static int mb_geometry[GEOM_TYPES] = {
|
||||
VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON,
|
||||
};
|
||||
|
||||
void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FILE *out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) {
|
||||
long long parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FILE *out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) {
|
||||
long long g = 0;
|
||||
|
||||
if (j == NULL || j->type != JSON_ARRAY) {
|
||||
fprintf(stderr, "%s:%d: expected array for type %d\n", fname, line, t);
|
||||
return;
|
||||
return g;
|
||||
}
|
||||
|
||||
int within = geometry_within[t];
|
||||
@ -80,7 +83,7 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL
|
||||
}
|
||||
}
|
||||
|
||||
parse_geometry(within, j->array[i], bbox, fpos, out, op, fname, line, wx, wy, initialized, initial_x, initial_y);
|
||||
g += parse_geometry(within, j->array[i], bbox, fpos, out, op, fname, line, wx, wy, initialized, initial_x, initial_y);
|
||||
}
|
||||
} else {
|
||||
if (j->length >= 2 && j->array[0]->type == JSON_NUMBER && j->array[1]->type == JSON_NUMBER) {
|
||||
@ -98,19 +101,17 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL
|
||||
}
|
||||
}
|
||||
|
||||
if (bbox != NULL) {
|
||||
if (x < bbox[0]) {
|
||||
bbox[0] = x;
|
||||
}
|
||||
if (y < bbox[1]) {
|
||||
bbox[1] = y;
|
||||
}
|
||||
if (x > bbox[2]) {
|
||||
bbox[2] = x;
|
||||
}
|
||||
if (y > bbox[3]) {
|
||||
bbox[3] = y;
|
||||
}
|
||||
if (x < bbox[0]) {
|
||||
bbox[0] = x;
|
||||
}
|
||||
if (y < bbox[1]) {
|
||||
bbox[1] = y;
|
||||
}
|
||||
if (x > bbox[2]) {
|
||||
bbox[2] = x;
|
||||
}
|
||||
if (y > bbox[3]) {
|
||||
bbox[3] = y;
|
||||
}
|
||||
|
||||
if (!*initialized) {
|
||||
@ -134,6 +135,7 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL
|
||||
serialize_long_long(out, (y >> geometry_scale) - (*wy >> geometry_scale), fpos, fname);
|
||||
*wx = x;
|
||||
*wy = y;
|
||||
g++;
|
||||
} else {
|
||||
fprintf(stderr, "%s:%d: malformed point\n", fname, line);
|
||||
}
|
||||
@ -151,9 +153,11 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL
|
||||
|
||||
serialize_byte(out, VT_CLOSEPATH, fpos, fname);
|
||||
}
|
||||
|
||||
return g;
|
||||
}
|
||||
|
||||
int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set<type_and_string> *file_keys) {
|
||||
int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set<type_and_string> *file_keys, int maxzoom) {
|
||||
json_object *geometry_type = json_hash_get(geometry, "type");
|
||||
if (geometry_type == NULL) {
|
||||
static int warned = 0;
|
||||
@ -208,7 +212,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha
|
||||
}
|
||||
}
|
||||
|
||||
long long bbox[] = {UINT_MAX, UINT_MAX, 0, 0};
|
||||
long long bbox[] = {LLONG_MAX, LLONG_MAX, LLONG_MIN, LLONG_MIN};
|
||||
|
||||
int nprop = 0;
|
||||
if (properties != NULL && properties->type == JSON_HASH) {
|
||||
@ -269,15 +273,6 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < m; i++) {
|
||||
serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname);
|
||||
serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname);
|
||||
|
||||
if (mustfree[i]) {
|
||||
free((void *) metaval[i]);
|
||||
}
|
||||
}
|
||||
|
||||
long long geomstart = *geompos;
|
||||
|
||||
serialize_byte(geomfile, mb_geometry[t], geompos, fname);
|
||||
@ -292,12 +287,56 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha
|
||||
}
|
||||
|
||||
serialize_int(geomfile, segment, geompos, fname);
|
||||
serialize_long_long(geomfile, metastart, geompos, fname);
|
||||
serialize_int(geomfile, m, geompos, fname);
|
||||
long long wx = *initial_x, wy = *initial_y;
|
||||
parse_geometry(t, coordinates, bbox, geompos, geomfile, VT_MOVETO, fname, line, &wx, &wy, initialized, initial_x, initial_y);
|
||||
long long g = parse_geometry(t, coordinates, bbox, geompos, geomfile, VT_MOVETO, fname, line, &wx, &wy, initialized, initial_x, initial_y);
|
||||
serialize_byte(geomfile, VT_END, geompos, fname);
|
||||
|
||||
bool inline_meta = true;
|
||||
// Don't inline metadata for features that will span several tiles at maxzoom
|
||||
if (g > 0 && (bbox[2] < bbox[0] || bbox[3] < bbox[1])) {
|
||||
fprintf(stderr, "Internal error: impossible feature bounding box %llx,%llx,%llx,%llx\n", bbox[0], bbox[1], bbox[2], bbox[3]);
|
||||
}
|
||||
if (bbox[2] - bbox[0] > (2LL << (32 - maxzoom)) || bbox[3] - bbox[1] > (2LL << (32 - maxzoom))) {
|
||||
inline_meta = false;
|
||||
|
||||
if (prevent[P_CLIPPING]) {
|
||||
static volatile long long warned = 0;
|
||||
long long extent = ((bbox[2] - bbox[0]) / ((1LL << (32 - maxzoom)) + 1)) * ((bbox[3] - bbox[1]) / ((1LL << (32 - maxzoom)) + 1));
|
||||
if (extent > warned) {
|
||||
fprintf(stderr, "Warning: %s:%d: Large unclipped (-pc) feature may be duplicated across %lld tiles\n", fname, line, extent);
|
||||
warned = extent;
|
||||
|
||||
if (extent > 10000) {
|
||||
fprintf(stderr, "Exiting because this can't be right.\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
serialize_int(geomfile, m, geompos, fname);
|
||||
if (inline_meta) {
|
||||
serialize_long_long(geomfile, -1, geompos, fname);
|
||||
|
||||
for (i = 0; i < m; i++) {
|
||||
serialize_long_long(geomfile, addpool(poolfile, treefile, metakey[i], VT_STRING), geompos, fname);
|
||||
serialize_long_long(geomfile, addpool(poolfile, treefile, metaval[i], metatype[i]), geompos, fname);
|
||||
}
|
||||
} else {
|
||||
serialize_long_long(geomfile, metastart, geompos, fname);
|
||||
|
||||
for (i = 0; i < m; i++) {
|
||||
serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname);
|
||||
serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname);
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < m; i++) {
|
||||
if (mustfree[i]) {
|
||||
free((void *) metaval[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Note that feature_minzoom for lines is the dimension
|
||||
* of the geometry in world coordinates, but
|
||||
@ -366,7 +405,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha
|
||||
return 1;
|
||||
}
|
||||
|
||||
void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set<type_and_string> *file_keys) {
|
||||
void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set<type_and_string> *file_keys, int maxzoom) {
|
||||
long long found_hashes = 0;
|
||||
long long found_features = 0;
|
||||
long long found_geometries = 0;
|
||||
@ -431,7 +470,7 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se
|
||||
}
|
||||
found_geometries++;
|
||||
|
||||
serialize_geometry(j, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers, file_keys);
|
||||
serialize_geometry(j, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom);
|
||||
json_free(j);
|
||||
continue;
|
||||
}
|
||||
@ -466,10 +505,10 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se
|
||||
if (geometries != NULL) {
|
||||
size_t g;
|
||||
for (g = 0; g < geometries->length; g++) {
|
||||
serialize_geometry(geometries->array[g], properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys);
|
||||
serialize_geometry(geometries->array[g], properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom);
|
||||
}
|
||||
} else {
|
||||
serialize_geometry(geometry, properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys);
|
||||
serialize_geometry(geometry, properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom);
|
||||
}
|
||||
|
||||
json_free(j);
|
||||
@ -481,7 +520,7 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se
|
||||
void *run_parse_json(void *v) {
|
||||
struct parse_json_args *pja = (struct parse_json_args *) v;
|
||||
|
||||
parse_json(pja->jp, pja->reading, pja->layer_seq, pja->progress_seq, pja->metapos, pja->geompos, pja->indexpos, pja->exclude, pja->include, pja->exclude_all, pja->metafile, pja->geomfile, pja->indexfile, pja->poolfile, pja->treefile, pja->fname, pja->basezoom, pja->layer, pja->droprate, pja->file_bbox, pja->segment, pja->initialized, pja->initial_x, pja->initial_y, pja->readers, pja->file_keys);
|
||||
parse_json(pja->jp, pja->reading, pja->layer_seq, pja->progress_seq, pja->metapos, pja->geompos, pja->indexpos, pja->exclude, pja->include, pja->exclude_all, pja->metafile, pja->geomfile, pja->indexfile, pja->poolfile, pja->treefile, pja->fname, pja->basezoom, pja->layer, pja->droprate, pja->file_bbox, pja->segment, pja->initialized, pja->initial_x, pja->initial_y, pja->readers, pja->file_keys, pja->maxzoom);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -25,8 +25,9 @@ struct parse_json_args {
|
||||
unsigned *initial_y;
|
||||
struct reader *readers;
|
||||
std::set<type_and_string> *file_keys;
|
||||
int maxzoom;
|
||||
};
|
||||
|
||||
struct json_pull *json_begin_map(char *map, long long len);
|
||||
void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set<type_and_string> *file_keys);
|
||||
void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set<type_and_string> *file_keys, int maxzoom);
|
||||
void *run_parse_json(void *v);
|
||||
|
||||
24
main.cpp
24
main.cpp
@ -56,8 +56,8 @@ static int min_detail = 7;
|
||||
int quiet = 0;
|
||||
int geometry_scale = 0;
|
||||
|
||||
static int prevent[256];
|
||||
static int additional[256];
|
||||
int prevent[256];
|
||||
int additional[256];
|
||||
|
||||
struct source {
|
||||
std::string layer;
|
||||
@ -298,7 +298,7 @@ void *run_sort(void *v) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys) {
|
||||
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys, int maxzoom) {
|
||||
long long segs[CPUS + 1];
|
||||
segs[0] = 0;
|
||||
segs[CPUS] = len;
|
||||
@ -354,6 +354,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
|
||||
pja[i].initial_y = &initial_y[i];
|
||||
pja[i].readers = reader;
|
||||
pja[i].file_keys = &file_subkeys[i];
|
||||
pja[i].maxzoom = maxzoom;
|
||||
|
||||
if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) {
|
||||
perror("pthread_create");
|
||||
@ -422,7 +423,7 @@ void *run_read_parallel(void *v) {
|
||||
}
|
||||
madvise(map, a->len, MADV_RANDOM); // sequential, but from several pointers at once
|
||||
|
||||
do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->droprate, a->initialized, a->initial_x, a->initial_y, a->file_keys);
|
||||
do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->droprate, a->initialized, a->initial_x, a->initial_y, a->file_keys, a->maxzoom);
|
||||
|
||||
madvise(map, a->len, MADV_DONTNEED);
|
||||
if (munmap(map, a->len) != 0) {
|
||||
@ -439,7 +440,7 @@ void *run_read_parallel(void *v) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys) {
|
||||
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys, int maxzoom) {
|
||||
// This has to kick off an intermediate thread to start the parser threads,
|
||||
// so the main thread can get back to reading the next input stage while
|
||||
// the intermediate thread waits for the completion of the parser threads.
|
||||
@ -473,6 +474,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i
|
||||
rpa->initial_x = initial_x;
|
||||
rpa->initial_y = initial_y;
|
||||
rpa->file_keys = file_keys;
|
||||
rpa->maxzoom = maxzoom;
|
||||
|
||||
if (pthread_create(parallel_parser, NULL, run_read_parallel, rpa) != 0) {
|
||||
perror("pthread_create");
|
||||
@ -1015,7 +1017,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
|
||||
}
|
||||
|
||||
if (map != NULL && map != MAP_FAILED) {
|
||||
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0]);
|
||||
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom);
|
||||
overall_offset += st.st_size - off;
|
||||
checkdisk(reader, CPUS);
|
||||
|
||||
@ -1081,7 +1083,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
|
||||
}
|
||||
|
||||
fflush(readfp);
|
||||
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0]);
|
||||
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom);
|
||||
|
||||
initial_offset += ahead;
|
||||
overall_offset += ahead;
|
||||
@ -1117,7 +1119,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
|
||||
fflush(readfp);
|
||||
|
||||
if (ahead > 0) {
|
||||
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0]);
|
||||
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom);
|
||||
|
||||
if (pthread_join(parallel_parser, NULL) != 0) {
|
||||
perror("pthread_join");
|
||||
@ -1131,7 +1133,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
|
||||
|
||||
long long layer_seq = overall_offset;
|
||||
json_pull *jp = json_begin_file(fp);
|
||||
parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, &file_keys[source < nlayers ? source : 0]);
|
||||
parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, &file_keys[source < nlayers ? source : 0], maxzoom);
|
||||
json_end(jp);
|
||||
overall_offset = layer_seq;
|
||||
checkdisk(reader, CPUS);
|
||||
@ -1391,7 +1393,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
|
||||
progress_seq = indexpos / sizeof(struct index);
|
||||
|
||||
if (!quiet) {
|
||||
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", progress_seq, geompos, metapos, poolpos);
|
||||
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", progress_seq, geompos, metapos, poolpos);
|
||||
}
|
||||
|
||||
if (indexpos == 0) {
|
||||
@ -1597,7 +1599,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
|
||||
}
|
||||
|
||||
unsigned midx = 0, midy = 0;
|
||||
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y);
|
||||
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y);
|
||||
|
||||
if (maxzoom != written) {
|
||||
fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written);
|
||||
|
||||
@ -15,3 +15,6 @@
|
||||
#define P_POLYGON_SPLIT ((int) 'p')
|
||||
#define P_CLIPPING ((int) 'c')
|
||||
#define P_DUPLICATION ((int) 'D')
|
||||
|
||||
extern int prevent[256];
|
||||
extern int additional[256];
|
||||
|
||||
104
tile.cpp
104
tile.cpp
@ -57,13 +57,15 @@ bool draws_something(drawvec &geom) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char *stringpool2);
|
||||
int metacmp(int m1, const std::vector<long long> &keys1, const std::vector<long long> &values1, char *stringpool1, int m2, const std::vector<long long> &keys2, const std::vector<long long> &values2, char *stringpool2);
|
||||
int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2);
|
||||
static int is_integer(const char *s, long long *v);
|
||||
|
||||
struct coalesce {
|
||||
char *meta;
|
||||
char *stringpool;
|
||||
std::vector<long long> keys;
|
||||
std::vector<long long> values;
|
||||
drawvec geom;
|
||||
unsigned long long index;
|
||||
unsigned long long index2;
|
||||
@ -97,10 +99,7 @@ int coalcmp(const void *v1, const void *v2) {
|
||||
return cmp;
|
||||
}
|
||||
|
||||
char *m1 = c1->meta;
|
||||
char *m2 = c2->meta;
|
||||
|
||||
return metacmp(c1->m, &m1, c1->stringpool, c2->m, &m2, c2->stringpool);
|
||||
return metacmp(c1->m, c1->keys, c1->values, c1->stringpool, c2->m, c2->keys, c2->values, c2->stringpool);
|
||||
}
|
||||
|
||||
int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2) {
|
||||
@ -123,10 +122,7 @@ int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2) {
|
||||
return cmp;
|
||||
}
|
||||
|
||||
mvt_value retrieve_string(char **f, char *stringpool, int *otype) {
|
||||
long long off;
|
||||
deserialize_long_long(f, &off);
|
||||
|
||||
mvt_value retrieve_string(long long off, char *stringpool, int *otype) {
|
||||
int type = stringpool[off];
|
||||
char *s = stringpool + off + 1;
|
||||
|
||||
@ -160,18 +156,18 @@ mvt_value retrieve_string(char **f, char *stringpool, int *otype) {
|
||||
return tv;
|
||||
}
|
||||
|
||||
void decode_meta(int m, char **meta, char *stringpool, mvt_layer &layer, mvt_feature &feature) {
|
||||
void decode_meta(int m, std::vector<long long> &metakeys, std::vector<long long> &metavals, char *stringpool, mvt_layer &layer, mvt_feature &feature) {
|
||||
int i;
|
||||
for (i = 0; i < m; i++) {
|
||||
int otype;
|
||||
mvt_value key = retrieve_string(meta, stringpool, NULL);
|
||||
mvt_value value = retrieve_string(meta, stringpool, &otype);
|
||||
mvt_value key = retrieve_string(metakeys[i], stringpool, NULL);
|
||||
mvt_value value = retrieve_string(metavals[i], stringpool, &otype);
|
||||
|
||||
layer.tag(feature, key.string_value, value);
|
||||
}
|
||||
}
|
||||
|
||||
int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char *stringpool2) {
|
||||
int metacmp(int m1, const std::vector<long long> &keys1, const std::vector<long long> &values1, char *stringpool1, int m2, const std::vector<long long> &keys2, const std::vector<long long> &values2, char *stringpool2) {
|
||||
// XXX
|
||||
// Ideally this would make identical features compare the same lexically
|
||||
// even if their attributes were declared in different orders in different instances.
|
||||
@ -179,8 +175,8 @@ int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char
|
||||
|
||||
int i;
|
||||
for (i = 0; i < m1 && i < m2; i++) {
|
||||
mvt_value key1 = retrieve_string(meta1, stringpool1, NULL);
|
||||
mvt_value key2 = retrieve_string(meta2, stringpool2, NULL);
|
||||
mvt_value key1 = retrieve_string(keys1[i], stringpool1, NULL);
|
||||
mvt_value key2 = retrieve_string(keys2[i], stringpool2, NULL);
|
||||
|
||||
if (key1.string_value < key2.string_value) {
|
||||
return -1;
|
||||
@ -188,13 +184,11 @@ int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char
|
||||
return 1;
|
||||
}
|
||||
|
||||
long long off1;
|
||||
deserialize_long_long(meta1, &off1);
|
||||
long long off1 = values1[i];
|
||||
int type1 = stringpool1[off1];
|
||||
char *s1 = stringpool1 + off1 + 1;
|
||||
|
||||
long long off2;
|
||||
deserialize_long_long(meta2, &off2);
|
||||
long long off2 = values2[i];
|
||||
int type2 = stringpool2[off2];
|
||||
char *s2 = stringpool2 + off2 + 1;
|
||||
|
||||
@ -265,7 +259,7 @@ struct sll {
|
||||
}
|
||||
};
|
||||
|
||||
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m) {
|
||||
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m, std::vector<long long> &metakeys, std::vector <long long> &metavals) {
|
||||
if (geom.size() > 0 && nextzoom <= maxzoom) {
|
||||
int xo, yo;
|
||||
int span = 1 << (nextzoom - z);
|
||||
@ -347,8 +341,6 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
|
||||
serialize_int(geomfile[j], tippecanoe_maxzoom, geompos, fname);
|
||||
}
|
||||
serialize_int(geomfile[j], segment, &geompos[j], fname);
|
||||
serialize_long_long(geomfile[j], metastart, &geompos[j], fname);
|
||||
serialize_int(geomfile[j], m, &geompos[j], fname);
|
||||
long long wx = initial_x[segment], wy = initial_y[segment];
|
||||
|
||||
for (size_t u = 0; u < geom.size(); u++) {
|
||||
@ -363,6 +355,16 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
|
||||
}
|
||||
|
||||
serialize_byte(geomfile[j], VT_END, &geompos[j], fname);
|
||||
|
||||
serialize_int(geomfile[j], m, &geompos[j], fname);
|
||||
serialize_long_long(geomfile[j], metastart, &geompos[j], fname);
|
||||
if (metastart < 0) {
|
||||
for (int i = 0; i < m; i++) {
|
||||
serialize_long_long(geomfile[j], metakeys[i], &geompos[j], fname);
|
||||
serialize_long_long(geomfile[j], metavals[i], &geompos[j], fname);
|
||||
}
|
||||
}
|
||||
|
||||
serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname);
|
||||
}
|
||||
}
|
||||
@ -372,9 +374,9 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
|
||||
|
||||
struct partial {
|
||||
std::vector<drawvec> geoms;
|
||||
std::vector<long long> keys;
|
||||
std::vector<long long> values;
|
||||
char *meta;
|
||||
int *prevent;
|
||||
int *additional;
|
||||
long long layer;
|
||||
long long original_seq;
|
||||
unsigned long long index;
|
||||
@ -445,8 +447,6 @@ void *partial_feature_worker(void *v) {
|
||||
signed char t = (*partials)[i].t;
|
||||
int z = (*partials)[i].z;
|
||||
int line_detail = (*partials)[i].line_detail;
|
||||
int *prevent = (*partials)[i].prevent;
|
||||
int *additional = (*partials)[i].additional;
|
||||
int maxzoom = (*partials)[i].maxzoom;
|
||||
|
||||
double area = 0;
|
||||
@ -557,7 +557,7 @@ int manage_gap(unsigned long long index, unsigned long long *previndex, double s
|
||||
return 0;
|
||||
}
|
||||
|
||||
long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, std::vector<std::string> *layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, double gamma, int nlayers, int *prevent, int *additional, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) {
|
||||
long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, std::vector<std::string> *layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, double gamma, int nlayers, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) {
|
||||
int line_detail;
|
||||
double fraction = 1;
|
||||
|
||||
@ -649,15 +649,37 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
|
||||
int segment;
|
||||
deserialize_int_io(geoms, &segment, geompos_in);
|
||||
|
||||
long long metastart;
|
||||
int m;
|
||||
deserialize_long_long_io(geoms, &metastart, geompos_in);
|
||||
deserialize_int_io(geoms, &m, geompos_in);
|
||||
char *meta = metabase + metastart + meta_off[segment];
|
||||
long long bbox[4];
|
||||
|
||||
drawvec geom = decode_geometry(geoms, geompos_in, z, tx, ty, line_detail, bbox, initial_x[segment], initial_y[segment]);
|
||||
|
||||
long long metastart;
|
||||
int m;
|
||||
deserialize_int_io(geoms, &m, geompos_in);
|
||||
deserialize_long_long_io(geoms, &metastart, geompos_in);
|
||||
char *meta = NULL;
|
||||
std::vector<long long> metakeys, metavals;
|
||||
|
||||
if (metastart >= 0) {
|
||||
meta = metabase + metastart + meta_off[segment];
|
||||
|
||||
for (int i = 0; i < m; i++) {
|
||||
long long k, v;
|
||||
deserialize_long_long(&meta, &k);
|
||||
deserialize_long_long(&meta, &v);
|
||||
metakeys.push_back(k);
|
||||
metavals.push_back(v);
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < m; i++) {
|
||||
long long k, v;
|
||||
deserialize_long_long_io(geoms, &k, geompos_in);
|
||||
deserialize_long_long_io(geoms, &v, geompos_in);
|
||||
metakeys.push_back(k);
|
||||
metavals.push_back(v);
|
||||
}
|
||||
}
|
||||
|
||||
signed char feature_minzoom;
|
||||
deserialize_byte_io(geoms, &feature_minzoom, geompos_in);
|
||||
|
||||
@ -752,7 +774,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
|
||||
}
|
||||
|
||||
if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */
|
||||
rewrite(geom, z, nextzoom, maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, child_shards, max_zoom_increment, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom, segment, initial_x, initial_y, m);
|
||||
rewrite(geom, z, nextzoom, maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, child_shards, max_zoom_increment, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom, segment, initial_x, initial_y, m, metakeys, metavals);
|
||||
}
|
||||
|
||||
if (z < minzoom) {
|
||||
@ -815,9 +837,9 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
|
||||
p.reduced = reduced;
|
||||
p.z = z;
|
||||
p.line_detail = line_detail;
|
||||
p.prevent = prevent;
|
||||
p.additional = additional;
|
||||
p.maxzoom = maxzoom;
|
||||
p.keys = metakeys;
|
||||
p.values = metavals;
|
||||
partials.push_back(p);
|
||||
}
|
||||
}
|
||||
@ -876,6 +898,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
|
||||
c.m = partials[i].m;
|
||||
c.meta = partials[i].meta;
|
||||
c.stringpool = stringpool + pool_off[partials[i].segment];
|
||||
c.keys = partials[i].keys;
|
||||
c.values = partials[i].values;
|
||||
|
||||
features[layer].push_back(c);
|
||||
}
|
||||
@ -970,7 +994,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
|
||||
count += features[k][x].geom.size();
|
||||
features[k][x].geom.clear();
|
||||
|
||||
decode_meta(features[k][x].m, &features[k][x].meta, features[k][x].stringpool, layer, feature);
|
||||
decode_meta(features[k][x].m, features[k][x].keys, features[k][x].values, features[k][x].stringpool, layer, feature);
|
||||
layer.features.push_back(feature);
|
||||
}
|
||||
|
||||
@ -1058,8 +1082,6 @@ struct write_tile_args {
|
||||
volatile long long *along;
|
||||
double gamma;
|
||||
int nlayers;
|
||||
int *prevent;
|
||||
int *additional;
|
||||
int child_shards;
|
||||
int *geomfd;
|
||||
off_t *geom_size;
|
||||
@ -1116,7 +1138,7 @@ void *run_thread(void *vargs) {
|
||||
|
||||
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
|
||||
|
||||
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running);
|
||||
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, arg->gamma, arg->nlayers, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running);
|
||||
|
||||
if (len < 0) {
|
||||
int *err = &arg->err;
|
||||
@ -1167,7 +1189,7 @@ void *run_thread(void *vargs) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, std::vector<std::string> &layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y) {
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, std::vector<std::string> &layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y) {
|
||||
int i;
|
||||
for (i = 0; i <= maxzoom; i++) {
|
||||
long long most = 0;
|
||||
@ -1287,8 +1309,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
args[thread].along = &along; // locked with var_lock
|
||||
args[thread].gamma = gamma;
|
||||
args[thread].nlayers = nlayers;
|
||||
args[thread].prevent = prevent;
|
||||
args[thread].additional = additional;
|
||||
args[thread].child_shards = TEMP_FILES / threads;
|
||||
|
||||
args[thread].geomfd = geomfd;
|
||||
|
||||
4
tile.hpp
4
tile.hpp
@ -1,5 +1,5 @@
|
||||
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, int *prevent, int *additional);
|
||||
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers);
|
||||
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, std::vector<std::string> &layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y);
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, std::vector<std::string> &layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y);
|
||||
|
||||
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);
|
||||
|
||||
@ -1 +1 @@
|
||||
#define VERSION "tippecanoe v1.11.6\n"
|
||||
#define VERSION "tippecanoe v1.11.7\n"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user