ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-02-08 03:50:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

More structure initializers

Browse Source
This commit is contained in:
Eric Fischer 2017-11-07 15:20:17 -08:00
parent 891f99f24c
commit ba62ab8596
10 changed files with 146 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
geobuf.cpp
View File

@ -26,7 +26,7 @@ struct queued_feature {
size_t dim; size_t dim;
double e; double e;
std::vector<std::string> *keys; std::vector<std::string> *keys;
struct serialization_state *sst; std::vector<struct serialization_state> *sst;
int layer; int layer;
std::string layername; std::string layername;
}; };
@ -432,7 +432,7 @@ void *run_parse_feature(void *v) {
for (size_t i = qra->start; i < qra->end; i++) { for (size_t i = qra->start; i < qra->end; i++) {
struct queued_feature &qf = feature_queue[i]; struct queued_feature &qf = feature_queue[i];
readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &qf.sst[qra->segment], qf.layer, qf.layername); readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &(*qf.sst)[qra->segment], qf.layer, qf.layername);
} }
return NULL; return NULL;
@ -447,7 +447,7 @@ void runQueue() {
pthread_t pthreads[CPUS]; pthread_t pthreads[CPUS];
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
*(feature_queue[0].sst[i].layer_seq) = *(feature_queue[0].sst[0].layer_seq) + feature_queue.size() * i / CPUS; *((*(feature_queue[0].sst))[i].layer_seq) = *((*(feature_queue[0].sst))[0].layer_seq) + feature_queue.size() * i / CPUS;
qra[i].start = feature_queue.size() * i / CPUS; qra[i].start = feature_queue.size() * i / CPUS;
qra[i].end = feature_queue.size() * (i + 1) / CPUS; qra[i].end = feature_queue.size() * (i + 1) / CPUS;
@ -467,11 +467,11 @@ void runQueue() {
} }
} }
*(feature_queue[0].sst[0].layer_seq) = *(feature_queue[0].sst[CPUS - 1].layer_seq); *((*(feature_queue[0].sst))[0].layer_seq) = *((*(feature_queue[0].sst))[CPUS - 1].layer_seq);
feature_queue.clear(); feature_queue.clear();
} }
void queueFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) { void queueFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, std::vector<struct serialization_state> *sst, int layer, std::string layername) {
struct queued_feature qf; struct queued_feature qf;
qf.pbf = pbf; qf.pbf = pbf;
qf.dim = dim; qf.dim = dim;
@ -506,7 +506,7 @@ void outBareGeometry(drawvec const &dv, int type, struct serialization_state *ss
serialize_feature(sst, sf); serialize_feature(sst, sf);
} }
void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) { void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, std::vector<struct serialization_state> *sst, int layer, std::string layername) {
while (pbf.next()) { while (pbf.next()) {
switch (pbf.tag()) { switch (pbf.tag()) {
case 1: { case 1: {
@ -521,7 +521,7 @@ void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std
} }
} }
void parse_geobuf(struct serialization_state *sst, const char *src, size_t len, int layer, std::string layername) { void parse_geobuf(std::vector<struct serialization_state> *sst, const char *src, size_t len, int layer, std::string layername) {
protozero::pbf_reader pbf(src, len); protozero::pbf_reader pbf(src, len);
size_t dim = 2; size_t dim = 2;
@ -558,7 +558,8 @@ void parse_geobuf(struct serialization_state *sst, const char *src, size_t len,
protozero::pbf_reader geometry_reader(pbf.get_message()); protozero::pbf_reader geometry_reader(pbf.get_message());
std::vector<drawvec_type> dv = readGeometry(geometry_reader, dim, e, keys); std::vector<drawvec_type> dv = readGeometry(geometry_reader, dim, e, keys);
for (size_t i = 0; i < dv.size(); i++) { for (size_t i = 0; i < dv.size(); i++) {
outBareGeometry(dv[i].dv, dv[i].type, sst, layer, layername); // Always on thread 0
outBareGeometry(dv[i].dv, dv[i].type, &(*sst)[0], layer, layername);
} }
break; break;
} }

2
geobuf.hpp
View File

@ -8,6 +8,6 @@
#include "mbtiles.hpp" #include "mbtiles.hpp"
#include "serial.hpp" #include "serial.hpp"
void parse_geobuf(struct serialization_state *sst, const char *s, size_t len, int layer, std::string layername); void parse_geobuf(std::vector<struct serialization_state> *sst, const char *s, size_t len, int layer, std::string layername);
#endif #endif

12
geojson.hpp
View File

@ -10,13 +10,13 @@
#include "serial.hpp" #include "serial.hpp"
struct parse_json_args { struct parse_json_args {
json_pull *jp; json_pull *jp = NULL;
int layer; int layer = 0;
std::string *layername; std::string *layername = NULL;
std::map<std::string, int> const *attribute_types; std::map<std::string, int> const *attribute_types = NULL;
bool want_dist; bool want_dist = false;
struct serialization_state *sst; struct serialization_state *sst = NULL;
}; };
struct json_pull *json_begin_map(char *map, long long len); struct json_pull *json_begin_map(char *map, long long len);

20
geometry.hpp
View File

@ -22,18 +22,18 @@ struct draw {
long long y : 40; long long y : 40;
signed char necessary; signed char necessary;
draw(int nop, long long nx, long long ny) { draw(int nop, long long nx, long long ny)
this->op = nop; : x(nx),
this->x = nx; op(nop),
this->y = ny; y(ny),
this->necessary = 0; necessary(0) {
} }
draw() { draw()
this->op = 0; : x(0),
this->x = 0; op(0),
this->y = 0; y(0),
this->necessary = 0; necessary(0) {
} }
bool operator<(draw const &s) const { bool operator<(draw const &s) const {

69
main.cpp
View File

@ -80,13 +80,12 @@ size_t TEMP_FILES;
long long MAX_FILES; long long MAX_FILES;
static long long diskfree; static long long diskfree;
void checkdisk(struct reader *r, int nreader) { void checkdisk(std::vector<struct reader> *r) {
long long used = 0; long long used = 0;
int i; for (size_t i = 0; i < r->size(); i++) {
for (i = 0; i < nreader; i++) {
// Meta, pool, and tree are used once. // Meta, pool, and tree are used once.
// Geometry and index will be duplicated during sorting and tiling. // Geometry and index will be duplicated during sorting and tiling.
used += r[i].metapos + 2 * r[i].geompos + 2 * r[i].indexpos + r[i].poolfile->len + r[i].treefile->len; used += (*r)[i].metapos + 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len;
} }
static int warned = 0; static int warned = 0;
@ -167,9 +166,9 @@ int indexcmp(const void *v1, const void *v2) {
const struct index *i1 = (const struct index *) v1; const struct index *i1 = (const struct index *) v1;
const struct index *i2 = (const struct index *) v2; const struct index *i2 = (const struct index *) v2;
if (i1->index < i2->index) { if (i1->ix < i2->ix) {
return -1; return -1;
} else if (i1->index > i2->index) { } else if (i1->ix > i2->ix) {
return 1; return 1;
} }
@ -212,7 +211,7 @@ struct drop_state {
int calc_feature_minzoom(struct index *ix, struct drop_state *ds, int maxzoom, double gamma) { int calc_feature_minzoom(struct index *ix, struct drop_state *ds, int maxzoom, double gamma) {
int feature_minzoom = 0; int feature_minzoom = 0;
unsigned xx, yy; unsigned xx, yy;
decode(ix->index, &xx, &yy); decode(ix->ix, &xx, &yy);
if (gamma >= 0 && (ix->t == VT_POINT || if (gamma >= 0 && (ix->t == VT_POINT ||
(additional[A_LINE_DROP] && ix->t == VT_LINE) || (additional[A_LINE_DROP] && ix->t == VT_LINE) ||
@ -350,7 +349,7 @@ void *run_sort(void *v) {
return NULL; return NULL;
} }
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector<struct reader> *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) {
long long segs[CPUS + 1]; long long segs[CPUS + 1];
segs[0] = 0; segs[0] = 0;
segs[CPUS] = len; segs[CPUS] = len;
@ -374,8 +373,12 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
dist_sums[i] = dist_counts[i] = 0; dist_sums[i] = dist_counts[i] = 0;
} }
struct parse_json_args pja[CPUS]; std::vector<parse_json_args> pja;
struct serialization_state sst[CPUS]; pja.resize(CPUS);
std::vector<serialization_state> sst;
sst.resize(CPUS);
pthread_t pthreads[CPUS]; pthread_t pthreads[CPUS];
std::vector<std::set<type_and_string> > file_subkeys; std::vector<std::set<type_and_string> > file_subkeys;
@ -442,7 +445,7 @@ struct read_parallel_arg {
int separator; int separator;
const char *reading; const char *reading;
struct reader *readers; std::vector<struct reader> *readers;
volatile long long *progress_seq; volatile long long *progress_seq;
std::set<std::string> *exclude; std::set<std::string> *exclude;
std::set<std::string> *include; std::set<std::string> *include;
@ -500,7 +503,7 @@ void *run_read_parallel(void *v) {
return NULL; return NULL;
} }
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector<struct reader> *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) {
// This has to kick off an intermediate thread to start the parser threads, // This has to kick off an intermediate thread to start the parser threads,
// so the main thread can get back to reading the next input stage while // so the main thread can get back to reading the next input stage while
// the intermediate thread waits for the completion of the parser threads. // the intermediate thread waits for the completion of the parser threads.
@ -627,7 +630,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
for (size_t a = 0; a < indexst.st_size / sizeof(struct index); a++) { for (size_t a = 0; a < indexst.st_size / sizeof(struct index); a++) {
struct index ix = indexmap[a]; struct index ix = indexmap[a];
unsigned long long which = (ix.index << prefix) >> (64 - splitbits); unsigned long long which = (ix.ix << prefix) >> (64 - splitbits);
long long pos = sub_geompos[which]; long long pos = sub_geompos[which];
fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], "geom"); fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], "geom");
@ -871,7 +874,7 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d
} }
} }
void radix(struct reader *readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, long long *geompos, int maxzoom, int basezoom, double droprate, double gamma) { void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, long long *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
// Run through the index and geometry for each reader, // Run through the index and geometry for each reader,
// splitting the contents out by index into as many // splitting the contents out by index into as many
// sub-files as we can write to simultaneously. // sub-files as we can write to simultaneously.
@ -948,7 +951,7 @@ void radix(struct reader *readers, int nreaders, FILE *geomfile, FILE *indexfile
} }
} }
void choose_first_zoom(long long *file_bbox, struct reader *readers, unsigned *iz, unsigned *ix, unsigned *iy, int minzoom, int buffer) { void choose_first_zoom(long long *file_bbox, std::vector<struct reader> &readers, unsigned *iz, unsigned *ix, unsigned *iy, int minzoom, int buffer) {
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
if (readers[i].file_bbox[0] < file_bbox[0]) { if (readers[i].file_bbox[0] < file_bbox[0]) {
file_bbox[0] = readers[i].file_bbox[0]; file_bbox[0] = readers[i].file_bbox[0];
@ -1002,7 +1005,8 @@ void choose_first_zoom(long long *file_bbox, struct reader *readers, unsigned *i
int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm) { int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm) {
int ret = EXIT_SUCCESS; int ret = EXIT_SUCCESS;
struct reader readers[CPUS]; std::vector<struct reader> readers;
readers.resize(CPUS);
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
struct reader *r = &readers[i]; struct reader *r = &readers[i];
@ -1209,7 +1213,8 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
long long layer_seq[CPUS]; long long layer_seq[CPUS];
double dist_sums[CPUS]; double dist_sums[CPUS];
size_t dist_counts[CPUS]; size_t dist_counts[CPUS];
struct serialization_state sst[CPUS]; std::vector<struct serialization_state> sst;
sst.resize(CPUS);
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
layer_seq[i] = overall_offset; layer_seq[i] = overall_offset;
@ -1220,7 +1225,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
sst[i].line = 0; sst[i].line = 0;
sst[i].layer_seq = &layer_seq[i]; sst[i].layer_seq = &layer_seq[i];
sst[i].progress_seq = &progress_seq; sst[i].progress_seq = &progress_seq;
sst[i].readers = readers; sst[i].readers = &readers;
sst[i].segment = i; sst[i].segment = i;
sst[i].initial_x = &initial_x[i]; sst[i].initial_x = &initial_x[i];
sst[i].initial_y = &initial_y[i]; sst[i].initial_y = &initial_y[i];
@ -1240,7 +1245,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
sst[i].attribute_types = attribute_types; sst[i].attribute_types = attribute_types;
} }
parse_geobuf(sst, map, st.st_size, layer, sources[layer].layer); parse_geobuf(&sst, map, st.st_size, layer, sources[layer].layer);
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
dist_sum += dist_sums[i]; dist_sum += dist_sums[i];
@ -1257,7 +1262,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
overall_offset = layer_seq[0]; overall_offset = layer_seq[0];
checkdisk(readers, CPUS); checkdisk(&readers);
continue; continue;
} }
@ -1298,9 +1303,9 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
if (map != NULL && map != MAP_FAILED && read_parallel_this) { if (map != NULL && map != MAP_FAILED && read_parallel_this) {
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), readers, &progress_seq, exclude, include, exclude_all, filter, basezoom, layer, &layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, uses_gamma, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, filter, basezoom, layer, &layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, uses_gamma, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL);
overall_offset += st.st_size - off; overall_offset += st.st_size - off;
checkdisk(readers, CPUS); checkdisk(&readers);
if (munmap(map, st.st_size - off) != 0) { if (munmap(map, st.st_size - off) != 0) {
perror("munmap source file"); perror("munmap source file");
@ -1374,11 +1379,11 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
fflush(readfp); fflush(readfp);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), readers, &progress_seq, exclude, include, exclude_all, filter, basezoom, layer, layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, filter, basezoom, layer, layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL);
initial_offset += ahead; initial_offset += ahead;
overall_offset += ahead; overall_offset += ahead;
checkdisk(readers, CPUS); checkdisk(&readers);
ahead = 0; ahead = 0;
sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX"); sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX");
@ -1411,7 +1416,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
fflush(readfp); fflush(readfp);
if (ahead > 0) { if (ahead > 0) {
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), readers, &progress_seq, exclude, include, exclude_all, filter, basezoom, layer, layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL); start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), &readers, &progress_seq, exclude, include, exclude_all, filter, basezoom, layer, layermaps, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types, read_parallel_this, &dist_sum, &dist_count, guess_maxzoom, prefilter != NULL || postfilter != NULL);
if (parser_created) { if (parser_created) {
if (pthread_join(parallel_parser, NULL) != 0) { if (pthread_join(parallel_parser, NULL) != 0) {
@ -1421,7 +1426,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
overall_offset += ahead; overall_offset += ahead;
checkdisk(readers, CPUS); checkdisk(&readers);
} }
} else { } else {
// Plain serial reading // Plain serial reading
@ -1434,7 +1439,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
sst.line = 0; sst.line = 0;
sst.layer_seq = &layer_seq; sst.layer_seq = &layer_seq;
sst.progress_seq = &progress_seq; sst.progress_seq = &progress_seq;
sst.readers = readers; sst.readers = &readers;
sst.segment = 0; sst.segment = 0;
sst.initial_x = &initial_x[0]; sst.initial_x = &initial_x[0];
sst.initial_y = &initial_y[0]; sst.initial_y = &initial_y[0];
@ -1456,7 +1461,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
parse_json(&sst, jp, layer, sources[layer].layer); parse_json(&sst, jp, layer, sources[layer].layer);
json_end(jp); json_end(jp);
overall_offset = layer_seq; overall_offset = layer_seq;
checkdisk(readers, CPUS); checkdisk(&readers);
} }
if (fclose(fp) != 0) { if (fclose(fp) != 0) {
@ -1699,9 +1704,9 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
long long progress = -1; long long progress = -1;
long long ip; long long ip;
for (ip = 1; ip < indices; ip++) { for (ip = 1; ip < indices; ip++) {
if (map[ip].index != map[ip - 1].index) { if (map[ip].ix != map[ip - 1].ix) {
count++; count++;
sum += log(map[ip].index - map[ip - 1].index); sum += log(map[ip].ix - map[ip - 1].ix);
} }
long long nprogress = 100 * ip / indices; long long nprogress = 100 * ip / indices;
@ -1798,7 +1803,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
long long ip; long long ip;
for (ip = 0; ip < indices; ip++) { for (ip = 0; ip < indices; ip++) {
unsigned xx, yy; unsigned xx, yy;
decode(map[ip].index, &xx, &yy); decode(map[ip].ix, &xx, &yy);
long long nprogress = 100 * ip / indices; long long nprogress = 100 * ip / indices;
if (nprogress != progress) { if (nprogress != progress) {
@ -1833,7 +1838,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
tile[z].fullcount++; tile[z].fullcount++;
if (manage_gap(map[ip].index, &tile[z].previndex, scale, gamma, &tile[z].gap)) { if (manage_gap(map[ip].ix, &tile[z].previndex, scale, gamma, &tile[z].gap)) {
continue; continue;
} }

15
main.hpp
View File

@ -4,15 +4,20 @@
#include <stddef.h> #include <stddef.h>
struct index { struct index {
long long start; long long start = 0;
long long end; long long end = 0;
unsigned long long index; unsigned long long ix = 0;
short segment; short segment = 0;
unsigned short t : 2; unsigned short t : 2;
unsigned long long seq : (64 - 18); // pack with segment and t to stay in 32 bytes unsigned long long seq : (64 - 18); // pack with segment and t to stay in 32 bytes
index()
: t(0),
seq(0) {
}
}; };
void checkdisk(struct reader *r, int nreader); void checkdisk(std::vector<struct reader> *r);
extern int geometry_scale; extern int geometry_scale;
extern int quiet; extern int quiet;

14
mbtiles.hpp
View File

@ -6,25 +6,25 @@
#include "mvt.hpp" #include "mvt.hpp"
struct type_and_string { struct type_and_string {
int type; int type = 0;
std::string string; std::string string = "";
bool operator<(const type_and_string &o) const; bool operator<(const type_and_string &o) const;
bool operator!=(const type_and_string &o) const; bool operator!=(const type_and_string &o) const;
}; };
struct type_and_string_stats { struct type_and_string_stats {
std::vector<type_and_string> sample_values; // sorted std::vector<type_and_string> sample_values = std::vector<type_and_string>(); // sorted
double min = INFINITY; double min = INFINITY;
double max = -INFINITY; double max = -INFINITY;
int type = 0; int type = 0;
}; };
struct layermap_entry { struct layermap_entry {
size_t id; size_t id = 0;
std::map<std::string, type_and_string_stats> file_keys; std::map<std::string, type_and_string_stats> file_keys = std::map<std::string, type_and_string_stats>();
int minzoom; int minzoom = 0;
int maxzoom; int maxzoom = 0;
size_t points = 0; size_t points = 0;
size_t lines = 0; size_t lines = 0;

4
mvt.hpp
View File

@ -100,8 +100,8 @@ struct mvt_layer {
void tag(mvt_feature &feature, std::string key, mvt_value value); void tag(mvt_feature &feature, std::string key, mvt_value value);
// For tracking the key-value constants already used in this layer // For tracking the key-value constants already used in this layer
std::map<std::string, size_t> key_map; std::map<std::string, size_t> key_map = std::map<std::string, size_t>();
std::map<mvt_value, size_t> value_map; std::map<mvt_value, size_t> value_map = std::map<mvt_value, size_t>();
}; };
struct mvt_tile { struct mvt_tile {

6
serial.cpp
View File

@ -379,7 +379,7 @@ static long long scale_geometry(struct serialization_state *sst, long long *bbox
} }
int serialize_feature(struct serialization_state *sst, serial_feature &sf) { int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
struct reader *r = &(sst->readers[sst->segment]); struct reader *r = &(*sst->readers)[sst->segment];
sf.bbox[0] = LLONG_MAX; sf.bbox[0] = LLONG_MAX;
sf.bbox[1] = LLONG_MAX; sf.bbox[1] = LLONG_MAX;
@ -611,7 +611,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
index.segment = sst->segment; index.segment = sst->segment;
index.seq = *(sst->layer_seq); index.seq = *(sst->layer_seq);
index.t = sf.t; index.t = sf.t;
index.index = bbox_index; index.ix = bbox_index;
fwrite_check(&index, sizeof(struct index), 1, r->indexfile, sst->fname); fwrite_check(&index, sizeof(struct index), 1, r->indexfile, sst->fname);
r->indexpos += sizeof(struct index); r->indexpos += sizeof(struct index);
@ -628,7 +628,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
} }
if (*(sst->progress_seq) % 10000 == 0) { if (*(sst->progress_seq) % 10000 == 0) {
checkdisk(sst->readers, CPUS); checkdisk(sst->readers);
if (!quiet) { if (!quiet) {
fprintf(stderr, "Read %.2f million features\r", *sst->progress_seq / 1000000.0); fprintf(stderr, "Read %.2f million features\r", *sst->progress_seq / 1000000.0);
} }

122
serial.hpp
View File

@ -31,102 +31,102 @@ int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos);
int deserialize_byte_io(FILE *f, signed char *n, long long *geompos); int deserialize_byte_io(FILE *f, signed char *n, long long *geompos);
struct serial_val { struct serial_val {
int type; int type = 0;
std::string s; std::string s = "";
}; };
struct serial_feature { struct serial_feature {
long long layer; long long layer = 0;
int segment; int segment = 0;
long long seq; long long seq = 0;
signed char t; signed char t = 0;
signed char feature_minzoom; signed char feature_minzoom = 0;
bool has_id; bool has_id = false;
unsigned long long id; unsigned long long id = 0;
bool has_tippecanoe_minzoom; bool has_tippecanoe_minzoom = false;
int tippecanoe_minzoom; int tippecanoe_minzoom = 0;
bool has_tippecanoe_maxzoom; bool has_tippecanoe_maxzoom = false;
int tippecanoe_maxzoom; int tippecanoe_maxzoom = 0;
drawvec geometry; drawvec geometry = drawvec();
unsigned long long index; unsigned long long index = 0;
long long extent; long long extent = 0;
size_t m; size_t m = 0;
std::vector<long long> keys; std::vector<long long> keys = std::vector<long long>();
std::vector<long long> values; std::vector<long long> values = std::vector<long long>();
long long metapos; long long metapos = 0;
// XXX This isn't serialized. Should it be here? // XXX This isn't serialized. Should it be here?
long long bbox[4]; long long bbox[4] = {0, 0, 0, 0};
std::vector<std::string> full_keys; std::vector<std::string> full_keys = std::vector<std::string>();
std::vector<serial_val> full_values; std::vector<serial_val> full_values = std::vector<serial_val>();
std::string layername; std::string layername = "";
}; };
void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, const char *fname, long long wx, long long wy, bool include_minzoom); void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, const char *fname, long long wx, long long wy, bool include_minzoom);
serial_feature deserialize_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y); serial_feature deserialize_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
struct reader { struct reader {
int metafd; int metafd = -1;
int poolfd; int poolfd = -1;
int treefd; int treefd = -1;
int geomfd; int geomfd = -1;
int indexfd; int indexfd = -1;
FILE *metafile; FILE *metafile = NULL;
struct memfile *poolfile; struct memfile *poolfile = NULL;
struct memfile *treefile; struct memfile *treefile = NULL;
FILE *geomfile; FILE *geomfile = NULL;
FILE *indexfile; FILE *indexfile = NULL;
long long metapos; long long metapos = 0;
long long geompos; long long geompos = 0;
long long indexpos; long long indexpos = 0;
long long file_bbox[4]; long long file_bbox[4] = {0, 0, 0, 0};
struct stat geomst; struct stat geomst;
struct stat metast; struct stat metast;
char *geom_map; char *geom_map = NULL;
}; };
struct serialization_state { struct serialization_state {
const char *fname; // source file name const char *fname = NULL; // source file name
int line; // user-oriented location within source for error reports int line = 0; // user-oriented location within source for error reports
volatile long long *layer_seq; // sequence within current layer volatile long long *layer_seq = NULL; // sequence within current layer
volatile long long *progress_seq; // overall sequence for progress indicator volatile long long *progress_seq = NULL; // overall sequence for progress indicator
struct reader *readers; // array of data for each input thread std::vector<struct reader> *readers = NULL; // array of data for each input thread
int segment; // the current input thread int segment = 0; // the current input thread
unsigned *initial_x; // relative offset of all geometries unsigned *initial_x = NULL; // relative offset of all geometries
unsigned *initial_y; unsigned *initial_y = NULL;
int *initialized; int *initialized = NULL;
double *dist_sum; // running tally for calculation of resolution within features double *dist_sum = NULL; // running tally for calculation of resolution within features
size_t *dist_count; size_t *dist_count = NULL;
bool want_dist; bool want_dist = false;
int maxzoom; int maxzoom = 0;
int basezoom; int basezoom = 0;
bool filters; bool filters = false;
bool uses_gamma; bool uses_gamma = false;
std::map<std::string, layermap_entry> *layermap; std::map<std::string, layermap_entry> *layermap = NULL;
std::map<std::string, int> const *attribute_types; std::map<std::string, int> const *attribute_types = NULL;
std::set<std::string> *exclude; std::set<std::string> *exclude = NULL;
std::set<std::string> *include; std::set<std::string> *include = NULL;
int exclude_all; int exclude_all = 0;
json_object *filter; json_object *filter = NULL;
}; };
int serialize_feature(struct serialization_state *sst, serial_feature &sf); int serialize_feature(struct serialization_state *sst, serial_feature &sf);