ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-02-24 02:41:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #540 from mapbox/progress-frequency

Browse Source 
Add --progress-interval setting to reduce progress indicator frequency
This commit is contained in:
Eric Fischer 2018-03-14 14:47:01 -07:00 committed by GitHub
commit e690a1b585
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 92 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,3 +1,7 @@
## 1.27.10
* Add --progress-interval setting to reduce progress indicator frequency
## 1.27.9 ## 1.27.9
* Make clusters look better by averaging locations of clustered points * Make clusters look better by averaging locations of clustered points

1
README.md
View File

@ -288,6 +288,7 @@ tippecanoe -z5 -o filtered.mbtiles -j '{ "ne_10m_admin_0_countries": [ "all", [
* `-q` or `--quiet`: Work quietly instead of reporting progress or warning messages * `-q` or `--quiet`: Work quietly instead of reporting progress or warning messages
* `-Q` or `--no-progress-indicator`: Don't report progress, but still give warnings * `-Q` or `--no-progress-indicator`: Don't report progress, but still give warnings
* `-U` _seconds_ or `--progress-interval=`_seconds_: Don't report progress more often than the specified number of _seconds_.
* `-v` or `--version`: Report Tippecanoe's version number * `-v` or `--version`: Report Tippecanoe's version number
### Filters ### Filters

4
geobuf.cpp
View File

@ -476,7 +476,9 @@ void runQueue() {
} }
} }
*((*(feature_queue[0].sst))[0].layer_seq) = *((*(feature_queue[0].sst))[CPUS - 1].layer_seq); // Lack of atomicity is OK, since we are single-threaded again here
long long was = *((*(feature_queue[0].sst))[CPUS - 1].layer_seq);
*((*(feature_queue[0].sst))[0].layer_seq) = was;
feature_queue.clear(); feature_queue.clear();
} }

69
main.cpp
View File

@ -25,6 +25,7 @@
#include <pthread.h> #include <pthread.h>
#include <getopt.h> #include <getopt.h>
#include <signal.h> #include <signal.h>
#include <sys/time.h>
#include <algorithm> #include <algorithm>
#include <vector> #include <vector>
#include <string> #include <string>
@ -66,6 +67,8 @@ static int min_detail = 7;
int quiet = 0; int quiet = 0;
int quiet_progress = 0; int quiet_progress = 0;
double progress_interval = 0;
std::atomic<double> last_progress(0);
int geometry_scale = 0; int geometry_scale = 0;
double simplification = 1; double simplification = 1;
size_t max_tile_size = 500000; size_t max_tile_size = 500000;
@ -293,6 +296,8 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map,
} }
} }
last_progress = 0;
while (head != NULL) { while (head != NULL) {
struct index ix = *((struct index *) (map + head->start)); struct index ix = *((struct index *) (map + head->start));
long long pos = *geompos; long long pos = *geompos;
@ -303,7 +308,7 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map,
// Count this as an 75%-accomplishment, since we already 25%-counted it // Count this as an 75%-accomplishment, since we already 25%-counted it
*progress += (ix.end - ix.start) * 3 / 4; *progress += (ix.end - ix.start) * 3 / 4;
if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) { if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) {
fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
*progress_reported = 100 * *progress / *progress_max; *progress_reported = 100 * *progress / *progress_max;
} }
@ -383,7 +388,7 @@ void *run_sort(void *v) {
return NULL; return NULL;
} }
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector<struct reader> *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector<struct reader> *readers, std::atomic<long long> *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) {
long long segs[CPUS + 1]; long long segs[CPUS + 1];
segs[0] = 0; segs[0] = 0;
segs[CPUS] = len; segs[CPUS] = len;
@ -399,7 +404,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
double dist_sums[CPUS]; double dist_sums[CPUS];
size_t dist_counts[CPUS]; size_t dist_counts[CPUS];
volatile long long layer_seq[CPUS]; std::atomic<long long> layer_seq[CPUS];
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
// To preserve feature ordering, unique id for each segment // To preserve feature ordering, unique id for each segment
// begins with that segment's offset into the input // begins with that segment's offset into the input
@ -476,12 +481,12 @@ struct read_parallel_arg {
FILE *fp = NULL; FILE *fp = NULL;
long long offset = 0; long long offset = 0;
long long len = 0; long long len = 0;
volatile int *is_parsing = NULL; std::atomic<int> *is_parsing = NULL;
int separator = 0; int separator = 0;
const char *reading = NULL; const char *reading = NULL;
std::vector<struct reader> *readers = NULL; std::vector<struct reader> *readers = NULL;
volatile long long *progress_seq = NULL; std::atomic<long long> *progress_seq = NULL;
std::set<std::string> *exclude = NULL; std::set<std::string> *exclude = NULL;
std::set<std::string> *include = NULL; std::set<std::string> *include = NULL;
int exclude_all = 0; int exclude_all = 0;
@ -538,7 +543,7 @@ void *run_read_parallel(void *v) {
return NULL; return NULL;
} }
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector<struct reader> *readers, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) { void start_parsing(int fd, FILE *fp, long long offset, long long len, std::atomic<int> *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector<struct reader> *readers, std::atomic<long long> *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, bool want_dist, bool filters) {
// This has to kick off an intermediate thread to start the parser threads, // This has to kick off an intermediate thread to start the parser threads,
// so the main thread can get back to reading the next input stage while // so the main thread can get back to reading the next input stage while
// the intermediate thread waits for the completion of the parser threads. // the intermediate thread waits for the completion of the parser threads.
@ -673,7 +678,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
// Count this as a 25%-accomplishment, since we will copy again // Count this as a 25%-accomplishment, since we will copy again
*progress += (ix.end - ix.start) / 4; *progress += (ix.end - ix.start) / 4;
if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) { if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) {
fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
*progress_reported = 100 * *progress / *progress_max; *progress_reported = 100 * *progress / *progress_max;
} }
@ -845,7 +850,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
// Count this as an 75%-accomplishment, since we already 25%-counted it // Count this as an 75%-accomplishment, since we already 25%-counted it
*progress += (ix.end - ix.start) * 3 / 4; *progress += (ix.end - ix.start) * 3 / 4;
if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) { if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) {
fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
*progress_reported = 100 * *progress / *progress_max; *progress_reported = 100 * *progress / *progress_max;
} }
@ -1140,7 +1145,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
diskfree = (long long) fsstat.f_bsize * fsstat.f_bavail; diskfree = (long long) fsstat.f_bsize * fsstat.f_bavail;
volatile long long progress_seq = 0; std::atomic<long long> progress_seq(0);
// 2 * CPUS: One per reader thread, one per tiling thread // 2 * CPUS: One per reader thread, one per tiling thread
int initialized[2 * CPUS]; int initialized[2 * CPUS];
@ -1277,7 +1282,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
long long layer_seq[CPUS]; std::atomic<long long> layer_seq[CPUS];
double dist_sums[CPUS]; double dist_sums[CPUS];
size_t dist_counts[CPUS]; size_t dist_counts[CPUS];
std::vector<struct serialization_state> sst; std::vector<struct serialization_state> sst;
@ -1334,7 +1339,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
if (sources[source].file.size() > 4 && sources[source].file.substr(sources[source].file.size() - 4) == std::string(".csv")) { if (sources[source].file.size() > 4 && sources[source].file.substr(sources[source].file.size() - 4) == std::string(".csv")) {
long long layer_seq[CPUS]; std::atomic<long long> layer_seq[CPUS];
double dist_sums[CPUS]; double dist_sums[CPUS];
size_t dist_counts[CPUS]; size_t dist_counts[CPUS];
@ -1469,7 +1474,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} }
unlink(readname); unlink(readname);
volatile int is_parsing = 0; std::atomic<int> is_parsing(0);
long long ahead = 0; long long ahead = 0;
long long initial_offset = overall_offset; long long initial_offset = overall_offset;
pthread_t parallel_parser; pthread_t parallel_parser;
@ -1553,7 +1558,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
} else { } else {
// Plain serial reading // Plain serial reading
long long layer_seq = overall_offset; std::atomic<long long> layer_seq(overall_offset);
json_pull *jp = json_begin_file(fp); json_pull *jp = json_begin_file(fp);
struct serialization_state sst; struct serialization_state sst;
@ -1819,8 +1824,10 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
long long indexpos = indexst.st_size; long long indexpos = indexst.st_size;
progress_seq = indexpos / sizeof(struct index); progress_seq = indexpos / sizeof(struct index);
last_progress = 0;
if (!quiet) { if (!quiet) {
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", progress_seq, geompos, metapos, poolpos); long long s = progress_seq;
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", s, geompos, metapos, poolpos);
} }
if (indexpos == 0) { if (indexpos == 0) {
@ -1856,7 +1863,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
long long nprogress = 100 * ip / indices; long long nprogress = 100 * ip / indices;
if (nprogress != progress) { if (nprogress != progress) {
progress = nprogress; progress = nprogress;
if (!quiet && !quiet_progress) { if (!quiet && !quiet_progress && progress_time()) {
fprintf(stderr, "Maxzoom: %lld%% \r", progress); fprintf(stderr, "Maxzoom: %lld%% \r", progress);
} }
} }
@ -1952,7 +1959,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
long long nprogress = 100 * ip / indices; long long nprogress = 100 * ip / indices;
if (nprogress != progress) { if (nprogress != progress) {
progress = nprogress; progress = nprogress;
if (!quiet && !quiet_progress) { if (!quiet && !quiet_progress && progress_time()) {
fprintf(stderr, "Base zoom/drop rate: %lld%% \r", progress); fprintf(stderr, "Base zoom/drop rate: %lld%% \r", progress);
} }
} }
@ -2144,7 +2151,8 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
size[j] = 0; size[j] = 0;
} }
unsigned midx = 0, midy = 0; std::atomic<unsigned> midx(0);
std::atomic<unsigned> midy(0);
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum); int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum);
if (maxzoom != written) { if (maxzoom != written) {
@ -2436,6 +2444,7 @@ int main(int argc, char **argv) {
{"Progress indicator", 0, 0, 0}, {"Progress indicator", 0, 0, 0},
{"quiet", no_argument, 0, 'q'}, {"quiet", no_argument, 0, 'q'},
{"no-progress-indicator", no_argument, 0, 'Q'}, {"no-progress-indicator", no_argument, 0, 'Q'},
{"progress-interval", required_argument, 0, 'U'},
{"version", no_argument, 0, 'v'}, {"version", no_argument, 0, 'v'},
{"", 0, 0, 0}, {"", 0, 0, 0},
@ -2686,6 +2695,10 @@ int main(int argc, char **argv) {
quiet_progress = 1; quiet_progress = 1;
break; break;
case 'U':
progress_interval = atof_require(optarg, "Progress interval");
break;
case 'p': { case 'p': {
char *cp; char *cp;
for (cp = optarg; *cp != '\0'; cp++) { for (cp = optarg; *cp != '\0'; cp++) {
@ -2937,3 +2950,25 @@ FILE *fopen_oflag(const char *name, const char *mode, int oflag) {
} }
return fdopen(fd, mode); return fdopen(fd, mode);
} }
bool progress_time() {
if (progress_interval == 0.0) {
return true;
}
struct timeval tv;
double now;
if (gettimeofday(&tv, NULL) != 0) {
fprintf(stderr, "%s: Can't get the time of day: %s\n", *av, strerror(errno));
now = 0;
} else {
now = tv.tv_sec + tv.tv_usec / 1000000.0;
}
if (now - last_progress >= progress_interval) {
last_progress = now;
return true;
} else {
return false;
}
}

4
main.hpp
View File

@ -2,6 +2,7 @@
#define MAIN_HPP #define MAIN_HPP
#include <stddef.h> #include <stddef.h>
#include <atomic>
struct index { struct index {
long long start = 0; long long start = 0;
@ -22,6 +23,8 @@ void checkdisk(std::vector<struct reader> *r);
extern int geometry_scale; extern int geometry_scale;
extern int quiet; extern int quiet;
extern int quiet_progress; extern int quiet_progress;
extern double progress_interval;
extern std::atomic<double> last_progress;
extern size_t CPUS; extern size_t CPUS;
extern size_t TEMP_FILES; extern size_t TEMP_FILES;
@ -32,6 +35,7 @@ extern int cluster_distance;
int mkstemp_cloexec(char *name); int mkstemp_cloexec(char *name);
FILE *fopen_oflag(const char *name, const char *mode, int oflag); FILE *fopen_oflag(const char *name, const char *mode, int oflag);
bool progress_time();
#define MAX_ZOOM 24 #define MAX_ZOOM 24

2
man/tippecanoe.1
View File

@ -355,6 +355,8 @@ If you don't specify, it will use \fB\fC/tmp\fR\&.
.IP \(bu 2 .IP \(bu 2
\fB\fC\-Q\fR or \fB\fC\-\-no\-progress\-indicator\fR: Don't report progress, but still give warnings \fB\fC\-Q\fR or \fB\fC\-\-no\-progress\-indicator\fR: Don't report progress, but still give warnings
.IP \(bu 2 .IP \(bu 2
\fB\fC\-U\fR \fIseconds\fP or \fB\fC\-\-progress\-interval=\fR\fIseconds\fP: Don't report progress more often than the specified number of \fIseconds\fP\&.
.IP \(bu 2
\fB\fC\-v\fR or \fB\fC\-\-version\fR: Report Tippecanoe's version number \fB\fC\-v\fR or \fB\fC\-\-version\fR: Report Tippecanoe's version number
.RE .RE
.SS Filters .SS Filters

4
serial.cpp
View File

@ -432,7 +432,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
inline_meta = false; inline_meta = false;
if (prevent[P_CLIPPING]) { if (prevent[P_CLIPPING]) {
static volatile long long warned = 0; static std::atomic<long long> warned(0);
long long extent = ((sf.bbox[2] - sf.bbox[0]) / ((1LL << (32 - sst->maxzoom)) + 1)) * ((sf.bbox[3] - sf.bbox[1]) / ((1LL << (32 - sst->maxzoom)) + 1)); long long extent = ((sf.bbox[2] - sf.bbox[0]) / ((1LL << (32 - sst->maxzoom)) + 1)) * ((sf.bbox[3] - sf.bbox[1]) / ((1LL << (32 - sst->maxzoom)) + 1));
if (extent > warned) { if (extent > warned) {
fprintf(stderr, "Warning: %s:%d: Large unclipped (-pc) feature may be duplicated across %lld tiles\n", sst->fname, sst->line, extent); fprintf(stderr, "Warning: %s:%d: Large unclipped (-pc) feature may be duplicated across %lld tiles\n", sst->fname, sst->line, extent);
@ -633,7 +633,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
if (*(sst->progress_seq) % 10000 == 0) { if (*(sst->progress_seq) % 10000 == 0) {
checkdisk(sst->readers); checkdisk(sst->readers);
if (!quiet && !quiet_progress) { if (!quiet && !quiet_progress && progress_time()) {
fprintf(stderr, "Read %.2f million features\r", *sst->progress_seq / 1000000.0); fprintf(stderr, "Read %.2f million features\r", *sst->progress_seq / 1000000.0);
} }
} }

5
serial.hpp
View File

@ -4,6 +4,7 @@
#include <stddef.h> #include <stddef.h>
#include <stdio.h> #include <stdio.h>
#include <vector> #include <vector>
#include <atomic>
#include <sys/stat.h> #include <sys/stat.h>
#include "geometry.hpp" #include "geometry.hpp"
#include "mbtiles.hpp" #include "mbtiles.hpp"
@ -101,8 +102,8 @@ struct serialization_state {
const char *fname = NULL; // source file name const char *fname = NULL; // source file name
int line = 0; // user-oriented location within source for error reports int line = 0; // user-oriented location within source for error reports
volatile long long *layer_seq = NULL; // sequence within current layer std::atomic<long long> *layer_seq = NULL; // sequence within current layer
volatile long long *progress_seq = NULL; // overall sequence for progress indicator std::atomic<long long> *progress_seq = NULL; // overall sequence for progress indicator
std::vector<struct reader> *readers = NULL; // array of data for each input thread std::vector<struct reader> *readers = NULL; // array of data for each input thread
int segment = 0; // the current input thread int segment = 0; // the current input thread

34
tile.cpp
View File

@ -1175,25 +1175,25 @@ struct write_tile_args {
const char *fname = NULL; const char *fname = NULL;
FILE **geomfile = NULL; FILE **geomfile = NULL;
double todo = 0; double todo = 0;
volatile long long *along = NULL; std::atomic<long long> *along = NULL;
double gamma = 0; double gamma = 0;
double gamma_out = 0; double gamma_out = 0;
int child_shards = 0; int child_shards = 0;
int *geomfd = NULL; int *geomfd = NULL;
off_t *geom_size = NULL; off_t *geom_size = NULL;
volatile unsigned *midx = NULL; std::atomic<unsigned> *midx = NULL;
volatile unsigned *midy = NULL; std::atomic<unsigned> *midy = NULL;
int maxzoom = 0; int maxzoom = 0;
int minzoom = 0; int minzoom = 0;
int full_detail = 0; int full_detail = 0;
int low_detail = 0; int low_detail = 0;
double simplification = 0; double simplification = 0;
volatile long long *most = NULL; std::atomic<long long> *most = NULL;
long long *meta_off = NULL; long long *meta_off = NULL;
long long *pool_off = NULL; long long *pool_off = NULL;
unsigned *initial_x = NULL; unsigned *initial_x = NULL;
unsigned *initial_y = NULL; unsigned *initial_y = NULL;
volatile int *running = NULL; std::atomic<int> *running = NULL;
int err = 0; int err = 0;
std::vector<std::map<std::string, layermap_entry>> *layermaps = NULL; std::vector<std::map<std::string, layermap_entry>> *layermaps = NULL;
std::vector<std::vector<std::string>> *layer_unmaps = NULL; std::vector<std::vector<std::string>> *layer_unmaps = NULL;
@ -1289,7 +1289,7 @@ bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
return false; return false;
} }
serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, volatile long long *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, long long *geompos, volatile double *oprogress, double todo, const char *fname, int child_shards) { serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, std::atomic<long long> *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, long long *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards) {
while (1) { while (1) {
serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y); serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y);
if (sf.t < 0) { if (sf.t < 0) {
@ -1298,7 +1298,7 @@ serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase,
double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10; double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10;
if (progress >= *oprogress + 0.1) { if (progress >= *oprogress + 0.1) {
if (!quiet && !quiet_progress) { if (!quiet && !quiet_progress && progress_time()) {
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
} }
*oprogress = progress; *oprogress = progress;
@ -1357,14 +1357,14 @@ struct run_prefilter_args {
int max_zoom_increment = 0; int max_zoom_increment = 0;
size_t pass = 0; size_t pass = 0;
size_t passes = 0; size_t passes = 0;
volatile long long *along = 0; std::atomic<long long> *along = 0;
long long alongminus = 0; long long alongminus = 0;
int buffer = 0; int buffer = 0;
int *within = NULL; int *within = NULL;
bool *first_time = NULL; bool *first_time = NULL;
FILE **geomfile = NULL; FILE **geomfile = NULL;
long long *geompos = NULL; long long *geompos = NULL;
volatile double *oprogress = NULL; std::atomic<double> *oprogress = NULL;
double todo = 0; double todo = 0;
const char *fname = 0; const char *fname = 0;
int child_shards = 0; int child_shards = 0;
@ -1593,13 +1593,13 @@ bool find_partial(std::vector<partial> &partials, serial_feature &sf, ssize_t &o
return false; return false;
} }
long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) { long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) {
int line_detail; int line_detail;
double merge_fraction = 1; double merge_fraction = 1;
double mingap_fraction = 1; double mingap_fraction = 1;
double minextent_fraction = 1; double minextent_fraction = 1;
static volatile double oprogress = 0; static std::atomic<double> oprogress(0);
long long og = *geompos_in; long long og = *geompos_in;
// XXX is there a way to do this without floating point? // XXX is there a way to do this without floating point?
@ -2163,7 +2163,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10; double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10;
if (progress >= oprogress + 0.1) { if (progress >= oprogress + 0.1) {
if (!quiet && !quiet_progress) { if (!quiet && !quiet_progress && progress_time()) {
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
} }
oprogress = progress; oprogress = progress;
@ -2476,7 +2476,9 @@ void *run_thread(void *vargs) {
return NULL; return NULL;
} }
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum) { int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum) {
last_progress = 0;
// The existing layermaps are one table per input thread. // The existing layermaps are one table per input thread.
// We need to add another one per *tiling* thread so that it can be // We need to add another one per *tiling* thread so that it can be
// safely changed during tiling. // safely changed during tiling.
@ -2500,7 +2502,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
int i; int i;
for (i = 0; i <= maxzoom; i++) { for (i = 0; i <= maxzoom; i++) {
long long most = 0; std::atomic<long long> most(0);
FILE *sub[TEMP_FILES]; FILE *sub[TEMP_FILES];
int subfd[TEMP_FILES]; int subfd[TEMP_FILES];
@ -2619,8 +2621,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
pthread_t pthreads[threads]; pthread_t pthreads[threads];
std::vector<write_tile_args> args; std::vector<write_tile_args> args;
args.resize(threads); args.resize(threads);
int running = threads; std::atomic<int> running(threads);
long long along = 0; std::atomic<long long> along(0);
for (size_t thread = 0; thread < threads; thread++) { for (size_t thread = 0; thread < threads; thread++) {
args[thread].metabase = metabase; args[thread].metabase = metabase;

3
tile.hpp
View File

@ -4,6 +4,7 @@
#include <stdio.h> #include <stdio.h>
#include <sqlite3.h> #include <sqlite3.h>
#include <vector> #include <vector>
#include <atomic>
#include <map> #include <map>
#include "mbtiles.hpp" #include "mbtiles.hpp"
@ -19,7 +20,7 @@ enum attribute_op {
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers); long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum); int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);

2
version.hpp
View File

@ -1,6 +1,6 @@
#ifndef VERSION_HPP #ifndef VERSION_HPP
#define VERSION_HPP #define VERSION_HPP
#define VERSION "tippecanoe v1.27.9\n" #define VERSION "tippecanoe v1.27.10\n"
#endif #endif