ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-04-21 09:11:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added percent

Browse Source
This commit is contained in:
Evgeny Kalashnikov 2020-09-23 18:19:31 +03:00
parent ddbeebb563
commit 252797a7c7
No known key found for this signature in database
GPG Key ID: 5564239CB45C62B0
4 changed files with 31 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
main.cpp
View File

@ -153,7 +153,7 @@ long long atoll_require(const char *s, const char *what) {
return ret;
}
void init_cpus(const char *tmpdir, long long max_files) {
void init_cpus() {
const char *TIPPECANOE_MAX_THREADS = getenv("TIPPECANOE_MAX_THREADS");
if (TIPPECANOE_MAX_THREADS != NULL) {
@ -184,8 +184,8 @@ void init_cpus(const char *tmpdir, long long max_files) {
// Don't really want too many temporary files, because the file system
// will start to bog down eventually
if (MAX_FILES > max_files) {
MAX_FILES = max_files;
if (MAX_FILES > 2000) {
MAX_FILES = 2000;
}
// MacOS can run out of system file descriptors
@ -194,7 +194,7 @@ void init_cpus(const char *tmpdir, long long max_files) {
long long fds[MAX_FILES];
long long i;
for (i = 0; i < MAX_FILES; i++) {
fds[i] = open(tmpdir, O_RDONLY | O_CLOEXEC);
fds[i] = open("/dev/null", O_RDONLY | O_CLOEXEC);
if (fds[i] < 0) {
break;
}
@ -1135,7 +1135,7 @@ void choose_first_zoom(long long *file_bbox, std::vector<struct reader> &readers
}
}
int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm, std::map<std::string, attribute_op> const *attribute_accum, std::map<std::string, std::string> const &attribute_descriptions, std::string const &commandline) {
int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm, std::map<std::string, attribute_op> const *attribute_accum, std::map<std::string, std::string> const &attribute_descriptions, std::string const &commandline, double *persent) {
int ret = EXIT_SUCCESS;
std::vector<struct reader> readers;
@ -2267,7 +2267,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
std::atomic<unsigned> midx(0);
std::atomic<unsigned> midy(0);
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum, filter);
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum, filter, persent);
if (maxzoom != written) {
if (written > minzoom) {
@ -2446,10 +2446,9 @@ void parse_json_source(const char *arg, struct source &src) {
}
#ifdef TARGET_OS_IPHONE
int tippecanoe_main(int argc, char **argv, const char *tmp, double *persent) {
int tippecanoe_main(int argc, char **argv, double *persent) {
#else
int main(int argc, char **argv) {
const char *tmp = "/dev/null";
double *persent = NULL;
#endif
@ -2458,9 +2457,10 @@ int main(int argc, char **argv) {
#endif
av = argv;
init_cpus(tmp, 2000);
init_cpus();
extern int optind;
optind = 1; // opt index reset
extern char *optarg;
int i;
@ -2479,13 +2479,7 @@ int main(int argc, char **argv) {
double droprate = 2.5;
double gamma = 0;
int buffer = 5;
#ifdef TARGET_OS_IPHONE
const char *tmpdir = tmp;
#else
const char *tmpdir = "/tmp";
#endif
const char *attribution = NULL;
std::vector<source> sources;
const char *prefilter = NULL;
@ -3081,9 +3075,9 @@ int main(int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
files_open_at_start = open(tmp, O_RDONLY | O_CLOEXEC);
files_open_at_start = open("/dev/null", O_RDONLY | O_CLOEXEC);
if (files_open_at_start < 0) {
perror("open tmp");
perror("open /dev/null");
exit(EXIT_FAILURE);
}
if (close(files_open_at_start) != 0) {
@ -3193,7 +3187,7 @@ int main(int argc, char **argv) {
long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0};
ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_dir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, prefilter, postfilter, description, guess_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline);
ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_dir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, prefilter, postfilter, description, guess_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, persent);
if (outdb != NULL) {
mbtiles_close(outdb, argv[0]);
@ -3203,7 +3197,7 @@ int main(int argc, char **argv) {
muntrace();
#endif
i = open(tmp, O_RDONLY | O_CLOEXEC);
i = open("/dev/null", O_RDONLY | O_CLOEXEC);
// i < files_open_at_start is not an error, because reading from a pipe closes stdin
if (i > files_open_at_start) {
fprintf(stderr, "Internal error: did not close all files: %d\n", i);

2
main.hpp
View File

@ -33,7 +33,7 @@ struct clipbbox {
extern std::vector<clipbbox> clipbboxes;
#ifdef TARGET_OS_IPHONE
int tippecanoe_main(int argc, char **argv, const char *tmp, double *persent);
int tippecanoe_main(int argc, char **argv, double *persent);
#endif
void checkdisk(std::vector<struct reader> *r);

22
tile.cpp
View File

@ -1221,6 +1221,7 @@ struct write_tile_args {
int wrote_zoom = 0;
size_t tiling_seg = 0;
struct json_object *filter = NULL;
double *percent;
};
bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
@ -1318,7 +1319,7 @@ void remove_attributes(serial_feature &sf, std::set<std::string> const &exclude_
}
}
serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, std::atomic<long long> *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps) {
serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, std::atomic<long long> *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, double *percent) {
while (1) {
serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y);
if (sf.t < 0) {
@ -1330,6 +1331,7 @@ serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, cha
if (!quiet && !quiet_progress && progress_time()) {
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
}
*percent = progress;
*oprogress = progress;
}
@ -1479,6 +1481,7 @@ struct run_prefilter_args {
long long *pool_off = NULL;
FILE *prefilter_fp = NULL;
struct json_object *filter = NULL;
double *percent;
};
void *run_prefilter(void *v) {
@ -1486,7 +1489,7 @@ void *run_prefilter(void *v) {
json_writer state(rpa->prefilter_fp);
while (1) {
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps);
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->percent);
if (sf.t < 0) {
break;
}
@ -1715,13 +1718,16 @@ static bool line_is_too_small(drawvec const &geometry, int z, int detail) {
return true;
}
long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg) {
long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, double *percent, write_tile_args *arg) {
int line_detail;
double merge_fraction = 1;
double mingap_fraction = 1;
double minextent_fraction = 1;
static std::atomic<double> oprogress(0);
if (*percent == 0) {
oprogress = 0;
}
long long og = *geompos_in;
// XXX is there a way to do this without floating point?
@ -1839,6 +1845,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
rpa.stringpool = stringpool;
rpa.pool_off = pool_off;
rpa.filter = filter;
rpa.percent = percent;
if (pthread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) {
perror("pthread_create (prefilter writer)");
@ -1858,7 +1865,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
ssize_t which_partial = -1;
if (prefilter == NULL) {
sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps);
sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, percent);
} else {
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
}
@ -2325,6 +2332,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
if (!quiet && !quiet_progress && progress_time()) {
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
}
*percent = progress;
oprogress = progress;
}
@ -2582,7 +2590,7 @@ void *run_thread(void *vargs) {
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg);
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg->percent, arg);
if (len < 0) {
int *err = &arg->err;
@ -2647,7 +2655,7 @@ void *run_thread(void *vargs) {
return NULL;
}
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter) {
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, double *percent) {
last_progress = 0;
// The existing layermaps are one table per input thread.
@ -2844,6 +2852,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
args[thread].passes = 2 - start;
args[thread].wrote_zoom = -1;
args[thread].still_dropping = false;
args[thread].percent = percent;
if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) {
perror("pthread_create");

2
tile.hpp
View File

@ -21,7 +21,7 @@ enum attribute_op {
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, double *percent);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);