ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-02-08 12:00:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #273 from mapbox/empty-parallel

Browse Source 
Fix error checking when reading empty files with parallel input
This commit is contained in:
Eric Fischer 2016-07-13 16:00:45 -07:00 committed by GitHub
commit a1f8564631
4 changed files with 31 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,3 +1,7 @@
## 1.12.6
* Fix error reports when reading from an empty file with parallel input
## 1.12.5 ## 1.12.5
* Add an option to vary the level of line and polygon simplification * Add an option to vary the level of line and polygon simplification

10
Makefile
View File

@ -1,6 +1,7 @@
PREFIX ?= /usr/local PREFIX ?= /usr/local
MANDIR ?= $(PREFIX)/share/man/man1/ MANDIR ?= $(PREFIX)/share/man/man1/
BUILDTYPE ?= Release BUILDTYPE ?= Release
SHELL = /bin/bash
# inherit from env if set # inherit from env if set
CC := $(CC) CC := $(CC)
@ -84,17 +85,22 @@ parallel-test:
perl -e 'for ($$i = 0; $$i < 20; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { \"yes\": \"no\", \"who\": 1 }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in1.json perl -e 'for ($$i = 0; $$i < 20; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { \"yes\": \"no\", \"who\": 1 }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in1.json
perl -e 'for ($$i = 0; $$i < 300000; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in2.json perl -e 'for ($$i = 0; $$i < 300000; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in2.json
perl -e 'for ($$i = 0; $$i < 20; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in3.json perl -e 'for ($$i = 0; $$i < 20; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in3.json
./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-file.mbtiles tests/parallel/in[123].json echo -n "" > tests/parallel/empty1.json
./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-file.mbtiles tests/parallel/in[123].json echo "" > tests/parallel/empty2.json
./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-file.mbtiles tests/parallel/in[123].json tests/parallel/empty[12].json
./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-file.mbtiles tests/parallel/in[123].json tests/parallel/empty[12].json
cat tests/parallel/in[123].json | ./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-pipe.mbtiles cat tests/parallel/in[123].json | ./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-pipe.mbtiles
cat tests/parallel/in[123].json | ./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-pipe.mbtiles cat tests/parallel/in[123].json | ./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-pipe.mbtiles
./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-pipes.mbtiles <(cat tests/parallel/in1.json) <(cat tests/parallel/empty1.json) <(cat tests/parallel/empty2.json) <(cat tests/parallel/in2.json) /dev/null <(cat tests/parallel/in3.json)
./tippecanoe-decode tests/parallel/linear-file.mbtiles > tests/parallel/linear-file.json ./tippecanoe-decode tests/parallel/linear-file.mbtiles > tests/parallel/linear-file.json
./tippecanoe-decode tests/parallel/parallel-file.mbtiles > tests/parallel/parallel-file.json ./tippecanoe-decode tests/parallel/parallel-file.mbtiles > tests/parallel/parallel-file.json
./tippecanoe-decode tests/parallel/linear-pipe.mbtiles > tests/parallel/linear-pipe.json ./tippecanoe-decode tests/parallel/linear-pipe.mbtiles > tests/parallel/linear-pipe.json
./tippecanoe-decode tests/parallel/parallel-pipe.mbtiles > tests/parallel/parallel-pipe.json ./tippecanoe-decode tests/parallel/parallel-pipe.mbtiles > tests/parallel/parallel-pipe.json
./tippecanoe-decode tests/parallel/parallel-pipes.mbtiles > tests/parallel/parallel-pipes.json
cmp tests/parallel/linear-file.json tests/parallel/parallel-file.json cmp tests/parallel/linear-file.json tests/parallel/parallel-file.json
cmp tests/parallel/linear-file.json tests/parallel/linear-pipe.json cmp tests/parallel/linear-file.json tests/parallel/linear-pipe.json
cmp tests/parallel/linear-file.json tests/parallel/parallel-pipe.json cmp tests/parallel/linear-file.json tests/parallel/parallel-pipe.json
cmp tests/parallel/linear-file.json tests/parallel/parallel-pipes.json
rm tests/parallel/*.mbtiles tests/parallel/*.json rm tests/parallel/*.mbtiles tests/parallel/*.json
pbf-test: pbf-test:

29
main.cpp
View File

@ -367,7 +367,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
void *retval; void *retval;
if (pthread_join(pthreads[i], &retval) != 0) { if (pthread_join(pthreads[i], &retval) != 0) {
perror("pthread_join"); perror("pthread_join 370");
} }
std::set<type_and_string>::iterator j; std::set<type_and_string>::iterator j;
@ -441,7 +441,7 @@ void *run_read_parallel(void *v) {
return NULL; return NULL;
} }
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys, int maxzoom) { void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys, int maxzoom) {
// This has to kick off an intermediate thread to start the parser threads, // This has to kick off an intermediate thread to start the parser threads,
// so the main thread can get back to reading the next input stage while // so the main thread can get back to reading the next input stage while
// the intermediate thread waits for the completion of the parser threads. // the intermediate thread waits for the completion of the parser threads.
@ -481,6 +481,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i
perror("pthread_create"); perror("pthread_create");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
parser_created = true;
} }
void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int splits, long long mem, const char *tmpdir, long long *availfiles, FILE *geomfile, FILE *indexfile, long long *geompos_out, long long *progress, long long *progress_max, long long *progress_reported) { void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int splits, long long mem, const char *tmpdir, long long *availfiles, FILE *geomfile, FILE *indexfile, long long *geompos_out, long long *progress, long long *progress_max, long long *progress_reported) {
@ -676,7 +677,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
void *retval; void *retval;
if (pthread_join(pthreads[a], &retval) != 0) { if (pthread_join(pthreads[a], &retval) != 0) {
perror("pthread_join"); perror("pthread_join 679");
} }
} }
@ -1065,6 +1066,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
long long ahead = 0; long long ahead = 0;
long long initial_offset = overall_offset; long long initial_offset = overall_offset;
pthread_t parallel_parser; pthread_t parallel_parser;
bool parser_created = false;
#define READ_BUF 2000 #define READ_BUF 2000
#define PARSE_MIN 10000000 #define PARSE_MIN 10000000
@ -1083,15 +1085,16 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
// wait for the parser thread instead of continuing to stream input. // wait for the parser thread instead of continuing to stream input.
if (is_parsing == 0 || ahead >= PARSE_MAX) { if (is_parsing == 0 || ahead >= PARSE_MAX) {
if (initial_offset != 0) { if (parser_created) {
if (pthread_join(parallel_parser, NULL) != 0) { if (pthread_join(parallel_parser, NULL) != 0) {
perror("pthread_join"); perror("pthread_join 1088");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
parser_created = false;
} }
fflush(readfp); fflush(readfp);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom); start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom);
initial_offset += ahead; initial_offset += ahead;
overall_offset += ahead; overall_offset += ahead;
@ -1117,20 +1120,24 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
perror(reading.c_str()); perror(reading.c_str());
} }
if (initial_offset != 0) { if (parser_created) {
if (pthread_join(parallel_parser, NULL) != 0) { if (pthread_join(parallel_parser, NULL) != 0) {
perror("pthread_join"); perror("pthread_join 1122");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
parser_created = false;
} }
fflush(readfp); fflush(readfp);
if (ahead > 0) { if (ahead > 0) {
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom); start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom);
if (pthread_join(parallel_parser, NULL) != 0) { if (parser_created) {
perror("pthread_join"); if (pthread_join(parallel_parser, NULL) != 0) {
perror("pthread_join 1133");
}
parser_created = false;
} }
overall_offset += ahead; overall_offset += ahead;

2
version.hpp
View File

@ -1 +1 @@
#define VERSION "tippecanoe v1.12.5\n" #define VERSION "tippecanoe v1.12.6\n"