ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-03-23 20:35:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Preserve input sequence across multiple input files

Browse Source
This commit is contained in:
Eric Fischer 2016-02-02 15:43:27 -08:00
parent 49324a3dc6
commit b649e99b32
2 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Makefile
View File

@ -76,11 +76,13 @@ test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) parallel-test
parallel-test:
mkdir -p tests/parallel
perl -e 'for ($$i = 0; $$i < 300000; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in.json
./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-file.mbtiles tests/parallel/in.json
./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-file.mbtiles tests/parallel/in.json
cat tests/parallel/in.json | ./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-pipe.mbtiles
cat tests/parallel/in.json | ./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-pipe.mbtiles
perl -e 'for ($$i = 0; $$i < 20; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in1.json
perl -e 'for ($$i = 0; $$i < 300000; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in2.json
perl -e 'for ($$i = 0; $$i < 20; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in3.json
./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-file.mbtiles tests/parallel/in[123].json
./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-file.mbtiles tests/parallel/in[123].json
cat tests/parallel/in[123].json | ./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-pipe.mbtiles
cat tests/parallel/in[123].json | ./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-pipe.mbtiles
./tippecanoe-decode tests/parallel/linear-file.mbtiles > tests/parallel/linear-file.json
./tippecanoe-decode tests/parallel/parallel-file.mbtiles > tests/parallel/parallel-file.json
./tippecanoe-decode tests/parallel/linear-pipe.mbtiles > tests/parallel/linear-pipe.json

13
geojson.c
View File

@ -1203,6 +1203,8 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
nsources = 1;
}
long overall_offset = 0;
int source;
for (source = 0; source < nsources; source++) {
const char *reading;
@ -1234,7 +1236,8 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
}
if (map != NULL && map != MAP_FAILED) {
do_read_parallel(map, st.st_size - off, 0, reading, reader, &progress_seq, exclude, include, exclude_all, fname, maxzoom, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y);
do_read_parallel(map, st.st_size - off, overall_offset, reading, reader, &progress_seq, exclude, include, exclude_all, fname, maxzoom, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y);
overall_offset += st.st_size - off;
if (munmap(map, st.st_size - off) != 0) {
perror("munmap source file");
@ -1266,7 +1269,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
volatile int is_parsing = 0;
long long ahead = 0;
long long initial_offset = 0;
long long initial_offset = overall_offset;
pthread_t parallel_parser;
#define READ_BUF 2000
@ -1290,6 +1293,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, reading, reader, &progress_seq, exclude, include, exclude_all, fname, maxzoom, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y);
initial_offset += ahead;
overall_offset += ahead;
ahead = 0;
sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX");
@ -1325,14 +1329,17 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
if (pthread_join(parallel_parser, NULL) != 0) {
perror("pthread_join");
}
overall_offset += ahead;
}
} else {
// Plain serial reading
long long layer_seq = 0;
long long layer_seq = overall_offset;
json_pull *jp = json_begin_file(fp);
parse_json(jp, reading, &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, maxzoom, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0]);
json_end(jp);
overall_offset = layer_seq;
}
fclose(fp);