diff --git a/CHANGELOG.md b/CHANGELOG.md index 362c2c9..1fcec72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.9.12 + +* Limit the size of the parallel parsing streaming input buffer +* Add an option to set the tileset's attribution + ## 1.9.11 * Fix a line simplification crash when a segment degenerates to a single point diff --git a/README.md b/README.md index 6eaf6f1..1a1d795 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ Options specified, the files are all merged into the single named layer, even if they try to specify individual names with -L. * -L _name_:_file.json_ or --named-layer=_name_:_file.json_: Specify layer names for individual files. If your shell supports it, you can use a subshell redirect like -L _name_:<(cat dir/*.json) to specify a layer name for the output of streamed input. * -n _name_ or --name=_name_: Human-readable name for the tileset (default file.json) + * -A _text_ or --attribution=_text_: Attribution (HTML) to be shown with maps that use data from this tileset. ### File control diff --git a/geojson.c b/geojson.c index 80eafc2..99f7972 100644 --- a/geojson.c +++ b/geojson.c @@ -332,6 +332,56 @@ void deserialize_byte(char **f, signed char *n) { *f += sizeof(signed char); } +int deserialize_long_long_io(FILE *f, long long *n, long long *geompos) { + unsigned long long zigzag = 0; + int shift = 0; + + while (1) { + int c = getc(f); + if (c == EOF) { + return 0; + } + (*geompos)++; + + if ((c & 0x80) == 0) { + zigzag |= ((unsigned long long) c) << shift; + shift += 7; + break; + } else { + zigzag |= ((unsigned long long) (c & 0x7F)) << shift; + shift += 7; + } + } + + *n = (zigzag >> 1) ^ (-(zigzag & 1)); + return 1; +} + +int deserialize_int_io(FILE *f, int *n, long long *geompos) { + long long ll = 0; + int ret = deserialize_long_long_io(f, &ll, geompos); + *n = ll; + return ret; +} + +int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos) { + if (fread(n, sizeof(unsigned), 1, f) != 1) { + return 0; + } + *geompos += sizeof(unsigned); + return 1; +} + +int deserialize_byte_io(FILE *f, signed char *n, long long *geompos) { + int c = getc(f); + if (c == EOF) { + return 0; + } + *n = c; + (*geompos)++; + return 1; +} + struct index { long long start; long long end; @@ -1498,7 +1548,7 @@ void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE radix1(geomfds, indexfds, nreaders, 0, splits, mem, tmpdir, availfiles, geomfile, indexfile, geompos, &progress, &progress_max, &progress_reported); } -int read_json(int argc, struct source **sourcelist, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int *prevent, int *additional, int read_parallel, int forcetable) { +int read_json(int argc, struct source **sourcelist, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int *prevent, int *additional, int read_parallel, int forcetable, const char *attribution) { int ret = EXIT_SUCCESS; struct reader reader[CPUS]; @@ -1711,6 +1761,8 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay #define READ_BUF 2000 #define PARSE_MIN 10000000 +#define PARSE_MAX (1LL * 1024 * 1024 * 1024) + char buf[READ_BUF]; int n; @@ -1718,34 +1770,40 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay fwrite_check(buf, sizeof(char), n, readfp, reading); ahead += n; - if (buf[n - 1] == '\n' && ahead > PARSE_MIN && is_parsing == 0) { - if (initial_offset != 0) { - if (pthread_join(parallel_parser, NULL) != 0) { - perror("pthread_join"); + if (buf[n - 1] == '\n' && ahead > PARSE_MIN) { + // Don't let the streaming reader get too far ahead of the parsers. + // If the buffered input gets huge, even if the parsers are still running, + // wait for the parser thread instead of continuing to stream input. + + if (is_parsing == 0 || ahead >= PARSE_MAX) { + if (initial_offset != 0) { + if (pthread_join(parallel_parser, NULL) != 0) { + perror("pthread_join"); + exit(EXIT_FAILURE); + } + } + + fflush(readfp); + start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y); + + initial_offset += ahead; + overall_offset += ahead; + checkdisk(reader, CPUS); + ahead = 0; + + sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX"); + readfd = mkstemp(readname); + if (readfd < 0) { + perror(readname); exit(EXIT_FAILURE); } + readfp = fdopen(readfd, "w"); + if (readfp == NULL) { + perror(readname); + exit(EXIT_FAILURE); + } + unlink(readname); } - - fflush(readfp); - start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y); - - initial_offset += ahead; - overall_offset += ahead; - checkdisk(reader, CPUS); - ahead = 0; - - sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX"); - readfd = mkstemp(readname); - if (readfd < 0) { - perror(readname); - exit(EXIT_FAILURE); - } - readfp = fdopen(readfd, "w"); - if (readfp == NULL) { - perror(readname); - exit(EXIT_FAILURE); - } - unlink(readname); } } if (n < 0) { @@ -2339,7 +2397,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay midlon = maxlon; } - mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable); + mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable, attribution); for (i = 0; i < nlayers; i++) { pool_free_strings(&file_keys1[i]); @@ -2385,6 +2443,7 @@ int main(int argc, char **argv) { double gamma = 0; int buffer = 5; const char *tmpdir = "/tmp"; + const char *attribution = NULL; int nsources = 0; struct source *sources = NULL; @@ -2403,6 +2462,7 @@ int main(int argc, char **argv) { static struct option long_options[] = { {"name", required_argument, 0, 'n'}, {"layer", required_argument, 0, 'l'}, + {"attribution", required_argument, 0, 'A'}, {"named-layer", required_argument, 0, 'L'}, {"maximum-zoom", required_argument, 0, 'z'}, {"minimum-zoom", required_argument, 0, 'Z'}, @@ -2445,7 +2505,7 @@ int main(int argc, char **argv) { {0, 0, 0, 0}, }; - while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:", long_options, NULL)) != -1) { + while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:", long_options, NULL)) != -1) { switch (i) { case 0: break; @@ -2458,6 +2518,10 @@ int main(int argc, char **argv) { layer = optarg; break; + case 'A': + attribution = optarg; + break; + case 'L': { char *cp = strchr(optarg, ':'); if (cp == NULL || cp == optarg) { @@ -2694,7 +2758,7 @@ int main(int argc, char **argv) { sourcelist[i--] = sources; } - ret = read_json(nsources, sourcelist, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel, forcetable); + ret = read_json(nsources, sourcelist, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel, forcetable, attribution); mbtiles_close(outdb, argv); diff --git a/geometry.cc b/geometry.cc index 27913be..dd36032 100644 --- a/geometry.cc +++ b/geometry.cc @@ -18,7 +18,7 @@ extern "C" { #include "projection.h" } -drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y) { +drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y) { drawvec out; bbox[0] = LLONG_MAX; @@ -31,7 +31,7 @@ drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail while (1) { draw d; - deserialize_byte(meta, &d.op); + deserialize_byte_io(meta, &d.op, geompos); if (d.op == VT_END) { break; } @@ -39,8 +39,8 @@ drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail if (d.op == VT_MOVETO || d.op == VT_LINETO) { long long dx, dy; - deserialize_long_long(meta, &dx); - deserialize_long_long(meta, &dy); + deserialize_long_long_io(meta, &dx, geompos); + deserialize_long_long_io(meta, &dy, geompos); wx += dx << geometry_scale; wy += dy << geometry_scale; diff --git a/geometry.hh b/geometry.hh index 907db38..6c94822 100644 --- a/geometry.hh +++ b/geometry.hh @@ -16,7 +16,7 @@ struct draw { typedef std::vector drawvec; -drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y); +drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y); void to_tile_scale(drawvec &geom, int z, int detail); drawvec remove_noop(drawvec geom, int type, int shift); drawvec clip_point(drawvec &geom, int z, int detail, long long buffer); diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 1fb86bd..793e490 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -65,6 +65,8 @@ specified, the files are all merged into the single named layer, even if they tr \-L \fIname\fP:\fIfile.json\fP or \-\-named\-layer=\fIname\fP:\fIfile.json\fP: Specify layer names for individual files. If your shell supports it, you can use a subshell redirect like \-L \fIname\fP:<(cat dir/*.json) to specify a layer name for the output of streamed input. .IP \(bu 2 \-n \fIname\fP or \-\-name=\fIname\fP: Human\-readable name for the tileset (default file.json) +.IP \(bu 2 +\-A \fItext\fP or \-\-attribution=\fItext\fP: Attribution (HTML) to be shown with maps that use data from this tileset. .RE .SS File control .RS diff --git a/mbtiles.c b/mbtiles.c index d46f16e..def1b94 100644 --- a/mbtiles.c +++ b/mbtiles.c @@ -138,7 +138,7 @@ static int pvcmp(const void *v1, const void *v2) { return (*pv1)->type - (*pv2)->type; } -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable) { +void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable, const char *attribution) { char *sql, *err; sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname); @@ -213,6 +213,17 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, } sqlite3_free(sql); + if (attribution != NULL) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", attribution); + if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "set type: %s\n", err); + if (!forcetable) { + exit(EXIT_FAILURE); + } + } + sqlite3_free(sql); + } + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", "pbf"); if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set format: %s\n", err); diff --git a/mbtiles.h b/mbtiles.h index faa3413..50f72cf 100644 --- a/mbtiles.h +++ b/mbtiles.h @@ -2,6 +2,6 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable); +void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable, const char *attribution); void mbtiles_close(sqlite3 *outdb, char **argv); diff --git a/tests/nullisland/out/-b0_-z4_-ANullIsland.json b/tests/nullisland/out/-b0_-z4_-ANullIsland.json new file mode 100644 index 0000000..33ffb0e --- /dev/null +++ b/tests/nullisland/out/-b0_-z4_-ANullIsland.json @@ -0,0 +1,251 @@ +{ "type": "FeatureCollection", "properties": { +"attribution": "NullIsland", +"bounds": "-1.000000,-1.000000,1.000000,1.000000", +"center": "-1.000000,1.000000,4", +"description": "tests/nullisland/out/-b0_-z4_-ANullIsland.json.check.mbtiles", +"format": "pbf", +"json": "{\"vector_layers\": [ { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 4, \"fields\": {} } ] }", +"maxzoom": "4", +"minzoom": "0", +"name": "tests/nullisland/out/-b0_-z4_-ANullIsland.json.check.mbtiles", +"type": "overlay", +"version": "2" +}, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.054688, 0.000000 ], [ -1.054688, 1.054628 ], [ 0.000000, 1.054628 ], [ 0.000000, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.054688, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.054688, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.966751 ], [ -1.054688, -0.966751 ], [ -1.054688, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.054628 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.966797, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.054628 ], [ 0.966797, 1.054628 ], [ 0.966797, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.966797, -0.966751 ], [ 0.000000, -0.966751 ], [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ], [ 0.966797, -0.966751 ] ] ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 0, "y": 1 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.966751 ], [ -1.010742, -0.966751 ], [ -1.010742, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ], [ -1.010742, 1.010690 ], [ 0.000000, 1.010690 ], [ 0.000000, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 1 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.966751 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.966797, -0.966751 ], [ 0.000000, -0.966751 ], [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ], [ 0.966797, -0.966751 ] ] ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.010690 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.966797, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ], [ 0.966797, 1.010690 ], [ 0.966797, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.966797, 0.000000 ] ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 1, "y": 2 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.988720 ], [ -1.010742, -0.988720 ], [ -1.010742, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.988720 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.988720 ] ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 1, "y": 1 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ], [ -1.010742, 1.010690 ], [ 0.000000, 1.010690 ], [ 0.000000, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 2, "y": 2 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.988720 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.988770, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.988770, -0.988720 ], [ 0.000000, -0.988720 ], [ 0.000000, 0.000000 ], [ 0.988770, 0.000000 ], [ 0.988770, -0.988720 ] ] ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 2, "y": 1 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.010690 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.988770, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ], [ 0.988770, 1.010690 ], [ 0.988770, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.988770, 0.000000 ] ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 3, "y": 4 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.999705 ], [ -1.010742, -0.999705 ], [ -1.010742, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, -0.999705 ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 3, "y": 3 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ], [ -1.010742, 1.010690 ], [ 0.000000, 1.010690 ], [ 0.000000, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.010742, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.010742, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 4, "y": 4 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.999756, -0.999705 ], [ 0.000000, -0.999705 ], [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ], [ 0.999756, -0.999705 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 0.000000 ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 4, "y": 3 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.010690 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.999756, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.010690 ], [ 0.999756, 1.010690 ], [ 0.999756, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 0.000000 ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 7, "y": 8 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.005249, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.005249, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, -0.999705 ], [ -1.005249, -0.999705 ], [ -1.005249, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, -0.999705 ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 7, "y": 7 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.000000, 0.000000 ], [ -1.005249, 0.000000 ], [ -1.005249, 1.005197 ], [ 0.000000, 1.005197 ], [ 0.000000, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ -1.005249, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ -1.005249, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.005197 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.005197 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 8, "y": 8 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, -0.999705 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.999756, -0.999705 ], [ 0.000000, -0.999705 ], [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ], [ 0.999756, -0.999705 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, -0.999705 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 0.000000 ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 8, "y": 7 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "in" }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 1.005197 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.000000, 1.005197 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0.999756, 0.000000 ], [ 0.000000, 0.000000 ], [ 0.000000, 1.005197 ], [ 0.999756, 1.005197 ], [ 0.999756, 0.000000 ] ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ 0.000000, 0.000000 ], [ 0.999756, 0.000000 ] ] } } +, +{ "type": "Feature", "properties": { }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 0.000000 ] } } +] } +] } +] } diff --git a/tile-join.cc b/tile-join.cc index 25794ce..8eedd8d 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -377,7 +377,7 @@ double max(double a, double b) { } } -void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector &header, std::map > &mapping, struct pool *exclude, int ifmatched) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector &header, std::map > &mapping, struct pool *exclude, int ifmatched, char **attribution) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -429,6 +429,12 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, } sqlite3_finalize(stmt); } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'attribution'", -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + *attribution = strdup((char *) sqlite3_column_text(stmt, 0)); + } + sqlite3_finalize(stmt); + } if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'bounds'", -1, &stmt, NULL) == SQLITE_OK) { if (sqlite3_step(stmt) == SQLITE_ROW) { const unsigned char *s = sqlite3_column_text(stmt, 0); @@ -593,9 +599,10 @@ int main(int argc, char **argv) { struct pool *file_keys = NULL; char **layernames = NULL; int nlayers = 0; + char *attribution = NULL; for (i = optind; i < argc; i++) { - decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping, &exclude, ifmatched); + decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping, &exclude, ifmatched, &attribution); } struct pool *fk[nlayers]; @@ -603,7 +610,7 @@ int main(int argc, char **argv) { fk[i] = &(file_keys[i]); } - mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, fk, nlayers, 0); + mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, fk, nlayers, 0, attribution); mbtiles_close(outdb, argv); return 0; diff --git a/tile.cc b/tile.cc index 6d3c8db..2db3201 100644 --- a/tile.cc +++ b/tile.cc @@ -596,11 +596,11 @@ int manage_gap(unsigned long long index, unsigned long long *previndex, double s return 0; } -long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, int *prevent, int *additional, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) { +long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, double gamma, int nlayers, int *prevent, int *additional, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) { int line_detail; double fraction = 1; - char *og = *geoms; + long long og = *geompos_in; // XXX is there a way to do this without floating point? int max_zoom_increment = log(child_shards) / log(4); @@ -669,45 +669,51 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsi memset(within, '\0', sizeof(within)); memset(geompos, '\0', sizeof(geompos)); - *geoms = og; + if (*geompos_in != og) { + if (fseek(geoms, og, SEEK_SET) != 0) { + perror("fseek geom"); + exit(EXIT_FAILURE); + } + *geompos_in = og; + } while (1) { signed char t; - deserialize_byte(geoms, &t); + deserialize_byte_io(geoms, &t, geompos_in); if (t < 0) { break; } long long original_seq; - deserialize_long_long(geoms, &original_seq); + deserialize_long_long_io(geoms, &original_seq, geompos_in); long long layer; - deserialize_long_long(geoms, &layer); + deserialize_long_long_io(geoms, &layer, geompos_in); int tippecanoe_minzoom = -1, tippecanoe_maxzoom = -1; if (layer & 2) { - deserialize_int(geoms, &tippecanoe_minzoom); + deserialize_int_io(geoms, &tippecanoe_minzoom, geompos_in); } if (layer & 1) { - deserialize_int(geoms, &tippecanoe_maxzoom); + deserialize_int_io(geoms, &tippecanoe_maxzoom, geompos_in); } layer >>= 2; int segment; - deserialize_int(geoms, &segment); + deserialize_int_io(geoms, &segment, geompos_in); long long metastart; int m; - deserialize_long_long(geoms, &metastart); - deserialize_int(geoms, &m); + deserialize_long_long_io(geoms, &metastart, geompos_in); + deserialize_int_io(geoms, &m, geompos_in); char *meta = metabase + metastart + meta_off[segment]; long long bbox[4]; - drawvec geom = decode_geometry(geoms, z, tx, ty, line_detail, bbox, initial_x[segment], initial_y[segment]); + drawvec geom = decode_geometry(geoms, geompos_in, z, tx, ty, line_detail, bbox, initial_x[segment], initial_y[segment]); signed char feature_minzoom; - deserialize_byte(geoms, &feature_minzoom); + deserialize_byte_io(geoms, &feature_minzoom, geompos_in); - double progress = floor((((*geoms - geomstart + *along) / (double) todo) + z) / (maxzoom + 1) * 1000) / 10; + double progress = floor((((*geompos_in + *along) / (double) todo) + z) / (maxzoom + 1) * 1000) / 10; if (progress >= oprogress + 0.1) { if (!quiet) { fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); @@ -1089,29 +1095,28 @@ void *run_thread(void *vargs) { // printf("%lld of geom_size\n", (long long) geom_size[j]); - char *geom = (char *) mmap(NULL, arg->geom_size[j], PROT_READ, MAP_PRIVATE, arg->geomfd[j], 0); - if (geom == MAP_FAILED) { + FILE *geom = fdopen(arg->geomfd[j], "rb"); + if (geom == NULL) { perror("mmap geom"); exit(EXIT_FAILURE); } - madvise(geom, arg->geom_size[j], MADV_SEQUENTIAL); - madvise(geom, arg->geom_size[j], MADV_WILLNEED); - char *geomstart = geom; - char *end = geom + arg->geom_size[j]; - char *prevgeom = geom; + long long geompos = 0; + long long prevgeom = 0; - while (geom < end) { + while (1) { int z; unsigned x, y; - deserialize_int(&geom, &z); - deserialize_uint(&geom, &x); - deserialize_uint(&geom, &y); + if (!deserialize_int_io(geom, &z, &geompos)) { + break; + } + deserialize_uint_io(geom, &x, &geompos); + deserialize_uint_io(geom, &y, &geompos); // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(&geom, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running); + long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running); if (len < 0) { int *err = (int *) malloc(sizeof(int)); @@ -1145,8 +1150,8 @@ void *run_thread(void *vargs) { } } - *arg->along += geom - prevgeom; - prevgeom = geom; + *arg->along += geompos - prevgeom; + prevgeom = geompos; if (pthread_mutex_unlock(&var_lock) != 0) { perror("pthread_mutex_unlock"); @@ -1154,10 +1159,12 @@ void *run_thread(void *vargs) { } } - madvise(geomstart, arg->geom_size[j], MADV_DONTNEED); - if (munmap(geomstart, arg->geom_size[j]) != 0) { - perror("munmap geom"); + if (fclose(geom) != 0) { + perror("close geom"); + exit(EXIT_FAILURE); } + // Since the fclose() has closed the underlying file descriptor + arg->geomfd[j] = -1; } arg->running--; diff --git a/tile.h b/tile.h index 3a0d3ef..904fef8 100644 --- a/tile.h +++ b/tile.h @@ -23,7 +23,11 @@ void deserialize_int(char **f, int *n); void deserialize_long_long(char **f, long long *n); void deserialize_uint(char **f, unsigned *n); void deserialize_byte(char **f, signed char *n); -struct pool_val *deserialize_string(char **f, struct pool *p, int type); + +int deserialize_int_io(FILE *f, int *n, long long *geompos); +int deserialize_long_long_io(FILE *f, long long *n, long long *geompos); +int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos); +int deserialize_byte_io(FILE *f, signed char *n, long long *geompos); long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, int *prevent, int *additional); diff --git a/version.h b/version.h index 0db3fcd..fb760ae 100644 --- a/version.h +++ b/version.h @@ -1 +1 @@ -#define VERSION "tippecanoe v1.9.11\n" +#define VERSION "tippecanoe v1.9.12\n"