diff --git a/CHANGELOG.md b/CHANGELOG.md index eebed5f..a686a0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.9.5 + +* Remove temporary files that were accidentally left behind +* Be more careful about checking memory allocations and array bounds +* Add GNU-style long options + ## 1.9.4 * Tippecanoe-decode can decode .pbf files that aren't in an .mbtiles container diff --git a/README.md b/README.md index f86fc63..60c6029 100644 --- a/README.md +++ b/README.md @@ -57,18 +57,18 @@ Options ### Naming - * -l _name_: Layer name (default "file" if source is file.json or output is file.mbtiles). If there are multiple input files + * -l _name_ or --layer=_name_: Layer name (default "file" if source is file.json or output is file.mbtiles). If there are multiple input files specified, the files are all merged into the single named layer. - * -n _name_: Human-readable name (default file.json) + * -n _name_ or --name=_name_: Human-readable name (default file.json) ### File control - * -o _file_.mbtiles: Name the output file. - * -f: Delete the mbtiles file if it already exists instead of giving an error - * -F: Proceed (without deleting existing data) if the metadata or tiles table already exists + * -o _file_.mbtiles or --output=_file_.mbtiles: Name the output file. + * -f or --force: Delete the mbtiles file if it already exists instead of giving an error + * -F or --allow-existing: Proceed (without deleting existing data) if the metadata or tiles table already exists or if metadata fields can't be set - * -t _directory_: Put the temporary files in _directory_. - * -P: Use multiple threads to read different parts of each input file at once. + * -t _directory_ or --temporary-directory=_directory_: Put the temporary files in _directory_. + * -P or --read-parallel: Use multiple threads to read different parts of each input file at once. This will only work if the input is line-delimited JSON with each Feature on its own line, because it knows nothing of the top-level structure around the Features. Spurious "EOF" error messages may result otherwise. @@ -77,16 +77,16 @@ Options ### Zoom levels and resolution - * -z _zoom_: Maxzoom: the highest zoom level for which tiles are generated (default 14) - * -Z _zoom_: Minzoom: the lowest zoom level for which tiles are generated (default 0) - * -B _zoom_: Base zoom, the level at and above which all points are included in the tiles (default maxzoom). + * -z _zoom_ or --maximum-zoom=_zoom_: Maxzoom: the highest zoom level for which tiles are generated (default 14) + * -Z _zoom_ or --minimum-zoom=_zoom_: Minzoom: the lowest zoom level for which tiles are generated (default 0) + * -B _zoom_ or --base-zoom=_zoom_: Base zoom, the level at and above which all points are included in the tiles (default maxzoom). If you use -Bg, it will guess a zoom level that will keep at most 50,000 features in the densest tile. You can also specify a marker-width with -Bg*width* to allow fewer features in the densest tile to compensate for the larger marker, or -Bf*number* to allow at most *number* features in the densest tile. - * -d _detail_: Detail at max zoom level (default 12, for tile resolution of 4096) - * -D _detail_: Detail at lower zoom levels (default 12, for tile resolution of 4096) - * -m _detail_: Minimum detail that it will try if tiles are too big at regular detail (default 7) - * -b _pixels_: Buffer size where features are duplicated from adjacent tiles. Units are "screen pixels"--1/256th of the tile width or height. (default 5) + * -d _detail_ or --full-detail=_detail_: Detail at max zoom level (default 12, for tile resolution of 4096) + * -D _detail_ or --low-detail=_detail_: Detail at lower zoom levels (default 12, for tile resolution of 4096) + * -m _detail_ or --minimum-detail=_detail_: Minimum detail that it will try if tiles are too big at regular detail (default 7) + * -b _pixels_ or --buffer=_pixels_: Buffer size where features are duplicated from adjacent tiles. Units are "screen pixels"--1/256th of the tile width or height. (default 5) All internal math is done in terms of a 32-bit tile coordinate system, so 1/(2^32) of the size of Earth, or about 1cm, is the smallest distinguishable distance. If _maxzoom_ + _detail_ > 32, no additional @@ -94,35 +94,35 @@ resolution is obtained than by using a smaller _maxzoom_ or _detail_. ### Properties - * -x _name_: Exclude the named properties from all features - * -y _name_: Include the named properties in all features, excluding all those not explicitly named - * -X: Exclude all properties and encode only geometries + * -x _name_ or --exclude=_name_: Exclude the named properties from all features + * -y _name_ or --include=_name_: Include the named properties in all features, excluding all those not explicitly named + * -X or --exclude-all: Exclude all properties and encode only geometries ### Point simplification - * -r _rate_: Rate at which dots are dropped at zoom levels below basezoom (default 2.5). + * -r _rate_ or --drop_rate=_rate_: Rate at which dots are dropped at zoom levels below basezoom (default 2.5). If you use -rg, it will guess a drop rate that will keep at most 50,000 features in the densest tile. You can also specify a marker-width with -rg*width* to allow fewer features in the densest tile to compensate for the larger marker, or -rf*number* to allow at most *number* features in the densest tile. - * -g _gamma_: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number. + * -g _gamma_ or --gamma=_gamma_: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number. ### Doing more - * -ac: Coalesce adjacent line and polygon features that have the same properties - * -ar: Try reversing the directions of lines to make them coalesce and compress better - * -ao: Reorder features to put ones with the same properties in sequence, to try to get them to coalesce - * -al: Let "dot" dropping at lower zooms apply to lines too + * -ac or --coalesce: Coalesce adjacent line and polygon features that have the same properties + * -ar or --reverse: Try reversing the directions of lines to make them coalesce and compress better + * -ao or --reorder: Reorder features to put ones with the same properties in sequence, to try to get them to coalesce + * -al or --drop-lines: Let "dot" dropping at lower zooms apply to lines too ### Doing less - * -ps: Don't simplify lines - * -pS: Don't simplify lines at maxzoom (but do simplify at lower zooms) - * -pf: Don't limit tiles to 200,000 features - * -pk: Don't limit tiles to 500K bytes - * -pd: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. - * -pi: Preserve the original input order of features as the drawing order instead of ordering geographically. (This is implemented as a restoration of the original order at the end, so that dot-dropping is still geographic, which means it also undoes -ao). - * -pp: Don't split complex polygons (over 700 vertices after simplification) into multiple features. - * -q: Work quietly instead of reporting progress + * -ps or --no-line-simplification: Don't simplify lines + * -pS or --simplify-only-low-zooms: Don't simplify lines at maxzoom (but do simplify at lower zooms) + * -pf or --no-feature-limit: Don't limit tiles to 200,000 features + * -pk or --no-tile-size-limit: Don't limit tiles to 500K bytes + * -pd or --force-feature-limit: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. + * -pi or --preserve-input-order: Preserve the original input order of features as the drawing order instead of ordering geographically. (This is implemented as a restoration of the original order at the end, so that dot-dropping is still geographic, which means it also undoes -ao). + * -pp or --no-polygon-splitting: Don't split complex polygons (over 700 vertices after simplification) into multiple features. + * -q or --quiet: Work quietly instead of reporting progress Example ------- diff --git a/geojson.c b/geojson.c index 1e8284d..ab6a0c5 100644 --- a/geojson.c +++ b/geojson.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "jsonpull.h" #include "tile.h" @@ -35,6 +36,9 @@ static int min_detail = 7; int quiet = 0; int geometry_scale = 0; +static int prevent[256]; +static int additional[256]; + #define GEOM_POINT 0 /* array of positions */ #define GEOM_MULTIPOINT 1 /* array of arrays of positions */ #define GEOM_LINESTRING 2 /* array of arrays of positions */ @@ -1080,7 +1084,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i } } -int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional, int read_parallel, int forcetable) { +int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int *prevent, int *additional, int read_parallel, int forcetable) { int ret = EXIT_SUCCESS; struct reader reader[CPUS]; @@ -1491,6 +1495,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max perror(geomname); exit(EXIT_FAILURE); } + unlink(geomname); int geomfd2; /* Sort the index by geometry */ @@ -1625,6 +1630,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max perror(geomname); exit(EXIT_FAILURE); } + unlink(geomname); for (i = 0; i < CPUS; i++) { if (reader[i].geomst.st_size > 0) { @@ -2106,8 +2112,6 @@ int main(int argc, char **argv) { double gamma = 0; int buffer = 5; const char *tmpdir = "/tmp"; - char prevent[256]; - char additional[256]; struct pool exclude, include; pool_init(&exclude, 0); @@ -2120,8 +2124,53 @@ int main(int argc, char **argv) { additional[i] = 0; } - while ((i = getopt(argc, argv, "l:n:z:Z:d:D:m:o:x:y:r:b:fFXt:g:p:vqa:B:P")) != -1) { + static struct option long_options[] = { + {"name", required_argument, 0, 'n'}, + {"layer", required_argument, 0, 'l'}, + {"maximum-zoom", required_argument, 0, 'z'}, + {"minimum-zoom", required_argument, 0, 'Z'}, + {"base-zoom", required_argument, 0, 'B'}, + {"full-detail", required_argument, 0, 'd'}, + {"low-detail", required_argument, 0, 'D'}, + {"minimum-detail", required_argument, 0, 'm'}, + {"output", required_argument, 0, 'o'}, + {"exclude", required_argument, 0, 'x'}, + {"include", required_argument, 0, 'y'}, + {"drop-rate", required_argument, 0, 'r'}, + {"buffer", required_argument, 0, 'b'}, + {"temporary-directory", required_argument, 0, 't'}, + {"gamma", required_argument, 0, 'g'}, + {"prevent", required_argument, 0, 'p'}, + {"additional", required_argument, 0, 'a'}, + + {"exclude-all", no_argument, 0, 'X'}, + {"force", no_argument, 0, 'f'}, + {"allow-existing", no_argument, 0, 'F'}, + {"quiet", no_argument, 0, 'q'}, + {"version", no_argument, 0, 'v'}, + {"read-parallel", no_argument, 0, 'P'}, + + {"coalesce", no_argument, &additional[A_COALESCE], 1}, + {"reverse", no_argument, &additional[A_REVERSE], 1}, + {"reorder", no_argument, &additional[A_REORDER], 1}, + {"drop-lines", no_argument, &additional[A_LINE_DROP], 1}, + + {"no-line-simplification", no_argument, &prevent[P_SIMPLIFY], 1}, + {"simplify-only-low-zooms", no_argument, &prevent[P_SIMPLIFY_LOW], 1}, + {"no-feature-limit", no_argument, &prevent[P_FEATURE_LIMIT], 1}, + {"no-tile-size-limit", no_argument, &prevent[P_KILOBYTE_LIMIT], 1}, + {"force-feature-limit", no_argument, &prevent[P_DYNAMIC_DROP], 1}, + {"preseve-input-order", no_argument, &prevent[P_INPUT_ORDER], 1}, + {"no-polygon-splitting", no_argument, &prevent[P_POLYGON_SPLIT], 1}, + + {0, 0, 0, 0}, + }; + + while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvP", long_options, NULL)) != -1) { switch (i) { + case 0: + break; + case 'n': name = optarg; break; diff --git a/tile.cc b/tile.cc index d71d6b5..7b61d11 100644 --- a/tile.cc +++ b/tile.cc @@ -480,8 +480,8 @@ struct partial { unsigned long long index2; int z; int line_detail; - char *prevent; - char *additional; + int *prevent; + int *additional; int maxzoom; }; @@ -501,8 +501,8 @@ void *partial_feature_worker(void *v) { signed char t = (*partials)[i].t; int z = (*partials)[i].z; int line_detail = (*partials)[i].line_detail; - char *prevent = (*partials)[i].prevent; - char *additional = (*partials)[i].additional; + int *prevent = (*partials)[i].prevent; + int *additional = (*partials)[i].additional; int maxzoom = (*partials)[i].maxzoom; if ((t == VT_LINE || t == VT_POLYGON) && !(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]))) { @@ -597,7 +597,7 @@ int manage_gap(unsigned long long index, unsigned long long *previndex, double s return 0; } -long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, char *prevent, char *additional, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) { +long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, int *prevent, int *additional, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) { int line_detail; double fraction = 1; @@ -1049,8 +1049,8 @@ struct write_tile_args { volatile long long *along; double gamma; int nlayers; - char *prevent; - char *additional; + int *prevent; + int *additional; int child_shards; int *geomfd; off_t *geom_size; @@ -1157,7 +1157,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y) { int i; for (i = 0; i <= maxzoom; i++) { long long most = 0; diff --git a/tile.h b/tile.h index 0930fc4..8d58deb 100644 --- a/tile.h +++ b/tile.h @@ -25,9 +25,9 @@ void deserialize_uint(char **f, unsigned *n); void deserialize_byte(char **f, signed char *n); struct pool_val *deserialize_string(char **f, struct pool *p, int type); -long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional); +long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, int *prevent, int *additional); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/version.h b/version.h index 4c2bdf0..ddb3937 100644 --- a/version.h +++ b/version.h @@ -1 +1 @@ -#define VERSION "tippecanoe v1.9.4\n" +#define VERSION "tippecanoe v1.9.5\n"