diff --git a/.gitignore b/.gitignore index 4581ef2..4bf996a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ *.exe *.out *.app + +# Mac +.DS_Store \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1d3f9..3d8bcc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.16.15 + +* Add --output-to-directory and --no-tile-compression options + ## 1.16.14 * Add --description option for mbtiles metadata diff --git a/Makefile b/Makefile index 67663cf..e9dc5f1 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o plugin.o read_json.o write_json.o +tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o rawtiles.o plugin.o read_json.o write_json.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o @@ -78,7 +78,7 @@ indent: TESTS = $(wildcard tests/*/out/*.json) SPACE = $(NULL) $(NULL) -test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) parallel-test pbf-test join-test enumerate-test decode-test unit +test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test unit ./unit # Work around Makefile and filename punctuation limits: _ for space, @ for :, % for / @@ -112,6 +112,11 @@ parallel-test: cmp tests/parallel/linear-file.json tests/parallel/parallel-pipes.json rm tests/parallel/*.mbtiles tests/parallel/*.json +raw-tiles-test: + ./tippecanoe -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC + diff -x '.*' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare + rm -rf tests/raw-tiles/raw-tiles + decode-test: mkdir -p tests/muni/decode ./tippecanoe -z11 -Z11 -f -o tests/muni/decode/multi.mbtiles tests/muni/*.json diff --git a/README.md b/README.md index 1332fc2..3b9aff4 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Options ### File control * -o _file_.mbtiles or --output=_file_.mbtiles: Name the output file. + * -e _directory_ or --output-directory=_directory_: Write tiles to the specified *directory* instead of to an mbtiles file. * -f or --force: Delete the mbtiles file if it already exists instead of giving an error * -F or --allow-existing: Proceed (without deleting existing data) if the metadata or tiles table already exists or if metadata fields can't be set @@ -147,6 +148,7 @@ resolution is obtained than by using a smaller _maxzoom_ or _detail_. * -pc or --no-clipping: Don't clip features to the size of the tile. If a feature overlaps the tile's bounds or buffer at all, it is included completely. Be careful: this can produce very large tilesets, especially with large polygons. * -pD or --no-duplication: As with --no-clipping, each feature is included intact instead of cut to tile boundaries. In addition, it is included only in a single tile per zoom level rather than potentially in multiple copies. Clients of the tileset must check adjacent tiles (possibly some distance away) to ensure they have all features. * -pt or --no-tiny-polygon-reduction: Don't combine the area of very small polygons into small squares that represent their combined area. + * -pC or --no-tile-compression: Don't compress the PBF vector tile data. * -q or --quiet: Work quietly instead of reporting progress ### Filters diff --git a/main.cpp b/main.cpp index 98ffd90..182cd29 100644 --- a/main.cpp +++ b/main.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -1010,7 +1011,7 @@ void choose_first_zoom(long long *file_bbox, struct reader *reader, unsigned *iz } } -int read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set *exclude, std::set *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description) { +int read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description) { int ret = EXIT_SUCCESS; struct reader reader[CPUS]; @@ -1789,7 +1790,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } unsigned midx = 0, midy = 0; - int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter); + int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, outdir, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); @@ -1853,7 +1854,8 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo ai->second.minzoom = minzoom; ai->second.maxzoom = maxzoom; } - mbtiles_write_metadata(outdb, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); + + mbtiles_write_metadata(outdb, outdir, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); return ret; } @@ -1882,7 +1884,9 @@ int main(int argc, char **argv) { char *name = NULL; char *description = NULL; char *layername = NULL; - char *outdir = NULL; + char *out_mbtiles = NULL; + char *out_directory = NULL; + sqlite3 *outdb = NULL; int maxzoom = 14; int minzoom = 0; int basezoom = -1; @@ -1910,6 +1914,7 @@ int main(int argc, char **argv) { static struct option long_options[] = { {"output", required_argument, 0, 'o'}, + {"output-to-directory", required_argument, 0, 'e'}, {"name", required_argument, 0, 'n'}, {"description", required_argument, 0, 'N'}, @@ -1969,6 +1974,7 @@ int main(int argc, char **argv) { {"no-clipping", no_argument, &prevent[P_CLIPPING], 1}, {"no-duplication", no_argument, &prevent[P_DUPLICATION], 1}, {"no-tiny-polygon-reduction", no_argument, &prevent[P_TINY_POLYGON_REDUCTION], 1}, + {"no-tile-compression", no_argument, &prevent[P_TILE_COMPRESSION], 1}, {0, 0, 0, 0}, }; @@ -1991,7 +1997,7 @@ int main(int argc, char **argv) { } } - while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:c:C:", long_options, NULL)) != -1) { + while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:e:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:c:C:", long_options, NULL)) != -1) { switch (i) { case 0: break; @@ -2068,7 +2074,11 @@ int main(int argc, char **argv) { break; case 'o': - outdir = optarg; + out_mbtiles = optarg; + break; + + case 'e': + out_directory = optarg; break; case 'x': @@ -2269,16 +2279,24 @@ int main(int argc, char **argv) { fprintf(stderr, "Forcing -g0 since -B or -r is not known\n"); } - if (outdir == NULL) { - fprintf(stderr, "%s: must specify -o out.mbtiles\n", argv[0]); + if (out_mbtiles == NULL && out_directory == NULL) { + fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory\n", argv[0]); exit(EXIT_FAILURE); } - if (force) { - unlink(outdir); + if (out_mbtiles != NULL && out_directory != NULL) { + fprintf(stderr, "%s: Options -o and -e cannot be used together\n", argv[0]); + exit(EXIT_FAILURE); + } + + if (out_mbtiles != NULL) { + if (force) { + unlink(out_mbtiles); + } + + outdb = mbtiles_open(out_mbtiles, argv, forcetable); } - sqlite3 *outdb = mbtiles_open(outdir, argv, forcetable); int ret = EXIT_SUCCESS; for (i = optind; i < argc; i++) { @@ -2303,9 +2321,11 @@ int main(int argc, char **argv) { long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0}; - ret = read_input(sources, name ? name : outdir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, prefilter, postfilter, description); + ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_directory, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_directory, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, prefilter, postfilter, description); - mbtiles_close(outdb, argv); + if (outdb != NULL) { + mbtiles_close(outdb, argv); + } #ifdef MTRACE muntrace(); diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index e6efd76..40615d1 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -60,12 +60,16 @@ specified, the files are all merged into the single named layer, even if they tr \-n \fIname\fP or \-\-name=\fIname\fP: Human\-readable name for the tileset (default file.json) .IP \(bu 2 \-A \fItext\fP or \-\-attribution=\fItext\fP: Attribution (HTML) to be shown with maps that use data from this tileset. +.IP \(bu 2 +\-N \fIdescription\fP or \-\-description=\fIdescription\fP: Description for the tileset (default file.mbtiles) .RE .SS File control .RS .IP \(bu 2 \-o \fIfile\fP\&.mbtiles or \-\-output=\fIfile\fP\&.mbtiles: Name the output file. .IP \(bu 2 +\-e \fIdirectory\fP or \-\-output\-directory=\fIdirectory\fP: Write tiles to the specified \fIdirectory\fP instead of to an mbtiles file. +.IP \(bu 2 \-f or \-\-force: Delete the mbtiles file if it already exists instead of giving an error .IP \(bu 2 \-F or \-\-allow\-existing: Proceed (without deleting existing data) if the metadata or tiles table already exists @@ -188,6 +192,8 @@ which may not be what you want. .IP \(bu 2 \-pt or \-\-no\-tiny\-polygon\-reduction: Don't combine the area of very small polygons into small squares that represent their combined area. .IP \(bu 2 +\-pC or \-\-no\-tile\-compression: Don't compress the PBF vector tile data. +.IP \(bu 2 \-q or \-\-quiet: Work quietly instead of reporting progress .RE .SS Filters diff --git a/mbtiles.cpp b/mbtiles.cpp index 7f6b5c2..d5291cf 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -126,11 +126,23 @@ bool type_and_string::operator<(const type_and_string &o) const { return false; } -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description) { +void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description) { char *sql, *err; + sqlite3 *db = outdb; + if (outdb == NULL) { + if (sqlite3_open("", &db) != SQLITE_OK) { + fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "Create metadata table: %s\n", err); + exit(EXIT_FAILURE); + } + } + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set name in metadata: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -139,7 +151,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", description != NULL ? description : fname); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set description in metadata: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -148,7 +160,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", 2); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set version : %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -157,7 +169,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", minzoom); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set minzoom: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -166,7 +178,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", maxzoom); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set maxzoom: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -175,7 +187,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", midlon, midlat, maxzoom); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set center: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -184,7 +196,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", minlon, minlat, maxlon, maxlat); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set bounds: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -193,7 +205,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", "overlay"); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set type: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -203,7 +215,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int if (attribution != NULL) { sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", attribution); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set type: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -213,7 +225,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int } sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", vector ? "pbf" : "png"); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set format: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -271,7 +283,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int aprintf(&buf, " ] }"); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str()); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set json: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -279,6 +291,45 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int } sqlite3_free(sql); } + + if (outdir != NULL) { + std::string metadata = std::string(outdir) + "/metadata.json"; + FILE *fp = fopen(metadata.c_str(), "w"); + if (fp == NULL) { + perror(metadata.c_str()); + exit(EXIT_FAILURE); + } + + fprintf(fp, "{\n"); + + sqlite3_stmt *stmt; + bool first = true; + if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) { + while (sqlite3_step(stmt) == SQLITE_ROW) { + std::string key, value; + + quote(key, (const char *) sqlite3_column_text(stmt, 0)); + quote(value, (const char *) sqlite3_column_text(stmt, 1)); + + if (!first) { + fprintf(fp, ",\n"); + } + fprintf(fp, " \"%s\": \"%s\"", key.c_str(), value.c_str()); + first = false; + } + sqlite3_finalize(stmt); + } + + fprintf(fp, "\n}\n"); + fclose(fp); + } + + if (outdb == NULL) { + if (sqlite3_close(db) != SQLITE_OK) { + fprintf(stderr, "Could not close temp database: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + } } void mbtiles_close(sqlite3 *outdb, char **argv) { diff --git a/mbtiles.hpp b/mbtiles.hpp index 4fb18b1..dbe93cd 100644 --- a/mbtiles.hpp +++ b/mbtiles.hpp @@ -20,7 +20,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description); +void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description); void mbtiles_close(sqlite3 *outdb, char **argv); diff --git a/mvt.cpp b/mvt.cpp index fa3f924..bee7473 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -370,10 +370,7 @@ std::string mvt_tile::encode() { writer.add_message(3, layer_string); } - std::string compressed; - compress(data, compressed); - - return compressed; + return data; } bool mvt_value::operator<(const mvt_value &o) const { diff --git a/options.hpp b/options.hpp index 4079369..6081d7d 100644 --- a/options.hpp +++ b/options.hpp @@ -25,6 +25,7 @@ #define P_CLIPPING ((int) 'c') #define P_DUPLICATION ((int) 'D') #define P_TINY_POLYGON_REDUCTION ((int) 't') +#define P_TILE_COMPRESSION ((int) 'C') extern int prevent[256]; extern int additional[256]; diff --git a/rawtiles.cpp b/rawtiles.cpp new file mode 100644 index 0000000..1c7ecc4 --- /dev/null +++ b/rawtiles.cpp @@ -0,0 +1,20 @@ +#include +#include +#include +#include +#include "rawtiles.hpp" + +void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) { + mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO); + std::string curdir(outdir); + std::string slash("/"); + std::string newdir = curdir + slash + std::to_string(z); + mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(tx); + mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(ty) + ".pbf"; + + std::ofstream pbfFile(newdir, std::ios::out | std::ios::binary); + pbfFile.write(pbf.data(), pbf.size()); + pbfFile.close(); +} diff --git a/rawtiles.hpp b/rawtiles.hpp new file mode 100644 index 0000000..1a332c9 --- /dev/null +++ b/rawtiles.hpp @@ -0,0 +1 @@ +void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf); diff --git a/tests/raw-tiles/compare/0/0/0.pbf b/tests/raw-tiles/compare/0/0/0.pbf new file mode 100644 index 0000000..3f518d3 Binary files /dev/null and b/tests/raw-tiles/compare/0/0/0.pbf differ diff --git a/tests/raw-tiles/compare/1/0/0.pbf b/tests/raw-tiles/compare/1/0/0.pbf new file mode 100644 index 0000000..49b703d Binary files /dev/null and b/tests/raw-tiles/compare/1/0/0.pbf differ diff --git a/tests/raw-tiles/compare/10/163/366.pbf b/tests/raw-tiles/compare/10/163/366.pbf new file mode 100644 index 0000000..9fd8fe0 Binary files /dev/null and b/tests/raw-tiles/compare/10/163/366.pbf differ diff --git a/tests/raw-tiles/compare/11/326/732.pbf b/tests/raw-tiles/compare/11/326/732.pbf new file mode 100644 index 0000000..ae7fcdb Binary files /dev/null and b/tests/raw-tiles/compare/11/326/732.pbf differ diff --git a/tests/raw-tiles/compare/12/652/1464.pbf b/tests/raw-tiles/compare/12/652/1464.pbf new file mode 100644 index 0000000..173f820 Binary files /dev/null and b/tests/raw-tiles/compare/12/652/1464.pbf differ diff --git a/tests/raw-tiles/compare/12/652/1465.pbf b/tests/raw-tiles/compare/12/652/1465.pbf new file mode 100644 index 0000000..385f335 Binary files /dev/null and b/tests/raw-tiles/compare/12/652/1465.pbf differ diff --git a/tests/raw-tiles/compare/13/1304/2928.pbf b/tests/raw-tiles/compare/13/1304/2928.pbf new file mode 100644 index 0000000..22531b0 Binary files /dev/null and b/tests/raw-tiles/compare/13/1304/2928.pbf differ diff --git a/tests/raw-tiles/compare/13/1304/2929.pbf b/tests/raw-tiles/compare/13/1304/2929.pbf new file mode 100644 index 0000000..6aad49f Binary files /dev/null and b/tests/raw-tiles/compare/13/1304/2929.pbf differ diff --git a/tests/raw-tiles/compare/13/1304/2930.pbf b/tests/raw-tiles/compare/13/1304/2930.pbf new file mode 100644 index 0000000..cecb84f Binary files /dev/null and b/tests/raw-tiles/compare/13/1304/2930.pbf differ diff --git a/tests/raw-tiles/compare/14/2608/5856.pbf b/tests/raw-tiles/compare/14/2608/5856.pbf new file mode 100644 index 0000000..b279fba Binary files /dev/null and b/tests/raw-tiles/compare/14/2608/5856.pbf differ diff --git a/tests/raw-tiles/compare/14/2608/5857.pbf b/tests/raw-tiles/compare/14/2608/5857.pbf new file mode 100644 index 0000000..2272a83 Binary files /dev/null and b/tests/raw-tiles/compare/14/2608/5857.pbf differ diff --git a/tests/raw-tiles/compare/14/2609/5860.pbf b/tests/raw-tiles/compare/14/2609/5860.pbf new file mode 100644 index 0000000..d8f0684 Binary files /dev/null and b/tests/raw-tiles/compare/14/2609/5860.pbf differ diff --git a/tests/raw-tiles/compare/2/0/1.pbf b/tests/raw-tiles/compare/2/0/1.pbf new file mode 100644 index 0000000..ecd69b4 Binary files /dev/null and b/tests/raw-tiles/compare/2/0/1.pbf differ diff --git a/tests/raw-tiles/compare/3/1/2.pbf b/tests/raw-tiles/compare/3/1/2.pbf new file mode 100644 index 0000000..acf10f5 Binary files /dev/null and b/tests/raw-tiles/compare/3/1/2.pbf differ diff --git a/tests/raw-tiles/compare/4/2/5.pbf b/tests/raw-tiles/compare/4/2/5.pbf new file mode 100644 index 0000000..2e4698b Binary files /dev/null and b/tests/raw-tiles/compare/4/2/5.pbf differ diff --git a/tests/raw-tiles/compare/5/5/11.pbf b/tests/raw-tiles/compare/5/5/11.pbf new file mode 100644 index 0000000..40ab997 Binary files /dev/null and b/tests/raw-tiles/compare/5/5/11.pbf differ diff --git a/tests/raw-tiles/compare/6/10/22.pbf b/tests/raw-tiles/compare/6/10/22.pbf new file mode 100644 index 0000000..75c556b Binary files /dev/null and b/tests/raw-tiles/compare/6/10/22.pbf differ diff --git a/tests/raw-tiles/compare/7/20/45.pbf b/tests/raw-tiles/compare/7/20/45.pbf new file mode 100644 index 0000000..b54f433 Binary files /dev/null and b/tests/raw-tiles/compare/7/20/45.pbf differ diff --git a/tests/raw-tiles/compare/8/40/91.pbf b/tests/raw-tiles/compare/8/40/91.pbf new file mode 100644 index 0000000..26c2f2a Binary files /dev/null and b/tests/raw-tiles/compare/8/40/91.pbf differ diff --git a/tests/raw-tiles/compare/9/81/183.pbf b/tests/raw-tiles/compare/9/81/183.pbf new file mode 100644 index 0000000..dd0721f Binary files /dev/null and b/tests/raw-tiles/compare/9/81/183.pbf differ diff --git a/tests/raw-tiles/compare/metadata.json b/tests/raw-tiles/compare/metadata.json new file mode 100644 index 0000000..c83c17c --- /dev/null +++ b/tests/raw-tiles/compare/metadata.json @@ -0,0 +1,12 @@ +{ + "name": "tests/raw-tiles/raw-tiles", + "description": "tests/raw-tiles/raw-tiles", + "version": "2", + "minzoom": "0", + "maxzoom": "14", + "center": "-122.662354,45.514045,14", + "bounds": "-122.682427,45.512332,-122.654961,45.569975", + "type": "overlay", + "format": "pbf", + "json": "{\"vector_layers\": [ { \"id\": \"hackspotsgeojson\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 14, \"fields\": {\"Address\": \"String\", \"Name\": \"String\", \"Notes\": \"String\"} } ] }" +} diff --git a/tests/raw-tiles/hackspots.geojson b/tests/raw-tiles/hackspots.geojson new file mode 100644 index 0000000..1f6e981 --- /dev/null +++ b/tests/raw-tiles/hackspots.geojson @@ -0,0 +1,65 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "Name": "Albina Press", + "Address": "4637 N Albina Ave Portland, OR 97217", + "Notes": "usually busy, outlets on side wall only" + }, + "geometry": { + "type": "Point", + "coordinates": [ + -122.67516911029816, + 45.55673233031101 + ] + } + }, + { + "type": "Feature", + "properties": { + "Name": "Arbor Lodge", + "Address": "1507 N Rosa Parks Way Portland, OR 97217", + "Notes": "" + }, + "geometry": { + "type": "Point", + "coordinates": [ + -122.68242716789246, + 45.56997505986905 + ] + } + }, + { + "type": "Feature", + "properties": { + "Name": "Three Friends Coffeehouse", + "Address": "201 SE 12th Ave, Portland, OR 97214", + "Notes": "" + }, + "geometry": { + "type": "Point", + "coordinates": [ + -122.65496134757996, + 45.5212590460849 + ] + } + }, + { + "type": "Feature", + "properties": { + "Name": "Lucky Labrador Brew Pub", + "Address": "915 SE Hawthorne Blvd. Portland, OR 97214", + "Notes": "Dog friendly" + }, + "geometry": { + "type": "Point", + "coordinates": [ + -122.65625953674315, + 45.512331780814186 + ] + } + } + ] +} \ No newline at end of file diff --git a/tile-join.cpp b/tile-join.cpp index f22c243..46f2c07 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -753,7 +753,7 @@ int main(int argc, char **argv) { decode(readers, csv, layermap, outdb, &st, header, mapping, exclude, ifmatched, attribution, description); - mbtiles_write_metadata(outdb, outfile, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str()); + mbtiles_write_metadata(outdb, NULL, outfile, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str()); mbtiles_close(outdb, argv); return 0; diff --git a/tile.cpp b/tile.cpp index 2752634..bb86f9d 100644 --- a/tile.cpp +++ b/tile.cpp @@ -28,6 +28,7 @@ #include #include "mvt.hpp" #include "mbtiles.hpp" +#include "rawtiles.hpp" #include "geometry.hpp" #include "tile.hpp" #include "pool.hpp" @@ -1137,6 +1138,7 @@ struct write_tile_args { int min_detail; int basezoom; sqlite3 *outdb; + const char *outdir; double droprate; int buffer; const char *fname; @@ -1390,7 +1392,7 @@ void *run_prefilter(void *v) { return NULL; } -long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) { +long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) { int line_detail; double merge_fraction = 1; double mingap_fraction = 1; @@ -1933,7 +1935,14 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s } } - std::string compressed = tile.encode(); + std::string compressed; + std::string pbf = tile.encode(); + + if (!prevent[P_TILE_COMPRESSION]) { + compress(pbf, compressed); + } else { + compressed = pbf; + } if (compressed.size() > max_tile_size && !prevent[P_KILOBYTE_LIMIT]) { if (!quiet) { @@ -2005,7 +2014,11 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s exit(EXIT_FAILURE); } - mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); + if (outdb != NULL) { + mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); + } else if (outdir != NULL) { + write_raw_tile(outdir, z, tx, ty, compressed); + } if (pthread_mutex_unlock(&db_lock) != 0) { perror("pthread_mutex_unlock"); @@ -2067,7 +2080,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg); + long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->outdir, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg); if (len < 0) { int *err = &arg->err; @@ -2132,7 +2145,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter) { // The existing layermaps are one table per input thread. // We need to add another one per *tiling* thread so that it can be // safely changed during tiling. @@ -2275,6 +2288,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].min_detail = min_detail; args[thread].basezoom = basezoom; args[thread].outdb = outdb; // locked with db_lock + args[thread].outdir = outdir; args[thread].droprate = droprate; args[thread].buffer = buffer; args[thread].fname = fname; diff --git a/tile.hpp b/tile.hpp index 6d3f0ca..b250300 100644 --- a/tile.hpp +++ b/tile.hpp @@ -1,3 +1,3 @@ -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap, const char *prefilter, const char *postfilter); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap, const char *prefilter, const char *postfilter); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/version.hpp b/version.hpp index 3b4aad7..b44031b 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.16.13\n" +#define VERSION "tippecanoe v1.16.14\n"