From aee12ac1fed0ad4f913c037e9ab400a6965869aa Mon Sep 17 00:00:00 2001 From: Shan-Chun Kuo Date: Wed, 5 Apr 2017 00:34:54 +0100 Subject: [PATCH 1/9] Add flag --raw-tiles or -pC to get raw protobuf --- main.cpp | 18 +++++++++++++++--- main.hpp | 2 ++ mvt.cpp | 5 +---- options.hpp | 1 + tile.cpp | 26 ++++++++++++++++++++++++-- 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/main.cpp b/main.cpp index 418ea7f..b2584d6 100644 --- a/main.cpp +++ b/main.cpp @@ -69,6 +69,8 @@ struct source { std::string file; }; +char *outpbfdir; + size_t CPUS; size_t TEMP_FILES; long long MAX_FILES; @@ -1843,7 +1845,9 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo ai->second.minzoom = minzoom; ai->second.maxzoom = maxzoom; } - mbtiles_write_metadata(outdb, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); + + if(!prevent[P_PBF_COMPRESSION]) + mbtiles_write_metadata(outdb, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); return ret; } @@ -1873,6 +1877,7 @@ int main(int argc, char **argv) { char *description = NULL; char *layername = NULL; char *outdir = NULL; + sqlite3 *outdb = NULL; int maxzoom = 14; int minzoom = 0; int basezoom = -1; @@ -1955,6 +1960,7 @@ int main(int argc, char **argv) { {"no-clipping", no_argument, &prevent[P_CLIPPING], 1}, {"no-duplication", no_argument, &prevent[P_DUPLICATION], 1}, {"no-tiny-polygon-reduction", no_argument, &prevent[P_TINY_POLYGON_REDUCTION], 1}, + {"raw-tiles", no_argument, &prevent[P_PBF_COMPRESSION], 1}, {0, 0, 0, 0}, }; @@ -2247,7 +2253,12 @@ int main(int argc, char **argv) { unlink(outdir); } - sqlite3 *outdb = mbtiles_open(outdir, argv, forcetable); + if(!prevent[P_PBF_COMPRESSION]){ + outdb = mbtiles_open(outdir, argv, forcetable); + }else{ + outpbfdir = outdir; + } + int ret = EXIT_SUCCESS; for (i = optind; i < argc; i++) { @@ -2274,7 +2285,8 @@ int main(int argc, char **argv) { ret = read_input(sources, name ? name : outdir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description); - mbtiles_close(outdb, argv); + if(!prevent[P_PBF_COMPRESSION]) + mbtiles_close(outdb, argv); #ifdef MTRACE muntrace(); diff --git a/main.hpp b/main.hpp index e06908b..d61c6de 100644 --- a/main.hpp +++ b/main.hpp @@ -12,6 +12,8 @@ void checkdisk(struct reader *r, int nreader); extern int geometry_scale; extern int quiet; +extern char *outpbfdir; + extern size_t CPUS; extern size_t TEMP_FILES; diff --git a/mvt.cpp b/mvt.cpp index f11164b..22198e9 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -367,10 +367,7 @@ std::string mvt_tile::encode() { writer.add_message(3, layer_string); } - std::string compressed; - compress(data, compressed); - - return compressed; + return data; } bool mvt_value::operator<(const mvt_value &o) const { diff --git a/options.hpp b/options.hpp index 4079369..97bda01 100644 --- a/options.hpp +++ b/options.hpp @@ -25,6 +25,7 @@ #define P_CLIPPING ((int) 'c') #define P_DUPLICATION ((int) 'D') #define P_TINY_POLYGON_REDUCTION ((int) 't') +#define P_PBF_COMPRESSION ((int) 'C') extern int prevent[256]; extern int additional[256]; diff --git a/tile.cpp b/tile.cpp index f972327..317295d 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1837,7 +1837,14 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s } } - std::string compressed = tile.encode(); + std::string compressed; + std::string pbf = tile.encode(); + + if(!prevent[P_PBF_COMPRESSION]){ + compress(pbf, compressed); + }else{ + compressed = pbf; + } if (compressed.size() > max_tile_size && !prevent[P_KILOBYTE_LIMIT]) { if (!quiet) { @@ -1909,7 +1916,22 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s exit(EXIT_FAILURE); } - mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); + if(!prevent[P_PBF_COMPRESSION]){ + mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); + }else{ + mkdir(outpbfdir, S_IRWXU | S_IRWXG | S_IRWXO); + std::string curdir(outpbfdir); + std::string slash( "/" ); + std::string newdir = curdir + slash + std::to_string(z); + mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(tx); + mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(ty) + ".pbf"; + + std::ofstream pbfFile (newdir, std::ios::out | std::ios::app | std::ios::binary); + pbfFile.write (pbf.data(), pbf.size()); + pbfFile.close(); + } if (pthread_mutex_unlock(&db_lock) != 0) { perror("pthread_mutex_unlock"); From f03b89656ca510c8ef79ff8ee83e014a04de4457 Mon Sep 17 00:00:00 2001 From: Shan-Chun Kuo Date: Wed, 5 Apr 2017 15:59:37 +0100 Subject: [PATCH 2/9] Seperated out to method and added test for --raw-tiles --- .gitignore | 3 ++ Makefile | 9 +++- main.cpp | 2 +- rawtiles.cpp | 20 +++++++ rawtiles.hpp | 1 + tests/raw-tiles/compare/0/0/0.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/1/0/0.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/10/163/366.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/11/326/732.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/12/652/1464.pbf | Bin 0 -> 218 bytes tests/raw-tiles/compare/12/652/1465.pbf | Bin 0 -> 138 bytes tests/raw-tiles/compare/13/1304/2928.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/13/1304/2929.pbf | Bin 0 -> 139 bytes tests/raw-tiles/compare/13/1304/2930.pbf | Bin 0 -> 139 bytes tests/raw-tiles/compare/14/2608/5856.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/14/2608/5857.pbf | Bin 0 -> 167 bytes tests/raw-tiles/compare/14/2609/5860.pbf | Bin 0 -> 246 bytes tests/raw-tiles/compare/2/0/1.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/3/1/2.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/4/2/5.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/5/5/11.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/6/10/22.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/7/20/45.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/8/40/91.pbf | Bin 0 -> 131 bytes tests/raw-tiles/compare/9/81/183.pbf | Bin 0 -> 131 bytes tests/raw-tiles/hackspots.geojson | 65 +++++++++++++++++++++++ tile.cpp | 14 +---- 27 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 rawtiles.cpp create mode 100644 rawtiles.hpp create mode 100644 tests/raw-tiles/compare/0/0/0.pbf create mode 100644 tests/raw-tiles/compare/1/0/0.pbf create mode 100644 tests/raw-tiles/compare/10/163/366.pbf create mode 100644 tests/raw-tiles/compare/11/326/732.pbf create mode 100644 tests/raw-tiles/compare/12/652/1464.pbf create mode 100644 tests/raw-tiles/compare/12/652/1465.pbf create mode 100644 tests/raw-tiles/compare/13/1304/2928.pbf create mode 100644 tests/raw-tiles/compare/13/1304/2929.pbf create mode 100644 tests/raw-tiles/compare/13/1304/2930.pbf create mode 100644 tests/raw-tiles/compare/14/2608/5856.pbf create mode 100644 tests/raw-tiles/compare/14/2608/5857.pbf create mode 100644 tests/raw-tiles/compare/14/2609/5860.pbf create mode 100644 tests/raw-tiles/compare/2/0/1.pbf create mode 100644 tests/raw-tiles/compare/3/1/2.pbf create mode 100644 tests/raw-tiles/compare/4/2/5.pbf create mode 100644 tests/raw-tiles/compare/5/5/11.pbf create mode 100644 tests/raw-tiles/compare/6/10/22.pbf create mode 100644 tests/raw-tiles/compare/7/20/45.pbf create mode 100644 tests/raw-tiles/compare/8/40/91.pbf create mode 100644 tests/raw-tiles/compare/9/81/183.pbf create mode 100644 tests/raw-tiles/hackspots.geojson diff --git a/.gitignore b/.gitignore index 4581ef2..4bf996a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ *.exe *.out *.app + +# Mac +.DS_Store \ No newline at end of file diff --git a/Makefile b/Makefile index 8e3f7fa..3be98a6 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o +tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o rawtiles.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o @@ -78,7 +78,7 @@ indent: TESTS = $(wildcard tests/*/out/*.json) SPACE = $(NULL) $(NULL) -test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) parallel-test pbf-test join-test enumerate-test decode-test unit +test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test unit ./unit # Work around Makefile and filename punctuation limits: _ for space, @ for :, % for / @@ -111,6 +111,11 @@ parallel-test: cmp tests/parallel/linear-file.json tests/parallel/parallel-pipes.json rm tests/parallel/*.mbtiles tests/parallel/*.json +raw-tiles-test: + ./tippecanoe -o tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson --raw-tiles + diff -x '.*' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare + rm -rf tests/raw-tiles/raw-tiles + decode-test: mkdir -p tests/muni/decode ./tippecanoe -z11 -Z11 -f -o tests/muni/decode/multi.mbtiles tests/muni/*.json diff --git a/main.cpp b/main.cpp index b2584d6..4b85373 100644 --- a/main.cpp +++ b/main.cpp @@ -2255,7 +2255,7 @@ int main(int argc, char **argv) { if(!prevent[P_PBF_COMPRESSION]){ outdb = mbtiles_open(outdir, argv, forcetable); - }else{ + }else{ outpbfdir = outdir; } diff --git a/rawtiles.cpp b/rawtiles.cpp new file mode 100644 index 0000000..07f28dc --- /dev/null +++ b/rawtiles.cpp @@ -0,0 +1,20 @@ +#include +#include +#include +#include +#include "rawtiles.hpp" + +void write_raw_tile(char *outdir, int z, int tx, int ty, std::string pbf) { + mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO); + std::string curdir(outdir); + std::string slash( "/" ); + std::string newdir = curdir + slash + std::to_string(z); + mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(tx); + mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(ty) + ".pbf"; + + std::ofstream pbfFile (newdir, std::ios::out | std::ios::binary); + pbfFile.write (pbf.data(), pbf.size()); + pbfFile.close(); +} \ No newline at end of file diff --git a/rawtiles.hpp b/rawtiles.hpp new file mode 100644 index 0000000..8a135ef --- /dev/null +++ b/rawtiles.hpp @@ -0,0 +1 @@ +void write_raw_tile(char *outdir, int z, int tx, int ty, std::string pbf); \ No newline at end of file diff --git a/tests/raw-tiles/compare/0/0/0.pbf b/tests/raw-tiles/compare/0/0/0.pbf new file mode 100644 index 0000000000000000000000000000000000000000..3f518d33c94233448f2ed74f2992cd6c8a2c19c8 GIT binary patch literal 131 zcmb1YV60%`639qQ&Mq#&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc<($FwK@0%znj(z= literal 0 HcmV?d00001 diff --git a/tests/raw-tiles/compare/1/0/0.pbf b/tests/raw-tiles/compare/1/0/0.pbf new file mode 100644 index 0000000000000000000000000000000000000000..49b703d2772eb386f39b2d47e244228d28c810a1 GIT binary patch literal 131 zcmb1YV60%`639qQ&Mq#&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc<=i0hKo&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc<-Ec2jST?sgd)8F literal 0 HcmV?d00001 diff --git a/tests/raw-tiles/compare/12/652/1464.pbf b/tests/raw-tiles/compare/12/652/1464.pbf new file mode 100644 index 0000000000000000000000000000000000000000..173f8208a654b8de6993272bab5688ca2842a657 GIT binary patch literal 218 zcmb2T&RD_3C6JMroLyXyUs9Z&nx9pipQq8FAjRUBn42oa?wFENlv-RY#p;(|l3J|9 z%f;s1T4?lwGV4o>-}%5RhL~l9QO1qNCs+ zq+n@oWN5C$#KoW_%_SLkn8~m02?AX7ytkO literal 0 HcmV?d00001 diff --git a/tests/raw-tiles/compare/12/652/1465.pbf b/tests/raw-tiles/compare/12/652/1465.pbf new file mode 100644 index 0000000000000000000000000000000000000000..385f335712a7c1dd36de79f220ea038220987977 GIT binary patch literal 138 zcmb1YXRKi2639qQ&Mq#&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}Ni9~A z=8_D_C`wILa4X78%}Xg(aL!LlOHIwlFD*`0Qsq)MGB8vKc2zJmD#=iAEKAi<2*@ug p$w|yh(NXXZQm`~PGBi&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc<$NS`OBw+1vm(v_ literal 0 HcmV?d00001 diff --git a/tests/raw-tiles/compare/13/1304/2929.pbf b/tests/raw-tiles/compare/13/1304/2929.pbf new file mode 100644 index 0000000000000000000000000000000000000000..6aad49f6f3f217b671330c32178472e33b32039f GIT binary patch literal 139 zcmb2@V60%`639qQ&Mq#&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}Ni9~A z=8_D_C`wILa4X78%}Xg(aL!LlOHIwlFD*`0Qsq)MGB8vKc2zJmD#=iAEKAi<2*@ug q$w|yh(NXXZQm`~PGBi&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}Ni9~A z=8_D_C`wILa4X78%}Xg(aL!LlOHIwlFD*`0Qsq)MGB8vKc2zJmD#=iAEKAi<2*@ug q$w|yh(NXXZQm`~PGBiSS6JJ literal 0 HcmV?d00001 diff --git a/tests/raw-tiles/compare/14/2608/5856.pbf b/tests/raw-tiles/compare/14/2608/5856.pbf new file mode 100644 index 0000000000000000000000000000000000000000..b279fba5a085aabd47c303346784b53d7a69de6e GIT binary patch literal 131 zcmb1YV60%`639qQ&Mq#&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWccL}!wmgJq$-pH<$#==N+CfBMjPXk R3NbUXFtGxK9vDtg2LNU$LazV- literal 0 HcmV?d00001 diff --git a/tests/raw-tiles/compare/2/0/1.pbf b/tests/raw-tiles/compare/2/0/1.pbf new file mode 100644 index 0000000000000000000000000000000000000000..ecd69b4ed080da2019fb461594c14326fe5c24e5 GIT binary patch literal 131 zcmb1YV60%`639qQ&Mq#&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc<$R!VKpFt>xgy8_ literal 0 HcmV?d00001 diff --git a/tests/raw-tiles/compare/3/1/2.pbf b/tests/raw-tiles/compare/3/1/2.pbf new file mode 100644 index 0000000000000000000000000000000000000000..acf10f52bbfd3b5fdba26ff4cde9d8460d4e92c5 GIT binary patch literal 131 zcmb1YV60%`639qQ&Mq#&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc<($Mb%K`xI{vuTX literal 0 HcmV?d00001 diff --git a/tests/raw-tiles/compare/7/20/45.pbf b/tests/raw-tiles/compare/7/20/45.pbf new file mode 100644 index 0000000000000000000000000000000000000000..b54f433ca26d946ab1a432d922ccb6b579125010 GIT binary patch literal 131 zcmb1YV60%`639qQ&Mq#&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWccV%Jnj(h) literal 0 HcmV?d00001 diff --git a/tests/raw-tiles/compare/8/40/91.pbf b/tests/raw-tiles/compare/8/40/91.pbf new file mode 100644 index 0000000000000000000000000000000000000000..26c2f2aab7ab3d5abf6fc8124fe76384582d8eac GIT binary patch literal 131 zcmb1YV60%`639qQ&Mq#&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc&Ce>%&(mm7kYe#m%uSVIcT7nsN-ZvyV)e@}NiA05 z<>GcMO3E)%@X1d}PgT<9(lj(RFjw$X2+A)`R0v2c$}Uz2Ppni>2*@ug$w|yh(NXXZ iQm`~PGBj6W;$jdIlwcHMV_;xpWMWcc<(#H?hY #include "mvt.hpp" #include "mbtiles.hpp" +#include "rawtiles.hpp" #include "geometry.hpp" #include "tile.hpp" #include "pool.hpp" @@ -1919,18 +1920,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s if(!prevent[P_PBF_COMPRESSION]){ mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); }else{ - mkdir(outpbfdir, S_IRWXU | S_IRWXG | S_IRWXO); - std::string curdir(outpbfdir); - std::string slash( "/" ); - std::string newdir = curdir + slash + std::to_string(z); - mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); - newdir = newdir + "/" + std::to_string(tx); - mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); - newdir = newdir + "/" + std::to_string(ty) + ".pbf"; - - std::ofstream pbfFile (newdir, std::ios::out | std::ios::app | std::ios::binary); - pbfFile.write (pbf.data(), pbf.size()); - pbfFile.close(); + write_raw_tile(outpbfdir, z, tx, ty, pbf); } if (pthread_mutex_unlock(&db_lock) != 0) { From 29fa3d85414aa8f39f1c2582871a0011618ddac7 Mon Sep 17 00:00:00 2001 From: Shan-Chun Kuo Date: Wed, 5 Apr 2017 16:02:57 +0100 Subject: [PATCH 3/9] Update the readme to add the flag --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e9c7f79..669e375 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ Options messages may result otherwise. Performance will be better if the input is a named file that can be mapped into memory rather than a stream that can only be read sequentially. + * -pC or --raw-tiles: Export to raw uncompressed pbf files rather than to an .mbtiles. This flag will use the output file name (-o) as the folder name. ### Zoom levels and resolution From 9fb1208b79388b75e7764c5119db390862003620 Mon Sep 17 00:00:00 2001 From: Shan-Chun Kuo Date: Wed, 5 Apr 2017 16:05:50 +0100 Subject: [PATCH 4/9] Fix indentation for rawtiles.cpp --- rawtiles.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/rawtiles.cpp b/rawtiles.cpp index 07f28dc..ced397a 100644 --- a/rawtiles.cpp +++ b/rawtiles.cpp @@ -7,14 +7,14 @@ void write_raw_tile(char *outdir, int z, int tx, int ty, std::string pbf) { mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO); std::string curdir(outdir); - std::string slash( "/" ); - std::string newdir = curdir + slash + std::to_string(z); + std::string slash( "/" ); + std::string newdir = curdir + slash + std::to_string(z); mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); - newdir = newdir + "/" + std::to_string(tx); - mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); - newdir = newdir + "/" + std::to_string(ty) + ".pbf"; - - std::ofstream pbfFile (newdir, std::ios::out | std::ios::binary); - pbfFile.write (pbf.data(), pbf.size()); - pbfFile.close(); + newdir = newdir + "/" + std::to_string(tx); + mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(ty) + ".pbf"; + + std::ofstream pbfFile (newdir, std::ios::out | std::ios::binary); + pbfFile.write (pbf.data(), pbf.size()); + pbfFile.close(); } \ No newline at end of file From a3202488570ef0be4ae443df897c26f5f21d1c60 Mon Sep 17 00:00:00 2001 From: Shan-Chun Kuo Date: Fri, 7 Apr 2017 13:33:01 +0100 Subject: [PATCH 5/9] Do make indent, add flag --output-to-directory or -e to write pbf files to a directory, also add flag --no-tile-compression or -pC to get raw protobuf --- Makefile | 2 +- main.cpp | 23 ++++++++++++++--------- main.hpp | 2 +- options.hpp | 2 +- rawtiles.cpp | 6 +++--- tile.cpp | 12 ++++++------ 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 3be98a6..f9d032b 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ parallel-test: rm tests/parallel/*.mbtiles tests/parallel/*.json raw-tiles-test: - ./tippecanoe -o tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson --raw-tiles + ./tippecanoe -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC diff -x '.*' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare rm -rf tests/raw-tiles/raw-tiles diff --git a/main.cpp b/main.cpp index 4b85373..b4566f8 100644 --- a/main.cpp +++ b/main.cpp @@ -57,6 +57,7 @@ static int full_detail = -1; static int min_detail = 7; int quiet = 0; +int outdirtable = 0; int geometry_scale = 0; double simplification = 1; size_t max_tile_size = 500000; @@ -1846,7 +1847,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo ai->second.maxzoom = maxzoom; } - if(!prevent[P_PBF_COMPRESSION]) + if (!outdirtable) mbtiles_write_metadata(outdb, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); return ret; @@ -1903,6 +1904,7 @@ int main(int argc, char **argv) { static struct option long_options[] = { {"output", required_argument, 0, 'o'}, + {"output-to-directory", required_argument, 0, 'e'}, {"name", required_argument, 0, 'n'}, {"description", required_argument, 0, 'N'}, @@ -1960,7 +1962,7 @@ int main(int argc, char **argv) { {"no-clipping", no_argument, &prevent[P_CLIPPING], 1}, {"no-duplication", no_argument, &prevent[P_DUPLICATION], 1}, {"no-tiny-polygon-reduction", no_argument, &prevent[P_TINY_POLYGON_REDUCTION], 1}, - {"raw-tiles", no_argument, &prevent[P_PBF_COMPRESSION], 1}, + {"no-tile-compression", no_argument, &prevent[P_TILE_COMPRESSION], 1}, {0, 0, 0, 0}, }; @@ -1983,7 +1985,7 @@ int main(int argc, char **argv) { } } - while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:", long_options, NULL)) != -1) { + while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:e:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:", long_options, NULL)) != -1) { switch (i) { case 0: break; @@ -2063,6 +2065,11 @@ int main(int argc, char **argv) { outdir = optarg; break; + case 'e': + outpbfdir = optarg; + outdirtable = 1; + break; + case 'x': exclude.insert(std::string(optarg)); break; @@ -2244,8 +2251,8 @@ int main(int argc, char **argv) { fprintf(stderr, "Forcing -g0 since -B or -r is not known\n"); } - if (outdir == NULL) { - fprintf(stderr, "%s: must specify -o out.mbtiles\n", argv[0]); + if (outdir == NULL && outpbfdir == NULL) { + fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory\n", argv[0]); exit(EXIT_FAILURE); } @@ -2253,10 +2260,8 @@ int main(int argc, char **argv) { unlink(outdir); } - if(!prevent[P_PBF_COMPRESSION]){ + if (!outdirtable) { outdb = mbtiles_open(outdir, argv, forcetable); - }else{ - outpbfdir = outdir; } int ret = EXIT_SUCCESS; @@ -2285,7 +2290,7 @@ int main(int argc, char **argv) { ret = read_input(sources, name ? name : outdir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description); - if(!prevent[P_PBF_COMPRESSION]) + if (!outdirtable) mbtiles_close(outdb, argv); #ifdef MTRACE diff --git a/main.hpp b/main.hpp index d61c6de..f1df878 100644 --- a/main.hpp +++ b/main.hpp @@ -11,7 +11,7 @@ void checkdisk(struct reader *r, int nreader); extern int geometry_scale; extern int quiet; - +extern int outdirtable; extern char *outpbfdir; extern size_t CPUS; diff --git a/options.hpp b/options.hpp index 97bda01..6081d7d 100644 --- a/options.hpp +++ b/options.hpp @@ -25,7 +25,7 @@ #define P_CLIPPING ((int) 'c') #define P_DUPLICATION ((int) 'D') #define P_TINY_POLYGON_REDUCTION ((int) 't') -#define P_PBF_COMPRESSION ((int) 'C') +#define P_TILE_COMPRESSION ((int) 'C') extern int prevent[256]; extern int additional[256]; diff --git a/rawtiles.cpp b/rawtiles.cpp index ced397a..1de0dec 100644 --- a/rawtiles.cpp +++ b/rawtiles.cpp @@ -7,14 +7,14 @@ void write_raw_tile(char *outdir, int z, int tx, int ty, std::string pbf) { mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO); std::string curdir(outdir); - std::string slash( "/" ); + std::string slash("/"); std::string newdir = curdir + slash + std::to_string(z); mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); newdir = newdir + "/" + std::to_string(tx); mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); newdir = newdir + "/" + std::to_string(ty) + ".pbf"; - std::ofstream pbfFile (newdir, std::ios::out | std::ios::binary); - pbfFile.write (pbf.data(), pbf.size()); + std::ofstream pbfFile(newdir, std::ios::out | std::ios::binary); + pbfFile.write(pbf.data(), pbf.size()); pbfFile.close(); } \ No newline at end of file diff --git a/tile.cpp b/tile.cpp index 521cdd3..ecac670 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1841,9 +1841,9 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s std::string compressed; std::string pbf = tile.encode(); - if(!prevent[P_PBF_COMPRESSION]){ + if (!prevent[P_TILE_COMPRESSION]) { compress(pbf, compressed); - }else{ + } else { compressed = pbf; } @@ -1917,10 +1917,10 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s exit(EXIT_FAILURE); } - if(!prevent[P_PBF_COMPRESSION]){ - mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); - }else{ - write_raw_tile(outpbfdir, z, tx, ty, pbf); + if (!outdirtable) { + mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); + } else { + write_raw_tile(outpbfdir, z, tx, ty, compressed); } if (pthread_mutex_unlock(&db_lock) != 0) { From 682a272f919da8654acd9f69d7705ffd6999bf3b Mon Sep 17 00:00:00 2001 From: Shan-Chun Kuo Date: Fri, 7 Apr 2017 16:24:29 +0100 Subject: [PATCH 6/9] Add condition to prevent options -o and -e being used together --- main.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/main.cpp b/main.cpp index b4566f8..9525aad 100644 --- a/main.cpp +++ b/main.cpp @@ -1863,6 +1863,15 @@ static bool has_name(struct option *long_options, int *pl) { return false; } +static bool findFlag(int argc, char **argv, std::string str) { + char **ptr = std::find(argv, argv + argc, str); + + if (ptr != argv + argc) + return true; + else + return false; +} + int main(int argc, char **argv) { #ifdef MTRACE mtrace(); @@ -2256,6 +2265,11 @@ int main(int argc, char **argv) { exit(EXIT_FAILURE); } + if ((findFlag(argc, argv, "-o") && findFlag(argc, argv, "-e")) || (findFlag(argc, argv, "--output") && findFlag(argc, argv, "--output-to-directory"))) { + fprintf(stderr, "%s: Options -o and -e cannot be used together\n", argv[0]); + exit(EXIT_FAILURE); + } + if (force) { unlink(outdir); } From b66ab10252010fae296ae88aca3b59241bf68137 Mon Sep 17 00:00:00 2001 From: Shan-Chun Kuo Date: Fri, 7 Apr 2017 17:03:30 +0100 Subject: [PATCH 7/9] Add algorithms library --- main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/main.cpp b/main.cpp index 9525aad..ea3c498 100644 --- a/main.cpp +++ b/main.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include From 1c1ae6a8be9587aba7efb2ca659ef28dfc358505 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 7 Apr 2017 12:36:34 -0700 Subject: [PATCH 8/9] Update documentation and reduce use of global variables --- README.md | 3 ++- main.cpp | 46 ++++++++++++++++++---------------------------- main.hpp | 2 -- man/tippecanoe.1 | 13 +++++++++---- rawtiles.cpp | 4 ++-- rawtiles.hpp | 2 +- tile.cpp | 14 ++++++++------ tile.hpp | 4 ++-- 8 files changed, 42 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 669e375..d9b8078 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Options ### File control * -o _file_.mbtiles or --output=_file_.mbtiles: Name the output file. + * -e _directory_ or --output-directory=_directory_: Write tiles to the specified *directory* instead of to an mbtiles file. * -f or --force: Delete the mbtiles file if it already exists instead of giving an error * -F or --allow-existing: Proceed (without deleting existing data) if the metadata or tiles table already exists or if metadata fields can't be set @@ -78,7 +79,6 @@ Options messages may result otherwise. Performance will be better if the input is a named file that can be mapped into memory rather than a stream that can only be read sequentially. - * -pC or --raw-tiles: Export to raw uncompressed pbf files rather than to an .mbtiles. This flag will use the output file name (-o) as the folder name. ### Zoom levels and resolution @@ -148,6 +148,7 @@ resolution is obtained than by using a smaller _maxzoom_ or _detail_. * -pc or --no-clipping: Don't clip features to the size of the tile. If a feature overlaps the tile's bounds or buffer at all, it is included completely. Be careful: this can produce very large tilesets, especially with large polygons. * -pD or --no-duplication: As with --no-clipping, each feature is included intact instead of cut to tile boundaries. In addition, it is included only in a single tile per zoom level rather than potentially in multiple copies. Clients of the tileset must check adjacent tiles (possibly some distance away) to ensure they have all features. * -pt or --no-tiny-polygon-reduction: Don't combine the area of very small polygons into small squares that represent their combined area. + * -pC or --no-tile-compression: Don't compress the PBF vector tile data. * -q or --quiet: Work quietly instead of reporting progress Environment diff --git a/main.cpp b/main.cpp index ea3c498..9e4926d 100644 --- a/main.cpp +++ b/main.cpp @@ -58,7 +58,6 @@ static int full_detail = -1; static int min_detail = 7; int quiet = 0; -int outdirtable = 0; int geometry_scale = 0; double simplification = 1; size_t max_tile_size = 500000; @@ -71,8 +70,6 @@ struct source { std::string file; }; -char *outpbfdir; - size_t CPUS; size_t TEMP_FILES; long long MAX_FILES; @@ -1009,7 +1006,7 @@ void choose_first_zoom(long long *file_bbox, struct reader *reader, unsigned *iz } } -int read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set *exclude, std::set *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description) { +int read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description) { int ret = EXIT_SUCCESS; struct reader reader[CPUS]; @@ -1783,7 +1780,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } unsigned midx = 0, midy = 0; - int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps); + int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, outdir, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); @@ -1848,8 +1845,9 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo ai->second.maxzoom = maxzoom; } - if (!outdirtable) + if (outdb != NULL) { mbtiles_write_metadata(outdb, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); + } return ret; } @@ -1864,15 +1862,6 @@ static bool has_name(struct option *long_options, int *pl) { return false; } -static bool findFlag(int argc, char **argv, std::string str) { - char **ptr = std::find(argv, argv + argc, str); - - if (ptr != argv + argc) - return true; - else - return false; -} - int main(int argc, char **argv) { #ifdef MTRACE mtrace(); @@ -1887,7 +1876,8 @@ int main(int argc, char **argv) { char *name = NULL; char *description = NULL; char *layername = NULL; - char *outdir = NULL; + char *out_mbtiles = NULL; + char *out_directory = NULL; sqlite3 *outdb = NULL; int maxzoom = 14; int minzoom = 0; @@ -2072,12 +2062,11 @@ int main(int argc, char **argv) { break; case 'o': - outdir = optarg; + out_mbtiles = optarg; break; case 'e': - outpbfdir = optarg; - outdirtable = 1; + out_directory = optarg; break; case 'x': @@ -2261,22 +2250,22 @@ int main(int argc, char **argv) { fprintf(stderr, "Forcing -g0 since -B or -r is not known\n"); } - if (outdir == NULL && outpbfdir == NULL) { + if (out_mbtiles == NULL && out_directory == NULL) { fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory\n", argv[0]); exit(EXIT_FAILURE); } - if ((findFlag(argc, argv, "-o") && findFlag(argc, argv, "-e")) || (findFlag(argc, argv, "--output") && findFlag(argc, argv, "--output-to-directory"))) { + if (out_mbtiles != NULL && out_directory != NULL) { fprintf(stderr, "%s: Options -o and -e cannot be used together\n", argv[0]); exit(EXIT_FAILURE); } - if (force) { - unlink(outdir); - } + if (out_mbtiles != NULL) { + if (force) { + unlink(out_mbtiles); + } - if (!outdirtable) { - outdb = mbtiles_open(outdir, argv, forcetable); + outdb = mbtiles_open(out_mbtiles, argv, forcetable); } int ret = EXIT_SUCCESS; @@ -2303,10 +2292,11 @@ int main(int argc, char **argv) { long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0}; - ret = read_input(sources, name ? name : outdir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description); + ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_directory, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_directory, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description); - if (!outdirtable) + if (outdb != NULL) { mbtiles_close(outdb, argv); + } #ifdef MTRACE muntrace(); diff --git a/main.hpp b/main.hpp index f1df878..e06908b 100644 --- a/main.hpp +++ b/main.hpp @@ -11,8 +11,6 @@ void checkdisk(struct reader *r, int nreader); extern int geometry_scale; extern int quiet; -extern int outdirtable; -extern char *outpbfdir; extern size_t CPUS; extern size_t TEMP_FILES; diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 5d36924..37a4468 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -60,12 +60,16 @@ specified, the files are all merged into the single named layer, even if they tr \-n \fIname\fP or \-\-name=\fIname\fP: Human\-readable name for the tileset (default file.json) .IP \(bu 2 \-A \fItext\fP or \-\-attribution=\fItext\fP: Attribution (HTML) to be shown with maps that use data from this tileset. +.IP \(bu 2 +\-N \fIdescription\fP or \-\-description=\fIdescription\fP: Description for the tileset (default file.mbtiles) .RE .SS File control .RS .IP \(bu 2 \-o \fIfile\fP\&.mbtiles or \-\-output=\fIfile\fP\&.mbtiles: Name the output file. .IP \(bu 2 +\-e \fIdirectory\fP or \-\-output\-directory=\fIdirectory\fP: Write tiles to the specified \fIdirectory\fP instead of to an mbtiles file. +.IP \(bu 2 \-f or \-\-force: Delete the mbtiles file if it already exists instead of giving an error .IP \(bu 2 \-F or \-\-allow\-existing: Proceed (without deleting existing data) if the metadata or tiles table already exists @@ -162,6 +166,8 @@ which may not be what you want. \-an or \-\-drop\-smallest\-as\-needed: Dynamically drop the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level to keep large tiles under the 500K size limit. This option will not work for point features. .IP \(bu 2 \-aL or \-\-grid\-low\-zooms: At all zoom levels below \fImaxzoom\fP, snap all lines and polygons to a stairstep grid instead of allowing diagonals. You will also want to specify a tile resolution, probably \fB\fC\-D8\fR\&. This option provides a way to display continuous parcel, gridded, or binned data at low zooms without overwhelming the tiles with tiny polygons, since features will either get stretched out to the grid unit or lost entirely, depending on how they happened to be aligned in the original data. +.IP \(bu 2 +\-aw or \-\-detect\-longitude\-wraparound: Detect when adjacent points within a feature jump to the other side of the world, and try to fix the geometry. .RE .SS Doing less .RS @@ -178,7 +184,7 @@ which may not be what you want. .IP \(bu 2 \-pi or \-\-preserve\-input\-order: Preserve the original input order of features as the drawing order instead of ordering geographically. (This is implemented as a restoration of the original order at the end, so that dot\-dropping is still geographic, which means it also undoes \-ao). .IP \(bu 2 -\-pp or \-\-no\-polygon\-splitting: Don't split complex polygons (over 700 vertices after simplification) into multiple features. +\-pp or \-\-no\-polygon\-splitting: This no longer has any effect. .IP \(bu 2 \-pc or \-\-no\-clipping: Don't clip features to the size of the tile. If a feature overlaps the tile's bounds or buffer at all, it is included completely. Be careful: this can produce very large tilesets, especially with large polygons. .IP \(bu 2 @@ -186,6 +192,8 @@ which may not be what you want. .IP \(bu 2 \-pt or \-\-no\-tiny\-polygon\-reduction: Don't combine the area of very small polygons into small squares that represent their combined area. .IP \(bu 2 +\-pC or \-\-no\-tile\-compression: Don't compress the PBF vector tile data. +.IP \(bu 2 \-q or \-\-quiet: Work quietly instead of reporting progress .RE .SH Environment @@ -301,9 +309,6 @@ have their probability diffused, so that some of them will be drawn as a square this minimum size and others will not be drawn at all, preserving the total area that all of them should have had together. .PP -Any polygons that have over 700 vertices after line simplification will be split into -multiple features so they can be rendered efficiently, unless you use \-pp to prevent this. -.PP Features in the same tile that share the same type and attributes are coalesced together into a single geometry if you use \fB\fC\-\-coalesce\fR\&. You are strongly encouraged to use \-x to exclude any unnecessary properties to reduce wasted file size. diff --git a/rawtiles.cpp b/rawtiles.cpp index 1de0dec..1c7ecc4 100644 --- a/rawtiles.cpp +++ b/rawtiles.cpp @@ -4,7 +4,7 @@ #include #include "rawtiles.hpp" -void write_raw_tile(char *outdir, int z, int tx, int ty, std::string pbf) { +void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) { mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO); std::string curdir(outdir); std::string slash("/"); @@ -17,4 +17,4 @@ void write_raw_tile(char *outdir, int z, int tx, int ty, std::string pbf) { std::ofstream pbfFile(newdir, std::ios::out | std::ios::binary); pbfFile.write(pbf.data(), pbf.size()); pbfFile.close(); -} \ No newline at end of file +} diff --git a/rawtiles.hpp b/rawtiles.hpp index 8a135ef..1a332c9 100644 --- a/rawtiles.hpp +++ b/rawtiles.hpp @@ -1 +1 @@ -void write_raw_tile(char *outdir, int z, int tx, int ty, std::string pbf); \ No newline at end of file +void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf); diff --git a/tile.cpp b/tile.cpp index ecac670..9d43bc5 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1185,6 +1185,7 @@ struct write_tile_args { int min_detail; int basezoom; sqlite3 *outdb; + const char *outdir; double droprate; int buffer; const char *fname; @@ -1222,7 +1223,7 @@ struct write_tile_args { double fraction_out; }; -long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, write_tile_args *arg) { +long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, write_tile_args *arg) { int line_detail; double merge_fraction = 1; double mingap_fraction = 1; @@ -1917,10 +1918,10 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s exit(EXIT_FAILURE); } - if (!outdirtable) { + if (outdb != NULL) { mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); - } else { - write_raw_tile(outpbfdir, z, tx, ty, compressed); + } else if (outdir != NULL) { + write_raw_tile(outdir, z, tx, ty, compressed); } if (pthread_mutex_unlock(&db_lock) != 0) { @@ -1983,7 +1984,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg); + long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->outdir, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg); if (len < 0) { int *err = &arg->err; @@ -2048,7 +2049,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps) { // Table to map segment and layer number back to layer name std::vector> layer_unmaps; for (size_t seg = 0; seg < layermaps.size(); seg++) { @@ -2183,6 +2184,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].min_detail = min_detail; args[thread].basezoom = basezoom; args[thread].outdb = outdb; // locked with db_lock + args[thread].outdir = outdir; args[thread].droprate = droprate; args[thread].buffer = buffer; args[thread].fname = fname; diff --git a/tile.hpp b/tile.hpp index ffbb960..2ae8d7f 100644 --- a/tile.hpp +++ b/tile.hpp @@ -1,5 +1,5 @@ -long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers); +long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); From f558b7838033a08947303bc9542cb4a5e23fe9d1 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 7 Apr 2017 13:21:38 -0700 Subject: [PATCH 9/9] Include tileset metadata when writing either to directory or mbtiles --- CHANGELOG.md | 4 ++ main.cpp | 4 +- mbtiles.cpp | 75 ++++++++++++++++++++++----- mbtiles.hpp | 2 +- tests/raw-tiles/compare/metadata.json | 12 +++++ tile-join.cpp | 2 +- version.hpp | 2 +- 7 files changed, 83 insertions(+), 18 deletions(-) create mode 100644 tests/raw-tiles/compare/metadata.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1d3f9..3d8bcc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.16.15 + +* Add --output-to-directory and --no-tile-compression options + ## 1.16.14 * Add --description option for mbtiles metadata diff --git a/main.cpp b/main.cpp index 9e4926d..78c6128 100644 --- a/main.cpp +++ b/main.cpp @@ -1845,9 +1845,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo ai->second.maxzoom = maxzoom; } - if (outdb != NULL) { - mbtiles_write_metadata(outdb, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); - } + mbtiles_write_metadata(outdb, outdir, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); return ret; } diff --git a/mbtiles.cpp b/mbtiles.cpp index 0a364da..ab9aeb3 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -131,11 +131,23 @@ bool type_and_string::operator<(const type_and_string &o) const { return false; } -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description) { +void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description) { char *sql, *err; + sqlite3 *db = outdb; + if (outdb == NULL) { + if (sqlite3_open("", &db) != SQLITE_OK) { + fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "Create metadata table: %s\n", err); + exit(EXIT_FAILURE); + } + } + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set name in metadata: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -144,7 +156,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", description != NULL ? description : fname); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set description in metadata: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -153,7 +165,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", 2); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set version : %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -162,7 +174,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", minzoom); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set minzoom: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -171,7 +183,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", maxzoom); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set maxzoom: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -180,7 +192,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", midlon, midlat, maxzoom); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set center: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -189,7 +201,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", minlon, minlat, maxlon, maxlat); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set bounds: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -198,7 +210,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", "overlay"); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set type: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -208,7 +220,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int if (attribution != NULL) { sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", attribution); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set type: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -218,7 +230,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int } sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", vector ? "pbf" : "png"); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set format: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -276,7 +288,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int aprintf(&buf, " ] }"); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str()); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set json: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -284,6 +296,45 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int } sqlite3_free(sql); } + + if (outdir != NULL) { + std::string metadata = std::string(outdir) + "/metadata.json"; + FILE *fp = fopen(metadata.c_str(), "w"); + if (fp == NULL) { + perror(metadata.c_str()); + exit(EXIT_FAILURE); + } + + fprintf(fp, "{\n"); + + sqlite3_stmt *stmt; + bool first = true; + if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) { + while (sqlite3_step(stmt) == SQLITE_ROW) { + std::string key, value; + + quote(&key, (const char *) sqlite3_column_text(stmt, 0)); + quote(&value, (const char *) sqlite3_column_text(stmt, 1)); + + if (!first) { + fprintf(fp, ",\n"); + } + fprintf(fp, " \"%s\": \"%s\"", key.c_str(), value.c_str()); + first = false; + } + sqlite3_finalize(stmt); + } + + fprintf(fp, "\n}\n"); + fclose(fp); + } + + if (outdb == NULL) { + if (sqlite3_close(db) != SQLITE_OK) { + fprintf(stderr, "Could not close temp database: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + } } void mbtiles_close(sqlite3 *outdb, char **argv) { diff --git a/mbtiles.hpp b/mbtiles.hpp index 4fb18b1..dbe93cd 100644 --- a/mbtiles.hpp +++ b/mbtiles.hpp @@ -20,7 +20,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description); +void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description); void mbtiles_close(sqlite3 *outdb, char **argv); diff --git a/tests/raw-tiles/compare/metadata.json b/tests/raw-tiles/compare/metadata.json new file mode 100644 index 0000000..c83c17c --- /dev/null +++ b/tests/raw-tiles/compare/metadata.json @@ -0,0 +1,12 @@ +{ + "name": "tests/raw-tiles/raw-tiles", + "description": "tests/raw-tiles/raw-tiles", + "version": "2", + "minzoom": "0", + "maxzoom": "14", + "center": "-122.662354,45.514045,14", + "bounds": "-122.682427,45.512332,-122.654961,45.569975", + "type": "overlay", + "format": "pbf", + "json": "{\"vector_layers\": [ { \"id\": \"hackspotsgeojson\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 14, \"fields\": {\"Address\": \"String\", \"Name\": \"String\", \"Notes\": \"String\"} } ] }" +} diff --git a/tile-join.cpp b/tile-join.cpp index 4920f2d..6ca1617 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -750,7 +750,7 @@ int main(int argc, char **argv) { decode(readers, csv, layermap, outdb, &st, header, mapping, exclude, ifmatched, attribution, description); - mbtiles_write_metadata(outdb, outfile, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str()); + mbtiles_write_metadata(outdb, NULL, outfile, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str()); mbtiles_close(outdb, argv); return 0; diff --git a/version.hpp b/version.hpp index 3b4aad7..b44031b 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.16.13\n" +#define VERSION "tippecanoe v1.16.14\n"