From 17a8aceaf30740d77b17cdc93c25bbbe906d2b8f Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Thu, 20 Dec 2018 22:36:30 +0100 Subject: [PATCH 01/42] [libdeflate] added smiple support for libdeflate | This work was sponsored by SkySight (@plantain) --- Makefile | 12 +++--- mvt.cpp | 109 ++++++++++++++++++++++++++----------------------------- 2 files changed, 57 insertions(+), 64 deletions(-) diff --git a/Makefile b/Makefile index 24f150e..80f4426 100644 --- a/Makefile +++ b/Makefile @@ -48,22 +48,22 @@ INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-enumerate: enumerate.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 -ldeflate tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate unit: unit.o text.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate -include $(wildcard *.d) diff --git a/mvt.cpp b/mvt.cpp index 27486f8..b66e537 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include #include "mvt.hpp" #include "geometry.hpp" #include "protozero/varint.hpp" @@ -27,84 +27,77 @@ bool is_compressed(std::string const &data) { // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int decompress(std::string const &input, std::string &output) { - z_stream inflate_s; - inflate_s.zalloc = Z_NULL; - inflate_s.zfree = Z_NULL; - inflate_s.opaque = Z_NULL; - inflate_s.avail_in = 0; - inflate_s.next_in = Z_NULL; - if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { - fprintf(stderr, "Decompression error: %s\n", inflate_s.msg); - } - inflate_s.next_in = (Bytef *) input.data(); - inflate_s.avail_in = input.size(); - inflate_s.next_out = (Bytef *) output.data(); - inflate_s.avail_out = output.size(); - + /* + if (inflateInit2(inflate_s, 32 + 15) != Z_OK) { + fprintf(stderr, "Decompression error: %s\n", msg); + }*/ + struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); + void* next_in = (void*)input.data(); + size_t avail_in = input.size(); + void* next_out = (void*)output.data(); + size_t avail_out = output.size(); while (true) { - size_t existing_output = inflate_s.next_out - (Bytef *) output.data(); + long unsigned int existing_output; - output.resize(existing_output + 2 * inflate_s.avail_in + 100); - inflate_s.next_out = (Bytef *) output.data() + existing_output; - inflate_s.avail_out = output.size() - existing_output; + output.resize(existing_output + 2 * avail_in + 100); + next_out = (void*)(output.data() + existing_output); + avail_out = (output.size() - existing_output); - int ret = inflate(&inflate_s, 0); - if (ret < 0) { - fprintf(stderr, "Decompression error: "); - if (ret == Z_DATA_ERROR) { + int ret = libdeflate_deflate_decompress_ex(decompressor, + next_in, avail_in, + next_out, avail_out, + &existing_output, + &existing_output); + if (ret != LIBDEFLATE_SUCCESS) { + fprintf(stderr, "Decompression error: "); + if (ret == LIBDEFLATE_BAD_DATA) { fprintf(stderr, "data error"); - } - if (ret == Z_STREAM_ERROR) { - fprintf(stderr, "stream error"); - } - if (ret == Z_MEM_ERROR) { + } + if (ret == LIBDEFLATE_INSUFFICIENT_SPACE) { fprintf(stderr, "out of memory"); - } - if (ret == Z_BUF_ERROR) { + } + if (ret == LIBDEFLATE_SHORT_OUTPUT) { fprintf(stderr, "no data in buffer"); - } + } fprintf(stderr, "\n"); return 0; } - if (ret == Z_STREAM_END) { - break; - } + if (ret == LIBDEFLATE_SHORT_OUTPUT) { + break; + } // ret must be Z_OK or Z_NEED_DICT; // continue decompresing } - output.resize(inflate_s.next_out - (Bytef *) output.data()); - inflateEnd(&inflate_s); + output.resize(avail_out - output.size()); + libdeflate_free_decompressor(decompressor); return 1; } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int compress(std::string const &input, std::string &output) { - z_stream deflate_s; - deflate_s.zalloc = Z_NULL; - deflate_s.zfree = Z_NULL; - deflate_s.opaque = Z_NULL; - deflate_s.avail_in = 0; - deflate_s.next_in = Z_NULL; - deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); - deflate_s.next_in = (Bytef *) input.data(); - deflate_s.avail_in = input.size(); - size_t length = 0; - do { - size_t increase = input.size() / 2 + 1024; - output.resize(length + increase); - deflate_s.avail_out = increase; - deflate_s.next_out = (Bytef *) (output.data() + length); - int ret = deflate(&deflate_s, Z_FINISH); - if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + void* next_in = (void*)input.data(); + size_t avail_in = input.size(); + void* next_out = (void*)output.data(); + size_t avail_out = output.size(); + struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); + do { + size_t increase = input.size() / 2 + 1024; + output.resize(avail_in + increase); + avail_out = increase; + next_out = ((void*)output.data() + avail_in); + int ret = libdeflate_deflate_compress(deflate_s, + next_in, avail_in, + next_out, avail_out); + if (ret != LIBDEFLATE_SUCCESS) { return -1; - } - length += (increase - deflate_s.avail_out); - } while (deflate_s.avail_out == 0); - deflateEnd(&deflate_s); - output.resize(length); + } + avail_in += (increase - avail_out); + } while (avail_out == 0); + libdeflate_free_compressor(deflate_s); + output.resize(avail_in); return 0; } From a86e6cab6caafe27ffecdf5116c11ca79dd5069d Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 00:21:14 +0100 Subject: [PATCH 02/42] [libdeflate] added smiple support for libdeflate | This work was sponsored by SkySight (@plantain) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 80f4426..4ac9455 100644 --- a/Makefile +++ b/Makefile @@ -331,6 +331,7 @@ layer-json-test: # Use this target to regenerate the standards that the tests are compared against # after making a change that legitimately changes their output +# prep-test: $(TESTS) From 0d764c7603e610bbbdeb0a3f5e4eb905bd37dcde Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 00:43:15 +0100 Subject: [PATCH 03/42] [libdeflate] added smiple support for libdeflate | This work was sponsored by SkySight (@plantain) --- .travis.yml | 1 + Makefile | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index f27b0b2..4a86f64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,7 @@ matrix: before_install: - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" - export PATH=${DEPS_DIR}/bin:${PATH} && mkdir -p ${DEPS_DIR} + - make -j12 libdeflate - | if [[ ${CLANG_VERSION:-false} != false ]]; then export CCOMPILER='clang' diff --git a/Makefile b/Makefile index 4ac9455..9a9d8ac 100644 --- a/Makefile +++ b/Makefile @@ -340,3 +340,8 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode ./tippecanoe-decode -x generator $@.check.mbtiles > $@ cmp $(patsubst %.check,%,$@) $@ rm $@.check.mbtiles + +libdeflate: + git clone https://github.com/ebiggers/libdeflate libdeflate + cd libdeflate/ + make -j12 install From 4a644276ee028d0763a7271f93c4f82eb3b16f0d Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 13:14:45 +0100 Subject: [PATCH 04/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- .gitmodules | 3 +++ .travis.yml | 2 +- libdeflate | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .gitmodules create mode 160000 libdeflate diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..7fdc574 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libdeflate"] + path = libdeflate + url = https://github.com/ebiggers/libdeflate diff --git a/.travis.yml b/.travis.yml index 4a86f64..061e88f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,7 @@ matrix: before_install: - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" - export PATH=${DEPS_DIR}/bin:${PATH} && mkdir -p ${DEPS_DIR} - - make -j12 libdeflate + - make -j12 -C libdeflate install - | if [[ ${CLANG_VERSION:-false} != false ]]; then export CCOMPILER='clang' diff --git a/libdeflate b/libdeflate new file mode 160000 index 0000000..d6d50c6 --- /dev/null +++ b/libdeflate @@ -0,0 +1 @@ +Subproject commit d6d50c69554ac25897fc988d671e8e39780bab92 From 44d5423ba098d68343e164622eb123639174eef2 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 13:24:37 +0100 Subject: [PATCH 05/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- .travis.yml | 1 - Makefile | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 061e88f..f27b0b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,6 @@ matrix: before_install: - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" - export PATH=${DEPS_DIR}/bin:${PATH} && mkdir -p ${DEPS_DIR} - - make -j12 -C libdeflate install - | if [[ ${CLANG_VERSION:-false} != false ]]; then export CCOMPILER='clang' diff --git a/Makefile b/Makefile index 9a9d8ac..38cbea7 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ else FINAL_FLAGS := -g $(WARNING_FLAGS) $(DEBUG_FLAGS) endif -all: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit tippecanoe-json-tool +all: libdeflate tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit tippecanoe-json-tool docs: man/tippecanoe.1 @@ -342,6 +342,4 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles libdeflate: - git clone https://github.com/ebiggers/libdeflate libdeflate - cd libdeflate/ - make -j12 install + make $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) $(LDFLAGS) -C libdeflate -j12 install From e866e2e53429d48d4b43d4e267d10bdb61b142ea Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 13:34:14 +0100 Subject: [PATCH 06/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- mvt.cpp | 94 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/mvt.cpp b/mvt.cpp index b66e537..7d1e3a6 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -27,77 +27,77 @@ bool is_compressed(std::string const &data) { // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int decompress(std::string const &input, std::string &output) { - /* + /* if (inflateInit2(inflate_s, 32 + 15) != Z_OK) { fprintf(stderr, "Decompression error: %s\n", msg); }*/ - struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); - void* next_in = (void*)input.data(); - size_t avail_in = input.size(); - void* next_out = (void*)output.data(); - size_t avail_out = output.size(); + struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); + void *next_in = (void *) input.data(); + size_t avail_in = input.size(); + void *next_out = (void *) output.data(); + size_t avail_out = output.size(); while (true) { - long unsigned int existing_output; + long unsigned int existing_output; - output.resize(existing_output + 2 * avail_in + 100); - next_out = (void*)(output.data() + existing_output); - avail_out = (output.size() - existing_output); + output.resize(existing_output + 2 * avail_in + 100); + next_out = (void *) (output.data() + existing_output); + avail_out = (output.size() - existing_output); - int ret = libdeflate_deflate_decompress_ex(decompressor, - next_in, avail_in, - next_out, avail_out, - &existing_output, - &existing_output); - if (ret != LIBDEFLATE_SUCCESS) { - fprintf(stderr, "Decompression error: "); - if (ret == LIBDEFLATE_BAD_DATA) { + int ret = libdeflate_deflate_decompress_ex(decompressor, + next_in, avail_in, + next_out, avail_out, + &existing_output, + &existing_output); + if (ret != LIBDEFLATE_SUCCESS) { + fprintf(stderr, "Decompression error: "); + if (ret == LIBDEFLATE_BAD_DATA) { fprintf(stderr, "data error"); - } - if (ret == LIBDEFLATE_INSUFFICIENT_SPACE) { + } + if (ret == LIBDEFLATE_INSUFFICIENT_SPACE) { fprintf(stderr, "out of memory"); - } - if (ret == LIBDEFLATE_SHORT_OUTPUT) { + } + if (ret == LIBDEFLATE_SHORT_OUTPUT) { fprintf(stderr, "no data in buffer"); - } + } fprintf(stderr, "\n"); return 0; } - if (ret == LIBDEFLATE_SHORT_OUTPUT) { - break; - } + if (ret == LIBDEFLATE_SHORT_OUTPUT) { + break; + } // ret must be Z_OK or Z_NEED_DICT; // continue decompresing } - output.resize(avail_out - output.size()); - libdeflate_free_decompressor(decompressor); + output.resize(avail_out - output.size()); + libdeflate_free_decompressor(decompressor); return 1; } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int compress(std::string const &input, std::string &output) { - void* next_in = (void*)input.data(); - size_t avail_in = input.size(); - void* next_out = (void*)output.data(); - size_t avail_out = output.size(); - struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); - do { - size_t increase = input.size() / 2 + 1024; - output.resize(avail_in + increase); - avail_out = increase; - next_out = ((void*)output.data() + avail_in); - int ret = libdeflate_deflate_compress(deflate_s, - next_in, avail_in, - next_out, avail_out); - if (ret != LIBDEFLATE_SUCCESS) { + void *next_in = (void *) input.data(); + size_t avail_in = input.size(); + void *next_out = (void *) output.data(); + size_t avail_out = output.size(); + struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); + do { + size_t increase = input.size() / 2 + 1024; + output.resize(avail_in + increase); + avail_out = increase; + next_out = ((void *) output.data() + avail_in); + int ret = libdeflate_deflate_compress(deflate_s, + next_in, avail_in, + next_out, avail_out); + if (ret != LIBDEFLATE_SUCCESS) { return -1; - } - avail_in += (increase - avail_out); - } while (avail_out == 0); - libdeflate_free_compressor(deflate_s); - output.resize(avail_in); + } + avail_in += (increase - avail_out); + } while (avail_out == 0); + libdeflate_free_compressor(deflate_s); + output.resize(avail_in); return 0; } From 65f0a6de0d9316461cc37e7dbc189fe64c40c766 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 13:38:38 +0100 Subject: [PATCH 07/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 38cbea7..e91078f 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ else FINAL_FLAGS := -g $(WARNING_FLAGS) $(DEBUG_FLAGS) endif -all: libdeflate tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit tippecanoe-json-tool +all: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit tippecanoe-json-tool docs: man/tippecanoe.1 @@ -47,7 +47,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o +tippecanoe: libdeflate geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-enumerate: enumerate.o From 03a21fb8b32e6dca8dca568114970934a5809be4 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 13:47:00 +0100 Subject: [PATCH 08/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e91078f..3a1c639 100644 --- a/Makefile +++ b/Makefile @@ -342,4 +342,4 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles libdeflate: - make $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) $(LDFLAGS) -C libdeflate -j12 install + make $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) $(LDFLAGS) -C libdeflate install From fb197a43a06506d7a83188d3447126afe56705d4 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 13:49:13 +0100 Subject: [PATCH 09/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 3a1c639..14ff526 100644 --- a/Makefile +++ b/Makefile @@ -342,4 +342,5 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles libdeflate: + git submodule update --remote libdeflate make $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) $(LDFLAGS) -C libdeflate install From 7e54e248d75e55b77681edc6407de9f38c93d1e4 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 14:00:12 +0100 Subject: [PATCH 10/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 14ff526..546d7bd 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: libdeflate geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o +tippecanoe: deflate geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-enumerate: enumerate.o @@ -341,6 +341,6 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode cmp $(patsubst %.check,%,$@) $@ rm $@.check.mbtiles -libdeflate: +deflate: git submodule update --remote libdeflate - make $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) $(LDFLAGS) -C libdeflate install + make -C libdeflate install From 3d5b7d607460faa87de7b6c34352c3367efbfe4c Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 14:03:28 +0100 Subject: [PATCH 11/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 546d7bd..1f737c0 100644 --- a/Makefile +++ b/Makefile @@ -342,5 +342,5 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles deflate: - git submodule update --remote libdeflate + git clone https://github.com/ebiggers/libdeflate make -C libdeflate install From 7c4e638ff654c8112cb981e9e47b412e69fe05de Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 14:04:26 +0100 Subject: [PATCH 12/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- .gitmodules | 3 --- libdeflate | 1 - 2 files changed, 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 libdeflate diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 7fdc574..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "libdeflate"] - path = libdeflate - url = https://github.com/ebiggers/libdeflate diff --git a/libdeflate b/libdeflate deleted file mode 160000 index d6d50c6..0000000 --- a/libdeflate +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d6d50c69554ac25897fc988d671e8e39780bab92 From 1c54f512129515e21eb0d0e1f6d1161e2659e688 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 14:17:21 +0100 Subject: [PATCH 13/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1f737c0..2b53375 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ else FINAL_FLAGS := -g $(WARNING_FLAGS) $(DEBUG_FLAGS) endif -all: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit tippecanoe-json-tool +all: deflate tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit tippecanoe-json-tool docs: man/tippecanoe.1 @@ -47,8 +47,8 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: deflate geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate +tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 -ldeflate @@ -342,5 +342,5 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles deflate: - git clone https://github.com/ebiggers/libdeflate + git clone https://github.com/ebiggers/libdeflate || true make -C libdeflate install From 31113da5591f36290d37876281daa13b8c3c8218 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 14:27:50 +0100 Subject: [PATCH 14/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- .gitmodules | 3 +++ Makefile | 3 +-- libdeflate | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 160000 libdeflate diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..7fdc574 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libdeflate"] + path = libdeflate + url = https://github.com/ebiggers/libdeflate diff --git a/Makefile b/Makefile index 2b53375..b2bf6b4 100644 --- a/Makefile +++ b/Makefile @@ -342,5 +342,4 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles deflate: - git clone https://github.com/ebiggers/libdeflate || true - make -C libdeflate install + make -C libdeflate diff --git a/libdeflate b/libdeflate new file mode 160000 index 0000000..d6d50c6 --- /dev/null +++ b/libdeflate @@ -0,0 +1 @@ +Subproject commit d6d50c69554ac25897fc988d671e8e39780bab92 From e28d035e2b8d61cd44e66139843290e9f8b00da6 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 14:36:42 +0100 Subject: [PATCH 15/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b2bf6b4..6c285fe 100644 --- a/Makefile +++ b/Makefile @@ -51,19 +51,19 @@ tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o pro $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-enumerate: enumerate.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate/libdeflate.a tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread unit: unit.o text.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -include $(wildcard *.d) From b351aca0f1f1b7fda54149751249a20acb3c129d Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 14:54:47 +0100 Subject: [PATCH 16/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6c285fe..0e4c9bc 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ PG= H = $(wildcard *.h) $(wildcard *.hpp) C = $(wildcard *.c) $(wildcard *.cpp) -INCLUDES = -I/usr/local/include -I. +INCLUDES = -I/usr/local/include -I. -Ilibdeflate LIBS = -L/usr/local/lib tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o From 52328b72c57671b2e3fe6ada8ad8e5826ec228c6 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 16:19:58 +0100 Subject: [PATCH 17/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- decode.cpp | 2 +- mvt.cpp | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/decode.cpp b/decode.cpp index 86e58fb..f1d3a5a 100644 --- a/decode.cpp +++ b/decode.cpp @@ -87,7 +87,7 @@ void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, u void handle(std::string message, int z, unsigned x, unsigned y, std::set const &to_decode, bool pipeline, bool stats, json_writer &state) { mvt_tile tile; - bool was_compressed; + bool was_compressed = true; try { if (!tile.decode(message, was_compressed)) { diff --git a/mvt.cpp b/mvt.cpp index 7d1e3a6..9a4da66 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -56,11 +56,8 @@ int decompress(std::string const &input, std::string &output) { if (ret == LIBDEFLATE_INSUFFICIENT_SPACE) { fprintf(stderr, "out of memory"); } - if (ret == LIBDEFLATE_SHORT_OUTPUT) { - fprintf(stderr, "no data in buffer"); - } fprintf(stderr, "\n"); - return 0; + return -ret; } if (ret == LIBDEFLATE_SHORT_OUTPUT) { From 5f8bee69cc47ae7b9905685c78f2aea34c1a1c24 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 16:27:47 +0100 Subject: [PATCH 18/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- mvt.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mvt.cpp b/mvt.cpp index 9a4da66..dca958b 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -37,17 +37,18 @@ int decompress(std::string const &input, std::string &output) { void *next_out = (void *) output.data(); size_t avail_out = output.size(); while (true) { - long unsigned int existing_output; - - output.resize(existing_output + 2 * avail_in + 100); - next_out = (void *) (output.data() + existing_output); - avail_out = (output.size() - existing_output); + long unsigned int existing_output = 0; int ret = libdeflate_deflate_decompress_ex(decompressor, next_in, avail_in, next_out, avail_out, &existing_output, &existing_output); + + output.resize(existing_output + 2 * avail_in + 100); + next_out = (void *) (output.data() + existing_output); + avail_out = (output.size() - existing_output); + if (ret != LIBDEFLATE_SUCCESS) { fprintf(stderr, "Decompression error: "); if (ret == LIBDEFLATE_BAD_DATA) { From fb07fbd68cd0d328b05adbaa3c4b306497540444 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 16:30:10 +0100 Subject: [PATCH 19/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- mvt.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mvt.cpp b/mvt.cpp index dca958b..a43c9a7 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -39,10 +39,9 @@ int decompress(std::string const &input, std::string &output) { while (true) { long unsigned int existing_output = 0; - int ret = libdeflate_deflate_decompress_ex(decompressor, + int ret = libdeflate_deflate_decompress(decompressor, next_in, avail_in, next_out, avail_out, - &existing_output, &existing_output); output.resize(existing_output + 2 * avail_in + 100); From eb13d6b56abe44fcd0e92bdae0d8e1146eb7c6a0 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 16:49:20 +0100 Subject: [PATCH 20/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- mvt.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mvt.cpp b/mvt.cpp index a43c9a7..4c86971 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -45,7 +45,7 @@ int decompress(std::string const &input, std::string &output) { &existing_output); output.resize(existing_output + 2 * avail_in + 100); - next_out = (void *) (output.data() + existing_output); + next_out = (void *) (next_out + existing_output); avail_out = (output.size() - existing_output); if (ret != LIBDEFLATE_SUCCESS) { @@ -84,7 +84,7 @@ int compress(std::string const &input, std::string &output) { size_t increase = input.size() / 2 + 1024; output.resize(avail_in + increase); avail_out = increase; - next_out = ((void *) output.data() + avail_in); + next_out = (void *) (output.data() + avail_in); int ret = libdeflate_deflate_compress(deflate_s, next_in, avail_in, next_out, avail_out); From ba3549ebb3d2df769580783e6b6495051512b4cc Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 18:56:47 +0100 Subject: [PATCH 21/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- mvt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvt.cpp b/mvt.cpp index 4c86971..91a5ca6 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -45,7 +45,7 @@ int decompress(std::string const &input, std::string &output) { &existing_output); output.resize(existing_output + 2 * avail_in + 100); - next_out = (void *) (next_out + existing_output); + next_out = (void *) (output.data() + existing_output); avail_out = (output.size() - existing_output); if (ret != LIBDEFLATE_SUCCESS) { From 586de1f5b1bf1fd9fbe3a06b639cb8ef8fa78b1a Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Sat, 22 Dec 2018 00:38:23 +0100 Subject: [PATCH 22/42] [tippecanoe] Added missing dependency for libdeflate as git submodule | This work was sponsored by SkySight (@plantain) --- mvt.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mvt.cpp b/mvt.cpp index 91a5ca6..07d7a1c 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -47,6 +47,7 @@ int decompress(std::string const &input, std::string &output) { output.resize(existing_output + 2 * avail_in + 100); next_out = (void *) (output.data() + existing_output); avail_out = (output.size() - existing_output); + avail_in = (input.size() - avail_in); if (ret != LIBDEFLATE_SUCCESS) { fprintf(stderr, "Decompression error: "); @@ -85,13 +86,14 @@ int compress(std::string const &input, std::string &output) { output.resize(avail_in + increase); avail_out = increase; next_out = (void *) (output.data() + avail_in); + next_in = (void *) (input.data() + avail_in); int ret = libdeflate_deflate_compress(deflate_s, next_in, avail_in, next_out, avail_out); if (ret != LIBDEFLATE_SUCCESS) { return -1; } - avail_in += (increase - avail_out); + avail_in -= (increase - avail_out); } while (avail_out == 0); libdeflate_free_compressor(deflate_s); output.resize(avail_in); From d25e79c3a2c21c6038972bb0f47508b0cea2301a Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Sat, 22 Dec 2018 07:53:03 +0100 Subject: [PATCH 23/42] [tippecanoe] Fixed compression errors --- mvt.cpp | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/mvt.cpp b/mvt.cpp index 07d7a1c..4baac9a 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -27,10 +27,6 @@ bool is_compressed(std::string const &data) { // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int decompress(std::string const &input, std::string &output) { - /* - if (inflateInit2(inflate_s, 32 + 15) != Z_OK) { - fprintf(stderr, "Decompression error: %s\n", msg); - }*/ struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); void *next_in = (void *) input.data(); size_t avail_in = input.size(); @@ -43,11 +39,12 @@ int decompress(std::string const &input, std::string &output) { next_in, avail_in, next_out, avail_out, &existing_output); - - output.resize(existing_output + 2 * avail_in + 100); - next_out = (void *) (output.data() + existing_output); - avail_out = (output.size() - existing_output); + existing_output += avail_in + 1024; + output.resize(existing_output); + next_out = (void *) (output.data() + existing_output + 1024); + next_in = (void *) (input.data() + existing_output / 2); avail_in = (input.size() - avail_in); + avail_out = avail_in; if (ret != LIBDEFLATE_SUCCESS) { fprintf(stderr, "Decompression error: "); @@ -68,7 +65,6 @@ int decompress(std::string const &input, std::string &output) { // ret must be Z_OK or Z_NEED_DICT; // continue decompresing } - output.resize(avail_out - output.size()); libdeflate_free_decompressor(decompressor); return 1; @@ -82,18 +78,18 @@ int compress(std::string const &input, std::string &output) { size_t avail_out = output.size(); struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); do { - size_t increase = input.size() / 2 + 1024; + size_t increase = 1024; output.resize(avail_in + increase); avail_out = increase; - next_out = (void *) (output.data() + avail_in); - next_in = (void *) (input.data() + avail_in); + next_out = (void *) (next_out + increase); + next_in = (void *) (next_in + increase * 2); int ret = libdeflate_deflate_compress(deflate_s, next_in, avail_in, next_out, avail_out); if (ret != LIBDEFLATE_SUCCESS) { return -1; } - avail_in -= (increase - avail_out); + avail_in -= increase; } while (avail_out == 0); libdeflate_free_compressor(deflate_s); output.resize(avail_in); From ede4a9509cafe4d436389bab88190fe837f84f70 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 13:28:33 +0100 Subject: [PATCH 24/42] [mvt.cpp] Fixed compression problems --- decode.cpp | 2 +- mvt.cpp | 112 +++++++++++++++++++++++------------------------------ 2 files changed, 50 insertions(+), 64 deletions(-) diff --git a/decode.cpp b/decode.cpp index f1d3a5a..86e58fb 100644 --- a/decode.cpp +++ b/decode.cpp @@ -87,7 +87,7 @@ void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, u void handle(std::string message, int z, unsigned x, unsigned y, std::set const &to_decode, bool pipeline, bool stats, json_writer &state) { mvt_tile tile; - bool was_compressed = true; + bool was_compressed; try { if (!tile.decode(message, was_compressed)) { diff --git a/mvt.cpp b/mvt.cpp index 4baac9a..e1fe6a6 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -22,102 +22,88 @@ mvt_geometry::mvt_geometry(int nop, long long nx, long long ny) { // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp bool is_compressed(std::string const &data) { - return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); + return data.size() > 2 && (((uint8_t) data[0] & 0x60) == 0x4 || ((uint8_t) data[0] & 0x60) == 0x20); } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int decompress(std::string const &input, std::string &output) { + size_t avail_in = 8192; + size_t avail_out = 8192; + long unsigned int actual_output = 0; + long unsigned int actual_input = 0; + long unsigned int actual_out = 0; + long unsigned int actual_in = 0; + output.resize(avail_out); + void *current_out = (void*)output.data(); + void *current_in = (void*)input.data(); + struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); - void *next_in = (void *) input.data(); - size_t avail_in = input.size(); - void *next_out = (void *) output.data(); - size_t avail_out = output.size(); - while (true) { - long unsigned int existing_output = 0; +decompress: + int ret = libdeflate_deflate_decompress_ex(decompressor, + current_in, avail_in, + current_out, avail_out, + &actual_input, + &actual_output); + actual_out += actual_output; + if (ret == LIBDEFLATE_SHORT_OUTPUT) { + output.resize(actual_output + avail_out); + current_out = (void*)((long)current_out + actual_out); + current_in = (void*)((long)current_in + actual_in); + goto decompress; + } + libdeflate_free_decompressor(decompressor); - int ret = libdeflate_deflate_decompress(decompressor, - next_in, avail_in, - next_out, avail_out, - &existing_output); - existing_output += avail_in + 1024; - output.resize(existing_output); - next_out = (void *) (output.data() + existing_output + 1024); - next_in = (void *) (input.data() + existing_output / 2); - avail_in = (input.size() - avail_in); - avail_out = avail_in; - - if (ret != LIBDEFLATE_SUCCESS) { + if (ret != LIBDEFLATE_SUCCESS) { + if (ret == LIBDEFLATE_BAD_DATA) { + fprintf(stderr, "data not compressed"); + } else { fprintf(stderr, "Decompression error: "); - if (ret == LIBDEFLATE_BAD_DATA) { - fprintf(stderr, "data error"); - } if (ret == LIBDEFLATE_INSUFFICIENT_SPACE) { fprintf(stderr, "out of memory"); } - fprintf(stderr, "\n"); - return -ret; } - - if (ret == LIBDEFLATE_SHORT_OUTPUT) { - break; - } - - // ret must be Z_OK or Z_NEED_DICT; - // continue decompresing + fprintf(stderr, "\n"); + return -ret; } - output.resize(avail_out - output.size()); - libdeflate_free_decompressor(decompressor); - return 1; + + output.resize(actual_out); + return 0; } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int compress(std::string const &input, std::string &output) { - void *next_in = (void *) input.data(); size_t avail_in = input.size(); - void *next_out = (void *) output.data(); - size_t avail_out = output.size(); + size_t avail_out = avail_in * 8; struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); - do { - size_t increase = 1024; - output.resize(avail_in + increase); - avail_out = increase; - next_out = (void *) (next_out + increase); - next_in = (void *) (next_in + increase * 2); - int ret = libdeflate_deflate_compress(deflate_s, - next_in, avail_in, - next_out, avail_out); - if (ret != LIBDEFLATE_SUCCESS) { - return -1; - } - avail_in -= increase; - } while (avail_out == 0); + output.resize(avail_out); + int ret = libdeflate_deflate_compress(deflate_s, + (void*)input.data(), avail_in, + (void*)output.data(), avail_out); libdeflate_free_compressor(deflate_s); - output.resize(avail_in); - return 0; + if (ret == 0) { + return -1; + } + output.resize(ret); + return 1; } bool mvt_tile::decode(std::string &message, bool &was_compressed) { layers.clear(); std::string src; + std::string uncompressed; - if (is_compressed(message)) { - std::string uncompressed; - if (decompress(message, uncompressed) == 0) { - exit(EXIT_FAILURE); - } + was_compressed = false; + src = message; + if (decompress(message, uncompressed) == 0) { src = uncompressed; was_compressed = true; - } else { - src = message; - was_compressed = false; } protozero::pbf_reader reader(src); while (reader.next()) { switch (reader.tag()) { - case 3: /* layer */ - { + case 3: /* layer */ { protozero::pbf_reader layer_reader(reader.get_message()); mvt_layer layer; From edd48c4708b3a4c6a8bed30f8489303a3efb78cc Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 14:02:41 +0100 Subject: [PATCH 25/42] [mvt.cpp] Fixed compression problems --- mvt.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mvt.cpp b/mvt.cpp index e1fe6a6..4e60796 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -44,11 +44,12 @@ decompress: current_out, avail_out, &actual_input, &actual_output); + actual_in += actual_input; actual_out += actual_output; if (ret == LIBDEFLATE_SHORT_OUTPUT) { - output.resize(actual_output + avail_out); - current_out = (void*)((long)current_out + actual_out); - current_in = (void*)((long)current_in + actual_in); + output.resize(actual_out + avail_out); + current_out = (void*)((long)output.data() + actual_out); + current_in = (void*)((long)input.data() + actual_in); goto decompress; } libdeflate_free_decompressor(decompressor); @@ -74,8 +75,8 @@ decompress: int compress(std::string const &input, std::string &output) { size_t avail_in = input.size(); size_t avail_out = avail_in * 8; - struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); output.resize(avail_out); + struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); int ret = libdeflate_deflate_compress(deflate_s, (void*)input.data(), avail_in, (void*)output.data(), avail_out); From 6fc4259105a27d920f47b5384567309e1e49afbd Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 14:17:55 +0100 Subject: [PATCH 26/42] [mvt.cpp] Fixed compression problems, added shared dependency to libdeflate --- .travis.yml | 2 +- Makefile | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index f27b0b2..11e10d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: node_js node_js: - "6" -sudo: false +sudo: true matrix: include: diff --git a/Makefile b/Makefile index 0e4c9bc..f57305e 100644 --- a/Makefile +++ b/Makefile @@ -48,16 +48,16 @@ INCLUDES = -I/usr/local/include -I. -Ilibdeflate LIBS = -L/usr/local/lib tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate/libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread @@ -342,4 +342,4 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles deflate: - make -C libdeflate + make -C libdeflate install From 9c15a070c99443f1289644fadc268d915d23c53e Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 14:24:01 +0100 Subject: [PATCH 27/42] [mvt.cpp] Fixed compression problems, switched to static libdeflate linking --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f57305e..0e4c9bc 100644 --- a/Makefile +++ b/Makefile @@ -48,16 +48,16 @@ INCLUDES = -I/usr/local/include -I. -Ilibdeflate LIBS = -L/usr/local/lib tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate/libdeflate.a tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread @@ -342,4 +342,4 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles deflate: - make -C libdeflate install + make -C libdeflate From df400e4ec60eb707a5eee1520b1095ad1be3b055 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 14:51:22 +0100 Subject: [PATCH 28/42] [tippecanoe] added libdeflate into ldpath --- .travis.yml | 4 +++- Makefile | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 11e10d2..5ec02b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: node_js node_js: - "6" -sudo: true +sudo: false matrix: include: @@ -100,9 +100,11 @@ before_install: fi install: + - echo "$(pwd)/libdeflate/" | sudo tee /etc/ld.so.conf.d/libdeflate.conf && sudo ldconfig - BUILDTYPE=${BUILDTYPE} make -j script: + - echo "$(pwd)/libdeflate/" | sudo tee /etc/ld.so.conf.d/libdeflate.conf && sudo ldconfig - npm install geobuf - if [ -n "${FEWER}" ]; then BUILDTYPE=${BUILDTYPE} make fewer-tests; else diff --git a/Makefile b/Makefile index 0e4c9bc..09df88c 100644 --- a/Makefile +++ b/Makefile @@ -45,19 +45,19 @@ H = $(wildcard *.h) $(wildcard *.hpp) C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. -Ilibdeflate -LIBS = -L/usr/local/lib +LIBS = -L/usr/local/lib -Llibdeflate tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate/libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread From 0ad4753f3561a8c5830b57020c5a6bbbd38d4d5d Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 14:55:42 +0100 Subject: [PATCH 29/42] [tippecanoe] added libdeflate into ldpath --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5ec02b5..4bc25ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -104,7 +104,7 @@ install: - BUILDTYPE=${BUILDTYPE} make -j script: - - echo "$(pwd)/libdeflate/" | sudo tee /etc/ld.so.conf.d/libdeflate.conf && sudo ldconfig + - echo "${TRAVIS_BUILD_DIR}/libdeflate/" | sudo tee /etc/ld.so.conf.d/libdeflate.conf && sudo ldconfig - npm install geobuf - if [ -n "${FEWER}" ]; then BUILDTYPE=${BUILDTYPE} make fewer-tests; else From acc28faef5b8a08ec2b0694ef3c96cd29439ff8b Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 14:58:54 +0100 Subject: [PATCH 30/42] [tippecanoe] added libdeflate into ldpath --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4bc25ec..bcc82bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,7 +100,7 @@ before_install: fi install: - - echo "$(pwd)/libdeflate/" | sudo tee /etc/ld.so.conf.d/libdeflate.conf && sudo ldconfig + - echo "${TRAVIS_BUILD_DIR}/libdeflate/" | sudo tee /etc/ld.so.conf.d/libdeflate.conf && sudo ldconfig - BUILDTYPE=${BUILDTYPE} make -j script: From b92ac213c607000ad09b558ba1c8389c75147d86 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 15:31:42 +0100 Subject: [PATCH 31/42] [tippecanoe] added libdeflate into ldpath --- Dockerfile | 1 + Dockerfile.centos7 | 1 + Makefile | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1df89e5..0eef762 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update \ RUN mkdir -p /tmp/tippecanoe-src WORKDIR /tmp/tippecanoe-src COPY . /tmp/tippecanoe-src +ENV LD_LIBRARY_PATH LD_LIBRARY_PATH:libdeflate # Build tippecanoe RUN make \ diff --git a/Dockerfile.centos7 b/Dockerfile.centos7 index c330de2..d180341 100644 --- a/Dockerfile.centos7 +++ b/Dockerfile.centos7 @@ -6,6 +6,7 @@ RUN yum install -y make sqlite-devel zlib-devel bash git gcc-c++ RUN mkdir -p /tmp/tippecanoe-src WORKDIR /tmp/tippecanoe-src COPY . /tmp/tippecanoe-src +ENV LD_LIBRARY_PATH LD_LIBRARY_PATH:libdeflate # Build tippecanoe RUN make \ diff --git a/Makefile b/Makefile index 09df88c..3d0ed8a 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ H = $(wildcard *.h) $(wildcard *.hpp) C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. -Ilibdeflate -LIBS = -L/usr/local/lib -Llibdeflate +LIBS = -L/usr/local/lib -L$(PWD)/libdeflate tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate From f76ed5658b3547075c710a43e19122a9f7d8c725 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 15:40:31 +0100 Subject: [PATCH 32/42] [tippecanoe] added libdeflate into ldpath --- .travis.yml | 2 -- Makefile | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcc82bf..f27b0b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,11 +100,9 @@ before_install: fi install: - - echo "${TRAVIS_BUILD_DIR}/libdeflate/" | sudo tee /etc/ld.so.conf.d/libdeflate.conf && sudo ldconfig - BUILDTYPE=${BUILDTYPE} make -j script: - - echo "${TRAVIS_BUILD_DIR}/libdeflate/" | sudo tee /etc/ld.so.conf.d/libdeflate.conf && sudo ldconfig - npm install geobuf - if [ -n "${FEWER}" ]; then BUILDTYPE=${BUILDTYPE} make fewer-tests; else diff --git a/Makefile b/Makefile index 3d0ed8a..e504f68 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. -Ilibdeflate LIBS = -L/usr/local/lib -L$(PWD)/libdeflate +LD_LIBRARY_PATH := /usr/local/lib $(PWD)/libdeflate tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate From 99a86cfc772fe6c7e1562010ff1c262b44c7fd6e Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 15:45:55 +0100 Subject: [PATCH 33/42] [tippecanoe] added libdeflate into ldpath --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index f27b0b2..1becc61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ matrix: - docker sudo: true dist: trusty - env: DOCKERFILE=Dockerfile.centos7 + env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate DOCKERFILE=Dockerfile.centos7 before_install: [] install: - docker build -t tippecanoe-image -f ${DOCKERFILE} . @@ -26,7 +26,7 @@ matrix: - docker sudo: true dist: trusty - env: DOCKERFILE=Dockerfile + env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate DOCKERFILE=Dockerfile before_install: [] install: - docker build -t tippecanoe-image -f ${DOCKERFILE} . @@ -35,7 +35,7 @@ matrix: # debug+integer-santizer build - os: linux compiler: clang - env: CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=integer" CFLAGS="-fsanitize=integer" LDFLAGS="-fsanitize=integer" + env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=integer" CFLAGS="-fsanitize=integer" LDFLAGS="-fsanitize=integer" addons: apt: sources: ['ubuntu-toolchain-r-test' ] @@ -43,7 +43,7 @@ matrix: # debug+leak+address-sanitizer build - os: linux compiler: clang - env: CLANG_VERSION='3.8.0' BUILDTYPE=Debug ASAN_OPTIONS=detect_leaks=1 CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=address,undefined" CFLAGS="-fsanitize=address,undefined" LDFLAGS="-fsanitize=address,undefined" FEWER=true + env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate CLANG_VERSION='3.8.0' BUILDTYPE=Debug ASAN_OPTIONS=detect_leaks=1 CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=address,undefined" CFLAGS="-fsanitize=address,undefined" LDFLAGS="-fsanitize=address,undefined" FEWER=true addons: apt: sources: ['ubuntu-toolchain-r-test' ] @@ -51,7 +51,7 @@ matrix: # coverage+debug build - os: linux compiler: clang - env: CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="--coverage" CFLAGS="--coverage" LDFLAGS="--coverage" + env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="--coverage" CFLAGS="--coverage" LDFLAGS="--coverage" after_script: - mason install llvm-cov 3.9.1 - mason link llvm-cov 3.9.1 @@ -66,7 +66,7 @@ matrix: # release+linux+g++ - os: linux compiler: gcc - env: BUILDTYPE=Release CC="gcc-4.9" CXX="g++-4.9" + env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate BUILDTYPE=Release CC="gcc-4.9" CXX="g++-4.9" addons: apt: sources: ['ubuntu-toolchain-r-test'] @@ -74,7 +74,7 @@ matrix: # release+linux+clang++ - os: linux compiler: clang - env: CLANG_VERSION='3.8.0' BUILDTYPE=Release CC="clang-3.8" CXX="clang++-3.8" + env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate CLANG_VERSION='3.8.0' BUILDTYPE=Release CC="clang-3.8" CXX="clang++-3.8" addons: apt: sources: ['ubuntu-toolchain-r-test' ] @@ -82,11 +82,11 @@ matrix: # release+osx - os: osx compiler: clang - env: BUILDTYPE=Release + env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate BUILDTYPE=Release # debug+osx - os: osx compiler: clang - env: BUILDTYPE=Debug + env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate BUILDTYPE=Debug before_install: - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" From b594b41d81c3592b146f04d3835fcf1fe3fced56 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 16:05:43 +0100 Subject: [PATCH 34/42] [tippecanoe] added libdeflate into ldpath --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e504f68..496ac10 100644 --- a/Makefile +++ b/Makefile @@ -49,16 +49,16 @@ LIBS = -L/usr/local/lib -L$(PWD)/libdeflate LD_LIBRARY_PATH := /usr/local/lib $(PWD)/libdeflate tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate/libdeflate.a tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread From 81d98e1aa1c0b4557c03002276673d46f879355a Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 16:23:01 +0100 Subject: [PATCH 35/42] [mvt.cpp] Fixed compression problems, switched to static libdeflate linking --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 496ac10..f56d317 100644 --- a/Makefile +++ b/Makefile @@ -343,4 +343,4 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles deflate: - make -C libdeflate + make -C libdeflate libdeflate.a From 314b4a105d0245c2cc25c6133dec3df66ee29c01 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 16:31:32 +0100 Subject: [PATCH 36/42] [tippecanoe] added libdeflate into ldpath --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f56d317..a868694 100644 --- a/Makefile +++ b/Makefile @@ -45,20 +45,20 @@ H = $(wildcard *.h) $(wildcard *.hpp) C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. -Ilibdeflate -LIBS = -L/usr/local/lib -L$(PWD)/libdeflate -LD_LIBRARY_PATH := /usr/local/lib $(PWD)/libdeflate +LIBS = -L/usr/local/lib -L$(PWD)/libdeflate -L$(PWD) +LD_LIBRARY_PATH := /usr/local/lib $(PWD)/libdeflate $(PWD) tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate.a tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate/libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate.a tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate.a tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread From 4019935fa5b2f3c4ac42c10ec4676f4aeb38099c Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 16:45:22 +0100 Subject: [PATCH 37/42] [tippecanoe] Fixed compression errors --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a868694..263fbf5 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ unit: unit.o text.o %.o: %.cpp $(CXX) -MMD $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c -o $@ $< -clean: +clean: deflate-clean rm -f ./tippecanoe ./tippecanoe-* ./tile-join ./unit *.o *.d */*.o */*.d tests/**/*.mbtiles tests/**/*.check indent: @@ -344,3 +344,6 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode deflate: make -C libdeflate libdeflate.a + +deflate-clean: + make -C libdeflate realclean From ee1c9c5c124e2e511271bee9e60dc6e5a28c5dd0 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 16:54:02 +0100 Subject: [PATCH 38/42] [tippecanoe] Fixed compression errors --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 263fbf5..caf3b23 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. -Ilibdeflate LIBS = -L/usr/local/lib -L$(PWD)/libdeflate -L$(PWD) -LD_LIBRARY_PATH := /usr/local/lib $(PWD)/libdeflate $(PWD) +LD_LIBRARY_PATH := /usr/local/lib:$(PWD)/libdeflate:$(PWD) tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate.a From ca1dbe2ae992b728fcd7d67c0f50687679819cd8 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 16:56:03 +0100 Subject: [PATCH 39/42] [tippecanoe] Fixed compression errors --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index caf3b23..5914adc 100644 --- a/Makefile +++ b/Makefile @@ -49,16 +49,16 @@ LIBS = -L/usr/local/lib -L$(PWD)/libdeflate -L$(PWD) LD_LIBRARY_PATH := /usr/local/lib:$(PWD)/libdeflate:$(PWD) tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate/libdeflate.a tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate.a + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread @@ -343,7 +343,7 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles deflate: - make -C libdeflate libdeflate.a + make -C libdeflate libdeflate/libdeflate.a deflate-clean: make -C libdeflate realclean From 05ac48e4cda296555013110700999843135a43db Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 16:57:25 +0100 Subject: [PATCH 40/42] [tippecanoe] Fixed compression errors --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5914adc..3083e58 100644 --- a/Makefile +++ b/Makefile @@ -343,7 +343,7 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles deflate: - make -C libdeflate libdeflate/libdeflate.a + make -C libdeflate libdeflate.a deflate-clean: make -C libdeflate realclean From cdf5322e9a529433507a3a39510c32838ca724ef Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 17:38:28 +0100 Subject: [PATCH 41/42] [tippecanoe] still trying to fix build errors --- Makefile | 4 ++-- mvt.cpp | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 3083e58..53492d9 100644 --- a/Makefile +++ b/Makefile @@ -343,7 +343,7 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode rm $@.check.mbtiles deflate: - make -C libdeflate libdeflate.a + cd libdeflate && make BUILDTYPE=$(BUILDTYPE) libdeflate.a deflate-clean: - make -C libdeflate realclean + cd libdeflate && make realclean diff --git a/mvt.cpp b/mvt.cpp index 4e60796..f841aa9 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -34,8 +34,8 @@ int decompress(std::string const &input, std::string &output) { long unsigned int actual_out = 0; long unsigned int actual_in = 0; output.resize(avail_out); - void *current_out = (void*)output.data(); - void *current_in = (void*)input.data(); + void *current_out = (void *) output.data(); + void *current_in = (void *) input.data(); struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); decompress: @@ -48,8 +48,8 @@ decompress: actual_out += actual_output; if (ret == LIBDEFLATE_SHORT_OUTPUT) { output.resize(actual_out + avail_out); - current_out = (void*)((long)output.data() + actual_out); - current_in = (void*)((long)input.data() + actual_in); + current_out = (void *) ((long) output.data() + actual_out); + current_in = (void *) ((long) input.data() + actual_in); goto decompress; } libdeflate_free_decompressor(decompressor); @@ -78,8 +78,8 @@ int compress(std::string const &input, std::string &output) { output.resize(avail_out); struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); int ret = libdeflate_deflate_compress(deflate_s, - (void*)input.data(), avail_in, - (void*)output.data(), avail_out); + (void *) input.data(), avail_in, + (void *) output.data(), avail_out); libdeflate_free_compressor(deflate_s); if (ret == 0) { return -1; @@ -104,7 +104,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { while (reader.next()) { switch (reader.tag()) { - case 3: /* layer */ { + case 3: /* layer */ { protozero::pbf_reader layer_reader(reader.get_message()); mvt_layer layer; From abd3e0f623cb6dfa398db747c2e5b68785a5e707 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Mon, 24 Dec 2018 18:14:53 +0100 Subject: [PATCH 42/42] [tippecanoe] Added missing dependency for libdeflate --- .travis.yml | 18 +++++++++--------- Makefile | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1becc61..f27b0b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ matrix: - docker sudo: true dist: trusty - env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate DOCKERFILE=Dockerfile.centos7 + env: DOCKERFILE=Dockerfile.centos7 before_install: [] install: - docker build -t tippecanoe-image -f ${DOCKERFILE} . @@ -26,7 +26,7 @@ matrix: - docker sudo: true dist: trusty - env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate DOCKERFILE=Dockerfile + env: DOCKERFILE=Dockerfile before_install: [] install: - docker build -t tippecanoe-image -f ${DOCKERFILE} . @@ -35,7 +35,7 @@ matrix: # debug+integer-santizer build - os: linux compiler: clang - env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=integer" CFLAGS="-fsanitize=integer" LDFLAGS="-fsanitize=integer" + env: CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=integer" CFLAGS="-fsanitize=integer" LDFLAGS="-fsanitize=integer" addons: apt: sources: ['ubuntu-toolchain-r-test' ] @@ -43,7 +43,7 @@ matrix: # debug+leak+address-sanitizer build - os: linux compiler: clang - env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate CLANG_VERSION='3.8.0' BUILDTYPE=Debug ASAN_OPTIONS=detect_leaks=1 CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=address,undefined" CFLAGS="-fsanitize=address,undefined" LDFLAGS="-fsanitize=address,undefined" FEWER=true + env: CLANG_VERSION='3.8.0' BUILDTYPE=Debug ASAN_OPTIONS=detect_leaks=1 CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=address,undefined" CFLAGS="-fsanitize=address,undefined" LDFLAGS="-fsanitize=address,undefined" FEWER=true addons: apt: sources: ['ubuntu-toolchain-r-test' ] @@ -51,7 +51,7 @@ matrix: # coverage+debug build - os: linux compiler: clang - env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="--coverage" CFLAGS="--coverage" LDFLAGS="--coverage" + env: CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="--coverage" CFLAGS="--coverage" LDFLAGS="--coverage" after_script: - mason install llvm-cov 3.9.1 - mason link llvm-cov 3.9.1 @@ -66,7 +66,7 @@ matrix: # release+linux+g++ - os: linux compiler: gcc - env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate BUILDTYPE=Release CC="gcc-4.9" CXX="g++-4.9" + env: BUILDTYPE=Release CC="gcc-4.9" CXX="g++-4.9" addons: apt: sources: ['ubuntu-toolchain-r-test'] @@ -74,7 +74,7 @@ matrix: # release+linux+clang++ - os: linux compiler: clang - env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate CLANG_VERSION='3.8.0' BUILDTYPE=Release CC="clang-3.8" CXX="clang++-3.8" + env: CLANG_VERSION='3.8.0' BUILDTYPE=Release CC="clang-3.8" CXX="clang++-3.8" addons: apt: sources: ['ubuntu-toolchain-r-test' ] @@ -82,11 +82,11 @@ matrix: # release+osx - os: osx compiler: clang - env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate BUILDTYPE=Release + env: BUILDTYPE=Release # debug+osx - os: osx compiler: clang - env: LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/libdeflate BUILDTYPE=Debug + env: BUILDTYPE=Debug before_install: - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" diff --git a/Makefile b/Makefile index 53492d9..c65f9a0 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ all: deflate tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit ti docs: man/tippecanoe.1 -install: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join tippecanoe-json-tool +install: deflate tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join tippecanoe-json-tool mkdir -p $(PREFIX)/bin mkdir -p $(MANDIR) cp tippecanoe $(PREFIX)/bin/tippecanoe @@ -342,7 +342,7 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode cmp $(patsubst %.check,%,$@) $@ rm $@.check.mbtiles -deflate: +deflate: deflate-clean cd libdeflate && make BUILDTYPE=$(BUILDTYPE) libdeflate.a deflate-clean: