From 17a8aceaf30740d77b17cdc93c25bbbe906d2b8f Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Thu, 20 Dec 2018 22:36:30 +0100 Subject: [PATCH 1/7] [libdeflate] added smiple support for libdeflate | This work was sponsored by SkySight (@plantain) --- Makefile | 12 +++--- mvt.cpp | 109 ++++++++++++++++++++++++++----------------------------- 2 files changed, 57 insertions(+), 64 deletions(-) diff --git a/Makefile b/Makefile index 24f150e..80f4426 100644 --- a/Makefile +++ b/Makefile @@ -48,22 +48,22 @@ INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-enumerate: enumerate.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 -ldeflate tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate unit: unit.o text.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate -include $(wildcard *.d) diff --git a/mvt.cpp b/mvt.cpp index 27486f8..b66e537 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include #include "mvt.hpp" #include "geometry.hpp" #include "protozero/varint.hpp" @@ -27,84 +27,77 @@ bool is_compressed(std::string const &data) { // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int decompress(std::string const &input, std::string &output) { - z_stream inflate_s; - inflate_s.zalloc = Z_NULL; - inflate_s.zfree = Z_NULL; - inflate_s.opaque = Z_NULL; - inflate_s.avail_in = 0; - inflate_s.next_in = Z_NULL; - if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { - fprintf(stderr, "Decompression error: %s\n", inflate_s.msg); - } - inflate_s.next_in = (Bytef *) input.data(); - inflate_s.avail_in = input.size(); - inflate_s.next_out = (Bytef *) output.data(); - inflate_s.avail_out = output.size(); - + /* + if (inflateInit2(inflate_s, 32 + 15) != Z_OK) { + fprintf(stderr, "Decompression error: %s\n", msg); + }*/ + struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); + void* next_in = (void*)input.data(); + size_t avail_in = input.size(); + void* next_out = (void*)output.data(); + size_t avail_out = output.size(); while (true) { - size_t existing_output = inflate_s.next_out - (Bytef *) output.data(); + long unsigned int existing_output; - output.resize(existing_output + 2 * inflate_s.avail_in + 100); - inflate_s.next_out = (Bytef *) output.data() + existing_output; - inflate_s.avail_out = output.size() - existing_output; + output.resize(existing_output + 2 * avail_in + 100); + next_out = (void*)(output.data() + existing_output); + avail_out = (output.size() - existing_output); - int ret = inflate(&inflate_s, 0); - if (ret < 0) { - fprintf(stderr, "Decompression error: "); - if (ret == Z_DATA_ERROR) { + int ret = libdeflate_deflate_decompress_ex(decompressor, + next_in, avail_in, + next_out, avail_out, + &existing_output, + &existing_output); + if (ret != LIBDEFLATE_SUCCESS) { + fprintf(stderr, "Decompression error: "); + if (ret == LIBDEFLATE_BAD_DATA) { fprintf(stderr, "data error"); - } - if (ret == Z_STREAM_ERROR) { - fprintf(stderr, "stream error"); - } - if (ret == Z_MEM_ERROR) { + } + if (ret == LIBDEFLATE_INSUFFICIENT_SPACE) { fprintf(stderr, "out of memory"); - } - if (ret == Z_BUF_ERROR) { + } + if (ret == LIBDEFLATE_SHORT_OUTPUT) { fprintf(stderr, "no data in buffer"); - } + } fprintf(stderr, "\n"); return 0; } - if (ret == Z_STREAM_END) { - break; - } + if (ret == LIBDEFLATE_SHORT_OUTPUT) { + break; + } // ret must be Z_OK or Z_NEED_DICT; // continue decompresing } - output.resize(inflate_s.next_out - (Bytef *) output.data()); - inflateEnd(&inflate_s); + output.resize(avail_out - output.size()); + libdeflate_free_decompressor(decompressor); return 1; } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int compress(std::string const &input, std::string &output) { - z_stream deflate_s; - deflate_s.zalloc = Z_NULL; - deflate_s.zfree = Z_NULL; - deflate_s.opaque = Z_NULL; - deflate_s.avail_in = 0; - deflate_s.next_in = Z_NULL; - deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); - deflate_s.next_in = (Bytef *) input.data(); - deflate_s.avail_in = input.size(); - size_t length = 0; - do { - size_t increase = input.size() / 2 + 1024; - output.resize(length + increase); - deflate_s.avail_out = increase; - deflate_s.next_out = (Bytef *) (output.data() + length); - int ret = deflate(&deflate_s, Z_FINISH); - if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + void* next_in = (void*)input.data(); + size_t avail_in = input.size(); + void* next_out = (void*)output.data(); + size_t avail_out = output.size(); + struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); + do { + size_t increase = input.size() / 2 + 1024; + output.resize(avail_in + increase); + avail_out = increase; + next_out = ((void*)output.data() + avail_in); + int ret = libdeflate_deflate_compress(deflate_s, + next_in, avail_in, + next_out, avail_out); + if (ret != LIBDEFLATE_SUCCESS) { return -1; - } - length += (increase - deflate_s.avail_out); - } while (deflate_s.avail_out == 0); - deflateEnd(&deflate_s); - output.resize(length); + } + avail_in += (increase - avail_out); + } while (avail_out == 0); + libdeflate_free_compressor(deflate_s); + output.resize(avail_in); return 0; } From a86e6cab6caafe27ffecdf5116c11ca79dd5069d Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 00:21:14 +0100 Subject: [PATCH 2/7] [libdeflate] added smiple support for libdeflate | This work was sponsored by SkySight (@plantain) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 80f4426..4ac9455 100644 --- a/Makefile +++ b/Makefile @@ -331,6 +331,7 @@ layer-json-test: # Use this target to regenerate the standards that the tests are compared against # after making a change that legitimately changes their output +# prep-test: $(TESTS) From 0d764c7603e610bbbdeb0a3f5e4eb905bd37dcde Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 21 Dec 2018 00:43:15 +0100 Subject: [PATCH 3/7] [libdeflate] added smiple support for libdeflate | This work was sponsored by SkySight (@plantain) --- .travis.yml | 1 + Makefile | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index f27b0b2..4a86f64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,7 @@ matrix: before_install: - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" - export PATH=${DEPS_DIR}/bin:${PATH} && mkdir -p ${DEPS_DIR} + - make -j12 libdeflate - | if [[ ${CLANG_VERSION:-false} != false ]]; then export CCOMPILER='clang' diff --git a/Makefile b/Makefile index 4ac9455..9a9d8ac 100644 --- a/Makefile +++ b/Makefile @@ -340,3 +340,8 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode ./tippecanoe-decode -x generator $@.check.mbtiles > $@ cmp $(patsubst %.check,%,$@) $@ rm $@.check.mbtiles + +libdeflate: + git clone https://github.com/ebiggers/libdeflate libdeflate + cd libdeflate/ + make -j12 install From dca42901b852426a738ab50ac3cfd783e2289a00 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Wed, 2 Jan 2019 12:49:43 +0100 Subject: [PATCH 4/7] [libdeflate] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9a9d8ac..90a1471 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ PG= H = $(wildcard *.h) $(wildcard *.hpp) C = $(wildcard *.c) $(wildcard *.cpp) -INCLUDES = -I/usr/local/include -I. +INCLUDES = -I/usr/local/include -Ilibdeflate -I. LIBS = -L/usr/local/lib tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o From 029333a398ac9109ae34985d244527bb850633df Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Wed, 2 Jan 2019 13:03:24 +0100 Subject: [PATCH 5/7] [libdeflate] changed pointer to void into long memory location --- mvt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvt.cpp b/mvt.cpp index b66e537..be1c894 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -87,7 +87,7 @@ int compress(std::string const &input, std::string &output) { size_t increase = input.size() / 2 + 1024; output.resize(avail_in + increase); avail_out = increase; - next_out = ((void*)output.data() + avail_in); + next_out = (void*)((long)output.data() + avail_in); int ret = libdeflate_deflate_compress(deflate_s, next_in, avail_in, next_out, avail_out); From 8c55d7c753563c39074ec848a5ac71badf8f3c30 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 18 Jan 2019 20:30:58 +0100 Subject: [PATCH 6/7] [SkySight] updated compress/decompress functions --- .gitmodules | 3 +++ Makefile | 21 ++++++++------------- libdeflate | 1 + 3 files changed, 12 insertions(+), 13 deletions(-) create mode 100644 .gitmodules create mode 160000 libdeflate diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..7fdc574 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libdeflate"] + path = libdeflate + url = https://github.com/ebiggers/libdeflate diff --git a/Makefile b/Makefile index 90a1471..35d16d4 100644 --- a/Makefile +++ b/Makefile @@ -44,26 +44,26 @@ PG= H = $(wildcard *.h) $(wildcard *.hpp) C = $(wildcard *.c) $(wildcard *.cpp) -INCLUDES = -I/usr/local/include -Ilibdeflate -I. -LIBS = -L/usr/local/lib +INCLUDES = -I/usr/local/include -I/usr/include -I. +LIBS = -L/usr/local/lib -L/usr/lib -L. tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-enumerate: enumerate.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 -ldeflate + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread unit: unit.o text.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -include $(wildcard *.d) @@ -340,8 +340,3 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode ./tippecanoe-decode -x generator $@.check.mbtiles > $@ cmp $(patsubst %.check,%,$@) $@ rm $@.check.mbtiles - -libdeflate: - git clone https://github.com/ebiggers/libdeflate libdeflate - cd libdeflate/ - make -j12 install diff --git a/libdeflate b/libdeflate new file mode 160000 index 0000000..cb7ee82 --- /dev/null +++ b/libdeflate @@ -0,0 +1 @@ +Subproject commit cb7ee82c22212d8830282b17c5e523c45be2c7e0 From 069ced973aeef6ca8ff4e930b7a998f12fa9f1a1 Mon Sep 17 00:00:00 2001 From: Ilia Platone Date: Fri, 18 Jan 2019 20:49:48 +0100 Subject: [PATCH 7/7] [SkySight] updated compress/decompress functions --- mvt.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mvt.cpp b/mvt.cpp index be1c894..7f12f04 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -79,25 +79,26 @@ int decompress(std::string const &input, std::string &output) { // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int compress(std::string const &input, std::string &output) { void* next_in = (void*)input.data(); - size_t avail_in = input.size(); void* next_out = (void*)output.data(); - size_t avail_out = output.size(); + size_t output_size = 0; + size_t input_size = 0; + size_t increase = 1024; struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); do { - size_t increase = input.size() / 2 + 1024; - output.resize(avail_in + increase); - avail_out = increase; - next_out = (void*)((long)output.data() + avail_in); + output.resize(output.size() + increase); int ret = libdeflate_deflate_compress(deflate_s, - next_in, avail_in, - next_out, avail_out); + next_in, increase, + next_out, increase); if (ret != LIBDEFLATE_SUCCESS) { return -1; } - avail_in += (increase - avail_out); - } while (avail_out == 0); + output_size += ret; + input_size += increase; + next_out = (void*)((long)next_out + ret); + next_in = (void*)((long)next_in + increase); + } while ((long)next_in < input.size()); libdeflate_free_compressor(deflate_s); - output.resize(avail_in); + output.resize(output_size); return 0; }