diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..7fdc574 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libdeflate"] + path = libdeflate + url = https://github.com/ebiggers/libdeflate diff --git a/.travis.yml b/.travis.yml index f27b0b2..4a86f64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,7 @@ matrix: before_install: - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps" - export PATH=${DEPS_DIR}/bin:${PATH} && mkdir -p ${DEPS_DIR} + - make -j12 libdeflate - | if [[ ${CLANG_VERSION:-false} != false ]]; then export CCOMPILER='clang' diff --git a/Makefile b/Makefile index ae542a1..4aed36b 100644 --- a/Makefile +++ b/Makefile @@ -44,26 +44,26 @@ PG= H = $(wildcard *.h) $(wildcard *.hpp) C = $(wildcard *.c) $(wildcard *.cpp) -INCLUDES = -I/usr/local/include -I. -LIBS = -L/usr/local/lib +INCLUDES = -I/usr/local/include -I/usr/include -I. +LIBS = -L/usr/local/lib -L/usr/lib -L. tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-enumerate: enumerate.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -ldeflate tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -ldeflate tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread unit: unit.o text.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -include $(wildcard *.d) @@ -338,6 +338,7 @@ layer-json-test: # Use this target to regenerate the standards that the tests are compared against # after making a change that legitimately changes their output +# prep-test: $(TESTS) diff --git a/libdeflate b/libdeflate new file mode 160000 index 0000000..cb7ee82 --- /dev/null +++ b/libdeflate @@ -0,0 +1 @@ +Subproject commit cb7ee82c22212d8830282b17c5e523c45be2c7e0 diff --git a/mvt.cpp b/mvt.cpp index 27486f8..7f12f04 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include #include "mvt.hpp" #include "geometry.hpp" #include "protozero/varint.hpp" @@ -27,84 +27,78 @@ bool is_compressed(std::string const &data) { // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int decompress(std::string const &input, std::string &output) { - z_stream inflate_s; - inflate_s.zalloc = Z_NULL; - inflate_s.zfree = Z_NULL; - inflate_s.opaque = Z_NULL; - inflate_s.avail_in = 0; - inflate_s.next_in = Z_NULL; - if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { - fprintf(stderr, "Decompression error: %s\n", inflate_s.msg); - } - inflate_s.next_in = (Bytef *) input.data(); - inflate_s.avail_in = input.size(); - inflate_s.next_out = (Bytef *) output.data(); - inflate_s.avail_out = output.size(); - + /* + if (inflateInit2(inflate_s, 32 + 15) != Z_OK) { + fprintf(stderr, "Decompression error: %s\n", msg); + }*/ + struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); + void* next_in = (void*)input.data(); + size_t avail_in = input.size(); + void* next_out = (void*)output.data(); + size_t avail_out = output.size(); while (true) { - size_t existing_output = inflate_s.next_out - (Bytef *) output.data(); + long unsigned int existing_output; - output.resize(existing_output + 2 * inflate_s.avail_in + 100); - inflate_s.next_out = (Bytef *) output.data() + existing_output; - inflate_s.avail_out = output.size() - existing_output; + output.resize(existing_output + 2 * avail_in + 100); + next_out = (void*)(output.data() + existing_output); + avail_out = (output.size() - existing_output); - int ret = inflate(&inflate_s, 0); - if (ret < 0) { - fprintf(stderr, "Decompression error: "); - if (ret == Z_DATA_ERROR) { + int ret = libdeflate_deflate_decompress_ex(decompressor, + next_in, avail_in, + next_out, avail_out, + &existing_output, + &existing_output); + if (ret != LIBDEFLATE_SUCCESS) { + fprintf(stderr, "Decompression error: "); + if (ret == LIBDEFLATE_BAD_DATA) { fprintf(stderr, "data error"); - } - if (ret == Z_STREAM_ERROR) { - fprintf(stderr, "stream error"); - } - if (ret == Z_MEM_ERROR) { + } + if (ret == LIBDEFLATE_INSUFFICIENT_SPACE) { fprintf(stderr, "out of memory"); - } - if (ret == Z_BUF_ERROR) { + } + if (ret == LIBDEFLATE_SHORT_OUTPUT) { fprintf(stderr, "no data in buffer"); - } + } fprintf(stderr, "\n"); return 0; } - if (ret == Z_STREAM_END) { - break; - } + if (ret == LIBDEFLATE_SHORT_OUTPUT) { + break; + } // ret must be Z_OK or Z_NEED_DICT; // continue decompresing } - output.resize(inflate_s.next_out - (Bytef *) output.data()); - inflateEnd(&inflate_s); + output.resize(avail_out - output.size()); + libdeflate_free_decompressor(decompressor); return 1; } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int compress(std::string const &input, std::string &output) { - z_stream deflate_s; - deflate_s.zalloc = Z_NULL; - deflate_s.zfree = Z_NULL; - deflate_s.opaque = Z_NULL; - deflate_s.avail_in = 0; - deflate_s.next_in = Z_NULL; - deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); - deflate_s.next_in = (Bytef *) input.data(); - deflate_s.avail_in = input.size(); - size_t length = 0; - do { - size_t increase = input.size() / 2 + 1024; - output.resize(length + increase); - deflate_s.avail_out = increase; - deflate_s.next_out = (Bytef *) (output.data() + length); - int ret = deflate(&deflate_s, Z_FINISH); - if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + void* next_in = (void*)input.data(); + void* next_out = (void*)output.data(); + size_t output_size = 0; + size_t input_size = 0; + size_t increase = 1024; + struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); + do { + output.resize(output.size() + increase); + int ret = libdeflate_deflate_compress(deflate_s, + next_in, increase, + next_out, increase); + if (ret != LIBDEFLATE_SUCCESS) { return -1; - } - length += (increase - deflate_s.avail_out); - } while (deflate_s.avail_out == 0); - deflateEnd(&deflate_s); - output.resize(length); + } + output_size += ret; + input_size += increase; + next_out = (void*)((long)next_out + ret); + next_in = (void*)((long)next_in + increase); + } while ((long)next_in < input.size()); + libdeflate_free_compressor(deflate_s); + output.resize(output_size); return 0; }