From b91e8f6d3ef04434edfca5e0bd65a19c12bd425e Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 22 Apr 2016 12:00:03 -0700 Subject: [PATCH] Start factoring out protocol buffer handling code --- Makefile | 6 ++--- decode.cc | 41 +---------------------------- protobuf.cc | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++ protobuf.hh | 4 +++ tile-join.cc | 65 +-------------------------------------------- tile.cc | 29 +------------------- 6 files changed, 84 insertions(+), 135 deletions(-) create mode 100644 protobuf.cc create mode 100644 protobuf.hh diff --git a/Makefile b/Makefile index 37c23d8..26181d1 100644 --- a/Makefile +++ b/Makefile @@ -35,16 +35,16 @@ C = $(shell find . '(' -name '*.c' -o -name '*.cc' ')') INCLUDES = -I/usr/local/include LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull.o vector_tile.pb.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o +tippecanoe: geojson.o jsonpull.o vector_tile.pb.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o protobuf.o $(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lprotobuf-lite -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o $(CC) $(PG) $(LIBS) -O3 -g -Wall $(CFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 -tippecanoe-decode: decode.o vector_tile.pb.o projection.o +tippecanoe-decode: decode.o vector_tile.pb.o projection.o protobuf.o $(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lprotobuf-lite -lsqlite3 -tile-join: tile-join.o vector_tile.pb.o projection.o pool.o mbtiles.o +tile-join: tile-join.o vector_tile.pb.o projection.o pool.o mbtiles.o protobuf.o $(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lprotobuf-lite -lsqlite3 libjsonpull.a: jsonpull.o diff --git a/decode.cc b/decode.cc index 5905e93..ec34b64 100644 --- a/decode.cc +++ b/decode.cc @@ -10,6 +10,7 @@ #include #include #include +#include "protobuf.hh" #include "vector_tile.pb.h" #include "tile.h" @@ -17,46 +18,6 @@ extern "C" { #include "projection.h" } -// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp -inline bool is_compressed(std::string const &data) { - return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); -} - -// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp -inline int decompress(std::string const &input, std::string &output) { - z_stream inflate_s; - inflate_s.zalloc = Z_NULL; - inflate_s.zfree = Z_NULL; - inflate_s.opaque = Z_NULL; - inflate_s.avail_in = 0; - inflate_s.next_in = Z_NULL; - if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { - fprintf(stderr, "error: %s\n", inflate_s.msg); - } - inflate_s.next_in = (Bytef *) input.data(); - inflate_s.avail_in = input.size(); - size_t length = 0; - do { - output.resize(length + 2 * input.size()); - inflate_s.avail_out = 2 * input.size(); - inflate_s.next_out = (Bytef *) (output.data() + length); - int ret = inflate(&inflate_s, Z_FINISH); - if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { - fprintf(stderr, "error: %s\n", inflate_s.msg); - return 0; - } - - length += (2 * input.size() - inflate_s.avail_out); - } while (inflate_s.avail_out == 0); - inflateEnd(&inflate_s); - output.resize(length); - return 1; -} - -int dezig(unsigned n) { - return (n >> 1) ^ (-(n & 1)); -} - void printq(const char *s) { putchar('"'); for (; *s; s++) { diff --git a/protobuf.cc b/protobuf.cc new file mode 100644 index 0000000..19bae74 --- /dev/null +++ b/protobuf.cc @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include "protobuf.hh" +#include "vector_tile.pb.h" + +// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp +bool is_compressed(std::string const &data) { + return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); +} + +// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp +int decompress(std::string const &input, std::string &output) { + z_stream inflate_s; + inflate_s.zalloc = Z_NULL; + inflate_s.zfree = Z_NULL; + inflate_s.opaque = Z_NULL; + inflate_s.avail_in = 0; + inflate_s.next_in = Z_NULL; + if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { + fprintf(stderr, "error: %s\n", inflate_s.msg); + } + inflate_s.next_in = (Bytef *) input.data(); + inflate_s.avail_in = input.size(); + size_t length = 0; + do { + output.resize(length + 2 * input.size()); + inflate_s.avail_out = 2 * input.size(); + inflate_s.next_out = (Bytef *) (output.data() + length); + int ret = inflate(&inflate_s, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + fprintf(stderr, "error: %s\n", inflate_s.msg); + return 0; + } + + length += (2 * input.size() - inflate_s.avail_out); + } while (inflate_s.avail_out == 0); + inflateEnd(&inflate_s); + output.resize(length); + return 1; +} + +// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp +int compress(std::string const &input, std::string &output) { + z_stream deflate_s; + deflate_s.zalloc = Z_NULL; + deflate_s.zfree = Z_NULL; + deflate_s.opaque = Z_NULL; + deflate_s.avail_in = 0; + deflate_s.next_in = Z_NULL; + deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); + deflate_s.next_in = (Bytef *) input.data(); + deflate_s.avail_in = input.size(); + size_t length = 0; + do { + size_t increase = input.size() / 2 + 1024; + output.resize(length + increase); + deflate_s.avail_out = increase; + deflate_s.next_out = (Bytef *) (output.data() + length); + int ret = deflate(&deflate_s, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + return -1; + } + length += (increase - deflate_s.avail_out); + } while (deflate_s.avail_out == 0); + deflateEnd(&deflate_s); + output.resize(length); + return 0; +} + +int dezig(unsigned n) { + return (n >> 1) ^ (-(n & 1)); +} diff --git a/protobuf.hh b/protobuf.hh new file mode 100644 index 0000000..541ac42 --- /dev/null +++ b/protobuf.hh @@ -0,0 +1,4 @@ +bool is_compressed(std::string const &data); +int decompress(std::string const &input, std::string &output); +int compress(std::string const &input, std::string &output); +int dezig(unsigned n); diff --git a/tile-join.cc b/tile-join.cc index 8eedd8d..32719d6 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -8,6 +8,7 @@ #include #include #include +#include "protobuf.hh" #include "vector_tile.pb.h" #include "tile.h" @@ -26,70 +27,6 @@ struct stats { double minlat, minlon, maxlat, maxlon; }; -// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp -inline bool is_compressed(std::string const &data) { - return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); -} - -// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp -inline int decompress(std::string const &input, std::string &output) { - z_stream inflate_s; - inflate_s.zalloc = Z_NULL; - inflate_s.zfree = Z_NULL; - inflate_s.opaque = Z_NULL; - inflate_s.avail_in = 0; - inflate_s.next_in = Z_NULL; - if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { - fprintf(stderr, "error: %s\n", inflate_s.msg); - } - inflate_s.next_in = (Bytef *) input.data(); - inflate_s.avail_in = input.size(); - size_t length = 0; - do { - output.resize(length + 2 * input.size()); - inflate_s.avail_out = 2 * input.size(); - inflate_s.next_out = (Bytef *) (output.data() + length); - int ret = inflate(&inflate_s, Z_FINISH); - if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { - fprintf(stderr, "error: %s\n", inflate_s.msg); - return 0; - } - - length += (2 * input.size() - inflate_s.avail_out); - } while (inflate_s.avail_out == 0); - inflateEnd(&inflate_s); - output.resize(length); - return 1; -} - -// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp -static inline int compress(std::string const &input, std::string &output) { - z_stream deflate_s; - deflate_s.zalloc = Z_NULL; - deflate_s.zfree = Z_NULL; - deflate_s.opaque = Z_NULL; - deflate_s.avail_in = 0; - deflate_s.next_in = Z_NULL; - deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); - deflate_s.next_in = (Bytef *) input.data(); - deflate_s.avail_in = input.size(); - size_t length = 0; - do { - size_t increase = input.size() / 2 + 1024; - output.resize(length + increase); - deflate_s.avail_out = increase; - deflate_s.next_out = (Bytef *) (output.data() + length); - int ret = deflate(&deflate_s, Z_FINISH); - if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { - return -1; - } - length += (increase - deflate_s.avail_out); - } while (deflate_s.avail_out == 0); - deflateEnd(&deflate_s); - output.resize(length); - return 0; -} - void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, std::vector &header, std::map > &mapping, struct pool *exclude, int ifmatched) { GOOGLE_PROTOBUF_VERIFY_VERSION; diff --git a/tile.cc b/tile.cc index ff3cf46..352bc38 100644 --- a/tile.cc +++ b/tile.cc @@ -18,6 +18,7 @@ #include #include #include +#include "protobuf.hh" #include "vector_tile.pb.h" #include "geometry.hh" @@ -37,34 +38,6 @@ extern "C" { pthread_mutex_t db_lock = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t var_lock = PTHREAD_MUTEX_INITIALIZER; -// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp -static inline int compress(std::string const &input, std::string &output) { - z_stream deflate_s; - deflate_s.zalloc = Z_NULL; - deflate_s.zfree = Z_NULL; - deflate_s.opaque = Z_NULL; - deflate_s.avail_in = 0; - deflate_s.next_in = Z_NULL; - deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); - deflate_s.next_in = (Bytef *) input.data(); - deflate_s.avail_in = input.size(); - size_t length = 0; - do { - size_t increase = input.size() / 2 + 1024; - output.resize(length + increase); - deflate_s.avail_out = increase; - deflate_s.next_out = (Bytef *) (output.data() + length); - int ret = deflate(&deflate_s, Z_FINISH); - if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { - return -1; - } - length += (increase - deflate_s.avail_out); - } while (deflate_s.avail_out == 0); - deflateEnd(&deflate_s); - output.resize(length); - return 0; -} - int to_feature(drawvec &geom, mapnik::vector::tile_feature *feature) { int px = 0, py = 0; int cmd_idx = -1;