Use protozero for writing tiles

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.9.16
+
+* Switch to protozero as the library for reading and writing protocol buffers
+
 ## 1.9.15
 
 * Add option not to clip features

Makefile

@@ -24,9 +24,6 @@ install: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join
 man/tippecanoe.1: README.md
 	md2man-roff README.md > man/tippecanoe.1
 
-vector_tile.pb.cc vector_tile.pb.h: vector_tile.proto
-	protoc --cpp_out=. vector_tile.proto
-
 PG=
 
 H = $(shell find . '(' -name '*.h' -o -name '*.hh' ')')
@@ -35,16 +32,16 @@ C = $(shell find . '(' -name '*.c' -o -name '*.cc' ')')
 INCLUDES = -I/usr/local/include -I.
 LIBS = -L/usr/local/lib
 
-tippecanoe: geojson.o jsonpull.o vector_tile.pb.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o mvt.o
+tippecanoe: geojson.o jsonpull.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o mvt.o
 	$(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lprotobuf-lite -lsqlite3 -lpthread
 
 tippecanoe-enumerate: enumerate.o
 	$(CC) $(PG) $(LIBS) -O3 -g -Wall $(CFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3
 
-tippecanoe-decode: decode.o vector_tile.pb.o projection.o mvt.o
+tippecanoe-decode: decode.o projection.o mvt.o
 	$(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lprotobuf-lite -lsqlite3
 
-tile-join: tile-join.o vector_tile.pb.o projection.o pool.o mbtiles.o mvt.o
+tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o
 	$(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lprotobuf-lite -lsqlite3
 
 libjsonpull.a: jsonpull.o

README.md

@@ -231,16 +231,12 @@ lower resolutions before failing if it still doesn't fit.
 Development
 -----------
 
-Requires protoc and sqlite3. Rebuilding the manpage
+Requires sqlite3 (should already be installed on MacOS). Rebuilding the manpage
 uses md2man (`gem install md2man`).
 
-MacOS:
-
-    brew install protobuf
-
 Linux:
 
-    sudo apt-get install libprotobuf-dev protobuf-compiler libsqlite3-dev
+    sudo apt-get install libsqlite3-dev
 
 Then build:
 

man/tippecanoe.1

@@ -267,22 +267,14 @@ If a tile is larger than 500K, it will try encoding that tile at progressively
 lower resolutions before failing if it still doesn't fit.
 .SH Development
 .PP
-Requires protoc and sqlite3. Rebuilding the manpage
+Requires sqlite3 (should already be installed on MacOS). Rebuilding the manpage
 uses md2man (\fB\fCgem install md2man\fR).
 .PP
-MacOS:
-.PP
-.RS
-.nf
-brew install protobuf
-.fi
-.RE
-.PP
 Linux:
 .PP
 .RS
 .nf
-sudo apt\-get install libprotobuf\-dev protobuf\-compiler libsqlite3\-dev
+sudo apt\-get install libsqlite3\-dev
 .fi
 .RE
 .PP

mvt.cc

@@ -2,11 +2,9 @@
 #include <string>
 #include <vector>
 #include <zlib.h>
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-#include <google/protobuf/io/coded_stream.h>
 #include "mvt.hh"
-#include "vector_tile.pb.h"
 #include "protozero/pbf_reader.hpp"
+#include "protozero/pbf_writer.hpp"
 
 // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
 bool is_compressed(std::string const &data) {
@@ -237,63 +235,54 @@ bool mvt_decode(std::string &message, mvt_tile &out) {
 }
 
 std::string mvt_encode(mvt_tile &in) {
-	GOOGLE_PROTOBUF_VERIFY_VERSION;
+	std::string data;
 
-	mapnik::vector::tile tile;
+	protozero::pbf_writer writer(data);
 
 	for (size_t i = 0; i < in.layers.size(); i++) {
-		mapnik::vector::tile_layer *layer = tile.add_layers();
+		std::string layer_string;
+		protozero::pbf_writer layer_writer(layer_string);
 
-		layer->set_name(in.layers[i].name);
+		layer_writer.add_uint32(15, 1); /* version */
-		layer->set_version(in.layers[i].version);
+		layer_writer.add_string(1, in.layers[i].name); /* name */
-		layer->set_extent(in.layers[i].extent);
+		layer_writer.add_uint32(5, in.layers[i].extent); /* extent */
 
-		for (size_t k = 0; k < in.layers[i].keys.size(); k++) {
+		for (size_t j = 0; j < in.layers[i].keys.size(); j++) {
-			layer->add_keys(in.layers[i].keys[k]);
+			layer_writer.add_string(3, in.layers[i].keys[j]); /* key */
 		}
+
 		for (size_t v = 0; v < in.layers[i].values.size(); v++) {
-			mapnik::vector::tile_value *tv = layer->add_values();
+			std::string value_string;
+			protozero::pbf_writer value_writer(value_string);
 			mvt_value &pbv = in.layers[i].values[v];
 
 			if (pbv.type == mvt_string) {
-				tv->set_string_value(pbv.string_value);
+				value_writer.add_string(1, pbv.string_value);
 			} else if (pbv.type == mvt_float) {
-				tv->set_float_value(pbv.numeric_value.float_value);
+				value_writer.add_float(2, pbv.numeric_value.float_value);
 			} else if (pbv.type == mvt_double) {
-				tv->set_double_value(pbv.numeric_value.double_value);
+				value_writer.add_double(3, pbv.numeric_value.double_value);
 			} else if (pbv.type == mvt_int) {
-				tv->set_int_value(pbv.numeric_value.int_value);
+				value_writer.add_int64(4, pbv.numeric_value.int_value);
 			} else if (pbv.type == mvt_uint) {
-				tv->set_uint_value(pbv.numeric_value.uint_value);
+				value_writer.add_uint64(5, pbv.numeric_value.uint_value);
 			} else if (pbv.type == mvt_sint) {
-				tv->set_sint_value(pbv.numeric_value.sint_value);
+				value_writer.add_sint64(6, pbv.numeric_value.sint_value);
 			} else if (pbv.type == mvt_bool) {
-				tv->set_bool_value(pbv.numeric_value.bool_value);
+				value_writer.add_bool(7, pbv.numeric_value.bool_value);
 			}
+
+			layer_writer.add_message(4, value_string);
 		}
 
 		for (size_t f = 0; f < in.layers[i].features.size(); f++) {
-			mapnik::vector::tile_feature *feature = layer->add_features();
+			std::string feature_string;
-			if (feature == NULL) {
+			protozero::pbf_writer feature_writer(feature_string);
-				perror("add feature");
-				exit(EXIT_FAILURE);
-			}
 
-			int type = in.layers[i].features[f].type;
+			feature_writer.add_enum(3, in.layers[i].features[f].type);
-			if (type == mvt_point) {
+			feature_writer.add_packed_uint32(2, std::begin(in.layers[i].features[f].tags), std::end(in.layers[i].features[f].tags));
-				feature->set_type(mapnik::vector::tile::Point);
-			} else if (type == mvt_linestring) {
-				feature->set_type(mapnik::vector::tile::LineString);
-			} else if (type == mvt_polygon) {
-				feature->set_type(mapnik::vector::tile::Polygon);
-			} else {
-				fprintf(stderr, "Corrupt geometry type\n");
-				exit(EXIT_FAILURE);
-			}
 
-			for (size_t t = 0; t < in.layers[i].features[f].tags.size(); t++) {
+			std::vector<uint32_t> geometry;
-				feature->add_tags(in.layers[i].features[f].tags[t]);
-			}
 
 			int px = 0, py = 0;
 			int cmd_idx = -1;
@@ -307,13 +296,13 @@ std::string mvt_encode(mvt_tile &in) {
 
 				if (op != cmd) {
 					if (cmd_idx >= 0) {
-						feature->set_geometry(cmd_idx, (length << 3) | (cmd & ((1 << 3) - 1)));
+						geometry[cmd_idx] = (length << 3) | (cmd & ((1 << 3) - 1));
 					}
 
 					cmd = op;
 					length = 0;
-					cmd_idx = feature->geometry_size();
+					cmd_idx = geometry.size();
-					feature->add_geometry(0);
+					geometry.push_back(0);
 				}
 
 				if (op == mvt_moveto || op == mvt_lineto) {
@@ -323,10 +312,8 @@ std::string mvt_encode(mvt_tile &in) {
 					int dx = wwx - px;
 					int dy = wwy - py;
 
-					if (feature != NULL) {
+					geometry.push_back((dx << 1) ^ (dx >> 31));
-						feature->add_geometry((dx << 1) ^ (dx >> 31));
+					geometry.push_back((dy << 1) ^ (dy >> 31));
-						feature->add_geometry((dy << 1) ^ (dy >> 31));
-					}
 
 					px = wwx;
 					py = wwy;
@@ -340,14 +327,18 @@ std::string mvt_encode(mvt_tile &in) {
 			}
 
 			if (cmd_idx >= 0) {
-				feature->set_geometry(cmd_idx, (length << 3) | (cmd & ((1 << 3) - 1)));
+				geometry[cmd_idx] = (length << 3) | (cmd & ((1 << 3) - 1));
-			}
-		}
 			}
 
-	std::string s, compressed;
+			feature_writer.add_packed_uint32(4, std::begin(geometry), std::end(geometry));
-	tile.SerializeToString(&s);
+			layer_writer.add_message(2, feature_string);
-	compress(s, compressed);
+		}
+
+		writer.add_message(3, layer_string);
+	}
+
+	std::string compressed;
+	compress(data, compressed);
 
 	return compressed;
 }

version.h

@@ -1 +1 @@
-#define VERSION "tippecanoe v1.9.15\n"
+#define VERSION "tippecanoe v1.9.16\n"