Merge pull request #416 from mapbox/tile-join-compression

Tile-join had not been compressing its tiles.

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.18.0
+
+* Fix compression of tiles in tile-join
+* Calculate the tileset bounding box in tile-join from the tile boundaries
+
 ## 1.17.7
 
 * Enforce polygon winding and closure rules in tippecanoe-decode

decode.cpp

@@ -54,9 +54,10 @@ struct lonlat {
 void handle(std::string message, int z, unsigned x, unsigned y, int describe, std::set<std::string> const &to_decode) {
 	int within = 0;
 	mvt_tile tile;
+	bool was_compressed;
 
 	try {
-		if (!tile.decode(message)) {
+		if (!tile.decode(message, was_compressed)) {
 			fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y);
 			exit(EXIT_FAILURE);
 		}
@@ -68,7 +69,12 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe, st
 	printf("{ \"type\": \"FeatureCollection\"");
 
 	if (describe) {
-		printf(", \"properties\": { \"zoom\": %d, \"x\": %d, \"y\": %d }", z, x, y);
+		printf(", \"properties\": { \"zoom\": %d, \"x\": %d, \"y\": %d", z, x, y);
+		if (!was_compressed) {
+			printf(", \"compressed\": false");
+		}
+
+		printf(" }");
 
 		if (projection != projections) {
 			printf(", \"crs\": { \"type\": \"name\", \"properties\": { \"name\": ");

mvt.cpp

@@ -79,7 +79,7 @@ int compress(std::string const &input, std::string &output) {
 	return 0;
 }
 
-bool mvt_tile::decode(std::string &message) {
+bool mvt_tile::decode(std::string &message, bool &was_compressed) {
 	layers.clear();
 	std::string src;
 
@@ -87,8 +87,10 @@ bool mvt_tile::decode(std::string &message) {
 		std::string uncompressed;
 		decompress(message, uncompressed);
 		src = uncompressed;
+		was_compressed = true;
 	} else {
 		src = message;
+		was_compressed = false;
 	}
 
 	protozero::pbf_reader reader(src);

mvt.hpp

@@ -91,7 +91,7 @@ struct mvt_tile {
 	std::vector<mvt_layer> layers;
 
 	std::string encode();
-	bool decode(std::string &message);
+	bool decode(std::string &message, bool &was_compressed);
 };
 
 bool is_compressed(std::string const &data);

tests/join-population/joined-i.mbtiles.json

@@ -1,5 +1,5 @@
 { "type": "FeatureCollection", "properties": {
-"bounds": "-122.334735,37.877356,-122.281639,37.898925",
+"bounds": "-122.343750,37.857507,-122.255859,37.926868",
 "center": "-122.299805,37.892187,12",
 "description": "tests/join-population/tabblock_06001420.mbtiles",
 "format": "pbf",

tests/join-population/joined.mbtiles.json

@@ -1,5 +1,5 @@
 { "type": "FeatureCollection", "properties": {
-"bounds": "-122.334735,37.877356,-122.281639,37.898925",
+"bounds": "-122.343750,37.857507,-122.255859,37.926868",
 "center": "-122.299805,37.892187,12",
 "description": "tests/join-population/tabblock_06001420.mbtiles",
 "format": "pbf",

tests/join-population/just-macarthur.mbtiles.json

@@ -1,6 +1,6 @@
 { "type": "FeatureCollection", "properties": {
 "attribution": "macarthur attribution",
-"bounds": "-122.334735,37.695438,-122.104097,37.898925",
+"bounds": "-122.343750,37.695438,-122.104097,37.926868",
 "center": "-122.299805,37.892187,12",
 "description": "macarthur description",
 "format": "pbf",

tests/join-population/merged.mbtiles.json

@@ -1,5 +1,5 @@
 { "type": "FeatureCollection", "properties": {
-"bounds": "-122.334735,37.695438,-122.104097,37.898925",
+"bounds": "-122.343750,37.695438,-122.104097,37.926868",
 "center": "-122.299805,37.892187,12",
 "description": "tests/join-population/tabblock_06001420.mbtiles",
 "format": "pbf",

tests/join-population/no-macarthur.mbtiles.json

@@ -1,5 +1,5 @@
 { "type": "FeatureCollection", "properties": {
-"bounds": "-122.334735,37.695438,-122.104097,37.898925",
+"bounds": "-122.343750,37.695438,-122.104097,37.926868",
 "center": "-122.299805,37.892187,12",
 "description": "tests/join-population/tabblock_06001420.mbtiles",
 "format": "pbf",

tests/join-population/windows.mbtiles.json

@@ -1,5 +1,5 @@
 { "type": "FeatureCollection", "properties": {
-"bounds": "-122.294563,37.695438,-122.104097,37.833010",
+"bounds": "-122.343750,37.439974,-121.992188,37.996163",
 "center": "-122.167969,37.833010,10",
 "description": "tests/join-population/macarthur.mbtiles",
 "format": "pbf",

tile-join.cpp

@@ -32,8 +32,9 @@ struct stats {
 void handle(std::string message, int z, unsigned x, unsigned y, std::map<std::string, layermap_entry> &layermap, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, int ifmatched, mvt_tile &outtile) {
 	mvt_tile tile;
 	int features_added = 0;
+	bool was_compressed;
 
-	if (!tile.decode(message)) {
+	if (!tile.decode(message, was_compressed)) {
 		fprintf(stderr, "Couldn't decompress tile %d/%u/%u\n", z, x, y);
 		exit(EXIT_FAILURE);
 	}
@@ -402,7 +403,9 @@ void *join_worker(void *v) {
 		}
 
 		if (anything) {
-			std::string compressed = tile.encode();
+			std::string pbf = tile.encode();
+			std::string compressed;
+			compress(pbf, compressed);
 
 			if (!pk && compressed.size() > 500000) {
 				fprintf(stderr, "Tile %lld/%lld/%lld size is %lld, >500000. Skipping this tile\n.", ai->first.z, ai->first.x, ai->first.y, (long long) compressed.size());
@@ -472,11 +475,34 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 
 	std::map<zxy, std::vector<std::string>> tasks;
 
+	double minlat = INT_MAX;
+	double minlon = INT_MAX;
+	double maxlat = INT_MIN;
+	double maxlon = INT_MIN;
+	int zoom_for_bbox = -1;
+
 	while (readers != NULL && readers->zoom < 32) {
 		reader *r = readers;
 		readers = readers->next;
 		r->next = NULL;
 
+		if (r->zoom != zoom_for_bbox) {
+			// Only use highest zoom for bbox calculation
+			// to avoid z0 always covering the world
+
+			minlat = minlon = INT_MAX;
+			maxlat = maxlon = INT_MIN;
+			zoom_for_bbox = r->zoom;
+		}
+
+		double lat1, lon1, lat2, lon2;
+		tile2lonlat(r->x, r->y, r->zoom, &lon1, &lat1);
+		tile2lonlat(r->x + 1, r->y + 1, r->zoom, &lon2, &lat2);
+		minlat = min(lat2, minlat);
+		minlon = min(lon1, minlon);
+		maxlat = max(lat1, maxlat);
+		maxlon = max(lon2, maxlon);
+
 		zxy tile = zxy(r->zoom, r->x, r->y);
 		if (tasks.count(tile) == 0) {
 			tasks.insert(std::pair<zxy, std::vector<std::string>>(tile, std::vector<std::string>()));
@@ -517,6 +543,11 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 		*rr = r;
 	}
 
+	st->minlon = min(minlon, st->minlon);
+	st->maxlon = max(maxlon, st->maxlon);
+	st->minlat = min(minlat, st->minlat);
+	st->maxlat = max(maxlat, st->maxlat);
+
 	handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
 	layermap = merge_layermaps(layermaps);
 
@@ -571,12 +602,12 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 		if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'bounds'", -1, &r->stmt, NULL) == SQLITE_OK) {
 			if (sqlite3_step(r->stmt) == SQLITE_ROW) {
 				const unsigned char *s = sqlite3_column_text(r->stmt, 0);
-				double minlon, minlat, maxlon, maxlat;
-				sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat);
-				st->minlon = min(minlon, st->minlon);
-				st->maxlon = max(maxlon, st->maxlon);
-				st->minlat = min(minlat, st->minlat);
-				st->maxlat = max(maxlat, st->maxlat);
+				if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) {
+					st->minlon = min(minlon, st->minlon);
+					st->maxlon = max(maxlon, st->maxlon);
+					st->minlat = min(minlat, st->minlat);
+					st->maxlat = max(maxlat, st->maxlat);
+				}
 			}
 			sqlite3_finalize(r->stmt);
 		}

version.hpp

@@ -1 +1 @@
-#define VERSION "tippecanoe v1.17.7\n"
+#define VERSION "tippecanoe v1.18.0\n"