Decode polygons, although probably not correctly

Work from the top down instead of from feature indices

Makefile

@@ -3,6 +3,7 @@ PREFIX ?= /usr/local
 all: tippecanoe enumerate decode
 
 install: tippecanoe
+	mkdir -p $(PREFIX)/bin
 	cp tippecanoe $(PREFIX)/bin/tippecanoe
 
 vector_tile.pb.cc vector_tile.pb.h: vector_tile.proto

README.md

@@ -3,6 +3,13 @@ tippecanoe
 
 Build vector tilesets from large collections of GeoJSON features.
 
+Installation
+------------
+
+The easiest way to install tippecanoe on OSX is with [Homebrew](http://brew.sh/):
+
+    brew install tippecanoe
+
 Usage
 -----
 

enumerate.c

@@ -24,7 +24,7 @@ void enumerate(char *fname) {
 		long long x = sqlite3_column_int(stmt, 1);
 		long long y = sqlite3_column_int(stmt, 2);
 
-		y = (1LL << zoom) - y;
+		y = (1LL << zoom) - 1 - y;
 		printf("%s %lld %lld %lld\n", fname, zoom, x, y);
 	}
 

geometry.cc

@@ -8,6 +8,7 @@
 #include <unistd.h>
 #include <math.h>
 #include <sqlite3.h>
+#include <limits.h>
 #include "geometry.hh"
 
 extern "C" {
@@ -16,9 +17,14 @@ extern "C" {
 	#include "projection.h"
 }
 
-drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail) {
+drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox) {
 	drawvec out;
 
+	bbox[0] = LONG_LONG_MAX;
+	bbox[1] = LONG_LONG_MAX;
+	bbox[2] = LONG_LONG_MIN;
+	bbox[3] = LONG_LONG_MIN;
+
 	while (1) {
 		draw d;
 
@@ -28,9 +34,9 @@ drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail
 		}
 
 		if (d.op == VT_MOVETO || d.op == VT_LINETO) {
-			int wx, wy;
-			deserialize_int(meta, &wx);
-			deserialize_int(meta, &wy);
+			unsigned wx, wy;
+			deserialize_uint(meta, &wx);
+			deserialize_uint(meta, &wy);
 
 			long long wwx = (unsigned) wx;
 			long long wwy = (unsigned) wy;
@@ -40,6 +46,19 @@ drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail
 				wwy -= ty << (32 - z);
 			}
 
+			if (wwx < bbox[0]) {
+				bbox[0] = wwx;
+			}
+			if (wwy < bbox[1]) {
+				bbox[1] = wwy;
+			}
+			if (wwx > bbox[2]) {
+				bbox[2] = wwx;
+			}
+			if (wwy > bbox[3]) {
+				bbox[3] = wwy;
+			}
+
 			d.x = wwx;
 			d.y = wwy;
 		}
@@ -308,7 +327,9 @@ drawvec clip_poly(drawvec &geom, int z, int detail, int buffer) {
 			}
 
 			if (j >= geom.size() || geom[j].op == VT_CLOSEPATH) {
-				out.push_back(draw(VT_CLOSEPATH, 0, 0));
+				if (out.size() > 0 && out[out.size() - 1].op != VT_CLOSEPATH) {
+					out.push_back(draw(VT_CLOSEPATH, 0, 0));
+				}
 				i = j;
 			} else {
 				i = j - 1;
@@ -374,7 +395,13 @@ drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double
 
 			i = j;
 		} else {
-			fprintf(stderr, "how did we get here with %d?\n", geom[i].op);
+			fprintf(stderr, "how did we get here with %d in %d?\n", geom[i].op, (int) geom.size());
+
+			for (unsigned n = 0; n < geom.size(); n++) {
+				fprintf(stderr, "%d/%lld/%lld ", geom[n].op, geom[n].x, geom[n].y);
+			}
+			fprintf(stderr, "\n");
+
 			out.push_back(geom[i]);
 		}
 	}
@@ -382,11 +409,68 @@ drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double
 	return out;
 }
 
+drawvec clip_point(drawvec &geom, int z, int detail, long long buffer) {
+	drawvec out;
+	unsigned i;
+
+	long long min = 0;
+	long long area = 0xFFFFFFFF;
+	if (z != 0) {
+		area = 1LL << (32 - z);
+
+		min -= buffer * area / 256;
+		area += buffer * area / 256;
+	}
+
+	for (i = 0; i < geom.size(); i++) {
+		if (geom[i].x >= min && geom[i].y >= min && geom[i].x <= area && geom[i].y <= area) {
+			out.push_back(geom[i]);
+		}
+	}
+
+	return out;
+}
+
+int quick_check(long long *bbox, int z, int detail, long long buffer) {
+	long long min = 0;
+	long long area = 0xFFFFFFFF;
+	if (z != 0) {
+		area = 1LL << (32 - z);
+
+		min -= buffer * area / 256;
+		area += buffer * area / 256;
+	}
+
+	// bbox entirely outside the tile
+	if (bbox[0] > area || bbox[1] > area) {
+		return 0;
+	}
+	if (bbox[2] < min || bbox[3] < min) {
+		return 0;
+	}
+
+	// bbox entirely within the tile
+	if (bbox[0] > min && bbox[1] > min && bbox[2] < area && bbox[3] < area) {
+		return 1;
+	}
+
+	// some overlap of edge
+	return 2;
+}
 
 drawvec clip_lines(drawvec &geom, int z, int detail, long long buffer) {
 	drawvec out;
 	unsigned i;
 
+	long long min = 0;
+	long long area = 0xFFFFFFFF;
+	if (z != 0) {
+		area = 1LL << (32 - z);
+
+		min -= buffer * area / 256;
+		area += buffer * area / 256;
+	}
+
 	for (i = 0; i < geom.size(); i++) {
 		if (i > 0 && (geom[i - 1].op == VT_MOVETO || geom[i - 1].op == VT_LINETO) && geom[i].op == VT_LINETO) {
 			double x1 = geom[i - 1].x;
@@ -395,15 +479,6 @@ drawvec clip_lines(drawvec &geom, int z, int detail, long long buffer) {
 			double x2 = geom[i - 0].x;
 			double y2 = geom[i - 0].y;
 
-			long long min = 0;
-			long long area = 0xFFFFFFFF;
-			if (z != 0) {
-				area = 1LL << (32 - z);
-
-				min -= buffer * area / 256;
-				area += buffer * area / 256;
-			}
-
 			int c = clip(&x1, &y1, &x2, &y2, min, min, area, area);
 
 			if (c > 1) { // clipped
@@ -548,6 +623,10 @@ drawvec simplify_lines(drawvec &geom, int z, int detail) {
 drawvec reorder_lines(drawvec &geom) {
 	// Only reorder simple linestrings with a single moveto
 
+	if (geom.size() == 0) {
+		return geom;
+	}
+
 	unsigned i;
 	for (i = 0; i < geom.size(); i++) {
 		if (geom[i].op == VT_MOVETO) {

geometry.hh

@@ -15,11 +15,13 @@ struct draw {
 
 typedef std::vector<draw> drawvec;
 
-drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail);
+drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox);
 void to_tile_scale(drawvec &geom, int z, int detail);
 drawvec remove_noop(drawvec geom, int type);
+drawvec clip_point(drawvec &geom, int z, int detail, long long buffer);
 drawvec clip_poly(drawvec &geom, int z, int detail, int buffer);
 drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double *accum_area);
 drawvec clip_lines(drawvec &geom, int z, int detail, long long buffer);
+int quick_check(long long *bbox, int z, int detail, long long buffer);
 drawvec simplify_lines(drawvec &geom, int z, int detail);
 drawvec reorder_lines(drawvec &geom);

jsonpull.c

@@ -161,7 +161,7 @@ static json_object *add_object(json_pull *j, json_type type) {
 	return o;
 }
 
-json_object *json_hash_get(json_object *o, char *s) {
+json_object *json_hash_get(json_object *o, const char *s) {
 	if (o == NULL || o->type != JSON_HASH) {
 		return NULL;
 	}

jsonpull.h

@@ -54,4 +54,4 @@ json_object *json_read(json_pull *j);
 json_object *json_read_separators(json_pull *j, json_separator_callback cb, void *state);
 void json_free(json_object *j);
 
-json_object *json_hash_get(json_object *o, char *s);
+json_object *json_hash_get(json_object *o, const char *s);

mbtiles.c

@@ -70,7 +70,7 @@ void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data,
 	}
 }
 
-static void quote(char **buf, char *s) {
+static void quote(char **buf, const char *s) {
 	char tmp[strlen(s) * 8 + 1];
 	char *out = tmp;
 
@@ -107,7 +107,7 @@ static void aprintf(char **buf, const char *format, ...) {
 	free(tmp);
 }
 
-void mbtiles_write_metadata(sqlite3 *outdb, char *fname, char *layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool *fields) {
+void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, const char *layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool *fields) {
 	char *sql, *err;
 
 	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname);

mbtiles.h

@@ -2,6 +2,6 @@ sqlite3 *mbtiles_open(char *dbname, char **argv);
 
 void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
 
-void mbtiles_write_metadata(sqlite3 *outdb, char *fname, char *layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool *fields);
+void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, const char *layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool *fields);
 
 void mbtiles_close(sqlite3 *outdb, char **argv);

tile.cc

@@ -214,7 +214,7 @@ void decode_meta(char **meta, struct pool *keys, struct pool *values, struct poo
 	}
 }
 
-mapnik::vector::tile create_tile(char *layername, int line_detail, std::vector<coalesce> &features, long long *count, struct pool *keys, struct pool *values) {
+mapnik::vector::tile create_tile(const char *layername, int line_detail, std::vector<coalesce> &features, long long *count, struct pool *keys, struct pool *values) {
 	mapnik::vector::tile tile;
 	mapnik::vector::tile_layer *layer = tile.add_layers();
 
@@ -284,7 +284,7 @@ struct sll {
 	}
 };
 
-void evaluate(std::vector<coalesce> &features, char *metabase, struct pool *file_keys, char *layername, int line_detail, long long orig) {
+void evaluate(std::vector<coalesce> &features, char *metabase, struct pool *file_keys, const char *layername, int line_detail, long long orig) {
 	std::vector<sll> options;
 
 	struct pool_val *pv;
@@ -342,9 +342,12 @@ void evaluate(std::vector<coalesce> &features, char *metabase, struct pool *file
 	pool_free(&keys);
 }
 
-long long write_tile(struct index *start, struct index *end, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, char *layername, sqlite3 *outdb, double droprate, int buffer) {
+long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along) {
 	int line_detail;
 	static bool evaluated = false;
+	double oprogress = 0;
+
+	char *og = *geoms;
 
 	for (line_detail = detail; line_detail >= MIN_DETAIL || line_detail == detail; line_detail--) {
 		GOOGLE_PROTOBUF_VERIFY_VERSION;
@@ -360,41 +363,128 @@ long long write_tile(struct index *start, struct index *end, char *metabase, uns
 
 		std::vector<coalesce> features;
 
-		struct index *i;
-		for (i = start; i < end; i++) {
-			int t = i->type;
+		int within[4] = { 0 };
+		long long geompos[4] = { 0 };
 
-			if (z > i->maxzoom) {
-				continue;
+		*geoms = og;
+
+		while (1) {
+			int t;
+			deserialize_int(geoms, &t);
+			if (t < 0) {
+				break;
 			}
-			if ((t == VT_LINE && z + line_detail <= i->minzoom) ||
-			    (t == VT_POINT && z < i->minzoom)) {
+
+			long long metastart;
+			deserialize_long_long(geoms, &metastart);
+			char *meta = metabase + metastart;
+			long long bbox[4];
+
+			drawvec geom = decode_geometry(geoms, z, tx, ty, line_detail, bbox);
+
+			signed char feature_minzoom;
+			deserialize_byte(geoms, &feature_minzoom);
+
+			double progress = floor((((*geoms - geomstart + along) / (double) todo) + z) / (file_maxzoom + 1) * 1000) / 10;
+			if (progress != oprogress) {
+				fprintf(stderr, "  %3.1f%%  %d/%u/%u  \r", progress, z, tx, ty);
+				oprogress = progress;
+			}
+
+			int quick = quick_check(bbox, z, line_detail, buffer);
+			if (quick == 0) {
 				continue;
 			}
 
-			if (i->candup) {
-				if (dup.count(i->fpos) != 0) {
-					continue;
+			if (quick != 1) {
+				if (t == VT_LINE) {
+					geom = clip_lines(geom, z, line_detail, buffer);
 				}
-				dup.insert(i->fpos);
+				if (t == VT_POLYGON) {
+					geom = clip_poly(geom, z, line_detail, buffer);
+				}
+				if (t == VT_POINT) {
+					geom = clip_point(geom, z, line_detail, buffer);
+				}
+
+				geom = remove_noop(geom, t);
 			}
 
-			char *meta = metabase + i->fpos;
-			drawvec geom = decode_geometry(&meta, z, tx, ty, line_detail);
+			if (line_detail == detail) { /* only write out the next zoom once, even if we retry */
+				if (geom.size() > 0 && z + 1 <= file_maxzoom) {
+					int j;
+					for (j = 0; j < 4; j++) {
+						int xo = j & 1;
+						int yo = (j >> 1) & 1;
+
+						long long bbox2[4];
+						int k;
+						for (k = 0; k < 4; k++) {
+							bbox2[k] = bbox[k];
+						}
+						if (z != 0) {
+							// Offset back to world-relative
+							bbox2[0] += tx << (32 - z);
+							bbox2[1] += ty << (32 - z);
+							bbox2[2] += tx << (32 - z);
+							bbox2[3] += ty << (32 - z);
+						}
+						// Offset to child tile-relative
+						bbox2[0] -= (tx * 2 + xo) << (32 - (z + 1));
+						bbox2[1] -= (ty * 2 + yo) << (32 - (z + 1));
+						bbox2[2] -= (tx * 2 + xo) << (32 - (z + 1));
+						bbox2[3] -= (ty * 2 + yo) << (32 - (z + 1));
+
+						int quick2 = quick_check(bbox2, z + 1, line_detail, buffer);
+						if (quick2 != 0) {
+							if (!within[j]) {
+								serialize_int(geomfile[j], z + 1, &geompos[j], fname, jp);
+								serialize_uint(geomfile[j], tx * 2 + xo, &geompos[j], fname, jp);
+								serialize_uint(geomfile[j], ty * 2 + yo, &geompos[j], fname, jp);
+								within[j] = 1;
+							}
+
+							// Offset from tile coordinates back to world coordinates
+							unsigned sx = 0, sy = 0;
+							if (z != 0) {
+								sx = tx << (32 - z);
+								sy = ty << (32 - z);
+							}
+
+							//printf("type %d, meta %lld\n", t, metastart);
+							serialize_int(geomfile[j], t, &geompos[j], fname, jp);
+							serialize_long_long(geomfile[j], metastart, &geompos[j], fname, jp);
+
+							for (unsigned u = 0; u < geom.size(); u++) {
+								serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname, jp);
+
+								if (geom[u].op != VT_CLOSEPATH) {
+									serialize_uint(geomfile[j], geom[u].x + sx, &geompos[j], fname, jp);
+									serialize_uint(geomfile[j], geom[u].y + sy, &geompos[j], fname, jp);
+								}
+							}
+
+							serialize_byte(geomfile[j], VT_END, &geompos[j], fname, jp);
+							serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname, jp);
+						}
+					}
+				}
+			}
+
+			if (z < file_minzoom) {
+				continue;
+			}
+
+			if ((t == VT_LINE && z + line_detail <= feature_minzoom) ||
+			    (t == VT_POINT && z < feature_minzoom)) {
+				continue;
+			}
 
 			bool reduced = false;
 			if (t == VT_POLYGON) {
 				geom = reduce_tiny_poly(geom, z, line_detail, &reduced, &accum_area);
 			}
 
-			if (t == VT_LINE) {
-				geom = clip_lines(geom, z, line_detail, buffer);
-			}
-
-			if (t == VT_POLYGON) {
-				geom = clip_poly(geom, z, line_detail, buffer);
-			}
-
 			if (t == VT_LINE || t == VT_POLYGON) {
 				if (!reduced) {
 					geom = simplify_lines(geom, z, line_detail);
@@ -428,8 +518,8 @@ long long write_tile(struct index *start, struct index *end, char *metabase, uns
 						c.index2 = ~0LL;
 					}
 				} else {
-					c.index = i->index;
-					c.index2 = i->index;
+					c.index = 0;
+					c.index2 = 0;
 				}
 				c.geom = geom;
 				c.metasrc = meta;
@@ -440,6 +530,14 @@ long long write_tile(struct index *start, struct index *end, char *metabase, uns
 			}
 		}
 
+		int j;
+		for (j = 0; j < 4; j++) {
+			if (within[j]) {
+				serialize_int(geomfile[j], -2, &geompos[j], fname, jp);
+				within[j] = 0;
+			}
+		}
+
 		std::sort(features.begin(), features.end());
 
 		std::vector<coalesce> out;
@@ -471,6 +569,12 @@ long long write_tile(struct index *start, struct index *end, char *metabase, uns
 		}
 
 		if (features.size() > 0) {
+			if (features.size() > 200000) {
+				fprintf(stderr, "tile %d/%u/%u has %lld features, >200000    \n", z, tx, ty, (long long) features.size());
+				fprintf(stderr, "Try using -z to set a higher base zoom level.\n");
+				exit(EXIT_FAILURE);
+			}
+
 			mapnik::vector::tile tile = create_tile(layername, line_detail, features, &count, &keys, &values);
 
 			pool_free(&keys);

tile.h

@@ -12,19 +12,18 @@
 #define VT_BOOLEAN 7
 
 struct pool;
+struct json_pull;
+
+void serialize_int(FILE *out, int n, long long *fpos, const char *fname, struct json_pull *source);
+void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname, struct json_pull *source);
+void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname, struct json_pull *source);
+void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname, struct json_pull *source);
+void serialize_string(FILE *out, const char *s, long long *fpos, const char *fname, struct json_pull *source);
 
 void deserialize_int(char **f, int *n);
+void deserialize_long_long(char **f, long long *n);
+void deserialize_uint(char **f, unsigned *n);
 void deserialize_byte(char **f, signed char *n);
 struct pool_val *deserialize_string(char **f, struct pool *p, int type);
 
-
-struct index {
-	unsigned long long index;
-	long long fpos : 44;
-	int maxzoom : 6;
-	int minzoom : 6;
-	int type : 7;
-	int candup : 1;
-};
-
-long long write_tile(struct index *start, struct index *end, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, char *layername, sqlite3 *outdb, double droprate, int buffer);
+long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along);