Merge pull request #65 from mapbox/skip

Skipping over zoom levels when there is a minzoom, and other performance improvements
This commit is contained in:
Eric Fischer 2015-07-08 14:44:55 -07:00
commit 3bc5a07e5b
9 changed files with 160 additions and 106 deletions

View File

@ -75,11 +75,17 @@ void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fn
unsigned char b = zigzag & 0x7F;
if ((zigzag >> 7) != 0) {
b |= 0x80;
fwrite_check(&b, sizeof(unsigned char), 1, out, fname);
if (putc(b, out) == EOF) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_FAILURE);
}
*fpos += 1;
zigzag >>= 7;
} else {
fwrite_check(&b, sizeof(unsigned char), 1, out, fname);
if (putc(b, out) == EOF) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_FAILURE);
}
*fpos += 1;
break;
}
@ -954,14 +960,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
exit(EXIT_FAILURE);
}
int fd[4];
off_t size[4];
int fd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)];
off_t size[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)];
fd[0] = geomfd;
size[0] = geomst.st_size;
int j;
for (j = 1; j < 4; j++) {
for (j = 1; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
fd[j] = -1;
size[j] = 0;
}

View File

@ -84,7 +84,7 @@ void to_tile_scale(drawvec &geom, int z, int detail) {
}
}
drawvec remove_noop(drawvec geom, int type) {
drawvec remove_noop(drawvec geom, int type, int shift) {
// first pass: remove empty linetos
long long x = 0, y = 0;
@ -92,7 +92,7 @@ drawvec remove_noop(drawvec geom, int type) {
unsigned i;
for (i = 0; i < geom.size(); i++) {
if (geom[i].op == VT_LINETO && geom[i].x == x && geom[i].y == y) {
if (geom[i].op == VT_LINETO && (geom[i].x >> shift) == x && (geom[i].y >> shift) == y) {
continue;
}
@ -100,8 +100,8 @@ drawvec remove_noop(drawvec geom, int type) {
out.push_back(geom[i]);
} else { /* moveto or lineto */
out.push_back(geom[i]);
x = geom[i].x;
y = geom[i].y;
x = geom[i].x >> shift;
y = geom[i].y >> shift;
}
}
@ -137,7 +137,7 @@ drawvec remove_noop(drawvec geom, int type) {
for (i = 0; i < geom.size(); i++) {
if (geom[i].op == VT_MOVETO) {
if (i > 0 && geom[i - 1].op == VT_LINETO && geom[i - 1].x == geom[i].x && geom[i - 1].y == geom[i].y) {
if (i > 0 && geom[i - 1].op == VT_LINETO && (geom[i - 1].x >> shift) == (geom[i].x >> shift) && (geom[i - 1].y >> shift) == (geom[i].y >> shift)) {
continue;
}
}

View File

@ -18,7 +18,7 @@ typedef std::vector<draw> drawvec;
drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox);
void to_tile_scale(drawvec &geom, int z, int detail);
drawvec remove_noop(drawvec geom, int type);
drawvec remove_noop(drawvec geom, int type, int shift);
drawvec clip_point(drawvec &geom, int z, int detail, long long buffer);
drawvec clip_poly(drawvec &geom, int z, int detail, int buffer);
drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double *accum_area);

View File

@ -499,7 +499,6 @@ again:
json_object *s = add_object(j, JSON_STRING);
if (s != NULL) {
val.buf = realloc(val.buf, val.n + 1);
s->string = val.buf;
s->length = val.n;
} else {

View File

@ -228,11 +228,7 @@ void mbtiles_close(sqlite3 *outdb, char **argv) {
char *err;
if (sqlite3_exec(outdb, "ANALYZE;", NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "%s: index metadata: %s\n", argv[0], err);
exit(EXIT_FAILURE);
}
if (sqlite3_exec(outdb, "VACUUM;", NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "%s: index tiles: %s\n", argv[0], err);
fprintf(stderr, "%s: ANALYZE failed: %s\n", argv[0], err);
exit(EXIT_FAILURE);
}
if (sqlite3_close(outdb) != SQLITE_OK) {

29
pool.c
View File

@ -2,11 +2,23 @@
#include <string.h>
#include "pool.h"
static struct pool_val *pool1(struct pool *p, char *s, int type, int (*compare)(const char *, const char *)) {
struct pool_val **v = &(p->vals);
#define POOL_WIDTH 256
static int hash(char *s) {
int h = 0;
for (; *s; s++) {
h = h * 37 + *s;
}
h = h & 0xFF;
return h;
}
struct pool_val *pool(struct pool *p, char *s, int type) {
int h = hash(s);
struct pool_val **v = &(p->vals[h]);
while (*v != NULL) {
int cmp = compare(s, (*v)->s);
int cmp = strcmp(s, (*v)->s);
if (cmp == 0) {
cmp = type - (*v)->type;
@ -41,7 +53,8 @@ static struct pool_val *pool1(struct pool *p, char *s, int type, int (*compare)(
}
int is_pooled(struct pool *p, char *s, int type) {
struct pool_val **v = &(p->vals);
int h = hash(s);
struct pool_val **v = &(p->vals[h]);
while (*v != NULL) {
int cmp = strcmp(s, (*v)->s);
@ -62,10 +75,6 @@ int is_pooled(struct pool *p, char *s, int type) {
return 0;
}
struct pool_val *pool(struct pool *p, char *s, int type) {
return pool1(p, s, type, strcmp);
}
void pool_free1(struct pool *p, void (*func)(void *)) {
while (p->head != NULL) {
if (func != NULL) {
@ -79,6 +88,8 @@ void pool_free1(struct pool *p, void (*func)(void *)) {
p->head = NULL;
p->tail = NULL;
free(p->vals);
p->vals = NULL;
}
@ -92,7 +103,7 @@ void pool_free_strings(struct pool *p) {
void pool_init(struct pool *p, int n) {
p->n = n;
p->vals = NULL;
p->vals = calloc(POOL_WIDTH, sizeof(struct pool_val *));
p->head = NULL;
p->tail = NULL;
}

2
pool.h
View File

@ -10,7 +10,7 @@ struct pool_val {
};
struct pool {
struct pool_val *vals;
struct pool_val **vals;
struct pool_val *head;
struct pool_val *tail;

194
tile.cc
View File

@ -236,7 +236,7 @@ mapnik::vector::tile create_tile(char **layernames, int line_detail, std::vector
unsigned x;
for (x = 0; x < features[i].size(); x++) {
if (features[i][x].type == VT_LINE || features[i][x].type == VT_POLYGON) {
features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type);
features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type, 0);
}
mapnik::vector::tile_feature *feature = layer->add_features();
@ -358,7 +358,96 @@ void evaluate(std::vector<coalesce> &features, char *metabase, struct pool *file
}
#endif
long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) {
void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, signed char layer, long long metastart, signed char feature_minzoom) {
if (geom.size() > 0 && nextzoom <= file_maxzoom) {
int xo, yo;
int span = 1 << (nextzoom - z);
// Get the feature bounding box in pixel (256) coordinates at the child zoom
// in order to calculate which sub-tiles it can touch including the buffer.
long long bbox2[4];
int k;
for (k = 0; k < 4; k++) {
// Division instead of right-shift because coordinates can be negative
bbox2[k] = bbox[k] / (1 << (32 - nextzoom - 8));
}
bbox2[0] -= buffer;
bbox2[1] -= buffer;
bbox2[2] += buffer;
bbox2[3] += buffer;
for (k = 0; k < 4; k++) {
if (bbox2[k] < 0) {
bbox2[k] = 0;
}
if (bbox2[k] >= 256 * span) {
bbox2[k] = 256 * (span - 1);
}
bbox2[k] /= 256;
}
for (xo = bbox2[0]; xo <= bbox2[2]; xo++) {
for (yo = bbox2[1]; yo <= bbox2[3]; yo++) {
unsigned jx = tx * span + xo;
unsigned jy = ty * span + yo;
// j is the shard that the child tile's data is being written to.
//
// Be careful: We can't jump more zoom levels than MAX_ZOOM_INCREMENT
// because that could break the constraint that each of the children
// of the current tile must have its own shard, because the data for
// the child tile must be contiguous within the shard.
//
// But it's OK to spread children across all the shards, not just
// the four that would normally result from splitting one tile,
// because it will go through all the shards when it does the
// next zoom.
int j = ((jx & ((1 << MAX_ZOOM_INCREMENT) - 1)) << MAX_ZOOM_INCREMENT) |
((jy & ((1 << MAX_ZOOM_INCREMENT) - 1)));
{
if (!within[j]) {
serialize_int(geomfile[j], nextzoom, &geompos[j], fname);
serialize_uint(geomfile[j], tx * span + xo, &geompos[j], fname);
serialize_uint(geomfile[j], ty * span + yo, &geompos[j], fname);
within[j] = 1;
}
// Offset from tile coordinates back to world coordinates
unsigned sx = 0, sy = 0;
if (z != 0) {
sx = tx << (32 - z);
sy = ty << (32 - z);
}
// printf("type %d, meta %lld\n", t, metastart);
serialize_byte(geomfile[j], t, &geompos[j], fname);
serialize_byte(geomfile[j], layer, &geompos[j], fname);
serialize_long_long(geomfile[j], metastart, &geompos[j], fname);
long long wx = initial_x, wy = initial_y;
for (unsigned u = 0; u < geom.size(); u++) {
serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname);
if (geom[u].op != VT_CLOSEPATH) {
serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname);
serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname);
wx = geom[u].x + sx;
wy = geom[u].y + sy;
}
}
serialize_byte(geomfile[j], VT_END, &geompos[j], fname);
serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname);
}
}
}
}
}
long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) {
int line_detail;
static bool evaluated = false;
double oprogress = 0;
@ -366,6 +455,15 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
char *og = *geoms;
int nextzoom = z + 1;
if (nextzoom < file_minzoom) {
if (z + MAX_ZOOM_INCREMENT > file_minzoom) {
nextzoom = file_minzoom;
} else {
nextzoom = z + MAX_ZOOM_INCREMENT;
}
}
for (line_detail = detail; line_detail >= min_detail || line_detail == detail; line_detail--) {
GOOGLE_PROTOBUF_VERIFY_VERSION;
@ -404,8 +502,8 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
features.push_back(std::vector<coalesce>());
}
int within[4] = {0};
long long geompos[4] = {0};
int within[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0};
long long geompos[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0};
*geoms = og;
@ -453,7 +551,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
geom = clip_point(geom, z, line_detail, buffer);
}
geom = remove_noop(geom, t);
geom = remove_noop(geom, t, 0);
}
if (geom.size() > 0) {
@ -461,68 +559,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
}
if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */
if (geom.size() > 0 && z + 1 <= file_maxzoom) {
int j;
for (j = 0; j < 4; j++) {
int xo = j & 1;
int yo = (j >> 1) & 1;
long long bbox2[4];
int k;
for (k = 0; k < 4; k++) {
bbox2[k] = bbox[k];
}
if (z != 0) {
// Offset back to world-relative
bbox2[0] += tx << (32 - z);
bbox2[1] += ty << (32 - z);
bbox2[2] += tx << (32 - z);
bbox2[3] += ty << (32 - z);
}
// Offset to child tile-relative
bbox2[0] -= (tx * 2 + xo) << (32 - (z + 1));
bbox2[1] -= (ty * 2 + yo) << (32 - (z + 1));
bbox2[2] -= (tx * 2 + xo) << (32 - (z + 1));
bbox2[3] -= (ty * 2 + yo) << (32 - (z + 1));
int quick2 = quick_check(bbox2, z + 1, line_detail, buffer);
if (quick2 != 0) {
if (!within[j]) {
serialize_int(geomfile[j], z + 1, &geompos[j], fname);
serialize_uint(geomfile[j], tx * 2 + xo, &geompos[j], fname);
serialize_uint(geomfile[j], ty * 2 + yo, &geompos[j], fname);
within[j] = 1;
}
// Offset from tile coordinates back to world coordinates
unsigned sx = 0, sy = 0;
if (z != 0) {
sx = tx << (32 - z);
sy = ty << (32 - z);
}
// printf("type %d, meta %lld\n", t, metastart);
serialize_byte(geomfile[j], t, &geompos[j], fname);
serialize_byte(geomfile[j], layer, &geompos[j], fname);
serialize_long_long(geomfile[j], metastart, &geompos[j], fname);
long long wx = initial_x, wy = initial_y;
for (unsigned u = 0; u < geom.size(); u++) {
serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname);
if (geom[u].op != VT_CLOSEPATH) {
serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname);
serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname);
wx = geom[u].x + sx;
wy = geom[u].y + sy;
}
}
serialize_byte(geomfile[j], VT_END, &geompos[j], fname);
serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname);
}
}
}
rewrite(geom, z, nextzoom, file_maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom);
}
if (z < file_minzoom) {
@ -588,6 +625,10 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
if ((t == VT_LINE || t == VT_POLYGON) && !prevent['s' & 0xFF]) {
if (!reduced) {
if (t == VT_LINE) {
geom = remove_noop(geom, t, 32 - z - line_detail);
}
geom = simplify_lines(geom, z, line_detail);
}
}
@ -632,7 +673,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
}
int j;
for (j = 0; j < 4; j++) {
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
if (within[j]) {
serialize_byte(geomfile[j], -2, &geompos[j], fname);
within[j] = 0;
@ -669,7 +710,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
for (x = 0; x < features[j].size(); x++) {
if (features[j][x].coalesced && features[j][x].type == VT_LINE) {
features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type);
features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type, 0);
features[j][x].geom = simplify_lines(features[j][x].geom, 32, 0);
}
}
@ -737,15 +778,15 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
return -1;
}
int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) {
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) {
int i;
for (i = 0; i <= maxzoom; i++) {
long long most = 0;
FILE *sub[4];
int subfd[4];
FILE *sub[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)];
int subfd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)];
int j;
for (j = 0; j < 4; j++) {
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1];
sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j);
subfd[j] = mkstemp(geomname);
@ -764,11 +805,11 @@ int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stri
long long todo = 0;
long long along = 0;
for (j = 0; j < 4; j++) {
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
todo += geom_size[j];
}
for (j = 0; j < 4; j++) {
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
if (geomfd[j] < 0) {
// only one source file for zoom level 0
continue;
@ -817,7 +858,7 @@ int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stri
along += geom_size[j];
}
for (j = 0; j < 4; j++) {
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
close(geomfd[j]);
fclose(sub[j]);
@ -835,4 +876,3 @@ int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stri
fprintf(stderr, "\n");
return maxzoom;
}

6
tile.h
View File

@ -25,9 +25,11 @@ void deserialize_uint(char **f, unsigned *n);
void deserialize_byte(char **f, signed char *n);
struct pool_val *deserialize_string(char **f, struct pool *p, int type);
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent);
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent);
int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail);
extern unsigned initial_x, initial_y;
extern int geometry_scale;
#define MAX_ZOOM_INCREMENT 3