From d69431e16b5e827263ee04bdc0f2f3f32b972a21 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 11 Dec 2014 15:46:54 -0800 Subject: [PATCH] Partition each tile into children instead of repeatedly clipping the parent. --- geojson.c | 141 ++++++++++++++++++++++++++++++------------------------ tile.cc | 90 +++++++++++++++++++++++----------- tile.h | 2 +- 3 files changed, 140 insertions(+), 93 deletions(-) diff --git a/geojson.c b/geojson.c index 9bf70b1..f7db816 100644 --- a/geojson.c +++ b/geojson.c @@ -210,86 +210,90 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type) { return ret; } -void check(int geomfd, off_t geom_size, char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, const char *tmpdir) { +void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, const char *tmpdir) { int i; for (i = 0; i <= maxzoom; i++) { fprintf(stderr, "\n"); long long most = 0; - printf("%lld of geom_size\n", (long long) geom_size); - - char *geom = mmap(NULL, geom_size, PROT_READ, MAP_PRIVATE, geomfd, 0); - if (geom == MAP_FAILED) { - perror("mmap geom"); - exit(EXIT_FAILURE); + FILE *sub[4]; + int subfd[4]; + int j; + for (j = 0; j < 4; j++) { + char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; + sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j); + subfd[j] = mkstemp(geomname); + //printf("%s\n", geomname); + if (subfd[j] < 0) { + perror(geomname); + exit(EXIT_FAILURE); + } + sub[j] = fopen(geomname, "wb"); + if (sub[j] == NULL) { + perror(geomname); + exit(EXIT_FAILURE); + } + unlink(geomname); } - char *end = geom + geom_size; - - char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; - sprintf(geomname, "%s%s", tmpdir, "/geom2.XXXXXXXX"); - int geomfd2 = mkstemp(geomname); - printf("%s\n", geomname); - if (geomfd2 < 0) { - perror(geomname); - exit(EXIT_FAILURE); - } - FILE *geomfile = fopen(geomname, "wb"); - if (geomfile == NULL) { - perror(geomname); - exit(EXIT_FAILURE); - } - unlink(geomname); - - while (geom < end) { - int zz; - unsigned xx, yy; - - deserialize_int(&geom, &zz); - deserialize_uint(&geom, &xx); - deserialize_uint(&geom, &yy); - - int z = zz + 1; - int dim = 2; - if (z == 0) { - dim = 1; // only one tile at z0 + for (j = 0; j < 4; j++) { + if (geomfd[j] < 0) { + // only one source file for zoom level 0 + continue; + } + if (geom_size[j] == 0) { + continue; } - char *ogeom = geom; + printf("%lld of geom_size\n", (long long) geom_size[j]); - unsigned x, y; - for (x = xx * 2; x < xx * 2 + dim; x++) { - for (y = yy * 2; y < yy * 2 + dim; y++) { - fprintf(stderr, "%d/%u/%u\n", z, x, y); - geom = ogeom; + char *geom = mmap(NULL, geom_size[j], PROT_READ, MAP_PRIVATE, geomfd[j], 0); + if (geom == MAP_FAILED) { + perror("mmap geom"); + exit(EXIT_FAILURE); + } - // fprintf(stderr, " %3.1f%% %d/%u/%u \r", (((i - ix) + (j - ix)) / 2.0 / n + (maxzoom - z)) / (maxzoom - minzoom + 1) * 100, z, tx, ty); + char *end = geom + geom_size[j]; - long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, geomfile); + while (geom < end) { + int z; + unsigned x, y; - if (z == maxzoom && len > most) { - *midx = x; - *midy = y; - most = len; - } + deserialize_int(&geom, &z); + deserialize_uint(&geom, &x); + deserialize_uint(&geom, &y); + + fprintf(stderr, "%d/%u/%u\n", z, x, y); + + // fprintf(stderr, " %3.1f%% %d/%u/%u \r", (((i - ix) + (j - ix)) / 2.0 / n + (maxzoom - z)) / (maxzoom - minzoom + 1) * 100, z, tx, ty); + + long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub); + + if (z == maxzoom && len > most) { + *midx = x; + *midy = y; + most = len; } + + fprintf(stderr, "\n"); } - fprintf(stderr, "\n"); + munmap(geom, geom_size[j]); } - munmap(geom, geom_size); - close(geomfd); - fclose(geomfile); + for (j = 0; j < 4; j++) { + close(geomfd[j]); + fclose(sub[j]); - struct stat geomst; - if (fstat(geomfd2, &geomst) != 0) { - perror("stat geom\n"); - exit(EXIT_FAILURE); + struct stat geomst; + if (fstat(subfd[j], &geomst) != 0) { + perror("stat geom\n"); + exit(EXIT_FAILURE); + } + + geomfd[j] = subfd[j]; + geom_size[j] = geomst.st_size; } - - geomfd = geomfd2; - geom_size = geomst.st_size; } } @@ -385,8 +389,8 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i json_pull *jp = json_begin_file(f); long long seq = 0; - /* zoom, x, y of enclosing tile, so 0/0/0's parent is -1/0/0 */ - serialize_int(geomfile, -1, &geompos, fname, jp); + /* initial tile is 0/0/0 */ + serialize_int(geomfile, 0, &geompos, fname, jp); serialize_uint(geomfile, 0, &geompos, fname, jp); serialize_uint(geomfile, 0, &geompos, fname, jp); @@ -771,8 +775,19 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i } #endif + int fd[4]; + off_t size[4]; - check(geomfd, geomst.st_size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir); + fd[0] = geomfd; + size[0] = geomst.st_size; + + int j; + for (j = 1; j < 4; j++) { + fd[j] = -1; + size[j] = 0; + } + + check(fd, size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir); munmap(meta, metast.st_size); diff --git a/tile.cc b/tile.cc index c2cd1bb..d96c0ef 100644 --- a/tile.cc +++ b/tile.cc @@ -342,7 +342,7 @@ void evaluate(std::vector &features, char *metabase, struct pool *file pool_free(&keys); } -long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile) { +long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4]) { int line_detail; static bool evaluated = false; @@ -362,8 +362,8 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u std::vector features; - int within = 0; - long long geompos = 0; + int within[4] = { 0 }; + long long geompos[4] = { 0 }; *geoms = og; @@ -420,35 +420,64 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u if (line_detail == detail) { /* only write out the next zoom once, even if we retry */ if (geom.size() > 0) { - if (!within) { - serialize_int(geomfile, z, &geompos, fname, jp); - serialize_uint(geomfile, tx, &geompos, fname, jp); - serialize_uint(geomfile, ty, &geompos, fname, jp); - within = 1; - } + int j; + for (j = 0; j < 4; j++) { + int xo = j & 1; + int yo = (j >> 1) & 1; - // Offset from tile coordinates back to world coordinates - unsigned sx = 0, sy = 0; - if (z != 0) { - sx = tx << (32 - z); - sy = ty << (32 - z); - } + long long bbox2[4]; + int k; + for (k = 0; k < 4; k++) { + bbox2[k] = bbox[k]; + } + if (z != 0) { + // Offset back to world-relative + bbox2[0] += tx << (32 - z); + bbox2[1] += ty << (32 - z); + bbox2[2] += tx << (32 - z); + bbox2[3] += ty << (32 - z); + } + // Offset to child tile-relative + bbox2[0] -= (tx * 2 + xo) << (32 - (z + 1)); + bbox2[1] -= (ty * 2 + yo) << (32 - (z + 1)); + bbox2[2] -= (tx * 2 + xo) << (32 - (z + 1)); + bbox2[3] -= (ty * 2 + yo) << (32 - (z + 1)); - //printf("type %d, meta %lld\n", t, metastart); - serialize_int(geomfile, t, &geompos, fname, jp); - serialize_long_long(geomfile, metastart, &geompos, fname, jp); + int quick2 = quick_check(bbox2, z + 1, line_detail, buffer); + if (quick2 != 0) { + if (!within[j]) { + printf("begin %d/%d/%d\n", z + 1, tx * 2 + xo, ty * 2 + yo); - for (unsigned u = 0; u < geom.size(); u++) { - serialize_byte(geomfile, geom[u].op, &geompos, fname, jp); + serialize_int(geomfile[j], z + 1, &geompos[j], fname, jp); + serialize_uint(geomfile[j], tx * 2 + xo, &geompos[j], fname, jp); + serialize_uint(geomfile[j], ty * 2 + yo, &geompos[j], fname, jp); + within[j] = 1; + } - if (geom[u].op != VT_CLOSEPATH) { - serialize_uint(geomfile, geom[u].x + sx, &geompos, fname, jp); - serialize_uint(geomfile, geom[u].y + sy, &geompos, fname, jp); + // Offset from tile coordinates back to world coordinates + unsigned sx = 0, sy = 0; + if (z != 0) { + sx = tx << (32 - z); + sy = ty << (32 - z); + } + + //printf("type %d, meta %lld\n", t, metastart); + serialize_int(geomfile[j], t, &geompos[j], fname, jp); + serialize_long_long(geomfile[j], metastart, &geompos[j], fname, jp); + + for (unsigned u = 0; u < geom.size(); u++) { + serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname, jp); + + if (geom[u].op != VT_CLOSEPATH) { + serialize_uint(geomfile[j], geom[u].x + sx, &geompos[j], fname, jp); + serialize_uint(geomfile[j], geom[u].y + sy, &geompos[j], fname, jp); + } + } + + serialize_byte(geomfile[j], VT_END, &geompos[j], fname, jp); + serialize_byte(geomfile[j], minzoom, &geompos[j], fname, jp); } } - - serialize_byte(geomfile, VT_END, &geompos, fname, jp); - serialize_byte(geomfile, minzoom, &geompos, fname, jp); } } @@ -507,9 +536,12 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u } } - if (within) { - serialize_int(geomfile, -2, &geompos, fname, jp); - within = 0; + int j; + for (j = 0; j < 4; j++) { + if (within[j]) { + serialize_int(geomfile[j], -2, &geompos[j], fname, jp); + within[j] = 0; + } } std::sort(features.begin(), features.end()); diff --git a/tile.h b/tile.h index 75422bb..51451b0 100644 --- a/tile.h +++ b/tile.h @@ -36,4 +36,4 @@ struct index { int candup : 1; }; -long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile); +long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4]);