From 62052cafab4c49d6096846e41082887e231d3ac1 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 19 Jun 2015 17:29:56 -0700 Subject: [PATCH] Move zoom traversal into C++ --- geojson.c | 101 +----------------------------------------------------- tile.cc | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ tile.h | 2 ++ 3 files changed, 104 insertions(+), 100 deletions(-) diff --git a/geojson.c b/geojson.c index 42572f8..467b511 100644 --- a/geojson.c +++ b/geojson.c @@ -229,105 +229,6 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type) { return ret; } -int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent) { - int i; - for (i = 0; i <= maxzoom; i++) { - long long most = 0; - - FILE *sub[4]; - int subfd[4]; - int j; - for (j = 0; j < 4; j++) { - char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; - sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j); - subfd[j] = mkstemp(geomname); - // printf("%s\n", geomname); - if (subfd[j] < 0) { - perror(geomname); - exit(EXIT_FAILURE); - } - sub[j] = fopen(geomname, "wb"); - if (sub[j] == NULL) { - perror(geomname); - exit(EXIT_FAILURE); - } - unlink(geomname); - } - - long long todo = 0; - long long along = 0; - for (j = 0; j < 4; j++) { - todo += geom_size[j]; - } - - for (j = 0; j < 4; j++) { - if (geomfd[j] < 0) { - // only one source file for zoom level 0 - continue; - } - if (geom_size[j] == 0) { - continue; - } - - // printf("%lld of geom_size\n", (long long) geom_size[j]); - - char *geom = mmap(NULL, geom_size[j], PROT_READ, MAP_PRIVATE, geomfd[j], 0); - if (geom == MAP_FAILED) { - perror("mmap geom"); - exit(EXIT_FAILURE); - } - - char *geomstart = geom; - char *end = geom + geom_size[j]; - - while (geom < end) { - int z; - unsigned x, y; - - deserialize_int(&geom, &z); - deserialize_uint(&geom, &x); - deserialize_uint(&geom, &y); - - // fprintf(stderr, "%d/%u/%u\n", z, x, y); - - long long len = write_tile(&geom, metabase, stringpool, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, min_detail, maxzoom, file_keys, layernames, outdb, droprate, buffer, fname, sub, minzoom, maxzoom, todo, geomstart, along, gamma, nlayers, prevent); - - if (len < 0) { - return i - 1; - } - - if (z == maxzoom && len > most) { - *midx = x; - *midy = y; - most = len; - } - } - - if (munmap(geomstart, geom_size[j]) != 0) { - perror("munmap geom"); - } - along += geom_size[j]; - } - - for (j = 0; j < 4; j++) { - close(geomfd[j]); - fclose(sub[j]); - - struct stat geomst; - if (fstat(subfd[j], &geomst) != 0) { - perror("stat geom\n"); - exit(EXIT_FAILURE); - } - - geomfd[j] = subfd[j]; - geom_size[j] = geomst.st_size; - } - } - - fprintf(stderr, "\n"); - return maxzoom; -} - struct index { long long start; long long end; @@ -1067,7 +968,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", seq, (long long) geomst.st_size, (long long) metast.st_size, poolfile->off); - int written = traverse_zooms(fd, size, meta, stringpool, file_bbox, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent); + int written = traverse_zooms(fd, size, meta, stringpool, file_bbox, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, full_detail, low_detail, min_detail); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); diff --git a/tile.cc b/tile.cc index 585783f..d24a5b8 100644 --- a/tile.cc +++ b/tile.cc @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include "vector_tile.pb.h" @@ -735,3 +736,103 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f fprintf(stderr, "could not make tile %d/%u/%u small enough\n", z, tx, ty); return -1; } + +int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) { + int i; + for (i = 0; i <= maxzoom; i++) { + long long most = 0; + + FILE *sub[4]; + int subfd[4]; + int j; + for (j = 0; j < 4; j++) { + char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; + sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j); + subfd[j] = mkstemp(geomname); + // printf("%s\n", geomname); + if (subfd[j] < 0) { + perror(geomname); + exit(EXIT_FAILURE); + } + sub[j] = fopen(geomname, "wb"); + if (sub[j] == NULL) { + perror(geomname); + exit(EXIT_FAILURE); + } + unlink(geomname); + } + + long long todo = 0; + long long along = 0; + for (j = 0; j < 4; j++) { + todo += geom_size[j]; + } + + for (j = 0; j < 4; j++) { + if (geomfd[j] < 0) { + // only one source file for zoom level 0 + continue; + } + if (geom_size[j] == 0) { + continue; + } + + // printf("%lld of geom_size\n", (long long) geom_size[j]); + + char *geom = (char *) mmap(NULL, geom_size[j], PROT_READ, MAP_PRIVATE, geomfd[j], 0); + if (geom == MAP_FAILED) { + perror("mmap geom"); + exit(EXIT_FAILURE); + } + + char *geomstart = geom; + char *end = geom + geom_size[j]; + + while (geom < end) { + int z; + unsigned x, y; + + deserialize_int(&geom, &z); + deserialize_uint(&geom, &x); + deserialize_uint(&geom, &y); + + // fprintf(stderr, "%d/%u/%u\n", z, x, y); + + long long len = write_tile(&geom, metabase, stringpool, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, min_detail, maxzoom, file_keys, layernames, outdb, droprate, buffer, fname, sub, minzoom, maxzoom, todo, geomstart, along, gamma, nlayers, prevent); + + if (len < 0) { + return i - 1; + } + + if (z == maxzoom && len > most) { + *midx = x; + *midy = y; + most = len; + } + } + + if (munmap(geomstart, geom_size[j]) != 0) { + perror("munmap geom"); + } + along += geom_size[j]; + } + + for (j = 0; j < 4; j++) { + close(geomfd[j]); + fclose(sub[j]); + + struct stat geomst; + if (fstat(subfd[j], &geomst) != 0) { + perror("stat geom\n"); + exit(EXIT_FAILURE); + } + + geomfd[j] = subfd[j]; + geom_size[j] = geomst.st_size; + } + } + + fprintf(stderr, "\n"); + return maxzoom; +} + diff --git a/tile.h b/tile.h index 644662e..96641e6 100644 --- a/tile.h +++ b/tile.h @@ -27,5 +27,7 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type); long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent); +int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail); + extern unsigned initial_x, initial_y; extern int geometry_scale;