From bd081c864ed30ba2c7fe6b79eb47da4c18bd0238 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 21 Dec 2015 18:00:07 -0800 Subject: [PATCH] It tiles again! --- geojson.c | 16 ++++++++-------- tile.cc | 14 +++++++++----- tile.h | 2 +- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/geojson.c b/geojson.c index 29f1598..37a12f8 100644 --- a/geojson.c +++ b/geojson.c @@ -1479,7 +1479,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max // segment+offset to find the data. long long pool_off[THREADS]; - long long metafile_off[THREADS]; + long long meta_off[THREADS]; char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX"); @@ -1532,7 +1532,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max munmap(map, reader[i].metapos); } - metafile_off[i] = metapos; + meta_off[i] = metapos; metapos += reader[i].metapos; close(reader[i].metafd); @@ -1563,10 +1563,8 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max exit(EXIT_FAILURE); } -#if 0 /* XXX */ - unsigned midx = 0, midy = 0; - int written = traverse_zooms(fd, size, meta, stringpool, file_keys, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail); + int written = traverse_zooms(fd, size, meta, stringpool, file_keys, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail, meta_off, pool_off); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); @@ -1574,14 +1572,17 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max ret = EXIT_FAILURE; } - if (munmap(meta, metast.st_size) != 0) { + if (munmap(meta, metapos) != 0) { perror("munmap meta"); } if (close(metafd) < 0) { perror("close meta"); } - if (memfile_close(poolfile) != 0) { + if (munmap(stringpool, poolpos) != 0) { + perror("munmap stringpool"); + } + if (close(poolfd) < 0) { perror("close pool"); } @@ -1650,7 +1651,6 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max free(layernames[i]); } -#endif return ret; } diff --git a/tile.cc b/tile.cc index 0f4eaac..6fe6509 100644 --- a/tile.cc +++ b/tile.cc @@ -464,7 +464,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u } } -long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, char *prevent, char *additional, int child_shards) { +long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, char *prevent, char *additional, int child_shards, long long *meta_off, long long *pool_off) { int line_detail; double fraction = 1; @@ -562,7 +562,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsi long long metastart; deserialize_long_long(geoms, &metastart); - char *meta = metabase + metastart; + char *meta = metabase + metastart + meta_off[segment]; long long bbox[4]; drawvec geom = decode_geometry(geoms, z, tx, ty, line_detail, bbox); @@ -757,7 +757,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsi c.coalesced = false; c.original_seq = original_seq; - decode_meta(&meta, stringpool, keys[layer], values[layer], file_keys[layer], &c.meta); + decode_meta(&meta, stringpool + pool_off[segment], keys[layer], values[layer], file_keys[layer], &c.meta); features[layer].push_back(c); } } @@ -926,6 +926,8 @@ struct write_tile_args { int full_detail; int low_detail; volatile long long *most; + long long *meta_off; + long long *pool_off; }; void *run_thread(void *vargs) { @@ -965,7 +967,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(&geom, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards); + long long len = write_tile(&geom, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards, arg->meta_off, arg->pool_off); if (len < 0) { int *err = (int *) malloc(sizeof(int)); @@ -1001,7 +1003,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off) { int i; for (i = 0; i <= maxzoom; i++) { long long most = 0; @@ -1125,6 +1127,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].full_detail = full_detail; args[thread].low_detail = low_detail; args[thread].most = &most; // locked with var_lock + args[thread].meta_off = meta_off; + args[thread].pool_off = pool_off; args[thread].tasks = dispatches[thread].tasks; diff --git a/tile.h b/tile.h index 2ac893f..f7df52f 100644 --- a/tile.h +++ b/tile.h @@ -27,7 +27,7 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type); long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off); extern unsigned initial_x, initial_y; extern int geometry_scale;