From eaeb55bf719a45a12231abe95f18128b5ed6bec3 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 2 Dec 2014 18:03:07 -0800 Subject: [PATCH 01/17] Distinguish between signed and unsigned in the temporary file --- geojson.c | 5 +++++ geometry.cc | 6 +++--- tile.h | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/geojson.c b/geojson.c index eaa64e0..79ef40e 100644 --- a/geojson.c +++ b/geojson.c @@ -179,6 +179,11 @@ void deserialize_int(char **f, int *n) { *f += sizeof(int); } +void deserialize_uint(char **f, unsigned *n) { + memcpy(n, *f, sizeof(unsigned)); + *f += sizeof(unsigned); +} + void deserialize_byte(char **f, signed char *n) { memcpy(n, *f, sizeof(signed char)); *f += sizeof(signed char); diff --git a/geometry.cc b/geometry.cc index daecdce..2b5491a 100644 --- a/geometry.cc +++ b/geometry.cc @@ -28,9 +28,9 @@ drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail } if (d.op == VT_MOVETO || d.op == VT_LINETO) { - int wx, wy; - deserialize_int(meta, &wx); - deserialize_int(meta, &wy); + unsigned wx, wy; + deserialize_uint(meta, &wx); + deserialize_uint(meta, &wy); long long wwx = (unsigned) wx; long long wwy = (unsigned) wy; diff --git a/tile.h b/tile.h index f4f38e6..0fe8a03 100644 --- a/tile.h +++ b/tile.h @@ -14,6 +14,7 @@ struct pool; void deserialize_int(char **f, int *n); +void deserialize_uint(char **f, unsigned *n); void deserialize_byte(char **f, signed char *n); struct pool_val *deserialize_string(char **f, struct pool *p, int type); From 08ff40e42fb84dacc6450f5e002053cf8415a3aa Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 3 Dec 2014 16:18:43 -0800 Subject: [PATCH 02/17] Trying to follow Vlad's suggestion and work from the top down. Doesn't quite work yet, but seems like the right thing to do. From the bottom up, indexing is a mess because of the buffering, and includes unneeded areas in the bounding boxes of big features. --- enumerate.c | 2 +- geojson.c | 229 +++++++++++++++++++++++++++++++--------------------- geometry.cc | 39 ++++++--- geometry.hh | 1 + tile.cc | 102 ++++++++++++++++++----- tile.h | 10 ++- 6 files changed, 258 insertions(+), 125 deletions(-) diff --git a/enumerate.c b/enumerate.c index 6d4cfdf..94a7359 100644 --- a/enumerate.c +++ b/enumerate.c @@ -24,7 +24,7 @@ void enumerate(char *fname) { long long x = sqlite3_column_int(stmt, 1); long long y = sqlite3_column_int(stmt, 2); - y = (1LL << zoom) - y; + y = (1LL << zoom) - 1 - y; printf("%s %lld %lld %lld\n", fname, zoom, x, y); } diff --git a/geojson.c b/geojson.c index 79ef40e..b0e14df 100644 --- a/geojson.c +++ b/geojson.c @@ -97,6 +97,11 @@ void serialize_int(FILE *out, int n, long long *fpos, const char *fname, json_pu *fpos += sizeof(int); } +void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname, json_pull *source) { + fwrite_check(&n, sizeof(long long), 1, out, fname, source); + *fpos += sizeof(long long); +} + void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname, json_pull *source) { fwrite_check(&n, sizeof(signed char), 1, out, fname, source); *fpos += sizeof(signed char); @@ -179,6 +184,11 @@ void deserialize_int(char **f, int *n) { *f += sizeof(int); } +void deserialize_long_long(char **f, long long *n) { + memcpy(n, *f, sizeof(long long)); + *f += sizeof(long long); +} + void deserialize_uint(char **f, unsigned *n) { memcpy(n, *f, sizeof(unsigned)); *f += sizeof(unsigned); @@ -200,57 +210,87 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type) { return ret; } -void check(struct index *ix, long long n, char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer) { - fprintf(stderr, "\n"); - long long most = 0; +void check(int geomfd, off_t geom_size, char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, const char *tmpdir) { + int i; + for (i = 0; i < maxzoom; i++) { + fprintf(stderr, "\n"); + long long most = 0; - int z; - for (z = maxzoom; z >= minzoom; z--) { - struct index *i, *j = NULL; - for (i = ix; i < ix + n && i != NULL; i = j) { - if (i > ix && indexcmp(i - 1, i) > 0) { - fprintf(stderr, "index out of order\n"); - exit(EXIT_FAILURE); - } - unsigned wx, wy; - decode(i->index, &wx, &wy); + printf("%lld of geom_size\n", (long long) geom_size); - unsigned tx = 0, ty = 0; - if (z != 0) { - tx = wx >> (32 - z); - ty = wy >> (32 - z); - } - - // printf("%lld in %lld\n", (long long)(i - ix), (long long)n); - - for (j = i + 1; j < ix + n; j++) { - unsigned wx2, wy2; - decode(j->index, &wx2, &wy2); - - unsigned tx2 = 0, ty2 = 0; - if (z != 0) { - tx2 = wx2 >> (32 - z); - ty2 = wy2 >> (32 - z); - } - - if (tx2 != tx || ty2 != ty) { - break; - } - } - - fprintf(stderr, " %3.1f%% %d/%u/%u \r", (((i - ix) + (j - ix)) / 2.0 / n + (maxzoom - z)) / (maxzoom - minzoom + 1) * 100, z, tx, ty); - - long long len = write_tile(i, j, metabase, file_bbox, z, tx, ty, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer); - - if (z == maxzoom && len > most) { - *midx = tx; - *midy = ty; - most = len; - } + char *geom = mmap(NULL, geom_size, PROT_READ, MAP_PRIVATE, geomfd, 0); + if (geom == MAP_FAILED) { + perror("mmap geom"); + exit(EXIT_FAILURE); } - } - fprintf(stderr, "\n"); + char *end = geom + geom_size; + + char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; + sprintf(geomname, "%s%s", tmpdir, "/geom2.XXXXXXXX"); + int geomfd2 = mkstemp(geomname); + printf("%s\n", geomname); + if (geomfd2 < 0) { + perror(geomname); + exit(EXIT_FAILURE); + } + FILE *geomfile = fopen(geomname, "wb"); + if (geomfile == NULL) { + perror(geomname); + exit(EXIT_FAILURE); + } + unlink(geomname); + + while (geom < end) { + int zz; + unsigned xx, yy; + + deserialize_int(&geom, &zz); + deserialize_uint(&geom, &xx); + deserialize_uint(&geom, &yy); + + int z = zz + 1; + int dim = 2; + if (z == 0) { + dim = 1; // only one tile at z0 + } + + char *ogeom = geom; + + unsigned x, y; + for (x = xx * 2; x < xx * 2 + dim; x++) { + for (y = yy * 2; y < yy * 2 + dim; y++) { + fprintf(stderr, "%d/%u/%u\n", z, x, y); + geom = ogeom; + + // fprintf(stderr, " %3.1f%% %d/%u/%u \r", (((i - ix) + (j - ix)) / 2.0 / n + (maxzoom - z)) / (maxzoom - minzoom + 1) * 100, z, tx, ty); + + long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, geomfile); + + if (z == maxzoom && len > most) { + *midx = x; + *midy = y; + most = len; + } + } + } + + fprintf(stderr, "\n"); + } + + munmap(geom, geom_size); + close(geomfd); + fclose(geomfile); + + struct stat geomst; + if (fstat(geomfd2, &geomst) != 0) { + perror("stat geom\n"); + exit(EXIT_FAILURE); + } + + geomfd = geomfd2; + geom_size = geomst.st_size; + } } struct merge { @@ -260,6 +300,7 @@ struct merge { struct merge *next; }; +#if 0 static void insert(struct merge *m, struct merge **head, unsigned char *map, int bytes) { while (*head != NULL && indexcmp(map + m->start, map + (*head)->start) > 0) { head = &((*head)->next); @@ -301,22 +342,24 @@ static void merge(struct merge *merges, int nmerges, unsigned char *map, FILE *f } } } +#endif void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir) { char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; - char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; + char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX"); - sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX"); + sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX"); int metafd = mkstemp(metaname); if (metafd < 0) { perror(metaname); exit(EXIT_FAILURE); } - int indexfd = mkstemp(indexname); - if (indexfd < 0) { - perror(indexname); + int geomfd = mkstemp(geomname); + printf("%s\n", geomname); + if (geomfd < 0) { + perror(geomname); exit(EXIT_FAILURE); } @@ -325,15 +368,16 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i perror(metaname); exit(EXIT_FAILURE); } - FILE *indexfile = fopen(indexname, "wb"); - if (indexfile == NULL) { - perror(indexname); + FILE *geomfile = fopen(geomname, "wb"); + if (geomfile == NULL) { + perror(geomname); exit(EXIT_FAILURE); } - long long fpos = 0; + long long metapos = 0; + long long geompos = 0; unlink(metaname); - unlink(indexname); + unlink(geomname); unsigned file_bbox[] = { UINT_MAX, UINT_MAX, 0, 0 }; unsigned midx = 0, midy = 0; @@ -341,6 +385,11 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i json_pull *jp = json_begin_file(f); long long seq = 0; + /* zoom, x, y of enclosing tile, so 0/0/0's parent is -1/0/0 */ + serialize_int(geomfile, -1, &geompos, fname, jp); + serialize_uint(geomfile, 0, &geompos, fname, jp); + serialize_uint(geomfile, 0, &geompos, fname, jp); + while (1) { json_object *j = json_read(jp); if (j == NULL) { @@ -398,13 +447,9 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i } { - long long start = fpos; - unsigned bbox[] = { UINT_MAX, UINT_MAX, 0, 0 }; - parse_geometry(t, coordinates, bbox, &fpos, metafile, VT_MOVETO, fname, jp); - serialize_byte(metafile, VT_END, &fpos, fname, jp); - + long long metastart = metapos; char *metakey[properties->length]; char *metaval[properties->length]; int metatype[properties->length]; @@ -445,17 +490,21 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i } } - serialize_int(metafile, m, &fpos, fname, jp); + serialize_int(metafile, m, &metapos, fname, jp); for (i = 0; i < m; i++) { - serialize_int(metafile, metatype[i], &fpos, fname, jp); - serialize_string(metafile, metakey[i], &fpos, fname, jp); - serialize_string(metafile, metaval[i], &fpos, fname, jp); + serialize_int(metafile, metatype[i], &metapos, fname, jp); + serialize_string(metafile, metakey[i], &metapos, fname, jp); + serialize_string(metafile, metaval[i], &metapos, fname, jp); } - int z = maxzoom; + serialize_int(geomfile, mb_geometry[t], &geompos, fname, jp); + serialize_long_long(geomfile, metastart, &geompos, fname, jp); + parse_geometry(t, coordinates, bbox, &geompos, geomfile, VT_MOVETO, fname, jp); + serialize_byte(geomfile, VT_END, &geompos, fname, jp); - unsigned cx = bbox[0] / 2 + bbox[2] / 2; - unsigned cy = bbox[1] / 2 + bbox[3] / 2; +#if 0 + int z = maxzoom; +#endif /* * Note that minzoom for lines is the dimension @@ -485,6 +534,12 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i minzoom = maxzoom - floor(log(r) / - log(droprate)); } + serialize_byte(geomfile, minzoom, &geompos, fname, jp); + +#if 0 + unsigned cx = bbox[0] / 2 + bbox[2] / 2; + unsigned cy = bbox[1] / 2 + bbox[3] / 2; + /* XXX do proper overlap instead of whole bounding box */ if (z == 0) { struct index ix; @@ -546,6 +601,7 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i } } } +#endif for (i = 0; i < 2; i++) { if (bbox[i] < file_bbox[i]) { @@ -569,17 +625,20 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i /* XXX check for any non-features in the outer object */ } + /* end of tile */ + serialize_int(geomfile, -2, &geompos, fname, jp); + json_end(jp); fclose(metafile); - fclose(indexfile); + fclose(geomfile); printf("bbox: %x %x %x %x\n", file_bbox[0], file_bbox[1], file_bbox[2], file_bbox[3]); - struct stat indexst; + struct stat geomst; struct stat metast; - if (fstat(indexfd, &indexst) != 0) { - perror("stat index\n"); + if (fstat(geomfd, &geomst) != 0) { + perror("stat geom\n"); exit(EXIT_FAILURE); } if (fstat(metafd, &metast) != 0) { @@ -587,7 +646,7 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i exit(EXIT_FAILURE); } - if (indexst.st_size == 0 || metast.st_size == 0) { + if (geomst.st_size == 0 || metast.st_size == 0) { fprintf(stderr, "%s: did not read any valid geometries\n", fname); exit(EXIT_FAILURE); } @@ -632,6 +691,7 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i printf("using layer name %s\n", trunc); } +#if 0 { int bytes = sizeof(struct index); @@ -709,29 +769,14 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i fclose(f); close(indexfd); } +#endif - indexfd = open(indexname, O_RDONLY); - if (indexfd < 0) { - perror(indexname); - exit(EXIT_FAILURE); - } - if (unlink(indexname) != 0) { - perror(indexname); - exit(EXIT_FAILURE); - } - struct index *index = (struct index *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfd, 0); - if (index == MAP_FAILED) { - perror("mmap index"); - exit(EXIT_FAILURE); - } + check(geomfd, geomst.st_size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir); - check(index, indexst.st_size / sizeof(struct index), meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer); - - munmap(index, indexst.st_size); munmap(meta, metast.st_size); - close(indexfd); + close(geomfd); close(metafd); diff --git a/geometry.cc b/geometry.cc index 2b5491a..c7efc32 100644 --- a/geometry.cc +++ b/geometry.cc @@ -382,11 +382,41 @@ drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double return out; } +drawvec clip_point(drawvec &geom, int z, int detail, long long buffer) { + drawvec out; + unsigned i; + + long long min = 0; + long long area = 0xFFFFFFFF; + if (z != 0) { + area = 1LL << (32 - z); + + min -= buffer * area / 256; + area += buffer * area / 256; + } + + for (i = 0; i < geom.size(); i++) { + if (geom[i].x >= min && geom[i].y >= min && geom[i].x <= area && geom[i].y <= area) { + out.push_back(geom[i]); + } + } + + return out; +} drawvec clip_lines(drawvec &geom, int z, int detail, long long buffer) { drawvec out; unsigned i; + long long min = 0; + long long area = 0xFFFFFFFF; + if (z != 0) { + area = 1LL << (32 - z); + + min -= buffer * area / 256; + area += buffer * area / 256; + } + for (i = 0; i < geom.size(); i++) { if (i > 0 && (geom[i - 1].op == VT_MOVETO || geom[i - 1].op == VT_LINETO) && geom[i].op == VT_LINETO) { double x1 = geom[i - 1].x; @@ -395,15 +425,6 @@ drawvec clip_lines(drawvec &geom, int z, int detail, long long buffer) { double x2 = geom[i - 0].x; double y2 = geom[i - 0].y; - long long min = 0; - long long area = 0xFFFFFFFF; - if (z != 0) { - area = 1LL << (32 - z); - - min -= buffer * area / 256; - area += buffer * area / 256; - } - int c = clip(&x1, &y1, &x2, &y2, min, min, area, area); if (c > 1) { // clipped diff --git a/geometry.hh b/geometry.hh index 7af9b4e..9a1487b 100644 --- a/geometry.hh +++ b/geometry.hh @@ -18,6 +18,7 @@ typedef std::vector drawvec; drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail); void to_tile_scale(drawvec &geom, int z, int detail); drawvec remove_noop(drawvec geom, int type); +drawvec clip_point(drawvec &geom, int z, int detail, long long buffer); drawvec clip_poly(drawvec &geom, int z, int detail, int buffer); drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double *accum_area); drawvec clip_lines(drawvec &geom, int z, int detail, long long buffer); diff --git a/tile.cc b/tile.cc index c61f0aa..bc2cdea 100644 --- a/tile.cc +++ b/tile.cc @@ -342,10 +342,12 @@ void evaluate(std::vector &features, char *metabase, struct pool *file pool_free(&keys); } -long long write_tile(struct index *start, struct index *end, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer) { +long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile) { int line_detail; static bool evaluated = false; + char *og = *geoms; + for (line_detail = detail; line_detail >= MIN_DETAIL || line_detail == detail; line_detail--) { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -360,41 +362,92 @@ long long write_tile(struct index *start, struct index *end, char *metabase, uns std::vector features; - struct index *i; - for (i = start; i < end; i++) { - int t = i->type; + int within = 0; + long long geompos = 0; - if (z > i->maxzoom) { - continue; - } - if ((t == VT_LINE && z + line_detail <= i->minzoom) || - (t == VT_POINT && z < i->minzoom)) { - continue; + *geoms = og; + + while (1) { + int t; + deserialize_int(geoms, &t); + printf("geometry %d\n", t); + if (t < 0) { + break; } +#if 0 if (i->candup) { if (dup.count(i->fpos) != 0) { continue; } dup.insert(i->fpos); } +#endif - char *meta = metabase + i->fpos; - drawvec geom = decode_geometry(&meta, z, tx, ty, line_detail); + long long metastart; + deserialize_long_long(geoms, &metastart); + char *meta = metabase + metastart; + + drawvec geom = decode_geometry(geoms, z, tx, ty, line_detail); + + signed char minzoom; + deserialize_byte(geoms, &minzoom); + +#if 0 + if (z > i->maxzoom) { + continue; + } +#endif + if (t == VT_LINE) { + geom = clip_lines(geom, z, line_detail, buffer); + } + if (t == VT_POLYGON) { + geom = clip_poly(geom, z, line_detail, buffer); + } + if (t == VT_POINT) { + geom = clip_point(geom, z, line_detail, buffer); + } + + geom = remove_noop(geom, t); + + if (line_detail == detail) { /* only write out the next zoom once, even if we retry */ + if (geom.size() > 0) { + if (!within) { + printf("writing %d/%u/%u\n", z, tx, ty); + serialize_int(geomfile, z, &geompos, fname, jp); + serialize_uint(geomfile, tx, &geompos, fname, jp); + serialize_uint(geomfile, ty, &geompos, fname, jp); + within = 1; + } + + //printf("type %d, meta %lld\n", t, metastart); + serialize_int(geomfile, t, &geompos, fname, jp); + serialize_long_long(geomfile, metastart, &geompos, fname, jp); + + for (unsigned u = 0; u < geom.size(); u++) { + serialize_byte(geomfile, geom[u].op, &geompos, fname, jp); + + if (geom[u].op != VT_CLOSEPATH) { + serialize_uint(geomfile, geom[u].x, &geompos, fname, jp); + serialize_uint(geomfile, geom[u].y, &geompos, fname, jp); + } + } + + serialize_byte(geomfile, VT_END, &geompos, fname, jp); + serialize_byte(geomfile, minzoom, &geompos, fname, jp); + } + } + + if ((t == VT_LINE && z + line_detail <= minzoom) || + (t == VT_POINT && z < minzoom)) { + continue; + } bool reduced = false; if (t == VT_POLYGON) { geom = reduce_tiny_poly(geom, z, line_detail, &reduced, &accum_area); } - if (t == VT_LINE) { - geom = clip_lines(geom, z, line_detail, buffer); - } - - if (t == VT_POLYGON) { - geom = clip_poly(geom, z, line_detail, buffer); - } - if (t == VT_LINE || t == VT_POLYGON) { if (!reduced) { geom = simplify_lines(geom, z, line_detail); @@ -428,8 +481,8 @@ long long write_tile(struct index *start, struct index *end, char *metabase, uns c.index2 = ~0LL; } } else { - c.index = i->index; - c.index2 = i->index; + c.index = 0; + c.index2 = 0; } c.geom = geom; c.metasrc = meta; @@ -440,6 +493,11 @@ long long write_tile(struct index *start, struct index *end, char *metabase, uns } } + if (within) { + serialize_int(geomfile, -2, &geompos, fname, jp); + within = 0; + } + std::sort(features.begin(), features.end()); std::vector out; diff --git a/tile.h b/tile.h index 0fe8a03..75422bb 100644 --- a/tile.h +++ b/tile.h @@ -12,8 +12,16 @@ #define VT_BOOLEAN 7 struct pool; +struct json_pull; + +void serialize_int(FILE *out, int n, long long *fpos, const char *fname, struct json_pull *source); +void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname, struct json_pull *source); +void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname, struct json_pull *source); +void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname, struct json_pull *source); +void serialize_string(FILE *out, const char *s, long long *fpos, const char *fname, struct json_pull *source); void deserialize_int(char **f, int *n); +void deserialize_long_long(char **f, long long *n); void deserialize_uint(char **f, unsigned *n); void deserialize_byte(char **f, signed char *n); struct pool_val *deserialize_string(char **f, struct pool *p, int type); @@ -28,4 +36,4 @@ struct index { int candup : 1; }; -long long write_tile(struct index *start, struct index *end, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer); +long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile); From b7b476b36c657a8033eb321daba8fb63cdbc8cdf Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 3 Dec 2014 16:30:35 -0800 Subject: [PATCH 03/17] Don't introduce bogus geometries in clipped-away features. --- geometry.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/geometry.cc b/geometry.cc index c7efc32..54fbf7c 100644 --- a/geometry.cc +++ b/geometry.cc @@ -569,6 +569,10 @@ drawvec simplify_lines(drawvec &geom, int z, int detail) { drawvec reorder_lines(drawvec &geom) { // Only reorder simple linestrings with a single moveto + if (geom.size() == 0) { + return geom; + } + unsigned i; for (i = 0; i < geom.size(); i++) { if (geom[i].op == VT_MOVETO) { From 028fef470e6f9ca038cfb7c8dbebca83d49c0132 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 4 Dec 2014 14:08:36 -0800 Subject: [PATCH 04/17] Fix the major bug: forgetting to offset back to world coordinates --- geojson.c | 2 +- tile.cc | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/geojson.c b/geojson.c index b0e14df..9bf70b1 100644 --- a/geojson.c +++ b/geojson.c @@ -212,7 +212,7 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type) { void check(int geomfd, off_t geom_size, char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, const char *tmpdir) { int i; - for (i = 0; i < maxzoom; i++) { + for (i = 0; i <= maxzoom; i++) { fprintf(stderr, "\n"); long long most = 0; diff --git a/tile.cc b/tile.cc index bc2cdea..d03bcd3 100644 --- a/tile.cc +++ b/tile.cc @@ -370,7 +370,6 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u while (1) { int t; deserialize_int(geoms, &t); - printf("geometry %d\n", t); if (t < 0) { break; } @@ -420,6 +419,13 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u within = 1; } + // Offset from tile coordinates back to world coordinates + unsigned sx = 0, sy = 0; + if (z != 0) { + sx = tx << (32 - z); + sy = ty << (32 - z); + } + //printf("type %d, meta %lld\n", t, metastart); serialize_int(geomfile, t, &geompos, fname, jp); serialize_long_long(geomfile, metastart, &geompos, fname, jp); @@ -428,8 +434,8 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u serialize_byte(geomfile, geom[u].op, &geompos, fname, jp); if (geom[u].op != VT_CLOSEPATH) { - serialize_uint(geomfile, geom[u].x, &geompos, fname, jp); - serialize_uint(geomfile, geom[u].y, &geompos, fname, jp); + serialize_uint(geomfile, geom[u].x + sx, &geompos, fname, jp); + serialize_uint(geomfile, geom[u].y + sy, &geompos, fname, jp); } } @@ -548,6 +554,7 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u evaluate(features, metabase, file_keys, layername, line_detail, compressed.size()); } } else { + printf("output %d/%u/%u\n", z, tx, ty); mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); return count; } From 380550ce85867c60d6f9be0241b41f6ec7020dde Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 4 Dec 2014 14:54:11 -0800 Subject: [PATCH 05/17] Remember to remove the closepath after clipping away a polygon --- geometry.cc | 12 ++++++++++-- tile.cc | 2 -- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/geometry.cc b/geometry.cc index 54fbf7c..8c4ad12 100644 --- a/geometry.cc +++ b/geometry.cc @@ -308,7 +308,9 @@ drawvec clip_poly(drawvec &geom, int z, int detail, int buffer) { } if (j >= geom.size() || geom[j].op == VT_CLOSEPATH) { - out.push_back(draw(VT_CLOSEPATH, 0, 0)); + if (out.size() > 0 && out[out.size() - 1].op != VT_CLOSEPATH) { + out.push_back(draw(VT_CLOSEPATH, 0, 0)); + } i = j; } else { i = j - 1; @@ -374,7 +376,13 @@ drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double i = j; } else { - fprintf(stderr, "how did we get here with %d?\n", geom[i].op); + fprintf(stderr, "how did we get here with %d in %d?\n", geom[i].op, (int) geom.size()); + + for (unsigned n = 0; n < geom.size(); n++) { + fprintf(stderr, "%d/%lld/%lld ", geom[n].op, geom[n].x, geom[n].y); + } + fprintf(stderr, "\n"); + out.push_back(geom[i]); } } diff --git a/tile.cc b/tile.cc index d03bcd3..767eafc 100644 --- a/tile.cc +++ b/tile.cc @@ -412,7 +412,6 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u if (line_detail == detail) { /* only write out the next zoom once, even if we retry */ if (geom.size() > 0) { if (!within) { - printf("writing %d/%u/%u\n", z, tx, ty); serialize_int(geomfile, z, &geompos, fname, jp); serialize_uint(geomfile, tx, &geompos, fname, jp); serialize_uint(geomfile, ty, &geompos, fname, jp); @@ -554,7 +553,6 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u evaluate(features, metabase, file_keys, layername, line_detail, compressed.size()); } } else { - printf("output %d/%u/%u\n", z, tx, ty); mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); return count; } From 105dfa73d79d752a59ff7bc13942f4a966931ef9 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 11 Dec 2014 13:34:50 -0800 Subject: [PATCH 06/17] Use bounding box for quick accept/reject before detailed clipping. --- geometry.cc | 48 +++++++++++++++++++++++++++++++++++++++++++++++- geometry.hh | 3 ++- tile.cc | 27 ++++++++++++++++++--------- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/geometry.cc b/geometry.cc index 8c4ad12..e74e970 100644 --- a/geometry.cc +++ b/geometry.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include "geometry.hh" extern "C" { @@ -16,9 +17,14 @@ extern "C" { #include "projection.h" } -drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail) { +drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox) { drawvec out; + bbox[0] = LONG_LONG_MAX; + bbox[1] = LONG_LONG_MAX; + bbox[2] = LONG_LONG_MIN; + bbox[3] = LONG_LONG_MIN; + while (1) { draw d; @@ -40,6 +46,19 @@ drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail wwy -= ty << (32 - z); } + if (wwx < bbox[0]) { + bbox[0] = wwx; + } + if (wwy < bbox[1]) { + bbox[1] = wwy; + } + if (wwx > bbox[2]) { + bbox[2] = wwx; + } + if (wwy > bbox[3]) { + bbox[3] = wwy; + } + d.x = wwx; d.y = wwy; } @@ -412,6 +431,33 @@ drawvec clip_point(drawvec &geom, int z, int detail, long long buffer) { return out; } +int quick_check(long long *bbox, int z, int detail, long long buffer) { + long long min = 0; + long long area = 0xFFFFFFFF; + if (z != 0) { + area = 1LL << (32 - z); + + min -= buffer * area / 256; + area += buffer * area / 256; + } + + // bbox entirely outside the tile + if (bbox[0] > area || bbox[1] > area) { + return 0; + } + if (bbox[2] < min || bbox[3] < min) { + return 0; + } + + // bbox entirely within the tile + if (bbox[0] > min && bbox[1] > min && bbox[2] < area && bbox[3] < area) { + return 1; + } + + // some overlap of edge + return 2; +} + drawvec clip_lines(drawvec &geom, int z, int detail, long long buffer) { drawvec out; unsigned i; diff --git a/geometry.hh b/geometry.hh index 9a1487b..d6fd72b 100644 --- a/geometry.hh +++ b/geometry.hh @@ -15,12 +15,13 @@ struct draw { typedef std::vector drawvec; -drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail); +drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox); void to_tile_scale(drawvec &geom, int z, int detail); drawvec remove_noop(drawvec geom, int type); drawvec clip_point(drawvec &geom, int z, int detail, long long buffer); drawvec clip_poly(drawvec &geom, int z, int detail, int buffer); drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double *accum_area); drawvec clip_lines(drawvec &geom, int z, int detail, long long buffer); +int quick_check(long long *bbox, int z, int detail, long long buffer); drawvec simplify_lines(drawvec &geom, int z, int detail); drawvec reorder_lines(drawvec &geom); diff --git a/tile.cc b/tile.cc index 767eafc..c2cd1bb 100644 --- a/tile.cc +++ b/tile.cc @@ -386,25 +386,34 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u long long metastart; deserialize_long_long(geoms, &metastart); char *meta = metabase + metastart; + long long bbox[4]; - drawvec geom = decode_geometry(geoms, z, tx, ty, line_detail); + drawvec geom = decode_geometry(geoms, z, tx, ty, line_detail, bbox); signed char minzoom; deserialize_byte(geoms, &minzoom); + int quick = quick_check(bbox, z, line_detail, buffer); + if (quick == 0) { + continue; + } + #if 0 if (z > i->maxzoom) { continue; } #endif - if (t == VT_LINE) { - geom = clip_lines(geom, z, line_detail, buffer); - } - if (t == VT_POLYGON) { - geom = clip_poly(geom, z, line_detail, buffer); - } - if (t == VT_POINT) { - geom = clip_point(geom, z, line_detail, buffer); + + if (quick != 1) { + if (t == VT_LINE) { + geom = clip_lines(geom, z, line_detail, buffer); + } + if (t == VT_POLYGON) { + geom = clip_poly(geom, z, line_detail, buffer); + } + if (t == VT_POINT) { + geom = clip_point(geom, z, line_detail, buffer); + } } geom = remove_noop(geom, t); From d69431e16b5e827263ee04bdc0f2f3f32b972a21 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 11 Dec 2014 15:46:54 -0800 Subject: [PATCH 07/17] Partition each tile into children instead of repeatedly clipping the parent. --- geojson.c | 141 ++++++++++++++++++++++++++++++------------------------ tile.cc | 90 +++++++++++++++++++++++----------- tile.h | 2 +- 3 files changed, 140 insertions(+), 93 deletions(-) diff --git a/geojson.c b/geojson.c index 9bf70b1..f7db816 100644 --- a/geojson.c +++ b/geojson.c @@ -210,86 +210,90 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type) { return ret; } -void check(int geomfd, off_t geom_size, char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, const char *tmpdir) { +void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, const char *tmpdir) { int i; for (i = 0; i <= maxzoom; i++) { fprintf(stderr, "\n"); long long most = 0; - printf("%lld of geom_size\n", (long long) geom_size); - - char *geom = mmap(NULL, geom_size, PROT_READ, MAP_PRIVATE, geomfd, 0); - if (geom == MAP_FAILED) { - perror("mmap geom"); - exit(EXIT_FAILURE); + FILE *sub[4]; + int subfd[4]; + int j; + for (j = 0; j < 4; j++) { + char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; + sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j); + subfd[j] = mkstemp(geomname); + //printf("%s\n", geomname); + if (subfd[j] < 0) { + perror(geomname); + exit(EXIT_FAILURE); + } + sub[j] = fopen(geomname, "wb"); + if (sub[j] == NULL) { + perror(geomname); + exit(EXIT_FAILURE); + } + unlink(geomname); } - char *end = geom + geom_size; - - char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; - sprintf(geomname, "%s%s", tmpdir, "/geom2.XXXXXXXX"); - int geomfd2 = mkstemp(geomname); - printf("%s\n", geomname); - if (geomfd2 < 0) { - perror(geomname); - exit(EXIT_FAILURE); - } - FILE *geomfile = fopen(geomname, "wb"); - if (geomfile == NULL) { - perror(geomname); - exit(EXIT_FAILURE); - } - unlink(geomname); - - while (geom < end) { - int zz; - unsigned xx, yy; - - deserialize_int(&geom, &zz); - deserialize_uint(&geom, &xx); - deserialize_uint(&geom, &yy); - - int z = zz + 1; - int dim = 2; - if (z == 0) { - dim = 1; // only one tile at z0 + for (j = 0; j < 4; j++) { + if (geomfd[j] < 0) { + // only one source file for zoom level 0 + continue; + } + if (geom_size[j] == 0) { + continue; } - char *ogeom = geom; + printf("%lld of geom_size\n", (long long) geom_size[j]); - unsigned x, y; - for (x = xx * 2; x < xx * 2 + dim; x++) { - for (y = yy * 2; y < yy * 2 + dim; y++) { - fprintf(stderr, "%d/%u/%u\n", z, x, y); - geom = ogeom; + char *geom = mmap(NULL, geom_size[j], PROT_READ, MAP_PRIVATE, geomfd[j], 0); + if (geom == MAP_FAILED) { + perror("mmap geom"); + exit(EXIT_FAILURE); + } - // fprintf(stderr, " %3.1f%% %d/%u/%u \r", (((i - ix) + (j - ix)) / 2.0 / n + (maxzoom - z)) / (maxzoom - minzoom + 1) * 100, z, tx, ty); + char *end = geom + geom_size[j]; - long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, geomfile); + while (geom < end) { + int z; + unsigned x, y; - if (z == maxzoom && len > most) { - *midx = x; - *midy = y; - most = len; - } + deserialize_int(&geom, &z); + deserialize_uint(&geom, &x); + deserialize_uint(&geom, &y); + + fprintf(stderr, "%d/%u/%u\n", z, x, y); + + // fprintf(stderr, " %3.1f%% %d/%u/%u \r", (((i - ix) + (j - ix)) / 2.0 / n + (maxzoom - z)) / (maxzoom - minzoom + 1) * 100, z, tx, ty); + + long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub); + + if (z == maxzoom && len > most) { + *midx = x; + *midy = y; + most = len; } + + fprintf(stderr, "\n"); } - fprintf(stderr, "\n"); + munmap(geom, geom_size[j]); } - munmap(geom, geom_size); - close(geomfd); - fclose(geomfile); + for (j = 0; j < 4; j++) { + close(geomfd[j]); + fclose(sub[j]); - struct stat geomst; - if (fstat(geomfd2, &geomst) != 0) { - perror("stat geom\n"); - exit(EXIT_FAILURE); + struct stat geomst; + if (fstat(subfd[j], &geomst) != 0) { + perror("stat geom\n"); + exit(EXIT_FAILURE); + } + + geomfd[j] = subfd[j]; + geom_size[j] = geomst.st_size; } - - geomfd = geomfd2; - geom_size = geomst.st_size; } } @@ -385,8 +389,8 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i json_pull *jp = json_begin_file(f); long long seq = 0; - /* zoom, x, y of enclosing tile, so 0/0/0's parent is -1/0/0 */ - serialize_int(geomfile, -1, &geompos, fname, jp); + /* initial tile is 0/0/0 */ + serialize_int(geomfile, 0, &geompos, fname, jp); serialize_uint(geomfile, 0, &geompos, fname, jp); serialize_uint(geomfile, 0, &geompos, fname, jp); @@ -771,8 +775,19 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i } #endif + int fd[4]; + off_t size[4]; - check(geomfd, geomst.st_size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir); + fd[0] = geomfd; + size[0] = geomst.st_size; + + int j; + for (j = 1; j < 4; j++) { + fd[j] = -1; + size[j] = 0; + } + + check(fd, size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir); munmap(meta, metast.st_size); diff --git a/tile.cc b/tile.cc index c2cd1bb..d96c0ef 100644 --- a/tile.cc +++ b/tile.cc @@ -342,7 +342,7 @@ void evaluate(std::vector &features, char *metabase, struct pool *file pool_free(&keys); } -long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile) { +long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4]) { int line_detail; static bool evaluated = false; @@ -362,8 +362,8 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u std::vector features; - int within = 0; - long long geompos = 0; + int within[4] = { 0 }; + long long geompos[4] = { 0 }; *geoms = og; @@ -420,35 +420,64 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u if (line_detail == detail) { /* only write out the next zoom once, even if we retry */ if (geom.size() > 0) { - if (!within) { - serialize_int(geomfile, z, &geompos, fname, jp); - serialize_uint(geomfile, tx, &geompos, fname, jp); - serialize_uint(geomfile, ty, &geompos, fname, jp); - within = 1; - } + int j; + for (j = 0; j < 4; j++) { + int xo = j & 1; + int yo = (j >> 1) & 1; - // Offset from tile coordinates back to world coordinates - unsigned sx = 0, sy = 0; - if (z != 0) { - sx = tx << (32 - z); - sy = ty << (32 - z); - } + long long bbox2[4]; + int k; + for (k = 0; k < 4; k++) { + bbox2[k] = bbox[k]; + } + if (z != 0) { + // Offset back to world-relative + bbox2[0] += tx << (32 - z); + bbox2[1] += ty << (32 - z); + bbox2[2] += tx << (32 - z); + bbox2[3] += ty << (32 - z); + } + // Offset to child tile-relative + bbox2[0] -= (tx * 2 + xo) << (32 - (z + 1)); + bbox2[1] -= (ty * 2 + yo) << (32 - (z + 1)); + bbox2[2] -= (tx * 2 + xo) << (32 - (z + 1)); + bbox2[3] -= (ty * 2 + yo) << (32 - (z + 1)); - //printf("type %d, meta %lld\n", t, metastart); - serialize_int(geomfile, t, &geompos, fname, jp); - serialize_long_long(geomfile, metastart, &geompos, fname, jp); + int quick2 = quick_check(bbox2, z + 1, line_detail, buffer); + if (quick2 != 0) { + if (!within[j]) { + printf("begin %d/%d/%d\n", z + 1, tx * 2 + xo, ty * 2 + yo); - for (unsigned u = 0; u < geom.size(); u++) { - serialize_byte(geomfile, geom[u].op, &geompos, fname, jp); + serialize_int(geomfile[j], z + 1, &geompos[j], fname, jp); + serialize_uint(geomfile[j], tx * 2 + xo, &geompos[j], fname, jp); + serialize_uint(geomfile[j], ty * 2 + yo, &geompos[j], fname, jp); + within[j] = 1; + } - if (geom[u].op != VT_CLOSEPATH) { - serialize_uint(geomfile, geom[u].x + sx, &geompos, fname, jp); - serialize_uint(geomfile, geom[u].y + sy, &geompos, fname, jp); + // Offset from tile coordinates back to world coordinates + unsigned sx = 0, sy = 0; + if (z != 0) { + sx = tx << (32 - z); + sy = ty << (32 - z); + } + + //printf("type %d, meta %lld\n", t, metastart); + serialize_int(geomfile[j], t, &geompos[j], fname, jp); + serialize_long_long(geomfile[j], metastart, &geompos[j], fname, jp); + + for (unsigned u = 0; u < geom.size(); u++) { + serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname, jp); + + if (geom[u].op != VT_CLOSEPATH) { + serialize_uint(geomfile[j], geom[u].x + sx, &geompos[j], fname, jp); + serialize_uint(geomfile[j], geom[u].y + sy, &geompos[j], fname, jp); + } + } + + serialize_byte(geomfile[j], VT_END, &geompos[j], fname, jp); + serialize_byte(geomfile[j], minzoom, &geompos[j], fname, jp); } } - - serialize_byte(geomfile, VT_END, &geompos, fname, jp); - serialize_byte(geomfile, minzoom, &geompos, fname, jp); } } @@ -507,9 +536,12 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u } } - if (within) { - serialize_int(geomfile, -2, &geompos, fname, jp); - within = 0; + int j; + for (j = 0; j < 4; j++) { + if (within[j]) { + serialize_int(geomfile[j], -2, &geompos[j], fname, jp); + within[j] = 0; + } } std::sort(features.begin(), features.end()); diff --git a/tile.h b/tile.h index 75422bb..51451b0 100644 --- a/tile.h +++ b/tile.h @@ -36,4 +36,4 @@ struct index { int candup : 1; }; -long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile); +long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4]); From c177b8bed248c8dceab15af141ce24c085f01023 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 11 Dec 2014 16:08:53 -0800 Subject: [PATCH 08/17] No need to remove no-ops here unless we clipped --- tile.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tile.cc b/tile.cc index d96c0ef..e4e55a9 100644 --- a/tile.cc +++ b/tile.cc @@ -414,9 +414,9 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u if (t == VT_POINT) { geom = clip_point(geom, z, line_detail, buffer); } - } - geom = remove_noop(geom, t); + geom = remove_noop(geom, t); + } if (line_detail == detail) { /* only write out the next zoom once, even if we retry */ if (geom.size() > 0) { From 3f2818a814013a29c4a28da2a77d3426e92c60e2 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 11 Dec 2014 17:59:22 -0800 Subject: [PATCH 09/17] More useful progress indicator --- geojson.c | 22 ++++++++++++++++------ tile.cc | 2 -- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/geojson.c b/geojson.c index f7db816..a5de42d 100644 --- a/geojson.c +++ b/geojson.c @@ -213,7 +213,6 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type) { void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, const char *tmpdir) { int i; for (i = 0; i <= maxzoom; i++) { - fprintf(stderr, "\n"); long long most = 0; FILE *sub[4]; @@ -236,6 +235,12 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo unlink(geomname); } + long long todo = 0; + long long along = 0; + for (j = 0; j < 4; j++) { + todo += geom_size[j]; + } + for (j = 0; j < 4; j++) { if (geomfd[j] < 0) { // only one source file for zoom level 0 @@ -245,7 +250,7 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo continue; } - printf("%lld of geom_size\n", (long long) geom_size[j]); + // printf("%lld of geom_size\n", (long long) geom_size[j]); char *geom = mmap(NULL, geom_size[j], PROT_READ, MAP_PRIVATE, geomfd[j], 0); if (geom == MAP_FAILED) { @@ -253,6 +258,7 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo exit(EXIT_FAILURE); } + char *geomstart = geom; char *end = geom + geom_size[j]; while (geom < end) { @@ -263,9 +269,10 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo deserialize_uint(&geom, &x); deserialize_uint(&geom, &y); - fprintf(stderr, "%d/%u/%u\n", z, x, y); + // fprintf(stderr, "%d/%u/%u\n", z, x, y); - // fprintf(stderr, " %3.1f%% %d/%u/%u \r", (((i - ix) + (j - ix)) / 2.0 / n + (maxzoom - z)) / (maxzoom - minzoom + 1) * 100, z, tx, ty); + fprintf(stderr, " %3.1f%% %d/%u/%u \r", + (((geom - geomstart + along) / (double) todo) + z) / (maxzoom + 1) * 100, z, x, y); long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub); @@ -274,11 +281,10 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo *midy = y; most = len; } - - fprintf(stderr, "\n"); } munmap(geom, geom_size[j]); + along += geom_size[j]; } for (j = 0; j < 4; j++) { @@ -295,6 +301,8 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo geom_size[j] = geomst.st_size; } } + + fprintf(stderr, "\n"); } struct merge { @@ -787,6 +795,8 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i size[j] = 0; } + fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata\n", seq, geomst.st_size, metast.st_size); + check(fd, size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir); munmap(meta, metast.st_size); diff --git a/tile.cc b/tile.cc index e4e55a9..eadf4e4 100644 --- a/tile.cc +++ b/tile.cc @@ -446,8 +446,6 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u int quick2 = quick_check(bbox2, z + 1, line_detail, buffer); if (quick2 != 0) { if (!within[j]) { - printf("begin %d/%d/%d\n", z + 1, tx * 2 + xo, ty * 2 + yo); - serialize_int(geomfile[j], z + 1, &geompos[j], fname, jp); serialize_uint(geomfile[j], tx * 2 + xo, &geompos[j], fname, jp); serialize_uint(geomfile[j], ty * 2 + yo, &geompos[j], fname, jp); From c90ba8511f66a14215914d9c696ad8cda380254a Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 12 Dec 2014 14:52:12 -0800 Subject: [PATCH 10/17] Fix compiler warning --- geojson.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geojson.c b/geojson.c index ade6583..f607b19 100644 --- a/geojson.c +++ b/geojson.c @@ -815,7 +815,7 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i size[j] = 0; } - fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata\n", seq, geomst.st_size, metast.st_size); + fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata\n", seq, (long long) geomst.st_size, (long long) metast.st_size); check(fd, size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir); From ad17f1f282f65b42f9a57f04843a1851d0db67a9 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 17 Dec 2014 11:05:14 -0800 Subject: [PATCH 11/17] Make -Z0 work again for file minimum zoom --- geojson.c | 2 +- tile.cc | 16 ++++++++++------ tile.h | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/geojson.c b/geojson.c index f607b19..f0dbbb1 100644 --- a/geojson.c +++ b/geojson.c @@ -283,7 +283,7 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo fprintf(stderr, " %3.1f%% %d/%u/%u \r", (((geom - geomstart + along) / (double) todo) + z) / (maxzoom + 1) * 100, z, x, y); - long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub); + long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub, minzoom); if (z == maxzoom && len > most) { *midx = x; diff --git a/tile.cc b/tile.cc index eadf4e4..8a54d4e 100644 --- a/tile.cc +++ b/tile.cc @@ -342,7 +342,7 @@ void evaluate(std::vector &features, char *metabase, struct pool *file pool_free(&keys); } -long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4]) { +long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4], int file_minzoom) { int line_detail; static bool evaluated = false; @@ -390,8 +390,8 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u drawvec geom = decode_geometry(geoms, z, tx, ty, line_detail, bbox); - signed char minzoom; - deserialize_byte(geoms, &minzoom); + signed char feature_minzoom; + deserialize_byte(geoms, &feature_minzoom); int quick = quick_check(bbox, z, line_detail, buffer); if (quick == 0) { @@ -473,14 +473,18 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u } serialize_byte(geomfile[j], VT_END, &geompos[j], fname, jp); - serialize_byte(geomfile[j], minzoom, &geompos[j], fname, jp); + serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname, jp); } } } } - if ((t == VT_LINE && z + line_detail <= minzoom) || - (t == VT_POINT && z < minzoom)) { + if (z < file_minzoom) { + continue; + } + + if ((t == VT_LINE && z + line_detail <= feature_minzoom) || + (t == VT_POINT && z < feature_minzoom)) { continue; } diff --git a/tile.h b/tile.h index 51451b0..98090db 100644 --- a/tile.h +++ b/tile.h @@ -36,4 +36,4 @@ struct index { int candup : 1; }; -long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4]); +long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4], int file_minzoom); From 0d0a546b1ec53235f2981e1e19810162cd80b1a5 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 17 Dec 2014 11:10:46 -0800 Subject: [PATCH 12/17] No need to split features into child tiles when processing the final zoom. --- geojson.c | 2 +- tile.cc | 4 ++-- tile.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/geojson.c b/geojson.c index f0dbbb1..5beb437 100644 --- a/geojson.c +++ b/geojson.c @@ -283,7 +283,7 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo fprintf(stderr, " %3.1f%% %d/%u/%u \r", (((geom - geomstart + along) / (double) todo) + z) / (maxzoom + 1) * 100, z, x, y); - long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub, minzoom); + long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub, minzoom, maxzoom); if (z == maxzoom && len > most) { *midx = x; diff --git a/tile.cc b/tile.cc index 8a54d4e..5fd82a7 100644 --- a/tile.cc +++ b/tile.cc @@ -342,7 +342,7 @@ void evaluate(std::vector &features, char *metabase, struct pool *file pool_free(&keys); } -long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4], int file_minzoom) { +long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom) { int line_detail; static bool evaluated = false; @@ -419,7 +419,7 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u } if (line_detail == detail) { /* only write out the next zoom once, even if we retry */ - if (geom.size() > 0) { + if (geom.size() > 0 && z + 1 <= file_maxzoom) { int j; for (j = 0; j < 4; j++) { int xo = j & 1; diff --git a/tile.h b/tile.h index 98090db..59e0e68 100644 --- a/tile.h +++ b/tile.h @@ -36,4 +36,4 @@ struct index { int candup : 1; }; -long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4], int file_minzoom); +long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom); From cba1b8ae7f8eeb333339f97e6ddc0e877b21f0f5 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 17 Dec 2014 11:13:51 -0800 Subject: [PATCH 13/17] Remove unneeded old indexing code --- geojson.c | 227 ------------------------------------------------------ tile.cc | 15 ---- tile.h | 10 --- 3 files changed, 252 deletions(-) diff --git a/geojson.c b/geojson.c index 5beb437..a6bdadd 100644 --- a/geojson.c +++ b/geojson.c @@ -58,31 +58,6 @@ int mb_geometry[GEOM_TYPES] = { VT_POLYGON, }; -int indexcmp(const void *v1, const void *v2) { - const struct index *i1 = (const struct index *) v1; - const struct index *i2 = (const struct index *) v2; - - if (i1->index < i2->index) { - return -1; - } else if (i1->index > i2->index) { - return 1; - } - - if (i1->fpos < i2->fpos) { - return -1; - } else if (i1->fpos > i2->fpos) { - return 1; - } - - if (i1->maxzoom < i2->maxzoom) { - return -1; - } else if (i1->maxzoom > i2->maxzoom) { - return 1; - } - - return 0; -} - size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname, json_pull *source) { size_t w = fwrite(ptr, size, nitems, stream); if (w != nitems) { @@ -314,57 +289,6 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo fprintf(stderr, "\n"); } -struct merge { - long long start; - long long end; - - struct merge *next; -}; - -#if 0 -static void insert(struct merge *m, struct merge **head, unsigned char *map, int bytes) { - while (*head != NULL && indexcmp(map + m->start, map + (*head)->start) > 0) { - head = &((*head)->next); - } - - m->next = *head; - *head = m; -} - -static void merge(struct merge *merges, int nmerges, unsigned char *map, FILE *f, int bytes, long long nrec) { - int i; - struct merge *head = NULL; - long long along = 0; - long long reported = -1; - - for (i = 0; i < nmerges; i++) { - if (merges[i].start < merges[i].end) { - insert(&(merges[i]), &head, map, bytes); - } - } - - while (head != NULL) { - fwrite(map + head->start, bytes, 1, f); - head->start += bytes; - - struct merge *m = head; - head = m->next; - m->next = NULL; - - if (m->start < m->end) { - insert(m, &head, map, bytes); - } - - along++; - long long report = 100 * along / nrec; - if (report != reported) { - fprintf(stderr, "Merging: %lld%%\r", report); - reported = report; - } - } -} -#endif - void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir) { char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; @@ -534,10 +458,6 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i parse_geometry(t, coordinates, bbox, &geompos, geomfile, VT_MOVETO, fname, jp); serialize_byte(geomfile, VT_END, &geompos, fname, jp); -#if 0 - int z = maxzoom; -#endif - /* * Note that minzoom for lines is the dimension * of the geometry in world coordinates, but @@ -568,73 +488,6 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i serialize_byte(geomfile, minzoom, &geompos, fname, jp); -#if 0 - unsigned cx = bbox[0] / 2 + bbox[2] / 2; - unsigned cy = bbox[1] / 2 + bbox[3] / 2; - - /* XXX do proper overlap instead of whole bounding box */ - if (z == 0) { - struct index ix; - ix.index = encode(cx, cy); - ix.fpos = start; - ix.type = mb_geometry[t]; - ix.maxzoom = z; - ix.minzoom = minzoom; - fwrite_check(&ix, sizeof(struct index), 1, indexfile, fname, jp); - } else { - int pass; - int instances = 0; - - for (pass = 0; pass < 2; pass++) { - for (z = maxzoom; z >= 1; z--) { - unsigned x, y; - for (x = (bbox[0] - (buffer << (32 - z - 8))) >> (32 - z); x <= (bbox[2] + (buffer << (32 - z - 8))) >> (32 - z); x++) { - for (y = (bbox[1] - (buffer << (32 - z - 8))) >> (32 - z); y <= (bbox[3] + (buffer << (32 - z - 8))) >> (32 - z); y++) { - if (z != maxzoom) { - // There must be a clearer way to write this, but the intent is - // not to add an additional index for a low-zoom tile - // if one of its children was already part of the - // buffered bounding box for the child's zoom. - - // So we are comparing this tile's x and y to the edges of the - // bounding box at the next zoom down, but divided by two - // to get it back into this zoom's tile coordinate scheme - - if ((x >= ((bbox[0] - (buffer << (32 - (z + 1) - 8))) >> (32 - (z + 1)) >> 1)) && - (x <= ((bbox[2] + (buffer << (32 - (z + 1) - 8))) >> (32 - (z + 1)) >> 1)) && - (y >= ((bbox[1] - (buffer << (32 - (z + 1) - 8))) >> (32 - (z + 1)) >> 1)) && - (y <= ((bbox[3] + (buffer << (32 - (z + 1) - 8))) >> (32 - (z + 1)) >> 1))) { - continue; - } - } - - if (pass == 0) { - instances++; - if (instances > 1) { - break; - } - } else { - struct index ix; - - if (x == cx >> (32 - z) && y == cy >> (32 - z)) { - ix.index = encode(cx, cy); - } else { - ix.index = encode(x << (32 - z), y << (32 - z)); - } - ix.fpos = start; - ix.type = mb_geometry[t]; - ix.maxzoom = z; - ix.candup = (instances > 1); - ix.minzoom = minzoom; - fwrite_check(&ix, sizeof(struct index), 1, indexfile, fname, jp); - } - } - } - } - } - } -#endif - for (i = 0; i < 2; i++) { if (bbox[i] < file_bbox[i]) { file_bbox[i] = bbox[i]; @@ -723,86 +576,6 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i printf("using layer name %s\n", trunc); } -#if 0 - { - int bytes = sizeof(struct index); - - fprintf(stderr, - "Sorting %lld indices for %lld features\n", - (long long) indexst.st_size / bytes, - seq); - - int page = sysconf(_SC_PAGESIZE); - long long unit = (50 * 1024 * 1024 / bytes) * bytes; - while (unit % page != 0) { - unit += bytes; - } - - int nmerges = (indexst.st_size + unit - 1) / unit; - struct merge merges[nmerges]; - - long long start; - for (start = 0; start < indexst.st_size; start += unit) { - long long end = start + unit; - if (end > indexst.st_size) { - end = indexst.st_size; - } - - if (nmerges != 1) { - fprintf(stderr, "Sorting part %lld of %d\r", start / unit + 1, nmerges); - } - - merges[start / unit].start = start; - merges[start / unit].end = end; - merges[start / unit].next = NULL; - - void *map = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_PRIVATE, indexfd, start); - if (map == MAP_FAILED) { - perror("mmap"); - exit(EXIT_FAILURE); - } - - qsort(map, (end - start) / bytes, bytes, indexcmp); - - // Sorting and then copying avoids the need to - // write out intermediate stages of the sort. - - void *map2 = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_SHARED, indexfd, start); - if (map2 == MAP_FAILED) { - perror("mmap (write)"); - exit(EXIT_FAILURE); - } - - memcpy(map2, map, end - start); - - munmap(map, end - start); - munmap(map2, end - start); - } - - if (nmerges != 1) { - fprintf(stderr, "\n"); - } - - void *map = mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfd, 0); - if (map == MAP_FAILED) { - perror("mmap"); - exit(EXIT_FAILURE); - } - - FILE *f = fopen(indexname, "w"); - if (f == NULL) { - perror(indexname); - exit(EXIT_FAILURE); - } - - merge(merges, nmerges, (unsigned char *) map, f, bytes, indexst.st_size / bytes); - - munmap(map, indexst.st_size); - fclose(f); - close(indexfd); - } -#endif - int fd[4]; off_t size[4]; diff --git a/tile.cc b/tile.cc index 5fd82a7..d8542ca 100644 --- a/tile.cc +++ b/tile.cc @@ -374,15 +374,6 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u break; } -#if 0 - if (i->candup) { - if (dup.count(i->fpos) != 0) { - continue; - } - dup.insert(i->fpos); - } -#endif - long long metastart; deserialize_long_long(geoms, &metastart); char *meta = metabase + metastart; @@ -398,12 +389,6 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u continue; } -#if 0 - if (z > i->maxzoom) { - continue; - } -#endif - if (quick != 1) { if (t == VT_LINE) { geom = clip_lines(geom, z, line_detail, buffer); diff --git a/tile.h b/tile.h index 59e0e68..9ac106e 100644 --- a/tile.h +++ b/tile.h @@ -26,14 +26,4 @@ void deserialize_uint(char **f, unsigned *n); void deserialize_byte(char **f, signed char *n); struct pool_val *deserialize_string(char **f, struct pool *p, int type); - -struct index { - unsigned long long index; - long long fpos : 44; - int maxzoom : 6; - int minzoom : 6; - int type : 7; - int candup : 1; -}; - long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom); From 7f3551070ebf61b288241dd0d89370eeb3c04103 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 17 Dec 2014 14:01:20 -0800 Subject: [PATCH 14/17] Remove unneeded debug output --- geojson.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/geojson.c b/geojson.c index c6c4a8c..11d1c33 100644 --- a/geojson.c +++ b/geojson.c @@ -302,7 +302,6 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i exit(EXIT_FAILURE); } int geomfd = mkstemp(geomname); - printf("%s\n", geomname); if (geomfd < 0) { perror(geomname); exit(EXIT_FAILURE); @@ -522,8 +521,6 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i fclose(metafile); fclose(geomfile); - printf("bbox: %x %x %x %x\n", file_bbox[0], file_bbox[1], file_bbox[2], file_bbox[3]); - struct stat geomst; struct stat metast; From 48b5db6ae53fed3851148a41946d506f92339b95 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 17 Dec 2014 16:01:33 -0800 Subject: [PATCH 15/17] Keep the progress indicator progressing while working through big tiles --- geojson.c | 5 +---- tile.cc | 8 +++++++- tile.h | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/geojson.c b/geojson.c index 11d1c33..eee23ac 100644 --- a/geojson.c +++ b/geojson.c @@ -255,10 +255,7 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo // fprintf(stderr, "%d/%u/%u\n", z, x, y); - fprintf(stderr, " %3.1f%% %d/%u/%u \r", - (((geom - geomstart + along) / (double) todo) + z) / (maxzoom + 1) * 100, z, x, y); - - long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub, minzoom, maxzoom); + long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub, minzoom, maxzoom, todo, geomstart, along); if (z == maxzoom && len > most) { *midx = x; diff --git a/tile.cc b/tile.cc index d8542ca..a7fcb97 100644 --- a/tile.cc +++ b/tile.cc @@ -342,9 +342,10 @@ void evaluate(std::vector &features, char *metabase, struct pool *file pool_free(&keys); } -long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom) { +long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along) { int line_detail; static bool evaluated = false; + double oprogress = 0; char *og = *geoms; @@ -384,6 +385,11 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u signed char feature_minzoom; deserialize_byte(geoms, &feature_minzoom); + double progress = floor((((*geoms - geomstart + along) / (double) todo) + z) / (file_maxzoom + 1) * 1000) / 10; + if (progress != oprogress) { + fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); + } + int quick = quick_check(bbox, z, line_detail, buffer); if (quick == 0) { continue; diff --git a/tile.h b/tile.h index 9ac106e..6807f08 100644 --- a/tile.h +++ b/tile.h @@ -26,4 +26,4 @@ void deserialize_uint(char **f, unsigned *n); void deserialize_byte(char **f, signed char *n); struct pool_val *deserialize_string(char **f, struct pool *p, int type); -long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom); +long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along); From 32010fc893b5f32c33ef2fe0359e9cf3b6cc9454 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 17 Dec 2014 17:08:04 -0800 Subject: [PATCH 16/17] Oops. Do a lot less progress indicator I/O. --- tile.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tile.cc b/tile.cc index a7fcb97..d8fe408 100644 --- a/tile.cc +++ b/tile.cc @@ -388,6 +388,7 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u double progress = floor((((*geoms - geomstart + along) / (double) todo) + z) / (file_maxzoom + 1) * 1000) / 10; if (progress != oprogress) { fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); + oprogress = progress; } int quick = quick_check(bbox, z, line_detail, buffer); From 5a2003cb2ceb7318d3a52b3051d5f786ea7c91ae Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 17 Dec 2014 17:41:57 -0800 Subject: [PATCH 17/17] Fix failure to release temporary memory --- geojson.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/geojson.c b/geojson.c index eee23ac..2965ccb 100644 --- a/geojson.c +++ b/geojson.c @@ -264,7 +264,9 @@ void check(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbo } } - munmap(geom, geom_size[j]); + if (munmap(geomstart, geom_size[j]) != 0) { + perror("munmap geom"); + } along += geom_size[j]; } @@ -591,7 +593,9 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i check(fd, size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir); - munmap(meta, metast.st_size); + if (munmap(meta, metast.st_size) != 0) { + perror("munmap meta"); + } close(geomfd); close(metafd);