From b12413eddb78cb4bea0ebac071067b1c0c022fea Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 12:22:47 -0700 Subject: [PATCH 01/19] Rename everything from .cc to .cpp to match other projects --- .travis.yml | 2 +- Makefile | 6 +++--- decode.cc => decode.cpp | 2 +- geometry.cc => geometry.cpp | 2 +- geometry.hh => geometry.hpp | 0 mvt.cc => mvt.cpp | 2 +- mvt.hh => mvt.hpp | 0 tile-join.cc => tile-join.cpp | 2 +- tile.cc => tile.cpp | 4 ++-- 9 files changed, 10 insertions(+), 10 deletions(-) rename decode.cc => decode.cpp (99%) rename geometry.cc => geometry.cpp (99%) rename geometry.hh => geometry.hpp (100%) rename mvt.cc => mvt.cpp (99%) rename mvt.hh => mvt.hpp (100%) rename tile-join.cc => tile-join.cpp (99%) rename tile.cc => tile.cpp (99%) diff --git a/.travis.yml b/.travis.yml index f2488cc..344b838 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: install: - - BUILDTYPE=${BUILDTYPE} make + - BUILDTYPE=${BUILDTYPE} make -j script: - BUILDTYPE=${BUILDTYPE} make test diff --git a/Makefile b/Makefile index 4dfce70..0ed76d1 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,8 @@ man/tippecanoe.1: README.md PG= -H = $(shell find . '(' -name '*.h' -o -name '*.hh' ')') -C = $(shell find . '(' -name '*.c' -o -name '*.cc' ')') +H = $(shell find . '(' -name '*.h' -o -name '*.hpp' ')') +C = $(shell find . '(' -name '*.c' -o -name '*.cpp' ')') INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib @@ -61,7 +61,7 @@ libjsonpull.a: jsonpull.o %.o: %.c $(H) $(CC) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CFLAGS) -c $< -%.o: %.cc $(H) +%.o: %.cpp $(H) $(CXX) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c $< clean: diff --git a/decode.cc b/decode.cpp similarity index 99% rename from decode.cc rename to decode.cpp index c177ec1..4982bc4 100644 --- a/decode.cc +++ b/decode.cpp @@ -11,7 +11,7 @@ #include #include #include -#include "mvt.hh" +#include "mvt.hpp" #include "tile.h" extern "C" { diff --git a/geometry.cc b/geometry.cpp similarity index 99% rename from geometry.cc rename to geometry.cpp index 687b00c..2e12e07 100644 --- a/geometry.cc +++ b/geometry.cpp @@ -8,7 +8,7 @@ #include #include #include -#include "geometry.hh" +#include "geometry.hpp" #include "clipper/clipper.hpp" extern "C" { diff --git a/geometry.hh b/geometry.hpp similarity index 100% rename from geometry.hh rename to geometry.hpp diff --git a/mvt.cc b/mvt.cpp similarity index 99% rename from mvt.cc rename to mvt.cpp index 679d0e8..9223a02 100644 --- a/mvt.cc +++ b/mvt.cpp @@ -4,7 +4,7 @@ #include #include #include -#include "mvt.hh" +#include "mvt.hpp" #include "protozero/varint.hpp" #include "protozero/pbf_reader.hpp" #include "protozero/pbf_writer.hpp" diff --git a/mvt.hh b/mvt.hpp similarity index 100% rename from mvt.hh rename to mvt.hpp diff --git a/tile-join.cc b/tile-join.cpp similarity index 99% rename from tile-join.cc rename to tile-join.cpp index 21d67c4..518ba4d 100644 --- a/tile-join.cc +++ b/tile-join.cpp @@ -9,7 +9,7 @@ #include #include #include -#include "mvt.hh" +#include "mvt.hpp" #include "tile.h" extern "C" { diff --git a/tile.cc b/tile.cpp similarity index 99% rename from tile.cc rename to tile.cpp index 9673909..f1c7d32 100644 --- a/tile.cc +++ b/tile.cpp @@ -19,8 +19,8 @@ #include #include #include -#include "mvt.hh" -#include "geometry.hh" +#include "mvt.hpp" +#include "geometry.hpp" extern "C" { #include "tile.h" From d32d4bb35f9b2b21cdc4e946fa7dc97dffc673c1 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 12:25:03 -0700 Subject: [PATCH 02/19] No need for line clipping to have its own source file --- Makefile | 7 +++-- clip.c | 84 --------------------------------------------------- clip.h | 1 - geometry.cpp | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++- tile.cpp | 1 - 5 files changed, 88 insertions(+), 90 deletions(-) delete mode 100644 clip.c delete mode 100644 clip.h diff --git a/Makefile b/Makefile index 0ed76d1..befde48 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ C = $(shell find . '(' -name '*.c' -o -name '*.cpp' ')') INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o mvt.o +tippecanoe: geojson.o jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper.o mvt.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o @@ -64,14 +64,15 @@ libjsonpull.a: jsonpull.o %.o: %.cpp $(H) $(CXX) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c $< +clipper.o: clipper/clipper.cpp $(H) + $(CXX) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c $< + clean: rm -f tippecanoe *.o indent: clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false}" $(C) $(H) -geometry.o: clipper/clipper.hpp - TESTS = $(wildcard tests/*/out/*.json) SPACE = $(NULL) $(NULL) diff --git a/clip.c b/clip.c deleted file mode 100644 index 5464b28..0000000 --- a/clip.c +++ /dev/null @@ -1,84 +0,0 @@ -#include "clip.h" - -#define INSIDE 0 -#define LEFT 1 -#define RIGHT 2 -#define BOTTOM 4 -#define TOP 8 - -static int computeOutCode(double x, double y, double xmin, double ymin, double xmax, double ymax) { - int code = INSIDE; - - if (x < xmin) { - code |= LEFT; - } else if (x > xmax) { - code |= RIGHT; - } - - if (y < ymin) { - code |= BOTTOM; - } else if (y > ymax) { - code |= TOP; - } - - return code; -} - -int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax) { - int outcode0 = computeOutCode(*x0, *y0, xmin, ymin, xmax, ymax); - int outcode1 = computeOutCode(*x1, *y1, xmin, ymin, xmax, ymax); - int accept = 0; - int changed = 0; - - while (1) { - if (!(outcode0 | outcode1)) { // Bitwise OR is 0. Trivially accept and get out of loop - accept = 1; - break; - } else if (outcode0 & outcode1) { // Bitwise AND is not 0. Trivially reject and get out of loop - break; - } else { - // failed both tests, so calculate the line segment to clip - // from an outside point to an intersection with clip edge - double x = *x0, y = *y0; - - // At least one endpoint is outside the clip rectangle; pick it. - int outcodeOut = outcode0 ? outcode0 : outcode1; - - // Now find the intersection point; - // use formulas y = y0 + slope * (x - x0), x = x0 + (1 / slope) * (y - y0) - if (outcodeOut & TOP) { // point is above the clip rectangle - x = *x0 + (*x1 - *x0) * (ymax - *y0) / (*y1 - *y0); - y = ymax; - } else if (outcodeOut & BOTTOM) { // point is below the clip rectangle - x = *x0 + (*x1 - *x0) * (ymin - *y0) / (*y1 - *y0); - y = ymin; - } else if (outcodeOut & RIGHT) { // point is to the right of clip rectangle - y = *y0 + (*y1 - *y0) * (xmax - *x0) / (*x1 - *x0); - x = xmax; - } else if (outcodeOut & LEFT) { // point is to the left of clip rectangle - y = *y0 + (*y1 - *y0) * (xmin - *x0) / (*x1 - *x0); - x = xmin; - } - - // Now we move outside point to intersection point to clip - // and get ready for next pass. - if (outcodeOut == outcode0) { - *x0 = x; - *y0 = y; - outcode0 = computeOutCode(*x0, *y0, xmin, ymin, xmax, ymax); - changed = 1; - } else { - *x1 = x; - *y1 = y; - outcode1 = computeOutCode(*x1, *y1, xmin, ymin, xmax, ymax); - changed = 1; - } - } - } - - if (accept == 0) { - return 0; - } else { - return changed + 1; - } -} diff --git a/clip.h b/clip.h deleted file mode 100644 index 513a980..0000000 --- a/clip.h +++ /dev/null @@ -1 +0,0 @@ -int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax); diff --git a/geometry.cpp b/geometry.cpp index 2e12e07..cb868f3 100644 --- a/geometry.cpp +++ b/geometry.cpp @@ -14,11 +14,11 @@ extern "C" { #include #include "tile.h" -#include "clip.h" #include "projection.h" } static int pnpoly(drawvec &vert, size_t start, size_t nvert, long long testx, long long testy); +static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax); drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y) { drawvec out; @@ -1310,3 +1310,86 @@ std::vector chop_polygon(std::vector &geoms) { geoms = out; } } + +#define INSIDE 0 +#define LEFT 1 +#define RIGHT 2 +#define BOTTOM 4 +#define TOP 8 + +static int computeOutCode(double x, double y, double xmin, double ymin, double xmax, double ymax) { + int code = INSIDE; + + if (x < xmin) { + code |= LEFT; + } else if (x > xmax) { + code |= RIGHT; + } + + if (y < ymin) { + code |= BOTTOM; + } else if (y > ymax) { + code |= TOP; + } + + return code; +} + +static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax) { + int outcode0 = computeOutCode(*x0, *y0, xmin, ymin, xmax, ymax); + int outcode1 = computeOutCode(*x1, *y1, xmin, ymin, xmax, ymax); + int accept = 0; + int changed = 0; + + while (1) { + if (!(outcode0 | outcode1)) { // Bitwise OR is 0. Trivially accept and get out of loop + accept = 1; + break; + } else if (outcode0 & outcode1) { // Bitwise AND is not 0. Trivially reject and get out of loop + break; + } else { + // failed both tests, so calculate the line segment to clip + // from an outside point to an intersection with clip edge + double x = *x0, y = *y0; + + // At least one endpoint is outside the clip rectangle; pick it. + int outcodeOut = outcode0 ? outcode0 : outcode1; + + // Now find the intersection point; + // use formulas y = y0 + slope * (x - x0), x = x0 + (1 / slope) * (y - y0) + if (outcodeOut & TOP) { // point is above the clip rectangle + x = *x0 + (*x1 - *x0) * (ymax - *y0) / (*y1 - *y0); + y = ymax; + } else if (outcodeOut & BOTTOM) { // point is below the clip rectangle + x = *x0 + (*x1 - *x0) * (ymin - *y0) / (*y1 - *y0); + y = ymin; + } else if (outcodeOut & RIGHT) { // point is to the right of clip rectangle + y = *y0 + (*y1 - *y0) * (xmax - *x0) / (*x1 - *x0); + x = xmax; + } else if (outcodeOut & LEFT) { // point is to the left of clip rectangle + y = *y0 + (*y1 - *y0) * (xmin - *x0) / (*x1 - *x0); + x = xmin; + } + + // Now we move outside point to intersection point to clip + // and get ready for next pass. + if (outcodeOut == outcode0) { + *x0 = x; + *y0 = y; + outcode0 = computeOutCode(*x0, *y0, xmin, ymin, xmax, ymax); + changed = 1; + } else { + *x1 = x; + *y1 = y; + outcode1 = computeOutCode(*x1, *y1, xmin, ymin, xmax, ymax); + changed = 1; + } + } + } + + if (accept == 0) { + return 0; + } else { + return changed + 1; + } +} diff --git a/tile.cpp b/tile.cpp index f1c7d32..389cb8e 100644 --- a/tile.cpp +++ b/tile.cpp @@ -25,7 +25,6 @@ extern "C" { #include "tile.h" #include "pool.h" -#include "clip.h" #include "mbtiles.h" #include "projection.h" } From 94db232a89cc79158aa515d2b0ffed946edfcb9b Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 12:38:04 -0700 Subject: [PATCH 03/19] Drag mbtiles handling into C++ --- mbtiles.c => mbtiles.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) rename mbtiles.c => mbtiles.cpp (96%) diff --git a/mbtiles.c b/mbtiles.cpp similarity index 96% rename from mbtiles.c rename to mbtiles.cpp index def1b94..2581d57 100644 --- a/mbtiles.c +++ b/mbtiles.cpp @@ -7,7 +7,10 @@ #include #include "pool.h" #include "tile.h" -#include "mbtiles.h" + +extern "C" { + #include "mbtiles.h" +}; sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) { sqlite3 *outdb; @@ -98,7 +101,7 @@ static void quote(char **buf, const char *s) { } *out = '\0'; - *buf = realloc(*buf, strlen(*buf) + strlen(tmp) + 1); + *buf = (char *) realloc(*buf, strlen(*buf) + strlen(tmp) + 1); if (*buf == NULL) { perror("realloc"); exit(EXIT_FAILURE); @@ -117,7 +120,7 @@ static void aprintf(char **buf, const char *format, ...) { } va_end(ap); - *buf = realloc(*buf, strlen(*buf) + strlen(tmp) + 1); + *buf = (char *) realloc(*buf, strlen(*buf) + strlen(tmp) + 1); if (*buf == NULL) { perror("Out of memory"); exit(EXIT_FAILURE); @@ -127,8 +130,8 @@ static void aprintf(char **buf, const char *format, ...) { } static int pvcmp(const void *v1, const void *v2) { - const struct pool_val *const *pv1 = v1; - const struct pool_val *const *pv2 = v2; + const struct pool_val *const *pv1 = (const struct pool_val *const *) v1; + const struct pool_val *const *pv2 = (const struct pool_val *const *) v2; int n = strcmp((*pv1)->s, (*pv2)->s); if (n != 0) { From 575072bb2fad421ce0edc45cbfb2f9ca667497f8 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 12:39:21 -0700 Subject: [PATCH 04/19] Drag projection math into C++ --- projection.c => projection.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) rename projection.c => projection.cpp (97%) diff --git a/projection.c b/projection.cpp similarity index 97% rename from projection.c rename to projection.cpp index d1ca865..0009215 100644 --- a/projection.c +++ b/projection.cpp @@ -1,5 +1,8 @@ #include -#include "projection.h" + +extern "C" { + #include "projection.h" +} // http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames void latlon2tile(double lat, double lon, int zoom, long long *x, long long *y) { From fb9f3b60689bc911470d71dc8a342fbce8a029e4 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 12:40:19 -0700 Subject: [PATCH 05/19] Drag tileset enumeration into C++ --- enumerate.c => enumerate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename enumerate.c => enumerate.cpp (94%) diff --git a/enumerate.c b/enumerate.cpp similarity index 94% rename from enumerate.c rename to enumerate.cpp index a66cfc8..00c63a9 100644 --- a/enumerate.c +++ b/enumerate.cpp @@ -11,7 +11,7 @@ void enumerate(char *fname) { exit(EXIT_FAILURE); } - char *sql = "SELECT zoom_level, tile_column, tile_row from tiles;"; + const char *sql = "SELECT zoom_level, tile_column, tile_row from tiles;"; sqlite3_stmt *stmt; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { From 3d56a56464b6ec61725e807954bc59fbe9c03516 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 12:41:49 -0700 Subject: [PATCH 06/19] Drag memory-mapped file handling into C++ --- memfile.c => memfile.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) rename memfile.c => memfile.cpp (77%) diff --git a/memfile.c b/memfile.cpp similarity index 77% rename from memfile.c rename to memfile.cpp index 68b7d7d..9519e5d 100644 --- a/memfile.c +++ b/memfile.cpp @@ -2,7 +2,10 @@ #include #include #include -#include "memfile.h" + +extern "C" { + #include "memfile.h" +} #define INCREMENT 131072 @@ -11,12 +14,12 @@ struct memfile *memfile_open(int fd) { return NULL; } - char *map = mmap(NULL, INCREMENT, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + char *map = (char *) mmap(NULL, INCREMENT, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { return NULL; } - struct memfile *mf = malloc(sizeof(struct memfile)); + struct memfile *mf = (struct memfile *) malloc(sizeof(struct memfile)); if (mf == NULL) { munmap(map, INCREMENT); return NULL; @@ -58,7 +61,7 @@ int memfile_write(struct memfile *file, void *s, long long len) { return -1; } - file->map = mmap(NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0); + file->map = (char *) mmap(NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0); if (file->map == MAP_FAILED) { return -1; } From 3f3a341c0a6fdf28e5a1d75ea891ed4af1ef5e80 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 12:44:46 -0700 Subject: [PATCH 07/19] Drag constant pool handling into C++ --- pool.c => pool.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) rename pool.c => pool.cpp (90%) diff --git a/pool.c b/pool.cpp similarity index 90% rename from pool.c rename to pool.cpp index 4c46bef..0859bb6 100644 --- a/pool.c +++ b/pool.cpp @@ -1,7 +1,10 @@ #include #include #include -#include "pool.h" + +extern "C" { + #include "pool.h" +} #define POOL_WIDTH 256 @@ -34,7 +37,7 @@ struct pool_val *pool(struct pool *p, const char *s, int type) { } } - struct pool_val *nv = malloc(sizeof(struct pool_val)); + struct pool_val *nv = (struct pool_val *) malloc(sizeof(struct pool_val)); if (nv == NULL) { fprintf(stderr, "out of memory making string pool\n"); exit(EXIT_FAILURE); @@ -109,7 +112,7 @@ void pool_free_strings(struct pool *p) { void pool_init(struct pool *p, int n) { p->n = n; - p->vals = calloc(POOL_WIDTH, sizeof(struct pool_val *)); + p->vals = (struct pool_val **) calloc(POOL_WIDTH, sizeof(struct pool_val *)); if (p->vals == NULL) { fprintf(stderr, "out of memory creating string pool\n"); exit(EXIT_FAILURE); From 7b0bb9a44340e195918d921991c64fe84a143eb2 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 13:54:00 -0700 Subject: [PATCH 08/19] Drag the main function and GeoJSON parsing into C++ --- geojson.c => geojson.cpp | 52 +++++++++++++++++++++------------------- mbtiles.cpp | 2 +- memfile.cpp | 2 +- pool.cpp | 2 +- projection.cpp | 2 +- 5 files changed, 31 insertions(+), 29 deletions(-) rename geojson.c => geojson.cpp (97%) diff --git a/geojson.c b/geojson.cpp similarity index 97% rename from geojson.c rename to geojson.cpp index 009ac9a..826daa3 100644 --- a/geojson.c +++ b/geojson.cpp @@ -30,6 +30,7 @@ #include #endif +extern "C" { #include "jsonpull.h" #include "tile.h" #include "pool.h" @@ -37,6 +38,7 @@ #include "projection.h" #include "version.h" #include "memfile.h" +} static int low_detail = 12; static int full_detail = -1; @@ -85,7 +87,7 @@ struct tofree { } *tofree = NULL; void mustfree(void *p) { - struct tofree *f = malloc(sizeof(struct tofree)); + struct tofree *f = (struct tofree *) malloc(sizeof(struct tofree)); if (f == NULL) { perror("malloc"); exit(EXIT_FAILURE); @@ -530,7 +532,7 @@ static unsigned char swizzle[256] = { 0xF9, 0xFE, 0x0C, 0x99, 0x2D, 0x0F, 0x3A, 0x41, 0x45, 0xA8, 0x30, 0x2B, 0x73, 0xBD, 0x86, 0x81, }; -int swizzlecmp(char *a, char *b) { +int swizzlecmp(const char *a, const char *b) { while (*a || *b) { int aa = swizzle[(unsigned char) *a]; int bb = swizzle[(unsigned char) *b]; @@ -547,7 +549,7 @@ int swizzlecmp(char *a, char *b) { return 0; } -long long addpool(struct memfile *poolfile, struct memfile *treefile, char *s, char type) { +long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type) { long long *sp = &treefile->tree; while (*sp != 0) { @@ -579,7 +581,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, char *s, c perror("memfile write"); exit(EXIT_FAILURE); } - if (memfile_write(poolfile, s, strlen(s) + 1) < 0) { + if (memfile_write(poolfile, (void *) s, strlen(s) + 1) < 0) { perror("memfile write"); exit(EXIT_FAILURE); } @@ -667,7 +669,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha long long metastart = *metapos; char *metakey[nprop]; - char *metaval[nprop]; + const char *metaval[nprop]; int metatype[nprop]; int mustfree[nprop]; int m = 0; @@ -714,7 +716,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname); if (mustfree[i]) { - free(metaval[i]); + free((void *) metaval[i]); } } @@ -947,7 +949,7 @@ struct parse_json_args { }; void *run_parse_json(void *v) { - struct parse_json_args *pja = v; + struct parse_json_args *pja = (struct parse_json_args *) v; parse_json(pja->jp, pja->reading, pja->layer_seq, pja->progress_seq, pja->metapos, pja->geompos, pja->indexpos, pja->exclude, pja->include, pja->exclude_all, pja->metafile, pja->geomfile, pja->indexfile, pja->poolfile, pja->treefile, pja->fname, pja->basezoom, pja->layer, pja->droprate, pja->file_bbox, pja->segment, pja->initialized, pja->initial_x, pja->initial_y, pja->readers); @@ -961,7 +963,7 @@ struct jsonmap { }; ssize_t json_map_read(struct json_pull *jp, char *buffer, size_t n) { - struct jsonmap *jm = jp->source; + struct jsonmap *jm = (struct jsonmap *) jp->source; if (jm->off + n >= jm->end) { n = jm->end - jm->off; @@ -974,7 +976,7 @@ ssize_t json_map_read(struct json_pull *jp, char *buffer, size_t n) { } struct json_pull *json_begin_map(char *map, long long len) { - struct jsonmap *jm = malloc(sizeof(struct jsonmap)); + struct jsonmap *jm = (struct jsonmap *) malloc(sizeof(struct jsonmap)); if (jm == NULL) { perror("Out of memory"); exit(EXIT_FAILURE); @@ -999,7 +1001,7 @@ struct sort_arg { }; void *run_sort(void *v) { - struct sort_arg *a = v; + struct sort_arg *a = (struct sort_arg *) v; long long start; for (start = a->task * a->unit; start < a->indexpos; start += a->unit * a->cpus) { @@ -1137,7 +1139,7 @@ struct read_parallel_arg { }; void *run_read_parallel(void *v) { - struct read_parallel_arg *a = v; + struct read_parallel_arg *a = (struct read_parallel_arg *) v; struct stat st; if (fstat(a->fd, &st) != 0) { @@ -1148,7 +1150,7 @@ void *run_read_parallel(void *v) { } a->len = st.st_size; - char *map = mmap(NULL, a->len, PROT_READ, MAP_PRIVATE, a->fd, 0); + char *map = (char *) mmap(NULL, a->len, PROT_READ, MAP_PRIVATE, a->fd, 0); if (map == NULL || map == MAP_FAILED) { perror("map intermediate input"); exit(EXIT_FAILURE); @@ -1179,7 +1181,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i *is_parsing = 1; - struct read_parallel_arg *rpa = malloc(sizeof(struct read_parallel_arg)); + struct read_parallel_arg *rpa = (struct read_parallel_arg *) malloc(sizeof(struct read_parallel_arg)); if (rpa == NULL) { perror("Out of memory"); exit(EXIT_FAILURE); @@ -1272,7 +1274,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split } if (indexst.st_size != 0) { - struct index *indexmap = mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds_in[i], 0); + struct index *indexmap = (struct index *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds_in[i], 0); if (indexmap == MAP_FAILED) { fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds_in[i], (long long) indexst.st_size); perror("map index"); @@ -1280,7 +1282,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split } madvise(indexmap, indexst.st_size, MADV_SEQUENTIAL); madvise(indexmap, indexst.st_size, MADV_WILLNEED); - char *geommap = mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds_in[i], 0); + char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds_in[i], 0); if (geommap == MAP_FAILED) { perror("map geom"); exit(EXIT_FAILURE); @@ -1410,7 +1412,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split } } - struct indexmap *indexmap = mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0); + struct indexmap *indexmap = (struct indexmap *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0); if (indexmap == MAP_FAILED) { fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds[i], (long long) indexst.st_size); perror("map index"); @@ -1418,7 +1420,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split } madvise(indexmap, indexst.st_size, MADV_RANDOM); // sequential, but from several pointers at once madvise(indexmap, indexst.st_size, MADV_WILLNEED); - char *geommap = mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0); + char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0); if (geommap == MAP_FAILED) { perror("map geom"); exit(EXIT_FAILURE); @@ -1439,7 +1441,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split exit(EXIT_FAILURE); } } else if (indexst.st_size == sizeof(struct index) || prefix + splitbits >= 64) { - struct index *indexmap = mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0); + struct index *indexmap = (struct index *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0); if (indexmap == MAP_FAILED) { fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds[i], (long long) indexst.st_size); perror("map index"); @@ -1447,7 +1449,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split } madvise(indexmap, indexst.st_size, MADV_SEQUENTIAL); madvise(indexmap, indexst.st_size, MADV_WILLNEED); - char *geommap = mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0); + char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0); if (geommap == MAP_FAILED) { perror("map geom"); exit(EXIT_FAILURE); @@ -1738,7 +1740,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay if (fstat(fd, &st) == 0) { off = lseek(fd, 0, SEEK_CUR); if (off >= 0) { - map = mmap(NULL, st.st_size - off, PROT_READ, MAP_PRIVATE, fd, off); + map = (char *) mmap(NULL, st.st_size - off, PROT_READ, MAP_PRIVATE, fd, off); // No error if MAP_FAILED because check is below if (map != MAP_FAILED) { madvise(map, st.st_size - off, MADV_RANDOM); // sequential, but from several pointers at once @@ -1932,7 +1934,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay src = sourcelist[i]->file; } - char *trunc = layernames[i] = malloc(strlen(src) + 1); + char *trunc = layernames[i] = (char *) malloc(strlen(src) + 1); if (trunc == NULL) { perror("Out of memory"); exit(EXIT_FAILURE); @@ -2146,7 +2148,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay } if (basezoom < 0 || droprate < 0) { - struct index *map = mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0); + struct index *map = (struct index *) mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0); if (map == MAP_FAILED) { perror("mmap index for basezoom"); exit(EXIT_FAILURE); @@ -2360,7 +2362,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay } if (poolpos > 0) { - madvise(pool, poolpos, MADV_DONTNEED); + madvise((void *) pool, poolpos, MADV_DONTNEED); if (munmap(stringpool, poolpos) != 0) { perror("munmap stringpool"); } @@ -2585,7 +2587,7 @@ int main(int argc, char **argv) { fprintf(stderr, "%s: -L requires layername:file\n", argv[0]); exit(EXIT_FAILURE); } - struct source *src = malloc(sizeof(struct source)); + struct source *src = (struct source *) malloc(sizeof(struct source)); if (src == NULL) { perror("Out of memory"); exit(EXIT_FAILURE); @@ -2830,7 +2832,7 @@ int main(int argc, char **argv) { int ret = EXIT_SUCCESS; for (i = optind; i < argc; i++) { - struct source *src = malloc(sizeof(struct source)); + struct source *src = (struct source *) malloc(sizeof(struct source)); if (src == NULL) { perror("Out of memory"); exit(EXIT_FAILURE); diff --git a/mbtiles.cpp b/mbtiles.cpp index 2581d57..f7f018e 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -9,7 +9,7 @@ #include "tile.h" extern "C" { - #include "mbtiles.h" +#include "mbtiles.h" }; sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) { diff --git a/memfile.cpp b/memfile.cpp index 9519e5d..d75cef3 100644 --- a/memfile.cpp +++ b/memfile.cpp @@ -4,7 +4,7 @@ #include extern "C" { - #include "memfile.h" +#include "memfile.h" } #define INCREMENT 131072 diff --git a/pool.cpp b/pool.cpp index 0859bb6..e6218fc 100644 --- a/pool.cpp +++ b/pool.cpp @@ -3,7 +3,7 @@ #include extern "C" { - #include "pool.h" +#include "pool.h" } #define POOL_WIDTH 256 diff --git a/projection.cpp b/projection.cpp index 0009215..534646c 100644 --- a/projection.cpp +++ b/projection.cpp @@ -1,7 +1,7 @@ #include extern "C" { - #include "projection.h" +#include "projection.h" } // http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames From a57c24750844dfdc06460e21d14509569553f328 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 13:55:28 -0700 Subject: [PATCH 09/19] Don't try to reindent Clipper and Protozero --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index befde48..2bff6ed 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,8 @@ man/tippecanoe.1: README.md PG= -H = $(shell find . '(' -name '*.h' -o -name '*.hpp' ')') -C = $(shell find . '(' -name '*.c' -o -name '*.cpp' ')') +H = $(wildcard *.h) $(wildcard *.hpp) +C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib From 65253cba50b4e8233ff6bd323959270e64fae918 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 14:00:14 -0700 Subject: [PATCH 10/19] Drag header files into C++ --- decode.cpp | 7 ++----- geojson.cpp | 13 +++++++------ geometry.cpp | 9 +++------ mbtiles.cpp | 9 +++------ mbtiles.h => mbtiles.hpp | 0 memfile.cpp | 5 +---- memfile.h => memfile.hpp | 0 pool.cpp | 5 +---- pool.h => pool.hpp | 0 projection.cpp | 5 +---- projection.h => projection.hpp | 0 tile-join.cpp | 11 ++++------- tile.cpp | 11 ++++------- tile.h => tile.hpp | 0 version.h => version.hpp | 0 15 files changed, 26 insertions(+), 49 deletions(-) rename mbtiles.h => mbtiles.hpp (100%) rename memfile.h => memfile.hpp (100%) rename pool.h => pool.hpp (100%) rename projection.h => projection.hpp (100%) rename tile.h => tile.hpp (100%) rename version.h => version.hpp (100%) diff --git a/decode.cpp b/decode.cpp index 4982bc4..f52d3d6 100644 --- a/decode.cpp +++ b/decode.cpp @@ -12,11 +12,8 @@ #include #include #include "mvt.hpp" -#include "tile.h" - -extern "C" { -#include "projection.h" -} +#include "tile.hpp" +#include "projection.hpp" void printq(const char *s) { putchar('"'); diff --git a/geojson.cpp b/geojson.cpp index 826daa3..2de135c 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -32,14 +32,15 @@ extern "C" { #include "jsonpull.h" -#include "tile.h" -#include "pool.h" -#include "mbtiles.h" -#include "projection.h" -#include "version.h" -#include "memfile.h" } +#include "tile.hpp" +#include "pool.hpp" +#include "mbtiles.hpp" +#include "projection.hpp" +#include "version.hpp" +#include "memfile.hpp" + static int low_detail = 12; static int full_detail = -1; static int min_detail = 7; diff --git a/geometry.cpp b/geometry.cpp index cb868f3..ce185d1 100644 --- a/geometry.cpp +++ b/geometry.cpp @@ -8,14 +8,11 @@ #include #include #include +#include #include "geometry.hpp" #include "clipper/clipper.hpp" - -extern "C" { -#include -#include "tile.h" -#include "projection.h" -} +#include "tile.hpp" +#include "projection.hpp" static int pnpoly(drawvec &vert, size_t start, size_t nvert, long long testx, long long testy); static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax); diff --git a/mbtiles.cpp b/mbtiles.cpp index f7f018e..216bdbb 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -5,12 +5,9 @@ #include #include #include -#include "pool.h" -#include "tile.h" - -extern "C" { -#include "mbtiles.h" -}; +#include "pool.hpp" +#include "tile.hpp" +#include "mbtiles.hpp" sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) { sqlite3 *outdb; diff --git a/mbtiles.h b/mbtiles.hpp similarity index 100% rename from mbtiles.h rename to mbtiles.hpp diff --git a/memfile.cpp b/memfile.cpp index d75cef3..a763c71 100644 --- a/memfile.cpp +++ b/memfile.cpp @@ -2,10 +2,7 @@ #include #include #include - -extern "C" { -#include "memfile.h" -} +#include "memfile.hpp" #define INCREMENT 131072 diff --git a/memfile.h b/memfile.hpp similarity index 100% rename from memfile.h rename to memfile.hpp diff --git a/pool.cpp b/pool.cpp index e6218fc..050a2c5 100644 --- a/pool.cpp +++ b/pool.cpp @@ -1,10 +1,7 @@ #include #include #include - -extern "C" { -#include "pool.h" -} +#include "pool.hpp" #define POOL_WIDTH 256 diff --git a/pool.h b/pool.hpp similarity index 100% rename from pool.h rename to pool.hpp diff --git a/projection.cpp b/projection.cpp index 534646c..521c7ed 100644 --- a/projection.cpp +++ b/projection.cpp @@ -1,8 +1,5 @@ #include - -extern "C" { -#include "projection.h" -} +#include "projection.hpp" // http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames void latlon2tile(double lat, double lon, int zoom, long long *x, long long *y) { diff --git a/projection.h b/projection.hpp similarity index 100% rename from projection.h rename to projection.hpp diff --git a/tile-join.cpp b/tile-join.cpp index 518ba4d..b265264 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -10,13 +10,10 @@ #include #include #include "mvt.hpp" -#include "tile.h" - -extern "C" { -#include "projection.h" -#include "pool.h" -#include "mbtiles.h" -} +#include "tile.hpp" +#include "projection.hpp" +#include "pool.hpp" +#include "mbtiles.hpp" std::string dequote(std::string s); diff --git a/tile.cpp b/tile.cpp index 389cb8e..c670019 100644 --- a/tile.cpp +++ b/tile.cpp @@ -21,13 +21,10 @@ #include #include "mvt.hpp" #include "geometry.hpp" - -extern "C" { -#include "tile.h" -#include "pool.h" -#include "mbtiles.h" -#include "projection.h" -} +#include "tile.hpp" +#include "pool.hpp" +#include "mbtiles.hpp" +#include "projection.hpp" #define CMD_BITS 3 diff --git a/tile.h b/tile.hpp similarity index 100% rename from tile.h rename to tile.hpp diff --git a/version.h b/version.hpp similarity index 100% rename from version.h rename to version.hpp From f3b9e15267e5c0c838798968027f391f6282ce8f Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 14:19:10 -0700 Subject: [PATCH 11/19] Move serialization code to its own file --- Makefile | 2 +- geojson.cpp | 134 +------------------------------------------------ geometry.cpp | 1 + serial.cpp | 138 +++++++++++++++++++++++++++++++++++++++++++++++++++ serial.hpp | 17 +++++++ tile.cpp | 1 + tile.hpp | 16 ------ 7 files changed, 159 insertions(+), 150 deletions(-) create mode 100644 serial.cpp create mode 100644 serial.hpp diff --git a/Makefile b/Makefile index 2bff6ed..5b81910 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper.o mvt.o +tippecanoe: geojson.o jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper.o mvt.o serial.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o diff --git a/geojson.cpp b/geojson.cpp index 2de135c..7357956 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -40,6 +40,7 @@ extern "C" { #include "projection.hpp" #include "version.hpp" #include "memfile.hpp" +#include "serial.hpp" static int low_detail = 12; static int full_detail = -1; @@ -206,53 +207,6 @@ void init_cpus() { } } -size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) { - size_t w = fwrite(ptr, size, nitems, stream); - if (w != nitems) { - fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); - exit(EXIT_FAILURE); - } - return w; -} - -void serialize_int(FILE *out, int n, long long *fpos, const char *fname) { - serialize_long_long(out, n, fpos, fname); -} - -void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname) { - unsigned long long zigzag = (n << 1) ^ (n >> 63); - - while (1) { - unsigned char b = zigzag & 0x7F; - if ((zigzag >> 7) != 0) { - b |= 0x80; - if (putc(b, out) == EOF) { - fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); - exit(EXIT_FAILURE); - } - *fpos += 1; - zigzag >>= 7; - } else { - if (putc(b, out) == EOF) { - fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); - exit(EXIT_FAILURE); - } - *fpos += 1; - break; - } - } -} - -void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname) { - fwrite_check(&n, sizeof(signed char), 1, out, fname); - *fpos += sizeof(signed char); -} - -void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname) { - fwrite_check(&n, sizeof(unsigned), 1, out, fname); - *fpos += sizeof(unsigned); -} - void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FILE *out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) { if (j == NULL || j->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: expected array for type %d\n", fname, line, t); @@ -344,92 +298,6 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL } } -void deserialize_int(char **f, int *n) { - long long ll; - deserialize_long_long(f, &ll); - *n = ll; -} - -void deserialize_long_long(char **f, long long *n) { - unsigned long long zigzag = 0; - int shift = 0; - - while (1) { - if ((**f & 0x80) == 0) { - zigzag |= ((unsigned long long) **f) << shift; - *f += 1; - shift += 7; - break; - } else { - zigzag |= ((unsigned long long) (**f & 0x7F)) << shift; - *f += 1; - shift += 7; - } - } - - *n = (zigzag >> 1) ^ (-(zigzag & 1)); -} - -void deserialize_uint(char **f, unsigned *n) { - memcpy(n, *f, sizeof(unsigned)); - *f += sizeof(unsigned); -} - -void deserialize_byte(char **f, signed char *n) { - memcpy(n, *f, sizeof(signed char)); - *f += sizeof(signed char); -} - -int deserialize_long_long_io(FILE *f, long long *n, long long *geompos) { - unsigned long long zigzag = 0; - int shift = 0; - - while (1) { - int c = getc(f); - if (c == EOF) { - return 0; - } - (*geompos)++; - - if ((c & 0x80) == 0) { - zigzag |= ((unsigned long long) c) << shift; - shift += 7; - break; - } else { - zigzag |= ((unsigned long long) (c & 0x7F)) << shift; - shift += 7; - } - } - - *n = (zigzag >> 1) ^ (-(zigzag & 1)); - return 1; -} - -int deserialize_int_io(FILE *f, int *n, long long *geompos) { - long long ll = 0; - int ret = deserialize_long_long_io(f, &ll, geompos); - *n = ll; - return ret; -} - -int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos) { - if (fread(n, sizeof(unsigned), 1, f) != 1) { - return 0; - } - *geompos += sizeof(unsigned); - return 1; -} - -int deserialize_byte_io(FILE *f, signed char *n, long long *geompos) { - int c = getc(f); - if (c == EOF) { - return 0; - } - *n = c; - (*geompos)++; - return 1; -} - struct index { long long start; long long end; diff --git a/geometry.cpp b/geometry.cpp index ce185d1..e932fb2 100644 --- a/geometry.cpp +++ b/geometry.cpp @@ -13,6 +13,7 @@ #include "clipper/clipper.hpp" #include "tile.hpp" #include "projection.hpp" +#include "serial.hpp" static int pnpoly(drawvec &vert, size_t start, size_t nvert, long long testx, long long testy); static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax); diff --git a/serial.cpp b/serial.cpp new file mode 100644 index 0000000..0150baa --- /dev/null +++ b/serial.cpp @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include "serial.hpp" + +size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) { + size_t w = fwrite(ptr, size, nitems, stream); + if (w != nitems) { + fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); + exit(EXIT_FAILURE); + } + return w; +} + +void serialize_int(FILE *out, int n, long long *fpos, const char *fname) { + serialize_long_long(out, n, fpos, fname); +} + +void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname) { + unsigned long long zigzag = (n << 1) ^ (n >> 63); + + while (1) { + unsigned char b = zigzag & 0x7F; + if ((zigzag >> 7) != 0) { + b |= 0x80; + if (putc(b, out) == EOF) { + fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); + exit(EXIT_FAILURE); + } + *fpos += 1; + zigzag >>= 7; + } else { + if (putc(b, out) == EOF) { + fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); + exit(EXIT_FAILURE); + } + *fpos += 1; + break; + } + } +} + +void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname) { + fwrite_check(&n, sizeof(signed char), 1, out, fname); + *fpos += sizeof(signed char); +} + +void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname) { + fwrite_check(&n, sizeof(unsigned), 1, out, fname); + *fpos += sizeof(unsigned); +} + +void deserialize_int(char **f, int *n) { + long long ll; + deserialize_long_long(f, &ll); + *n = ll; +} + +void deserialize_long_long(char **f, long long *n) { + unsigned long long zigzag = 0; + int shift = 0; + + while (1) { + if ((**f & 0x80) == 0) { + zigzag |= ((unsigned long long) **f) << shift; + *f += 1; + shift += 7; + break; + } else { + zigzag |= ((unsigned long long) (**f & 0x7F)) << shift; + *f += 1; + shift += 7; + } + } + + *n = (zigzag >> 1) ^ (-(zigzag & 1)); +} + +void deserialize_uint(char **f, unsigned *n) { + memcpy(n, *f, sizeof(unsigned)); + *f += sizeof(unsigned); +} + +void deserialize_byte(char **f, signed char *n) { + memcpy(n, *f, sizeof(signed char)); + *f += sizeof(signed char); +} + +int deserialize_long_long_io(FILE *f, long long *n, long long *geompos) { + unsigned long long zigzag = 0; + int shift = 0; + + while (1) { + int c = getc(f); + if (c == EOF) { + return 0; + } + (*geompos)++; + + if ((c & 0x80) == 0) { + zigzag |= ((unsigned long long) c) << shift; + shift += 7; + break; + } else { + zigzag |= ((unsigned long long) (c & 0x7F)) << shift; + shift += 7; + } + } + + *n = (zigzag >> 1) ^ (-(zigzag & 1)); + return 1; +} + +int deserialize_int_io(FILE *f, int *n, long long *geompos) { + long long ll = 0; + int ret = deserialize_long_long_io(f, &ll, geompos); + *n = ll; + return ret; +} + +int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos) { + if (fread(n, sizeof(unsigned), 1, f) != 1) { + return 0; + } + *geompos += sizeof(unsigned); + return 1; +} + +int deserialize_byte_io(FILE *f, signed char *n, long long *geompos) { + int c = getc(f); + if (c == EOF) { + return 0; + } + *n = c; + (*geompos)++; + return 1; +} diff --git a/serial.hpp b/serial.hpp new file mode 100644 index 0000000..6113cb7 --- /dev/null +++ b/serial.hpp @@ -0,0 +1,17 @@ +size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname); + +void serialize_int(FILE *out, int n, long long *fpos, const char *fname); +void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname); +void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname); +void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname); +void serialize_string(FILE *out, const char *s, long long *fpos, const char *fname); + +void deserialize_int(char **f, int *n); +void deserialize_long_long(char **f, long long *n); +void deserialize_uint(char **f, unsigned *n); +void deserialize_byte(char **f, signed char *n); + +int deserialize_int_io(FILE *f, int *n, long long *geompos); +int deserialize_long_long_io(FILE *f, long long *n, long long *geompos); +int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos); +int deserialize_byte_io(FILE *f, signed char *n, long long *geompos); diff --git a/tile.cpp b/tile.cpp index c670019..99370f0 100644 --- a/tile.cpp +++ b/tile.cpp @@ -25,6 +25,7 @@ #include "pool.hpp" #include "mbtiles.hpp" #include "projection.hpp" +#include "serial.hpp" #define CMD_BITS 3 diff --git a/tile.hpp b/tile.hpp index cf8883c..b16a142 100644 --- a/tile.hpp +++ b/tile.hpp @@ -13,22 +13,6 @@ struct pool; -void serialize_int(FILE *out, int n, long long *fpos, const char *fname); -void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname); -void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname); -void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname); -void serialize_string(FILE *out, const char *s, long long *fpos, const char *fname); - -void deserialize_int(char **f, int *n); -void deserialize_long_long(char **f, long long *n); -void deserialize_uint(char **f, unsigned *n); -void deserialize_byte(char **f, signed char *n); - -int deserialize_int_io(FILE *f, int *n, long long *geompos); -int deserialize_long_long_io(FILE *f, long long *n, long long *geompos); -int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos); -int deserialize_byte_io(FILE *f, signed char *n, long long *geompos); - long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, int *prevent, int *additional); int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y); From a52733eb07a124cdf5dd44f6666e52306ca82223 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 14:22:44 -0700 Subject: [PATCH 12/19] Use protozero functions for zigzag encoding and decoding --- serial.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/serial.cpp b/serial.cpp index 0150baa..d0cb4b4 100644 --- a/serial.cpp +++ b/serial.cpp @@ -2,6 +2,7 @@ #include #include #include +#include "protozero/varint.hpp" #include "serial.hpp" size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) { @@ -18,7 +19,7 @@ void serialize_int(FILE *out, int n, long long *fpos, const char *fname) { } void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname) { - unsigned long long zigzag = (n << 1) ^ (n >> 63); + unsigned long long zigzag = protozero::encode_zigzag32(n); while (1) { unsigned char b = zigzag & 0x7F; @@ -74,7 +75,7 @@ void deserialize_long_long(char **f, long long *n) { } } - *n = (zigzag >> 1) ^ (-(zigzag & 1)); + *n = protozero::decode_zigzag32(zigzag); } void deserialize_uint(char **f, unsigned *n) { @@ -108,7 +109,7 @@ int deserialize_long_long_io(FILE *f, long long *n, long long *geompos) { } } - *n = (zigzag >> 1) ^ (-(zigzag & 1)); + *n = protozero::decode_zigzag32(zigzag); return 1; } From 3662f1a66bdfdeb4e243042a65ca4b0dff31d0c2 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 14:59:20 -0700 Subject: [PATCH 13/19] Split main program functions apart from GeoJSON parsing --- Makefile | 4 +- geojson.cpp | 2250 +-------------------------------------------------- geojson.hpp | 31 + main.hpp | 9 + pool.cpp | 93 +++ pool.hpp | 8 + 6 files changed, 145 insertions(+), 2250 deletions(-) create mode 100644 geojson.hpp create mode 100644 main.hpp diff --git a/Makefile b/Makefile index 5b81910..4e62b3e 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper.o mvt.o serial.o +tippecanoe: geojson.o jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper.o mvt.o serial.o main.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o @@ -51,7 +51,7 @@ tippecanoe-enumerate: enumerate.o tippecanoe-decode: decode.o projection.o mvt.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o +tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libjsonpull.a: jsonpull.o diff --git a/geojson.cpp b/geojson.cpp index 7357956..741b8ee 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -19,16 +19,6 @@ #include #include #include -#include - -#ifdef __APPLE__ -#include -#include -#include -#include -#else -#include -#endif extern "C" { #include "jsonpull.h" @@ -41,16 +31,8 @@ extern "C" { #include "version.hpp" #include "memfile.hpp" #include "serial.hpp" - -static int low_detail = 12; -static int full_detail = -1; -static int min_detail = 7; - -int quiet = 0; -int geometry_scale = 0; - -static int prevent[256]; -static int additional[256]; +#include "main.hpp" +#include "geojson.hpp" #define GEOM_POINT 0 /* array of positions */ #define GEOM_MULTIPOINT 1 /* array of arrays of positions */ @@ -77,136 +59,6 @@ static int mb_geometry[GEOM_TYPES] = { VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON, }; -struct source { - char *layer; - char *file; - struct source *next; -}; - -struct tofree { - void *p; - struct tofree *next; -} *tofree = NULL; - -void mustfree(void *p) { - struct tofree *f = (struct tofree *) malloc(sizeof(struct tofree)); - if (f == NULL) { - perror("malloc"); - exit(EXIT_FAILURE); - } - f->p = p; - f->next = tofree; - tofree = f; -} - -int CPUS; -int TEMP_FILES; -long long MAX_FILES; -static long long diskfree; - -#define MAX_ZOOM 24 - -struct reader { - int metafd; - int poolfd; - int treefd; - int geomfd; - int indexfd; - - FILE *metafile; - struct memfile *poolfile; - struct memfile *treefile; - FILE *geomfile; - FILE *indexfile; - - long long metapos; - long long geompos; - long long indexpos; - - long long file_bbox[4]; - - struct stat geomst; - struct stat metast; - - char *geom_map; -}; - -void checkdisk(struct reader *r, int nreader) { - long long used = 0; - int i; - for (i = 0; i < nreader; i++) { - // Meta, pool, and tree are used once. - // Geometry and index will be duplicated during sorting and tiling. - used += r[i].metapos + 2 * r[i].geompos + 2 * r[i].indexpos + r[i].poolfile->len + r[i].treefile->len; - } - - static int warned = 0; - if (used > diskfree * .9 && !warned) { - fprintf(stderr, "You will probably run out of disk space.\n%lld bytes used or committed, of %lld originally available\n", used, diskfree); - warned = 1; - } -}; - -void init_cpus() { - CPUS = sysconf(_SC_NPROCESSORS_ONLN); - if (CPUS < 1) { - CPUS = 1; - } - - // Guard against short struct index.segment - if (CPUS > 32767) { - CPUS = 32767; - } - - // Round down to a power of 2 - CPUS = 1 << (int) (log(CPUS) / log(2)); - - struct rlimit rl; - if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { - perror("getrlimit"); - exit(EXIT_FAILURE); - } else { - MAX_FILES = rl.rlim_cur; - } - - // Don't really want too many temporary files, because the file system - // will start to bog down eventually - if (MAX_FILES > 2000) { - MAX_FILES = 2000; - } - - // MacOS can run out of system file descriptors - // even if we stay under the rlimit, so try to - // find out the real limit. - long long fds[MAX_FILES]; - long long i; - for (i = 0; i < MAX_FILES; i++) { - fds[i] = open("/dev/null", O_RDONLY); - if (fds[i] < 0) { - break; - } - } - long long j; - for (j = 0; j < i; j++) { - if (close(fds[j]) < 0) { - perror("close"); - exit(EXIT_FAILURE); - } - } - - // Scale down because we really don't want to run the system out of files - MAX_FILES = i * 3 / 4; - if (MAX_FILES < 32) { - fprintf(stderr, "Can't open a useful number of files: %lld\n", MAX_FILES); - exit(EXIT_FAILURE); - } - - TEMP_FILES = (MAX_FILES - 10) / 2; - if (TEMP_FILES > CPUS * 4) { - TEMP_FILES = CPUS * 4; - } -} - void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FILE *out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) { if (j == NULL || j->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: expected array for type %d\n", fname, line, t); @@ -298,182 +150,6 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL } } -struct index { - long long start; - long long end; - unsigned long long index; - short segment; - unsigned long long seq : (64 - 16); // pack with segment to stay in 32 bytes -}; - -int indexcmp(const void *v1, const void *v2) { - const struct index *i1 = (const struct index *) v1; - const struct index *i2 = (const struct index *) v2; - - if (i1->index < i2->index) { - return -1; - } else if (i1->index > i2->index) { - return 1; - } - - if (i1->seq < i2->seq) { - return -1; - } else if (i1->seq > i2->seq) { - return 1; - } - - return 0; -} - -struct merge { - long long start; - long long end; - - struct merge *next; -}; - -static void insert(struct merge *m, struct merge **head, unsigned char *map) { - while (*head != NULL && indexcmp(map + m->start, map + (*head)->start) > 0) { - head = &((*head)->next); - } - - m->next = *head; - *head = m; -} - -static void merge(struct merge *merges, int nmerges, unsigned char *map, FILE *f, int bytes, long long nrec, char *geom_map, FILE *geom_out, long long *geompos, long long *progress, long long *progress_max, long long *progress_reported) { - int i; - struct merge *head = NULL; - - for (i = 0; i < nmerges; i++) { - if (merges[i].start < merges[i].end) { - insert(&(merges[i]), &head, map); - } - } - - while (head != NULL) { - struct index *ix = (struct index *) (map + head->start); - fwrite_check(geom_map + ix->start, 1, ix->end - ix->start, geom_out, "merge geometry"); - *geompos += ix->end - ix->start; - - // Count this as an 75%-accomplishment, since we already 25%-counted it - *progress += (ix->end - ix->start) * 3 / 4; - if (!quiet && 100 * *progress / *progress_max != *progress_reported) { - fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); - *progress_reported = 100 * *progress / *progress_max; - } - - fwrite_check(map + head->start, bytes, 1, f, "merge temporary"); - head->start += bytes; - - struct merge *m = head; - head = m->next; - m->next = NULL; - - if (m->start < m->end) { - insert(m, &head, map); - } - } -} - -struct stringpool { - long long left; - long long right; - long long off; -}; - -static unsigned char swizzle[256] = { - 0x00, 0xBF, 0x18, 0xDE, 0x93, 0xC9, 0xB1, 0x5E, 0xDF, 0xBE, 0x72, 0x5A, 0xBB, 0x42, 0x64, 0xC6, - 0xD8, 0xB7, 0x15, 0x74, 0x1C, 0x8B, 0x91, 0xF5, 0x29, 0x46, 0xEC, 0x6F, 0xCA, 0x20, 0xF0, 0x06, - 0x27, 0x61, 0x87, 0xE0, 0x6E, 0x43, 0x50, 0xC5, 0x1B, 0xB4, 0x37, 0xC3, 0x69, 0xA6, 0xEE, 0x80, - 0xAF, 0x9B, 0xA1, 0x76, 0x23, 0x24, 0x53, 0xF3, 0x5B, 0x65, 0x19, 0xF4, 0xFC, 0xDD, 0x26, 0xE8, - 0x10, 0xF7, 0xCE, 0x92, 0x48, 0xF6, 0x94, 0x60, 0x07, 0xC4, 0xB9, 0x97, 0x6D, 0xA4, 0x11, 0x0D, - 0x1F, 0x4D, 0x13, 0xB0, 0x5D, 0xBA, 0x31, 0xD5, 0x8D, 0x51, 0x36, 0x96, 0x7A, 0x03, 0x7F, 0xDA, - 0x17, 0xDB, 0xD4, 0x83, 0xE2, 0x79, 0x6A, 0xE1, 0x95, 0x38, 0xFF, 0x28, 0xB2, 0xB3, 0xA7, 0xAE, - 0xF8, 0x54, 0xCC, 0xDC, 0x9A, 0x6B, 0xFB, 0x3F, 0xD7, 0xBC, 0x21, 0xC8, 0x71, 0x09, 0x16, 0xAC, - 0x3C, 0x8A, 0x62, 0x05, 0xC2, 0x8C, 0x32, 0x4E, 0x35, 0x9C, 0x5F, 0x75, 0xCD, 0x2E, 0xA2, 0x3E, - 0x1A, 0xC1, 0x8E, 0x14, 0xA0, 0xD3, 0x7D, 0xD9, 0xEB, 0x5C, 0x70, 0xE6, 0x9E, 0x12, 0x3B, 0xEF, - 0x1E, 0x49, 0xD2, 0x98, 0x39, 0x7E, 0x44, 0x4B, 0x6C, 0x88, 0x02, 0x2C, 0xAD, 0xE5, 0x9F, 0x40, - 0x7B, 0x4A, 0x3D, 0xA9, 0xAB, 0x0B, 0xD6, 0x2F, 0x90, 0x2A, 0xB6, 0x1D, 0xC7, 0x22, 0x55, 0x34, - 0x0A, 0xD0, 0xB5, 0x68, 0xE3, 0x59, 0xFD, 0xFA, 0x57, 0x77, 0x25, 0xA3, 0x04, 0xB8, 0x33, 0x89, - 0x78, 0x82, 0xE4, 0xC0, 0x0E, 0x8F, 0x85, 0xD1, 0x84, 0x08, 0x67, 0x47, 0x9D, 0xCB, 0x58, 0x4C, - 0xAA, 0xED, 0x52, 0xF2, 0x4F, 0xF1, 0x66, 0xCF, 0xA5, 0x56, 0xEA, 0x7C, 0xE9, 0x63, 0xE7, 0x01, - 0xF9, 0xFE, 0x0C, 0x99, 0x2D, 0x0F, 0x3A, 0x41, 0x45, 0xA8, 0x30, 0x2B, 0x73, 0xBD, 0x86, 0x81, -}; - -int swizzlecmp(const char *a, const char *b) { - while (*a || *b) { - int aa = swizzle[(unsigned char) *a]; - int bb = swizzle[(unsigned char) *b]; - - int cmp = aa - bb; - if (cmp != 0) { - return cmp; - } - - a++; - b++; - } - - return 0; -} - -long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type) { - long long *sp = &treefile->tree; - - while (*sp != 0) { - int cmp = swizzlecmp(s, poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off + 1); - - if (cmp == 0) { - cmp = type - (poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off)[0]; - } - - if (cmp < 0) { - sp = &(((struct stringpool *) (treefile->map + *sp))->left); - } else if (cmp > 0) { - sp = &(((struct stringpool *) (treefile->map + *sp))->right); - } else { - return ((struct stringpool *) (treefile->map + *sp))->off; - } - } - - // *sp is probably in the memory-mapped file, and will move if the file grows. - long long ssp; - if (sp == &treefile->tree) { - ssp = -1; - } else { - ssp = ((char *) sp) - treefile->map; - } - - long long off = poolfile->off; - if (memfile_write(poolfile, &type, 1) < 0) { - perror("memfile write"); - exit(EXIT_FAILURE); - } - if (memfile_write(poolfile, (void *) s, strlen(s) + 1) < 0) { - perror("memfile write"); - exit(EXIT_FAILURE); - } - - struct stringpool tsp; - tsp.left = 0; - tsp.right = 0; - tsp.off = off; - - long long p = treefile->off; - if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) { - perror("memfile write"); - exit(EXIT_FAILURE); - } - - if (ssp == -1) { - treefile->tree = p; - } else { - *((long long *) (treefile->map + ssp)) = p; - } - return off; -} - int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers) { json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == NULL) { @@ -789,34 +465,6 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se } } -struct parse_json_args { - json_pull *jp; - const char *reading; - volatile long long *layer_seq; - volatile long long *progress_seq; - long long *metapos; - long long *geompos; - long long *indexpos; - struct pool *exclude; - struct pool *include; - int exclude_all; - FILE *metafile; - FILE *geomfile; - FILE *indexfile; - struct memfile *poolfile; - struct memfile *treefile; - char *fname; - int basezoom; - int layer; - double droprate; - long long *file_bbox; - int segment; - int *initialized; - unsigned *initial_x; - unsigned *initial_y; - struct reader *readers; -}; - void *run_parse_json(void *v) { struct parse_json_args *pja = (struct parse_json_args *) v; @@ -857,1897 +505,3 @@ struct json_pull *json_begin_map(char *map, long long len) { return json_begin(json_map_read, jm); } - -struct sort_arg { - int task; - int cpus; - long long indexpos; - struct merge *merges; - int indexfd; - int nmerges; - long long unit; - int bytes; -}; - -void *run_sort(void *v) { - struct sort_arg *a = (struct sort_arg *) v; - - long long start; - for (start = a->task * a->unit; start < a->indexpos; start += a->unit * a->cpus) { - long long end = start + a->unit; - if (end > a->indexpos) { - end = a->indexpos; - } - - a->merges[start / a->unit].start = start; - a->merges[start / a->unit].end = end; - a->merges[start / a->unit].next = NULL; - - // MAP_PRIVATE to avoid disk writes if it fits in memory - void *map = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_PRIVATE, a->indexfd, start); - if (map == MAP_FAILED) { - perror("mmap in run_sort"); - exit(EXIT_FAILURE); - } - madvise(map, end - start, MADV_RANDOM); - madvise(map, end - start, MADV_WILLNEED); - - qsort(map, (end - start) / a->bytes, a->bytes, indexcmp); - - // Sorting and then copying avoids disk access to - // write out intermediate stages of the sort. - - void *map2 = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_SHARED, a->indexfd, start); - if (map2 == MAP_FAILED) { - perror("mmap (write)"); - exit(EXIT_FAILURE); - } - madvise(map2, end - start, MADV_SEQUENTIAL); - - memcpy(map2, map, end - start); - - // No madvise, since caller will want the sorted data - munmap(map, end - start); - munmap(map2, end - start); - } - - return NULL; -} - -void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, struct pool *exclude, struct pool *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y) { - long long segs[CPUS + 1]; - segs[0] = 0; - segs[CPUS] = len; - - int i; - for (i = 1; i < CPUS; i++) { - segs[i] = len * i / CPUS; - - while (segs[i] < len && map[segs[i]] != '\n') { - segs[i]++; - } - } - - volatile long long layer_seq[CPUS]; - for (i = 0; i < CPUS; i++) { - // To preserve feature ordering, unique id for each segment - // begins with that segment's offset into the input - layer_seq[i] = segs[i] + initial_offset; - } - - struct parse_json_args pja[CPUS]; - pthread_t pthreads[CPUS]; - - for (i = 0; i < CPUS; i++) { - pja[i].jp = json_begin_map(map + segs[i], segs[i + 1] - segs[i]); - pja[i].reading = reading; - pja[i].layer_seq = &layer_seq[i]; - pja[i].progress_seq = progress_seq; - pja[i].metapos = &reader[i].metapos; - pja[i].geompos = &reader[i].geompos; - pja[i].indexpos = &reader[i].indexpos; - pja[i].exclude = exclude; - pja[i].include = include; - pja[i].exclude_all = exclude_all; - pja[i].metafile = reader[i].metafile; - pja[i].geomfile = reader[i].geomfile; - pja[i].indexfile = reader[i].indexfile; - pja[i].poolfile = reader[i].poolfile; - pja[i].treefile = reader[i].treefile; - pja[i].fname = fname; - pja[i].basezoom = basezoom; - pja[i].layer = source < nlayers ? source : 0; - pja[i].droprate = droprate; - pja[i].file_bbox = reader[i].file_bbox; - pja[i].segment = i; - pja[i].initialized = &initialized[i]; - pja[i].initial_x = &initial_x[i]; - pja[i].initial_y = &initial_y[i]; - pja[i].readers = reader; - - if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) { - perror("pthread_create"); - exit(EXIT_FAILURE); - } - } - - for (i = 0; i < CPUS; i++) { - void *retval; - - if (pthread_join(pthreads[i], &retval) != 0) { - perror("pthread_join"); - } - - free(pja[i].jp->source); - json_end(pja[i].jp); - } -} - -struct read_parallel_arg { - int fd; - FILE *fp; - long long offset; - long long len; - volatile int *is_parsing; - - const char *reading; - struct reader *reader; - volatile long long *progress_seq; - struct pool *exclude; - struct pool *include; - int exclude_all; - char *fname; - int maxzoom; - int basezoom; - int source; - int nlayers; - double droprate; - int *initialized; - unsigned *initial_x; - unsigned *initial_y; -}; - -void *run_read_parallel(void *v) { - struct read_parallel_arg *a = (struct read_parallel_arg *) v; - - struct stat st; - if (fstat(a->fd, &st) != 0) { - perror("stat read temp"); - } - if (a->len != st.st_size) { - fprintf(stderr, "wrong number of bytes in temporary: %lld vs %lld\n", a->len, (long long) st.st_size); - } - a->len = st.st_size; - - char *map = (char *) mmap(NULL, a->len, PROT_READ, MAP_PRIVATE, a->fd, 0); - if (map == NULL || map == MAP_FAILED) { - perror("map intermediate input"); - exit(EXIT_FAILURE); - } - madvise(map, a->len, MADV_RANDOM); // sequential, but from several pointers at once - - do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->droprate, a->initialized, a->initial_x, a->initial_y); - - madvise(map, a->len, MADV_DONTNEED); - if (munmap(map, a->len) != 0) { - perror("munmap source file"); - } - if (fclose(a->fp) != 0) { - perror("close source file"); - exit(EXIT_FAILURE); - } - - *(a->is_parsing) = 0; - free(a); - - return NULL; -} - -void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, const char *reading, struct reader *reader, volatile long long *progress_seq, struct pool *exclude, struct pool *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y) { - // This has to kick off an intermediate thread to start the parser threads, - // so the main thread can get back to reading the next input stage while - // the intermediate thread waits for the completion of the parser threads. - - *is_parsing = 1; - - struct read_parallel_arg *rpa = (struct read_parallel_arg *) malloc(sizeof(struct read_parallel_arg)); - if (rpa == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - rpa->fd = fd; - rpa->fp = fp; - rpa->offset = offset; - rpa->len = len; - rpa->is_parsing = is_parsing; - - rpa->reading = reading; - rpa->reader = reader; - rpa->progress_seq = progress_seq; - rpa->exclude = exclude; - rpa->include = include; - rpa->exclude_all = exclude_all; - rpa->fname = fname; - rpa->basezoom = basezoom; - rpa->source = source; - rpa->nlayers = nlayers; - rpa->droprate = droprate; - rpa->initialized = initialized; - rpa->initial_x = initial_x; - rpa->initial_y = initial_y; - - if (pthread_create(parallel_parser, NULL, run_read_parallel, rpa) != 0) { - perror("pthread_create"); - exit(EXIT_FAILURE); - } -} - -void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int splits, long long mem, const char *tmpdir, long long *availfiles, FILE *geomfile, FILE *indexfile, long long *geompos_out, long long *progress, long long *progress_max, long long *progress_reported) { - // Arranged as bits to facilitate subdividing again if a subdivided file is still huge - int splitbits = log(splits) / log(2); - splits = 1 << splitbits; - - FILE *geomfiles[splits]; - FILE *indexfiles[splits]; - int geomfds[splits]; - int indexfds[splits]; - long long sub_geompos[splits]; - - int i; - for (i = 0; i < splits; i++) { - sub_geompos[i] = 0; - - char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; - sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX"); - char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; - sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX"); - - geomfds[i] = mkstemp(geomname); - if (geomfds[i] < 0) { - perror(geomname); - exit(EXIT_FAILURE); - } - indexfds[i] = mkstemp(indexname); - if (indexfds[i] < 0) { - perror(indexname); - exit(EXIT_FAILURE); - } - - geomfiles[i] = fopen(geomname, "wb"); - if (geomfiles[i] == NULL) { - perror(geomname); - exit(EXIT_FAILURE); - } - indexfiles[i] = fopen(indexname, "wb"); - if (indexfiles[i] == NULL) { - perror(indexname); - exit(EXIT_FAILURE); - } - - *availfiles -= 4; - - unlink(geomname); - unlink(indexname); - } - - for (i = 0; i < inputs; i++) { - struct stat geomst, indexst; - if (fstat(geomfds_in[i], &geomst) < 0) { - perror("stat geom"); - exit(EXIT_FAILURE); - } - if (fstat(indexfds_in[i], &indexst) < 0) { - perror("stat index"); - exit(EXIT_FAILURE); - } - - if (indexst.st_size != 0) { - struct index *indexmap = (struct index *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds_in[i], 0); - if (indexmap == MAP_FAILED) { - fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds_in[i], (long long) indexst.st_size); - perror("map index"); - exit(EXIT_FAILURE); - } - madvise(indexmap, indexst.st_size, MADV_SEQUENTIAL); - madvise(indexmap, indexst.st_size, MADV_WILLNEED); - char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds_in[i], 0); - if (geommap == MAP_FAILED) { - perror("map geom"); - exit(EXIT_FAILURE); - } - madvise(geommap, geomst.st_size, MADV_SEQUENTIAL); - madvise(geommap, geomst.st_size, MADV_WILLNEED); - - long long a; - for (a = 0; a < indexst.st_size / sizeof(struct index); a++) { - struct index ix = indexmap[a]; - unsigned long long which = (ix.index << prefix) >> (64 - splitbits); - long long pos = sub_geompos[which]; - - fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], "geom"); - sub_geompos[which] += ix.end - ix.start; - - // Count this as a 25%-accomplishment, since we will copy again - *progress += (ix.end - ix.start) / 4; - if (!quiet && 100 * *progress / *progress_max != *progress_reported) { - fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); - *progress_reported = 100 * *progress / *progress_max; - } - - ix.start = pos; - ix.end = sub_geompos[which]; - - fwrite_check(&ix, sizeof(struct index), 1, indexfiles[which], "index"); - } - - madvise(indexmap, indexst.st_size, MADV_DONTNEED); - if (munmap(indexmap, indexst.st_size) < 0) { - perror("unmap index"); - exit(EXIT_FAILURE); - } - madvise(geommap, geomst.st_size, MADV_DONTNEED); - if (munmap(geommap, geomst.st_size) < 0) { - perror("unmap geom"); - exit(EXIT_FAILURE); - } - } - - if (close(geomfds_in[i]) < 0) { - perror("close geom"); - exit(EXIT_FAILURE); - } - if (close(indexfds_in[i]) < 0) { - perror("close index"); - exit(EXIT_FAILURE); - } - - *availfiles += 2; - } - - for (i = 0; i < splits; i++) { - if (fclose(geomfiles[i]) != 0) { - perror("fclose geom"); - exit(EXIT_FAILURE); - } - if (fclose(indexfiles[i]) != 0) { - perror("fclose index"); - exit(EXIT_FAILURE); - } - - *availfiles += 2; - } - - for (i = 0; i < splits; i++) { - int already_closed = 0; - - struct stat geomst, indexst; - if (fstat(geomfds[i], &geomst) < 0) { - perror("stat geom"); - exit(EXIT_FAILURE); - } - if (fstat(indexfds[i], &indexst) < 0) { - perror("stat index"); - exit(EXIT_FAILURE); - } - - if (indexst.st_size > 0) { - if (indexst.st_size > sizeof(struct index) && indexst.st_size + geomst.st_size < mem) { - long long indexpos = indexst.st_size; - int bytes = sizeof(struct index); - - int page = sysconf(_SC_PAGESIZE); - // Don't try to sort more than 2GB at once, - // which used to crash Macs and may still - long long max_unit = 2LL * 1024 * 1024 * 1024; - long long unit = ((indexpos / CPUS + bytes - 1) / bytes) * bytes; - if (unit > max_unit) { - unit = max_unit; - } - unit = ((unit + page - 1) / page) * page; - - int nmerges = (indexpos + unit - 1) / unit; - struct merge merges[nmerges]; - - int a; - for (a = 0; a < nmerges; a++) { - merges[a].start = merges[a].end = 0; - } - - pthread_t pthreads[CPUS]; - struct sort_arg args[CPUS]; - - for (a = 0; a < CPUS; a++) { - args[a].task = a; - args[a].cpus = CPUS; - args[a].indexpos = indexpos; - args[a].merges = merges; - args[a].indexfd = indexfds[i]; - args[a].nmerges = nmerges; - args[a].unit = unit; - args[a].bytes = bytes; - - if (pthread_create(&pthreads[a], NULL, run_sort, &args[a]) != 0) { - perror("pthread_create"); - exit(EXIT_FAILURE); - } - } - - for (a = 0; a < CPUS; a++) { - void *retval; - - if (pthread_join(pthreads[a], &retval) != 0) { - perror("pthread_join"); - } - } - - struct indexmap *indexmap = (struct indexmap *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0); - if (indexmap == MAP_FAILED) { - fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds[i], (long long) indexst.st_size); - perror("map index"); - exit(EXIT_FAILURE); - } - madvise(indexmap, indexst.st_size, MADV_RANDOM); // sequential, but from several pointers at once - madvise(indexmap, indexst.st_size, MADV_WILLNEED); - char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0); - if (geommap == MAP_FAILED) { - perror("map geom"); - exit(EXIT_FAILURE); - } - madvise(geommap, geomst.st_size, MADV_RANDOM); - madvise(geommap, geomst.st_size, MADV_WILLNEED); - - merge(merges, nmerges, (unsigned char *) indexmap, indexfile, bytes, indexpos / bytes, geommap, geomfile, geompos_out, progress, progress_max, progress_reported); - - madvise(indexmap, indexst.st_size, MADV_DONTNEED); - if (munmap(indexmap, indexst.st_size) < 0) { - perror("unmap index"); - exit(EXIT_FAILURE); - } - madvise(geommap, geomst.st_size, MADV_DONTNEED); - if (munmap(geommap, geomst.st_size) < 0) { - perror("unmap geom"); - exit(EXIT_FAILURE); - } - } else if (indexst.st_size == sizeof(struct index) || prefix + splitbits >= 64) { - struct index *indexmap = (struct index *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0); - if (indexmap == MAP_FAILED) { - fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds[i], (long long) indexst.st_size); - perror("map index"); - exit(EXIT_FAILURE); - } - madvise(indexmap, indexst.st_size, MADV_SEQUENTIAL); - madvise(indexmap, indexst.st_size, MADV_WILLNEED); - char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0); - if (geommap == MAP_FAILED) { - perror("map geom"); - exit(EXIT_FAILURE); - } - madvise(geommap, geomst.st_size, MADV_RANDOM); - madvise(geommap, geomst.st_size, MADV_WILLNEED); - - long long a; - for (a = 0; a < indexst.st_size / sizeof(struct index); a++) { - struct index ix = indexmap[a]; - long long pos = *geompos_out; - - fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfile, "geom"); - *geompos_out += ix.end - ix.start; - - // Count this as an 75%-accomplishment, since we already 25%-counted it - *progress += (ix.end - ix.start) * 3 / 4; - if (!quiet && 100 * *progress / *progress_max != *progress_reported) { - fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); - *progress_reported = 100 * *progress / *progress_max; - } - - ix.start = pos; - ix.end = *geompos_out; - fwrite_check(&ix, sizeof(struct index), 1, indexfile, "index"); - } - - madvise(indexmap, indexst.st_size, MADV_DONTNEED); - if (munmap(indexmap, indexst.st_size) < 0) { - perror("unmap index"); - exit(EXIT_FAILURE); - } - madvise(geommap, geomst.st_size, MADV_DONTNEED); - if (munmap(geommap, geomst.st_size) < 0) { - perror("unmap geom"); - exit(EXIT_FAILURE); - } - } else { - // We already reported the progress from splitting this radix out - // but we need to split it again, which will be credited with more - // progress. So increase the total amount of progress to report by - // the additional progress that will happpen, which may move the - // counter backward but will be an honest estimate of the work remaining. - *progress_max += geomst.st_size / 4; - - radix1(&geomfds[i], &indexfds[i], 1, prefix + splitbits, *availfiles / 4, mem, tmpdir, availfiles, geomfile, indexfile, geompos_out, progress, progress_max, progress_reported); - already_closed = 1; - } - } - - if (!already_closed) { - if (close(geomfds[i]) < 0) { - perror("close geom"); - exit(EXIT_FAILURE); - } - if (close(indexfds[i]) < 0) { - perror("close index"); - exit(EXIT_FAILURE); - } - - *availfiles += 2; - } - } -} - -void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE *indexfile, int indexfd, const char *tmpdir, long long *geompos) { - // Run through the index and geometry for each reader, - // splitting the contents out by index into as many - // sub-files as we can write to simultaneously. - - // Then sort each of those by index, recursively if it is - // too big to fit in memory. - - // Then concatenate each of the sub-outputs into a final output. - - long long mem; - -#ifdef __APPLE__ - int64_t hw_memsize; - size_t len = sizeof(int64_t); - if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) { - perror("sysctl hw.memsize"); - exit(EXIT_FAILURE); - } - mem = hw_memsize; -#else - long long pagesize = sysconf(_SC_PAGESIZE); - long long pages = sysconf(_SC_PHYS_PAGES); - if (pages < 0 || pagesize < 0) { - perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES"); - exit(EXIT_FAILURE); - } - - mem = (long long) pages * pagesize; -#endif - - // Just for code coverage testing. Deeply recursive sorting is very slow - // compared to sorting in memory. - if (additional[A_PREFER_RADIX_SORT]) { - mem = 8192; - } - - long long availfiles = MAX_FILES - 2 * nreaders // each reader has a geom and an index - - 4 // pool, meta, mbtiles, mbtiles journal - - 4 // top-level geom and index output, both FILE and fd - - 3; // stdin, stdout, stderr - - // 4 because for each we have output and input FILE and fd for geom and index - int splits = availfiles / 4; - - // Be somewhat conservative about memory availability because the whole point of this - // is to keep from thrashing by working on chunks that will fit in memory. - mem /= 2; - - long long geom_total = 0; - int geomfds[nreaders]; - int indexfds[nreaders]; - int i; - for (i = 0; i < nreaders; i++) { - geomfds[i] = reader[i].geomfd; - indexfds[i] = reader[i].indexfd; - - struct stat geomst; - if (fstat(reader[i].geomfd, &geomst) < 0) { - perror("stat geom"); - exit(EXIT_FAILURE); - } - geom_total += geomst.st_size; - } - - long long progress = 0, progress_max = geom_total, progress_reported = -1; - long long availfiles_before = availfiles; - radix1(geomfds, indexfds, nreaders, 0, splits, mem, tmpdir, &availfiles, geomfile, indexfile, geompos, &progress, &progress_max, &progress_reported); - - if (availfiles - 2 * nreaders != availfiles_before) { - fprintf(stderr, "Internal error: miscounted available file descriptors: %lld vs %lld\n", availfiles - 2 * nreaders, availfiles); - exit(EXIT_FAILURE); - } -} - -int read_json(int argc, struct source **sourcelist, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int *prevent, int *additional, int read_parallel, int forcetable, const char *attribution) { - int ret = EXIT_SUCCESS; - - struct reader reader[CPUS]; - int i; - for (i = 0; i < CPUS; i++) { - struct reader *r = reader + i; - - char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; - char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; - char treename[strlen(tmpdir) + strlen("/tree.XXXXXXXX") + 1]; - char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; - char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; - - sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX"); - sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX"); - sprintf(treename, "%s%s", tmpdir, "/tree.XXXXXXXX"); - sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX"); - sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX"); - - r->metafd = mkstemp(metaname); - if (r->metafd < 0) { - perror(metaname); - exit(EXIT_FAILURE); - } - r->poolfd = mkstemp(poolname); - if (r->poolfd < 0) { - perror(poolname); - exit(EXIT_FAILURE); - } - r->treefd = mkstemp(treename); - if (r->treefd < 0) { - perror(treename); - exit(EXIT_FAILURE); - } - r->geomfd = mkstemp(geomname); - if (r->geomfd < 0) { - perror(geomname); - exit(EXIT_FAILURE); - } - r->indexfd = mkstemp(indexname); - if (r->indexfd < 0) { - perror(indexname); - exit(EXIT_FAILURE); - } - - r->metafile = fopen(metaname, "wb"); - if (r->metafile == NULL) { - perror(metaname); - exit(EXIT_FAILURE); - } - r->poolfile = memfile_open(r->poolfd); - if (r->poolfile == NULL) { - perror(poolname); - exit(EXIT_FAILURE); - } - r->treefile = memfile_open(r->treefd); - if (r->treefile == NULL) { - perror(treename); - exit(EXIT_FAILURE); - } - r->geomfile = fopen(geomname, "wb"); - if (r->geomfile == NULL) { - perror(geomname); - exit(EXIT_FAILURE); - } - r->indexfile = fopen(indexname, "wb"); - if (r->indexfile == NULL) { - perror(indexname); - exit(EXIT_FAILURE); - } - r->metapos = 0; - r->geompos = 0; - r->indexpos = 0; - - unlink(metaname); - unlink(poolname); - unlink(treename); - unlink(geomname); - unlink(indexname); - - // To distinguish a null value - { - struct stringpool p; - memfile_write(r->treefile, &p, sizeof(struct stringpool)); - } - // Keep metadata file from being completely empty if no attributes - serialize_int(r->metafile, 0, &r->metapos, "meta"); - - r->file_bbox[0] = r->file_bbox[1] = UINT_MAX; - r->file_bbox[2] = r->file_bbox[3] = 0; - } - - struct statfs fsstat; - if (fstatfs(reader[0].geomfd, &fsstat) != 0) { - perror("fstatfs"); - exit(EXIT_FAILURE); - } - diskfree = (long long) fsstat.f_bsize * fsstat.f_bavail; - - volatile long long progress_seq = 0; - - int initialized[CPUS]; - unsigned initial_x[CPUS], initial_y[CPUS]; - for (i = 0; i < CPUS; i++) { - initialized[i] = initial_x[i] = initial_y[i] = 0; - } - - int nlayers; - if (layername != NULL) { - nlayers = 1; - } else { - nlayers = argc; - if (nlayers == 0) { - nlayers = 1; - } - } - - int nsources = argc; - if (nsources == 0) { - nsources = 1; - } - - long overall_offset = 0; - - int source; - for (source = 0; source < nsources; source++) { - const char *reading; - int fd; - - if (source >= argc) { - reading = "standard input"; - fd = 0; - } else { - reading = sourcelist[source]->file; - fd = open(sourcelist[source]->file, O_RDONLY); - if (fd < 0) { - perror(sourcelist[source]->file); - continue; - } - } - - struct stat st; - char *map = NULL; - off_t off = 0; - - if (read_parallel) { - if (fstat(fd, &st) == 0) { - off = lseek(fd, 0, SEEK_CUR); - if (off >= 0) { - map = (char *) mmap(NULL, st.st_size - off, PROT_READ, MAP_PRIVATE, fd, off); - // No error if MAP_FAILED because check is below - if (map != MAP_FAILED) { - madvise(map, st.st_size - off, MADV_RANDOM); // sequential, but from several pointers at once - } - } - } - } - - if (map != NULL && map != MAP_FAILED) { - do_read_parallel(map, st.st_size - off, overall_offset, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y); - overall_offset += st.st_size - off; - checkdisk(reader, CPUS); - - if (munmap(map, st.st_size - off) != 0) { - madvise(map, st.st_size, MADV_DONTNEED); - perror("munmap source file"); - } - } else { - FILE *fp = fdopen(fd, "r"); - if (fp == NULL) { - perror(sourcelist[source]->file); - if (close(fd) != 0) { - perror("close source file"); - exit(EXIT_FAILURE); - } - continue; - } - - if (read_parallel) { - // Serial reading of chunks that are then parsed in parallel - - char readname[strlen(tmpdir) + strlen("/read.XXXXXXXX") + 1]; - sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX"); - int readfd = mkstemp(readname); - if (readfd < 0) { - perror(readname); - exit(EXIT_FAILURE); - } - FILE *readfp = fdopen(readfd, "w"); - if (readfp == NULL) { - perror(readname); - exit(EXIT_FAILURE); - } - unlink(readname); - - volatile int is_parsing = 0; - long long ahead = 0; - long long initial_offset = overall_offset; - pthread_t parallel_parser; - -#define READ_BUF 2000 -#define PARSE_MIN 10000000 -#define PARSE_MAX (1LL * 1024 * 1024 * 1024) - - char buf[READ_BUF]; - int n; - - while ((n = fread(buf, sizeof(char), READ_BUF, fp)) > 0) { - fwrite_check(buf, sizeof(char), n, readfp, reading); - ahead += n; - - if (buf[n - 1] == '\n' && ahead > PARSE_MIN) { - // Don't let the streaming reader get too far ahead of the parsers. - // If the buffered input gets huge, even if the parsers are still running, - // wait for the parser thread instead of continuing to stream input. - - if (is_parsing == 0 || ahead >= PARSE_MAX) { - if (initial_offset != 0) { - if (pthread_join(parallel_parser, NULL) != 0) { - perror("pthread_join"); - exit(EXIT_FAILURE); - } - } - - fflush(readfp); - start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y); - - initial_offset += ahead; - overall_offset += ahead; - checkdisk(reader, CPUS); - ahead = 0; - - sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX"); - readfd = mkstemp(readname); - if (readfd < 0) { - perror(readname); - exit(EXIT_FAILURE); - } - readfp = fdopen(readfd, "w"); - if (readfp == NULL) { - perror(readname); - exit(EXIT_FAILURE); - } - unlink(readname); - } - } - } - if (n < 0) { - perror(reading); - } - - if (initial_offset != 0) { - if (pthread_join(parallel_parser, NULL) != 0) { - perror("pthread_join"); - exit(EXIT_FAILURE); - } - } - - fflush(readfp); - - if (ahead > 0) { - start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y); - - if (pthread_join(parallel_parser, NULL) != 0) { - perror("pthread_join"); - } - - overall_offset += ahead; - checkdisk(reader, CPUS); - } - } else { - // Plain serial reading - - long long layer_seq = overall_offset; - json_pull *jp = json_begin_file(fp); - parse_json(jp, reading, &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader); - json_end(jp); - overall_offset = layer_seq; - checkdisk(reader, CPUS); - } - - if (fclose(fp) != 0) { - perror("fclose input"); - exit(EXIT_FAILURE); - } - } - } - - if (!quiet) { - fprintf(stderr, " \r"); - // (stderr, "Read 10000.00 million features\r", *progress_seq / 1000000.0); - } - - for (i = 0; i < CPUS; i++) { - if (fclose(reader[i].metafile) != 0) { - perror("fclose meta"); - exit(EXIT_FAILURE); - } - if (fclose(reader[i].geomfile) != 0) { - perror("fclose geom"); - exit(EXIT_FAILURE); - } - if (fclose(reader[i].indexfile) != 0) { - perror("fclose index"); - exit(EXIT_FAILURE); - } - memfile_close(reader[i].treefile); - - if (fstat(reader[i].geomfd, &reader[i].geomst) != 0) { - perror("stat geom\n"); - exit(EXIT_FAILURE); - } - if (fstat(reader[i].metafd, &reader[i].metast) != 0) { - perror("stat meta\n"); - exit(EXIT_FAILURE); - } - } - - struct pool file_keys1[nlayers]; - struct pool *file_keys[nlayers]; - for (i = 0; i < nlayers; i++) { - pool_init(&file_keys1[i], 0); - file_keys[i] = &file_keys1[i]; - } - - char *layernames[nlayers]; - for (i = 0; i < nlayers; i++) { - if (layername != NULL) { - layernames[i] = strdup(layername); - if (layernames[i] == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } else { - char *src; - if (argc < 1) { - src = fname; - } else if (sourcelist[i]->layer != NULL) { - src = sourcelist[i]->layer; - } else { - src = sourcelist[i]->file; - } - - char *trunc = layernames[i] = (char *) malloc(strlen(src) + 1); - if (trunc == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - const char *ocp, *use = src; - for (ocp = src; *ocp; ocp++) { - if (*ocp == '/' && ocp[1] != '\0') { - use = ocp + 1; - } - } - strcpy(trunc, use); - - char *cp = strstr(trunc, ".json"); - if (cp != NULL) { - *cp = '\0'; - } - cp = strstr(trunc, ".mbtiles"); - if (cp != NULL) { - *cp = '\0'; - } - - char *out = trunc; - for (cp = trunc; *cp; cp++) { - if (isalpha(*cp) || isdigit(*cp) || *cp == '_') { - *out++ = *cp; - } - } - *out = '\0'; - - if (!quiet) { - fprintf(stderr, "For layer %d, using name \"%s\"\n", i, trunc); - } - } - } - - // Create a combined string pool and a combined metadata file - // but keep track of the offsets into it since we still need - // segment+offset to find the data. - - long long pool_off[CPUS]; - long long meta_off[CPUS]; - - char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; - sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX"); - - int poolfd = mkstemp(poolname); - if (poolfd < 0) { - perror(poolname); - exit(EXIT_FAILURE); - } - - FILE *poolfile = fopen(poolname, "wb"); - if (poolfile == NULL) { - perror(poolname); - exit(EXIT_FAILURE); - } - - unlink(poolname); - - char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; - sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX"); - - int metafd = mkstemp(metaname); - if (metafd < 0) { - perror(metaname); - exit(EXIT_FAILURE); - } - - FILE *metafile = fopen(metaname, "wb"); - if (metafile == NULL) { - perror(metaname); - exit(EXIT_FAILURE); - } - - unlink(metaname); - - long long metapos = 0; - long long poolpos = 0; - - for (i = 0; i < CPUS; i++) { - if (reader[i].metapos > 0) { - void *map = mmap(NULL, reader[i].metapos, PROT_READ, MAP_PRIVATE, reader[i].metafd, 0); - if (map == MAP_FAILED) { - perror("mmap unmerged meta"); - exit(EXIT_FAILURE); - } - madvise(map, reader[i].metapos, MADV_SEQUENTIAL); - madvise(map, reader[i].metapos, MADV_WILLNEED); - if (fwrite(map, reader[i].metapos, 1, metafile) != 1) { - perror("Reunify meta"); - exit(EXIT_FAILURE); - } - madvise(map, reader[i].metapos, MADV_DONTNEED); - if (munmap(map, reader[i].metapos) != 0) { - perror("unmap unmerged meta"); - } - } - - meta_off[i] = metapos; - metapos += reader[i].metapos; - if (close(reader[i].metafd) != 0) { - perror("close unmerged meta"); - } - - if (reader[i].poolfile->off > 0) { - if (fwrite(reader[i].poolfile->map, reader[i].poolfile->off, 1, poolfile) != 1) { - perror("Reunify string pool"); - exit(EXIT_FAILURE); - } - } - - pool_off[i] = poolpos; - poolpos += reader[i].poolfile->off; - memfile_close(reader[i].poolfile); - } - - if (fclose(poolfile) != 0) { - perror("fclose pool"); - exit(EXIT_FAILURE); - } - if (fclose(metafile) != 0) { - perror("fclose meta"); - exit(EXIT_FAILURE); - } - - char *meta = (char *) mmap(NULL, metapos, PROT_READ, MAP_PRIVATE, metafd, 0); - if (meta == MAP_FAILED) { - perror("mmap meta"); - exit(EXIT_FAILURE); - } - madvise(meta, metapos, MADV_RANDOM); - - char *stringpool = NULL; - if (poolpos > 0) { // Will be 0 if -X was specified - stringpool = (char *) mmap(NULL, poolpos, PROT_READ, MAP_PRIVATE, poolfd, 0); - if (stringpool == MAP_FAILED) { - perror("mmap string pool"); - exit(EXIT_FAILURE); - } - madvise(stringpool, poolpos, MADV_RANDOM); - } - - char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; - sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX"); - - int indexfd = mkstemp(indexname); - if (indexfd < 0) { - perror(indexname); - exit(EXIT_FAILURE); - } - FILE *indexfile = fopen(indexname, "wb"); - if (indexfile == NULL) { - perror(indexname); - exit(EXIT_FAILURE); - } - - unlink(indexname); - - char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; - sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX"); - - int geomfd = mkstemp(geomname); - if (geomfd < 0) { - perror(geomname); - exit(EXIT_FAILURE); - } - FILE *geomfile = fopen(geomname, "wb"); - if (geomfile == NULL) { - perror(geomname); - exit(EXIT_FAILURE); - } - unlink(geomname); - - long long geompos = 0; - - /* initial tile is 0/0/0 */ - serialize_int(geomfile, 0, &geompos, fname); - serialize_uint(geomfile, 0, &geompos, fname); - serialize_uint(geomfile, 0, &geompos, fname); - - radix(reader, CPUS, geomfile, geomfd, indexfile, indexfd, tmpdir, &geompos); - - /* end of tile */ - serialize_byte(geomfile, -2, &geompos, fname); - - if (fclose(geomfile) != 0) { - perror("fclose geom"); - exit(EXIT_FAILURE); - } - if (fclose(indexfile) != 0) { - perror("fclose index"); - exit(EXIT_FAILURE); - } - - struct stat indexst; - if (fstat(indexfd, &indexst) < 0) { - perror("stat index"); - exit(EXIT_FAILURE); - } - long long indexpos = indexst.st_size; - progress_seq = indexpos / sizeof(struct index); - - if (!quiet) { - fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", progress_seq, geompos, metapos, poolpos); - } - - if (indexpos == 0) { - fprintf(stderr, "Did not read any valid geometries\n"); - exit(EXIT_FAILURE); - } - - if (basezoom < 0 || droprate < 0) { - struct index *map = (struct index *) mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0); - if (map == MAP_FAILED) { - perror("mmap index for basezoom"); - exit(EXIT_FAILURE); - } - madvise(map, indexpos, MADV_SEQUENTIAL); - madvise(map, indexpos, MADV_WILLNEED); - - struct tile { - unsigned x; - unsigned y; - long long count; - long long fullcount; - double gap; - unsigned long long previndex; - } tile[MAX_ZOOM + 1], max[MAX_ZOOM + 1]; - - { - int i; - for (i = 0; i <= MAX_ZOOM; i++) { - tile[i].x = tile[i].y = tile[i].count = tile[i].fullcount = tile[i].gap = tile[i].previndex = 0; - max[i].x = max[i].y = max[i].count = max[i].fullcount = 0; - } - } - - long long progress = -1; - - long long indices = indexpos / sizeof(struct index); - long long i; - for (i = 0; i < indices; i++) { - unsigned xx, yy; - decode(map[i].index, &xx, &yy); - - long long nprogress = 100 * i / indices; - if (nprogress != progress) { - progress = nprogress; - if (!quiet) { - fprintf(stderr, "Base zoom/drop rate: %lld%% \r", progress); - } - } - - int z; - for (z = 0; z <= MAX_ZOOM; z++) { - unsigned xxx = 0, yyy = 0; - if (z != 0) { - xxx = xx >> (32 - z); - yyy = yy >> (32 - z); - } - - double scale = (double) (1LL << (64 - 2 * (z + 8))); - - if (tile[z].x != xxx || tile[z].y != yyy) { - if (tile[z].count > max[z].count) { - max[z] = tile[z]; - } - - tile[z].x = xxx; - tile[z].y = yyy; - tile[z].count = 0; - tile[z].fullcount = 0; - tile[z].gap = 0; - tile[z].previndex = 0; - } - - tile[z].fullcount++; - - if (manage_gap(map[i].index, &tile[z].previndex, scale, gamma, &tile[z].gap)) { - continue; - } - - tile[z].count++; - } - } - - int z; - for (z = MAX_ZOOM; z >= 0; z--) { - if (tile[z].count > max[z].count) { - max[z] = tile[z]; - } - } - - int max_features = 50000 / (basezoom_marker_width * basezoom_marker_width); - - int obasezoom = basezoom; - if (basezoom < 0) { - basezoom = MAX_ZOOM; - - for (z = MAX_ZOOM; z >= 0; z--) { - if (max[z].count < max_features) { - basezoom = z; - } - - // printf("%d/%u/%u %lld\n", z, max[z].x, max[z].y, max[z].count); - } - - fprintf(stderr, "Choosing a base zoom of -B%d to keep %lld features in tile %d/%u/%u.\n", basezoom, max[basezoom].count, basezoom, max[basezoom].x, max[basezoom].y); - } - - if (obasezoom < 0 && basezoom > maxzoom) { - fprintf(stderr, "Couldn't find a suitable base zoom. Working from the other direction.\n"); - if (gamma == 0) { - fprintf(stderr, "You might want to try -g1 to limit near-duplicates.\n"); - } - - if (droprate < 0) { - if (maxzoom == 0) { - droprate = 2.5; - } else { - droprate = exp(log((long double) max[0].count / max[maxzoom].count) / (maxzoom)); - fprintf(stderr, "Choosing a drop rate of -r%f to get from %lld to %lld in %d zooms\n", droprate, max[maxzoom].count, max[0].count, maxzoom); - } - } - - basezoom = 0; - for (z = 0; z <= maxzoom; z++) { - double zoomdiff = log((long double) max[z].count / max_features) / log(droprate); - if (zoomdiff + z > basezoom) { - basezoom = ceil(zoomdiff + z); - } - } - - fprintf(stderr, "Choosing a base zoom of -B%d to keep %f features in tile %d/%u/%u.\n", basezoom, max[maxzoom].count * exp(log(droprate) * (maxzoom - basezoom)), maxzoom, max[maxzoom].x, max[maxzoom].y); - } else if (droprate < 0) { - droprate = 1; - - for (z = basezoom - 1; z >= 0; z--) { - double interval = exp(log(droprate) * (basezoom - z)); - - if (max[z].count / interval >= max_features) { - interval = (long double) max[z].count / max_features; - droprate = exp(log(interval) / (basezoom - z)); - interval = exp(log(droprate) * (basezoom - z)); - - fprintf(stderr, "Choosing a drop rate of -r%f to keep %f features in tile %d/%u/%u.\n", droprate, max[z].count / interval, z, max[z].x, max[z].y); - } - } - } - - if (gamma > 0) { - int effective = 0; - - for (z = 0; z < maxzoom; z++) { - if (max[z].count < max[z].fullcount) { - effective = z + 1; - } - } - - if (effective == 0) { - fprintf(stderr, "With gamma, effective base zoom is 0, so no effective drop rate\n"); - } else { - double interval_0 = exp(log(droprate) * (basezoom - 0)); - double interval_eff = exp(log(droprate) * (basezoom - effective)); - if (effective > basezoom) { - interval_eff = 1; - } - - double scaled_0 = max[0].count / interval_0; - double scaled_eff = max[effective].count / interval_eff; - - double rate_at_0 = scaled_0 / max[0].fullcount; - double rate_at_eff = scaled_eff / max[effective].fullcount; - - double eff_drop = exp(log(rate_at_eff / rate_at_0) / (effective - 0)); - - fprintf(stderr, "With gamma, effective base zoom of %d, effective drop rate of %f\n", effective, eff_drop); - } - } - - madvise(map, indexpos, MADV_DONTNEED); - munmap(map, indexpos); - } - - if (close(indexfd) != 0) { - perror("close sorted index"); - } - - /* Traverse and split the geometries for each zoom level */ - - struct stat geomst; - if (fstat(geomfd, &geomst) != 0) { - perror("stat sorted geom\n"); - exit(EXIT_FAILURE); - } - - int fd[TEMP_FILES]; - off_t size[TEMP_FILES]; - - fd[0] = geomfd; - size[0] = geomst.st_size; - - int j; - for (j = 1; j < TEMP_FILES; j++) { - fd[j] = -1; - size[j] = 0; - } - - unsigned midx = 0, midy = 0; - int written = traverse_zooms(fd, size, meta, stringpool, file_keys, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y); - - if (maxzoom != written) { - fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); - maxzoom = written; - ret = EXIT_FAILURE; - } - - madvise(meta, metapos, MADV_DONTNEED); - if (munmap(meta, metapos) != 0) { - perror("munmap meta"); - } - if (close(metafd) < 0) { - perror("close meta"); - } - - if (poolpos > 0) { - madvise((void *) pool, poolpos, MADV_DONTNEED); - if (munmap(stringpool, poolpos) != 0) { - perror("munmap stringpool"); - } - } - if (close(poolfd) < 0) { - perror("close pool"); - } - - double minlat = 0, minlon = 0, maxlat = 0, maxlon = 0, midlat = 0, midlon = 0; - - tile2latlon(midx, midy, maxzoom, &maxlat, &minlon); - tile2latlon(midx + 1, midy + 1, maxzoom, &minlat, &maxlon); - - midlat = (maxlat + minlat) / 2; - midlon = (maxlon + minlon) / 2; - - long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0}; - for (i = 0; i < CPUS; i++) { - if (reader[i].file_bbox[0] < file_bbox[0]) { - file_bbox[0] = reader[i].file_bbox[0]; - } - if (reader[i].file_bbox[1] < file_bbox[1]) { - file_bbox[1] = reader[i].file_bbox[1]; - } - if (reader[i].file_bbox[2] > file_bbox[2]) { - file_bbox[2] = reader[i].file_bbox[2]; - } - if (reader[i].file_bbox[3] > file_bbox[3]) { - file_bbox[3] = reader[i].file_bbox[3]; - } - } - - // If the bounding box extends off the plane on either side, - // a feature wrapped across the date line, so the width of the - // bounding box is the whole world. - if (file_bbox[0] < 0) { - file_bbox[0] = 0; - file_bbox[2] = (1LL << 32) - 1; - } - if (file_bbox[2] > (1LL << 32) - 1) { - file_bbox[0] = 0; - file_bbox[2] = (1LL << 32) - 1; - } - if (file_bbox[1] < 0) { - file_bbox[1] = 0; - } - if (file_bbox[3] > (1LL << 32) - 1) { - file_bbox[3] = (1LL << 32) - 1; - } - - tile2latlon(file_bbox[0], file_bbox[1], 32, &maxlat, &minlon); - tile2latlon(file_bbox[2], file_bbox[3], 32, &minlat, &maxlon); - - if (midlat < minlat) { - midlat = minlat; - } - if (midlat > maxlat) { - midlat = maxlat; - } - if (midlon < minlon) { - midlon = minlon; - } - if (midlon > maxlon) { - midlon = maxlon; - } - - mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable, attribution); - - for (i = 0; i < nlayers; i++) { - pool_free_strings(&file_keys1[i]); - free(layernames[i]); - } - - return ret; -} - -int int_in(int v, int *a, int len) { - int i; - - for (i = 0; i < len; i++) { - if (a[i] == v) { - return 1; - } - } - - return 0; -} - -int main(int argc, char **argv) { -#ifdef MTRACE - mtrace(); -#endif - - init_cpus(); - - extern int optind; - extern char *optarg; - int i; - - char *name = NULL; - char *layer = NULL; - char *outdir = NULL; - int maxzoom = 14; - int minzoom = 0; - int basezoom = -1; - double basezoom_marker_width = 1; - int force = 0; - int forcetable = 0; - double droprate = 2.5; - double gamma = 0; - int buffer = 5; - const char *tmpdir = "/tmp"; - const char *attribution = NULL; - - int nsources = 0; - struct source *sources = NULL; - - struct pool exclude, include; - pool_init(&exclude, 0); - pool_init(&include, 0); - int exclude_all = 0; - int read_parallel = 0; - int files_open_at_start; - - for (i = 0; i < 256; i++) { - prevent[i] = 0; - additional[i] = 0; - } - - { - char dup[256]; - - memset(dup, 0, sizeof(dup)); - for (i = 0; i < sizeof(additional_options) / sizeof(additional_options[0]); i++) { - if (dup[additional_options[i]]) { - fprintf(stderr, "Internal error: reused -a%c\n", additional_options[i]); - exit(EXIT_FAILURE); - } - dup[additional_options[i]] = 1; - } - - memset(dup, 0, sizeof(dup)); - for (i = 0; i < sizeof(prevent_options) / sizeof(prevent_options[0]); i++) { - if (dup[prevent_options[i]]) { - fprintf(stderr, "Internal error: reused -p%c\n", prevent_options[i]); - exit(EXIT_FAILURE); - } - dup[prevent_options[i]] = 1; - } - } - - static struct option long_options[] = { - {"output", required_argument, 0, 'o'}, - - {"name", required_argument, 0, 'n'}, - {"layer", required_argument, 0, 'l'}, - {"attribution", required_argument, 0, 'A'}, - {"named-layer", required_argument, 0, 'L'}, - {"maximum-zoom", required_argument, 0, 'z'}, - {"minimum-zoom", required_argument, 0, 'Z'}, - {"base-zoom", required_argument, 0, 'B'}, - {"full-detail", required_argument, 0, 'd'}, - {"low-detail", required_argument, 0, 'D'}, - {"minimum-detail", required_argument, 0, 'm'}, - {"exclude", required_argument, 0, 'x'}, - {"include", required_argument, 0, 'y'}, - {"drop-rate", required_argument, 0, 'r'}, - {"buffer", required_argument, 0, 'b'}, - {"temporary-directory", required_argument, 0, 't'}, - {"gamma", required_argument, 0, 'g'}, - {"prevent", required_argument, 0, 'p'}, - {"additional", required_argument, 0, 'a'}, - - {"exclude-all", no_argument, 0, 'X'}, - {"force", no_argument, 0, 'f'}, - {"allow-existing", no_argument, 0, 'F'}, - {"quiet", no_argument, 0, 'q'}, - {"version", no_argument, 0, 'v'}, - {"read-parallel", no_argument, 0, 'P'}, - - {"coalesce", no_argument, &additional[A_COALESCE], 1}, - {"reverse", no_argument, &additional[A_REVERSE], 1}, - {"reorder", no_argument, &additional[A_REORDER], 1}, - {"drop-lines", no_argument, &additional[A_LINE_DROP], 1}, - {"check-polygons", no_argument, &additional[A_DEBUG_POLYGON], 1}, - {"drop-polygons", no_argument, &additional[A_POLYGON_DROP], 1}, - {"prefer-radix-sort", no_argument, &additional[A_PREFER_RADIX_SORT], 1}, - - {"no-line-simplification", no_argument, &prevent[P_SIMPLIFY], 1}, - {"simplify-only-low-zooms", no_argument, &prevent[P_SIMPLIFY_LOW], 1}, - {"no-feature-limit", no_argument, &prevent[P_FEATURE_LIMIT], 1}, - {"no-tile-size-limit", no_argument, &prevent[P_KILOBYTE_LIMIT], 1}, - {"force-feature-limit", no_argument, &prevent[P_DYNAMIC_DROP], 1}, - {"preseve-input-order", no_argument, &prevent[P_INPUT_ORDER], 1}, - {"no-polygon-splitting", no_argument, &prevent[P_POLYGON_SPLIT], 1}, - {"no-clipping", no_argument, &prevent[P_CLIPPING], 1}, - {"no-duplication", no_argument, &prevent[P_DUPLICATION], 1}, - - {0, 0, 0, 0}, - }; - - while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:", long_options, NULL)) != -1) { - switch (i) { - case 0: - break; - - case 'n': - name = optarg; - break; - - case 'l': - layer = optarg; - break; - - case 'A': - attribution = optarg; - break; - - case 'L': { - char *cp = strchr(optarg, ':'); - if (cp == NULL || cp == optarg) { - fprintf(stderr, "%s: -L requires layername:file\n", argv[0]); - exit(EXIT_FAILURE); - } - struct source *src = (struct source *) malloc(sizeof(struct source)); - if (src == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - src->layer = strdup(optarg); - src->file = strdup(cp + 1); - if (src->layer == NULL || src->file == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - mustfree(src->layer); - mustfree(src->file); - src->layer[cp - optarg] = '\0'; - src->next = sources; - sources = src; - nsources++; - } break; - - case 'z': - maxzoom = atoi(optarg); - break; - - case 'Z': - minzoom = atoi(optarg); - break; - - case 'B': - if (strcmp(optarg, "g") == 0) { - basezoom = -2; - } else if (optarg[0] == 'g' || optarg[0] == 'f') { - basezoom = -2; - if (optarg[0] == 'g') { - basezoom_marker_width = atof(optarg + 1); - } else { - basezoom_marker_width = sqrt(50000 / atof(optarg + 1)); - } - if (basezoom_marker_width == 0 || atof(optarg + 1) == 0) { - fprintf(stderr, "%s: Must specify value >0 with -B%c\n", argv[0], optarg[0]); - exit(EXIT_FAILURE); - } - } else { - basezoom = atoi(optarg); - if (basezoom == 0 && strcmp(optarg, "0") != 0) { - fprintf(stderr, "%s: Couldn't understand -B%s\n", argv[0], optarg); - exit(EXIT_FAILURE); - } - } - break; - - case 'd': - full_detail = atoi(optarg); - break; - - case 'D': - low_detail = atoi(optarg); - break; - - case 'm': - min_detail = atoi(optarg); - break; - - case 'o': - outdir = optarg; - break; - - case 'x': - pool(&exclude, optarg, VT_STRING); - break; - - case 'y': - exclude_all = 1; - pool(&include, optarg, VT_STRING); - break; - - case 'X': - exclude_all = 1; - break; - - case 'r': - if (strcmp(optarg, "g") == 0) { - droprate = -2; - } else if (optarg[0] == 'g' || optarg[0] == 'f') { - droprate = -2; - if (optarg[0] == 'g') { - basezoom_marker_width = atof(optarg + 1); - } else { - basezoom_marker_width = sqrt(50000 / atof(optarg + 1)); - } - if (basezoom_marker_width == 0 || atof(optarg + 1) == 0) { - fprintf(stderr, "%s: Must specify value >0 with -r%c\n", argv[0], optarg[0]); - exit(EXIT_FAILURE); - } - } else { - droprate = atof(optarg); - } - break; - - case 'b': - buffer = atoi(optarg); - break; - - case 'f': - force = 1; - break; - - case 'F': - forcetable = 1; - break; - - case 't': - tmpdir = optarg; - if (tmpdir[0] != '/') { - fprintf(stderr, "Warning: temp directory %s doesn't begin with /\n", tmpdir); - } - break; - - case 'g': - gamma = atof(optarg); - break; - - case 'q': - quiet = 1; - break; - - case 'p': { - char *cp; - for (cp = optarg; *cp != '\0'; cp++) { - if (int_in(*cp, prevent_options, sizeof(prevent_options) / sizeof(prevent_options[0]))) { - prevent[*cp & 0xFF] = 1; - } else { - fprintf(stderr, "%s: Unknown option -p%c\n", argv[0], *cp); - exit(EXIT_FAILURE); - } - } - } break; - - case 'a': { - char *cp; - for (cp = optarg; *cp != '\0'; cp++) { - if (int_in(*cp, additional_options, sizeof(additional_options) / sizeof(additional_options[0]))) { - additional[*cp & 0xFF] = 1; - } else { - fprintf(stderr, "%s: Unknown option -a%c\n", argv[0], *cp); - exit(EXIT_FAILURE); - } - } - } break; - - case 'v': - fprintf(stderr, VERSION); - exit(EXIT_FAILURE); - - case 'P': - read_parallel = 1; - break; - - default: { - int width = 7 + strlen(argv[0]); - fprintf(stderr, "Usage: %s", argv[0]); - int i; - for (i = 0; long_options[i].name != NULL; i++) { - if (width + strlen(long_options[i].name) + 9 >= 80) { - fprintf(stderr, "\n "); - width = 8; - } - width += strlen(long_options[i].name) + 9; - if (strcmp(long_options[i].name, "output") == 0) { - fprintf(stderr, " --%s=output.mbtiles", long_options[i].name); - width += 9; - } else if (long_options[i].has_arg) { - fprintf(stderr, " [--%s=...]", long_options[i].name); - } else { - fprintf(stderr, " [--%s]", long_options[i].name); - } - } - if (width + 16 >= 80) { - fprintf(stderr, "\n "); - width = 8; - } - fprintf(stderr, " [file.json ...]"); - } - exit(EXIT_FAILURE); - } - } - - files_open_at_start = open("/dev/null", O_RDONLY); - close(files_open_at_start); - - if (full_detail <= 0) { - full_detail = 12; - } - - if (full_detail < min_detail || low_detail < min_detail) { - fprintf(stderr, "%s: Full detail and low detail must be at least minimum detail\n", argv[0]); - exit(EXIT_FAILURE); - } - - // Need two checks: one for geometry representation, the other for - // index traversal when guessing base zoom and drop rate - if (maxzoom > 32 - full_detail) { - maxzoom = 32 - full_detail; - fprintf(stderr, "Highest supported zoom with detail %d is %d\n", full_detail, maxzoom); - } - if (maxzoom > MAX_ZOOM) { - maxzoom = MAX_ZOOM; - fprintf(stderr, "Highest supported zoom is %d\n", maxzoom); - } - - if (minzoom > maxzoom) { - fprintf(stderr, "minimum zoom -Z cannot be greater than maxzoom -z\n"); - exit(EXIT_FAILURE); - } - - if (basezoom == -1) { - basezoom = maxzoom; - } - - geometry_scale = 32 - (full_detail + maxzoom); - if (geometry_scale < 0) { - geometry_scale = 0; - fprintf(stderr, "Full detail + maxzoom > 32, so you are asking for more detail than is available.\n"); - } - - if ((basezoom < 0 || droprate < 0) && (gamma < 0)) { - // Can't use randomized (as opposed to evenly distributed) dot dropping - // if rate and base aren't known during feature reading. - gamma = 0; - fprintf(stderr, "Forcing -g0 since -B or -r is not known\n"); - } - - if (outdir == NULL) { - fprintf(stderr, "%s: must specify -o out.mbtiles\n", argv[0]); - exit(EXIT_FAILURE); - } - - if (force) { - unlink(outdir); - } - - sqlite3 *outdb = mbtiles_open(outdir, argv, forcetable); - int ret = EXIT_SUCCESS; - - for (i = optind; i < argc; i++) { - struct source *src = (struct source *) malloc(sizeof(struct source)); - if (src == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - src->layer = NULL; - src->file = argv[i]; - src->next = sources; - sources = src; - nsources++; - } - - struct source *sourcelist[nsources]; - i = nsources - 1; - for (; sources != NULL; sources = sources->next) { - sourcelist[i--] = sources; - } - - ret = read_json(nsources, sourcelist, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel, forcetable, attribution); - - mbtiles_close(outdb, argv); - -#ifdef MTRACE - muntrace(); -#endif - - i = open("/dev/null", O_RDONLY); - // i < files_open_at_start is not an error, because reading from a pipe closes stdin - if (i > files_open_at_start) { - fprintf(stderr, "Internal error: did not close all files: %d\n", i); - exit(EXIT_FAILURE); - } - - for (i = 0; i < nsources; i++) { - free(sourcelist[i]); - } - - pool_free(&exclude); - pool_free(&include); - - struct tofree *tf, *next; - for (tf = tofree; tf != NULL; tf = next) { - next = tf->next; - free(tf->p); - free(tf); - } - - return ret; -} diff --git a/geojson.hpp b/geojson.hpp new file mode 100644 index 0000000..925145b --- /dev/null +++ b/geojson.hpp @@ -0,0 +1,31 @@ +struct parse_json_args { + json_pull *jp; + const char *reading; + volatile long long *layer_seq; + volatile long long *progress_seq; + long long *metapos; + long long *geompos; + long long *indexpos; + struct pool *exclude; + struct pool *include; + int exclude_all; + FILE *metafile; + FILE *geomfile; + FILE *indexfile; + struct memfile *poolfile; + struct memfile *treefile; + char *fname; + int basezoom; + int layer; + double droprate; + long long *file_bbox; + int segment; + int *initialized; + unsigned *initial_x; + unsigned *initial_y; + struct reader *readers; +}; + +struct json_pull *json_begin_map(char *map, long long len); +void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers); +void *run_parse_json(void *v); diff --git a/main.hpp b/main.hpp new file mode 100644 index 0000000..105397e --- /dev/null +++ b/main.hpp @@ -0,0 +1,9 @@ +struct index { + long long start; + long long end; + unsigned long long index; + short segment; + unsigned long long seq : (64 - 16); // pack with segment to stay in 32 bytes +}; + +void checkdisk(struct reader *r, int nreader); diff --git a/pool.cpp b/pool.cpp index 050a2c5..349f0c6 100644 --- a/pool.cpp +++ b/pool.cpp @@ -1,6 +1,7 @@ #include #include #include +#include "memfile.hpp" #include "pool.hpp" #define POOL_WIDTH 256 @@ -117,3 +118,95 @@ void pool_init(struct pool *p, int n) { p->head = NULL; p->tail = NULL; } + +static unsigned char swizzle[256] = { + 0x00, 0xBF, 0x18, 0xDE, 0x93, 0xC9, 0xB1, 0x5E, 0xDF, 0xBE, 0x72, 0x5A, 0xBB, 0x42, 0x64, 0xC6, + 0xD8, 0xB7, 0x15, 0x74, 0x1C, 0x8B, 0x91, 0xF5, 0x29, 0x46, 0xEC, 0x6F, 0xCA, 0x20, 0xF0, 0x06, + 0x27, 0x61, 0x87, 0xE0, 0x6E, 0x43, 0x50, 0xC5, 0x1B, 0xB4, 0x37, 0xC3, 0x69, 0xA6, 0xEE, 0x80, + 0xAF, 0x9B, 0xA1, 0x76, 0x23, 0x24, 0x53, 0xF3, 0x5B, 0x65, 0x19, 0xF4, 0xFC, 0xDD, 0x26, 0xE8, + 0x10, 0xF7, 0xCE, 0x92, 0x48, 0xF6, 0x94, 0x60, 0x07, 0xC4, 0xB9, 0x97, 0x6D, 0xA4, 0x11, 0x0D, + 0x1F, 0x4D, 0x13, 0xB0, 0x5D, 0xBA, 0x31, 0xD5, 0x8D, 0x51, 0x36, 0x96, 0x7A, 0x03, 0x7F, 0xDA, + 0x17, 0xDB, 0xD4, 0x83, 0xE2, 0x79, 0x6A, 0xE1, 0x95, 0x38, 0xFF, 0x28, 0xB2, 0xB3, 0xA7, 0xAE, + 0xF8, 0x54, 0xCC, 0xDC, 0x9A, 0x6B, 0xFB, 0x3F, 0xD7, 0xBC, 0x21, 0xC8, 0x71, 0x09, 0x16, 0xAC, + 0x3C, 0x8A, 0x62, 0x05, 0xC2, 0x8C, 0x32, 0x4E, 0x35, 0x9C, 0x5F, 0x75, 0xCD, 0x2E, 0xA2, 0x3E, + 0x1A, 0xC1, 0x8E, 0x14, 0xA0, 0xD3, 0x7D, 0xD9, 0xEB, 0x5C, 0x70, 0xE6, 0x9E, 0x12, 0x3B, 0xEF, + 0x1E, 0x49, 0xD2, 0x98, 0x39, 0x7E, 0x44, 0x4B, 0x6C, 0x88, 0x02, 0x2C, 0xAD, 0xE5, 0x9F, 0x40, + 0x7B, 0x4A, 0x3D, 0xA9, 0xAB, 0x0B, 0xD6, 0x2F, 0x90, 0x2A, 0xB6, 0x1D, 0xC7, 0x22, 0x55, 0x34, + 0x0A, 0xD0, 0xB5, 0x68, 0xE3, 0x59, 0xFD, 0xFA, 0x57, 0x77, 0x25, 0xA3, 0x04, 0xB8, 0x33, 0x89, + 0x78, 0x82, 0xE4, 0xC0, 0x0E, 0x8F, 0x85, 0xD1, 0x84, 0x08, 0x67, 0x47, 0x9D, 0xCB, 0x58, 0x4C, + 0xAA, 0xED, 0x52, 0xF2, 0x4F, 0xF1, 0x66, 0xCF, 0xA5, 0x56, 0xEA, 0x7C, 0xE9, 0x63, 0xE7, 0x01, + 0xF9, 0xFE, 0x0C, 0x99, 0x2D, 0x0F, 0x3A, 0x41, 0x45, 0xA8, 0x30, 0x2B, 0x73, 0xBD, 0x86, 0x81, +}; + +int swizzlecmp(const char *a, const char *b) { + while (*a || *b) { + int aa = swizzle[(unsigned char) *a]; + int bb = swizzle[(unsigned char) *b]; + + int cmp = aa - bb; + if (cmp != 0) { + return cmp; + } + + a++; + b++; + } + + return 0; +} + +long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type) { + long long *sp = &treefile->tree; + + while (*sp != 0) { + int cmp = swizzlecmp(s, poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off + 1); + + if (cmp == 0) { + cmp = type - (poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off)[0]; + } + + if (cmp < 0) { + sp = &(((struct stringpool *) (treefile->map + *sp))->left); + } else if (cmp > 0) { + sp = &(((struct stringpool *) (treefile->map + *sp))->right); + } else { + return ((struct stringpool *) (treefile->map + *sp))->off; + } + } + + // *sp is probably in the memory-mapped file, and will move if the file grows. + long long ssp; + if (sp == &treefile->tree) { + ssp = -1; + } else { + ssp = ((char *) sp) - treefile->map; + } + + long long off = poolfile->off; + if (memfile_write(poolfile, &type, 1) < 0) { + perror("memfile write"); + exit(EXIT_FAILURE); + } + if (memfile_write(poolfile, (void *) s, strlen(s) + 1) < 0) { + perror("memfile write"); + exit(EXIT_FAILURE); + } + + struct stringpool tsp; + tsp.left = 0; + tsp.right = 0; + tsp.off = off; + + long long p = treefile->off; + if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) { + perror("memfile write"); + exit(EXIT_FAILURE); + } + + if (ssp == -1) { + treefile->tree = p; + } else { + *((long long *) (treefile->map + ssp)) = p; + } + return off; +} diff --git a/pool.hpp b/pool.hpp index 100dd76..378aadf 100644 --- a/pool.hpp +++ b/pool.hpp @@ -22,3 +22,11 @@ void pool_free(struct pool *p); void pool_free_strings(struct pool *p); void pool_init(struct pool *p, int n); int is_pooled(struct pool *p, const char *s, int type); + +struct stringpool { + long long left; + long long right; + long long off; +}; + +long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type); From ee97e6c3078ffae350706942d21ab8027685a83f Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 15:09:06 -0700 Subject: [PATCH 14/19] Whittle down tile.hpp to the things that actually related to tile.cpp --- decode.cpp | 16 +++++++------- geojson.cpp | 3 ++- geometry.cpp | 2 +- geometry.hpp | 13 ++++++++++++ main.hpp | 6 ++++++ mbtiles.cpp | 3 ++- options.hpp | 37 ++++++++++++++++++++++++++++++++ tile-join.cpp | 2 +- tile.cpp | 2 ++ tile.hpp | 59 --------------------------------------------------- 10 files changed, 72 insertions(+), 71 deletions(-) create mode 100644 options.hpp diff --git a/decode.cpp b/decode.cpp index f52d3d6..8b58319 100644 --- a/decode.cpp +++ b/decode.cpp @@ -12,8 +12,8 @@ #include #include #include "mvt.hpp" -#include "tile.hpp" #include "projection.hpp" +#include "geometry.hpp" void printq(const char *s) { putchar('"'); @@ -29,12 +29,12 @@ void printq(const char *s) { putchar('"'); } -struct draw { +struct lonlat { int op; double lon; double lat; - draw(int op, double lon, double lat) { + lonlat(int op, double lon, double lat) { this->op = op; this->lon = lon; this->lat = lat; @@ -141,7 +141,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) { printf(" }, \"geometry\": { "); - std::vector ops; + std::vector ops; for (size_t g = 0; g < feat.geometry.size(); g++) { int op = feat.geometry[g].op; @@ -156,9 +156,9 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) { double lat, lon; tile2latlon(wx, wy, 32, &lat, &lon); - ops.push_back(draw(op, lon, lat)); + ops.push_back(lonlat(op, lon, lat)); } else { - ops.push_back(draw(op, 0, 0)); + ops.push_back(lonlat(op, 0, 0)); } } @@ -212,12 +212,12 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) { printf(" ] ]"); } } else if (feat.type == VT_POLYGON) { - std::vector > rings; + std::vector > rings; std::vector areas; for (size_t i = 0; i < ops.size(); i++) { if (ops[i].op == VT_MOVETO) { - rings.push_back(std::vector()); + rings.push_back(std::vector()); areas.push_back(0); } diff --git a/geojson.cpp b/geojson.cpp index 741b8ee..f8f0c75 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -19,12 +19,12 @@ #include #include #include +#include extern "C" { #include "jsonpull.h" } -#include "tile.hpp" #include "pool.hpp" #include "mbtiles.hpp" #include "projection.hpp" @@ -33,6 +33,7 @@ extern "C" { #include "serial.hpp" #include "main.hpp" #include "geojson.hpp" +#include "geometry.hpp" #define GEOM_POINT 0 /* array of positions */ #define GEOM_MULTIPOINT 1 /* array of arrays of positions */ diff --git a/geometry.cpp b/geometry.cpp index e932fb2..1cefe65 100644 --- a/geometry.cpp +++ b/geometry.cpp @@ -11,9 +11,9 @@ #include #include "geometry.hpp" #include "clipper/clipper.hpp" -#include "tile.hpp" #include "projection.hpp" #include "serial.hpp" +#include "main.hpp" static int pnpoly(drawvec &vert, size_t start, size_t nvert, long long testx, long long testy); static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax); diff --git a/geometry.hpp b/geometry.hpp index 77fdc6c..760a383 100644 --- a/geometry.hpp +++ b/geometry.hpp @@ -1,3 +1,16 @@ +#define VT_POINT 1 +#define VT_LINE 2 +#define VT_POLYGON 3 + +#define VT_END 0 +#define VT_MOVETO 1 +#define VT_LINETO 2 +#define VT_CLOSEPATH 7 + +#define VT_STRING 1 +#define VT_NUMBER 2 +#define VT_BOOLEAN 7 + struct draw { signed char op; long long x; diff --git a/main.hpp b/main.hpp index 105397e..ff0338d 100644 --- a/main.hpp +++ b/main.hpp @@ -7,3 +7,9 @@ struct index { }; void checkdisk(struct reader *r, int nreader); + +extern int geometry_scale; +extern int quiet; + +extern int CPUS; +extern int TEMP_FILES; diff --git a/mbtiles.cpp b/mbtiles.cpp index 216bdbb..697630c 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -5,9 +5,10 @@ #include #include #include +#include #include "pool.hpp" -#include "tile.hpp" #include "mbtiles.hpp" +#include "geometry.hpp" sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) { sqlite3 *outdb; diff --git a/options.hpp b/options.hpp new file mode 100644 index 0000000..8e7d130 --- /dev/null +++ b/options.hpp @@ -0,0 +1,37 @@ +static int additional_options[] = { +#define A_COALESCE ((int) 'c') + A_COALESCE, +#define A_REVERSE ((int) 'r') + A_REVERSE, +#define A_REORDER ((int) 'o') + A_REORDER, +#define A_LINE_DROP ((int) 'l') + A_LINE_DROP, +#define A_DEBUG_POLYGON ((int) 'd') + A_DEBUG_POLYGON, +#define A_POLYGON_DROP ((int) 'p') + A_POLYGON_DROP, +#define A_PREFER_RADIX_SORT ((int) 'R') + A_PREFER_RADIX_SORT, +}; + +static int prevent_options[] = { +#define P_SIMPLIFY ((int) 's') + P_SIMPLIFY, +#define P_SIMPLIFY_LOW ((int) 'S') + P_SIMPLIFY_LOW, +#define P_FEATURE_LIMIT ((int) 'f') + P_FEATURE_LIMIT, +#define P_KILOBYTE_LIMIT ((int) 'k') + P_KILOBYTE_LIMIT, +#define P_DYNAMIC_DROP ((int) 'd') + P_DYNAMIC_DROP, +#define P_INPUT_ORDER ((int) 'i') + P_INPUT_ORDER, +#define P_POLYGON_SPLIT ((int) 'p') + P_POLYGON_SPLIT, +#define P_CLIPPING ((int) 'c') + P_CLIPPING, +#define P_DUPLICATION ((int) 'D') + P_DUPLICATION, +}; diff --git a/tile-join.cpp b/tile-join.cpp index b265264..acccbb3 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -10,10 +10,10 @@ #include #include #include "mvt.hpp" -#include "tile.hpp" #include "projection.hpp" #include "pool.hpp" #include "mbtiles.hpp" +#include "geometry.hpp" std::string dequote(std::string s); diff --git a/tile.cpp b/tile.cpp index 99370f0..9a235f4 100644 --- a/tile.cpp +++ b/tile.cpp @@ -26,6 +26,8 @@ #include "mbtiles.hpp" #include "projection.hpp" #include "serial.hpp" +#include "options.hpp" +#include "main.hpp" #define CMD_BITS 3 diff --git a/tile.hpp b/tile.hpp index b16a142..e37a02a 100644 --- a/tile.hpp +++ b/tile.hpp @@ -1,64 +1,5 @@ -#define VT_POINT 1 -#define VT_LINE 2 -#define VT_POLYGON 3 - -#define VT_END 0 -#define VT_MOVETO 1 -#define VT_LINETO 2 -#define VT_CLOSEPATH 7 - -#define VT_STRING 1 -#define VT_NUMBER 2 -#define VT_BOOLEAN 7 - -struct pool; - long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, int *prevent, int *additional); int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); - -extern int geometry_scale; -extern int quiet; - -extern int CPUS; -extern int TEMP_FILES; - -static int additional_options[] = { -#define A_COALESCE ((int) 'c') - A_COALESCE, -#define A_REVERSE ((int) 'r') - A_REVERSE, -#define A_REORDER ((int) 'o') - A_REORDER, -#define A_LINE_DROP ((int) 'l') - A_LINE_DROP, -#define A_DEBUG_POLYGON ((int) 'd') - A_DEBUG_POLYGON, -#define A_POLYGON_DROP ((int) 'p') - A_POLYGON_DROP, -#define A_PREFER_RADIX_SORT ((int) 'R') - A_PREFER_RADIX_SORT, -}; - -static int prevent_options[] = { -#define P_SIMPLIFY ((int) 's') - P_SIMPLIFY, -#define P_SIMPLIFY_LOW ((int) 'S') - P_SIMPLIFY_LOW, -#define P_FEATURE_LIMIT ((int) 'f') - P_FEATURE_LIMIT, -#define P_KILOBYTE_LIMIT ((int) 'k') - P_KILOBYTE_LIMIT, -#define P_DYNAMIC_DROP ((int) 'd') - P_DYNAMIC_DROP, -#define P_INPUT_ORDER ((int) 'i') - P_INPUT_ORDER, -#define P_POLYGON_SPLIT ((int) 'p') - P_POLYGON_SPLIT, -#define P_CLIPPING ((int) 'c') - P_CLIPPING, -#define P_DUPLICATION ((int) 'D') - P_DUPLICATION, -}; From c4274303ea59e59097127bcc5e4a401dc32a7b86 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 15:10:26 -0700 Subject: [PATCH 15/19] Fix indentation --- geojson.hpp | 50 +++++++++++++++++++++++++------------------------- main.hpp | 10 +++++----- pool.hpp | 6 +++--- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/geojson.hpp b/geojson.hpp index 925145b..f96d6e3 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -1,29 +1,29 @@ struct parse_json_args { - json_pull *jp; - const char *reading; - volatile long long *layer_seq; - volatile long long *progress_seq; - long long *metapos; - long long *geompos; - long long *indexpos; - struct pool *exclude; - struct pool *include; - int exclude_all; - FILE *metafile; - FILE *geomfile; - FILE *indexfile; - struct memfile *poolfile; - struct memfile *treefile; - char *fname; - int basezoom; - int layer; - double droprate; - long long *file_bbox; - int segment; - int *initialized; - unsigned *initial_x; - unsigned *initial_y; - struct reader *readers; + json_pull *jp; + const char *reading; + volatile long long *layer_seq; + volatile long long *progress_seq; + long long *metapos; + long long *geompos; + long long *indexpos; + struct pool *exclude; + struct pool *include; + int exclude_all; + FILE *metafile; + FILE *geomfile; + FILE *indexfile; + struct memfile *poolfile; + struct memfile *treefile; + char *fname; + int basezoom; + int layer; + double droprate; + long long *file_bbox; + int segment; + int *initialized; + unsigned *initial_x; + unsigned *initial_y; + struct reader *readers; }; struct json_pull *json_begin_map(char *map, long long len); diff --git a/main.hpp b/main.hpp index ff0338d..e842511 100644 --- a/main.hpp +++ b/main.hpp @@ -1,9 +1,9 @@ struct index { - long long start; - long long end; - unsigned long long index; - short segment; - unsigned long long seq : (64 - 16); // pack with segment to stay in 32 bytes + long long start; + long long end; + unsigned long long index; + short segment; + unsigned long long seq : (64 - 16); // pack with segment to stay in 32 bytes }; void checkdisk(struct reader *r, int nreader); diff --git a/pool.hpp b/pool.hpp index 378aadf..c064524 100644 --- a/pool.hpp +++ b/pool.hpp @@ -24,9 +24,9 @@ void pool_init(struct pool *p, int n); int is_pooled(struct pool *p, const char *s, int type); struct stringpool { - long long left; - long long right; - long long off; + long long left; + long long right; + long long off; }; long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type); From 24db559f0bf05c2a4e61f9f1fdb7ee0393a0d6b5 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 15:12:03 -0700 Subject: [PATCH 16/19] Fix complaint about duplicate #define --- mbtiles.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mbtiles.cpp b/mbtiles.cpp index 697630c..8015334 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -1,5 +1,7 @@ // for vasprintf() on Linux +#ifndef _GNU_SOURCE #define _GNU_SOURCE +#endif #include #include From 48a82039ce08f1572d8c33d02faabdff68599729 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 15:14:09 -0700 Subject: [PATCH 17/19] Oops, I didn't check main.cpp in. --- main.cpp | 2152 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2152 insertions(+) create mode 100644 main.cpp diff --git a/main.cpp b/main.cpp new file mode 100644 index 0000000..eb27e83 --- /dev/null +++ b/main.cpp @@ -0,0 +1,2152 @@ +#ifdef MTRACE +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#include +#include +#else +#include +#endif + +extern "C" { +#include "jsonpull.h" +} + +#include "tile.hpp" +#include "pool.hpp" +#include "mbtiles.hpp" +#include "projection.hpp" +#include "version.hpp" +#include "memfile.hpp" +#include "serial.hpp" +#include "main.hpp" +#include "geojson.hpp" +#include "geometry.hpp" +#include "options.hpp" + +static int low_detail = 12; +static int full_detail = -1; +static int min_detail = 7; + +int quiet = 0; +int geometry_scale = 0; + +static int prevent[256]; +static int additional[256]; + +struct source { + char *layer; + char *file; + struct source *next; +}; + +struct tofree { + void *p; + struct tofree *next; +} *tofree = NULL; + +void mustfree(void *p) { + struct tofree *f = (struct tofree *) malloc(sizeof(struct tofree)); + if (f == NULL) { + perror("malloc"); + exit(EXIT_FAILURE); + } + f->p = p; + f->next = tofree; + tofree = f; +} + +int CPUS; +int TEMP_FILES; +long long MAX_FILES; +static long long diskfree; + +#define MAX_ZOOM 24 + +struct reader { + int metafd; + int poolfd; + int treefd; + int geomfd; + int indexfd; + + FILE *metafile; + struct memfile *poolfile; + struct memfile *treefile; + FILE *geomfile; + FILE *indexfile; + + long long metapos; + long long geompos; + long long indexpos; + + long long file_bbox[4]; + + struct stat geomst; + struct stat metast; + + char *geom_map; +}; + +void checkdisk(struct reader *r, int nreader) { + long long used = 0; + int i; + for (i = 0; i < nreader; i++) { + // Meta, pool, and tree are used once. + // Geometry and index will be duplicated during sorting and tiling. + used += r[i].metapos + 2 * r[i].geompos + 2 * r[i].indexpos + r[i].poolfile->len + r[i].treefile->len; + } + + static int warned = 0; + if (used > diskfree * .9 && !warned) { + fprintf(stderr, "You will probably run out of disk space.\n%lld bytes used or committed, of %lld originally available\n", used, diskfree); + warned = 1; + } +}; + +void init_cpus() { + CPUS = sysconf(_SC_NPROCESSORS_ONLN); + if (CPUS < 1) { + CPUS = 1; + } + + // Guard against short struct index.segment + if (CPUS > 32767) { + CPUS = 32767; + } + + // Round down to a power of 2 + CPUS = 1 << (int) (log(CPUS) / log(2)); + + struct rlimit rl; + if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { + perror("getrlimit"); + exit(EXIT_FAILURE); + } else { + MAX_FILES = rl.rlim_cur; + } + + // Don't really want too many temporary files, because the file system + // will start to bog down eventually + if (MAX_FILES > 2000) { + MAX_FILES = 2000; + } + + // MacOS can run out of system file descriptors + // even if we stay under the rlimit, so try to + // find out the real limit. + long long fds[MAX_FILES]; + long long i; + for (i = 0; i < MAX_FILES; i++) { + fds[i] = open("/dev/null", O_RDONLY); + if (fds[i] < 0) { + break; + } + } + long long j; + for (j = 0; j < i; j++) { + if (close(fds[j]) < 0) { + perror("close"); + exit(EXIT_FAILURE); + } + } + + // Scale down because we really don't want to run the system out of files + MAX_FILES = i * 3 / 4; + if (MAX_FILES < 32) { + fprintf(stderr, "Can't open a useful number of files: %lld\n", MAX_FILES); + exit(EXIT_FAILURE); + } + + TEMP_FILES = (MAX_FILES - 10) / 2; + if (TEMP_FILES > CPUS * 4) { + TEMP_FILES = CPUS * 4; + } +} + +int indexcmp(const void *v1, const void *v2) { + const struct index *i1 = (const struct index *) v1; + const struct index *i2 = (const struct index *) v2; + + if (i1->index < i2->index) { + return -1; + } else if (i1->index > i2->index) { + return 1; + } + + if (i1->seq < i2->seq) { + return -1; + } else if (i1->seq > i2->seq) { + return 1; + } + + return 0; +} + +struct merge { + long long start; + long long end; + + struct merge *next; +}; + +static void insert(struct merge *m, struct merge **head, unsigned char *map) { + while (*head != NULL && indexcmp(map + m->start, map + (*head)->start) > 0) { + head = &((*head)->next); + } + + m->next = *head; + *head = m; +} + +static void merge(struct merge *merges, int nmerges, unsigned char *map, FILE *f, int bytes, long long nrec, char *geom_map, FILE *geom_out, long long *geompos, long long *progress, long long *progress_max, long long *progress_reported) { + int i; + struct merge *head = NULL; + + for (i = 0; i < nmerges; i++) { + if (merges[i].start < merges[i].end) { + insert(&(merges[i]), &head, map); + } + } + + while (head != NULL) { + struct index *ix = (struct index *) (map + head->start); + fwrite_check(geom_map + ix->start, 1, ix->end - ix->start, geom_out, "merge geometry"); + *geompos += ix->end - ix->start; + + // Count this as an 75%-accomplishment, since we already 25%-counted it + *progress += (ix->end - ix->start) * 3 / 4; + if (!quiet && 100 * *progress / *progress_max != *progress_reported) { + fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); + *progress_reported = 100 * *progress / *progress_max; + } + + fwrite_check(map + head->start, bytes, 1, f, "merge temporary"); + head->start += bytes; + + struct merge *m = head; + head = m->next; + m->next = NULL; + + if (m->start < m->end) { + insert(m, &head, map); + } + } +} + +struct sort_arg { + int task; + int cpus; + long long indexpos; + struct merge *merges; + int indexfd; + int nmerges; + long long unit; + int bytes; +}; + +void *run_sort(void *v) { + struct sort_arg *a = (struct sort_arg *) v; + + long long start; + for (start = a->task * a->unit; start < a->indexpos; start += a->unit * a->cpus) { + long long end = start + a->unit; + if (end > a->indexpos) { + end = a->indexpos; + } + + a->merges[start / a->unit].start = start; + a->merges[start / a->unit].end = end; + a->merges[start / a->unit].next = NULL; + + // MAP_PRIVATE to avoid disk writes if it fits in memory + void *map = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_PRIVATE, a->indexfd, start); + if (map == MAP_FAILED) { + perror("mmap in run_sort"); + exit(EXIT_FAILURE); + } + madvise(map, end - start, MADV_RANDOM); + madvise(map, end - start, MADV_WILLNEED); + + qsort(map, (end - start) / a->bytes, a->bytes, indexcmp); + + // Sorting and then copying avoids disk access to + // write out intermediate stages of the sort. + + void *map2 = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_SHARED, a->indexfd, start); + if (map2 == MAP_FAILED) { + perror("mmap (write)"); + exit(EXIT_FAILURE); + } + madvise(map2, end - start, MADV_SEQUENTIAL); + + memcpy(map2, map, end - start); + + // No madvise, since caller will want the sorted data + munmap(map, end - start); + munmap(map2, end - start); + } + + return NULL; +} + +void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, struct pool *exclude, struct pool *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y) { + long long segs[CPUS + 1]; + segs[0] = 0; + segs[CPUS] = len; + + int i; + for (i = 1; i < CPUS; i++) { + segs[i] = len * i / CPUS; + + while (segs[i] < len && map[segs[i]] != '\n') { + segs[i]++; + } + } + + volatile long long layer_seq[CPUS]; + for (i = 0; i < CPUS; i++) { + // To preserve feature ordering, unique id for each segment + // begins with that segment's offset into the input + layer_seq[i] = segs[i] + initial_offset; + } + + struct parse_json_args pja[CPUS]; + pthread_t pthreads[CPUS]; + + for (i = 0; i < CPUS; i++) { + pja[i].jp = json_begin_map(map + segs[i], segs[i + 1] - segs[i]); + pja[i].reading = reading; + pja[i].layer_seq = &layer_seq[i]; + pja[i].progress_seq = progress_seq; + pja[i].metapos = &reader[i].metapos; + pja[i].geompos = &reader[i].geompos; + pja[i].indexpos = &reader[i].indexpos; + pja[i].exclude = exclude; + pja[i].include = include; + pja[i].exclude_all = exclude_all; + pja[i].metafile = reader[i].metafile; + pja[i].geomfile = reader[i].geomfile; + pja[i].indexfile = reader[i].indexfile; + pja[i].poolfile = reader[i].poolfile; + pja[i].treefile = reader[i].treefile; + pja[i].fname = fname; + pja[i].basezoom = basezoom; + pja[i].layer = source < nlayers ? source : 0; + pja[i].droprate = droprate; + pja[i].file_bbox = reader[i].file_bbox; + pja[i].segment = i; + pja[i].initialized = &initialized[i]; + pja[i].initial_x = &initial_x[i]; + pja[i].initial_y = &initial_y[i]; + pja[i].readers = reader; + + if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) { + perror("pthread_create"); + exit(EXIT_FAILURE); + } + } + + for (i = 0; i < CPUS; i++) { + void *retval; + + if (pthread_join(pthreads[i], &retval) != 0) { + perror("pthread_join"); + } + + free(pja[i].jp->source); + json_end(pja[i].jp); + } +} + +struct read_parallel_arg { + int fd; + FILE *fp; + long long offset; + long long len; + volatile int *is_parsing; + + const char *reading; + struct reader *reader; + volatile long long *progress_seq; + struct pool *exclude; + struct pool *include; + int exclude_all; + char *fname; + int maxzoom; + int basezoom; + int source; + int nlayers; + double droprate; + int *initialized; + unsigned *initial_x; + unsigned *initial_y; +}; + +void *run_read_parallel(void *v) { + struct read_parallel_arg *a = (struct read_parallel_arg *) v; + + struct stat st; + if (fstat(a->fd, &st) != 0) { + perror("stat read temp"); + } + if (a->len != st.st_size) { + fprintf(stderr, "wrong number of bytes in temporary: %lld vs %lld\n", a->len, (long long) st.st_size); + } + a->len = st.st_size; + + char *map = (char *) mmap(NULL, a->len, PROT_READ, MAP_PRIVATE, a->fd, 0); + if (map == NULL || map == MAP_FAILED) { + perror("map intermediate input"); + exit(EXIT_FAILURE); + } + madvise(map, a->len, MADV_RANDOM); // sequential, but from several pointers at once + + do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->droprate, a->initialized, a->initial_x, a->initial_y); + + madvise(map, a->len, MADV_DONTNEED); + if (munmap(map, a->len) != 0) { + perror("munmap source file"); + } + if (fclose(a->fp) != 0) { + perror("close source file"); + exit(EXIT_FAILURE); + } + + *(a->is_parsing) = 0; + free(a); + + return NULL; +} + +void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, const char *reading, struct reader *reader, volatile long long *progress_seq, struct pool *exclude, struct pool *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y) { + // This has to kick off an intermediate thread to start the parser threads, + // so the main thread can get back to reading the next input stage while + // the intermediate thread waits for the completion of the parser threads. + + *is_parsing = 1; + + struct read_parallel_arg *rpa = (struct read_parallel_arg *) malloc(sizeof(struct read_parallel_arg)); + if (rpa == NULL) { + perror("Out of memory"); + exit(EXIT_FAILURE); + } + + rpa->fd = fd; + rpa->fp = fp; + rpa->offset = offset; + rpa->len = len; + rpa->is_parsing = is_parsing; + + rpa->reading = reading; + rpa->reader = reader; + rpa->progress_seq = progress_seq; + rpa->exclude = exclude; + rpa->include = include; + rpa->exclude_all = exclude_all; + rpa->fname = fname; + rpa->basezoom = basezoom; + rpa->source = source; + rpa->nlayers = nlayers; + rpa->droprate = droprate; + rpa->initialized = initialized; + rpa->initial_x = initial_x; + rpa->initial_y = initial_y; + + if (pthread_create(parallel_parser, NULL, run_read_parallel, rpa) != 0) { + perror("pthread_create"); + exit(EXIT_FAILURE); + } +} + +void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int splits, long long mem, const char *tmpdir, long long *availfiles, FILE *geomfile, FILE *indexfile, long long *geompos_out, long long *progress, long long *progress_max, long long *progress_reported) { + // Arranged as bits to facilitate subdividing again if a subdivided file is still huge + int splitbits = log(splits) / log(2); + splits = 1 << splitbits; + + FILE *geomfiles[splits]; + FILE *indexfiles[splits]; + int geomfds[splits]; + int indexfds[splits]; + long long sub_geompos[splits]; + + int i; + for (i = 0; i < splits; i++) { + sub_geompos[i] = 0; + + char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; + sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX"); + char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; + sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX"); + + geomfds[i] = mkstemp(geomname); + if (geomfds[i] < 0) { + perror(geomname); + exit(EXIT_FAILURE); + } + indexfds[i] = mkstemp(indexname); + if (indexfds[i] < 0) { + perror(indexname); + exit(EXIT_FAILURE); + } + + geomfiles[i] = fopen(geomname, "wb"); + if (geomfiles[i] == NULL) { + perror(geomname); + exit(EXIT_FAILURE); + } + indexfiles[i] = fopen(indexname, "wb"); + if (indexfiles[i] == NULL) { + perror(indexname); + exit(EXIT_FAILURE); + } + + *availfiles -= 4; + + unlink(geomname); + unlink(indexname); + } + + for (i = 0; i < inputs; i++) { + struct stat geomst, indexst; + if (fstat(geomfds_in[i], &geomst) < 0) { + perror("stat geom"); + exit(EXIT_FAILURE); + } + if (fstat(indexfds_in[i], &indexst) < 0) { + perror("stat index"); + exit(EXIT_FAILURE); + } + + if (indexst.st_size != 0) { + struct index *indexmap = (struct index *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds_in[i], 0); + if (indexmap == MAP_FAILED) { + fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds_in[i], (long long) indexst.st_size); + perror("map index"); + exit(EXIT_FAILURE); + } + madvise(indexmap, indexst.st_size, MADV_SEQUENTIAL); + madvise(indexmap, indexst.st_size, MADV_WILLNEED); + char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds_in[i], 0); + if (geommap == MAP_FAILED) { + perror("map geom"); + exit(EXIT_FAILURE); + } + madvise(geommap, geomst.st_size, MADV_SEQUENTIAL); + madvise(geommap, geomst.st_size, MADV_WILLNEED); + + long long a; + for (a = 0; a < indexst.st_size / sizeof(struct index); a++) { + struct index ix = indexmap[a]; + unsigned long long which = (ix.index << prefix) >> (64 - splitbits); + long long pos = sub_geompos[which]; + + fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], "geom"); + sub_geompos[which] += ix.end - ix.start; + + // Count this as a 25%-accomplishment, since we will copy again + *progress += (ix.end - ix.start) / 4; + if (!quiet && 100 * *progress / *progress_max != *progress_reported) { + fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); + *progress_reported = 100 * *progress / *progress_max; + } + + ix.start = pos; + ix.end = sub_geompos[which]; + + fwrite_check(&ix, sizeof(struct index), 1, indexfiles[which], "index"); + } + + madvise(indexmap, indexst.st_size, MADV_DONTNEED); + if (munmap(indexmap, indexst.st_size) < 0) { + perror("unmap index"); + exit(EXIT_FAILURE); + } + madvise(geommap, geomst.st_size, MADV_DONTNEED); + if (munmap(geommap, geomst.st_size) < 0) { + perror("unmap geom"); + exit(EXIT_FAILURE); + } + } + + if (close(geomfds_in[i]) < 0) { + perror("close geom"); + exit(EXIT_FAILURE); + } + if (close(indexfds_in[i]) < 0) { + perror("close index"); + exit(EXIT_FAILURE); + } + + *availfiles += 2; + } + + for (i = 0; i < splits; i++) { + if (fclose(geomfiles[i]) != 0) { + perror("fclose geom"); + exit(EXIT_FAILURE); + } + if (fclose(indexfiles[i]) != 0) { + perror("fclose index"); + exit(EXIT_FAILURE); + } + + *availfiles += 2; + } + + for (i = 0; i < splits; i++) { + int already_closed = 0; + + struct stat geomst, indexst; + if (fstat(geomfds[i], &geomst) < 0) { + perror("stat geom"); + exit(EXIT_FAILURE); + } + if (fstat(indexfds[i], &indexst) < 0) { + perror("stat index"); + exit(EXIT_FAILURE); + } + + if (indexst.st_size > 0) { + if (indexst.st_size > sizeof(struct index) && indexst.st_size + geomst.st_size < mem) { + long long indexpos = indexst.st_size; + int bytes = sizeof(struct index); + + int page = sysconf(_SC_PAGESIZE); + // Don't try to sort more than 2GB at once, + // which used to crash Macs and may still + long long max_unit = 2LL * 1024 * 1024 * 1024; + long long unit = ((indexpos / CPUS + bytes - 1) / bytes) * bytes; + if (unit > max_unit) { + unit = max_unit; + } + unit = ((unit + page - 1) / page) * page; + + int nmerges = (indexpos + unit - 1) / unit; + struct merge merges[nmerges]; + + int a; + for (a = 0; a < nmerges; a++) { + merges[a].start = merges[a].end = 0; + } + + pthread_t pthreads[CPUS]; + struct sort_arg args[CPUS]; + + for (a = 0; a < CPUS; a++) { + args[a].task = a; + args[a].cpus = CPUS; + args[a].indexpos = indexpos; + args[a].merges = merges; + args[a].indexfd = indexfds[i]; + args[a].nmerges = nmerges; + args[a].unit = unit; + args[a].bytes = bytes; + + if (pthread_create(&pthreads[a], NULL, run_sort, &args[a]) != 0) { + perror("pthread_create"); + exit(EXIT_FAILURE); + } + } + + for (a = 0; a < CPUS; a++) { + void *retval; + + if (pthread_join(pthreads[a], &retval) != 0) { + perror("pthread_join"); + } + } + + struct indexmap *indexmap = (struct indexmap *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0); + if (indexmap == MAP_FAILED) { + fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds[i], (long long) indexst.st_size); + perror("map index"); + exit(EXIT_FAILURE); + } + madvise(indexmap, indexst.st_size, MADV_RANDOM); // sequential, but from several pointers at once + madvise(indexmap, indexst.st_size, MADV_WILLNEED); + char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0); + if (geommap == MAP_FAILED) { + perror("map geom"); + exit(EXIT_FAILURE); + } + madvise(geommap, geomst.st_size, MADV_RANDOM); + madvise(geommap, geomst.st_size, MADV_WILLNEED); + + merge(merges, nmerges, (unsigned char *) indexmap, indexfile, bytes, indexpos / bytes, geommap, geomfile, geompos_out, progress, progress_max, progress_reported); + + madvise(indexmap, indexst.st_size, MADV_DONTNEED); + if (munmap(indexmap, indexst.st_size) < 0) { + perror("unmap index"); + exit(EXIT_FAILURE); + } + madvise(geommap, geomst.st_size, MADV_DONTNEED); + if (munmap(geommap, geomst.st_size) < 0) { + perror("unmap geom"); + exit(EXIT_FAILURE); + } + } else if (indexst.st_size == sizeof(struct index) || prefix + splitbits >= 64) { + struct index *indexmap = (struct index *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0); + if (indexmap == MAP_FAILED) { + fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds[i], (long long) indexst.st_size); + perror("map index"); + exit(EXIT_FAILURE); + } + madvise(indexmap, indexst.st_size, MADV_SEQUENTIAL); + madvise(indexmap, indexst.st_size, MADV_WILLNEED); + char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0); + if (geommap == MAP_FAILED) { + perror("map geom"); + exit(EXIT_FAILURE); + } + madvise(geommap, geomst.st_size, MADV_RANDOM); + madvise(geommap, geomst.st_size, MADV_WILLNEED); + + long long a; + for (a = 0; a < indexst.st_size / sizeof(struct index); a++) { + struct index ix = indexmap[a]; + long long pos = *geompos_out; + + fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfile, "geom"); + *geompos_out += ix.end - ix.start; + + // Count this as an 75%-accomplishment, since we already 25%-counted it + *progress += (ix.end - ix.start) * 3 / 4; + if (!quiet && 100 * *progress / *progress_max != *progress_reported) { + fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); + *progress_reported = 100 * *progress / *progress_max; + } + + ix.start = pos; + ix.end = *geompos_out; + fwrite_check(&ix, sizeof(struct index), 1, indexfile, "index"); + } + + madvise(indexmap, indexst.st_size, MADV_DONTNEED); + if (munmap(indexmap, indexst.st_size) < 0) { + perror("unmap index"); + exit(EXIT_FAILURE); + } + madvise(geommap, geomst.st_size, MADV_DONTNEED); + if (munmap(geommap, geomst.st_size) < 0) { + perror("unmap geom"); + exit(EXIT_FAILURE); + } + } else { + // We already reported the progress from splitting this radix out + // but we need to split it again, which will be credited with more + // progress. So increase the total amount of progress to report by + // the additional progress that will happpen, which may move the + // counter backward but will be an honest estimate of the work remaining. + *progress_max += geomst.st_size / 4; + + radix1(&geomfds[i], &indexfds[i], 1, prefix + splitbits, *availfiles / 4, mem, tmpdir, availfiles, geomfile, indexfile, geompos_out, progress, progress_max, progress_reported); + already_closed = 1; + } + } + + if (!already_closed) { + if (close(geomfds[i]) < 0) { + perror("close geom"); + exit(EXIT_FAILURE); + } + if (close(indexfds[i]) < 0) { + perror("close index"); + exit(EXIT_FAILURE); + } + + *availfiles += 2; + } + } +} + +void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE *indexfile, int indexfd, const char *tmpdir, long long *geompos) { + // Run through the index and geometry for each reader, + // splitting the contents out by index into as many + // sub-files as we can write to simultaneously. + + // Then sort each of those by index, recursively if it is + // too big to fit in memory. + + // Then concatenate each of the sub-outputs into a final output. + + long long mem; + +#ifdef __APPLE__ + int64_t hw_memsize; + size_t len = sizeof(int64_t); + if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) { + perror("sysctl hw.memsize"); + exit(EXIT_FAILURE); + } + mem = hw_memsize; +#else + long long pagesize = sysconf(_SC_PAGESIZE); + long long pages = sysconf(_SC_PHYS_PAGES); + if (pages < 0 || pagesize < 0) { + perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES"); + exit(EXIT_FAILURE); + } + + mem = (long long) pages * pagesize; +#endif + + // Just for code coverage testing. Deeply recursive sorting is very slow + // compared to sorting in memory. + if (additional[A_PREFER_RADIX_SORT]) { + mem = 8192; + } + + long long availfiles = MAX_FILES - 2 * nreaders // each reader has a geom and an index + - 4 // pool, meta, mbtiles, mbtiles journal + - 4 // top-level geom and index output, both FILE and fd + - 3; // stdin, stdout, stderr + + // 4 because for each we have output and input FILE and fd for geom and index + int splits = availfiles / 4; + + // Be somewhat conservative about memory availability because the whole point of this + // is to keep from thrashing by working on chunks that will fit in memory. + mem /= 2; + + long long geom_total = 0; + int geomfds[nreaders]; + int indexfds[nreaders]; + int i; + for (i = 0; i < nreaders; i++) { + geomfds[i] = reader[i].geomfd; + indexfds[i] = reader[i].indexfd; + + struct stat geomst; + if (fstat(reader[i].geomfd, &geomst) < 0) { + perror("stat geom"); + exit(EXIT_FAILURE); + } + geom_total += geomst.st_size; + } + + long long progress = 0, progress_max = geom_total, progress_reported = -1; + long long availfiles_before = availfiles; + radix1(geomfds, indexfds, nreaders, 0, splits, mem, tmpdir, &availfiles, geomfile, indexfile, geompos, &progress, &progress_max, &progress_reported); + + if (availfiles - 2 * nreaders != availfiles_before) { + fprintf(stderr, "Internal error: miscounted available file descriptors: %lld vs %lld\n", availfiles - 2 * nreaders, availfiles); + exit(EXIT_FAILURE); + } +} + +int read_input(int argc, struct source **sourcelist, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int *prevent, int *additional, int read_parallel, int forcetable, const char *attribution) { + int ret = EXIT_SUCCESS; + + struct reader reader[CPUS]; + int i; + for (i = 0; i < CPUS; i++) { + struct reader *r = reader + i; + + char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; + char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; + char treename[strlen(tmpdir) + strlen("/tree.XXXXXXXX") + 1]; + char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; + char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; + + sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX"); + sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX"); + sprintf(treename, "%s%s", tmpdir, "/tree.XXXXXXXX"); + sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX"); + sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX"); + + r->metafd = mkstemp(metaname); + if (r->metafd < 0) { + perror(metaname); + exit(EXIT_FAILURE); + } + r->poolfd = mkstemp(poolname); + if (r->poolfd < 0) { + perror(poolname); + exit(EXIT_FAILURE); + } + r->treefd = mkstemp(treename); + if (r->treefd < 0) { + perror(treename); + exit(EXIT_FAILURE); + } + r->geomfd = mkstemp(geomname); + if (r->geomfd < 0) { + perror(geomname); + exit(EXIT_FAILURE); + } + r->indexfd = mkstemp(indexname); + if (r->indexfd < 0) { + perror(indexname); + exit(EXIT_FAILURE); + } + + r->metafile = fopen(metaname, "wb"); + if (r->metafile == NULL) { + perror(metaname); + exit(EXIT_FAILURE); + } + r->poolfile = memfile_open(r->poolfd); + if (r->poolfile == NULL) { + perror(poolname); + exit(EXIT_FAILURE); + } + r->treefile = memfile_open(r->treefd); + if (r->treefile == NULL) { + perror(treename); + exit(EXIT_FAILURE); + } + r->geomfile = fopen(geomname, "wb"); + if (r->geomfile == NULL) { + perror(geomname); + exit(EXIT_FAILURE); + } + r->indexfile = fopen(indexname, "wb"); + if (r->indexfile == NULL) { + perror(indexname); + exit(EXIT_FAILURE); + } + r->metapos = 0; + r->geompos = 0; + r->indexpos = 0; + + unlink(metaname); + unlink(poolname); + unlink(treename); + unlink(geomname); + unlink(indexname); + + // To distinguish a null value + { + struct stringpool p; + memfile_write(r->treefile, &p, sizeof(struct stringpool)); + } + // Keep metadata file from being completely empty if no attributes + serialize_int(r->metafile, 0, &r->metapos, "meta"); + + r->file_bbox[0] = r->file_bbox[1] = UINT_MAX; + r->file_bbox[2] = r->file_bbox[3] = 0; + } + + struct statfs fsstat; + if (fstatfs(reader[0].geomfd, &fsstat) != 0) { + perror("fstatfs"); + exit(EXIT_FAILURE); + } + diskfree = (long long) fsstat.f_bsize * fsstat.f_bavail; + + volatile long long progress_seq = 0; + + int initialized[CPUS]; + unsigned initial_x[CPUS], initial_y[CPUS]; + for (i = 0; i < CPUS; i++) { + initialized[i] = initial_x[i] = initial_y[i] = 0; + } + + int nlayers; + if (layername != NULL) { + nlayers = 1; + } else { + nlayers = argc; + if (nlayers == 0) { + nlayers = 1; + } + } + + int nsources = argc; + if (nsources == 0) { + nsources = 1; + } + + long overall_offset = 0; + + int source; + for (source = 0; source < nsources; source++) { + const char *reading; + int fd; + + if (source >= argc) { + reading = "standard input"; + fd = 0; + } else { + reading = sourcelist[source]->file; + fd = open(sourcelist[source]->file, O_RDONLY); + if (fd < 0) { + perror(sourcelist[source]->file); + continue; + } + } + + struct stat st; + char *map = NULL; + off_t off = 0; + + if (read_parallel) { + if (fstat(fd, &st) == 0) { + off = lseek(fd, 0, SEEK_CUR); + if (off >= 0) { + map = (char *) mmap(NULL, st.st_size - off, PROT_READ, MAP_PRIVATE, fd, off); + // No error if MAP_FAILED because check is below + if (map != MAP_FAILED) { + madvise(map, st.st_size - off, MADV_RANDOM); // sequential, but from several pointers at once + } + } + } + } + + if (map != NULL && map != MAP_FAILED) { + do_read_parallel(map, st.st_size - off, overall_offset, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y); + overall_offset += st.st_size - off; + checkdisk(reader, CPUS); + + if (munmap(map, st.st_size - off) != 0) { + madvise(map, st.st_size, MADV_DONTNEED); + perror("munmap source file"); + } + } else { + FILE *fp = fdopen(fd, "r"); + if (fp == NULL) { + perror(sourcelist[source]->file); + if (close(fd) != 0) { + perror("close source file"); + exit(EXIT_FAILURE); + } + continue; + } + + if (read_parallel) { + // Serial reading of chunks that are then parsed in parallel + + char readname[strlen(tmpdir) + strlen("/read.XXXXXXXX") + 1]; + sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX"); + int readfd = mkstemp(readname); + if (readfd < 0) { + perror(readname); + exit(EXIT_FAILURE); + } + FILE *readfp = fdopen(readfd, "w"); + if (readfp == NULL) { + perror(readname); + exit(EXIT_FAILURE); + } + unlink(readname); + + volatile int is_parsing = 0; + long long ahead = 0; + long long initial_offset = overall_offset; + pthread_t parallel_parser; + +#define READ_BUF 2000 +#define PARSE_MIN 10000000 +#define PARSE_MAX (1LL * 1024 * 1024 * 1024) + + char buf[READ_BUF]; + int n; + + while ((n = fread(buf, sizeof(char), READ_BUF, fp)) > 0) { + fwrite_check(buf, sizeof(char), n, readfp, reading); + ahead += n; + + if (buf[n - 1] == '\n' && ahead > PARSE_MIN) { + // Don't let the streaming reader get too far ahead of the parsers. + // If the buffered input gets huge, even if the parsers are still running, + // wait for the parser thread instead of continuing to stream input. + + if (is_parsing == 0 || ahead >= PARSE_MAX) { + if (initial_offset != 0) { + if (pthread_join(parallel_parser, NULL) != 0) { + perror("pthread_join"); + exit(EXIT_FAILURE); + } + } + + fflush(readfp); + start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y); + + initial_offset += ahead; + overall_offset += ahead; + checkdisk(reader, CPUS); + ahead = 0; + + sprintf(readname, "%s%s", tmpdir, "/read.XXXXXXXX"); + readfd = mkstemp(readname); + if (readfd < 0) { + perror(readname); + exit(EXIT_FAILURE); + } + readfp = fdopen(readfd, "w"); + if (readfp == NULL) { + perror(readname); + exit(EXIT_FAILURE); + } + unlink(readname); + } + } + } + if (n < 0) { + perror(reading); + } + + if (initial_offset != 0) { + if (pthread_join(parallel_parser, NULL) != 0) { + perror("pthread_join"); + exit(EXIT_FAILURE); + } + } + + fflush(readfp); + + if (ahead > 0) { + start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading, reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y); + + if (pthread_join(parallel_parser, NULL) != 0) { + perror("pthread_join"); + } + + overall_offset += ahead; + checkdisk(reader, CPUS); + } + } else { + // Plain serial reading + + long long layer_seq = overall_offset; + json_pull *jp = json_begin_file(fp); + parse_json(jp, reading, &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader); + json_end(jp); + overall_offset = layer_seq; + checkdisk(reader, CPUS); + } + + if (fclose(fp) != 0) { + perror("fclose input"); + exit(EXIT_FAILURE); + } + } + } + + if (!quiet) { + fprintf(stderr, " \r"); + // (stderr, "Read 10000.00 million features\r", *progress_seq / 1000000.0); + } + + for (i = 0; i < CPUS; i++) { + if (fclose(reader[i].metafile) != 0) { + perror("fclose meta"); + exit(EXIT_FAILURE); + } + if (fclose(reader[i].geomfile) != 0) { + perror("fclose geom"); + exit(EXIT_FAILURE); + } + if (fclose(reader[i].indexfile) != 0) { + perror("fclose index"); + exit(EXIT_FAILURE); + } + memfile_close(reader[i].treefile); + + if (fstat(reader[i].geomfd, &reader[i].geomst) != 0) { + perror("stat geom\n"); + exit(EXIT_FAILURE); + } + if (fstat(reader[i].metafd, &reader[i].metast) != 0) { + perror("stat meta\n"); + exit(EXIT_FAILURE); + } + } + + struct pool file_keys1[nlayers]; + struct pool *file_keys[nlayers]; + for (i = 0; i < nlayers; i++) { + pool_init(&file_keys1[i], 0); + file_keys[i] = &file_keys1[i]; + } + + char *layernames[nlayers]; + for (i = 0; i < nlayers; i++) { + if (layername != NULL) { + layernames[i] = strdup(layername); + if (layernames[i] == NULL) { + perror("Out of memory"); + exit(EXIT_FAILURE); + } + } else { + char *src; + if (argc < 1) { + src = fname; + } else if (sourcelist[i]->layer != NULL) { + src = sourcelist[i]->layer; + } else { + src = sourcelist[i]->file; + } + + char *trunc = layernames[i] = (char *) malloc(strlen(src) + 1); + if (trunc == NULL) { + perror("Out of memory"); + exit(EXIT_FAILURE); + } + + const char *ocp, *use = src; + for (ocp = src; *ocp; ocp++) { + if (*ocp == '/' && ocp[1] != '\0') { + use = ocp + 1; + } + } + strcpy(trunc, use); + + char *cp = strstr(trunc, ".json"); + if (cp != NULL) { + *cp = '\0'; + } + cp = strstr(trunc, ".mbtiles"); + if (cp != NULL) { + *cp = '\0'; + } + + char *out = trunc; + for (cp = trunc; *cp; cp++) { + if (isalpha(*cp) || isdigit(*cp) || *cp == '_') { + *out++ = *cp; + } + } + *out = '\0'; + + if (!quiet) { + fprintf(stderr, "For layer %d, using name \"%s\"\n", i, trunc); + } + } + } + + // Create a combined string pool and a combined metadata file + // but keep track of the offsets into it since we still need + // segment+offset to find the data. + + long long pool_off[CPUS]; + long long meta_off[CPUS]; + + char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; + sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX"); + + int poolfd = mkstemp(poolname); + if (poolfd < 0) { + perror(poolname); + exit(EXIT_FAILURE); + } + + FILE *poolfile = fopen(poolname, "wb"); + if (poolfile == NULL) { + perror(poolname); + exit(EXIT_FAILURE); + } + + unlink(poolname); + + char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; + sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX"); + + int metafd = mkstemp(metaname); + if (metafd < 0) { + perror(metaname); + exit(EXIT_FAILURE); + } + + FILE *metafile = fopen(metaname, "wb"); + if (metafile == NULL) { + perror(metaname); + exit(EXIT_FAILURE); + } + + unlink(metaname); + + long long metapos = 0; + long long poolpos = 0; + + for (i = 0; i < CPUS; i++) { + if (reader[i].metapos > 0) { + void *map = mmap(NULL, reader[i].metapos, PROT_READ, MAP_PRIVATE, reader[i].metafd, 0); + if (map == MAP_FAILED) { + perror("mmap unmerged meta"); + exit(EXIT_FAILURE); + } + madvise(map, reader[i].metapos, MADV_SEQUENTIAL); + madvise(map, reader[i].metapos, MADV_WILLNEED); + if (fwrite(map, reader[i].metapos, 1, metafile) != 1) { + perror("Reunify meta"); + exit(EXIT_FAILURE); + } + madvise(map, reader[i].metapos, MADV_DONTNEED); + if (munmap(map, reader[i].metapos) != 0) { + perror("unmap unmerged meta"); + } + } + + meta_off[i] = metapos; + metapos += reader[i].metapos; + if (close(reader[i].metafd) != 0) { + perror("close unmerged meta"); + } + + if (reader[i].poolfile->off > 0) { + if (fwrite(reader[i].poolfile->map, reader[i].poolfile->off, 1, poolfile) != 1) { + perror("Reunify string pool"); + exit(EXIT_FAILURE); + } + } + + pool_off[i] = poolpos; + poolpos += reader[i].poolfile->off; + memfile_close(reader[i].poolfile); + } + + if (fclose(poolfile) != 0) { + perror("fclose pool"); + exit(EXIT_FAILURE); + } + if (fclose(metafile) != 0) { + perror("fclose meta"); + exit(EXIT_FAILURE); + } + + char *meta = (char *) mmap(NULL, metapos, PROT_READ, MAP_PRIVATE, metafd, 0); + if (meta == MAP_FAILED) { + perror("mmap meta"); + exit(EXIT_FAILURE); + } + madvise(meta, metapos, MADV_RANDOM); + + char *stringpool = NULL; + if (poolpos > 0) { // Will be 0 if -X was specified + stringpool = (char *) mmap(NULL, poolpos, PROT_READ, MAP_PRIVATE, poolfd, 0); + if (stringpool == MAP_FAILED) { + perror("mmap string pool"); + exit(EXIT_FAILURE); + } + madvise(stringpool, poolpos, MADV_RANDOM); + } + + char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; + sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX"); + + int indexfd = mkstemp(indexname); + if (indexfd < 0) { + perror(indexname); + exit(EXIT_FAILURE); + } + FILE *indexfile = fopen(indexname, "wb"); + if (indexfile == NULL) { + perror(indexname); + exit(EXIT_FAILURE); + } + + unlink(indexname); + + char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; + sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX"); + + int geomfd = mkstemp(geomname); + if (geomfd < 0) { + perror(geomname); + exit(EXIT_FAILURE); + } + FILE *geomfile = fopen(geomname, "wb"); + if (geomfile == NULL) { + perror(geomname); + exit(EXIT_FAILURE); + } + unlink(geomname); + + long long geompos = 0; + + /* initial tile is 0/0/0 */ + serialize_int(geomfile, 0, &geompos, fname); + serialize_uint(geomfile, 0, &geompos, fname); + serialize_uint(geomfile, 0, &geompos, fname); + + radix(reader, CPUS, geomfile, geomfd, indexfile, indexfd, tmpdir, &geompos); + + /* end of tile */ + serialize_byte(geomfile, -2, &geompos, fname); + + if (fclose(geomfile) != 0) { + perror("fclose geom"); + exit(EXIT_FAILURE); + } + if (fclose(indexfile) != 0) { + perror("fclose index"); + exit(EXIT_FAILURE); + } + + struct stat indexst; + if (fstat(indexfd, &indexst) < 0) { + perror("stat index"); + exit(EXIT_FAILURE); + } + long long indexpos = indexst.st_size; + progress_seq = indexpos / sizeof(struct index); + + if (!quiet) { + fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", progress_seq, geompos, metapos, poolpos); + } + + if (indexpos == 0) { + fprintf(stderr, "Did not read any valid geometries\n"); + exit(EXIT_FAILURE); + } + + if (basezoom < 0 || droprate < 0) { + struct index *map = (struct index *) mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0); + if (map == MAP_FAILED) { + perror("mmap index for basezoom"); + exit(EXIT_FAILURE); + } + madvise(map, indexpos, MADV_SEQUENTIAL); + madvise(map, indexpos, MADV_WILLNEED); + + struct tile { + unsigned x; + unsigned y; + long long count; + long long fullcount; + double gap; + unsigned long long previndex; + } tile[MAX_ZOOM + 1], max[MAX_ZOOM + 1]; + + { + int i; + for (i = 0; i <= MAX_ZOOM; i++) { + tile[i].x = tile[i].y = tile[i].count = tile[i].fullcount = tile[i].gap = tile[i].previndex = 0; + max[i].x = max[i].y = max[i].count = max[i].fullcount = 0; + } + } + + long long progress = -1; + + long long indices = indexpos / sizeof(struct index); + long long i; + for (i = 0; i < indices; i++) { + unsigned xx, yy; + decode(map[i].index, &xx, &yy); + + long long nprogress = 100 * i / indices; + if (nprogress != progress) { + progress = nprogress; + if (!quiet) { + fprintf(stderr, "Base zoom/drop rate: %lld%% \r", progress); + } + } + + int z; + for (z = 0; z <= MAX_ZOOM; z++) { + unsigned xxx = 0, yyy = 0; + if (z != 0) { + xxx = xx >> (32 - z); + yyy = yy >> (32 - z); + } + + double scale = (double) (1LL << (64 - 2 * (z + 8))); + + if (tile[z].x != xxx || tile[z].y != yyy) { + if (tile[z].count > max[z].count) { + max[z] = tile[z]; + } + + tile[z].x = xxx; + tile[z].y = yyy; + tile[z].count = 0; + tile[z].fullcount = 0; + tile[z].gap = 0; + tile[z].previndex = 0; + } + + tile[z].fullcount++; + + if (manage_gap(map[i].index, &tile[z].previndex, scale, gamma, &tile[z].gap)) { + continue; + } + + tile[z].count++; + } + } + + int z; + for (z = MAX_ZOOM; z >= 0; z--) { + if (tile[z].count > max[z].count) { + max[z] = tile[z]; + } + } + + int max_features = 50000 / (basezoom_marker_width * basezoom_marker_width); + + int obasezoom = basezoom; + if (basezoom < 0) { + basezoom = MAX_ZOOM; + + for (z = MAX_ZOOM; z >= 0; z--) { + if (max[z].count < max_features) { + basezoom = z; + } + + // printf("%d/%u/%u %lld\n", z, max[z].x, max[z].y, max[z].count); + } + + fprintf(stderr, "Choosing a base zoom of -B%d to keep %lld features in tile %d/%u/%u.\n", basezoom, max[basezoom].count, basezoom, max[basezoom].x, max[basezoom].y); + } + + if (obasezoom < 0 && basezoom > maxzoom) { + fprintf(stderr, "Couldn't find a suitable base zoom. Working from the other direction.\n"); + if (gamma == 0) { + fprintf(stderr, "You might want to try -g1 to limit near-duplicates.\n"); + } + + if (droprate < 0) { + if (maxzoom == 0) { + droprate = 2.5; + } else { + droprate = exp(log((long double) max[0].count / max[maxzoom].count) / (maxzoom)); + fprintf(stderr, "Choosing a drop rate of -r%f to get from %lld to %lld in %d zooms\n", droprate, max[maxzoom].count, max[0].count, maxzoom); + } + } + + basezoom = 0; + for (z = 0; z <= maxzoom; z++) { + double zoomdiff = log((long double) max[z].count / max_features) / log(droprate); + if (zoomdiff + z > basezoom) { + basezoom = ceil(zoomdiff + z); + } + } + + fprintf(stderr, "Choosing a base zoom of -B%d to keep %f features in tile %d/%u/%u.\n", basezoom, max[maxzoom].count * exp(log(droprate) * (maxzoom - basezoom)), maxzoom, max[maxzoom].x, max[maxzoom].y); + } else if (droprate < 0) { + droprate = 1; + + for (z = basezoom - 1; z >= 0; z--) { + double interval = exp(log(droprate) * (basezoom - z)); + + if (max[z].count / interval >= max_features) { + interval = (long double) max[z].count / max_features; + droprate = exp(log(interval) / (basezoom - z)); + interval = exp(log(droprate) * (basezoom - z)); + + fprintf(stderr, "Choosing a drop rate of -r%f to keep %f features in tile %d/%u/%u.\n", droprate, max[z].count / interval, z, max[z].x, max[z].y); + } + } + } + + if (gamma > 0) { + int effective = 0; + + for (z = 0; z < maxzoom; z++) { + if (max[z].count < max[z].fullcount) { + effective = z + 1; + } + } + + if (effective == 0) { + fprintf(stderr, "With gamma, effective base zoom is 0, so no effective drop rate\n"); + } else { + double interval_0 = exp(log(droprate) * (basezoom - 0)); + double interval_eff = exp(log(droprate) * (basezoom - effective)); + if (effective > basezoom) { + interval_eff = 1; + } + + double scaled_0 = max[0].count / interval_0; + double scaled_eff = max[effective].count / interval_eff; + + double rate_at_0 = scaled_0 / max[0].fullcount; + double rate_at_eff = scaled_eff / max[effective].fullcount; + + double eff_drop = exp(log(rate_at_eff / rate_at_0) / (effective - 0)); + + fprintf(stderr, "With gamma, effective base zoom of %d, effective drop rate of %f\n", effective, eff_drop); + } + } + + madvise(map, indexpos, MADV_DONTNEED); + munmap(map, indexpos); + } + + if (close(indexfd) != 0) { + perror("close sorted index"); + } + + /* Traverse and split the geometries for each zoom level */ + + struct stat geomst; + if (fstat(geomfd, &geomst) != 0) { + perror("stat sorted geom\n"); + exit(EXIT_FAILURE); + } + + int fd[TEMP_FILES]; + off_t size[TEMP_FILES]; + + fd[0] = geomfd; + size[0] = geomst.st_size; + + int j; + for (j = 1; j < TEMP_FILES; j++) { + fd[j] = -1; + size[j] = 0; + } + + unsigned midx = 0, midy = 0; + int written = traverse_zooms(fd, size, meta, stringpool, file_keys, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y); + + if (maxzoom != written) { + fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); + maxzoom = written; + ret = EXIT_FAILURE; + } + + madvise(meta, metapos, MADV_DONTNEED); + if (munmap(meta, metapos) != 0) { + perror("munmap meta"); + } + if (close(metafd) < 0) { + perror("close meta"); + } + + if (poolpos > 0) { + madvise((void *) pool, poolpos, MADV_DONTNEED); + if (munmap(stringpool, poolpos) != 0) { + perror("munmap stringpool"); + } + } + if (close(poolfd) < 0) { + perror("close pool"); + } + + double minlat = 0, minlon = 0, maxlat = 0, maxlon = 0, midlat = 0, midlon = 0; + + tile2latlon(midx, midy, maxzoom, &maxlat, &minlon); + tile2latlon(midx + 1, midy + 1, maxzoom, &minlat, &maxlon); + + midlat = (maxlat + minlat) / 2; + midlon = (maxlon + minlon) / 2; + + long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0}; + for (i = 0; i < CPUS; i++) { + if (reader[i].file_bbox[0] < file_bbox[0]) { + file_bbox[0] = reader[i].file_bbox[0]; + } + if (reader[i].file_bbox[1] < file_bbox[1]) { + file_bbox[1] = reader[i].file_bbox[1]; + } + if (reader[i].file_bbox[2] > file_bbox[2]) { + file_bbox[2] = reader[i].file_bbox[2]; + } + if (reader[i].file_bbox[3] > file_bbox[3]) { + file_bbox[3] = reader[i].file_bbox[3]; + } + } + + // If the bounding box extends off the plane on either side, + // a feature wrapped across the date line, so the width of the + // bounding box is the whole world. + if (file_bbox[0] < 0) { + file_bbox[0] = 0; + file_bbox[2] = (1LL << 32) - 1; + } + if (file_bbox[2] > (1LL << 32) - 1) { + file_bbox[0] = 0; + file_bbox[2] = (1LL << 32) - 1; + } + if (file_bbox[1] < 0) { + file_bbox[1] = 0; + } + if (file_bbox[3] > (1LL << 32) - 1) { + file_bbox[3] = (1LL << 32) - 1; + } + + tile2latlon(file_bbox[0], file_bbox[1], 32, &maxlat, &minlon); + tile2latlon(file_bbox[2], file_bbox[3], 32, &minlat, &maxlon); + + if (midlat < minlat) { + midlat = minlat; + } + if (midlat > maxlat) { + midlat = maxlat; + } + if (midlon < minlon) { + midlon = minlon; + } + if (midlon > maxlon) { + midlon = maxlon; + } + + mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable, attribution); + + for (i = 0; i < nlayers; i++) { + pool_free_strings(&file_keys1[i]); + free(layernames[i]); + } + + return ret; +} + +static int int_in(int v, int *a, int len) { + int i; + + for (i = 0; i < len; i++) { + if (a[i] == v) { + return 1; + } + } + + return 0; +} + +int main(int argc, char **argv) { +#ifdef MTRACE + mtrace(); +#endif + + init_cpus(); + + extern int optind; + extern char *optarg; + int i; + + char *name = NULL; + char *layer = NULL; + char *outdir = NULL; + int maxzoom = 14; + int minzoom = 0; + int basezoom = -1; + double basezoom_marker_width = 1; + int force = 0; + int forcetable = 0; + double droprate = 2.5; + double gamma = 0; + int buffer = 5; + const char *tmpdir = "/tmp"; + const char *attribution = NULL; + + int nsources = 0; + struct source *sources = NULL; + + struct pool exclude, include; + pool_init(&exclude, 0); + pool_init(&include, 0); + int exclude_all = 0; + int read_parallel = 0; + int files_open_at_start; + + for (i = 0; i < 256; i++) { + prevent[i] = 0; + additional[i] = 0; + } + + { + char dup[256]; + + memset(dup, 0, sizeof(dup)); + for (i = 0; i < sizeof(additional_options) / sizeof(additional_options[0]); i++) { + if (dup[additional_options[i]]) { + fprintf(stderr, "Internal error: reused -a%c\n", additional_options[i]); + exit(EXIT_FAILURE); + } + dup[additional_options[i]] = 1; + } + + memset(dup, 0, sizeof(dup)); + for (i = 0; i < sizeof(prevent_options) / sizeof(prevent_options[0]); i++) { + if (dup[prevent_options[i]]) { + fprintf(stderr, "Internal error: reused -p%c\n", prevent_options[i]); + exit(EXIT_FAILURE); + } + dup[prevent_options[i]] = 1; + } + } + + static struct option long_options[] = { + {"output", required_argument, 0, 'o'}, + + {"name", required_argument, 0, 'n'}, + {"layer", required_argument, 0, 'l'}, + {"attribution", required_argument, 0, 'A'}, + {"named-layer", required_argument, 0, 'L'}, + {"maximum-zoom", required_argument, 0, 'z'}, + {"minimum-zoom", required_argument, 0, 'Z'}, + {"base-zoom", required_argument, 0, 'B'}, + {"full-detail", required_argument, 0, 'd'}, + {"low-detail", required_argument, 0, 'D'}, + {"minimum-detail", required_argument, 0, 'm'}, + {"exclude", required_argument, 0, 'x'}, + {"include", required_argument, 0, 'y'}, + {"drop-rate", required_argument, 0, 'r'}, + {"buffer", required_argument, 0, 'b'}, + {"temporary-directory", required_argument, 0, 't'}, + {"gamma", required_argument, 0, 'g'}, + {"prevent", required_argument, 0, 'p'}, + {"additional", required_argument, 0, 'a'}, + + {"exclude-all", no_argument, 0, 'X'}, + {"force", no_argument, 0, 'f'}, + {"allow-existing", no_argument, 0, 'F'}, + {"quiet", no_argument, 0, 'q'}, + {"version", no_argument, 0, 'v'}, + {"read-parallel", no_argument, 0, 'P'}, + + {"coalesce", no_argument, &additional[A_COALESCE], 1}, + {"reverse", no_argument, &additional[A_REVERSE], 1}, + {"reorder", no_argument, &additional[A_REORDER], 1}, + {"drop-lines", no_argument, &additional[A_LINE_DROP], 1}, + {"check-polygons", no_argument, &additional[A_DEBUG_POLYGON], 1}, + {"drop-polygons", no_argument, &additional[A_POLYGON_DROP], 1}, + {"prefer-radix-sort", no_argument, &additional[A_PREFER_RADIX_SORT], 1}, + + {"no-line-simplification", no_argument, &prevent[P_SIMPLIFY], 1}, + {"simplify-only-low-zooms", no_argument, &prevent[P_SIMPLIFY_LOW], 1}, + {"no-feature-limit", no_argument, &prevent[P_FEATURE_LIMIT], 1}, + {"no-tile-size-limit", no_argument, &prevent[P_KILOBYTE_LIMIT], 1}, + {"force-feature-limit", no_argument, &prevent[P_DYNAMIC_DROP], 1}, + {"preseve-input-order", no_argument, &prevent[P_INPUT_ORDER], 1}, + {"no-polygon-splitting", no_argument, &prevent[P_POLYGON_SPLIT], 1}, + {"no-clipping", no_argument, &prevent[P_CLIPPING], 1}, + {"no-duplication", no_argument, &prevent[P_DUPLICATION], 1}, + + {0, 0, 0, 0}, + }; + + while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:", long_options, NULL)) != -1) { + switch (i) { + case 0: + break; + + case 'n': + name = optarg; + break; + + case 'l': + layer = optarg; + break; + + case 'A': + attribution = optarg; + break; + + case 'L': { + char *cp = strchr(optarg, ':'); + if (cp == NULL || cp == optarg) { + fprintf(stderr, "%s: -L requires layername:file\n", argv[0]); + exit(EXIT_FAILURE); + } + struct source *src = (struct source *) malloc(sizeof(struct source)); + if (src == NULL) { + perror("Out of memory"); + exit(EXIT_FAILURE); + } + + src->layer = strdup(optarg); + src->file = strdup(cp + 1); + if (src->layer == NULL || src->file == NULL) { + perror("Out of memory"); + exit(EXIT_FAILURE); + } + mustfree(src->layer); + mustfree(src->file); + src->layer[cp - optarg] = '\0'; + src->next = sources; + sources = src; + nsources++; + } break; + + case 'z': + maxzoom = atoi(optarg); + break; + + case 'Z': + minzoom = atoi(optarg); + break; + + case 'B': + if (strcmp(optarg, "g") == 0) { + basezoom = -2; + } else if (optarg[0] == 'g' || optarg[0] == 'f') { + basezoom = -2; + if (optarg[0] == 'g') { + basezoom_marker_width = atof(optarg + 1); + } else { + basezoom_marker_width = sqrt(50000 / atof(optarg + 1)); + } + if (basezoom_marker_width == 0 || atof(optarg + 1) == 0) { + fprintf(stderr, "%s: Must specify value >0 with -B%c\n", argv[0], optarg[0]); + exit(EXIT_FAILURE); + } + } else { + basezoom = atoi(optarg); + if (basezoom == 0 && strcmp(optarg, "0") != 0) { + fprintf(stderr, "%s: Couldn't understand -B%s\n", argv[0], optarg); + exit(EXIT_FAILURE); + } + } + break; + + case 'd': + full_detail = atoi(optarg); + break; + + case 'D': + low_detail = atoi(optarg); + break; + + case 'm': + min_detail = atoi(optarg); + break; + + case 'o': + outdir = optarg; + break; + + case 'x': + pool(&exclude, optarg, VT_STRING); + break; + + case 'y': + exclude_all = 1; + pool(&include, optarg, VT_STRING); + break; + + case 'X': + exclude_all = 1; + break; + + case 'r': + if (strcmp(optarg, "g") == 0) { + droprate = -2; + } else if (optarg[0] == 'g' || optarg[0] == 'f') { + droprate = -2; + if (optarg[0] == 'g') { + basezoom_marker_width = atof(optarg + 1); + } else { + basezoom_marker_width = sqrt(50000 / atof(optarg + 1)); + } + if (basezoom_marker_width == 0 || atof(optarg + 1) == 0) { + fprintf(stderr, "%s: Must specify value >0 with -r%c\n", argv[0], optarg[0]); + exit(EXIT_FAILURE); + } + } else { + droprate = atof(optarg); + } + break; + + case 'b': + buffer = atoi(optarg); + break; + + case 'f': + force = 1; + break; + + case 'F': + forcetable = 1; + break; + + case 't': + tmpdir = optarg; + if (tmpdir[0] != '/') { + fprintf(stderr, "Warning: temp directory %s doesn't begin with /\n", tmpdir); + } + break; + + case 'g': + gamma = atof(optarg); + break; + + case 'q': + quiet = 1; + break; + + case 'p': { + char *cp; + for (cp = optarg; *cp != '\0'; cp++) { + if (int_in(*cp, prevent_options, sizeof(prevent_options) / sizeof(prevent_options[0]))) { + prevent[*cp & 0xFF] = 1; + } else { + fprintf(stderr, "%s: Unknown option -p%c\n", argv[0], *cp); + exit(EXIT_FAILURE); + } + } + } break; + + case 'a': { + char *cp; + for (cp = optarg; *cp != '\0'; cp++) { + if (int_in(*cp, additional_options, sizeof(additional_options) / sizeof(additional_options[0]))) { + additional[*cp & 0xFF] = 1; + } else { + fprintf(stderr, "%s: Unknown option -a%c\n", argv[0], *cp); + exit(EXIT_FAILURE); + } + } + } break; + + case 'v': + fprintf(stderr, VERSION); + exit(EXIT_FAILURE); + + case 'P': + read_parallel = 1; + break; + + default: { + int width = 7 + strlen(argv[0]); + fprintf(stderr, "Usage: %s", argv[0]); + int i; + for (i = 0; long_options[i].name != NULL; i++) { + if (width + strlen(long_options[i].name) + 9 >= 80) { + fprintf(stderr, "\n "); + width = 8; + } + width += strlen(long_options[i].name) + 9; + if (strcmp(long_options[i].name, "output") == 0) { + fprintf(stderr, " --%s=output.mbtiles", long_options[i].name); + width += 9; + } else if (long_options[i].has_arg) { + fprintf(stderr, " [--%s=...]", long_options[i].name); + } else { + fprintf(stderr, " [--%s]", long_options[i].name); + } + } + if (width + 16 >= 80) { + fprintf(stderr, "\n "); + width = 8; + } + fprintf(stderr, " [file.json ...]"); + } + exit(EXIT_FAILURE); + } + } + + files_open_at_start = open("/dev/null", O_RDONLY); + close(files_open_at_start); + + if (full_detail <= 0) { + full_detail = 12; + } + + if (full_detail < min_detail || low_detail < min_detail) { + fprintf(stderr, "%s: Full detail and low detail must be at least minimum detail\n", argv[0]); + exit(EXIT_FAILURE); + } + + // Need two checks: one for geometry representation, the other for + // index traversal when guessing base zoom and drop rate + if (maxzoom > 32 - full_detail) { + maxzoom = 32 - full_detail; + fprintf(stderr, "Highest supported zoom with detail %d is %d\n", full_detail, maxzoom); + } + if (maxzoom > MAX_ZOOM) { + maxzoom = MAX_ZOOM; + fprintf(stderr, "Highest supported zoom is %d\n", maxzoom); + } + + if (minzoom > maxzoom) { + fprintf(stderr, "minimum zoom -Z cannot be greater than maxzoom -z\n"); + exit(EXIT_FAILURE); + } + + if (basezoom == -1) { + basezoom = maxzoom; + } + + geometry_scale = 32 - (full_detail + maxzoom); + if (geometry_scale < 0) { + geometry_scale = 0; + fprintf(stderr, "Full detail + maxzoom > 32, so you are asking for more detail than is available.\n"); + } + + if ((basezoom < 0 || droprate < 0) && (gamma < 0)) { + // Can't use randomized (as opposed to evenly distributed) dot dropping + // if rate and base aren't known during feature reading. + gamma = 0; + fprintf(stderr, "Forcing -g0 since -B or -r is not known\n"); + } + + if (outdir == NULL) { + fprintf(stderr, "%s: must specify -o out.mbtiles\n", argv[0]); + exit(EXIT_FAILURE); + } + + if (force) { + unlink(outdir); + } + + sqlite3 *outdb = mbtiles_open(outdir, argv, forcetable); + int ret = EXIT_SUCCESS; + + for (i = optind; i < argc; i++) { + struct source *src = (struct source *) malloc(sizeof(struct source)); + if (src == NULL) { + perror("Out of memory"); + exit(EXIT_FAILURE); + } + + src->layer = NULL; + src->file = argv[i]; + src->next = sources; + sources = src; + nsources++; + } + + struct source *sourcelist[nsources]; + i = nsources - 1; + for (; sources != NULL; sources = sources->next) { + sourcelist[i--] = sources; + } + + ret = read_input(nsources, sourcelist, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel, forcetable, attribution); + + mbtiles_close(outdb, argv); + +#ifdef MTRACE + muntrace(); +#endif + + i = open("/dev/null", O_RDONLY); + // i < files_open_at_start is not an error, because reading from a pipe closes stdin + if (i > files_open_at_start) { + fprintf(stderr, "Internal error: did not close all files: %d\n", i); + exit(EXIT_FAILURE); + } + + for (i = 0; i < nsources; i++) { + free(sourcelist[i]); + } + + pool_free(&exclude); + pool_free(&include); + + struct tofree *tf, *next; + for (tf = tofree; tf != NULL; tf = next) { + next = tf->next; + free(tf->p); + free(tf); + } + + return ret; +} From 2b393ad8e51bec058f46021dede679ba1a969f0c Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 15:33:30 -0700 Subject: [PATCH 18/19] Move jsonpull into a subdirectory like other included libraries --- CHANGELOG.md | 4 ++++ Makefile | 13 +++---------- geojson.cpp | 2 +- jsonpull.c => jsonpull/jsonpull.c | 0 jsonpull.h => jsonpull/jsonpull.h | 0 main.cpp | 2 +- version.hpp | 2 +- 7 files changed, 10 insertions(+), 13 deletions(-) rename jsonpull.c => jsonpull/jsonpull.c (100%) rename jsonpull.h => jsonpull/jsonpull.h (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5ce486..01663e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.11.0 + +* Convert C source files to C++ + ## 1.10.0 * Upgrade Clipper to fix potential crashes and improve polygon topology diff --git a/Makefile b/Makefile index 4e62b3e..90ecedb 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper.o mvt.o serial.o main.o +tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o mvt.o serial.o main.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o @@ -54,18 +54,11 @@ tippecanoe-decode: decode.o projection.o mvt.o tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -libjsonpull.a: jsonpull.o - $(AR) rc $@ $^ - ranlib $@ - %.o: %.c $(H) - $(CC) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CFLAGS) -c $< + $(CC) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CFLAGS) -c -o $@ $< %.o: %.cpp $(H) - $(CXX) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c $< - -clipper.o: clipper/clipper.cpp $(H) - $(CXX) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c $< + $(CXX) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c -o $@ $< clean: rm -f tippecanoe *.o diff --git a/geojson.cpp b/geojson.cpp index f8f0c75..1ae9805 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -22,7 +22,7 @@ #include extern "C" { -#include "jsonpull.h" +#include "jsonpull/jsonpull.h" } #include "pool.hpp" diff --git a/jsonpull.c b/jsonpull/jsonpull.c similarity index 100% rename from jsonpull.c rename to jsonpull/jsonpull.c diff --git a/jsonpull.h b/jsonpull/jsonpull.h similarity index 100% rename from jsonpull.h rename to jsonpull/jsonpull.h diff --git a/main.cpp b/main.cpp index eb27e83..27b55e8 100644 --- a/main.cpp +++ b/main.cpp @@ -32,7 +32,7 @@ #endif extern "C" { -#include "jsonpull.h" +#include "jsonpull/jsonpull.h" } #include "tile.hpp" diff --git a/version.hpp b/version.hpp index c3cfbb4..3539096 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.10.0\n" +#define VERSION "tippecanoe v1.11.0\n" From 744915025d3dad538527beabb9c2238524f7b97a Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 27 Apr 2016 15:41:40 -0700 Subject: [PATCH 19/19] Restore dependency on headers in subdirectories --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 90ecedb..d6033e6 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,7 @@ man/tippecanoe.1: README.md PG= +ALL_H = $(shell find . '(' -name '*.h' -o -name '*.hpp' ')') H = $(wildcard *.h) $(wildcard *.hpp) C = $(wildcard *.c) $(wildcard *.cpp) @@ -54,10 +55,10 @@ tippecanoe-decode: decode.o projection.o mvt.o tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -%.o: %.c $(H) +%.o: %.c $(ALL_H) $(CC) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CFLAGS) -c -o $@ $< -%.o: %.cpp $(H) +%.o: %.cpp $(ALL_H) $(CXX) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c -o $@ $< clean: