ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-04-08 19:44:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request from mapbox/cplusplus

Browse Source 
Convert everything to C++
This commit is contained in:
Eric Fischer 2016-04-27 15:52:05 -07:00
commit 666565e820
35 changed files with 1166 additions and 1119 deletions

2
.travis.yml

@ -39,7 +39,7 @@ matrix:
install:
- BUILDTYPE=${BUILDTYPE} make
- BUILDTYPE=${BUILDTYPE} make -j
script:
- BUILDTYPE=${BUILDTYPE} make test

4
CHANGELOG.md

@ -1,3 +1,7 @@
## 1.11.0
* Convert C source files to C++
## 1.10.0
* Upgrade Clipper to fix potential crashes and improve polygon topology

23
Makefile

@ -36,13 +36,14 @@ man/tippecanoe.1: README.md
PG=
H = $(shell find . '(' -name '*.h' -o -name '*.hh' ')')
C = $(shell find . '(' -name '*.c' -o -name '*.cc' ')')
ALL_H = $(shell find . '(' -name '*.h' -o -name '*.hpp' ')')
H = $(wildcard *.h) $(wildcard *.hpp)
C = $(wildcard *.c) $(wildcard *.cpp)
INCLUDES = -I/usr/local/include -I.
LIBS = -L/usr/local/lib
tippecanoe: geojson.o jsonpull.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o mvt.o
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o mvt.o serial.o main.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
tippecanoe-enumerate: enumerate.o
@ -51,18 +52,14 @@ tippecanoe-enumerate: enumerate.o
tippecanoe-decode: decode.o projection.o mvt.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o
tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
libjsonpull.a: jsonpull.o
$(AR) rc $@ $^
ranlib $@
%.o: %.c $(ALL_H)
$(CC) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CFLAGS) -c -o $@ $<
%.o: %.c $(H)
$(CC) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CFLAGS) -c $<
%.o: %.cc $(H)
$(CXX) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c $<
%.o: %.cpp $(ALL_H)
$(CXX) $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c -o $@ $<
clean:
rm -f tippecanoe *.o
@ -70,8 +67,6 @@ clean:
indent:
clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false}" $(C) $(H)
geometry.o: clipper/clipper.hpp
TESTS = $(wildcard tests/*/out/*.json)
SPACE = $(NULL) $(NULL)

84
clip.c

@ -1,84 +0,0 @@
#include "clip.h"
#define INSIDE 0
#define LEFT 1
#define RIGHT 2
#define BOTTOM 4
#define TOP 8
static int computeOutCode(double x, double y, double xmin, double ymin, double xmax, double ymax) {
int code = INSIDE;
if (x < xmin) {
code |= LEFT;
} else if (x > xmax) {
code |= RIGHT;
}
if (y < ymin) {
code |= BOTTOM;
} else if (y > ymax) {
code |= TOP;
}
return code;
}
int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax) {
int outcode0 = computeOutCode(*x0, *y0, xmin, ymin, xmax, ymax);
int outcode1 = computeOutCode(*x1, *y1, xmin, ymin, xmax, ymax);
int accept = 0;
int changed = 0;
while (1) {
if (!(outcode0 | outcode1)) { // Bitwise OR is 0. Trivially accept and get out of loop
accept = 1;
break;
} else if (outcode0 & outcode1) { // Bitwise AND is not 0. Trivially reject and get out of loop
break;
} else {
// failed both tests, so calculate the line segment to clip
// from an outside point to an intersection with clip edge
double x = *x0, y = *y0;
// At least one endpoint is outside the clip rectangle; pick it.
int outcodeOut = outcode0 ? outcode0 : outcode1;
// Now find the intersection point;
// use formulas y = y0 + slope * (x - x0), x = x0 + (1 / slope) * (y - y0)
if (outcodeOut & TOP) { // point is above the clip rectangle
x = *x0 + (*x1 - *x0) * (ymax - *y0) / (*y1 - *y0);
y = ymax;
} else if (outcodeOut & BOTTOM) { // point is below the clip rectangle
x = *x0 + (*x1 - *x0) * (ymin - *y0) / (*y1 - *y0);
y = ymin;
} else if (outcodeOut & RIGHT) { // point is to the right of clip rectangle
y = *y0 + (*y1 - *y0) * (xmax - *x0) / (*x1 - *x0);
x = xmax;
} else if (outcodeOut & LEFT) { // point is to the left of clip rectangle
y = *y0 + (*y1 - *y0) * (xmin - *x0) / (*x1 - *x0);
x = xmin;
}
// Now we move outside point to intersection point to clip
// and get ready for next pass.
if (outcodeOut == outcode0) {
*x0 = x;
*y0 = y;
outcode0 = computeOutCode(*x0, *y0, xmin, ymin, xmax, ymax);
changed = 1;
} else {
*x1 = x;
*y1 = y;
outcode1 = computeOutCode(*x1, *y1, xmin, ymin, xmax, ymax);
changed = 1;
}
}
}
if (accept == 0) {
return 0;
} else {
return changed + 1;
}
}

1
clip.h

@ -1 +0,0 @@
int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax);

23
decode.cc → decode.cpp

@ -11,12 +11,9 @@
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include "mvt.hh"
#include "tile.h"
extern "C" {
#include "projection.h"
}
#include "mvt.hpp"
#include "projection.hpp"
#include "geometry.hpp"
void printq(const char *s) {
putchar('"');
@ -32,12 +29,12 @@ void printq(const char *s) {
putchar('"');
}
struct draw {
struct lonlat {
int op;
double lon;
double lat;
draw(int op, double lon, double lat) {
lonlat(int op, double lon, double lat) {
this->op = op;
this->lon = lon;
this->lat = lat;
@ -144,7 +141,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
printf(" }, \"geometry\": { ");
std::vector<draw> ops;
std::vector<lonlat> ops;
for (size_t g = 0; g < feat.geometry.size(); g++) {
int op = feat.geometry[g].op;
@ -159,9 +156,9 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
double lat, lon;
tile2latlon(wx, wy, 32, &lat, &lon);
ops.push_back(draw(op, lon, lat));
ops.push_back(lonlat(op, lon, lat));
} else {
ops.push_back(draw(op, 0, 0));
ops.push_back(lonlat(op, 0, 0));
}
}
@ -215,12 +212,12 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
printf(" ] ]");
}
} else if (feat.type == VT_POLYGON) {
std::vector<std::vector<draw> > rings;
std::vector<std::vector<lonlat> > rings;
std::vector<double> areas;
for (size_t i = 0; i < ops.size(); i++) {
if (ops[i].op == VT_MOVETO) {
rings.push_back(std::vector<draw>());
rings.push_back(std::vector<lonlat>());
areas.push_back(0);
}

2
enumerate.c → enumerate.cpp

@ -11,7 +11,7 @@ void enumerate(char *fname) {
exit(EXIT_FAILURE);
}
char *sql = "SELECT zoom_level, tile_column, tile_row from tiles;";
const char *sql = "SELECT zoom_level, tile_column, tile_row from tiles;";
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {

508
geojson.cpp Normal file

@ -0,0 +1,508 @@
#ifdef MTRACE
#include <mcheck.h>
#endif
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <string.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <string.h>
#include <fcntl.h>
#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <sqlite3.h>
#include <stdarg.h>
#include <sys/resource.h>
#include <pthread.h>
#include <vector>
extern "C" {
#include "jsonpull/jsonpull.h"
}
#include "pool.hpp"
#include "mbtiles.hpp"
#include "projection.hpp"
#include "version.hpp"
#include "memfile.hpp"
#include "serial.hpp"
#include "main.hpp"
#include "geojson.hpp"
#include "geometry.hpp"
#define GEOM_POINT 0 /* array of positions */
#define GEOM_MULTIPOINT 1 /* array of arrays of positions */
#define GEOM_LINESTRING 2 /* array of arrays of positions */
#define GEOM_MULTILINESTRING 3 /* array of arrays of arrays of positions */
#define GEOM_POLYGON 4 /* array of arrays of arrays of positions */
#define GEOM_MULTIPOLYGON 5 /* array of arrays of arrays of arrays of positions */
#define GEOM_TYPES 6
static const char *geometry_names[GEOM_TYPES] = {
"Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon",
};
static int geometry_within[GEOM_TYPES] = {
-1, /* point */
GEOM_POINT, /* multipoint */
GEOM_POINT, /* linestring */
GEOM_LINESTRING, /* multilinestring */
GEOM_LINESTRING, /* polygon */
GEOM_POLYGON, /* multipolygon */
};
static int mb_geometry[GEOM_TYPES] = {
VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON,
};
void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FILE *out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) {
if (j == NULL || j->type != JSON_ARRAY) {
fprintf(stderr, "%s:%d: expected array for type %d\n", fname, line, t);
return;
}
int within = geometry_within[t];
if (within >= 0) {
size_t i;
for (i = 0; i < j->length; i++) {
if (within == GEOM_POINT) {
if (i == 0 || mb_geometry[t] == GEOM_MULTIPOINT) {
op = VT_MOVETO;
} else {
op = VT_LINETO;
}
}
parse_geometry(within, j->array[i], bbox, fpos, out, op, fname, line, wx, wy, initialized, initial_x, initial_y);
}
} else {
if (j->length >= 2 && j->array[0]->type == JSON_NUMBER && j->array[1]->type == JSON_NUMBER) {
long long x, y;
double lon = j->array[0]->number;
double lat = j->array[1]->number;
latlon2tile(lat, lon, 32, &x, &y);
if (j->length > 2) {
static int warned = 0;
if (!warned) {
fprintf(stderr, "%s:%d: ignoring dimensions beyond two\n", fname, line);
warned = 1;
}
}
if (bbox != NULL) {
if (x < bbox[0]) {
bbox[0] = x;
}
if (y < bbox[1]) {
bbox[1] = y;
}
if (x > bbox[2]) {
bbox[2] = x;
}
if (y > bbox[3]) {
bbox[3] = y;
}
}
if (!*initialized) {
if (x < 0 || x >= (1LL << 32) || y < 0 || y >= (1LL < 32)) {
*initial_x = 1LL << 31;
*initial_y = 1LL << 31;
*wx = 1LL << 31;
*wy = 1LL << 31;
} else {
*initial_x = (x >> geometry_scale) << geometry_scale;
*initial_y = (y >> geometry_scale) << geometry_scale;
*wx = x;
*wy = y;
}
*initialized = 1;
}
serialize_byte(out, op, fpos, fname);
serialize_long_long(out, (x >> geometry_scale) - (*wx >> geometry_scale), fpos, fname);
serialize_long_long(out, (y >> geometry_scale) - (*wy >> geometry_scale), fpos, fname);
*wx = x;
*wy = y;
} else {
fprintf(stderr, "%s:%d: malformed point\n", fname, line);
}
}
if (t == GEOM_POLYGON) {
// Note that this is not using the correct meaning of closepath.
//
// We are using it here to close an entire Polygon, to distinguish
// the Polygons within a MultiPolygon from each other.
//
// This will be undone in fix_polygon(), which needs to know which
// rings come from which Polygons so that it can make the winding order
// of the outer ring be the opposite of the order of the inner rings.
serialize_byte(out, VT_CLOSEPATH, fpos, fname);
}
}
int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers) {
json_object *geometry_type = json_hash_get(geometry, "type");
if (geometry_type == NULL) {
static int warned = 0;
if (!warned) {
fprintf(stderr, "%s:%d: null geometry (additional not reported)\n", reading, line);
warned = 1;
}
return 0;
}
if (geometry_type->type != JSON_STRING) {
fprintf(stderr, "%s:%d: geometry without type\n", reading, line);
return 0;
}
json_object *coordinates = json_hash_get(geometry, "coordinates");
if (coordinates == NULL || coordinates->type != JSON_ARRAY) {
fprintf(stderr, "%s:%d: feature without coordinates array\n", reading, line);
return 0;
}
int t;
for (t = 0; t < GEOM_TYPES; t++) {
if (strcmp(geometry_type->string, geometry_names[t]) == 0) {
break;
}
}
if (t >= GEOM_TYPES) {
fprintf(stderr, "%s:%d: Can't handle geometry type %s\n", reading, line, geometry_type->string);
return 0;
}
int tippecanoe_minzoom = -1;
int tippecanoe_maxzoom = -1;
if (tippecanoe != NULL) {
json_object *min = json_hash_get(tippecanoe, "minzoom");
if (min != NULL && min->type == JSON_NUMBER) {
tippecanoe_minzoom = min->number;
}
if (min != NULL && min->type == JSON_STRING) {
tippecanoe_minzoom = atoi(min->string);
}
json_object *max = json_hash_get(tippecanoe, "maxzoom");
if (max != NULL && max->type == JSON_NUMBER) {
tippecanoe_maxzoom = max->number;
}
if (max != NULL && max->type == JSON_STRING) {
tippecanoe_maxzoom = atoi(max->string);
}
}
long long bbox[] = {UINT_MAX, UINT_MAX, 0, 0};
int nprop = 0;
if (properties != NULL && properties->type == JSON_HASH) {
nprop = properties->length;
}
long long metastart = *metapos;
char *metakey[nprop];
const char *metaval[nprop];
int metatype[nprop];
int mustfree[nprop];
int m = 0;
int i;
for (i = 0; i < nprop; i++) {
if (properties->keys[i]->type == JSON_STRING) {
if (exclude_all) {
if (!is_pooled(include, properties->keys[i]->string, VT_STRING)) {
continue;
}
} else if (is_pooled(exclude, properties->keys[i]->string, VT_STRING)) {
continue;
}
metakey[m] = properties->keys[i]->string;
mustfree[m] = 0;
if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) {
metatype[m] = VT_STRING;
metaval[m] = properties->values[i]->string;
m++;
} else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) {
metatype[m] = VT_NUMBER;
metaval[m] = properties->values[i]->string;
m++;
} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) {
metatype[m] = VT_BOOLEAN;
metaval[m] = properties->values[i]->type == JSON_TRUE ? "true" : "false";
m++;
} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) {
;
} else {
metatype[m] = VT_STRING;
metaval[m] = json_stringify(properties->values[i]);
mustfree[m] = 1;
m++;
}
}
}
for (i = 0; i < m; i++) {
serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname);
serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname);
if (mustfree[i]) {
free((void *) metaval[i]);
}
}
long long geomstart = *geompos;
serialize_byte(geomfile, mb_geometry[t], geompos, fname);
serialize_long_long(geomfile, *layer_seq, geompos, fname);
serialize_long_long(geomfile, (layer << 2) | ((tippecanoe_minzoom != -1) << 1) | (tippecanoe_maxzoom != -1), geompos, fname);
if (tippecanoe_minzoom != -1) {
serialize_int(geomfile, tippecanoe_minzoom, geompos, fname);
}
if (tippecanoe_maxzoom != -1) {
serialize_int(geomfile, tippecanoe_maxzoom, geompos, fname);
}
serialize_int(geomfile, segment, geompos, fname);
serialize_long_long(geomfile, metastart, geompos, fname);
serialize_int(geomfile, m, geompos, fname);
long long wx = *initial_x, wy = *initial_y;
parse_geometry(t, coordinates, bbox, geompos, geomfile, VT_MOVETO, fname, line, &wx, &wy, initialized, initial_x, initial_y);
serialize_byte(geomfile, VT_END, geompos, fname);
/*
* Note that feature_minzoom for lines is the dimension
* of the geometry in world coordinates, but
* for points is the lowest zoom level (in tiles,
* not in pixels) at which it should be drawn.
*
* So a line that is too small for, say, z8
* will have feature_minzoom of 18 (if tile detail is 10),
* not 8.
*/
int feature_minzoom = 0;
if (mb_geometry[t] == VT_LINE) {
// Skip z0 check because everything is always in the one z0 tile
for (feature_minzoom = 1; feature_minzoom < 31; feature_minzoom++) {
unsigned mask = 1 << (32 - (feature_minzoom + 1));
if (((bbox[0] & mask) != (bbox[2] & mask)) || ((bbox[1] & mask) != (bbox[3] & mask))) {
break;
}
}
} else if (mb_geometry[t] == VT_POINT) {
double r = ((double) rand()) / RAND_MAX;
if (r == 0) {
r = .00000001;
}
feature_minzoom = basezoom - floor(log(r) / -log(droprate));
}
serialize_byte(geomfile, feature_minzoom, geompos, fname);
struct index index;
index.start = geomstart;
index.end = *geompos;
index.segment = segment;
index.seq = *layer_seq;
// Calculate the center even if off the edge of the plane,
// and then mask to bring it back into the addressable area
long long midx = (bbox[0] / 2 + bbox[2] / 2) & ((1LL << 32) - 1);
long long midy = (bbox[1] / 2 + bbox[3] / 2) & ((1LL << 32) - 1);
index.index = encode(midx, midy);
fwrite_check(&index, sizeof(struct index), 1, indexfile, fname);
*indexpos += sizeof(struct index);
for (i = 0; i < 2; i++) {
if (bbox[i] < file_bbox[i]) {
file_bbox[i] = bbox[i];
}
}
for (i = 2; i < 4; i++) {
if (bbox[i] > file_bbox[i]) {
file_bbox[i] = bbox[i];
}
}
if (*progress_seq % 10000 == 0) {
checkdisk(readers, CPUS);
if (!quiet) {
fprintf(stderr, "Read %.2f million features\r", *progress_seq / 1000000.0);
}
}
(*progress_seq)++;
(*layer_seq)++;
return 1;
}
void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers) {
long long found_hashes = 0;
long long found_features = 0;
long long found_geometries = 0;
while (1) {
json_object *j = json_read(jp);
if (j == NULL) {
if (jp->error != NULL) {
fprintf(stderr, "%s:%d: %s\n", reading, jp->line, jp->error);
}
json_free(jp->root);
break;
}
if (j->type == JSON_HASH) {
found_hashes++;
if (found_hashes == 50 && found_features == 0 && found_geometries == 0) {
fprintf(stderr, "%s:%d: Warning: not finding any GeoJSON features or geometries in input yet after 50 objects.\n", reading, jp->line);
}
}
json_object *type = json_hash_get(j, "type");
if (type == NULL || type->type != JSON_STRING) {
continue;
}
if (found_features == 0) {
int i;
int is_geometry = 0;
for (i = 0; i < GEOM_TYPES; i++) {
if (strcmp(type->string, geometry_names[i]) == 0) {
is_geometry = 1;
break;
}
}
if (is_geometry) {
if (j->parent != NULL) {
if (j->parent->type == JSON_ARRAY) {
if (j->parent->parent->type == JSON_HASH) {
json_object *geometries = json_hash_get(j->parent->parent, "geometries");
if (geometries != NULL) {
// Parent of Parent must be a GeometryCollection
is_geometry = 0;
}
}
} else if (j->parent->type == JSON_HASH) {
json_object *geometry = json_hash_get(j->parent, "geometry");
if (geometry != NULL) {
// Parent must be a Feature
is_geometry = 0;
}
}
}
}
if (is_geometry) {
if (found_features != 0 && found_geometries == 0) {
fprintf(stderr, "%s:%d: Warning: found a mixture of features and bare geometries\n", reading, jp->line);
}
found_geometries++;
serialize_geometry(j, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers);
json_free(j);
continue;
}
}
if (strcmp(type->string, "Feature") != 0) {
continue;
}
if (found_features == 0 && found_geometries != 0) {
fprintf(stderr, "%s:%d: Warning: found a mixture of features and bare geometries\n", reading, jp->line);
}
found_features++;
json_object *geometry = json_hash_get(j, "geometry");
if (geometry == NULL) {
fprintf(stderr, "%s:%d: feature with no geometry\n", reading, jp->line);
json_free(j);
continue;
}
json_object *properties = json_hash_get(j, "properties");
if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
fprintf(stderr, "%s:%d: feature without properties hash\n", reading, jp->line);
json_free(j);
continue;
}
json_object *tippecanoe = json_hash_get(j, "tippecanoe");
json_object *geometries = json_hash_get(geometry, "geometries");
if (geometries != NULL) {
size_t g;
for (g = 0; g < geometries->length; g++) {
serialize_geometry(geometries->array[g], properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers);
}
} else {
serialize_geometry(geometry, properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers);
}
json_free(j);
/* XXX check for any non-features in the outer object */
}
}
void *run_parse_json(void *v) {
struct parse_json_args *pja = (struct parse_json_args *) v;
parse_json(pja->jp, pja->reading, pja->layer_seq, pja->progress_seq, pja->metapos, pja->geompos, pja->indexpos, pja->exclude, pja->include, pja->exclude_all, pja->metafile, pja->geomfile, pja->indexfile, pja->poolfile, pja->treefile, pja->fname, pja->basezoom, pja->layer, pja->droprate, pja->file_bbox, pja->segment, pja->initialized, pja->initial_x, pja->initial_y, pja->readers);
return NULL;
}
struct jsonmap {
char *map;
unsigned long long off;
unsigned long long end;
};
ssize_t json_map_read(struct json_pull *jp, char *buffer, size_t n) {
struct jsonmap *jm = (struct jsonmap *) jp->source;
if (jm->off + n >= jm->end) {
n = jm->end - jm->off;
}
memcpy(buffer, jm->map + jm->off, n);
jm->off += n;
return n;
}
struct json_pull *json_begin_map(char *map, long long len) {
struct jsonmap *jm = (struct jsonmap *) malloc(sizeof(struct jsonmap));
if (jm == NULL) {
perror("Out of memory");
exit(EXIT_FAILURE);
}
jm->map = map;
jm->off = 0;
jm->end = len;
return json_begin(json_map_read, jm);
}

31
geojson.hpp Normal file

@ -0,0 +1,31 @@
struct parse_json_args {
json_pull *jp;
const char *reading;
volatile long long *layer_seq;
volatile long long *progress_seq;
long long *metapos;
long long *geompos;
long long *indexpos;
struct pool *exclude;
struct pool *include;
int exclude_all;
FILE *metafile;
FILE *geomfile;
FILE *indexfile;
struct memfile *poolfile;
struct memfile *treefile;
char *fname;
int basezoom;
int layer;
double droprate;
long long *file_bbox;
int segment;
int *initialized;
unsigned *initial_x;
unsigned *initial_y;
struct reader *readers;
};
struct json_pull *json_begin_map(char *map, long long len);
void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers);
void *run_parse_json(void *v);

97
geometry.cc → geometry.cpp

@ -8,17 +8,15 @@
#include <unistd.h>
#include <cmath>
#include <limits.h>
#include "geometry.hh"
#include "clipper/clipper.hpp"
extern "C" {
#include <sqlite3.h>
#include "tile.h"
#include "clip.h"
#include "projection.h"
}
#include "geometry.hpp"
#include "clipper/clipper.hpp"
#include "projection.hpp"
#include "serial.hpp"
#include "main.hpp"
static int pnpoly(drawvec &vert, size_t start, size_t nvert, long long testx, long long testy);
static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax);
drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y) {
drawvec out;
@ -1310,3 +1308,86 @@ std::vector<drawvec> chop_polygon(std::vector<drawvec> &geoms) {
geoms = out;
}
}
#define INSIDE 0
#define LEFT 1
#define RIGHT 2
#define BOTTOM 4
#define TOP 8
static int computeOutCode(double x, double y, double xmin, double ymin, double xmax, double ymax) {
int code = INSIDE;
if (x < xmin) {
code |= LEFT;
} else if (x > xmax) {
code |= RIGHT;
}
if (y < ymin) {
code |= BOTTOM;
} else if (y > ymax) {
code |= TOP;
}
return code;
}
static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax) {
int outcode0 = computeOutCode(*x0, *y0, xmin, ymin, xmax, ymax);
int outcode1 = computeOutCode(*x1, *y1, xmin, ymin, xmax, ymax);
int accept = 0;
int changed = 0;
while (1) {
if (!(outcode0 | outcode1)) { // Bitwise OR is 0. Trivially accept and get out of loop
accept = 1;
break;
} else if (outcode0 & outcode1) { // Bitwise AND is not 0. Trivially reject and get out of loop
break;
} else {
// failed both tests, so calculate the line segment to clip
// from an outside point to an intersection with clip edge
double x = *x0, y = *y0;
// At least one endpoint is outside the clip rectangle; pick it.
int outcodeOut = outcode0 ? outcode0 : outcode1;
// Now find the intersection point;
// use formulas y = y0 + slope * (x - x0), x = x0 + (1 / slope) * (y - y0)
if (outcodeOut & TOP) { // point is above the clip rectangle
x = *x0 + (*x1 - *x0) * (ymax - *y0) / (*y1 - *y0);
y = ymax;
} else if (outcodeOut & BOTTOM) { // point is below the clip rectangle
x = *x0 + (*x1 - *x0) * (ymin - *y0) / (*y1 - *y0);
y = ymin;
} else if (outcodeOut & RIGHT) { // point is to the right of clip rectangle
y = *y0 + (*y1 - *y0) * (xmax - *x0) / (*x1 - *x0);
x = xmax;
} else if (outcodeOut & LEFT) { // point is to the left of clip rectangle
y = *y0 + (*y1 - *y0) * (xmin - *x0) / (*x1 - *x0);
x = xmin;
}
// Now we move outside point to intersection point to clip
// and get ready for next pass.
if (outcodeOut == outcode0) {
*x0 = x;
*y0 = y;
outcode0 = computeOutCode(*x0, *y0, xmin, ymin, xmax, ymax);
changed = 1;
} else {
*x1 = x;
*y1 = y;
outcode1 = computeOutCode(*x1, *y1, xmin, ymin, xmax, ymax);
changed = 1;
}
}
}
if (accept == 0) {
return 0;
} else {
return changed + 1;
}
}

13
geometry.hh → geometry.hpp

@ -1,3 +1,16 @@
#define VT_POINT 1
#define VT_LINE 2
#define VT_POLYGON 3
#define VT_END 0
#define VT_MOVETO 1
#define VT_LINETO 2
#define VT_CLOSEPATH 7
#define VT_STRING 1
#define VT_NUMBER 2
#define VT_BOOLEAN 7
struct draw {
signed char op;
long long x;

0
jsonpull.c → jsonpull/jsonpull.c

0
jsonpull.h → jsonpull/jsonpull.h

802
geojson.c → main.cpp

@ -20,6 +20,7 @@
#include <sys/resource.h>
#include <pthread.h>
#include <getopt.h>
#include <vector>
#ifdef __APPLE__
#include <sys/types.h>
@ -30,13 +31,21 @@
#include <sys/statfs.h>
#endif
#include "jsonpull.h"
#include "tile.h"
#include "pool.h"
#include "mbtiles.h"
#include "projection.h"
#include "version.h"
#include "memfile.h"
extern "C" {
#include "jsonpull/jsonpull.h"
}
#include "tile.hpp"
#include "pool.hpp"
#include "mbtiles.hpp"
#include "projection.hpp"
#include "version.hpp"
#include "memfile.hpp"
#include "serial.hpp"
#include "main.hpp"
#include "geojson.hpp"
#include "geometry.hpp"
#include "options.hpp"
static int low_detail = 12;
static int full_detail = -1;
@ -48,31 +57,6 @@ int geometry_scale = 0;
static int prevent[256];
static int additional[256];
#define GEOM_POINT 0 /* array of positions */
#define GEOM_MULTIPOINT 1 /* array of arrays of positions */
#define GEOM_LINESTRING 2 /* array of arrays of positions */
#define GEOM_MULTILINESTRING 3 /* array of arrays of arrays of positions */
#define GEOM_POLYGON 4 /* array of arrays of arrays of positions */
#define GEOM_MULTIPOLYGON 5 /* array of arrays of arrays of arrays of positions */
#define GEOM_TYPES 6
static const char *geometry_names[GEOM_TYPES] = {
"Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon",
};
static int geometry_within[GEOM_TYPES] = {
-1, /* point */
GEOM_POINT, /* multipoint */
GEOM_POINT, /* linestring */
GEOM_LINESTRING, /* multilinestring */
GEOM_LINESTRING, /* polygon */
GEOM_POLYGON, /* multipolygon */
};
static int mb_geometry[GEOM_TYPES] = {
VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON,
};
struct source {
char *layer;
char *file;
@ -85,7 +69,7 @@ struct tofree {
} *tofree = NULL;
void mustfree(void *p) {
struct tofree *f = malloc(sizeof(struct tofree));
struct tofree *f = (struct tofree *) malloc(sizeof(struct tofree));
if (f == NULL) {
perror("malloc");
exit(EXIT_FAILURE);
@ -203,238 +187,6 @@ void init_cpus() {
}
}
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) {
size_t w = fwrite(ptr, size, nitems, stream);
if (w != nitems) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_FAILURE);
}
return w;
}
void serialize_int(FILE *out, int n, long long *fpos, const char *fname) {
serialize_long_long(out, n, fpos, fname);
}
void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname) {
unsigned long long zigzag = (n << 1) ^ (n >> 63);
while (1) {
unsigned char b = zigzag & 0x7F;
if ((zigzag >> 7) != 0) {
b |= 0x80;
if (putc(b, out) == EOF) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_FAILURE);
}
*fpos += 1;
zigzag >>= 7;
} else {
if (putc(b, out) == EOF) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_FAILURE);
}
*fpos += 1;
break;
}
}
}
void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname) {
fwrite_check(&n, sizeof(signed char), 1, out, fname);
*fpos += sizeof(signed char);
}
void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname) {
fwrite_check(&n, sizeof(unsigned), 1, out, fname);
*fpos += sizeof(unsigned);
}
void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FILE *out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) {
if (j == NULL || j->type != JSON_ARRAY) {
fprintf(stderr, "%s:%d: expected array for type %d\n", fname, line, t);
return;
}
int within = geometry_within[t];
if (within >= 0) {
size_t i;
for (i = 0; i < j->length; i++) {
if (within == GEOM_POINT) {
if (i == 0 || mb_geometry[t] == GEOM_MULTIPOINT) {
op = VT_MOVETO;
} else {
op = VT_LINETO;
}
}
parse_geometry(within, j->array[i], bbox, fpos, out, op, fname, line, wx, wy, initialized, initial_x, initial_y);
}
} else {
if (j->length >= 2 && j->array[0]->type == JSON_NUMBER && j->array[1]->type == JSON_NUMBER) {
long long x, y;
double lon = j->array[0]->number;
double lat = j->array[1]->number;
latlon2tile(lat, lon, 32, &x, &y);
if (j->length > 2) {
static int warned = 0;
if (!warned) {
fprintf(stderr, "%s:%d: ignoring dimensions beyond two\n", fname, line);
warned = 1;
}
}
if (bbox != NULL) {
if (x < bbox[0]) {
bbox[0] = x;
}
if (y < bbox[1]) {
bbox[1] = y;
}
if (x > bbox[2]) {
bbox[2] = x;
}
if (y > bbox[3]) {
bbox[3] = y;
}
}
if (!*initialized) {
if (x < 0 || x >= (1LL << 32) || y < 0 || y >= (1LL < 32)) {
*initial_x = 1LL << 31;
*initial_y = 1LL << 31;
*wx = 1LL << 31;
*wy = 1LL << 31;
} else {
*initial_x = (x >> geometry_scale) << geometry_scale;
*initial_y = (y >> geometry_scale) << geometry_scale;
*wx = x;
*wy = y;
}
*initialized = 1;
}
serialize_byte(out, op, fpos, fname);
serialize_long_long(out, (x >> geometry_scale) - (*wx >> geometry_scale), fpos, fname);
serialize_long_long(out, (y >> geometry_scale) - (*wy >> geometry_scale), fpos, fname);
*wx = x;
*wy = y;
} else {
fprintf(stderr, "%s:%d: malformed point\n", fname, line);
}
}
if (t == GEOM_POLYGON) {
// Note that this is not using the correct meaning of closepath.
//
// We are using it here to close an entire Polygon, to distinguish
// the Polygons within a MultiPolygon from each other.
//
// This will be undone in fix_polygon(), which needs to know which
// rings come from which Polygons so that it can make the winding order
// of the outer ring be the opposite of the order of the inner rings.
serialize_byte(out, VT_CLOSEPATH, fpos, fname);
}
}
void deserialize_int(char **f, int *n) {
long long ll;
deserialize_long_long(f, &ll);
*n = ll;
}
void deserialize_long_long(char **f, long long *n) {
unsigned long long zigzag = 0;
int shift = 0;
while (1) {
if ((**f & 0x80) == 0) {
zigzag |= ((unsigned long long) **f) << shift;
*f += 1;
shift += 7;
break;
} else {
zigzag |= ((unsigned long long) (**f & 0x7F)) << shift;
*f += 1;
shift += 7;
}
}
*n = (zigzag >> 1) ^ (-(zigzag & 1));
}
void deserialize_uint(char **f, unsigned *n) {
memcpy(n, *f, sizeof(unsigned));
*f += sizeof(unsigned);
}
void deserialize_byte(char **f, signed char *n) {
memcpy(n, *f, sizeof(signed char));
*f += sizeof(signed char);
}
int deserialize_long_long_io(FILE *f, long long *n, long long *geompos) {
unsigned long long zigzag = 0;
int shift = 0;
while (1) {
int c = getc(f);
if (c == EOF) {
return 0;
}
(*geompos)++;
if ((c & 0x80) == 0) {
zigzag |= ((unsigned long long) c) << shift;
shift += 7;
break;
} else {
zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
shift += 7;
}
}
*n = (zigzag >> 1) ^ (-(zigzag & 1));
return 1;
}
int deserialize_int_io(FILE *f, int *n, long long *geompos) {
long long ll = 0;
int ret = deserialize_long_long_io(f, &ll, geompos);
*n = ll;
return ret;
}
int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos) {
if (fread(n, sizeof(unsigned), 1, f) != 1) {
return 0;
}
*geompos += sizeof(unsigned);
return 1;
}
int deserialize_byte_io(FILE *f, signed char *n, long long *geompos) {
int c = getc(f);
if (c == EOF) {
return 0;
}
*n = c;
(*geompos)++;
return 1;
}
struct index {
long long start;
long long end;
unsigned long long index;
short segment;
unsigned long long seq : (64 - 16); // pack with segment to stay in 32 bytes
};
int indexcmp(const void *v1, const void *v2) {
const struct index *i1 = (const struct index *) v1;
const struct index *i2 = (const struct index *) v2;
@ -505,488 +257,6 @@ static void merge(struct merge *merges, int nmerges, unsigned char *map, FILE *f
}
}
struct stringpool {
long long left;
long long right;
long long off;
};
static unsigned char swizzle[256] = {
0x00, 0xBF, 0x18, 0xDE, 0x93, 0xC9, 0xB1, 0x5E, 0xDF, 0xBE, 0x72, 0x5A, 0xBB, 0x42, 0x64, 0xC6,
0xD8, 0xB7, 0x15, 0x74, 0x1C, 0x8B, 0x91, 0xF5, 0x29, 0x46, 0xEC, 0x6F, 0xCA, 0x20, 0xF0, 0x06,
0x27, 0x61, 0x87, 0xE0, 0x6E, 0x43, 0x50, 0xC5, 0x1B, 0xB4, 0x37, 0xC3, 0x69, 0xA6, 0xEE, 0x80,
0xAF, 0x9B, 0xA1, 0x76, 0x23, 0x24, 0x53, 0xF3, 0x5B, 0x65, 0x19, 0xF4, 0xFC, 0xDD, 0x26, 0xE8,
0x10, 0xF7, 0xCE, 0x92, 0x48, 0xF6, 0x94, 0x60, 0x07, 0xC4, 0xB9, 0x97, 0x6D, 0xA4, 0x11, 0x0D,
0x1F, 0x4D, 0x13, 0xB0, 0x5D, 0xBA, 0x31, 0xD5, 0x8D, 0x51, 0x36, 0x96, 0x7A, 0x03, 0x7F, 0xDA,
0x17, 0xDB, 0xD4, 0x83, 0xE2, 0x79, 0x6A, 0xE1, 0x95, 0x38, 0xFF, 0x28, 0xB2, 0xB3, 0xA7, 0xAE,
0xF8, 0x54, 0xCC, 0xDC, 0x9A, 0x6B, 0xFB, 0x3F, 0xD7, 0xBC, 0x21, 0xC8, 0x71, 0x09, 0x16, 0xAC,
0x3C, 0x8A, 0x62, 0x05, 0xC2, 0x8C, 0x32, 0x4E, 0x35, 0x9C, 0x5F, 0x75, 0xCD, 0x2E, 0xA2, 0x3E,
0x1A, 0xC1, 0x8E, 0x14, 0xA0, 0xD3, 0x7D, 0xD9, 0xEB, 0x5C, 0x70, 0xE6, 0x9E, 0x12, 0x3B, 0xEF,
0x1E, 0x49, 0xD2, 0x98, 0x39, 0x7E, 0x44, 0x4B, 0x6C, 0x88, 0x02, 0x2C, 0xAD, 0xE5, 0x9F, 0x40,
0x7B, 0x4A, 0x3D, 0xA9, 0xAB, 0x0B, 0xD6, 0x2F, 0x90, 0x2A, 0xB6, 0x1D, 0xC7, 0x22, 0x55, 0x34,
0x0A, 0xD0, 0xB5, 0x68, 0xE3, 0x59, 0xFD, 0xFA, 0x57, 0x77, 0x25, 0xA3, 0x04, 0xB8, 0x33, 0x89,
0x78, 0x82, 0xE4, 0xC0, 0x0E, 0x8F, 0x85, 0xD1, 0x84, 0x08, 0x67, 0x47, 0x9D, 0xCB, 0x58, 0x4C,
0xAA, 0xED, 0x52, 0xF2, 0x4F, 0xF1, 0x66, 0xCF, 0xA5, 0x56, 0xEA, 0x7C, 0xE9, 0x63, 0xE7, 0x01,
0xF9, 0xFE, 0x0C, 0x99, 0x2D, 0x0F, 0x3A, 0x41, 0x45, 0xA8, 0x30, 0x2B, 0x73, 0xBD, 0x86, 0x81,
};
int swizzlecmp(char *a, char *b) {
while (*a || *b) {
int aa = swizzle[(unsigned char) *a];
int bb = swizzle[(unsigned char) *b];
int cmp = aa - bb;
if (cmp != 0) {
return cmp;
}
a++;
b++;
}
return 0;
}
long long addpool(struct memfile *poolfile, struct memfile *treefile, char *s, char type) {
long long *sp = &treefile->tree;
while (*sp != 0) {
int cmp = swizzlecmp(s, poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off + 1);
if (cmp == 0) {
cmp = type - (poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off)[0];
}
if (cmp < 0) {
sp = &(((struct stringpool *) (treefile->map + *sp))->left);
} else if (cmp > 0) {
sp = &(((struct stringpool *) (treefile->map + *sp))->right);
} else {
return ((struct stringpool *) (treefile->map + *sp))->off;
}
}
// *sp is probably in the memory-mapped file, and will move if the file grows.
long long ssp;
if (sp == &treefile->tree) {
ssp = -1;
} else {
ssp = ((char *) sp) - treefile->map;
}
long long off = poolfile->off;
if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write");
exit(EXIT_FAILURE);
}
if (memfile_write(poolfile, s, strlen(s) + 1) < 0) {
perror("memfile write");
exit(EXIT_FAILURE);
}
struct stringpool tsp;
tsp.left = 0;
tsp.right = 0;
tsp.off = off;
long long p = treefile->off;
if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) {
perror("memfile write");
exit(EXIT_FAILURE);
}
if (ssp == -1) {
treefile->tree = p;
} else {
*((long long *) (treefile->map + ssp)) = p;
}
return off;
}
int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers) {
json_object *geometry_type = json_hash_get(geometry, "type");
if (geometry_type == NULL) {
static int warned = 0;
if (!warned) {
fprintf(stderr, "%s:%d: null geometry (additional not reported)\n", reading, line);
warned = 1;
}
return 0;
}
if (geometry_type->type != JSON_STRING) {
fprintf(stderr, "%s:%d: geometry without type\n", reading, line);
return 0;
}
json_object *coordinates = json_hash_get(geometry, "coordinates");
if (coordinates == NULL || coordinates->type != JSON_ARRAY) {
fprintf(stderr, "%s:%d: feature without coordinates array\n", reading, line);
return 0;
}
int t;
for (t = 0; t < GEOM_TYPES; t++) {
if (strcmp(geometry_type->string, geometry_names[t]) == 0) {
break;
}
}
if (t >= GEOM_TYPES) {
fprintf(stderr, "%s:%d: Can't handle geometry type %s\n", reading, line, geometry_type->string);
return 0;
}
int tippecanoe_minzoom = -1;
int tippecanoe_maxzoom = -1;
if (tippecanoe != NULL) {
json_object *min = json_hash_get(tippecanoe, "minzoom");
if (min != NULL && min->type == JSON_NUMBER) {
tippecanoe_minzoom = min->number;
}
if (min != NULL && min->type == JSON_STRING) {
tippecanoe_minzoom = atoi(min->string);
}
json_object *max = json_hash_get(tippecanoe, "maxzoom");
if (max != NULL && max->type == JSON_NUMBER) {
tippecanoe_maxzoom = max->number;
}
if (max != NULL && max->type == JSON_STRING) {
tippecanoe_maxzoom = atoi(max->string);
}
}
long long bbox[] = {UINT_MAX, UINT_MAX, 0, 0};
int nprop = 0;
if (properties != NULL && properties->type == JSON_HASH) {
nprop = properties->length;
}
long long metastart = *metapos;
char *metakey[nprop];
char *metaval[nprop];
int metatype[nprop];
int mustfree[nprop];
int m = 0;
int i;
for (i = 0; i < nprop; i++) {
if (properties->keys[i]->type == JSON_STRING) {
if (exclude_all) {
if (!is_pooled(include, properties->keys[i]->string, VT_STRING)) {
continue;
}
} else if (is_pooled(exclude, properties->keys[i]->string, VT_STRING)) {
continue;
}
metakey[m] = properties->keys[i]->string;
mustfree[m] = 0;
if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) {
metatype[m] = VT_STRING;
metaval[m] = properties->values[i]->string;
m++;
} else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) {
metatype[m] = VT_NUMBER;
metaval[m] = properties->values[i]->string;
m++;
} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) {
metatype[m] = VT_BOOLEAN;
metaval[m] = properties->values[i]->type == JSON_TRUE ? "true" : "false";
m++;
} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) {
;
} else {
metatype[m] = VT_STRING;
metaval[m] = json_stringify(properties->values[i]);
mustfree[m] = 1;
m++;
}
}
}
for (i = 0; i < m; i++) {
serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname);
serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname);
if (mustfree[i]) {
free(metaval[i]);
}
}
long long geomstart = *geompos;
serialize_byte(geomfile, mb_geometry[t], geompos, fname);
serialize_long_long(geomfile, *layer_seq, geompos, fname);
serialize_long_long(geomfile, (layer << 2) | ((tippecanoe_minzoom != -1) << 1) | (tippecanoe_maxzoom != -1), geompos, fname);
if (tippecanoe_minzoom != -1) {
serialize_int(geomfile, tippecanoe_minzoom, geompos, fname);
}
if (tippecanoe_maxzoom != -1) {
serialize_int(geomfile, tippecanoe_maxzoom, geompos, fname);
}
serialize_int(geomfile, segment, geompos, fname);
serialize_long_long(geomfile, metastart, geompos, fname);
serialize_int(geomfile, m, geompos, fname);
long long wx = *initial_x, wy = *initial_y;
parse_geometry(t, coordinates, bbox, geompos, geomfile, VT_MOVETO, fname, line, &wx, &wy, initialized, initial_x, initial_y);
serialize_byte(geomfile, VT_END, geompos, fname);
/*
* Note that feature_minzoom for lines is the dimension
* of the geometry in world coordinates, but
* for points is the lowest zoom level (in tiles,
* not in pixels) at which it should be drawn.
*
* So a line that is too small for, say, z8
* will have feature_minzoom of 18 (if tile detail is 10),
* not 8.
*/
int feature_minzoom = 0;
if (mb_geometry[t] == VT_LINE) {
// Skip z0 check because everything is always in the one z0 tile
for (feature_minzoom = 1; feature_minzoom < 31; feature_minzoom++) {
unsigned mask = 1 << (32 - (feature_minzoom + 1));
if (((bbox[0] & mask) != (bbox[2] & mask)) || ((bbox[1] & mask) != (bbox[3] & mask))) {
break;
}
}
} else if (mb_geometry[t] == VT_POINT) {
double r = ((double) rand()) / RAND_MAX;
if (r == 0) {
r = .00000001;
}
feature_minzoom = basezoom - floor(log(r) / -log(droprate));
}
serialize_byte(geomfile, feature_minzoom, geompos, fname);
struct index index;
index.start = geomstart;
index.end = *geompos;
index.segment = segment;
index.seq = *layer_seq;
// Calculate the center even if off the edge of the plane,
// and then mask to bring it back into the addressable area
long long midx = (bbox[0] / 2 + bbox[2] / 2) & ((1LL << 32) - 1);
long long midy = (bbox[1] / 2 + bbox[3] / 2) & ((1LL << 32) - 1);
index.index = encode(midx, midy);
fwrite_check(&index, sizeof(struct index), 1, indexfile, fname);
*indexpos += sizeof(struct index);
for (i = 0; i < 2; i++) {
if (bbox[i] < file_bbox[i]) {
file_bbox[i] = bbox[i];
}
}
for (i = 2; i < 4; i++) {
if (bbox[i] > file_bbox[i]) {
file_bbox[i] = bbox[i];
}
}
if (*progress_seq % 10000 == 0) {
checkdisk(readers, CPUS);
if (!quiet) {
fprintf(stderr, "Read %.2f million features\r", *progress_seq / 1000000.0);
}
}
(*progress_seq)++;
(*layer_seq)++;
return 1;
}
void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers) {
long long found_hashes = 0;
long long found_features = 0;
long long found_geometries = 0;
while (1) {
json_object *j = json_read(jp);
if (j == NULL) {
if (jp->error != NULL) {
fprintf(stderr, "%s:%d: %s\n", reading, jp->line, jp->error);
}
json_free(jp->root);
break;
}
if (j->type == JSON_HASH) {
found_hashes++;
if (found_hashes == 50 && found_features == 0 && found_geometries == 0) {
fprintf(stderr, "%s:%d: Warning: not finding any GeoJSON features or geometries in input yet after 50 objects.\n", reading, jp->line);
}
}
json_object *type = json_hash_get(j, "type");
if (type == NULL || type->type != JSON_STRING) {
continue;
}
if (found_features == 0) {
int i;
int is_geometry = 0;
for (i = 0; i < GEOM_TYPES; i++) {
if (strcmp(type->string, geometry_names[i]) == 0) {
is_geometry = 1;
break;
}
}
if (is_geometry) {
if (j->parent != NULL) {
if (j->parent->type == JSON_ARRAY) {
if (j->parent->parent->type == JSON_HASH) {
json_object *geometries = json_hash_get(j->parent->parent, "geometries");
if (geometries != NULL) {
// Parent of Parent must be a GeometryCollection
is_geometry = 0;
}
}
} else if (j->parent->type == JSON_HASH) {
json_object *geometry = json_hash_get(j->parent, "geometry");
if (geometry != NULL) {
// Parent must be a Feature
is_geometry = 0;
}
}
}
}
if (is_geometry) {
if (found_features != 0 && found_geometries == 0) {
fprintf(stderr, "%s:%d: Warning: found a mixture of features and bare geometries\n", reading, jp->line);
}
found_geometries++;
serialize_geometry(j, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers);
json_free(j);
continue;
}
}
if (strcmp(type->string, "Feature") != 0) {
continue;
}
if (found_features == 0 && found_geometries != 0) {
fprintf(stderr, "%s:%d: Warning: found a mixture of features and bare geometries\n", reading, jp->line);
}
found_features++;
json_object *geometry = json_hash_get(j, "geometry");
if (geometry == NULL) {
fprintf(stderr, "%s:%d: feature with no geometry\n", reading, jp->line);
json_free(j);
continue;
}
json_object *properties = json_hash_get(j, "properties");
if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
fprintf(stderr, "%s:%d: feature without properties hash\n", reading, jp->line);
json_free(j);
continue;
}
json_object *tippecanoe = json_hash_get(j, "tippecanoe");
json_object *geometries = json_hash_get(geometry, "geometries");
if (geometries != NULL) {
size_t g;
for (g = 0; g < geometries->length; g++) {
serialize_geometry(geometries->array[g], properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers);
}
} else {
serialize_geometry(geometry, properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers);
}
json_free(j);
/* XXX check for any non-features in the outer object */
}
}
struct parse_json_args {
json_pull *jp;
const char *reading;
volatile long long *layer_seq;
volatile long long *progress_seq;
long long *metapos;
long long *geompos;
long long *indexpos;
struct pool *exclude;
struct pool *include;
int exclude_all;
FILE *metafile;
FILE *geomfile;
FILE *indexfile;
struct memfile *poolfile;
struct memfile *treefile;
char *fname;
int basezoom;
int layer;
double droprate;
long long *file_bbox;
int segment;
int *initialized;
unsigned *initial_x;
unsigned *initial_y;
struct reader *readers;
};
void *run_parse_json(void *v) {
struct parse_json_args *pja = v;
parse_json(pja->jp, pja->reading, pja->layer_seq, pja->progress_seq, pja->metapos, pja->geompos, pja->indexpos, pja->exclude, pja->include, pja->exclude_all, pja->metafile, pja->geomfile, pja->indexfile, pja->poolfile, pja->treefile, pja->fname, pja->basezoom, pja->layer, pja->droprate, pja->file_bbox, pja->segment, pja->initialized, pja->initial_x, pja->initial_y, pja->readers);
return NULL;
}
struct jsonmap {
char *map;
unsigned long long off;
unsigned long long end;
};
ssize_t json_map_read(struct json_pull *jp, char *buffer, size_t n) {
struct jsonmap *jm = jp->source;
if (jm->off + n >= jm->end) {
n = jm->end - jm->off;
}
memcpy(buffer, jm->map + jm->off, n);
jm->off += n;
return n;
}
struct json_pull *json_begin_map(char *map, long long len) {
struct jsonmap *jm = malloc(sizeof(struct jsonmap));
if (jm == NULL) {
perror("Out of memory");
exit(EXIT_FAILURE);
}
jm->map = map;
jm->off = 0;
jm->end = len;
return json_begin(json_map_read, jm);
}
struct sort_arg {
int task;
int cpus;
@ -999,7 +269,7 @@ struct sort_arg {
};
void *run_sort(void *v) {
struct sort_arg *a = v;
struct sort_arg *a = (struct sort_arg *) v;
long long start;
for (start = a->task * a->unit; start < a->indexpos; start += a->unit * a->cpus) {
@ -1137,7 +407,7 @@ struct read_parallel_arg {
};
void *run_read_parallel(void *v) {
struct read_parallel_arg *a = v;
struct read_parallel_arg *a = (struct read_parallel_arg *) v;
struct stat st;
if (fstat(a->fd, &st) != 0) {
@ -1148,7 +418,7 @@ void *run_read_parallel(void *v) {
}
a->len = st.st_size;
char *map = mmap(NULL, a->len, PROT_READ, MAP_PRIVATE, a->fd, 0);
char *map = (char *) mmap(NULL, a->len, PROT_READ, MAP_PRIVATE, a->fd, 0);
if (map == NULL || map == MAP_FAILED) {
perror("map intermediate input");
exit(EXIT_FAILURE);
@ -1179,7 +449,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i
*is_parsing = 1;
struct read_parallel_arg *rpa = malloc(sizeof(struct read_parallel_arg));
struct read_parallel_arg *rpa = (struct read_parallel_arg *) malloc(sizeof(struct read_parallel_arg));
if (rpa == NULL) {
perror("Out of memory");
exit(EXIT_FAILURE);
@ -1272,7 +542,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
}
if (indexst.st_size != 0) {
struct index *indexmap = mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds_in[i], 0);
struct index *indexmap = (struct index *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds_in[i], 0);
if (indexmap == MAP_FAILED) {
fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds_in[i], (long long) indexst.st_size);
perror("map index");
@ -1280,7 +550,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
}
madvise(indexmap, indexst.st_size, MADV_SEQUENTIAL);
madvise(indexmap, indexst.st_size, MADV_WILLNEED);
char *geommap = mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds_in[i], 0);
char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds_in[i], 0);
if (geommap == MAP_FAILED) {
perror("map geom");
exit(EXIT_FAILURE);
@ -1410,7 +680,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
}
}
struct indexmap *indexmap = mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0);
struct indexmap *indexmap = (struct indexmap *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0);
if (indexmap == MAP_FAILED) {
fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds[i], (long long) indexst.st_size);
perror("map index");
@ -1418,7 +688,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
}
madvise(indexmap, indexst.st_size, MADV_RANDOM); // sequential, but from several pointers at once
madvise(indexmap, indexst.st_size, MADV_WILLNEED);
char *geommap = mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0);
char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0);
if (geommap == MAP_FAILED) {
perror("map geom");
exit(EXIT_FAILURE);
@ -1439,7 +709,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
exit(EXIT_FAILURE);
}
} else if (indexst.st_size == sizeof(struct index) || prefix + splitbits >= 64) {
struct index *indexmap = mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0);
struct index *indexmap = (struct index *) mmap(NULL, indexst.st_size, PROT_READ, MAP_PRIVATE, indexfds[i], 0);
if (indexmap == MAP_FAILED) {
fprintf(stderr, "fd %lld, len %lld\n", (long long) indexfds[i], (long long) indexst.st_size);
perror("map index");
@ -1447,7 +717,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
}
madvise(indexmap, indexst.st_size, MADV_SEQUENTIAL);
madvise(indexmap, indexst.st_size, MADV_WILLNEED);
char *geommap = mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0);
char *geommap = (char *) mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfds[i], 0);
if (geommap == MAP_FAILED) {
perror("map geom");
exit(EXIT_FAILURE);
@ -1588,7 +858,7 @@ void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE
}
}
int read_json(int argc, struct source **sourcelist, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int *prevent, int *additional, int read_parallel, int forcetable, const char *attribution) {
int read_input(int argc, struct source **sourcelist, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int *prevent, int *additional, int read_parallel, int forcetable, const char *attribution) {
int ret = EXIT_SUCCESS;
struct reader reader[CPUS];
@ -1738,7 +1008,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay
if (fstat(fd, &st) == 0) {
off = lseek(fd, 0, SEEK_CUR);
if (off >= 0) {
map = mmap(NULL, st.st_size - off, PROT_READ, MAP_PRIVATE, fd, off);
map = (char *) mmap(NULL, st.st_size - off, PROT_READ, MAP_PRIVATE, fd, off);
// No error if MAP_FAILED because check is below
if (map != MAP_FAILED) {
madvise(map, st.st_size - off, MADV_RANDOM); // sequential, but from several pointers at once
@ -1932,7 +1202,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay
src = sourcelist[i]->file;
}
char *trunc = layernames[i] = malloc(strlen(src) + 1);
char *trunc = layernames[i] = (char *) malloc(strlen(src) + 1);
if (trunc == NULL) {
perror("Out of memory");
exit(EXIT_FAILURE);
@ -2146,7 +1416,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay
}
if (basezoom < 0 || droprate < 0) {
struct index *map = mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0);
struct index *map = (struct index *) mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0);
if (map == MAP_FAILED) {
perror("mmap index for basezoom");
exit(EXIT_FAILURE);
@ -2360,7 +1630,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay
}
if (poolpos > 0) {
madvise(pool, poolpos, MADV_DONTNEED);
madvise((void *) pool, poolpos, MADV_DONTNEED);
if (munmap(stringpool, poolpos) != 0) {
perror("munmap stringpool");
}
@ -2437,7 +1707,7 @@ int read_json(int argc, struct source **sourcelist, char *fname, const char *lay
return ret;
}
int int_in(int v, int *a, int len) {
static int int_in(int v, int *a, int len) {
int i;
for (i = 0; i < len; i++) {
@ -2585,7 +1855,7 @@ int main(int argc, char **argv) {
fprintf(stderr, "%s: -L requires layername:file\n", argv[0]);
exit(EXIT_FAILURE);
}
struct source *src = malloc(sizeof(struct source));
struct source *src = (struct source *) malloc(sizeof(struct source));
if (src == NULL) {
perror("Out of memory");
exit(EXIT_FAILURE);
@ -2830,7 +2100,7 @@ int main(int argc, char **argv) {
int ret = EXIT_SUCCESS;
for (i = optind; i < argc; i++) {
struct source *src = malloc(sizeof(struct source));
struct source *src = (struct source *) malloc(sizeof(struct source));
if (src == NULL) {
perror("Out of memory");
exit(EXIT_FAILURE);
@ -2849,7 +2119,7 @@ int main(int argc, char **argv) {
sourcelist[i--] = sources;
}
ret = read_json(nsources, sourcelist, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel, forcetable, attribution);
ret = read_input(nsources, sourcelist, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel, forcetable, attribution);
mbtiles_close(outdb, argv);

15
main.hpp Normal file

@ -0,0 +1,15 @@
struct index {
long long start;
long long end;
unsigned long long index;
short segment;
unsigned long long seq : (64 - 16); // pack with segment to stay in 32 bytes
};
void checkdisk(struct reader *r, int nreader);
extern int geometry_scale;
extern int quiet;
extern int CPUS;
extern int TEMP_FILES;

17
mbtiles.c → mbtiles.cpp

@ -1,13 +1,16 @@
// for vasprintf() on Linux
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sqlite3.h>
#include "pool.h"
#include "tile.h"
#include "mbtiles.h"
#include <vector>
#include "pool.hpp"
#include "mbtiles.hpp"
#include "geometry.hpp"
sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) {
sqlite3 *outdb;
@ -98,7 +101,7 @@ static void quote(char **buf, const char *s) {
}
*out = '\0';
*buf = realloc(*buf, strlen(*buf) + strlen(tmp) + 1);
*buf = (char *) realloc(*buf, strlen(*buf) + strlen(tmp) + 1);
if (*buf == NULL) {
perror("realloc");
exit(EXIT_FAILURE);
@ -117,7 +120,7 @@ static void aprintf(char **buf, const char *format, ...) {
}
va_end(ap);
*buf = realloc(*buf, strlen(*buf) + strlen(tmp) + 1);
*buf = (char *) realloc(*buf, strlen(*buf) + strlen(tmp) + 1);
if (*buf == NULL) {
perror("Out of memory");
exit(EXIT_FAILURE);
@ -127,8 +130,8 @@ static void aprintf(char **buf, const char *format, ...) {
}
static int pvcmp(const void *v1, const void *v2) {
const struct pool_val *const *pv1 = v1;
const struct pool_val *const *pv2 = v2;
const struct pool_val *const *pv1 = (const struct pool_val *const *) v1;
const struct pool_val *const *pv2 = (const struct pool_val *const *) v2;
int n = strcmp((*pv1)->s, (*pv2)->s);
if (n != 0) {

0
mbtiles.h → mbtiles.hpp

8
memfile.c → memfile.cpp

@ -2,7 +2,7 @@
#include <string.h>
#include <unistd.h>
#include <sys/mman.h>
#include "memfile.h"
#include "memfile.hpp"
#define INCREMENT 131072
@ -11,12 +11,12 @@ struct memfile *memfile_open(int fd) {
return NULL;
}
char *map = mmap(NULL, INCREMENT, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
char *map = (char *) mmap(NULL, INCREMENT, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
return NULL;
}
struct memfile *mf = malloc(sizeof(struct memfile));
struct memfile *mf = (struct memfile *) malloc(sizeof(struct memfile));
if (mf == NULL) {
munmap(map, INCREMENT);
return NULL;
@ -58,7 +58,7 @@ int memfile_write(struct memfile *file, void *s, long long len) {
return -1;
}
file->map = mmap(NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0);
file->map = (char *) mmap(NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0);
if (file->map == MAP_FAILED) {
return -1;
}

0
memfile.h → memfile.hpp

2
mvt.cc → mvt.cpp

@ -4,7 +4,7 @@
#include <vector>
#include <map>
#include <zlib.h>
#include "mvt.hh"
#include "mvt.hpp"
#include "protozero/varint.hpp"
#include "protozero/pbf_reader.hpp"
#include "protozero/pbf_writer.hpp"

0
mvt.hh → mvt.hpp

37
options.hpp Normal file

@ -0,0 +1,37 @@
static int additional_options[] = {
#define A_COALESCE ((int) 'c')
A_COALESCE,
#define A_REVERSE ((int) 'r')
A_REVERSE,
#define A_REORDER ((int) 'o')
A_REORDER,
#define A_LINE_DROP ((int) 'l')
A_LINE_DROP,
#define A_DEBUG_POLYGON ((int) 'd')
A_DEBUG_POLYGON,
#define A_POLYGON_DROP ((int) 'p')
A_POLYGON_DROP,
#define A_PREFER_RADIX_SORT ((int) 'R')
A_PREFER_RADIX_SORT,
};
static int prevent_options[] = {
#define P_SIMPLIFY ((int) 's')
P_SIMPLIFY,
#define P_SIMPLIFY_LOW ((int) 'S')
P_SIMPLIFY_LOW,
#define P_FEATURE_LIMIT ((int) 'f')
P_FEATURE_LIMIT,
#define P_KILOBYTE_LIMIT ((int) 'k')
P_KILOBYTE_LIMIT,
#define P_DYNAMIC_DROP ((int) 'd')
P_DYNAMIC_DROP,
#define P_INPUT_ORDER ((int) 'i')
P_INPUT_ORDER,
#define P_POLYGON_SPLIT ((int) 'p')
P_POLYGON_SPLIT,
#define P_CLIPPING ((int) 'c')
P_CLIPPING,
#define P_DUPLICATION ((int) 'D')
P_DUPLICATION,
};

119
pool.c

@ -1,119 +0,0 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "pool.h"
#define POOL_WIDTH 256
static int hash(const char *s) {
int h = 0;
for (; *s; s++) {
h = h * 37 + *s;
}
h = h & 0xFF;
return h;
}
struct pool_val *pool(struct pool *p, const char *s, int type) {
int h = hash(s);
struct pool_val **v = &(p->vals[h]);
while (*v != NULL) {
int cmp = strcmp(s, (*v)->s);
if (cmp == 0) {
cmp = type - (*v)->type;
}
if (cmp == 0) {
return *v;
} else if (cmp < 0) {
v = &((*v)->left);
} else {
v = &((*v)->right);
}
}
struct pool_val *nv = malloc(sizeof(struct pool_val));
if (nv == NULL) {
fprintf(stderr, "out of memory making string pool\n");
exit(EXIT_FAILURE);
}
nv->left = NULL;
nv->right = NULL;
nv->next = NULL;
nv->s = s;
nv->type = type;
nv->n = p->n++;
if (p->tail != NULL) {
p->tail->next = nv;
}
p->tail = nv;
if (p->head == NULL) {
p->head = nv;
}
*v = nv;
return *v;
}
int is_pooled(struct pool *p, const char *s, int type) {
int h = hash(s);
struct pool_val **v = &(p->vals[h]);
while (*v != NULL) {
int cmp = strcmp(s, (*v)->s);
if (cmp == 0) {
cmp = type - (*v)->type;
}
if (cmp == 0) {
return 1;
} else if (cmp < 0) {
v = &((*v)->left);
} else {
v = &((*v)->right);
}
}
return 0;
}
void pool_free1(struct pool *p, void (*func)(void *)) {
while (p->head != NULL) {
if (func != NULL) {
func((void *) p->head->s);
}
struct pool_val *next = p->head->next;
free(p->head);
p->head = next;
}
p->head = NULL;
p->tail = NULL;
free(p->vals);
p->vals = NULL;
}
void pool_free(struct pool *p) {
pool_free1(p, NULL);
}
void pool_free_strings(struct pool *p) {
pool_free1(p, free);
}
void pool_init(struct pool *p, int n) {
p->n = n;
p->vals = calloc(POOL_WIDTH, sizeof(struct pool_val *));
if (p->vals == NULL) {
fprintf(stderr, "out of memory creating string pool\n");
exit(EXIT_FAILURE);
}
p->head = NULL;
p->tail = NULL;
}

212
pool.cpp Normal file

@ -0,0 +1,212 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "memfile.hpp"
#include "pool.hpp"
#define POOL_WIDTH 256
static int hash(const char *s) {
int h = 0;
for (; *s; s++) {
h = h * 37 + *s;
}
h = h & 0xFF;
return h;
}
struct pool_val *pool(struct pool *p, const char *s, int type) {
int h = hash(s);
struct pool_val **v = &(p->vals[h]);
while (*v != NULL) {
int cmp = strcmp(s, (*v)->s);
if (cmp == 0) {
cmp = type - (*v)->type;
}
if (cmp == 0) {
return *v;
} else if (cmp < 0) {
v = &((*v)->left);
} else {
v = &((*v)->right);
}
}
struct pool_val *nv = (struct pool_val *) malloc(sizeof(struct pool_val));
if (nv == NULL) {
fprintf(stderr, "out of memory making string pool\n");
exit(EXIT_FAILURE);
}
nv->left = NULL;
nv->right = NULL;
nv->next = NULL;
nv->s = s;
nv->type = type;
nv->n = p->n++;
if (p->tail != NULL) {
p->tail->next = nv;
}
p->tail = nv;
if (p->head == NULL) {
p->head = nv;
}
*v = nv;
return *v;
}
int is_pooled(struct pool *p, const char *s, int type) {
int h = hash(s);
struct pool_val **v = &(p->vals[h]);
while (*v != NULL) {
int cmp = strcmp(s, (*v)->s);
if (cmp == 0) {
cmp = type - (*v)->type;
}
if (cmp == 0) {
return 1;
} else if (cmp < 0) {
v = &((*v)->left);
} else {
v = &((*v)->right);
}
}
return 0;
}
void pool_free1(struct pool *p, void (*func)(void *)) {
while (p->head != NULL) {
if (func != NULL) {
func((void *) p->head->s);
}
struct pool_val *next = p->head->next;
free(p->head);
p->head = next;
}
p->head = NULL;
p->tail = NULL;
free(p->vals);
p->vals = NULL;
}
void pool_free(struct pool *p) {
pool_free1(p, NULL);
}
void pool_free_strings(struct pool *p) {
pool_free1(p, free);
}
void pool_init(struct pool *p, int n) {
p->n = n;
p->vals = (struct pool_val **) calloc(POOL_WIDTH, sizeof(struct pool_val *));
if (p->vals == NULL) {
fprintf(stderr, "out of memory creating string pool\n");
exit(EXIT_FAILURE);
}
p->head = NULL;
p->tail = NULL;
}
static unsigned char swizzle[256] = {
0x00, 0xBF, 0x18, 0xDE, 0x93, 0xC9, 0xB1, 0x5E, 0xDF, 0xBE, 0x72, 0x5A, 0xBB, 0x42, 0x64, 0xC6,
0xD8, 0xB7, 0x15, 0x74, 0x1C, 0x8B, 0x91, 0xF5, 0x29, 0x46, 0xEC, 0x6F, 0xCA, 0x20, 0xF0, 0x06,
0x27, 0x61, 0x87, 0xE0, 0x6E, 0x43, 0x50, 0xC5, 0x1B, 0xB4, 0x37, 0xC3, 0x69, 0xA6, 0xEE, 0x80,
0xAF, 0x9B, 0xA1, 0x76, 0x23, 0x24, 0x53, 0xF3, 0x5B, 0x65, 0x19, 0xF4, 0xFC, 0xDD, 0x26, 0xE8,
0x10, 0xF7, 0xCE, 0x92, 0x48, 0xF6, 0x94, 0x60, 0x07, 0xC4, 0xB9, 0x97, 0x6D, 0xA4, 0x11, 0x0D,
0x1F, 0x4D, 0x13, 0xB0, 0x5D, 0xBA, 0x31, 0xD5, 0x8D, 0x51, 0x36, 0x96, 0x7A, 0x03, 0x7F, 0xDA,
0x17, 0xDB, 0xD4, 0x83, 0xE2, 0x79, 0x6A, 0xE1, 0x95, 0x38, 0xFF, 0x28, 0xB2, 0xB3, 0xA7, 0xAE,
0xF8, 0x54, 0xCC, 0xDC, 0x9A, 0x6B, 0xFB, 0x3F, 0xD7, 0xBC, 0x21, 0xC8, 0x71, 0x09, 0x16, 0xAC,
0x3C, 0x8A, 0x62, 0x05, 0xC2, 0x8C, 0x32, 0x4E, 0x35, 0x9C, 0x5F, 0x75, 0xCD, 0x2E, 0xA2, 0x3E,
0x1A, 0xC1, 0x8E, 0x14, 0xA0, 0xD3, 0x7D, 0xD9, 0xEB, 0x5C, 0x70, 0xE6, 0x9E, 0x12, 0x3B, 0xEF,
0x1E, 0x49, 0xD2, 0x98, 0x39, 0x7E, 0x44, 0x4B, 0x6C, 0x88, 0x02, 0x2C, 0xAD, 0xE5, 0x9F, 0x40,
0x7B, 0x4A, 0x3D, 0xA9, 0xAB, 0x0B, 0xD6, 0x2F, 0x90, 0x2A, 0xB6, 0x1D, 0xC7, 0x22, 0x55, 0x34,
0x0A, 0xD0, 0xB5, 0x68, 0xE3, 0x59, 0xFD, 0xFA, 0x57, 0x77, 0x25, 0xA3, 0x04, 0xB8, 0x33, 0x89,
0x78, 0x82, 0xE4, 0xC0, 0x0E, 0x8F, 0x85, 0xD1, 0x84, 0x08, 0x67, 0x47, 0x9D, 0xCB, 0x58, 0x4C,
0xAA, 0xED, 0x52, 0xF2, 0x4F, 0xF1, 0x66, 0xCF, 0xA5, 0x56, 0xEA, 0x7C, 0xE9, 0x63, 0xE7, 0x01,
0xF9, 0xFE, 0x0C, 0x99, 0x2D, 0x0F, 0x3A, 0x41, 0x45, 0xA8, 0x30, 0x2B, 0x73, 0xBD, 0x86, 0x81,
};
int swizzlecmp(const char *a, const char *b) {
while (*a || *b) {
int aa = swizzle[(unsigned char) *a];
int bb = swizzle[(unsigned char) *b];
int cmp = aa - bb;
if (cmp != 0) {
return cmp;
}
a++;
b++;
}
return 0;
}
long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type) {
long long *sp = &treefile->tree;
while (*sp != 0) {
int cmp = swizzlecmp(s, poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off + 1);
if (cmp == 0) {
cmp = type - (poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off)[0];
}
if (cmp < 0) {
sp = &(((struct stringpool *) (treefile->map + *sp))->left);
} else if (cmp > 0) {
sp = &(((struct stringpool *) (treefile->map + *sp))->right);
} else {
return ((struct stringpool *) (treefile->map + *sp))->off;
}
}
// *sp is probably in the memory-mapped file, and will move if the file grows.
long long ssp;
if (sp == &treefile->tree) {
ssp = -1;
} else {
ssp = ((char *) sp) - treefile->map;
}
long long off = poolfile->off;
if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write");
exit(EXIT_FAILURE);
}
if (memfile_write(poolfile, (void *) s, strlen(s) + 1) < 0) {
perror("memfile write");
exit(EXIT_FAILURE);
}
struct stringpool tsp;
tsp.left = 0;
tsp.right = 0;
tsp.off = off;
long long p = treefile->off;
if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) {
perror("memfile write");
exit(EXIT_FAILURE);
}
if (ssp == -1) {
treefile->tree = p;
} else {
*((long long *) (treefile->map + ssp)) = p;
}
return off;
}

8
pool.h → pool.hpp

@ -22,3 +22,11 @@ void pool_free(struct pool *p);
void pool_free_strings(struct pool *p);
void pool_init(struct pool *p, int n);
int is_pooled(struct pool *p, const char *s, int type);
struct stringpool {
long long left;
long long right;
long long off;
};
long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type);

2
projection.c → projection.cpp

@ -1,5 +1,5 @@
#include <math.h>
#include "projection.h"
#include "projection.hpp"
// http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames
void latlon2tile(double lat, double lon, int zoom, long long *x, long long *y) {

0
projection.h → projection.hpp

139
serial.cpp Normal file

@ -0,0 +1,139 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include "protozero/varint.hpp"
#include "serial.hpp"
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) {
size_t w = fwrite(ptr, size, nitems, stream);
if (w != nitems) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_FAILURE);
}
return w;
}
void serialize_int(FILE *out, int n, long long *fpos, const char *fname) {
serialize_long_long(out, n, fpos, fname);
}
void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname) {
unsigned long long zigzag = protozero::encode_zigzag32(n);
while (1) {
unsigned char b = zigzag & 0x7F;
if ((zigzag >> 7) != 0) {
b |= 0x80;
if (putc(b, out) == EOF) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_FAILURE);
}
*fpos += 1;
zigzag >>= 7;
} else {
if (putc(b, out) == EOF) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_FAILURE);
}
*fpos += 1;
break;
}
}
}
void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname) {
fwrite_check(&n, sizeof(signed char), 1, out, fname);
*fpos += sizeof(signed char);
}
void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname) {
fwrite_check(&n, sizeof(unsigned), 1, out, fname);
*fpos += sizeof(unsigned);
}
void deserialize_int(char **f, int *n) {
long long ll;
deserialize_long_long(f, &ll);
*n = ll;
}
void deserialize_long_long(char **f, long long *n) {
unsigned long long zigzag = 0;
int shift = 0;
while (1) {
if ((**f & 0x80) == 0) {
zigzag |= ((unsigned long long) **f) << shift;
*f += 1;
shift += 7;
break;
} else {
zigzag |= ((unsigned long long) (**f & 0x7F)) << shift;
*f += 1;
shift += 7;
}
}
*n = protozero::decode_zigzag32(zigzag);
}
void deserialize_uint(char **f, unsigned *n) {
memcpy(n, *f, sizeof(unsigned));
*f += sizeof(unsigned);
}
void deserialize_byte(char **f, signed char *n) {
memcpy(n, *f, sizeof(signed char));
*f += sizeof(signed char);
}
int deserialize_long_long_io(FILE *f, long long *n, long long *geompos) {
unsigned long long zigzag = 0;
int shift = 0;
while (1) {
int c = getc(f);
if (c == EOF) {
return 0;
}
(*geompos)++;
if ((c & 0x80) == 0) {
zigzag |= ((unsigned long long) c) << shift;
shift += 7;
break;
} else {
zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
shift += 7;
}
}
*n = protozero::decode_zigzag32(zigzag);
return 1;
}
int deserialize_int_io(FILE *f, int *n, long long *geompos) {
long long ll = 0;
int ret = deserialize_long_long_io(f, &ll, geompos);
*n = ll;
return ret;
}
int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos) {
if (fread(n, sizeof(unsigned), 1, f) != 1) {
return 0;
}
*geompos += sizeof(unsigned);
return 1;
}
int deserialize_byte_io(FILE *f, signed char *n, long long *geompos) {
int c = getc(f);
if (c == EOF) {
return 0;
}
*n = c;
(*geompos)++;
return 1;
}

17
serial.hpp Normal file

@ -0,0 +1,17 @@
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname);
void serialize_int(FILE *out, int n, long long *fpos, const char *fname);
void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname);
void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname);
void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname);
void serialize_string(FILE *out, const char *s, long long *fpos, const char *fname);
void deserialize_int(char **f, int *n);
void deserialize_long_long(char **f, long long *n);
void deserialize_uint(char **f, unsigned *n);
void deserialize_byte(char **f, signed char *n);
int deserialize_int_io(FILE *f, int *n, long long *geompos);
int deserialize_long_long_io(FILE *f, long long *n, long long *geompos);
int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos);
int deserialize_byte_io(FILE *f, signed char *n, long long *geompos);

13
tile-join.cc → tile-join.cpp

@ -9,14 +9,11 @@
#include <map>
#include <zlib.h>
#include <math.h>
#include "mvt.hh"
#include "tile.h"
extern "C" {
#include "projection.h"
#include "pool.h"
#include "mbtiles.h"
}
#include "mvt.hpp"
#include "projection.hpp"
#include "pool.hpp"
#include "mbtiles.hpp"
#include "geometry.hpp"
std::string dequote(std::string s);

19
tile.cc → tile.cpp

@ -19,16 +19,15 @@
#include <sqlite3.h>
#include <pthread.h>
#include <errno.h>
#include "mvt.hh"
#include "geometry.hh"
extern "C" {
#include "tile.h"
#include "pool.h"
#include "clip.h"
#include "mbtiles.h"
#include "projection.h"
}
#include "mvt.hpp"
#include "geometry.hpp"
#include "tile.hpp"
#include "pool.hpp"
#include "mbtiles.hpp"
#include "projection.hpp"
#include "serial.hpp"
#include "options.hpp"
#include "main.hpp"
#define CMD_BITS 3

80
tile.h

@ -1,80 +0,0 @@
#define VT_POINT 1
#define VT_LINE 2
#define VT_POLYGON 3
#define VT_END 0
#define VT_MOVETO 1
#define VT_LINETO 2
#define VT_CLOSEPATH 7
#define VT_STRING 1
#define VT_NUMBER 2
#define VT_BOOLEAN 7
struct pool;
void serialize_int(FILE *out, int n, long long *fpos, const char *fname);
void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fname);
void serialize_byte(FILE *out, signed char n, long long *fpos, const char *fname);
void serialize_uint(FILE *out, unsigned n, long long *fpos, const char *fname);
void serialize_string(FILE *out, const char *s, long long *fpos, const char *fname);
void deserialize_int(char **f, int *n);
void deserialize_long_long(char **f, long long *n);
void deserialize_uint(char **f, unsigned *n);
void deserialize_byte(char **f, signed char *n);
int deserialize_int_io(FILE *f, int *n, long long *geompos);
int deserialize_long_long_io(FILE *f, long long *n, long long *geompos);
int deserialize_uint_io(FILE *f, unsigned *n, long long *geompos);
int deserialize_byte_io(FILE *f, signed char *n, long long *geompos);
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, int *prevent, int *additional);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);
extern int geometry_scale;
extern int quiet;
extern int CPUS;
extern int TEMP_FILES;
static int additional_options[] = {
#define A_COALESCE ((int) 'c')
A_COALESCE,
#define A_REVERSE ((int) 'r')
A_REVERSE,
#define A_REORDER ((int) 'o')
A_REORDER,
#define A_LINE_DROP ((int) 'l')
A_LINE_DROP,
#define A_DEBUG_POLYGON ((int) 'd')
A_DEBUG_POLYGON,
#define A_POLYGON_DROP ((int) 'p')
A_POLYGON_DROP,
#define A_PREFER_RADIX_SORT ((int) 'R')
A_PREFER_RADIX_SORT,
};
static int prevent_options[] = {
#define P_SIMPLIFY ((int) 's')
P_SIMPLIFY,
#define P_SIMPLIFY_LOW ((int) 'S')
P_SIMPLIFY_LOW,
#define P_FEATURE_LIMIT ((int) 'f')
P_FEATURE_LIMIT,
#define P_KILOBYTE_LIMIT ((int) 'k')
P_KILOBYTE_LIMIT,
#define P_DYNAMIC_DROP ((int) 'd')
P_DYNAMIC_DROP,
#define P_INPUT_ORDER ((int) 'i')
P_INPUT_ORDER,
#define P_POLYGON_SPLIT ((int) 'p')
P_POLYGON_SPLIT,
#define P_CLIPPING ((int) 'c')
P_CLIPPING,
#define P_DUPLICATION ((int) 'D')
P_DUPLICATION,
};

5
tile.hpp Normal file

@ -0,0 +1,5 @@
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, int *prevent, int *additional);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);

1
version.h

@ -1 +0,0 @@
#define VERSION "tippecanoe v1.10.0\n"

1
version.hpp Normal file

@ -0,0 +1 @@
#define VERSION "tippecanoe v1.11.0\n"