Compare commits

..

2 Commits

6 changed files with 85 additions and 347 deletions

View File

@ -37,6 +37,7 @@ Options
* -f: Delete the mbtiles file if it already exists instead of giving an error
* -r <i>rate</i>: Rate at which dots are dropped at lower zoom levels (default 2.5)
* -b <i>pixels</i>: Buffer size where features are duplicated from adjacent tiles (default 5)
* -m <i>max</i>: Drop additional points at the same location after <i>max</i> points overlap
Example
-------

333
geojson.c
View File

@ -58,318 +58,6 @@ int mb_geometry[GEOM_TYPES] = {
VT_POLYGON,
};
struct feature_attribute {
int type; /* JSON_STRING, JSON_NUMBER, JSON_TRUE, JSON_FALSE, JSON_NULL */
char *key;
char *value;
};
struct feature_geometry {
int op; /* VT_MOVETO, VT_LINETO, VT_CLOSEPATH */
double lat;
double lon;
struct feature_geometry *next;
};
struct feature {
int type; /* VT_POINT, VT_LINE, VT_POLYGON */
int nattributes;
struct feature_attribute *attributes;
struct feature_geometry *geometries;
};
void geo_free(struct feature *geo) {
int i;
for (i = 0; i < geo->nattributes; i++) {
free(geo->attributes[i].key);
free(geo->attributes[i].value);
}
free(geo->attributes);
while (geo->geometries != NULL) {
struct feature_geometry *next = geo->geometries->next;
free(geo->geometries);
geo->geometries = next;
}
free(geo);
}
void geo_print(struct feature *geo) {
printf("{ \"type\": \"Feature\", \"properties\": {");
int i;
for (i = 0; i < geo->nattributes; i++) {
if (i != 0) {
printf(", ");
}
printf("\"%s\": \"%s\"", geo->attributes[i].key, geo->attributes[i].value);
}
printf("}, \"geometry\": { \"type\": \"%s\", \"coordinates\": [",
geo->type == VT_POLYGON ? "MultiPolygon" :
geo->type == VT_LINE ? "MultiLineString" :
geo->type == VT_POINT ? "MultiPoint" :
"???");
struct feature_geometry *g;
if (geo->type == VT_POINT) {
for (g = geo->geometries; g != NULL; g = g->next) {
printf(" [ %f, %f ]", g->lon, g->lat);
if (g->next != NULL) {
printf(",");
}
}
} else if (geo->type == VT_LINE) {
printf(" [");
for (g = geo->geometries; g != NULL; g = g->next) {
printf(" [ %f, %f ]", g->lon, g->lat);
if (g->next != NULL && g->next->op == VT_MOVETO) {
printf(" ], [");
} else if (g->next != NULL) {
printf(",");
}
}
printf(" ] ");
} else if (geo->type == VT_POLYGON) {
printf(" [ [");
for (g = geo->geometries; g != NULL; g = g->next) {
if (g->op == VT_CLOSEPATH) {
if (g->next != NULL) {
printf(" ], [");
}
} else {
printf(" [ %f, %f ]", g->lon, g->lat);
if (g->next != NULL && g->next->op == VT_MOVETO) {
printf(" ], [");
} else if (g->next != NULL && g->next->op != VT_CLOSEPATH) {
printf(",");
}
}
}
printf("] ] ");
}
printf("] } }\n");
}
struct feature_geometry **geo_parse(int t, json_object *j, int op, const char *fname, int line, struct feature_geometry **geom) {
if (j == NULL || j->type != JSON_ARRAY) {
fprintf(stderr, "%s:%d: expected array for type %d\n", fname, line, t);
return geom;
}
int within = geometry_within[t];
struct feature_geometry **began = geom;
if (within >= 0) {
int i;
for (i = 0; i < j->length; i++) {
if (within == GEOM_POINT) {
if (i == 0 || mb_geometry[t] == GEOM_MULTIPOINT) {
op = VT_MOVETO;
} else {
op = VT_LINETO;
}
}
geom = geo_parse(within, j->array[i], op, fname, line, geom);
}
} else {
if (j->length >= 2 && j->array[0]->type == JSON_NUMBER && j->array[1]->type == JSON_NUMBER) {
unsigned x, y;
double lon = j->array[0]->number;
double lat = j->array[1]->number;
latlon2tile(lat, lon, 32, &x, &y);
if (j->length > 2) {
static int warned = 0;
if (!warned) {
fprintf(stderr, "%s:%d: ignoring dimensions beyond two\n", fname, line);
warned = 1;
}
}
*geom = malloc(sizeof(struct feature_geometry));
(*geom)->op = op;
(*geom)->lat = lat;
(*geom)->lon = lon;
(*geom)->next = NULL;
geom = &((*geom)->next);
} else {
fprintf(stderr, "%s:%d: malformed point\n", fname, line);
}
}
if (t == GEOM_POLYGON) {
if (geom != began) {
*geom = malloc(sizeof(struct feature_geometry));
(*geom)->op = VT_CLOSEPATH;
(*geom)->lat = 0;
(*geom)->lon = 0;
(*geom)->next = NULL;
geom = &((*geom)->next);
}
}
return geom;
}
struct feature *parse_feature(json_object *geometry, json_object *properties, const char *fname, int line) {
json_object *geometry_type = json_hash_get(geometry, "type");
if (geometry_type == NULL) {
static int warned = 0;
if (!warned) {
fprintf(stderr, "%s:%d: null geometry (additional not reported)\n", fname, line);
warned = 1;
}
return NULL;
}
if (geometry_type->type != JSON_STRING) {
fprintf(stderr, "%s:%d: geometry without type\n", fname, line);
return NULL;
}
if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
fprintf(stderr, "%s:%d: feature without properties hash\n", fname, line);
return NULL;
}
json_object *coordinates = json_hash_get(geometry, "coordinates");
if (coordinates == NULL || coordinates->type != JSON_ARRAY) {
fprintf(stderr, "%s:%d: feature without coordinates array\n", fname, line);
return NULL;
}
int t;
for (t = 0; t < GEOM_TYPES; t++) {
if (strcmp(geometry_type->string, geometry_names[t]) == 0) {
break;
}
}
if (t >= GEOM_TYPES) {
fprintf(stderr, "%s:%d: Can't handle geometry type %s\n", fname, line, geometry_type->string);
return NULL;
}
struct feature *g = malloc(sizeof(struct feature));
g->type = mb_geometry[t];
g->attributes = NULL;
g->geometries = NULL;
g->nattributes = 0;
int nprop = 0;
if (properties->type == JSON_HASH) {
nprop = properties->length;
}
g->attributes = malloc(nprop * sizeof(struct feature_attribute));
int m = 0;
int i;
for (i = 0; i < nprop; i++) {
if (properties->keys[i]->type == JSON_STRING) {
g->attributes[m].key = strdup(properties->keys[i]->string);
if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) {
g->attributes[m].type = VT_STRING;
g->attributes[m].value = strdup(properties->values[i]->string);
m++;
} else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) {
g->attributes[m].type = VT_NUMBER;
g->attributes[m].value = strdup(properties->values[i]->string);
m++;
} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) {
g->attributes[m].type = VT_BOOLEAN;
g->attributes[m].value = strdup(properties->values[i]->string);
m++;
} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) {
;
} else {
fprintf(stderr, "%s:%d: Unsupported property type for %s\n", fname, line, properties->keys[i]->string);
continue;
}
}
}
g->nattributes = m;
geo_parse(t, coordinates, VT_MOVETO, fname, line, &(g->geometries));
return g;
}
/* XXX check for any non-features in the outer object */
/*
if (bbox != NULL) {
if (x < bbox[0]) {
bbox[0] = x;
}
if (y < bbox[1]) {
bbox[1] = y;
}
if (x > bbox[2]) {
bbox[2] = x;
}
if (y > bbox[3]) {
bbox[3] = y;
}
}
*/
void parse_outer(FILE *f, char *fname) {
json_pull *jp = json_begin_file(f);
long long seq = 0;
while (1) {
json_object *j = json_read(jp);
if (j == NULL) {
if (jp->error != NULL) {
fprintf(stderr, "%s:%d: %s\n", fname, jp->line, jp->error);
}
json_free(jp->root);
break;
}
json_object *type = json_hash_get(j, "type");
json_object *geometry = json_hash_get(j, "geometry");
json_object *properties = json_hash_get(j, "properties");
if (type != NULL && type->type == JSON_STRING && strcmp(type->string, "Feature") == 0 &&
geometry != NULL && geometry->type == JSON_HASH &&
properties != NULL && properties->type == JSON_HASH) {
struct feature *g = parse_feature(geometry, properties, fname, jp->line);
json_free(j);
geo_print(g);
geo_free(g);
if (seq % 10000 == 0) {
fprintf(stderr, "Read %.2f million features\r", seq / 1000000.0);
}
seq++;
}
}
json_end(jp);
}
int main(int argc, char **argv) {
parse_outer(stdin, "standard input");
}
#define main xmain
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname, json_pull *source) {
size_t w = fwrite(ptr, size, nitems, stream);
if (w != nitems) {
@ -506,7 +194,7 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type) {
return ret;
}
void traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, const char *tmpdir) {
void traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, unsigned *file_bbox, struct pool *file_keys, unsigned *midx, unsigned *midy, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, const char *tmpdir, int maxdup) {
int i;
for (i = 0; i <= maxzoom; i++) {
long long most = 0;
@ -567,7 +255,7 @@ void traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, unsigned
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub, minzoom, maxzoom, todo, geomstart, along);
long long len = write_tile(&geom, metabase, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, maxzoom, file_keys, layername, outdb, droprate, buffer, fname, jp, sub, minzoom, maxzoom, todo, geomstart, along, maxdup);
if (z == maxzoom && len > most) {
*midx = x;
@ -600,7 +288,7 @@ void traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, unsigned
fprintf(stderr, "\n");
}
void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir) {
void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, int maxdup) {
char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1];
char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1];
@ -903,7 +591,7 @@ void read_json(FILE *f, const char *fname, const char *layername, int maxzoom, i
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata\n", seq, (long long) geomst.st_size, (long long) metast.st_size);
traverse_zooms(fd, size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir);
traverse_zooms(fd, size, meta, file_bbox, &file_keys, &midx, &midy, layername, maxzoom, minzoom, outdb, droprate, buffer, fname, jp, tmpdir, maxdup);
if (munmap(meta, metast.st_size) != 0) {
perror("munmap meta");
@ -956,13 +644,14 @@ int main(int argc, char **argv) {
double droprate = 2.5;
int buffer = 5;
const char *tmpdir = "/tmp";
int maxdup = INT_MAX;
struct pool exclude, include;
pool_init(&exclude, 0);
pool_init(&include, 0);
int exclude_all = 0;
while ((i = getopt(argc, argv, "l:n:z:Z:d:D:o:x:y:r:b:fXt:")) != -1) {
while ((i = getopt(argc, argv, "l:n:z:Z:d:D:o:x:y:r:b:fXt:m:")) != -1) {
switch (i) {
case 'n':
name = optarg;
@ -1021,8 +710,12 @@ int main(int argc, char **argv) {
tmpdir = optarg;
break;
case 'm':
maxdup = atoi(optarg);
break;
default:
fprintf(stderr, "Usage: %s -o out.mbtiles [-n name] [-l layername] [-z maxzoom] [-Z minzoom] [-d detail] [-D lower-detail] [-x excluded-field ...] [-y included-field ...] [-X] [-r droprate] [-b buffer] [-t tmpdir] [file.json]\n", argv[0]);
fprintf(stderr, "Usage: %s -o out.mbtiles [-n name] [-l layername] [-z maxzoom] [-Z minzoom] [-d detail] [-D lower-detail] [-x excluded-field ...] [-y included-field ...] [-X] [-r droprate] [-b buffer] [-t tmpdir] [-m max-overlap] [file.json]\n", argv[0]);
exit(EXIT_FAILURE);
}
}
@ -1052,7 +745,7 @@ int main(int argc, char **argv) {
if (f == NULL) {
fprintf(stderr, "%s: %s: %s\n", argv[0], argv[i], strerror(errno));
} else {
read_json(f, name ? name : argv[i], layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir);
read_json(f, name ? name : argv[i], layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, maxdup);
fclose(f);
}
}
@ -1060,7 +753,7 @@ int main(int argc, char **argv) {
fprintf(stderr, "%s: Only accepts one input file\n", argv[0]);
exit(EXIT_FAILURE);
} else {
read_json(stdin, name ? name : outdir, layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir);
read_json(stdin, name ? name : outdir, layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, maxdup);
}
mbtiles_close(outdb, argv);

View File

@ -661,3 +661,28 @@ drawvec reorder_lines(drawvec &geom) {
return geom;
}
drawvec trim_dup_points(drawvec &geom, int *grid, int size, int max) {
drawvec out;
unsigned i;
for (i = 0; i < geom.size(); i++) {
if (geom[i].op == VT_MOVETO || geom[i].op == VT_LINETO) {
long long x = geom[i].x + size / 4;
long long y = geom[i].y + size / 4;
if (x < 0 || y < 0 || x >= size || y >= size) {
out.push_back(geom[i]);
} else {
if (grid[y * size + x] < max) {
grid[y * size + x]++;
out.push_back(geom[i]);
}
}
} else {
out.push_back(geom[i]);
}
}
return out;
}

View File

@ -25,3 +25,4 @@ drawvec clip_lines(drawvec &geom, int z, int detail, long long buffer);
int quick_check(long long *bbox, int z, int detail, long long buffer);
drawvec simplify_lines(drawvec &geom, int z, int detail);
drawvec reorder_lines(drawvec &geom);
drawvec trim_dup_points(drawvec &geom, int *grid, int size, int max);

70
tile.cc
View File

@ -342,7 +342,7 @@ void evaluate(std::vector<coalesce> &features, char *metabase, struct pool *file
pool_free(&keys);
}
long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along) {
long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, int maxdup) {
int line_detail;
static bool evaluated = false;
double oprogress = 0;
@ -366,6 +366,13 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
int within[4] = { 0 };
long long geompos[4] = { 0 };
int *dupcount = (int *) malloc((2 << line_detail) * (2 << line_detail) * sizeof(int));
if (dupcount == NULL) {
perror("memory allocation for duplicates");
exit(EXIT_FAILURE);
}
memset(dupcount, '\0', (2 << line_detail) * (2 << line_detail) * sizeof(int));
*geoms = og;
while (1) {
@ -503,6 +510,10 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
to_tile_scale(geom, z, line_detail);
if (t == VT_POINT) {
geom = trim_dup_points(geom, dupcount, 2 << line_detail, maxdup);
}
if (t == VT_POINT || to_feature(geom, NULL)) {
struct coalesce c;
@ -526,7 +537,9 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
c.coalesced = false;
decode_meta(&meta, &keys, &values, file_keys, &c.meta, NULL);
features.push_back(c);
if (geom.size() > 0) {
features.push_back(c);
}
}
}
@ -538,6 +551,8 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
}
}
free(dupcount);
std::sort(features.begin(), features.end());
std::vector<coalesce> out;
@ -570,32 +585,35 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
if (features.size() > 0) {
if (features.size() > 200000) {
fprintf(stderr, "tile %d/%u/%u has %lld features, >200000 \n", z, tx, ty, (long long) features.size());
fprintf(stderr, "Try using -z to set a higher base zoom level.\n");
exit(EXIT_FAILURE);
}
fprintf(stderr, "tile %d/%u/%u has %lld features with detail %d, >200000 \n", z, tx, ty, (long long) features.size(), line_detail);
fprintf(stderr, "Try using -z to set a higher base zoom level or lower -m to eliminate duplicate points.\n");
mapnik::vector::tile tile = create_tile(layername, line_detail, features, &count, &keys, &values);
pool_free(&keys);
pool_free(&values);
std::string s;
std::string compressed;
tile.SerializeToString(&s);
compress(s, compressed);
if (compressed.size() > 500000) {
fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >500000 \n", z, tx, ty, (long long) compressed.size(), line_detail);
if (line_detail == MIN_DETAIL || !evaluated) {
evaluated = true;
evaluate(features, metabase, file_keys, layername, line_detail, compressed.size());
}
// Don't return, to try the next smaller grid size
} else {
mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
return count;
mapnik::vector::tile tile = create_tile(layername, line_detail, features, &count, &keys, &values);
pool_free(&keys);
pool_free(&values);
std::string s;
std::string compressed;
tile.SerializeToString(&s);
compress(s, compressed);
if (compressed.size() > 500000) {
fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >500000 \n", z, tx, ty, (long long) compressed.size(), line_detail);
if (line_detail == MIN_DETAIL || !evaluated) {
evaluated = true;
evaluate(features, metabase, file_keys, layername, line_detail, compressed.size());
}
// Don't return, to try the next smaller grid size
} else {
mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
return count;
}
}
} else {
return count;

2
tile.h
View File

@ -26,4 +26,4 @@ void deserialize_uint(char **f, unsigned *n);
void deserialize_byte(char **f, signed char *n);
struct pool_val *deserialize_string(char **f, struct pool *p, int type);
long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along);
long long write_tile(char **geom, char *metabase, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int basezoom, struct pool *file_keys, const char *layername, sqlite3 *outdb, double droprate, int buffer, const char *fname, struct json_pull *jp, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, int maxdup);