From a5b1378d1a1cd278a0fcdf87a9b8cfb3fca6019c Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 30 Aug 2017 15:25:47 -0700 Subject: [PATCH] Minimize external dependencies for Geobuf testing --- .travis.yml | 3 +- Makefile | 11 ++- README.md | 5 +- geojson2nd.cpp | 153 ++++++++++++++++++++++++++++++++++++ main.cpp | 2 +- man/tippecanoe.1 | 6 +- tests/geometry/onebare.json | 1 + tests/geometry/out/-z3.json | 24 +++++- 8 files changed, 193 insertions(+), 12 deletions(-) create mode 100644 geojson2nd.cpp create mode 100644 tests/geometry/onebare.json diff --git a/.travis.yml b/.travis.yml index 6361da2..d1b389f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,7 +68,8 @@ install: - BUILDTYPE=${BUILDTYPE} make -j script: - - BUILDTYPE=${BUILDTYPE} make test + - npm install geobuf + - BUILDTYPE=${BUILDTYPE} make test geobuf-test - if [ -n "${COVERAGE}" ]; then /usr/bin/llvm-cov-3.5 -lp *.o; pip install --user cpp-coveralls; diff --git a/Makefile b/Makefile index b309f58..ebb8d40 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ else FINAL_FLAGS := -g $(WARNING_FLAGS) $(DEBUG_FLAGS) endif -all: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit +all: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit geojson2nd docs: man/tippecanoe.1 @@ -58,6 +58,9 @@ tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread +geojson2nd: geojson2nd.o jsonpull/jsonpull.o + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + unit: unit.o text.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread @@ -88,12 +91,12 @@ test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test p cmp $@.out $(patsubst %.check,%,$@) rm $@.out $@.mbtiles -geobuf-test: $(addsuffix .checkbuf,$(TESTS)) +geobuf-test: geojson2nd $(addsuffix .checkbuf,$(TESTS)) # XXX Use proper makefile rules instead of a for loop %.json.checkbuf: - for i in $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json); do ./tests/fc-wrap $$i | json2geobuf > $$i.pbf; done - ./tippecanoe -aD -f -o $@.mbtiles $(subst @,:,$(subst %,/,$(subst _, ,$(patsubst %.json.checkbuf,%,$(word 4,$(subst /, ,$@)))))) $(addsuffix .pbf,$(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json)) < /dev/null + for i in $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json); do ./geojson2nd -w $$i | json2geobuf > $$i.geobuf; done + ./tippecanoe -aD -f -o $@.mbtiles $(subst @,:,$(subst %,/,$(subst _, ,$(patsubst %.json.checkbuf,%,$(word 4,$(subst /, ,$@)))))) $(addsuffix .geobuf,$(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json)) < /dev/null ./tippecanoe-decode $@.mbtiles | sed 's/checkbuf/check/g' > $@.out cmp $@.out $(patsubst %.checkbuf,%,$@) rm $@.out $@.mbtiles diff --git a/README.md b/README.md index 1f3e631..97f9dc6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ tippecanoe ========== -Builds [vector tilesets](https://www.mapbox.com/developers/vector-tiles/) from large (or small) collections of [GeoJSON](http://geojson.org/) features, +Builds [vector tilesets](https://www.mapbox.com/developers/vector-tiles/) from large (or small) collections of [GeoJSON](http://geojson.org/) or [Geobuf](https://github.com/mapbox/geobuf) features, [like these](MADE_WITH.md). [![Build Status](https://travis-ci.org/mapbox/tippecanoe.svg)](https://travis-ci.org/mapbox/tippecanoe) @@ -41,7 +41,7 @@ Usage ----- ```sh -$ tippecanoe -o file.mbtiles [file.json ...] +$ tippecanoe -o file.mbtiles [file.json file.geobuf ...] ``` If no files are specified, it reads GeoJSON from the standard input. @@ -109,6 +109,7 @@ If your input is formatted as newline-delimited GeoJSON, use `-P` to make input ### Input files and layer names * _name_`.json` or _name_`.geojson`: Read the named GeoJSON input file into a layer called _name_. + * _name_`.geobuf` or _name_`.geobuf`: Read the named Geobuf input file into a layer called _name_. * `-l` _name_ or `--layer=`_name_: Use the specified layer name instead of deriving a name from the input filename or output tileset. If there are multiple input files specified, the files are all merged into the single named layer, even if they try to specify individual names with `-L`. * `-L` _name_`:`_file.json_ or `--named-layer=`_name_`:`_file.json_: Specify layer names for individual files. If your shell supports it, you can use a subshell redirect like `-L` _name_`:<(cat dir/*.json)` to specify a layer name for the output of streamed input. diff --git a/geojson2nd.cpp b/geojson2nd.cpp new file mode 100644 index 0000000..09be263 --- /dev/null +++ b/geojson2nd.cpp @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include +#include +#include +#include "jsonpull.h" + +int fail = EXIT_SUCCESS; +bool wrap = false; + +std::string buffered; +int buffered_type = -1; +// 0: nothing yet +// 1: buffered a line +// 2: wrote the line and the wrapper +int buffer_state = 0; + +void out(std::string s, int type) { + if (!wrap) { + printf("%s\n", s.c_str()); + return; + } + + if (buffer_state == 0) { + buffered = s; + buffered_type = type; + buffer_state = 1; + return; + } + + if (buffer_state == 1) { + if (buffered_type == 1) { + printf("{\"type\":\"FeatureCollection\",\"features\":[\n"); + } else { + printf("{\"type\":\"GeometryCollection\",\"geometries\":[\n"); + } + + printf("%s\n", buffered.c_str()); + buffer_state = 2; + } + + printf(",\n%s\n", s.c_str()); + + if (type != buffered_type) { + fprintf(stderr, "Error: mix of bare geometries and features\n"); + exit(EXIT_FAILURE); + } +} + +void process(FILE *fp, const char *fname) { + json_pull *jp = json_begin_file(fp); + + while (1) { + json_object *j = json_read(jp); + if (j == NULL) { + if (jp->error != NULL) { + fprintf(stderr, "%s:%d: %s\n", fname, jp->line, jp->error); + } + + json_free(jp->root); + break; + } + + json_object *type = json_hash_get(j, "type"); + if (type == NULL || type->type != JSON_STRING) { + continue; + } + + if (strcmp(type->string, "Feature") == 0) { + char *s = json_stringify(j); + out(s, 1); + free(s); + json_free(j); + } else if (strcmp(type->string, "Point") == 0 || + strcmp(type->string, "MultiPoint") == 0 || + strcmp(type->string, "LineString") == 0 || + strcmp(type->string, "MultiLineString") == 0 || + strcmp(type->string, "MultiPolygon") == 0) { + int is_geometry = 1; + + if (j->parent != NULL) { + if (j->parent->type == JSON_ARRAY) { + if (j->parent->parent->type == JSON_HASH) { + json_object *geometries = json_hash_get(j->parent->parent, "geometries"); + if (geometries != NULL) { + // Parent of Parent must be a GeometryCollection + is_geometry = 0; + } + } + } else if (j->parent->type == JSON_HASH) { + json_object *geometry = json_hash_get(j->parent, "geometry"); + if (geometry != NULL) { + // Parent must be a Feature + is_geometry = 0; + } + } + } + + if (is_geometry) { + char *s = json_stringify(j); + out(s, 2); + free(s); + json_free(j); + } + } else if (strcmp(type->string, "FeatureCollection") == 0) { + json_free(j); + } + } + + json_end(jp); +} + +int main(int argc, char **argv) { + extern int optind; + int i; + + while ((i = getopt(argc, argv, "w")) != -1) { + switch (i) { + case 'w': + wrap = true; + break; + + default: + fprintf(stderr, "Unexpected option -%c\n", i); + exit(EXIT_FAILURE); + } + } + + if (optind >= argc) { + process(stdin, "standard input"); + } else { + for (i = optind; i < argc; i++) { + FILE *f = fopen(argv[i], "r"); + if (f == NULL) { + perror(argv[i]); + exit(EXIT_FAILURE); + } + + process(f, argv[i]); + fclose(f); + } + } + + if (buffer_state == 1) { + printf("%s\n", buffered.c_str()); + } else if (buffer_state == 2) { + printf("]}\n"); + } + + return fail; +} diff --git a/main.cpp b/main.cpp index 72c6062..c17a328 100644 --- a/main.cpp +++ b/main.cpp @@ -1193,7 +1193,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } size_t layer = a->second.id; - if (sources[source].file.size() > 4 && sources[source].file.substr(sources[source].file.size() - 4) == std::string(".pbf")) { + if (sources[source].file.size() > 7 && sources[source].file.substr(sources[source].file.size() - 7) == std::string(".geobuf")) { struct stat st; if (fstat(fd, &st) != 0) { perror("fstat"); diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 7bea721..e435732 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -1,6 +1,6 @@ .TH tippecanoe .PP -Builds vector tilesets \[la]https://www.mapbox.com/developers/vector-tiles/\[ra] from large (or small) collections of GeoJSON \[la]http://geojson.org/\[ra] features, +Builds vector tilesets \[la]https://www.mapbox.com/developers/vector-tiles/\[ra] from large (or small) collections of GeoJSON \[la]http://geojson.org/\[ra] or Geobuf \[la]https://github.com/mapbox/geobuf\[ra] features, like these \[la]MADE_WITH.md\[ra]\&. .PP [Build Status](https://travis\-ci.org/mapbox/tippecanoe.svg) \[la]https://travis-ci.org/mapbox/tippecanoe\[ra] @@ -37,7 +37,7 @@ $ brew install tippecanoe .PP .RS .nf -$ tippecanoe \-o file.mbtiles [file.json ...] +$ tippecanoe \-o file.mbtiles [file.json file.geobuf ...] .fi .RE .PP @@ -112,6 +112,8 @@ or if metadata fields can't be set. You probably don't want to use this. .IP \(bu 2 \fIname\fP\fB\fC\&.json\fR or \fIname\fP\fB\fC\&.geojson\fR: Read the named GeoJSON input file into a layer called \fIname\fP\&. .IP \(bu 2 +\fIname\fP\fB\fC\&.geobuf\fR or \fIname\fP\fB\fC\&.geobuf\fR: Read the named Geobuf input file into a layer called \fIname\fP\&. +.IP \(bu 2 \fB\fC\-l\fR \fIname\fP or \fB\fC\-\-layer=\fR\fIname\fP: Use the specified layer name instead of deriving a name from the input filename or output tileset. If there are multiple input files specified, the files are all merged into the single named layer, even if they try to specify individual names with \fB\fC\-L\fR\&. .IP \(bu 2 diff --git a/tests/geometry/onebare.json b/tests/geometry/onebare.json new file mode 100644 index 0000000..54533ab --- /dev/null +++ b/tests/geometry/onebare.json @@ -0,0 +1 @@ +{ "type": "LineString", "coordinates": [ [ -122, 37 ], [ -118, 34 ] ] } diff --git a/tests/geometry/out/-z3.json b/tests/geometry/out/-z3.json index 9e8043a..2236f54 100644 --- a/tests/geometry/out/-z3.json +++ b/tests/geometry/out/-z3.json @@ -1,9 +1,9 @@ { "type": "FeatureCollection", "properties": { -"bounds": "-79.453125,-23.563987,102.000000,59.900000", +"bounds": "-122.000000,-23.563987,102.000000,59.900000", "center": "22.500000,20.489949,3", "description": "tests/geometry/out/-z3.json.check.mbtiles", "format": "pbf", -"json": "{\"vector_layers\": [ { \"id\": \"bare\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {} }, { \"id\": \"geometrycollection\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {\"collection\": \"Boolean\"} }, { \"id\": \"multipoint\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {\"point\": \"String\"} } ],\"tilestats\": {\"layerCount\": 3,\"layers\": [{\"layer\": \"bare\",\"count\": 2,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []},{\"layer\": \"geometrycollection\",\"count\": 2,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"collection\",\"count\": 1,\"type\": \"boolean\",\"values\": [true]}]},{\"layer\": \"multipoint\",\"count\": 1,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"point\",\"count\": 1,\"type\": \"string\",\"values\": [\"multi\"]}]}]}}", +"json": "{\"vector_layers\": [ { \"id\": \"bare\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {} }, { \"id\": \"geometrycollection\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {\"collection\": \"Boolean\"} }, { \"id\": \"multipoint\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {\"point\": \"String\"} }, { \"id\": \"onebare\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {} } ],\"tilestats\": {\"layerCount\": 4,\"layers\": [{\"layer\": \"bare\",\"count\": 2,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []},{\"layer\": \"geometrycollection\",\"count\": 2,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"collection\",\"count\": 1,\"type\": \"boolean\",\"values\": [true]}]},{\"layer\": \"multipoint\",\"count\": 1,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"point\",\"count\": 1,\"type\": \"string\",\"values\": [\"multi\"]}]},{\"layer\": \"onebare\",\"count\": 1,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []}]}}", "maxzoom": "3", "minzoom": "0", "name": "tests/geometry/out/-z3.json.check.mbtiles", @@ -24,6 +24,10 @@ { "type": "FeatureCollection", "properties": { "layer": "multipoint", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "point": "multi" }, "geometry": { "type": "MultiPoint", "coordinates": [ [ -79.453125, 49.382373 ], [ 11.513672, 52.052490 ], [ -60.468750, -6.926427 ], [ 23.906250, -12.468760 ] ] } } ] } +, +{ "type": "FeatureCollection", "properties": { "layer": "onebare", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ -122.080078, 37.020098 ], [ -118.037109, 34.016242 ] ] } } +] } ] } , { "type": "FeatureCollection", "properties": { "zoom": 1, "x": 0, "y": 1 }, "features": [ @@ -42,6 +46,10 @@ { "type": "FeatureCollection", "properties": { "layer": "multipoint", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "point": "multi" }, "geometry": { "type": "Point", "coordinates": [ -79.453125, 49.382373 ] } } ] } +, +{ "type": "FeatureCollection", "properties": { "layer": "onebare", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ -122.036133, 37.020098 ], [ -118.037109, 34.016242 ] ] } } +] } ] } , { "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 1 }, "features": [ @@ -78,6 +86,12 @@ ] } ] } , +{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 0, "y": 1 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "onebare", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ -122.014160, 37.002553 ], [ -118.015137, 34.016242 ] ] } } +] } +] } +, { "type": "FeatureCollection", "properties": { "zoom": 2, "x": 1, "y": 2 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "multipoint", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "point": "multi" }, "geometry": { "type": "Point", "coordinates": [ -60.468750, -6.991859 ] } } @@ -132,6 +146,12 @@ ] } ] } , +{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 1, "y": 3 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "onebare", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ -122.003174, 37.002553 ], [ -118.004150, 34.007135 ] ] } } +] } +] } +, { "type": "FeatureCollection", "properties": { "zoom": 3, "x": 2, "y": 4 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "multipoint", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "point": "multi" }, "geometry": { "type": "Point", "coordinates": [ -60.468750, -7.002764 ] } }