From 3cb09d0e2a1b7100951c2bcc360b0795264695cb Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 17 Jul 2018 14:57:56 -0700 Subject: [PATCH] =?UTF-8?q?Add=20the=20ability=20to=20specify=20the=20file?= =?UTF-8?q?=20format=20with=20-L{"format":"=E2=80=A6"}?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 12 ++++++++++++ README.md | 2 +- geocsv.cpp | 22 +++++++++++++++------- main.cpp | 13 +++++++++---- man/tippecanoe.1 | 2 +- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index e2fbed4..691b0bd 100644 --- a/Makefile +++ b/Makefile @@ -278,16 +278,28 @@ allow-existing-test: rm -r tests/allow-existing/both.dir.json.check tests/allow-existing/both.dir tests/allow-existing/both.mbtiles.json.check tests/allow-existing/both.mbtiles csv-test: + # Reading from named CSV ./tippecanoe -q -zg -f -o tests/csv/out.mbtiles tests/csv/ne_110m_populated_places_simple.csv ./tippecanoe-decode -x generator tests/csv/out.mbtiles > tests/csv/out.mbtiles.json.check cmp tests/csv/out.mbtiles.json.check tests/csv/out.mbtiles.json rm -f tests/csv/out.mbtiles.json.check tests/csv/out.mbtiles + # Same, but specifying csv with -L format + ./tippecanoe -q -zg -f -o tests/csv/out.mbtiles -L'{"file":"", "format":"csv", "layer":"ne_110m_populated_places_simplecsv"}' < tests/csv/ne_110m_populated_places_simple.csv + ./tippecanoe-decode -x generator tests/csv/out.mbtiles > tests/csv/out.mbtiles.json.check + cmp tests/csv/out.mbtiles.json.check tests/csv/out.mbtiles.json + rm -f tests/csv/out.mbtiles.json.check tests/csv/out.mbtiles layer-json-test: + # GeoJSON with description and named layer ./tippecanoe -q -z0 -r1 -yNAME -f -o tests/layer-json/out.mbtiles -L'{"file":"tests/ne_110m_populated_places/in.json", "description":"World cities", "layer":"places"}' ./tippecanoe-decode -x generator tests/layer-json/out.mbtiles > tests/layer-json/out.mbtiles.json.check cmp tests/layer-json/out.mbtiles.json.check tests/layer-json/out.mbtiles.json rm -f tests/layer-json/out.mbtiles.json.check tests/layer-json/out.mbtiles + # Same, but reading from the standard input + ./tippecanoe -q -z0 -r1 -yNAME -f -o tests/layer-json/out.mbtiles -L'{"file":"", "description":"World cities", "layer":"places"}' < tests/ne_110m_populated_places/in.json + ./tippecanoe-decode -x generator tests/layer-json/out.mbtiles > tests/layer-json/out.mbtiles.json.check + cmp tests/layer-json/out.mbtiles.json.check tests/layer-json/out.mbtiles.json + rm -f tests/layer-json/out.mbtiles.json.check tests/layer-json/out.mbtiles # Use this target to regenerate the standards that the tests are compared against # after making a change that legitimately changes their output diff --git a/README.md b/README.md index 08d8f53..26f3815 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ If your input is formatted as newline-delimited GeoJSON, use `-P` to make input * `-l` _name_ or `--layer=`_name_: Use the specified layer name instead of deriving a name from the input filename or output tileset. If there are multiple input files specified, the files are all merged into the single named layer, even if they try to specify individual names with `-L`. * `-L` _name_`:`_file.json_ or `--named-layer=`_name_`:`_file.json_: Specify layer names for individual files. If your shell supports it, you can use a subshell redirect like `-L` _name_`:<(cat dir/*.json)` to specify a layer name for the output of streamed input. - * `-L{`_layer-json_`}` or `--named-layer={`_layer-json_`}`: Specify an input file and layer options by a JSON object. The JSON object must contain a `"file"` key to specify the filename to read from. It may also contain a `"layer"` field to specify the name of the layer, and/or a `"description"` field to specify the layer's description in the tileset metadata. Example: + * `-L{`_layer-json_`}` or `--named-layer={`_layer-json_`}`: Specify an input file and layer options by a JSON object. The JSON object must contain a `"file"` key to specify the filename to read from. (If the `"file"` key is an empty string, it means to read from the standard input stream.) It may also contain a `"layer"` field to specify the name of the layer, and/or a `"description"` field to specify the layer's description in the tileset metadata, and/or a `"format"` field to specify `csv` or `geobuf` file format if it is not obvious from the `name`. Example: ``` tippecanoe -z5 -o world.mbtiles -L'{"file":"ne_10m_admin_0_countries.json", "layer":"countries", "description":"Natural Earth countries"}' diff --git a/geocsv.cpp b/geocsv.cpp index 4fb5a4b..7da7112 100644 --- a/geocsv.cpp +++ b/geocsv.cpp @@ -10,10 +10,16 @@ #include "milo/dtoa_milo.h" void parse_geocsv(std::vector &sst, std::string fname, int layer, std::string layername) { - FILE *f = fopen(fname.c_str(), "r"); - if (f == NULL) { - perror(fname.c_str()); - exit(EXIT_FAILURE); + FILE *f; + + if (fname.size() == 0) { + f = stdin; + } else { + f = fopen(fname.c_str(), "r"); + if (f == NULL) { + perror(fname.c_str()); + exit(EXIT_FAILURE); + } } std::string s; @@ -120,8 +126,10 @@ void parse_geocsv(std::vector &sst, std::string fnam serialize_feature(&sst[0], sf); } - if (fclose(f) != 0) { - perror("fclose"); - exit(EXIT_FAILURE); + if (fname.size() != 0) { + if (fclose(f) != 0) { + perror("fclose"); + exit(EXIT_FAILURE); + } } } diff --git a/main.cpp b/main.cpp index 7149638..674d19f 100644 --- a/main.cpp +++ b/main.cpp @@ -84,6 +84,7 @@ struct source { std::string layer = ""; std::string file = ""; std::string description = ""; + std::string format = ""; }; size_t CPUS; @@ -1351,7 +1352,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } size_t layer = a->second.id; - if (sources[source].file.size() > 7 && sources[source].file.substr(sources[source].file.size() - 7) == std::string(".geobuf")) { + if (sources[source].format == "geobuf" || (sources[source].file.size() > 7 && sources[source].file.substr(sources[source].file.size() - 7) == std::string(".geobuf"))) { struct stat st; if (fstat(fd, &st) != 0) { perror("fstat"); @@ -1361,8 +1362,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo char *map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (map == MAP_FAILED) { - perror("mmap"); - perror(sources[source].file.c_str()); + fprintf(stderr, "%s: mmap: %s: %s\n", *av, reading.c_str(), strerror(errno)); exit(EXIT_FAILURE); } @@ -1422,7 +1422,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo continue; } - if (sources[source].file.size() > 4 && sources[source].file.substr(sources[source].file.size() - 4) == std::string(".csv")) { + if (sources[source].format == "csv" || (sources[source].file.size() > 4 && sources[source].file.substr(sources[source].file.size() - 4) == std::string(".csv"))) { std::atomic layer_seq[CPUS]; double dist_sums[CPUS]; size_t dist_counts[CPUS]; @@ -2418,6 +2418,11 @@ void parse_json_source(const char *arg, struct source &src) { src.description = std::string(description->string); } + json_object *format = json_hash_get(o, "format"); + if (format != NULL && format->type == JSON_STRING) { + src.format = std::string(format->string); + } + json_free(o); json_end(jp); } diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 72773d6..166192f 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -160,7 +160,7 @@ specified, the files are all merged into the single named layer, even if they tr .IP \(bu 2 \fB\fC\-L\fR \fIname\fP\fB\fC:\fR\fIfile.json\fP or \fB\fC\-\-named\-layer=\fR\fIname\fP\fB\fC:\fR\fIfile.json\fP: Specify layer names for individual files. If your shell supports it, you can use a subshell redirect like \fB\fC\-L\fR \fIname\fP\fB\fC:<(cat dir/*.json)\fR to specify a layer name for the output of streamed input. .IP \(bu 2 -\fB\fC\-L{\fR\fIlayer\-json\fP\fB\fC}\fR or \fB\fC\-\-named\-layer={\fR\fIlayer\-json\fP\fB\fC}\fR: Specify an input file and layer options by a JSON object. The JSON object must contain a \fB\fC"file"\fR key to specify the filename to read from. It may also contain a \fB\fC"layer"\fR field to specify the name of the layer, and/or a \fB\fC"description"\fR field to specify the layer's description in the tileset metadata. Example: +\fB\fC\-L{\fR\fIlayer\-json\fP\fB\fC}\fR or \fB\fC\-\-named\-layer={\fR\fIlayer\-json\fP\fB\fC}\fR: Specify an input file and layer options by a JSON object. The JSON object must contain a \fB\fC"file"\fR key to specify the filename to read from. (If the \fB\fC"file"\fR key is an empty string, it means to read from the standard input stream.) It may also contain a \fB\fC"layer"\fR field to specify the name of the layer, and/or a \fB\fC"description"\fR field to specify the layer's description in the tileset metadata, and/or a \fB\fC"format"\fR field to specify \fB\fCcsv\fR or \fB\fCgeobuf\fR file format if it is not obvious from the \fB\fCname\fR\&. Example: .RE .PP .RS