From 7be21f60469f611f3b3327b336699caaed4bbec1 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 28 Aug 2017 13:26:11 -0700 Subject: [PATCH] First (untested) pass at handling GL Style Spec filters --- Makefile | 2 +- README.md | 1 + evaluator.cpp | 196 ++++++++++++++++++++++++++++++++++++++++++++++++++ evaluator.hpp | 11 +++ tile-join.cpp | 74 +++++++++++++++++-- 5 files changed, 276 insertions(+), 8 deletions(-) create mode 100644 evaluator.cpp create mode 100644 evaluator.hpp diff --git a/Makefile b/Makefile index f83ccb1..c3f0e79 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ tippecanoe-enumerate: enumerate.o tippecanoe-decode: decode.o projection.o mvt.o write_json.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o +tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread unit: unit.o text.o diff --git a/README.md b/README.md index d8f197a..63d9344 100644 --- a/README.md +++ b/README.md @@ -482,6 +482,7 @@ The options are: * `-x` *key* or `--exclude=`*key*: Remove attributes of type *key* from the output. You can use this to remove the field you are matching against if you no longer need it after joining, or to remove any other attributes you don't want. * `-i` or `--if-matched`: Only include features that matched the CSV. + * `-J` *filter-file* or `--gl-filter-file`=*filter-file*: Check features against a per-layer filter (as defined in the [Mapbox GL Style Specification](https://www.mapbox.com/mapbox-gl-js/style-spec/#types-filter)) and only include those that match. ### Setting or disabling tile size limits diff --git a/evaluator.cpp b/evaluator.cpp new file mode 100644 index 0000000..1da9778 --- /dev/null +++ b/evaluator.cpp @@ -0,0 +1,196 @@ +#include +#include +#include +#include "mvt.hpp" +#include "evaluator.hpp" + +int compare(mvt_value one, json_object *two, bool &fail) { + if (one.type == mvt_string) { + if (two->type != JSON_STRING) { + fail = true; + return false; // string vs non-string + } + + return strcmp(one.string_value.c_str(), two->string); + } + + if (one.type == mvt_double || one.type == mvt_float || one.type == mvt_int || one.type == mvt_uint || one.type == mvt_sint) { + if (two->type != JSON_NUMBER) { + fail = true; + return false; // number vs non-number + } + + double v; + if (one.type == mvt_double) { + v = one.numeric_value.double_value; + } else if (one.type == mvt_float) { + v = one.numeric_value.float_value; + } else if (one.type == mvt_int) { + v = one.numeric_value.int_value; + } else if (one.type == mvt_uint) { + v = one.numeric_value.uint_value; + } else if (one.type == mvt_sint) { + v = one.numeric_value.sint_value; + } else { + fprintf(stderr, "Internal error: bad mvt type %d\n", one.type); + exit(EXIT_FAILURE); + } + + if (v < two->number) { + return -1; + } else if (v > two->number) { + return 1; + } else { + return 0; + } + } + + if (one.type == mvt_bool) { + if (two->type != JSON_TRUE && two->type != JSON_FALSE) { + fail = true; + return false; // bool vs non-bool + } + + bool b = two->type = JSON_TRUE; + return one.numeric_value.bool_value > b; + } + + fprintf(stderr, "Internal error: bad mvt type %d\n", one.type); + exit(EXIT_FAILURE); +} + +bool eval(std::map const &feature, json_object *f) { + if (f == NULL || f->type != JSON_ARRAY) { + fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f)); + exit(EXIT_FAILURE); + } + + if (f->length < 2) { + fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f)); + exit(EXIT_FAILURE); + } + + if (f->array[0]->type != JSON_STRING) { + fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f)); + exit(EXIT_FAILURE); + } + + if (strcmp(f->array[0]->string, "has") == 0) { + if (f->array[1]->type != JSON_STRING) { + fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f)); + exit(EXIT_FAILURE); + } + return feature.count(std::string(f->array[1]->string)) != 0; + } + + if (strcmp(f->array[0]->string, "!has") == 0) { + if (f->array[1]->type != JSON_STRING) { + fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f)); + exit(EXIT_FAILURE); + } + return feature.count(std::string(f->array[1]->string)) == 0; + } + + if (strcmp(f->array[0]->string, "==") == 0 || + strcmp(f->array[0]->string, "!=") == 0 || + strcmp(f->array[0]->string, ">") == 0 || + strcmp(f->array[0]->string, ">=") == 0 || + strcmp(f->array[0]->string, "<") == 0 || + strcmp(f->array[0]->string, "<=") == 0) { + if (f->length < 3) { + fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f)); + exit(EXIT_FAILURE); + } + if (f->array[1]->type != JSON_STRING) { + fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f)); + exit(EXIT_FAILURE); + } + + auto ff = feature.find(std::string(f->array[1]->string)); + if (ff == feature.end()) { + return false; // not found: comparison is false + } + + bool fail = false; + int cmp = compare(ff->second, f->array[2], fail); + + if (fail) { + return false; + } + + if (strcmp(f->array[0]->string, "==") == 0) { + return cmp == 0; + } + if (strcmp(f->array[0]->string, "!=") == 0) { + return cmp != 0; + } + if (strcmp(f->array[0]->string, ">") == 0) { + return cmp > 0; + } + if (strcmp(f->array[0]->string, ">=") == 0) { + return cmp >= 0; + } + if (strcmp(f->array[0]->string, "<") == 0) { + return cmp < 0; + } + if (strcmp(f->array[0]->string, "<=") == 0) { + return cmp <= 0; + } + + fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f)); + exit(EXIT_FAILURE); + } + + if (strcmp(f->array[0]->string, "all") == 0 || + strcmp(f->array[0]->string, "any") == 0 || + strcmp(f->array[0]->string, "none") == 0) { + bool v; + + if (strcmp(f->array[0]->string, "all") == 0) { + v = true; + } else { + v = false; + } + + for (size_t i = 1; i < f->length; i++) { + bool out = eval(feature, f->array[i]); + + if (strcmp(f->array[0]->string, "all") == 0) { + v = v && out; + } else { + v = v || out; + } + } + + if (strcmp(f->array[0]->string, "none") == 0) { + return !v; + } else { + return v; + } + } + + fprintf(stderr, "Unknown filter %s\n", json_stringify(f)); + exit(EXIT_FAILURE); +} + +bool evaluate(std::map const &feature, std::string const &layer, json_object *filter) { + json_object *layers = json_hash_get(filter, "layers"); + + if (layers == NULL) { + fprintf(stderr, "Filter: no \"layers\" key at top level\n"); + exit(EXIT_FAILURE); + } + + if (layers->type != JSON_HASH) { + fprintf(stderr, "Filter: \"layers\" is not a hash\n"); + exit(EXIT_FAILURE); + } + + json_object *f = json_hash_get(layers, layer.c_str()); + + if (f != NULL) { + return eval(feature, f); + } else { + return true; // no filter for this layer; + } +} diff --git a/evaluator.hpp b/evaluator.hpp new file mode 100644 index 0000000..8c3b25b --- /dev/null +++ b/evaluator.hpp @@ -0,0 +1,11 @@ +#ifndef EVALUATOR_HPP +#define EVALUATOR HPP + +#include +#include +#include "jsonpull/jsonpull.h" +#include "mvt.hpp" + +bool evaluate(std::map const &feature, std::string const &layer, json_pull *filter); + +#endif diff --git a/tile-join.cpp b/tile-join.cpp index aee4f3c..5a0422b 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -45,7 +45,7 @@ struct stats { double minlat, minlon, maxlat, maxlon; }; -void handle(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, std::set &exclude, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile) { +void handle(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, std::set &exclude, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter) { mvt_tile tile; int features_added = 0; bool was_compressed; @@ -99,6 +99,39 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::map attributes; + + for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { + std::string key = layer.keys[feat.tags[t]]; + mvt_value &val = layer.values[feat.tags[t + 1]]; + + attributes.insert(std::pair(key, val)); + } + + if (feat.has_id) { + mvt_value v; + v.type = mvt_uint; + v.numeric_value.uint_value = feat.id; + + attributes.insert(std::pair("$id", v)); + } + + mvt_value v; + v.type = mvt_string; + + if (feat.type == mvt_point) { + v.string_value = "Point"; + } else if (feat.type == mvt_linestring) { + v.string_value = "LineString"; + } else if (feat.type == mvt_polygon) { + v.string_value = "Polygon"; + } + + attributes.insert(std::pair("$type", v)); + } + mvt_feature outfeature; int matched = 0; @@ -563,6 +596,7 @@ struct arg { std::set *keep_layers; std::set *remove_layers; int ifmatched; + json_object *filter; }; void *join_worker(void *v) { @@ -572,7 +606,7 @@ void *join_worker(void *v) { mvt_tile tile; for (size_t i = 0; i < ai->second.size(); i++) { - handle(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), *(a->exclude), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile); + handle(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), *(a->exclude), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter); } ai->second.clear(); @@ -606,7 +640,7 @@ void *join_worker(void *v) { return NULL; } -void handle_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::set &keep_layers, std::set &remove_layers) { +void handle_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object *filter) { pthread_t pthreads[CPUS]; std::vector args; @@ -620,6 +654,7 @@ void handle_tasks(std::map> &tasks, std::vector> &tasks, std::vector &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name) { +void decode(struct reader *readers, char *map, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object *filter) { std::vector> layermaps; for (size_t i = 0; i < CPUS; i++) { layermaps.push_back(std::map()); @@ -706,7 +741,7 @@ void decode(struct reader *readers, char *map, std::mapzoom != r->zoom || readers->x != r->x || readers->y != r->y) { if (tasks.size() > 100 * CPUS) { - handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers); + handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers, filter); tasks.clear(); } } @@ -759,7 +794,7 @@ void decode(struct reader *readers, char *map, std::mapminlat = min(minlat, st->minlat); st->maxlat = max(maxlat, st->maxlat); - handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers); + handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers, filter); layermap = merge_layermaps(layermaps); struct reader *next; @@ -996,6 +1031,25 @@ void readcsv(char *fn, std::vector &header, std::maperror); + exit(EXIT_FAILURE); + } + + // XXX clone tree instead of leaving pull open + fclose(fp); + return filter; +} + int main(int argc, char **argv) { char *out_mbtiles = NULL; char *out_dir = NULL; @@ -1003,6 +1057,7 @@ int main(int argc, char **argv) { char *csv = NULL; int force = 0; int ifmatched = 0; + json_object *filter = NULL; CPUS = sysconf(_SC_NPROCESSORS_ONLN); @@ -1039,6 +1094,7 @@ int main(int argc, char **argv) { {"quiet", no_argument, 0, 'q'}, {"maximum-zoom", required_argument, 0, 'z'}, {"minimum-zoom", required_argument, 0, 'Z'}, + {"gl-filter-file", required_argument, 0, 'J'}, {"no-tile-size-limit", no_argument, &pk, 1}, {"no-tile-compression", no_argument, &pC, 1}, @@ -1103,6 +1159,10 @@ int main(int argc, char **argv) { minzoom = atoi(optarg); break; + case 'J': + filter = read_filter(optarg); + break; + case 'p': if (strcmp(optarg, "k") == 0) { pk = true; @@ -1200,7 +1260,7 @@ int main(int argc, char **argv) { *rr = r; } - decode(readers, csv, layermap, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name); + decode(readers, csv, layermap, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name, filter); if (set_attribution.size() != 0) { attribution = set_attribution;