mirror of
https://github.com/mapbox/tippecanoe.git
synced 2025-01-21 12:05:05 +00:00
First (untested) pass at handling GL Style Spec filters
This commit is contained in:
Eric Fischer
parent
e7f264fa51
commit
7be21f6046
2
Makefile
2
Makefile
@ -55,7 +55,7 @@ tippecanoe-enumerate: enumerate.o
|
||||
tippecanoe-decode: decode.o projection.o mvt.o write_json.o
|
||||
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
|
||||
|
||||
tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o
|
||||
tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o
|
||||
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
|
||||
|
||||
unit: unit.o text.o
|
||||
|
||||
@ -482,6 +482,7 @@ The options are:
|
||||
|
||||
* `-x` *key* or `--exclude=`*key*: Remove attributes of type *key* from the output. You can use this to remove the field you are matching against if you no longer need it after joining, or to remove any other attributes you don't want.
|
||||
* `-i` or `--if-matched`: Only include features that matched the CSV.
|
||||
* `-J` *filter-file* or `--gl-filter-file`=*filter-file*: Check features against a per-layer filter (as defined in the [Mapbox GL Style Specification](https://www.mapbox.com/mapbox-gl-js/style-spec/#types-filter)) and only include those that match.
|
||||
|
||||
### Setting or disabling tile size limits
|
||||
|
||||
|
||||
196
evaluator.cpp
Normal file
196
evaluator.cpp
Normal file
@ -0,0 +1,196 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <map>
|
||||
#include "mvt.hpp"
|
||||
#include "evaluator.hpp"
|
||||
|
||||
int compare(mvt_value one, json_object *two, bool &fail) {
|
||||
if (one.type == mvt_string) {
|
||||
if (two->type != JSON_STRING) {
|
||||
fail = true;
|
||||
return false; // string vs non-string
|
||||
}
|
||||
|
||||
return strcmp(one.string_value.c_str(), two->string);
|
||||
}
|
||||
|
||||
if (one.type == mvt_double || one.type == mvt_float || one.type == mvt_int || one.type == mvt_uint || one.type == mvt_sint) {
|
||||
if (two->type != JSON_NUMBER) {
|
||||
fail = true;
|
||||
return false; // number vs non-number
|
||||
}
|
||||
|
||||
double v;
|
||||
if (one.type == mvt_double) {
|
||||
v = one.numeric_value.double_value;
|
||||
} else if (one.type == mvt_float) {
|
||||
v = one.numeric_value.float_value;
|
||||
} else if (one.type == mvt_int) {
|
||||
v = one.numeric_value.int_value;
|
||||
} else if (one.type == mvt_uint) {
|
||||
v = one.numeric_value.uint_value;
|
||||
} else if (one.type == mvt_sint) {
|
||||
v = one.numeric_value.sint_value;
|
||||
} else {
|
||||
fprintf(stderr, "Internal error: bad mvt type %d\n", one.type);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (v < two->number) {
|
||||
return -1;
|
||||
} else if (v > two->number) {
|
||||
return 1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (one.type == mvt_bool) {
|
||||
if (two->type != JSON_TRUE && two->type != JSON_FALSE) {
|
||||
fail = true;
|
||||
return false; // bool vs non-bool
|
||||
}
|
||||
|
||||
bool b = two->type = JSON_TRUE;
|
||||
return one.numeric_value.bool_value > b;
|
||||
}
|
||||
|
||||
fprintf(stderr, "Internal error: bad mvt type %d\n", one.type);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
bool eval(std::map<std::string, mvt_value> const &feature, json_object *f) {
|
||||
if (f == NULL || f->type != JSON_ARRAY) {
|
||||
fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (f->length < 2) {
|
||||
fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (f->array[0]->type != JSON_STRING) {
|
||||
fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (strcmp(f->array[0]->string, "has") == 0) {
|
||||
if (f->array[1]->type != JSON_STRING) {
|
||||
fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
return feature.count(std::string(f->array[1]->string)) != 0;
|
||||
}
|
||||
|
||||
if (strcmp(f->array[0]->string, "!has") == 0) {
|
||||
if (f->array[1]->type != JSON_STRING) {
|
||||
fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
return feature.count(std::string(f->array[1]->string)) == 0;
|
||||
}
|
||||
|
||||
if (strcmp(f->array[0]->string, "==") == 0 ||
|
||||
strcmp(f->array[0]->string, "!=") == 0 ||
|
||||
strcmp(f->array[0]->string, ">") == 0 ||
|
||||
strcmp(f->array[0]->string, ">=") == 0 ||
|
||||
strcmp(f->array[0]->string, "<") == 0 ||
|
||||
strcmp(f->array[0]->string, "<=") == 0) {
|
||||
if (f->length < 3) {
|
||||
fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (f->array[1]->type != JSON_STRING) {
|
||||
fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
auto ff = feature.find(std::string(f->array[1]->string));
|
||||
if (ff == feature.end()) {
|
||||
return false; // not found: comparison is false
|
||||
}
|
||||
|
||||
bool fail = false;
|
||||
int cmp = compare(ff->second, f->array[2], fail);
|
||||
|
||||
if (fail) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (strcmp(f->array[0]->string, "==") == 0) {
|
||||
return cmp == 0;
|
||||
}
|
||||
if (strcmp(f->array[0]->string, "!=") == 0) {
|
||||
return cmp != 0;
|
||||
}
|
||||
if (strcmp(f->array[0]->string, ">") == 0) {
|
||||
return cmp > 0;
|
||||
}
|
||||
if (strcmp(f->array[0]->string, ">=") == 0) {
|
||||
return cmp >= 0;
|
||||
}
|
||||
if (strcmp(f->array[0]->string, "<") == 0) {
|
||||
return cmp < 0;
|
||||
}
|
||||
if (strcmp(f->array[0]->string, "<=") == 0) {
|
||||
return cmp <= 0;
|
||||
}
|
||||
|
||||
fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (strcmp(f->array[0]->string, "all") == 0 ||
|
||||
strcmp(f->array[0]->string, "any") == 0 ||
|
||||
strcmp(f->array[0]->string, "none") == 0) {
|
||||
bool v;
|
||||
|
||||
if (strcmp(f->array[0]->string, "all") == 0) {
|
||||
v = true;
|
||||
} else {
|
||||
v = false;
|
||||
}
|
||||
|
||||
for (size_t i = 1; i < f->length; i++) {
|
||||
bool out = eval(feature, f->array[i]);
|
||||
|
||||
if (strcmp(f->array[0]->string, "all") == 0) {
|
||||
v = v && out;
|
||||
} else {
|
||||
v = v || out;
|
||||
}
|
||||
}
|
||||
|
||||
if (strcmp(f->array[0]->string, "none") == 0) {
|
||||
return !v;
|
||||
} else {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
fprintf(stderr, "Unknown filter %s\n", json_stringify(f));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
bool evaluate(std::map<std::string, mvt_value> const &feature, std::string const &layer, json_object *filter) {
|
||||
json_object *layers = json_hash_get(filter, "layers");
|
||||
|
||||
if (layers == NULL) {
|
||||
fprintf(stderr, "Filter: no \"layers\" key at top level\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (layers->type != JSON_HASH) {
|
||||
fprintf(stderr, "Filter: \"layers\" is not a hash\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
json_object *f = json_hash_get(layers, layer.c_str());
|
||||
|
||||
if (f != NULL) {
|
||||
return eval(feature, f);
|
||||
} else {
|
||||
return true; // no filter for this layer;
|
||||
}
|
||||
}
|
||||
11
evaluator.hpp
Normal file
11
evaluator.hpp
Normal file
@ -0,0 +1,11 @@
|
||||
#ifndef EVALUATOR_HPP
|
||||
#define EVALUATOR HPP
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include "jsonpull/jsonpull.h"
|
||||
#include "mvt.hpp"
|
||||
|
||||
bool evaluate(std::map<std::string, mvt_value> const &feature, std::string const &layer, json_pull *filter);
|
||||
|
||||
#endif
|
||||
@ -45,7 +45,7 @@ struct stats {
|
||||
double minlat, minlon, maxlat, maxlon;
|
||||
};
|
||||
|
||||
void handle(std::string message, int z, unsigned x, unsigned y, std::map<std::string, layermap_entry> &layermap, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, int ifmatched, mvt_tile &outtile) {
|
||||
void handle(std::string message, int z, unsigned x, unsigned y, std::map<std::string, layermap_entry> &layermap, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter) {
|
||||
mvt_tile tile;
|
||||
int features_added = 0;
|
||||
bool was_compressed;
|
||||
@ -99,6 +99,39 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::map<std::st
|
||||
|
||||
for (size_t f = 0; f < layer.features.size(); f++) {
|
||||
mvt_feature feat = layer.features[f];
|
||||
|
||||
if (filter != NULL) {
|
||||
std::map<std::string, mvt_value> attributes;
|
||||
|
||||
for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) {
|
||||
std::string key = layer.keys[feat.tags[t]];
|
||||
mvt_value &val = layer.values[feat.tags[t + 1]];
|
||||
|
||||
attributes.insert(std::pair<std::string, mvt_value>(key, val));
|
||||
}
|
||||
|
||||
if (feat.has_id) {
|
||||
mvt_value v;
|
||||
v.type = mvt_uint;
|
||||
v.numeric_value.uint_value = feat.id;
|
||||
|
||||
attributes.insert(std::pair<std::string, mvt_value>("$id", v));
|
||||
}
|
||||
|
||||
mvt_value v;
|
||||
v.type = mvt_string;
|
||||
|
||||
if (feat.type == mvt_point) {
|
||||
v.string_value = "Point";
|
||||
} else if (feat.type == mvt_linestring) {
|
||||
v.string_value = "LineString";
|
||||
} else if (feat.type == mvt_polygon) {
|
||||
v.string_value = "Polygon";
|
||||
}
|
||||
|
||||
attributes.insert(std::pair<std::string, mvt_value>("$type", v));
|
||||
}
|
||||
|
||||
mvt_feature outfeature;
|
||||
int matched = 0;
|
||||
|
||||
@ -563,6 +596,7 @@ struct arg {
|
||||
std::set<std::string> *keep_layers;
|
||||
std::set<std::string> *remove_layers;
|
||||
int ifmatched;
|
||||
json_object *filter;
|
||||
};
|
||||
|
||||
void *join_worker(void *v) {
|
||||
@ -572,7 +606,7 @@ void *join_worker(void *v) {
|
||||
mvt_tile tile;
|
||||
|
||||
for (size_t i = 0; i < ai->second.size(); i++) {
|
||||
handle(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), *(a->exclude), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile);
|
||||
handle(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), *(a->exclude), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter);
|
||||
}
|
||||
|
||||
ai->second.clear();
|
||||
@ -606,7 +640,7 @@ void *join_worker(void *v) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, const char *outdir, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers) {
|
||||
void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, const char *outdir, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, json_object *filter) {
|
||||
pthread_t pthreads[CPUS];
|
||||
std::vector<arg> args;
|
||||
|
||||
@ -620,6 +654,7 @@ void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<st
|
||||
args[i].keep_layers = &keep_layers;
|
||||
args[i].remove_layers = &remove_layers;
|
||||
args[i].ifmatched = ifmatched;
|
||||
args[i].filter = filter;
|
||||
}
|
||||
|
||||
size_t count = 0;
|
||||
@ -661,7 +696,7 @@ void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<st
|
||||
}
|
||||
}
|
||||
|
||||
void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name) {
|
||||
void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name, json_object *filter) {
|
||||
std::vector<std::map<std::string, layermap_entry>> layermaps;
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
layermaps.push_back(std::map<std::string, layermap_entry>());
|
||||
@ -706,7 +741,7 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
|
||||
|
||||
if (readers == NULL || readers->zoom != r->zoom || readers->x != r->x || readers->y != r->y) {
|
||||
if (tasks.size() > 100 * CPUS) {
|
||||
handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
|
||||
handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers, filter);
|
||||
tasks.clear();
|
||||
}
|
||||
}
|
||||
@ -759,7 +794,7 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
|
||||
st->minlat = min(minlat, st->minlat);
|
||||
st->maxlat = max(maxlat, st->maxlat);
|
||||
|
||||
handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
|
||||
handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers, filter);
|
||||
layermap = merge_layermaps(layermaps);
|
||||
|
||||
struct reader *next;
|
||||
@ -996,6 +1031,25 @@ void readcsv(char *fn, std::vector<std::string> &header, std::map<std::string, s
|
||||
}
|
||||
}
|
||||
|
||||
json_object *read_filter(const char *fname) {
|
||||
FILE *fp = fopen(fname, "r");
|
||||
if (fp == NULL) {
|
||||
perror(fname);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
json_pull *jp = json_begin_file(fp);
|
||||
json_object *filter = json_read_tree(jp);
|
||||
if (filter == NULL) {
|
||||
fprintf(stderr, "%s: %s\n", fname, jp->error);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// XXX clone tree instead of leaving pull open
|
||||
fclose(fp);
|
||||
return filter;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
char *out_mbtiles = NULL;
|
||||
char *out_dir = NULL;
|
||||
@ -1003,6 +1057,7 @@ int main(int argc, char **argv) {
|
||||
char *csv = NULL;
|
||||
int force = 0;
|
||||
int ifmatched = 0;
|
||||
json_object *filter = NULL;
|
||||
|
||||
CPUS = sysconf(_SC_NPROCESSORS_ONLN);
|
||||
|
||||
@ -1039,6 +1094,7 @@ int main(int argc, char **argv) {
|
||||
{"quiet", no_argument, 0, 'q'},
|
||||
{"maximum-zoom", required_argument, 0, 'z'},
|
||||
{"minimum-zoom", required_argument, 0, 'Z'},
|
||||
{"gl-filter-file", required_argument, 0, 'J'},
|
||||
|
||||
{"no-tile-size-limit", no_argument, &pk, 1},
|
||||
{"no-tile-compression", no_argument, &pC, 1},
|
||||
@ -1103,6 +1159,10 @@ int main(int argc, char **argv) {
|
||||
minzoom = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 'J':
|
||||
filter = read_filter(optarg);
|
||||
break;
|
||||
|
||||
case 'p':
|
||||
if (strcmp(optarg, "k") == 0) {
|
||||
pk = true;
|
||||
@ -1200,7 +1260,7 @@ int main(int argc, char **argv) {
|
||||
*rr = r;
|
||||
}
|
||||
|
||||
decode(readers, csv, layermap, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name);
|
||||
decode(readers, csv, layermap, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name, filter);
|
||||
|
||||
if (set_attribution.size() != 0) {
|
||||
attribution = set_attribution;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user