ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-01-21 20:08:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move tilestats management out of parsing and into serialization

Browse Source
This commit is contained in:
Eric Fischer 2017-08-24 16:30:01 -07:00
parent ed8fbd0236
commit 34b1b215f4
4 changed files with 53 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
geojson.cpp
View File

@ -99,7 +99,7 @@ static long long scale_geometry(struct serialization_state *sst, long long *bbox
return geom.size();
}
int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int layer, json_object *tippecanoe, json_object *feature, std::map<std::string, layermap_entry> *layermap, std::string layername, std::map<std::string, int> const *attribute_types) {
int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int layer, json_object *tippecanoe, json_object *feature, std::string layername, std::map<std::string, int> const *attribute_types) {
json_object *geometry_type = json_hash_get(geometry, "type");
if (geometry_type == NULL) {
static int warned = 0;
@ -202,42 +202,6 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
}
}
if (!sst->filters) {
if (tippecanoe_layername.size() != 0) {
if (layermap->count(tippecanoe_layername) == 0) {
layermap->insert(std::pair<std::string, layermap_entry>(tippecanoe_layername, layermap_entry(layermap->size())));
}
auto ai = layermap->find(tippecanoe_layername);
if (ai != layermap->end()) {
layer = ai->second.id;
layername = tippecanoe_layername;
if (mb_geometry[t] == VT_POINT) {
ai->second.points++;
} else if (mb_geometry[t] == VT_LINE) {
ai->second.lines++;
} else if (mb_geometry[t] == VT_POLYGON) {
ai->second.polygons++;
}
} else {
fprintf(stderr, "Internal error: can't find layer name %s\n", tippecanoe_layername.c_str());
exit(EXIT_FAILURE);
}
} else {
auto fk = layermap->find(layername);
if (fk != layermap->end()) {
if (mb_geometry[t] == VT_POINT) {
fk->second.points++;
} else if (mb_geometry[t] == VT_LINE) {
fk->second.lines++;
} else if (mb_geometry[t] == VT_POLYGON) {
fk->second.polygons++;
}
}
}
}
size_t nprop = 0;
if (properties != NULL && properties->type == JSON_HASH) {
nprop = properties->length;
@ -270,15 +234,6 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
metatype[m] = type;
metaval[m] = val;
m++;
type_and_string attrib;
attrib.type = metatype[m - 1];
attrib.string = metaval[m - 1];
if (!sst->filters) {
auto fk = layermap->find(layername);
add_to_file_keys(fk->second.file_keys, metakey[m - 1], attrib);
}
}
}
}
@ -301,6 +256,12 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
sf.feature_minzoom = 0; // Will be filled in during index merging
sf.seq = *(sst->layer_seq);
if (tippecanoe_layername.size() != 0) {
sf.layername = tippecanoe_layername;
} else {
sf.layername = layername;
}
for (size_t i = 0; i < m; i++) {
sf.full_keys.push_back(metakey[i]);
@ -426,6 +387,39 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
sf.index = 0;
}
if (sst->layermap->count(sf.layername) == 0) {
sst->layermap->insert(std::pair<std::string, layermap_entry>(sf.layername, layermap_entry(sst->layermap->size())));
}
auto ai = sst->layermap->find(sf.layername);
if (ai != sst->layermap->end()) {
sf.layer = ai->second.id;
if (!sst->filters) {
if (sf.t == VT_POINT) {
ai->second.points++;
} else if (sf.t == VT_LINE) {
ai->second.lines++;
} else if (sf.t == VT_POLYGON) {
ai->second.polygons++;
}
}
} else {
fprintf(stderr, "Internal error: can't find layer name %s\n", sf.layername.c_str());
exit(EXIT_FAILURE);
}
if (!sst->filters) {
for (size_t i = 0; i < sf.full_keys.size(); i++) {
type_and_string attrib;
attrib.type = sf.full_values[i].type;
attrib.string = sf.full_values[i].s;
auto fk = sst->layermap->find(sf.layername);
add_to_file_keys(fk->second.file_keys, sf.full_keys[i], attrib);
}
}
if (inline_meta) {
sf.metapos = -1;
for (size_t i = 0; i < sf.full_keys.size(); i++) {
@ -493,7 +487,7 @@ void check_crs(json_object *j, const char *reading) {
}
}
void parse_json(struct serialization_state *sst, json_pull *jp, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int layer, std::map<std::string, layermap_entry> *layermap, std::string layername, std::map<std::string, int> const *attribute_types) {
void parse_json(struct serialization_state *sst, json_pull *jp, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int layer, std::string layername, std::map<std::string, int> const *attribute_types) {
long long found_hashes = 0;
long long found_features = 0;
long long found_geometries = 0;
@ -561,7 +555,7 @@ void parse_json(struct serialization_state *sst, json_pull *jp, std::set<std::st
}
found_geometries++;
serialize_geojson_feature(sst, j, NULL, NULL, exclude, include, exclude_all, basezoom, layer, NULL, j, layermap, layername, attribute_types);
serialize_geojson_feature(sst, j, NULL, NULL, exclude, include, exclude_all, basezoom, layer, NULL, j, layername, attribute_types);
json_free(j);
continue;
}
@ -604,10 +598,10 @@ void parse_json(struct serialization_state *sst, json_pull *jp, std::set<std::st
if (geometries != NULL) {
size_t g;
for (g = 0; g < geometries->length; g++) {
serialize_geojson_feature(sst, geometries->array[g], properties, id, exclude, include, exclude_all, basezoom, layer, tippecanoe, j, layermap, layername, attribute_types);
serialize_geojson_feature(sst, geometries->array[g], properties, id, exclude, include, exclude_all, basezoom, layer, tippecanoe, j, layername, attribute_types);
}
} else {
serialize_geojson_feature(sst, geometry, properties, id, exclude, include, exclude_all, basezoom, layer, tippecanoe, j, layermap, layername, attribute_types);
serialize_geojson_feature(sst, geometry, properties, id, exclude, include, exclude_all, basezoom, layer, tippecanoe, j, layername, attribute_types);
}
json_free(j);
@ -619,7 +613,7 @@ void parse_json(struct serialization_state *sst, json_pull *jp, std::set<std::st
void *run_parse_json(void *v) {
struct parse_json_args *pja = (struct parse_json_args *) v;
parse_json(pja->sst, pja->jp, pja->exclude, pja->include, pja->exclude_all, pja->basezoom, pja->layer, pja->layermap, *pja->layername, pja->attribute_types);
parse_json(pja->sst, pja->jp, pja->exclude, pja->include, pja->exclude_all, pja->basezoom, pja->layer, *pja->layername, pja->attribute_types);
return NULL;
}

3
geojson.hpp
View File

@ -16,7 +16,6 @@ struct parse_json_args {
int exclude_all;
int basezoom;
int layer;
std::map<std::string, layermap_entry> *layermap;
std::string *layername;
std::map<std::string, int> const *attribute_types;
bool want_dist;
@ -27,7 +26,7 @@ struct parse_json_args {
struct json_pull *json_begin_map(char *map, long long len);
void json_end_map(struct json_pull *jp);
void parse_json(struct serialization_state *sst, json_pull *jp, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int layer, std::map<std::string, layermap_entry> *layermap, std::string layername, std::map<std::string, int> const *attribute_types);
void parse_json(struct serialization_state *sst, json_pull *jp, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int layer, std::string layername, std::map<std::string, int> const *attribute_types);
void *run_parse_json(void *v);
#endif

5
main.cpp
View File

@ -398,6 +398,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
sst[i].maxzoom = maxzoom;
sst[i].uses_gamma = uses_gamma;
sst[i].filters = filters;
sst[i].layermap = &(*layermaps)[i];
pja[i].jp = json_begin_map(map + segs[i], segs[i + 1] - segs[i]);
pja[i].exclude = exclude;
@ -405,7 +406,6 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
pja[i].exclude_all = exclude_all;
pja[i].basezoom = basezoom;
pja[i].layer = source;
pja[i].layermap = &(*layermaps)[i];
pja[i].layername = &layername;
pja[i].attribute_types = attribute_types;
@ -1377,8 +1377,9 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
sst.maxzoom = maxzoom;
sst.filters = prefilter != NULL || postfilter != NULL;
sst.uses_gamma = uses_gamma;
sst.layermap = &layermaps[0];
parse_json(&sst, jp, exclude, include, exclude_all, basezoom, layer, &layermaps[0], sources[layer].layer, attribute_types);
parse_json(&sst, jp, exclude, include, exclude_all, basezoom, layer, sources[layer].layer, attribute_types);
json_end(jp);
overall_offset = layer_seq;
checkdisk(reader, CPUS);

4
serial.hpp
View File

@ -6,6 +6,7 @@
#include <vector>
#include <sys/stat.h>
#include "geometry.hpp"
#include "mbtiles.hpp"
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname);
@ -63,6 +64,7 @@ struct serial_feature {
long long bbox[4];
std::vector<std::string> full_keys;
std::vector<serial_val> full_values;
std::string layername;
};
void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, const char *fname, long long wx, long long wy, bool include_minzoom);
@ -114,6 +116,8 @@ struct serialization_state {
int maxzoom;
bool filters;
bool uses_gamma;
std::map<std::string, layermap_entry> *layermap;
};
#endif