ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-02-22 10:10:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add encoding details to tippecanoe-decode --stats output

Browse Source
This commit is contained in:
Eric Fischer 2021-02-26 16:50:46 -08:00
parent 4bb73aa6bc
commit 4656453906
4 changed files with 273 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
decode.cpp
View File

@ -79,7 +79,117 @@ void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, u
state.json_end_hash();
}
state.json_end_hash();
state.json_end_hash(); // layers
state.json_write_string("encoding");
state.json_write_hash();
state.json_write_string("wrapper_overhead_bytes");
state.json_write_unsigned(tile.reader_overhead);
state.json_write_string("wrapper_overhead_count");
state.json_write_unsigned(tile.reader_overhead_count);
state.json_write_string("layer_overhead_bytes");
state.json_write_unsigned(tile.layer_read_overhead);
state.json_write_string("layer_overhead_count");
state.json_write_unsigned(tile.layer_read_overhead_count);
state.json_write_string("layer_name_bytes");
state.json_write_unsigned(tile.layer_names);
state.json_write_string("layer_name_count");
state.json_write_unsigned(tile.layer_names_count);
state.json_write_string("attribute_name_bytes");
state.json_write_unsigned(tile.layer_keys);
state.json_write_string("attribute_name_count");
state.json_write_unsigned(tile.layer_keys_count);
state.json_write_string("attribute_value_overhead_bytes");
state.json_write_unsigned(tile.value_overhead);
state.json_write_string("attribute_value_overhead_count");
state.json_write_unsigned(tile.value_overhead_count);
state.json_write_string("attribute_string_value_bytes");
state.json_write_unsigned(tile.string_values);
state.json_write_string("attribute_string_value_count");
state.json_write_unsigned(tile.string_values_count);
state.json_write_string("attribute_float_value_bytes");
state.json_write_unsigned(tile.float_values);
state.json_write_string("attribute_float_value_count");
state.json_write_unsigned(tile.float_values_count);
state.json_write_string("attribute_double_value_bytes");
state.json_write_unsigned(tile.double_values);
state.json_write_string("attribute_double_value_count");
state.json_write_unsigned(tile.double_values_count);
state.json_write_string("attribute_int_value_bytes");
state.json_write_unsigned(tile.int_values);
state.json_write_string("attribute_int_value_count");
state.json_write_unsigned(tile.int_values_count);
state.json_write_string("attribute_bool_value_bytes");
state.json_write_unsigned(tile.bool_values);
state.json_write_string("attribute_bool_value_count");
state.json_write_unsigned(tile.bool_values_count);
state.json_write_string("layer_extent_bytes");
state.json_write_unsigned(tile.layer_extents);
state.json_write_string("layer_extent_count");
state.json_write_unsigned(tile.layer_extents_count);
state.json_write_string("layer_version_bytes");
state.json_write_unsigned(tile.layer_versions);
state.json_write_string("layer_version_count");
state.json_write_unsigned(tile.layer_versions_count);
state.json_write_string("feature_overhead_bytes");
state.json_write_unsigned(tile.feature_reader_overhead);
state.json_write_string("feature_overhead_count");
state.json_write_unsigned(tile.feature_reader_overhead_count);
state.json_write_string("feature_id_bytes");
state.json_write_unsigned(tile.ids);
state.json_write_string("feature_id_count");
state.json_write_unsigned(tile.ids_count);
state.json_write_string("feature_attribute_references_bytes");
state.json_write_unsigned(tile.attribute_refs);
state.json_write_string("feature_attribute_references_count");
state.json_write_unsigned(tile.attribute_refs_count);
state.json_write_string("feature_type_bytes");
state.json_write_unsigned(tile.feature_types);
state.json_write_string("feature_type_count");
state.json_write_unsigned(tile.feature_types_count);
state.json_write_string("feature_geometry_bytes");
state.json_write_unsigned(tile.feature_geometries);
state.json_write_string("feature_geometry_count");
state.json_write_unsigned(tile.feature_geometries_count);
state.json_write_string("feature_geometry_nodes");
state.json_write_unsigned(tile.feature_geometries_nodes);
state.json_end_hash(); // encoding
state.json_end_hash();
state.json_write_newline();

126
mvt.cpp
View File

@ -124,23 +124,79 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
was_compressed = false;
}
reader_overhead = 0;
reader_overhead_count = 0;
layer_read_overhead = 0;
layer_read_overhead_count = 0;
layer_names = 0;
layer_names_count = 0;
layer_keys = 0;
layer_keys_count = 0;
value_overhead = 0;
value_overhead_count = 0;
string_values = 0;
string_values_count = 0;
float_values = 0;
float_values_count = 0;
double_values = 0;
double_values_count = 0;
int_values = 0;
int_values_count = 0;
bool_values = 0;
bool_values_count = 0;
layer_extents = 0;
layer_extents_count = 0;
layer_versions = 0;
layer_versions_count = 0;
feature_reader_overhead = 0;
feature_reader_overhead_count = 0;
ids = 0;
ids_count = 0;
attribute_refs = 0;
attribute_refs_count = 0;
feature_types = 0;
feature_types_count = 0;
feature_geometries = 0;
feature_geometries_count = 0;
feature_geometries_nodes = 0;
protozero::pbf_reader reader(src);
while (reader.next()) {
while (1) {
reader_overhead += reader.length();
if (!reader.next()) {
break;
}
reader_overhead -= reader.length();
reader_overhead_count++;
switch (reader.tag()) {
case 3: /* layer */
{
protozero::pbf_reader layer_reader(reader.get_message());
mvt_layer layer;
while (layer_reader.next()) {
while (1) {
layer_read_overhead += layer_reader.length();
if (!layer_reader.next()) {
break;
}
layer_read_overhead -= layer_reader.length();
layer_read_overhead_count++;
switch (layer_reader.tag()) {
case 1: /* name */
layer_names += layer_reader.length();
layer.name = layer_reader.get_string();
layer_names -= layer_reader.length();
layer_names_count++;
break;
case 3: /* key */
layer_keys += layer_reader.length();
layer.keys.push_back(layer_reader.get_string());
layer_keys -= layer_reader.length();
layer_keys_count++;
break;
case 4: /* value */
@ -151,45 +207,75 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
value.type = mvt_null;
value.numeric_value.null_value = 0;
while (value_reader.next()) {
while (1) {
value_overhead += value_reader.length();
if (!value_reader.next()) {
break;
}
value_overhead -= value_reader.length();
value_overhead_count++;
switch (value_reader.tag()) {
case 1: /* string */
value.type = mvt_string;
string_values += value_reader.length();
value.string_value = value_reader.get_string();
string_values -= value_reader.length();
string_values_count++;
break;
case 2: /* float */
value.type = mvt_float;
float_values += value_reader.length();
value.numeric_value.float_value = value_reader.get_float();
float_values -= value_reader.length();
float_values_count++;
break;
case 3: /* double */
value.type = mvt_double;
double_values += value_reader.length();
value.numeric_value.double_value = value_reader.get_double();
double_values -= value_reader.length();
double_values_count++;
break;
case 4: /* int */
value.type = mvt_int;
int_values += value_reader.length();
value.numeric_value.int_value = value_reader.get_int64();
int_values -= value_reader.length();
int_values_count++;
break;
case 5: /* uint */
value.type = mvt_uint;
int_values += value_reader.length();
value.numeric_value.uint_value = value_reader.get_uint64();
int_values -= value_reader.length();
int_values_count++;
break;
case 6: /* sint */
value.type = mvt_sint;
int_values += value_reader.length();
value.numeric_value.sint_value = value_reader.get_sint64();
int_values -= value_reader.length();
int_values_count++;
break;
case 7: /* bool */
value.type = mvt_bool;
bool_values += value_reader.length();
value.numeric_value.bool_value = value_reader.get_bool();
bool_values -= value_reader.length();
bool_values_count++;
break;
default:
value_overhead += value_reader.length();
value_reader.skip();
value_overhead -= value_reader.length();
break;
}
}
@ -199,11 +285,17 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
}
case 5: /* extent */
layer_extents += layer_reader.length();
layer.extent = layer_reader.get_uint32();
layer_extents -= layer_reader.length();
layer_extents_count++;
break;
case 15: /* version */
layer_versions += layer_reader.length();
layer.version = layer_reader.get_uint32();
layer_versions -= layer_reader.length();
layer_versions_count++;
break;
case 2: /* feature */
@ -212,16 +304,30 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
mvt_feature feature;
std::vector<uint32_t> geoms;
while (feature_reader.next()) {
while (1) {
feature_reader_overhead += feature_reader.length();
if (!feature_reader.next()) {
break;
}
feature_reader_overhead -= feature_reader.length();
feature_reader_overhead_count++;
switch (feature_reader.tag()) {
case 1: /* id */
ids += feature_reader.length();
feature.id = feature_reader.get_uint64();
ids -= feature_reader.length();
ids_count++;
feature.has_id = true;
break;
case 2: /* tag */
{
attribute_refs += feature_reader.length();
auto pi = feature_reader.get_packed_uint32();
attribute_refs -= feature_reader.length();
attribute_refs_count++;
for (auto it = pi.first; it != pi.second; ++it) {
feature.tags.push_back(*it);
}
@ -229,12 +335,19 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
}
case 3: /* feature type */
feature_types += feature_reader.length();
feature.type = feature_reader.get_enum();
feature_types -= feature_reader.length();
feature_types_count++;
break;
case 4: /* geometry */
{
feature_geometries += feature_reader.length();
auto pi = feature_reader.get_packed_uint32();
feature_geometries -= feature_reader.length();
feature_geometries_count++;
for (auto it = pi.first; it != pi.second; ++it) {
geoms.push_back(*it);
}
@ -242,7 +355,9 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
}
default:
feature_reader_overhead += feature_reader.length();
feature_reader.skip();
feature_reader_overhead -= feature_reader.length();
break;
}
}
@ -266,6 +381,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
}
}
feature_geometries_nodes += feature.geometry.size();
layer.features.push_back(feature);
break;
}
@ -288,7 +404,9 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
}
default:
reader_overhead += reader.length();
reader.skip();
reader_overhead -= reader.length();
break;
}
}

36
mvt.hpp
View File

@ -111,6 +111,42 @@ struct mvt_tile {
std::string encode();
bool decode(std::string &message, bool &was_compressed);
size_t reader_overhead = 0;
size_t reader_overhead_count = 0;
size_t layer_read_overhead = 0;
size_t layer_read_overhead_count = 0;
size_t layer_names = 0;
size_t layer_names_count = 0;
size_t layer_keys = 0;
size_t layer_keys_count = 0;
size_t value_overhead = 0;
size_t value_overhead_count = 0;
size_t string_values = 0;
size_t string_values_count = 0;
size_t float_values = 0;
size_t float_values_count = 0;
size_t double_values = 0;
size_t double_values_count = 0;
size_t int_values = 0;
size_t int_values_count = 0;
size_t bool_values = 0;
size_t bool_values_count = 0;
size_t layer_extents = 0;
size_t layer_extents_count = 0;
size_t layer_versions = 0;
size_t layer_versions_count = 0;
size_t feature_reader_overhead = 0;
size_t feature_reader_overhead_count = 0;
size_t ids = 0;
size_t ids_count = 0;
size_t attribute_refs = 0;
size_t attribute_refs_count = 0;
size_t feature_types = 0;
size_t feature_types_count = 0;
size_t feature_geometries = 0;
size_t feature_geometries_count = 0;
size_t feature_geometries_nodes = 0;
};
bool is_compressed(std::string const &data);

8
tests/muni/decode/multi.mbtiles.stats.json
View File

@ -1,9 +1,9 @@
[
{ "zoom": 11, "x": 326, "y": 791, "bytes": 372, "compressed": true, "layers": { "muni": { "points": 14, "lines": 0, "polygons": 0, "extent": 4096 } } }
{ "zoom": 11, "x": 326, "y": 791, "bytes": 372, "compressed": true, "layers": { "muni": { "points": 14, "lines": 0, "polygons": 0, "extent": 4096 } }, "encoding": { "wrapper_overhead_bytes": 1, "wrapper_overhead_count": 1, "layer_overhead_bytes": 30, "layer_overhead_count": 30, "layer_name_bytes": 5, "layer_name_count": 1, "attribute_name_bytes": 5, "attribute_name_count": 1, "attribute_value_overhead_bytes": 12, "attribute_value_overhead_count": 12, "attribute_string_value_bytes": 283, "attribute_string_value_count": 12, "attribute_float_value_bytes": 0, "attribute_float_value_count": 0, "attribute_double_value_bytes": 0, "attribute_double_value_count": 0, "attribute_int_value_bytes": 0, "attribute_int_value_count": 0, "attribute_bool_value_bytes": 0, "attribute_bool_value_count": 0, "layer_extent_bytes": 2, "layer_extent_count": 1, "layer_version_bytes": 1, "layer_version_count": 1, "feature_overhead_bytes": 42, "feature_overhead_count": 42, "feature_id_bytes": 0, "feature_id_count": 0, "feature_attribute_references_bytes": 42, "feature_attribute_references_count": 14, "feature_type_bytes": 14, "feature_type_count": 14, "feature_geometry_bytes": 84, "feature_geometry_count": 14, "feature_geometry_nodes": 14 } }
,
{ "zoom": 11, "x": 327, "y": 792, "bytes": 6481, "compressed": true, "layers": { "muni": { "points": 528, "lines": 0, "polygons": 0, "extent": 4096 } } }
{ "zoom": 11, "x": 327, "y": 792, "bytes": 6481, "compressed": true, "layers": { "muni": { "points": 528, "lines": 0, "polygons": 0, "extent": 4096 } }, "encoding": { "wrapper_overhead_bytes": 1, "wrapper_overhead_count": 1, "layer_overhead_bytes": 888, "layer_overhead_count": 888, "layer_name_bytes": 5, "layer_name_count": 1, "attribute_name_bytes": 5, "attribute_name_count": 1, "attribute_value_overhead_bytes": 356, "attribute_value_overhead_count": 356, "attribute_string_value_bytes": 9075, "attribute_string_value_count": 356, "attribute_float_value_bytes": 0, "attribute_float_value_count": 0, "attribute_double_value_bytes": 0, "attribute_double_value_count": 0, "attribute_int_value_bytes": 0, "attribute_int_value_count": 0, "attribute_bool_value_bytes": 0, "attribute_bool_value_count": 0, "layer_extent_bytes": 2, "layer_extent_count": 1, "layer_version_bytes": 1, "layer_version_count": 1, "feature_overhead_bytes": 1584, "feature_overhead_count": 1584, "feature_id_bytes": 0, "feature_id_count": 0, "feature_attribute_references_bytes": 1918, "feature_attribute_references_count": 528, "feature_type_bytes": 528, "feature_type_count": 528, "feature_geometry_bytes": 2982, "feature_geometry_count": 528, "feature_geometry_nodes": 528 } }
,
{ "zoom": 11, "x": 327, "y": 791, "bytes": 44376, "compressed": true, "layers": { "muni": { "points": 4285, "lines": 0, "polygons": 0, "extent": 4096 }, "subway": { "points": 19, "lines": 0, "polygons": 0, "extent": 4096 } } }
{ "zoom": 11, "x": 327, "y": 791, "bytes": 44376, "compressed": true, "layers": { "muni": { "points": 4285, "lines": 0, "polygons": 0, "extent": 4096 }, "subway": { "points": 19, "lines": 0, "polygons": 0, "extent": 4096 } }, "encoding": { "wrapper_overhead_bytes": 2, "wrapper_overhead_count": 2, "layer_overhead_bytes": 7156, "layer_overhead_count": 7156, "layer_name_bytes": 12, "layer_name_count": 2, "attribute_name_bytes": 10, "attribute_name_count": 2, "attribute_value_overhead_bytes": 2844, "attribute_value_overhead_count": 2844, "attribute_string_value_bytes": 67705, "attribute_string_value_count": 2844, "attribute_float_value_bytes": 0, "attribute_float_value_count": 0, "attribute_double_value_bytes": 0, "attribute_double_value_count": 0, "attribute_int_value_bytes": 0, "attribute_int_value_count": 0, "attribute_bool_value_bytes": 0, "attribute_bool_value_count": 0, "layer_extent_bytes": 4, "layer_extent_count": 2, "layer_version_bytes": 2, "layer_version_count": 2, "feature_overhead_bytes": 12912, "feature_overhead_count": 12912, "feature_id_bytes": 0, "feature_id_count": 0, "feature_attribute_references_bytes": 17010, "feature_attribute_references_count": 4304, "feature_type_bytes": 4304, "feature_type_count": 4304, "feature_geometry_bytes": 25824, "feature_geometry_count": 4304, "feature_geometry_nodes": 4304 } }
,
{ "zoom": 11, "x": 954, "y": 791, "bytes": 75, "compressed": true, "layers": { "muni": { "points": 12, "lines": 0, "polygons": 0, "extent": 4096 } } }
{ "zoom": 11, "x": 954, "y": 791, "bytes": 75, "compressed": true, "layers": { "muni": { "points": 12, "lines": 0, "polygons": 0, "extent": 4096 } }, "encoding": { "wrapper_overhead_bytes": 1, "wrapper_overhead_count": 1, "layer_overhead_bytes": 17, "layer_overhead_count": 17, "layer_name_bytes": 5, "layer_name_count": 1, "attribute_name_bytes": 5, "attribute_name_count": 1, "attribute_value_overhead_bytes": 1, "attribute_value_overhead_count": 1, "attribute_string_value_bytes": 22, "attribute_string_value_count": 1, "attribute_float_value_bytes": 0, "attribute_float_value_count": 0, "attribute_double_value_bytes": 0, "attribute_double_value_count": 0, "attribute_int_value_bytes": 0, "attribute_int_value_count": 0, "attribute_bool_value_bytes": 0, "attribute_bool_value_count": 0, "layer_extent_bytes": 2, "layer_extent_count": 1, "layer_version_bytes": 1, "layer_version_count": 1, "feature_overhead_bytes": 36, "feature_overhead_count": 36, "feature_id_bytes": 0, "feature_id_count": 0, "feature_attribute_references_bytes": 36, "feature_attribute_references_count": 12, "feature_type_bytes": 12, "feature_type_count": 12, "feature_geometry_bytes": 72, "feature_geometry_count": 12, "feature_geometry_nodes": 12 } }
]