// for vasprintf() on Linux #define _GNU_SOURCE #include #include #include #include #include #include #include #include "vector_tile.pb.h" #include "tile.h" extern "C" { #include "projection.h" #include "pool.h" #include "mbtiles.h" } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp inline bool is_compressed(std::string const &data) { return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp inline int decompress(std::string const &input, std::string &output) { z_stream inflate_s; inflate_s.zalloc = Z_NULL; inflate_s.zfree = Z_NULL; inflate_s.opaque = Z_NULL; inflate_s.avail_in = 0; inflate_s.next_in = Z_NULL; if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { fprintf(stderr, "error: %s\n", inflate_s.msg); } inflate_s.next_in = (Bytef *) input.data(); inflate_s.avail_in = input.size(); size_t length = 0; do { output.resize(length + 2 * input.size()); inflate_s.avail_out = 2 * input.size(); inflate_s.next_out = (Bytef *) (output.data() + length); int ret = inflate(&inflate_s, Z_FINISH); if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { fprintf(stderr, "error: %s\n", inflate_s.msg); return 0; } length += (2 * input.size() - inflate_s.avail_out); } while (inflate_s.avail_out == 0); inflateEnd(&inflate_s); output.resize(length); return 1; } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp static inline int compress(std::string const &input, std::string &output) { z_stream deflate_s; deflate_s.zalloc = Z_NULL; deflate_s.zfree = Z_NULL; deflate_s.opaque = Z_NULL; deflate_s.avail_in = 0; deflate_s.next_in = Z_NULL; deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); deflate_s.next_in = (Bytef *) input.data(); deflate_s.avail_in = input.size(); size_t length = 0; do { size_t increase = input.size() / 2 + 1024; output.resize(length + increase); deflate_s.avail_out = increase; deflate_s.next_out = (Bytef *) (output.data() + length); int ret = deflate(&deflate_s, Z_FINISH); if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { return -1; } length += (increase - deflate_s.avail_out); } while (deflate_s.avail_out == 0); deflateEnd(&deflate_s); output.resize(length); return 0; } void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { GOOGLE_PROTOBUF_VERIFY_VERSION; // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp mapnik::vector::tile tile; mapnik::vector::tile outtile; if (is_compressed(message)) { std::string uncompressed; decompress(message, uncompressed); if (!tile.ParseFromString(uncompressed)) { fprintf(stderr, "Couldn't decompress tile %d/%u/%u\n", z, x, y); exit(EXIT_FAILURE); } } else if (!tile.ParseFromString(message)) { fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y); exit(EXIT_FAILURE); } for (int l = 0; l < tile.layers_size(); l++) { mapnik::vector::tile_layer layer = tile.layers(l); mapnik::vector::tile_layer *outlayer = outtile.add_layers(); outlayer->set_name(layer.name()); outlayer->set_version(layer.version()); outlayer->set_extent(layer.extent()); const char *ln = layer.name().c_str(); int ll; for (ll = 0; ll < *nlayers; ll++) { if (strcmp((*layernames)[ll], ln) == 0) { break; } } if (ll == *nlayers) { *file_keys = (struct pool *) realloc(*file_keys, (ll + 1) * sizeof(struct pool)); *layernames = (char **) realloc(*layernames, (ll + 1) * sizeof(char *)); pool_init(file_keys[ll], 0); (*layernames)[ll] = strdup(ln); *nlayers = ll + 1; } struct pool keys, values; pool_init(&keys, 0); pool_init(&values, 0); for (int f = 0; f < layer.features_size(); f++) { mapnik::vector::tile_feature feat = layer.features(f); mapnik::vector::tile_feature *outfeature = outlayer->add_features(); outfeature->set_type(feat.type()); for (int g = 0; g < feat.geometry_size(); g++) { outfeature->add_geometry(feat.geometry(g)); } for (int t = 0; t + 1 < feat.tags_size(); t += 2) { if (feat.tags(t) >= layer.keys_size() || feat.tags(t + 1) >= layer.values_size()) { printf("out of range: %d=%d\n", feat.tags(t), feat.tags(t + 1)); continue; } const char *key = layer.keys(feat.tags(t)).c_str(); mapnik::vector::tile_value const &val = layer.values(feat.tags(t + 1)); char *value; int type; if (val.has_string_value()) { value = strdup(val.string_value().c_str()); type = VT_STRING; } else if (val.has_int_value()) { asprintf(&value, "%lld", val.int_value()); type = VT_NUMBER; } else if (val.has_double_value()) { asprintf(&value, "%g", val.double_value()); type = VT_NUMBER; } else if (val.has_float_value()) { asprintf(&value, "%g", val.float_value()); type = VT_NUMBER; } else if (val.has_bool_value()) { asprintf(&value, "%s", val.bool_value() ? "true" : "false"); type = VT_BOOLEAN; } else if (val.has_sint_value()) { asprintf(&value, "%lld", val.sint_value()); type = VT_NUMBER; } else if (val.has_uint_value()) { asprintf(&value, "%llu", val.uint_value()); type = VT_NUMBER; } else { continue; } if (!is_pooled(file_keys[ll], key, type)) { pool(file_keys[ll], strdup(key), type); } struct pool_val *k, *v; if (is_pooled(&keys, key, VT_STRING)) { k = pool(&keys, key, VT_STRING); } else { k = pool(&keys, strdup(key), VT_STRING); } if (is_pooled(&values, value, type)) { v = pool(&values, value, type); } else { v = pool(&values, strdup(value), type); } outfeature->add_tags(k->n); outfeature->add_tags(v->n); free(value); } } struct pool_val *pv; for (pv = keys.head; pv != NULL; pv = pv->next) { outlayer->add_keys(pv->s, strlen(pv->s)); } for (pv = values.head; pv != NULL; pv = pv->next) { mapnik::vector::tile_value *tv = outlayer->add_values(); if (pv->type == VT_NUMBER) { tv->set_double_value(atof(pv->s)); } else if (pv->type == VT_BOOLEAN) { tv->set_bool_value(pv->s[0] == 't'); } else { tv->set_string_value(pv->s); } } pool_free_strings(&keys); pool_free_strings(&values); } std::string s; std::string compressed; outtile.SerializeToString(&s); compress(s, compressed); mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_FAILURE); } const char *sql = "SELECT zoom_level, tile_column, tile_row, tile_data from tiles;"; sqlite3_stmt *stmt; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_FAILURE); } while (sqlite3_step(stmt) == SQLITE_ROW) { long long zoom = sqlite3_column_int(stmt, 0); long long x = sqlite3_column_int(stmt, 1); long long y = sqlite3_column_int(stmt, 2); y = (1LL << zoom) - 1 - y; int len = sqlite3_column_bytes(stmt, 3); const char *s = (const char *) sqlite3_column_blob(stmt, 3); fprintf(stderr, "%lld/%lld/%lld \r", zoom, x, y); handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb); } sqlite3_finalize(stmt); if (sqlite3_close(db) != SQLITE_OK) { fprintf(stderr, "%s: could not close database: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_FAILURE); } } void usage(char **argv) { fprintf(stderr, "Usage: %s [-f] -o new.mbtiles source.mbtiles map.csv\n", argv[0]); exit(EXIT_FAILURE); } int main(int argc, char **argv) { char *outfile = NULL; int force = 0; extern int optind; extern char *optarg; int i; while ((i = getopt(argc, argv, "fo:")) != -1) { switch (i) { case 'o': outfile = optarg; break; case 'f': force = 1; break; default: usage(argv); } } if (argc != optind + 2 || outfile == NULL) { usage(argv); } if (force) { unlink(outfile); } sqlite3 *outdb = mbtiles_open(outfile, argv); struct pool *file_keys = NULL; char **layernames = NULL; int nlayers = 0; decode(argv[optind], argv[optind + 1], &file_keys, &layernames, &nlayers, outdb); for (i = 0; i < nlayers; i++) { printf("%s\n", layernames[i]); } mbtiles_write_metadata(outdb, outfile, layernames, 0, 0, 0, 0, 0, 0, 0, 0, &file_keys, nlayers); mbtiles_close(outdb, argv); return 0; }