From c26fa235647d1334c331eec365c7e6c81edfc627 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 29 Aug 2016 14:59:28 -0700 Subject: [PATCH] Per-thread layermap will contain the file keys, not just a layer ID --- geojson.cpp | 8 ++++---- geojson.hpp | 4 ++-- main.cpp | 12 ++++++------ mbtiles.hpp | 9 +++++++++ 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/geojson.cpp b/geojson.cpp index aaf0143..72330de 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -167,7 +167,7 @@ long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, i return g; } -int serialize_geometry(json_object *geometry, json_object *properties, json_object *id, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, json_object *feature, std::map *layermap) { +int serialize_geometry(json_object *geometry, json_object *properties, json_object *id, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, json_object *feature, std::map *layermap) { json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == NULL) { static int warned = 0; @@ -373,12 +373,12 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje if (tippecanoe_layername.size() != 0) { if (layermap->count(tippecanoe_layername) == 0) { - layermap->insert(std::pair(tippecanoe_layername, layermap->size())); + layermap->insert(std::pair(tippecanoe_layername, layermap_entry(layermap->size()))); } auto ai = layermap->find(tippecanoe_layername); if (ai != layermap->end()) { - layer = ai->second; + layer = ai->second.id; } else { fprintf(stderr, "Internal error: can't find layer name %s\n", tippecanoe_layername.c_str()); exit(EXIT_FAILURE); @@ -471,7 +471,7 @@ void check_crs(json_object *j, const char *reading) { } } -void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, std::map *layermap) { +void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, std::map *layermap) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; diff --git a/geojson.hpp b/geojson.hpp index 83ebc47..6d4b1fe 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -26,9 +26,9 @@ struct parse_json_args { struct reader *readers; std::set *file_keys; int maxzoom; - std::map *layermap; + std::map *layermap; }; struct json_pull *json_begin_map(char *map, long long len); -void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, std::map *layermap); +void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, std::map *layermap); void *run_parse_json(void *v); diff --git a/main.cpp b/main.cpp index 66289b4..849643b 100644 --- a/main.cpp +++ b/main.cpp @@ -300,7 +300,7 @@ void *run_sort(void *v) { return NULL; } -void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector > *layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys, int maxzoom) { +void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector > *layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys, int maxzoom) { long long segs[CPUS + 1]; segs[0] = 0; segs[CPUS] = len; @@ -400,7 +400,7 @@ struct read_parallel_arg { int basezoom; int source; int nlayers; - std::vector > *layermaps; + std::vector > *layermaps; double droprate; int *initialized; unsigned *initial_x; @@ -444,7 +444,7 @@ void *run_read_parallel(void *v) { return NULL; } -void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector > &layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys, int maxzoom) { +void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector > &layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys, int maxzoom) { // This has to kick off an intermediate thread to start the parser threads, // so the main thread can get back to reading the next input stage while // the intermediate thread waits for the completion of the parser threads. @@ -1029,11 +1029,11 @@ int read_input(std::vector &sources, char *fname, const char *layername, } } - std::map layermap; + std::map layermap; for (size_t l = 0; l < nlayers; l++) { - layermap.insert(std::pair(layernames[l], l)); + layermap.insert(std::pair(layernames[l], layermap_entry(l))); } - std::vector > layermaps; + std::vector > layermaps; for (size_t l = 0; l < CPUS; l++) { layermaps.push_back(layermap); } diff --git a/mbtiles.hpp b/mbtiles.hpp index 93d0462..e7ddc23 100644 --- a/mbtiles.hpp +++ b/mbtiles.hpp @@ -5,6 +5,15 @@ struct type_and_string { bool operator<(const type_and_string &o) const; }; +struct layermap_entry { + size_t id; + std::set file_keys; + + layermap_entry(size_t _id) { + id = _id; + } +}; + sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);