diff --git a/geojson.cpp b/geojson.cpp index 32f6ddd..10c1b35 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -167,7 +167,7 @@ long long parse_geometry(int t, json_object *j, long long *bbox, drawvec &out, i return g; } -int serialize_geometry(json_object *geometry, json_object *properties, json_object *id, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, json_object *feature, std::map *layermap, std::string const &layername) { +int serialize_geometry(json_object *geometry, json_object *properties, json_object *id, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, int maxzoom, json_object *feature, std::map *layermap, std::string const &layername) { json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == NULL) { static int warned = 0; @@ -309,8 +309,6 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje } if (tas.type >= 0) { - file_keys->insert(tas); - auto fk = layermap->find(layername); fk->second.file_keys.insert(tas); } @@ -474,7 +472,7 @@ void check_crs(json_object *j, const char *reading) { } } -void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, std::map *layermap, std::string layername) { +void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, int maxzoom, std::map *layermap, std::string layername) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; @@ -542,7 +540,7 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se } found_geometries++; - serialize_geometry(j, NULL, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom, j, layermap, layername); + serialize_geometry(j, NULL, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers, maxzoom, j, layermap, layername); json_free(j); continue; } @@ -584,10 +582,10 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se if (geometries != NULL) { size_t g; for (g = 0; g < geometries->length; g++) { - serialize_geometry(geometries->array[g], properties, id, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom, j, layermap, layername); + serialize_geometry(geometries->array[g], properties, id, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, maxzoom, j, layermap, layername); } } else { - serialize_geometry(geometry, properties, id, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom, j, layermap, layername); + serialize_geometry(geometry, properties, id, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, maxzoom, j, layermap, layername); } json_free(j); @@ -599,7 +597,7 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se void *run_parse_json(void *v) { struct parse_json_args *pja = (struct parse_json_args *) v; - parse_json(pja->jp, pja->reading, pja->layer_seq, pja->progress_seq, pja->metapos, pja->geompos, pja->indexpos, pja->exclude, pja->include, pja->exclude_all, pja->metafile, pja->geomfile, pja->indexfile, pja->poolfile, pja->treefile, pja->fname, pja->basezoom, pja->layer, pja->droprate, pja->file_bbox, pja->segment, pja->initialized, pja->initial_x, pja->initial_y, pja->readers, pja->file_keys, pja->maxzoom, pja->layermap, *pja->layername); + parse_json(pja->jp, pja->reading, pja->layer_seq, pja->progress_seq, pja->metapos, pja->geompos, pja->indexpos, pja->exclude, pja->include, pja->exclude_all, pja->metafile, pja->geomfile, pja->indexfile, pja->poolfile, pja->treefile, pja->fname, pja->basezoom, pja->layer, pja->droprate, pja->file_bbox, pja->segment, pja->initialized, pja->initial_x, pja->initial_y, pja->readers, pja->maxzoom, pja->layermap, *pja->layername); return NULL; } diff --git a/geojson.hpp b/geojson.hpp index b055bae..2ee4bfc 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -24,12 +24,11 @@ struct parse_json_args { unsigned *initial_x; unsigned *initial_y; struct reader *readers; - std::set *file_keys; int maxzoom; std::map *layermap; std::string *layername; }; struct json_pull *json_begin_map(char *map, long long len); -void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom, std::map *layermap, std::string layername); +void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, int maxzoom, std::map *layermap, std::string layername); void *run_parse_json(void *v); diff --git a/main.cpp b/main.cpp index ac620e0..53bbf0a 100644 --- a/main.cpp +++ b/main.cpp @@ -300,7 +300,7 @@ void *run_sort(void *v) { return NULL; } -void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector > *layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys, int maxzoom, std::string layername) { +void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector > *layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername) { long long segs[CPUS + 1]; segs[0] = 0; segs[CPUS] = len; @@ -355,7 +355,6 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const pja[i].initial_x = &initial_x[i]; pja[i].initial_y = &initial_y[i]; pja[i].readers = reader; - pja[i].file_keys = &file_subkeys[i]; pja[i].maxzoom = maxzoom; pja[i].layermap = &(*layermaps)[i]; pja[i].layername = &layername; @@ -373,11 +372,6 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const perror("pthread_join 370"); } - std::set::iterator j; - for (j = file_subkeys[i].begin(); j != file_subkeys[i].end(); ++j) { - file_keys->insert(*j); - } - free(pja[i].jp->source); json_end(pja[i].jp); } @@ -406,7 +400,6 @@ struct read_parallel_arg { int *initialized; unsigned *initial_x; unsigned *initial_y; - std::set *file_keys; std::string layername; }; @@ -429,7 +422,7 @@ void *run_read_parallel(void *v) { } madvise(map, a->len, MADV_RANDOM); // sequential, but from several pointers at once - do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->layermaps, a->droprate, a->initialized, a->initial_x, a->initial_y, a->file_keys, a->maxzoom, a->layername); + do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->layermaps, a->droprate, a->initialized, a->initial_x, a->initial_y, a->maxzoom, a->layername); madvise(map, a->len, MADV_DONTNEED); if (munmap(map, a->len) != 0) { @@ -446,7 +439,7 @@ void *run_read_parallel(void *v) { return NULL; } -void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector > &layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys, int maxzoom, std::string layername) { +void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector > &layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername) { // This has to kick off an intermediate thread to start the parser threads, // so the main thread can get back to reading the next input stage while // the intermediate thread waits for the completion of the parser threads. @@ -480,7 +473,6 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i rpa->initialized = initialized; rpa->initial_x = initial_x; rpa->initial_y = initial_y; - rpa->file_keys = file_keys; rpa->maxzoom = maxzoom; rpa->layername = layername; @@ -1077,21 +1069,12 @@ int read_input(std::vector &sources, char *fname, const char *layername, nsources = 1; } - std::vector > file_keys; long overall_offset = 0; for (size_t source = 0; source < nsources; source++) { - file_keys.push_back(std::set()); std::string reading; int fd; - if (additional[A_CALCULATE_FEATURE_DENSITY]) { - type_and_string tas; - tas.type = VT_NUMBER; - tas.string = "tippecanoe_feature_density"; - file_keys[source].insert(tas); - } - if (source >= sources.size()) { reading = "standard input"; fd = 0; @@ -1122,7 +1105,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, } if (map != NULL && map != MAP_FAILED) { - do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, &layermaps, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom, layernames[source < nlayers ? source : 0]); + do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, &layermaps, droprate, initialized, initial_x, initial_y, maxzoom, layernames[source < nlayers ? source : 0]); overall_offset += st.st_size - off; checkdisk(reader, CPUS); @@ -1190,7 +1173,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, } fflush(readfp); - start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, layermaps, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom, layernames[source < nlayers ? source : 0]); + start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, layermaps, droprate, initialized, initial_x, initial_y, maxzoom, layernames[source < nlayers ? source : 0]); initial_offset += ahead; overall_offset += ahead; @@ -1227,7 +1210,8 @@ int read_input(std::vector &sources, char *fname, const char *layername, fflush(readfp); if (ahead > 0) { - start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, layermaps, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom, layernames[source < nlayers ? source : 0]); + printf("layer name %s\n", layernames[source < nlayers ? source : 0].c_str()); + start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, layermaps, droprate, initialized, initial_x, initial_y, maxzoom, layernames[source < nlayers ? source : 0]); if (parser_created) { if (pthread_join(parallel_parser, NULL) != 0) { @@ -1244,7 +1228,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, long long layer_seq = overall_offset; json_pull *jp = json_begin_file(fp); - parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, &file_keys[source < nlayers ? source : 0], maxzoom, &layermaps[0], layernames[source < nlayers ? source : 0]); + parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, maxzoom, &layermaps[0], layernames[source < nlayers ? source : 0]); json_end(jp); overall_offset = layer_seq; checkdisk(reader, CPUS); @@ -1747,7 +1731,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, std::map merged_lm = merge_layermaps(layermaps); - mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable, attribution, merged_lm); + mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, nlayers, forcetable, attribution, merged_lm); return ret; } diff --git a/mbtiles.cpp b/mbtiles.cpp index 748e81d..4fa0f51 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -133,7 +133,7 @@ bool type_and_string::operator<(const type_and_string &o) const { return false; } -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, std::vector &layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, std::vector > &file_keys, int nlayers, int forcetable, const char *attribution, std::map const &layermap) { +void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, std::vector &layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int nlayers, int forcetable, const char *attribution, std::map const &layermap) { char *sql, *err; sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname); diff --git a/mbtiles.hpp b/mbtiles.hpp index 0f14881..c5e2e51 100644 --- a/mbtiles.hpp +++ b/mbtiles.hpp @@ -18,7 +18,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, std::vector &layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, std::vector > &file_keys, int nlayers, int forcetable, const char *attribution, std::map const &layermap); +void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, std::vector &layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int nlayers, int forcetable, const char *attribution, std::map const &layermap); void mbtiles_close(sqlite3 *outdb, char **argv); diff --git a/tile-join.cpp b/tile-join.cpp index a64e496..8485e3b 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -429,7 +429,7 @@ int main(int argc, char **argv) { layermap.find(layernames[i])->second.file_keys = file_keys[i]; } - mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, file_keys, nlayers, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap); + mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, nlayers, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap); mbtiles_close(outdb, argv); return 0; diff --git a/tile.cpp b/tile.cpp index 0b38900..d96a2b0 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1155,7 +1155,7 @@ struct write_tile_args { unsigned *initial_y; volatile int *running; int err; - std::vector > *layermaps; + std::vector > *layermaps; }; void *run_thread(void *vargs) { @@ -1383,7 +1383,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].pool_off = pool_off; args[thread].initial_x = initial_x; args[thread].initial_y = initial_y; - args[thread].layermaps = &layermaps; + args[thread].layermaps = &layermaps; args[thread].tasks = dispatches[thread].tasks; args[thread].running = &running;