ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-01-21 03:55:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Memory-map geobuf input instead of reading it into a temporary string

Browse Source
This commit is contained in:
Eric Fischer 2017-08-28 11:30:30 -07:00
parent 5943c82457
commit 7c031a9796
3 changed files with 22 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
geobuf.cpp
View File

@ -313,8 +313,8 @@ void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std
}
}
void parse_geobuf(struct serialization_state *sst, std::string const &src, int layer, std::string layername) {
protozero::pbf_reader pbf(src);
void parse_geobuf(struct serialization_state *sst, const char *src, size_t len, int layer, std::string layername) {
protozero::pbf_reader pbf(src, len);
size_t dim = 2;
double e = 1e6;

2
geobuf.hpp
View File

@ -8,6 +8,6 @@
#include "mbtiles.hpp"
#include "serial.hpp"
void parse_geobuf(struct serialization_state *sst, std::string const &s, int layer, std::string layername);
void parse_geobuf(struct serialization_state *sst, const char *s, size_t len, int layer, std::string layername);
#endif

28
main.cpp
View File

@ -1194,16 +1194,17 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
size_t layer = a->second.id;
if (sources[source].file.size() > 4 && sources[source].file.substr(sources[source].file.size() - 4) == std::string(".pbf")) {
std::string s = "";
char buf[20000];
ssize_t n;
while ((n = read(fd, buf, 20000)) > 0) {
s.append(std::string(buf, n));
struct stat st;
if (fstat(fd, &st) != 0) {
perror("fstat");
perror(sources[source].file.c_str());
exit(EXIT_FAILURE);
}
if (close(fd) != 0) {
perror("close");
char *map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (map == MAP_FAILED) {
perror("mmap");
perror(sources[source].file.c_str());
exit(EXIT_FAILURE);
}
@ -1232,7 +1233,16 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
sst.basezoom = basezoom;
sst.attribute_types = attribute_types;
parse_geobuf(&sst, s, layer, sources[layer].layer);
parse_geobuf(&sst, map, st.st_size, layer, sources[layer].layer);
if (munmap(map, st.st_size) != 0) {
perror("munmap source file");
exit(EXIT_FAILURE);
}
if (close(fd) != 0) {
perror("close");
exit(EXIT_FAILURE);
}
overall_offset = layer_seq;
checkdisk(reader, CPUS);