ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-02-02 01:08:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Clean up the maximum-files-open check a little

Browse Source
This commit is contained in:
Eric Fischer 2016-04-13 20:19:41 -07:00
parent cae20bb5e6
commit 137bb46db5
3 changed files with 43 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,3 +1,7 @@
## 1.9.13
* Don't trust the OS so much about how many files can be open
## 1.9.12 ## 1.9.12
* Limit the size of the parallel parsing streaming input buffer * Limit the size of the parallel parsing streaming input buffer

70
geojson.c
View File

@ -147,43 +147,49 @@ void init_cpus() {
// Round down to a power of 2 // Round down to a power of 2
CPUS = 1 << (int) (log(CPUS) / log(2)); CPUS = 1 << (int) (log(CPUS) / log(2));
TEMP_FILES = 64;
MAX_FILES = 64;
struct rlimit rl; struct rlimit rl;
if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
perror("getrlimit"); perror("getrlimit");
exit(EXIT_FAILURE);
} else { } else {
// MacOS can run out of system file descriptors MAX_FILES = rl.rlim_cur;
// even if we stay under the rlimit, so try to }
// find out the real limit.
long long fds[rl.rlim_cur]; // Don't really want too many temporary files, because the file system
long long i; // will start to bog down eventually
for (i = 0; i < rl.rlim_cur; i++) { if (MAX_FILES > 2000) {
fds[i] = open("/dev/null", O_RDONLY); MAX_FILES = 2000;
if (fds[i] < 0) { }
break;
}
}
MAX_FILES = i * 3 / 4; // MacOS can run out of system file descriptors
if (MAX_FILES > 2000) { // even if we stay under the rlimit, so try to
MAX_FILES = 2000; // find out the real limit.
long long fds[MAX_FILES];
long long i;
for (i = 0; i < MAX_FILES; i++) {
fds[i] = open("/dev/null", O_RDONLY);
if (fds[i] < 0) {
break;
} }
}
long long j;
for (j = 0; j < i; j++) {
if (close(fds[j]) < 0) {
perror("close");
exit(EXIT_FAILURE);
}
}
long long j; // Scale down because we really don't want to run the system out of files
for (j = 0; j < i; j++) { MAX_FILES = i * 3 / 4;
if (close(fds[j]) < 0) { if (MAX_FILES < 32) {
perror("close"); fprintf(stderr, "Can't open a useful number of files: %lld\n", MAX_FILES);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
TEMP_FILES = MAX_FILES / 3; TEMP_FILES = (MAX_FILES - 10) / 2;
if (TEMP_FILES > CPUS * 4) { if (TEMP_FILES > CPUS * 4) {
TEMP_FILES = CPUS * 4; TEMP_FILES = CPUS * 4;
}
} }
} }
@ -1533,10 +1539,10 @@ void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE
mem = 8192; mem = 8192;
} }
long long availfiles = MAX_FILES - 2 * nreaders // each reader has a geom and an index long long availfiles = MAX_FILES - 2 * nreaders // each reader has a geom and an index
- 4 // pool, meta, mbtiles, mbtiles journal - 4 // pool, meta, mbtiles, mbtiles journal
- 4 // top-level geom and index output, both FILE and fd - 4 // top-level geom and index output, both FILE and fd
- 3; // stdin, stdout, stderr - 3; // stdin, stdout, stderr
// 4 because for each we have output and input FILE and fd for geom and index // 4 because for each we have output and input FILE and fd for geom and index
int splits = availfiles / 4; int splits = availfiles / 4;

2
version.h
View File

@ -1 +1 @@
#define VERSION "tippecanoe v1.9.12\n" #define VERSION "tippecanoe v1.9.13\n"