ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-02-02 01:08:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Clean up the maximum-files-open check a little

Browse Source
This commit is contained in:
Eric Fischer 2016-04-13 20:19:41 -07:00
parent cae20bb5e6
commit 137bb46db5
3 changed files with 43 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,3 +1,7 @@
## 1.9.13
* Don't trust the OS so much about how many files can be open
## 1.9.12 ## 1.9.12
* Limit the size of the parallel parsing streaming input buffer * Limit the size of the parallel parsing streaming input buffer

34
geojson.c
View File

@ -147,31 +147,31 @@ void init_cpus() {
// Round down to a power of 2 // Round down to a power of 2
CPUS = 1 << (int) (log(CPUS) / log(2)); CPUS = 1 << (int) (log(CPUS) / log(2));
TEMP_FILES = 64;
MAX_FILES = 64;
struct rlimit rl; struct rlimit rl;
if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
perror("getrlimit"); perror("getrlimit");
exit(EXIT_FAILURE);
} else { } else {
MAX_FILES = rl.rlim_cur;
}
// Don't really want too many temporary files, because the file system
// will start to bog down eventually
if (MAX_FILES > 2000) {
MAX_FILES = 2000;
}
// MacOS can run out of system file descriptors // MacOS can run out of system file descriptors
// even if we stay under the rlimit, so try to // even if we stay under the rlimit, so try to
// find out the real limit. // find out the real limit.
long long fds[MAX_FILES];
long long fds[rl.rlim_cur];
long long i; long long i;
for (i = 0; i < rl.rlim_cur; i++) { for (i = 0; i < MAX_FILES; i++) {
fds[i] = open("/dev/null", O_RDONLY); fds[i] = open("/dev/null", O_RDONLY);
if (fds[i] < 0) { if (fds[i] < 0) {
break; break;
} }
} }
MAX_FILES = i * 3 / 4;
if (MAX_FILES > 2000) {
MAX_FILES = 2000;
}
long long j; long long j;
for (j = 0; j < i; j++) { for (j = 0; j < i; j++) {
if (close(fds[j]) < 0) { if (close(fds[j]) < 0) {
@ -180,11 +180,17 @@ void init_cpus() {
} }
} }
TEMP_FILES = MAX_FILES / 3; // Scale down because we really don't want to run the system out of files
MAX_FILES = i * 3 / 4;
if (MAX_FILES < 32) {
fprintf(stderr, "Can't open a useful number of files: %lld\n", MAX_FILES);
exit(EXIT_FAILURE);
}
TEMP_FILES = (MAX_FILES - 10) / 2;
if (TEMP_FILES > CPUS * 4) { if (TEMP_FILES > CPUS * 4) {
TEMP_FILES = CPUS * 4; TEMP_FILES = CPUS * 4;
} }
}
} }
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) { size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) {

2
version.h
View File

@ -1 +1 @@
#define VERSION "tippecanoe v1.9.12\n" #define VERSION "tippecanoe v1.9.13\n"