diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fcec72..a312786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.9.13 + +* Don't trust the OS so much about how many files can be open + ## 1.9.12 * Limit the size of the parallel parsing streaming input buffer diff --git a/geojson.c b/geojson.c index 7e42f4f..ef16bca 100644 --- a/geojson.c +++ b/geojson.c @@ -147,43 +147,49 @@ void init_cpus() { // Round down to a power of 2 CPUS = 1 << (int) (log(CPUS) / log(2)); - TEMP_FILES = 64; - MAX_FILES = 64; - struct rlimit rl; if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { perror("getrlimit"); + exit(EXIT_FAILURE); } else { - // MacOS can run out of system file descriptors - // even if we stay under the rlimit, so try to - // find out the real limit. + MAX_FILES = rl.rlim_cur; + } - long long fds[rl.rlim_cur]; - long long i; - for (i = 0; i < rl.rlim_cur; i++) { - fds[i] = open("/dev/null", O_RDONLY); - if (fds[i] < 0) { - break; - } - } + // Don't really want too many temporary files, because the file system + // will start to bog down eventually + if (MAX_FILES > 2000) { + MAX_FILES = 2000; + } - MAX_FILES = i * 3 / 4; - if (MAX_FILES > 2000) { - MAX_FILES = 2000; + // MacOS can run out of system file descriptors + // even if we stay under the rlimit, so try to + // find out the real limit. + long long fds[MAX_FILES]; + long long i; + for (i = 0; i < MAX_FILES; i++) { + fds[i] = open("/dev/null", O_RDONLY); + if (fds[i] < 0) { + break; } + } + long long j; + for (j = 0; j < i; j++) { + if (close(fds[j]) < 0) { + perror("close"); + exit(EXIT_FAILURE); + } + } - long long j; - for (j = 0; j < i; j++) { - if (close(fds[j]) < 0) { - perror("close"); - exit(EXIT_FAILURE); - } - } + // Scale down because we really don't want to run the system out of files + MAX_FILES = i * 3 / 4; + if (MAX_FILES < 32) { + fprintf(stderr, "Can't open a useful number of files: %lld\n", MAX_FILES); + exit(EXIT_FAILURE); + } - TEMP_FILES = MAX_FILES / 3; - if (TEMP_FILES > CPUS * 4) { - TEMP_FILES = CPUS * 4; - } + TEMP_FILES = (MAX_FILES - 10) / 2; + if (TEMP_FILES > CPUS * 4) { + TEMP_FILES = CPUS * 4; } } @@ -1533,10 +1539,10 @@ void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE mem = 8192; } - long long availfiles = MAX_FILES - 2 * nreaders // each reader has a geom and an index - - 4 // pool, meta, mbtiles, mbtiles journal - - 4 // top-level geom and index output, both FILE and fd - - 3; // stdin, stdout, stderr + long long availfiles = MAX_FILES - 2 * nreaders // each reader has a geom and an index + - 4 // pool, meta, mbtiles, mbtiles journal + - 4 // top-level geom and index output, both FILE and fd + - 3; // stdin, stdout, stderr // 4 because for each we have output and input FILE and fd for geom and index int splits = availfiles / 4; diff --git a/version.h b/version.h index fb760ae..06899da 100644 --- a/version.h +++ b/version.h @@ -1 +1 @@ -#define VERSION "tippecanoe v1.9.12\n" +#define VERSION "tippecanoe v1.9.13\n"