diff --git a/geojson.c b/geojson.c index 194f87d..cc9d499 100644 --- a/geojson.c +++ b/geojson.c @@ -17,6 +17,7 @@ #include <limits.h> #include <sqlite3.h> #include <stdarg.h> +#include <sys/resource.h> #include "jsonpull.h" #include "tile.h" @@ -60,6 +61,27 @@ int mb_geometry[GEOM_TYPES] = { VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON, }; +int CPUS; +int TEMP_FILES; + +void init_cpus() { + CPUS = sysconf(_SC_NPROCESSORS_ONLN); + if (CPUS < 1) { + CPUS = 1; + } + + TEMP_FILES = 64; + struct rlimit rl; + if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { + perror("getrlimit"); + } else { + TEMP_FILES = rl.rlim_cur / 3; + if (TEMP_FILES > CPUS * 4) { + TEMP_FILES = CPUS * 4; + } + } +} + size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) { size_t w = fwrite(ptr, size, nitems, stream); if (w != nitems) { @@ -1167,6 +1189,8 @@ int main(int argc, char **argv) { mtrace(); #endif + init_cpus(); + extern int optind; extern char *optarg; int i; diff --git a/tile.cc b/tile.cc index ee1c928..bdd95a6 100644 --- a/tile.cc +++ b/tile.cc @@ -818,7 +818,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsi struct task { int fileno; struct task *next; -} tasks[TEMP_FILES]; +}; struct write_tile_args { struct task *tasks; @@ -961,9 +961,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo } } -#define MAX_THREADS 20 // XXX Obtain from sysctl(hw.ncpu), /proc/cpuinfo, etc. - - int threads = MAX_THREADS; + int threads = CPUS; if (threads > TEMP_FILES / 4) { threads = TEMP_FILES / 4; } @@ -978,6 +976,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo // Assign temporary files to threads + struct task tasks[TEMP_FILES]; struct dispatch { struct task *tasks; long long todo; diff --git a/tile.h b/tile.h index 8d73566..b2746e7 100644 --- a/tile.h +++ b/tile.h @@ -33,4 +33,5 @@ extern unsigned initial_x, initial_y; extern int geometry_scale; extern int quiet; -#define TEMP_FILES 64 +extern int CPUS; +extern int TEMP_FILES;