diff --git a/src/include/nfdump.h b/src/include/nfdump.h index d44c51ac..b9af9e3e 100755 --- a/src/include/nfdump.h +++ b/src/include/nfdump.h @@ -132,6 +132,6 @@ typedef struct stat_record_s { // allocate space for this number of workers #define MAXWORKERS 64 // If number of CPUs can not be determined -#define DEFAULTWORKERS 4 +#define DEFAULTWORKERS 2 #endif //_NFDUMP_H diff --git a/src/libnffile/barrier.c b/src/libnffile/barrier.c index 21810c92..cc833fc4 100755 --- a/src/libnffile/barrier.c +++ b/src/libnffile/barrier.c @@ -42,33 +42,44 @@ #include #include "barrier.h" +#include "nfconf.h" #include "nfdump.h" #include "util.h" // get decent number of workers depending // on the number of cores online uint32_t GetNumWorkers(uint32_t requested) { - long CoresOnline = sysconf(_SC_NPROCESSORS_ONLN); + // get conf value for maxworkers + int confMaxWorkers = ConfGetValue("maxworkers"); + if (confMaxWorkers == 0) confMaxWorkers = DEFAULTWORKERS; + long CoresOnline = sysconf(_SC_NPROCESSORS_ONLN); if (CoresOnline < 0) { LogError("sysconf(_SC_NPROCESSORS_ONLN) error in %s line %d: %s", __FILE__, __LINE__, strerror(errno)); CoresOnline = 1; } - uint32_t numWorkers = DEFAULTWORKERS; - if (requested && requested < CoresOnline) { - numWorkers = requested; - } else { - numWorkers = CoresOnline; + // no more than cores online + if (requested && (requested > CoresOnline)) { + LogError("Number of workers should not be greater than number of cores online. %d is > %d", requested, CoresOnline); } - // limit to MAXWORKERS - if (numWorkers > MAXWORKERS) numWorkers = DEFAULTWORKERS; - - LogInfo("CoresOnline: %lld, Requested: %u, Set workers to: %u", CoresOnline, requested, numWorkers); + // try to find optimal number of workers + // if nothing requested, use default, unless we are on a high number of cores + // system, so double the workers + if (requested == 0) { + requested = confMaxWorkers; + if ((2 * requested) < CoresOnline) requested = 2 * requested; + } + if (requested > CoresOnline) requested = CoresOnline; - return numWorkers; + // no more than internal array limit + if (requested > MAXWORKERS) { + LogError("Number of workers is limited to %s", MAXWORKERS); + requested = MAXWORKERS; + } + return requested; } // End of GetNumWorkers // initialize barrier for numWorkers + 1 controller diff --git a/src/libnffile/nffile.c b/src/libnffile/nffile.c index b4dfb1ea..427f6ff2 100644 --- a/src/libnffile/nffile.c +++ b/src/libnffile/nffile.c @@ -74,8 +74,8 @@ #include "lz4.h" #include "lz4hc.h" #endif +#include "barrier.h" #include "minilzo.h" -#include "nfconf.h" #include "nfdump.h" #include "nffileV2.h" #include "util.h" @@ -169,32 +169,7 @@ int Init_nffile(int workers, queue_t *fileList) { atomic_init(&blocksInUse, 0); - // get conf value for maxworkers - int confMaxWorkers = ConfGetValue("maxworkers"); - if (confMaxWorkers == 0) confMaxWorkers = DEFAULTWORKERS; - - long CoresOnline = sysconf(_SC_NPROCESSORS_ONLN); - if (CoresOnline < 0) { - LogError("sysconf(_SC_NPROCESSORS_ONLN) error in %s line %d: %s", __FILE__, __LINE__, strerror(errno)); - CoresOnline = 1; - } - - // no more than cores online - if (workers && (workers > CoresOnline)) { - LogError("Number of workers should not be greater than number of cores online. %d is > %d", workers, CoresOnline); - } - - // set to default if not set - if (workers == 0) workers = confMaxWorkers; - if (workers > CoresOnline) workers = CoresOnline; - - // no more than internal array limit - if (workers > MAXWORKERS) { - LogError("Number of workers is limited to %s", MAXWORKERS); - workers = MAXWORKERS; - } - - NumWorkers = workers; + NumWorkers = GetNumWorkers(workers); return 1; } // End of Init_nffile