Fix #469. Update usage and error messages and man pages

man/nfcapd.1

@@ -61,6 +61,7 @@
 .Op Fl e
 .Op Fl x Ar command
 .Op Fl X Ar extensionList
+.Op Fl W Ar workers
 .Op Fl E
 .Op Fl v
 .Op Fl V
@@ -256,15 +257,22 @@ In that case the announced sampling rate is applied. If
 is negative, this will hard overwrite any device specific announced sampling rates.
 The sampling rate is used to multiply the number of packets and bytes in a record. Please
 note, this may vary from other volume counters such as SNMP etc.
-.It Fl z
-Compress flow files with LZO1X-1 compression. Fastest compression.
 .It Fl z=lzo
 Compress flow files with LZO1X-1 compression. Fastest compression.
-.It Fl z=lz4
-Compress flow files with LZ4 compression. Fast and efficient.
 .It Fl z=bz2
 Compress flow files with bz2 compression. Slow but most efficient. It is not recommended 
 to use bz2 in a real time capturing.
+.It Fl z=lz4[:level]
+Compress flow files with LZ4 compression. Fast and efficient. Optional level should be between 1..10
+Changing the level results in smaller files but uses up more time to compress. Levels > 5 may need more
+workers. See -W.
+.It Fl z=zstd[:level]
+Compress flow files with ZSTD compression. Fast and efficient. Optional level should be between 1..10
+Changing the level results in smaller files but uses up more time to compress. Levels > 5 may need more
+workers. See -W.
+.It Fl W Ar num
+Sets the number of workers to compress flows. Defaults to 4. Must not be greater than the number of
+cores online. Useful for higher levels of compression for lz4 or zstd and large amount of flows per second.
 .It Fl e
 Sets auto-expire mode. At the end of every rotate interval
 .Fl t

man/nfdump.1

@@ -59,6 +59,7 @@
 .Op Fl v Ar flowfile
 .Op Fl E Ar flowfile
 .Op Fl x Ar flowfile
+.Op Fl W Ar workers
 .Op Fl z=<compress>
 .Op Fl J Ar compress
 .Op Fl X
@@ -667,11 +668,21 @@ Scans and prints extension maps located in
 .Ar flowfile
 .It Fl z=lzo
 Compress flow files with LZO1X-1 compression. Fastest compression.
-.It Fl z=lz4
-Compress flow files with LZ4 compression. Fast and efficient.
 .It Fl z=bz2
 Compress flow files with bz2 compression. Slow but most efficient. May be used
 for archiving files or if you are really short of spce.
+.It Fl z=lz4[:level]
+Compress flow files with LZ4 compression. Fast and efficient. Optional level should be between 1..10
+Changing the level results in smaller files but uses up more time to compress. Levels > 5 may need more
+workers. See -W.
+.It Fl z=zstd[:level]
+Compress flow files with ZSTD compression. Fast and efficient. Optional level should be between 1..10
+Changing the level results in smaller files but uses up more time to compress. Levels > 5 may need more
+workers. See -W.
+.It Fl W Ar num
+Sets the number of workers to compress flows. Defaults to 4. Must not be greater than the number of
+cores online. Useful for higher levels of compression for lz4 or zstd and large amount of flows per second.
+Please not, -W affects only writing flows.
 .It Fl J Ar compress
 Change compression for any number of files given by option
 .Fl r Ar flowpath

man/nfpcapd.1

@@ -114,12 +114,19 @@ Adds options to nfpcapd. Two options are available:
 .B -z=lzo
 Compress flows. Use fast LZO1X\-1 compression in output file.
 .TP 3
-.B -z=lz4
-Compress flows. Use LZ4 compression in output file.
-.TP 3
 .B -z=bz2
 Compress flows. Use bz2 compression in output file. Note: not recommended while collecting
 .TP 3
+.B -z=lz4[:level]
+Compress flows. Use LZ4 compression in output file.
+.TP 3
+.B -z=zstd[:level]
+Compress flows. Use zstd compression in output file.
+.TP 3
+.B -W \fIworkers
+Sets the number of workers to compress flows. Defaults to 4. Must not be greater than the number of
+cores online. Useful for higher levels of compression for lz4 or zstd and large amount of flows per second.
+.TP 3
 .B -V
 Print nfpcapd version and exit.
 .TP 3

man/sfcapd.1

@@ -59,6 +59,7 @@
 .Op Fl m Ar metricpath
 .Op Fl e
 .Op Fl x Ar command
+.Op Fl W Ar workers
 .Op Fl E
 .Op Fl v
 .Op Fl V
@@ -218,11 +219,20 @@ Sets the time interval in seconds to rotate files. The default value is 300s ( 5
 The smallest available interval is 2s.
 .It Fl z=lzo
 Compress flow files with LZO1X-1 compression. Fastest compression.
-.It Fl z=lz4
-Compress flow files with LZ4 compression. Fast and efficient.
 .It Fl z=bz2
 Compress flow files with bz2 compression. Slow but most efficient. It is not recommended 
 to use bz2 in a real time capturing.
+.It Fl z=lz4[:level]
+Compress flow files with LZ4 compression. Fast and efficient. Optional level should be between 1..10
+Changing the level results in smaller files but uses up more time to compress. Levels > 5 may need more
+workers. See -W.
+.It Fl z=zstd[:level]
+Compress flow files with ZSTD compression. Fast and efficient. Optional level should be between 1..10
+Changing the level results in smaller files but uses up more time to compress. Levels > 5 may need more
+workers. See -W.
+.It Fl W Ar num
+Sets the number of workers to compress flows. Defaults to 4. Must not be greater than the number of
+cores online. Useful for higher levels of compression for lz4 or zstd and large amount of flows per second.
 .It Fl e
 Sets auto-expire mode. At the end of every rotate interval
 .Fl t

src/ft2nfdump/ft2nfdump.c

@@ -95,9 +95,10 @@ void usage(char *name) {
         "-V\t\tPrint version and exit.\n"
         "-r <file>\tread flow-tools records from file\n"
         "-w <file>\twrite nfdump records to file\n"
-        "-j\t\tBZ2 compress flows in output file.\n"
-        "-y\t\tLZ4 compress flows in output file.\n"
-        "-z\t\tLZO compress flows in output file.\n"
+        "-z=lzo\t\tLZO compress flows in output file.\n"
+        "-z=bz2\t\tBZIP2 compress flows in output file.\n"
+        "-z=lz4[:level]\tLZ4 compress flows in output file.\n"
+        "-z=zstd[:level]\tZSTD compress flows in output file.\n"
         "Convert flow-tools format to nfdump format:\n"
         "ft2nfdump -r <flow-tools-data-file> -w <nfdump-file> [-z]\n",
         name);
@@ -343,7 +344,15 @@ int main(int argc, char **argv) {
                     LogError("Use one compression: -z for LZO, -j for BZ2 or -y for LZ4 compression");
                     exit(EXIT_FAILURE);
                 }
-                compress = LZO_COMPRESSED;
+                if (optarg == NULL) {
+                    compress = LZO_COMPRESSED;
+                } else {
+                    compress = ParseCompression(optarg);
+                }
+                if (compress == -1) {
+                    LogError("Usage for option -z: set -z=lzo, -z=lz4, -z=bz2 or z=zstd for valid compression formats");
+                    exit(EXIT_FAILURE);
+                }
                 break;
             case 'r':
                 ftfile = optarg;

src/nfcapd/nfcapd.c

@@ -146,9 +146,11 @@ static void usage(char *name) {
         "-A\t\tEnable source address spoofing for packet repeater -R.\n"
         "-s rate\tset default sampling rate (default 1)\n"
         "-x process\tlaunch process after a new file becomes available\n"
+        "-W workers\toptionally set the number of workers to compress flows\n"
         "-z=lzo\t\tLZO compress flows in output file.\n"
         "-z=bz2\t\tBZIP2 compress flows in output file.\n"
         "-z=lz4[:level]\tLZ4 compress flows in output file.\n"
+        "-z=zstd[:level]\tZSTD compress flows in output file.\n"
         "-B bufflen\tSet socket buffer to bufflen bytes\n"
         "-e\t\tExpire data at each cycle.\n"
         "-D\t\tFork to background\n"
@@ -595,7 +597,7 @@ int main(int argc, char **argv) {
     FlowSource_t *fs;
     int family, bufflen, metricInterval;
     time_t twin;
-    int sock, do_daemonize, expire, spec_time_extension;
+    int sock, do_daemonize, expire, spec_time_extension, workers;
     int subdir_index, sampling_rate, compress, srcSpoofing;
 #ifdef PCAP
     char *pcap_file = NULL;
@@ -629,9 +631,10 @@ int main(int argc, char **argv) {
     metricSocket = NULL;
     metricInterval = 60;
     extensionList = NULL;
+    workers = 0;
 
     int c;
-    while ((c = getopt(argc, argv, "46AB:b:C:d:DeEf:g:hI:i:jJ:l:m:M:n:p:P:R:s:S:t:T:u:vVw:x:X:yz::Z")) != EOF) {
+    while ((c = getopt(argc, argv, "46AB:b:C:d:DeEf:g:hI:i:jJ:l:m:M:n:p:P:R:s:S:t:T:u:vVW:w:x:X:yz::Z")) != EOF) {
         switch (c) {
             case 'h':
                 usage(argv[0]);
@@ -819,6 +822,14 @@ int main(int argc, char **argv) {
                 CheckArgLen(optarg, 128);
                 extensionList = strdup(optarg);
                 break;
+            case 'W':
+                CheckArgLen(optarg, 16);
+                workers = atoi(optarg);
+                if (workers < 0 || workers > MAXWORKERS) {
+                    LogError("Number of working threads out of range 1..%d", MAXWORKERS);
+                    exit(EXIT_FAILURE);
+                }
+                break;
             case 'j':
                 if (compress) {
                     LogError("Use one compression: -z for LZO, -j for BZ2 or -y for LZ4 compression");
@@ -844,7 +855,7 @@ int main(int argc, char **argv) {
                     compress = ParseCompression(optarg);
                 }
                 if (compress == -1) {
-                    LogError("Usage for option -z: set -z=lzo, -z=lz4 or -z=bz2 for valid compression formats");
+                    LogError("Usage for option -z: set -z=lzo, -z=lz4, -z=bz2 or z=zstd for valid compression formats");
                     exit(EXIT_FAILURE);
                 }
                 break;
@@ -925,7 +936,7 @@ int main(int argc, char **argv) {
         exit(EXIT_FAILURE);
     }
 
-    if (!Init_nffile(0, NULL)) exit(254);
+    if (!Init_nffile(workers, NULL)) exit(254);
 
     if (expire && spec_time_extension) {
         LogError("ERROR, -Z timezone extension breaks expire -e");

src/nfdump/nfdump.c

@@ -135,6 +135,7 @@ static void usage(char *name) {
         "-z=lzo\t\tLZO compress flows in output file.\n"
         "-z=bz2\t\tBZIP2 compress flows in output file.\n"
         "-z=lz4[:level]\tLZ4 compress flows in output file.\n"
+        "-z=zstd[:level]\tZSTD compress flows in output file.\n"
         "-l <expr>\tSet limit on packets for line and packed output format.\n"
         "\t\tkey: 32 character string or 64 digit hex string starting with 0x.\n"
         "-L <expr>\tSet limit on bytes for line and packed output format.\n"
@@ -163,7 +164,7 @@ static void usage(char *name) {
         "\t\t\tmode may be extended by '6' for full IPv6 listing. e.g.long6, extended6.\n"
         "-E <file>\tPrint exporter and sampling info for collected flows.\n"
         "-v <file>\tverify netflow data file. Print version and blocks.\n"
-        "-W <num>\tNumber of working threads. Default number of cores online.\n"
+        "-W <num>\tOptionally set the number of workers to compress flows\n"
         "-x <file>\tverify extension records in netflow data file.\n"
         "-X\t\tDump Filtertable and exit (debug option).\n"
         "-Z\t\tCheck filter syntax and exit.\n"
@@ -690,21 +691,21 @@ int main(int argc, char **argv) {
                 break;
             case 'j':
                 if (compress) {
-                    LogError("Use one compression: -z for LZO, -j for BZ2 or -y for LZ4 compression");
+                    LogError("Use one compression only: set -z=lzo, -z=lz4, -z=bz2 or z=zstd for valid compression formats");
                     exit(EXIT_FAILURE);
                 }
                 compress = BZ2_COMPRESSED;
                 break;
             case 'y':
                 if (compress) {
-                    LogError("Use one compression: -z for LZO, -j for BZ2 or -y for LZ4 compression");
+                    LogError("Use one compression only: set -z=lzo, -z=lz4, -z=bz2 or z=zstd for valid compression formats");
                     exit(EXIT_FAILURE);
                 }
                 compress = LZ4_COMPRESSED;
                 break;
             case 'z':
                 if (compress) {
-                    LogError("Use one compression: -z for LZO, -j for BZ2 or -y for LZ4 compression");
+                    LogError("Use one compression only: set -z=lzo, -z=lz4, -z=bz2 or z=zstd for valid compression formats");
                     exit(EXIT_FAILURE);
                 }
                 if (optarg == NULL) {
@@ -713,7 +714,7 @@ int main(int argc, char **argv) {
                     compress = ParseCompression(optarg);
                 }
                 if (compress == -1) {
-                    LogError("Usage for option -z: set -z=lzo, -z=lz4 or -z=bz2 for valid compression formats");
+                    LogError("Usage for option -z: set -z=lzo, -z=lz4, -z=bz2 or z=zstd for valid compression formats");
                     exit(EXIT_FAILURE);
                 }
                 break;

src/nfpcapd/nfpcapd.c

@@ -146,8 +146,9 @@ static void usage(char *name) {
         "-P pidfile\tset the PID file\n"
         "-t time frame\tset the time window to rotate pcap/nfcapd file\n"
         "-z=lzo\t\tLZO compress flows in output file.\n"
-        "-z=lz4\t\tLZ4 compress flows in output file.\n"
         "-z=bz2\t\tBZIP2 compress flows in output file.\n"
+        "-z=lz4[:level]\tLZ4 compress flows in output file.\n"
+        "-z=zstd[:level]\tZSTD compress flows in output file.\n"
         "-v\t\tverbose logging.\n"
         "-D\t\tdetach from terminal (daemonize)\n",
         name);
@@ -281,7 +282,7 @@ int main(int argc, char *argv[]) {
     struct sigaction sa;
     int c, snaplen, bufflen, err, do_daemonize;
     int subdir_index, compress, expire, cache_size, buff_size;
-    int activeTimeout, inactiveTimeout, metricInterval;
+    int activeTimeout, inactiveTimeout, metricInterval, workers;
     dirstat_t *dirstat;
     repeater_t *sendHost;
     time_t t_win;
@@ -316,6 +317,8 @@ int main(int argc, char *argv[]) {
     buff_size = 20;
     activeTimeout = 0;
     inactiveTimeout = 0;
+    workers = 0;
+
     while ((c = getopt(argc, argv, "b:B:C:De:g:hH:I:i:j:l:m:o:p:P:r:s:S:T:t:u:vVw:yz::")) != EOF) {
         switch (c) {
             struct stat fstat;
@@ -471,6 +474,14 @@ int main(int argc, char *argv[]) {
                     time_extension = "%Y%m%d%H%M%S";
                 }
                 break;
+            case 'W':
+                CheckArgLen(optarg, 16);
+                workers = atoi(optarg);
+                if (workers < 0 || workers > MAXWORKERS) {
+                    LogError("Number of working threads out of range 1..%d", MAXWORKERS);
+                    exit(EXIT_FAILURE);
+                }
+                break;
             case 'j':
                 if (compress) {
                     LogError("Use either -z for LZO or -j for BZ2 compression, but not both");
@@ -496,7 +507,7 @@ int main(int argc, char *argv[]) {
                     compress = ParseCompression(optarg);
                 }
                 if (compress == -1) {
-                    LogError("Usage for option -z: set -z=lzo, -z=lz4 or -z=bz2 for valid compression formats");
+                    LogError("Usage for option -z: set -z=lzo, -z=lz4, -z=bz2 or z=zstd for valid compression formats");
                     exit(EXIT_FAILURE);
                 }
                 break;
@@ -626,7 +637,7 @@ int main(int argc, char *argv[]) {
             exit(EXIT_FAILURE);
         }
 
-        if (!Init_nffile(0, NULL)) exit(EXIT_FAILURE);
+        if (!Init_nffile(workers, NULL)) exit(EXIT_FAILURE);
 
         if (subdir_index && !InitHierPath(subdir_index)) {
             pcap_close(packetParam.pcap_dev);

src/nfsen/nfprofile.c

@@ -91,6 +91,7 @@ static void usage(char *name) {
         "-z=lzo\t\tLZO compress flows in output file.\n"
         "-z=bz2\t\tBZIP2 compress flows in output file.\n"
         "-z=lz4[:level]\tLZ4 compress flows in output file.\n"
+        "-z=zstd[:level]\tZSTD compress flows in output file.\n"
 #ifdef HAVE_INFLUXDB
         "-i <influxurl>\tInfluxdb url for stats (example: http://localhost:8086/write?db=mydb&u=pippo&p=paperino)\n"
 #endif
@@ -524,7 +525,7 @@ int main(int argc, char **argv) {
                 break;
             case 'z':
                 if (compress) {
-                    LogError("Use one compression: -z for LZO, -j for BZ2 or -y for LZ4 compression");
+                    LogError("Use one compression only: set -z=lzo, -z=lz4, -z=bz2 or z=zstd for valid compression formats");
                     exit(EXIT_FAILURE);
                 }
                 if (optarg == NULL) {
@@ -533,7 +534,7 @@ int main(int argc, char **argv) {
                     compress = ParseCompression(optarg);
                 }
                 if (compress == -1) {
-                    LogError("Usage for option -z: set -z=lzo, -z=lz4 or -z=bz2 for valid compression formats");
+                    LogError("Usage for option -z: set -z=lzo, -z=lz4, -z=bz2 or z=zstd for valid compression formats");
                     exit(EXIT_FAILURE);
                 }
                 break;