Update nfdump(1) man page for ndjson and cleanup

man/nfdump.1

@@ -634,8 +634,8 @@ format. See the section
 below for more details on how to compile your own csv format.
 .It Cm json
 Print full record as a separate json object.
-.It Cm json-log
-Print full record as a one line separate json object. Suitable for log processors such as logstash.
+.It Cm ndjson
+Print full record as a one line json object, sepatated by newline. Suitable for log processors such as logstash.
 .It Cm csv
 Print reocrd in csv format - format compatible to fmt
 .Sy line
@@ -1564,7 +1564,7 @@ If comp is omitted, '==' is assumed.
 This section describes how output formats are compiled.
 .Nm
 has a lot of already pre-defined output formats such as
-.Ar raw, json, json-log, csv
+.Ar raw, json, ndjson, csv
 etc. One line formats supplied with option
 .Fl o
 can be compiled from various elements of a flow record. As a flow record contains many different

src/maxmind/mmcreate.c

@@ -119,6 +119,18 @@ static FILE *checkFile(char *fileName, char **fieldNames) {
 
 }  // End of checkFile
 
+static void strCopyReplace(char *dst, char *src) {
+    int i = 0;
+    for (i = 0; src[i] != 0; i++) {
+        // convert " to ' in org name, otherwise it breaks json output
+        if (src[i] == '"')
+            dst[i] = '\'';
+        else
+            dst[i] = src[i];
+    }
+    dst[i] = '\0';
+}  // End of strCopyReplace
+
 static int loadLocalMap(char *fileName) {
     FILE *fp = checkFile(fileName, localFieldNames);
     if (!fp) {
@@ -180,7 +192,7 @@ static int loadLocalMap(char *fileName) {
                     } else if (divisionName && strlen(divisionName) > 0) {
                         strcpy(locationInfo.city, divisionName);
                     } else if (countryName && strlen(countryName) > 0) {
-                        strcpy(locationInfo.city, countryName);
+                        strCopyReplace(locationInfo.city, countryName);
                     } else {
                         strcpy(locationInfo.city, "unknown");
                     }
@@ -402,9 +414,9 @@ static int loadASV4tree(char *fileName) {
         }
 
         // extract org name
-        strncpy(asV4Node.orgName, field, orgNameLength);
+        strCopyReplace(asV4Node.orgName, field);
         asV4Node.orgName[orgNameLength - 1] = '\0';
-        strncpy(asOrgNode.orgName, field, orgNameLength);
+        strCopyReplace(asOrgNode.orgName, field);
         asOrgNode.orgName[orgNameLength - 1] = '\0';
 
         // insert node
@@ -481,9 +493,9 @@ static int loadASV6tree(char *fileName) {
         field = sep;
 
         // extract org name
-        strncpy(asV6Node.orgName, field, orgNameLength);
+        strCopyReplace(asV6Node.orgName, field);
         asV6Node.orgName[orgNameLength - 1] = '\0';
-        strncpy(asOrgNode.orgName, field, orgNameLength);
+        strCopyReplace(asOrgNode.orgName, field);
         asOrgNode.orgName[orgNameLength - 1] = '\0';
 
         PutasV6Node(&asV6Node);

src/nfdump/nfdump.c

@@ -186,7 +186,7 @@ static void usage(char *name) {
         "\t\t extended Even more information.\n"
         "\t\t csv      ',' separated, machine parseable output format.\n"
         "\t\t json     json output format.\n"
-        "\t\t json-log json log output format (one json record per line).\n"
+        "\t\t ndjson   ndjson log output format (one json object per line).\n"
         "\t\t null     no flow records, only statistics output.\n"
         "\t\t\tmode may be extended by '6' for full IPv6 listing. e.g.long6, extended6.\n"
         "-E <file>\tPrint exporter and sampling info for collected flows.\n"
@@ -1303,8 +1303,7 @@ int main(int argc, char **argv) {
             case MODE_CSV_FAST:
                 break;
             case MODE_JSON:
-                break;
-            case MODE_JSON_LOG:
+            case MODE_NDJSON:
                 break;
         }
 

src/nfdump/nfstat.c

@@ -77,7 +77,8 @@ typedef enum {
     IS_ASORG
 } elementType_t;
 
-typedef enum { DESCENDING = 0, ASCENDING } direction_t;
+typedef enum { DESCENDING = 0,
+               ASCENDING } direction_t;
 
 /*
  * pre-process functions:
@@ -256,7 +257,9 @@ typedef struct StatRecord {
  * orderby functions:
  * retrieve or calculate value, records want to be ordered by.
  */
-typedef enum flowDir { IN = 0, OUT, INOUT } flowDir_t;
+typedef enum flowDir { IN = 0,
+                       OUT,
+                       INOUT } flowDir_t;
 typedef uint64_t (*order_proc_element_t)(StatRecord_t *);
 
 static uint64_t order_bytes_in(StatRecord_t *record);
@@ -1446,7 +1449,7 @@ void PrintElementStat(stat_record_t *sum_stat, outputParams_t *outputParams, Rec
                                              outputParams->doTag, orderByTable[order_index].inout);
                             break;
                         case MODE_JSON:
-                        case MODE_JSON_LOG:
+                        case MODE_NDJSON:
                             PrintJsonStatLine(StatParameters[stat].statname, sum_stat, outputParams, &topN_element_list[index], type,
                                               StatRequest[hash_num].order_proto, orderByTable[order_index].inout);
                             break;

src/output/Makefile.am

@@ -6,7 +6,8 @@ noinst_LIBRARIES = liboutput.a
 liboutput_a_SOURCES = output.c output.h \
 	output_util.c output_util.h  output_raw.c output_raw.h \
 	output_csv.c output_csv.h output_csv_fast.c \
-	output_fmt.c output_fmt.h output_json.c output_json.h
+	output_fmt.c output_fmt.h \
+	output_json.c output_json.h output_ndjson.c output_ndjson.h
 
 EXTRA_DIST = itoa.c
 

src/output/output.c

@@ -43,6 +43,7 @@
 #include "output_csv.h"
 #include "output_fmt.h"
 #include "output_json.h"
+#include "output_ndjson.h"
 #include "output_raw.h"
 #include "util.h"
 
@@ -81,9 +82,9 @@ static void AddFormat(char *format, char *name, char *fmtString);
 
 static void null_record(FILE *stream, recordHandle_t *record, int tag);
 
-static void null_prolog(void);
+static void null_prolog(outputParams_t *outputParam);
 
-static void null_epilog(void);
+static void null_epilog(outputParams_t *outputParam);
 
 // Assign print functions for all output options -o
 // Terminated with a NULL record
@@ -103,7 +104,7 @@ static struct printmap_s {
                           {"nsel", MODE_FMT, FORMAT_nsel, "predefined"},
                           {"nat", MODE_FMT, FORMAT_nat, "predefined"},
                           {"json", MODE_JSON, NULL, "json output"},
-                          {"json-log", MODE_JSON_LOG, NULL, "json output for logging"},
+                          {"ndjson", MODE_NDJSON, NULL, "ndjson output formart"},
                           {"csv", MODE_CSV, FORMAT_CSV, "csv predefined"},
                           {"csv-fast", MODE_CSV_FAST, NULL, "csv fast predefined"},
                           {"null", MODE_NULL, NULL, "do not print any output"},
@@ -121,8 +122,8 @@ static struct printerFunc_s {
                     [MODE_RAW] = {raw_record, raw_prolog, raw_epilog},
                     [MODE_CSV] = {csv_record, csv_prolog, csv_epilog},
                     [MODE_CSV_FAST] = {csv_record_fast, csv_prolog_fast, csv_epilog_fast},
-                    [MODE_JSON] = {flow_record_to_json_human, json_prolog, json_epilog},
-                    [MODE_JSON_LOG] = {flow_record_to_json_log, json_prolog, json_epilog}};
+                    [MODE_JSON] = {flow_record_to_json, json_prolog, json_epilog},
+                    [MODE_NDJSON] = {flow_record_to_ndjson, ndjson_prolog, ndjson_epilog}};
 
 static PrologPrinter_t print_prolog;  // prints the output prolog
 static PrologPrinter_t print_epilog;  // prints the output epilog
@@ -133,11 +134,11 @@ static void null_record(FILE *stream, recordHandle_t *record, int tag) {
     // empty - do not list any flows
 }  // End of null_record
 
-static void null_prolog(void) {
+static void null_prolog(outputParams_t *outputParam) {
     // empty prolog
 }  // End of null_prolog
 
-static void null_epilog(void) {
+static void null_epilog(outputParams_t *outputParam) {
     // empty epilog
 }  // End of null_epilog
 
@@ -308,11 +309,11 @@ RecordPrinter_t SetupOutputMode(char *print_format, outputParams_t *outputParams
 }  // End of SetupOutputMode
 
 void PrintProlog(outputParams_t *outputParams) {
-    if (!outputParams->quiet) print_prolog();
+    print_prolog(outputParams);
 }  // End of PrintProlog
 
 void PrintEpilog(outputParams_t *outputParams) {
-    if (!outputParams->quiet) print_epilog();
+    print_epilog(outputParams);
 }  // End of PrintEpilog
 
 void PrintOutputHelp(void) {

src/output/output.h

@@ -36,11 +36,13 @@
 
 #include "nfdump.h"
 
-typedef void (*RecordPrinter_t)(FILE *, recordHandle_t *, int);
-typedef void (*PrologPrinter_t)(void);
-typedef void (*EpilogPrinter_t)(void);
-
-typedef enum { MODE_NULL = 0, MODE_RAW, MODE_FMT, MODE_CSV, MODE_CSV_FAST, MODE_JSON, MODE_JSON_LOG } outputMode_t;
+typedef enum { MODE_NULL = 0,
+               MODE_RAW,
+               MODE_FMT,
+               MODE_CSV,
+               MODE_CSV_FAST,
+               MODE_JSON,
+               MODE_NDJSON } outputMode_t;
 
 typedef struct outputParams_s {
     bool printPlain;
@@ -53,6 +55,10 @@ typedef struct outputParams_s {
     void *postFilter;
 } outputParams_t;
 
+typedef void (*RecordPrinter_t)(FILE *, recordHandle_t *, int);
+typedef void (*PrologPrinter_t)(outputParams_t *);
+typedef void (*EpilogPrinter_t)(outputParams_t *);
+
 RecordPrinter_t SetupOutputMode(char *print_format, outputParams_t *outputParams);
 
 void PrintProlog(outputParams_t *outputParams);

src/output/output_csv.c

@@ -645,7 +645,7 @@ void csv_record(FILE *stream, recordHandle_t *recordHandle, int tag) {
 
 }  // End of csv_record
 
-void csv_prolog(void) {
+void csv_prolog(outputParams_t *outputParam) {
     streamBuff = malloc(STREAMBUFFSIZE);
     if (!streamBuff) {
         LogError("malloc() error in %s line %d: %s", __FILE__, __LINE__, strerror(errno));
@@ -657,10 +657,9 @@ void csv_prolog(void) {
     printf("%s\n", header_string);
 }  // End of csv_prolog
 
-void csv_epilog(void) {
+void csv_epilog(outputParams_t *outputParam) {
     free(streamBuff);
     streamBuff = NULL;
-    // empty
 }  // End of csv_epilog
 
 static void InitFormatParser(void) {

src/output/output_csv.h

@@ -38,15 +38,15 @@
 
 int ParseCSVOutputFormat(char *format);
 
-void csv_prolog(void);
+void csv_prolog(outputParams_t *outputParam);
 
-void csv_epilog(void);
+void csv_epilog(outputParams_t *outputParam);
 
 void csv_record(FILE *stream, recordHandle_t *recordHandle, int tag);
 
-void csv_prolog_fast(void);
+void csv_prolog_fast(outputParams_t *outputParam);
 
-void csv_epilog_fast(void);
+void csv_epilog_fast(outputParams_t *outputParam);
 
 void csv_record_fast(FILE *stream, recordHandle_t *recordHandle, int tag);
 

src/output/output_csv_fast.c

@@ -76,7 +76,7 @@ static uint32_t recordCount;
 #define STREAMBUFFSIZE 1014
 static char *streamBuff = NULL;
 
-void csv_prolog_fast(void) {
+void csv_prolog_fast(outputParams_t *outputParam) {
     // empty prolog
     recordCount = 0;
     streamBuff = malloc(STREAMBUFFSIZE);
@@ -88,7 +88,7 @@ void csv_prolog_fast(void) {
     printf("cnt,af,firstSeen,lastSeen,proto,srcAddr,srcPort,dstAddr,dstPort,srcAS,dstAS,input,output,flags,srcTos,packets,bytes\n");
 }  // End of csv_prolog_fast
 
-void csv_epilog_fast(void) {
+void csv_epilog_fast(outputParams_t *outputParam) {
     // empty epilog
     free(streamBuff);
     streamBuff = NULL;

src/output/output_fmt.c

@@ -672,12 +672,13 @@ void fmt_record(FILE *stream, recordHandle_t *recordHandle, int tag) {
 
 }  // End of fmt_record
 
-void fmt_prolog(void) {
+void fmt_prolog(outputParams_t *outputParam) {
     // header
-    printf("%s\n", header_string);
+    if (outputParam->quiet == 0)
+        printf("%s\n", header_string);
 }  // End of fmt_prolog
 
-void fmt_epilog(void) {
+void fmt_epilog(outputParams_t *outputParam) {
     // empty
 }  // End of fmt_epilog
 

src/output/output_fmt.h

@@ -47,9 +47,9 @@ void Setv6Mode(int mode);
 
 int Getv6Mode(void);
 
-void fmt_prolog(void);
+void fmt_prolog(outputParams_t *outputParam);
 
-void fmt_epilog(void);
+void fmt_epilog(outputParams_t *outputParam);
 
 void CondenseV6(char *s);