From cc0f5938514abad0c4e520dcce2a25a32fe50ada Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Mon, 29 Jul 2024 14:47:00 +0200 Subject: [PATCH] Add token %tsg, %teg for GMT start and end time in fmt and csv format. Update man page --- man/nfdump.1 | 10 ++++ src/output/output_csv.c | 68 ++++++++++++++++++++++- src/output/output_fmt.c | 116 +++++++++++++++++++++++++++++++--------- 3 files changed, 168 insertions(+), 26 deletions(-) diff --git a/man/nfdump.1 b/man/nfdump.1 index 518d210b..18a8e614 100755 --- a/man/nfdump.1 +++ b/man/nfdump.1 @@ -634,10 +634,14 @@ format. See the section below for more details on how to compile your own csv format. .It Cm json Print full record as a separate json object. +.It Cm json-log +Print full record as a one line separate json object. Suitable for log processors such as logstash. .It Cm csv Print reocrd in csv format - format compatible to fmt .Sy line format. +.It Cm csv-fast +Replaces old pipe format. Basic record information only. Fast implementation. .El .Pp Already predefined fmt formats: @@ -1659,14 +1663,20 @@ Start Time - first seen First seen - identical to %ts .It Cm %tsr Start Time, but in fractional seconds since the epoch (1970-01-01) UNIX format. +.It Cm %tsg +Start Time GMT - first seem .It Cm %te End Time - last seen .It Cm %ter End Time, in fractional seconds +.It Cm %teg +End Time GMT - last seen .It Cm %tr Time the flow was received by the collector .It Cm %trr Time the flow was received, in fractional seconds +.It Cm %trg +Time GMT the flow was received by the collector .It Cm %td Duration of flow. Displayed in ddHHMMSS.msec .It Cm %pr diff --git a/src/output/output_csv.c b/src/output/output_csv.c index 1953cbd1..abf4cec8 100644 --- a/src/output/output_csv.c +++ b/src/output/output_csv.c @@ -107,6 +107,12 @@ static void String_LastSeenRaw(FILE *stream, recordHandle_t *recordHandle); static void String_ReceivedRaw(FILE *stream, recordHandle_t *recordHandle); +static void String_FirstSeenGMT(FILE *stream, recordHandle_t *recordHandle); + +static void String_LastSeenGMT(FILE *stream, recordHandle_t *recordHandle); + +static void String_ReceivedGMT(FILE *stream, recordHandle_t *recordHandle); + static void String_Duration(FILE *stream, recordHandle_t *recordHandle); static void String_Duration_Seconds(FILE *stream, recordHandle_t *recordHandle); @@ -343,10 +349,13 @@ static struct format_entry_s { {"%tfs", 0, "firstSeen", String_FirstSeen}, // Start Time - first seen {"%ts", 0, "firstSeen", String_FirstSeen}, // Start Time - first seen {"%tsr", 0, "firstSeen", String_FirstSeenRaw}, // Start Time - first seen, seconds + {"%tsg", 0, "firstSeen", String_FirstSeenGMT}, // Start Time GMT - first seen, seconds {"%te", 0, "lastSeen", String_LastSeen}, // End Time - last seen {"%ter", 0, "lastSeen", String_LastSeenRaw}, // End Time - first seen, seconds - {"%trr", 0, "received", String_ReceivedRaw}, // Received Time, seconds + {"%teg", 0, "lastSeen", String_LastSeenGMT}, // End Time GMT - first seen, seconds {"%tr", 0, "received", String_Received}, // Received Time + {"%trr", 0, "received", String_ReceivedRaw}, // Received Time, seconds + {"%trg", 0, "received", String_ReceivedGMT}, // Received Time GMT, seconds {"%td", 0, "duration", String_Duration}, // Duration {"%tds", 0, "duration", String_Duration_Seconds}, // Duration always in seconds {"%pkt", 0, "packets", String_InPackets}, // Packets - default input - compat @@ -863,6 +872,63 @@ static void String_LastSeenRaw(FILE *stream, recordHandle_t *recordHandle) { } // End of String_LastSeenRaw +static void String_FirstSeenGMT(FILE *stream, recordHandle_t *recordHandle) { + EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; + + uint64_t msecFirst = genericFlow ? genericFlow->msecFirst : 0; + + if (msecFirst) { + time_t tt = msecFirst / 1000LL; + struct tm ts; + gmtime_r(&tt, &ts); + char s[128]; + strftime(s, 128, "%Y-%m-%d %H:%M:%S", &ts); + s[127] = '\0'; + fprintf(stream, "%s.%03u", s, (unsigned)(msecFirst % 1000LL)); + } else { + fprintf(stream, "%s", "0000-00-00 00:00:00.000"); + } + +} // End of String_FirstSeenGMT + +static void String_LastSeenGMT(FILE *stream, recordHandle_t *recordHandle) { + EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; + + uint64_t msecLast = genericFlow ? genericFlow->msecLast : 0; + + if (msecLast) { + time_t tt = msecLast / 1000LL; + struct tm ts; + gmtime_r(&tt, &ts); + char s[128]; + strftime(s, 128, "%Y-%m-%d %H:%M:%S", &ts); + s[127] = '\0'; + fprintf(stream, "%s.%03u", s, (unsigned)(msecLast % 1000LL)); + } else { + fprintf(stream, "%s", "0000-00-00 00:00:00.000"); + } + +} // End of String_LastSeenGMT + +static void String_ReceivedGMT(FILE *stream, recordHandle_t *recordHandle) { + EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; + + uint64_t msecReceived = genericFlow ? genericFlow->msecReceived : 0; + + if (msecReceived) { + time_t tt = msecReceived / 1000LL; + struct tm ts; + gmtime_r(&tt, &ts); + char s[128]; + strftime(s, 128, "%Y-%m-%d %H:%M:%S", &ts); + s[127] = '\0'; + fprintf(stream, "%s.%03llu", s, msecReceived % 1000LL); + } else { + fprintf(stream, "%s", "0000-00-00 00:00:00.000"); + } + +} // End of String_ReceivedGMT + static void String_nbarID(FILE *stream, recordHandle_t *recordHandle) { uint8_t *nbar = (uint8_t *)recordHandle->extensionList[EXnbarAppID]; diff --git a/src/output/output_fmt.c b/src/output/output_fmt.c index 358b23de..d8947012 100755 --- a/src/output/output_fmt.c +++ b/src/output/output_fmt.c @@ -111,8 +111,14 @@ static void String_FirstSeenRaw(FILE *stream, recordHandle_t *recordHandle); static void String_LastSeenRaw(FILE *stream, recordHandle_t *recordHandle); +static void String_FirstSeenGMT(FILE *stream, recordHandle_t *recordHandle); + +static void String_LastSeenGMT(FILE *stream, recordHandle_t *recordHandle); + static void String_ReceivedRaw(FILE *stream, recordHandle_t *recordHandle); +static void String_ReceivedGMT(FILE *stream, recordHandle_t *recordHandle); + static void String_Duration(FILE *stream, recordHandle_t *recordHandle); static void String_Duration_Seconds(FILE *stream, recordHandle_t *recordHandle); @@ -368,31 +374,34 @@ static struct format_entry_s { {"%exp", 0, "Exp ID", String_ExpSysID}, // Exporter SysID // EXgenericFlowID - {"%tfs", 0, "Date first seen ", String_FirstSeen}, // Start Time - first seen - {"%ts", 0, "Date first seen ", String_FirstSeen}, // Start Time - first seen - {"%tsr", 0, "First seen raw", String_FirstSeenRaw}, // Start Time - first seen, seconds - {"%te", 0, "Date last seen ", String_LastSeen}, // End Time - last seen - {"%ter", 0, "Last seen raw ", String_LastSeenRaw}, // End Time - first seen, seconds - {"%trr", 0, "Received raw ", String_ReceivedRaw}, // Received Time, seconds - {"%tr", 0, "Date flow received ", String_Received}, // Received Time - {"%td", 0, "Duration ", String_Duration}, // Duration - {"%tds", 0, "Duration ", String_Duration_Seconds}, // Duration always in seconds - {"%pkt", 0, " Packets", String_InPackets}, // Packets - default input - compat - {"%ipkt", 0, " In Pkt", String_InPackets}, // In Packets - {"%byt", 0, " Bytes", String_InBytes}, // Bytes - default input - compat - {"%ibyt", 0, " In Byte", String_InBytes}, // In Bytes - {"%sp", 0, "Src Pt", String_SrcPort}, // Source Port - {"%dp", 0, "Dst Pt", String_DstPort}, // Destination Port - {"%it", 0, "ICMP-T", String_ICMP_type}, // ICMP type - {"%ic", 0, "ICMP-C", String_ICMP_code}, // ICMP code - {"%pr", 0, "Proto", String_Protocol}, // Protocol - {"%flg", 0, " Flags", String_Flags}, // TCP Flags - {"%fwd", 0, "Fwd", String_FwdStatus}, // Forwarding Status - {"%tos", 0, " Tos", String_Tos}, // Tos - compat - {"%stos", 0, "STos", String_SrcTos}, // Tos - Src tos - {"%bps", 0, " bps", String_bps}, // bps - bits per second - {"%pps", 0, " pps", String_pps}, // pps - packets per second - {"%bpp", 0, " Bpp", String_bpp}, // bpp - Bytes per package + {"%tfs", 0, "Date first seen ", String_FirstSeen}, // Start Time - first seen + {"%ts", 0, "Date first seen ", String_FirstSeen}, // Start Time - first seen + {"%tsr", 0, "First seen raw", String_FirstSeenRaw}, // Start Time - first seen, seconds + {"%tsg", 0, "First seen GMT ", String_FirstSeenGMT}, // Start Time - first seen GMT, seconds + {"%te", 0, "Date last seen ", String_LastSeen}, // End Time - last seen + {"%ter", 0, "Last seen raw ", String_LastSeenRaw}, // End Time - first seen, seconds + {"%teg", 0, "Last seen GMT ", String_LastSeenGMT}, // End Time - first seen GMT, seconds + {"%tr", 0, "Date flow received ", String_Received}, // Received Time + {"%trr", 0, "Received raw ", String_ReceivedRaw}, // Received Time, seconds + {"%trg", 0, "Date flow received GMT ", String_ReceivedGMT}, // Received Time GMT + {"%td", 0, "Duration ", String_Duration}, // Duration + {"%tds", 0, "Duration ", String_Duration_Seconds}, // Duration always in seconds + {"%pkt", 0, " Packets", String_InPackets}, // Packets - default input - compat + {"%ipkt", 0, " In Pkt", String_InPackets}, // In Packets + {"%byt", 0, " Bytes", String_InBytes}, // Bytes - default input - compat + {"%ibyt", 0, " In Byte", String_InBytes}, // In Bytes + {"%sp", 0, "Src Pt", String_SrcPort}, // Source Port + {"%dp", 0, "Dst Pt", String_DstPort}, // Destination Port + {"%it", 0, "ICMP-T", String_ICMP_type}, // ICMP type + {"%ic", 0, "ICMP-C", String_ICMP_code}, // ICMP code + {"%pr", 0, "Proto", String_Protocol}, // Protocol + {"%flg", 0, " Flags", String_Flags}, // TCP Flags + {"%fwd", 0, "Fwd", String_FwdStatus}, // Forwarding Status + {"%tos", 0, " Tos", String_Tos}, // Tos - compat + {"%stos", 0, "STos", String_SrcTos}, // Tos - Src tos + {"%bps", 0, " bps", String_bps}, // bps - bits per second + {"%pps", 0, " pps", String_pps}, // pps - packets per second + {"%bpp", 0, " Bpp", String_bpp}, // bpp - Bytes per package // EXipv4FlowID EXipv6FlowID {"%sa", 1, " Src IP Addr", String_SrcAddr}, // Source Address @@ -928,6 +937,25 @@ static void String_Received(FILE *stream, recordHandle_t *recordHandle) { } // End of String_Received +static void String_ReceivedGMT(FILE *stream, recordHandle_t *recordHandle) { + EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; + + uint64_t msecReceived = genericFlow ? genericFlow->msecReceived : 0; + + if (msecReceived) { + time_t tt = msecReceived / 1000LL; + struct tm ts; + gmtime_r(&tt, &ts); + char s[128]; + strftime(s, 128, "%Y-%m-%d %H:%M:%S", &ts); + s[127] = '\0'; + fprintf(stream, "%s.%03llu", s, msecReceived % 1000LL); + } else { + fprintf(stream, "%s", "0000-00-00 00:00:00.000"); + } + +} // End of String_ReceivedGMT + static void String_ReceivedRaw(FILE *stream, recordHandle_t *recordHandle) { EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; @@ -952,6 +980,44 @@ static void String_LastSeenRaw(FILE *stream, recordHandle_t *recordHandle) { } // End of String_LastSeenRaw +static void String_FirstSeenGMT(FILE *stream, recordHandle_t *recordHandle) { + EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; + + uint64_t msecFirst = genericFlow ? genericFlow->msecFirst : 0; + + if (msecFirst) { + time_t tt = msecFirst / 1000LL; + struct tm ts; + gmtime_r(&tt, &ts); + char s[128]; + strftime(s, 128, "%Y-%m-%d %H:%M:%S", &ts); + s[127] = '\0'; + fprintf(stream, "%s.%03u", s, (unsigned)(msecFirst % 1000LL)); + } else { + fprintf(stream, "%s", "0000-00-00 00:00:00.000"); + } + +} // End of String_FirstSeenGMT + +static void String_LastSeenGMT(FILE *stream, recordHandle_t *recordHandle) { + EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; + + uint64_t msecLast = genericFlow ? genericFlow->msecLast : 0; + + if (msecLast) { + time_t tt = msecLast / 1000LL; + struct tm ts; + gmtime_r(&tt, &ts); + char s[128]; + strftime(s, 128, "%Y-%m-%d %H:%M:%S", &ts); + s[127] = '\0'; + fprintf(stream, "%s.%03u", s, (unsigned)(msecLast % 1000LL)); + } else { + fprintf(stream, "%s", "0000-00-00 00:00:00.000"); + } + +} // End of String_LastSeenGMT + static void String_Payload(FILE *stream, uint8_t *payload, EXgenericFlow_t *genericFlow) { uint32_t payloadLength = 0; if (payload) {