From fa2785bfc2298bc8c3b61e78a1c9b1765e12ed9c Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Mon, 29 Jul 2024 13:05:55 +0200 Subject: [PATCH] Add fixed but fast csv output mode csv-fast. See #548 --- src/output/Makefile.am | 6 +- src/output/output.c | 2 + src/output/output.h | 2 +- src/output/output_csv.h | 6 ++ src/output/output_csv_fast.c | 106 +++++++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 src/output/output_csv_fast.c diff --git a/src/output/Makefile.am b/src/output/Makefile.am index 56744311..c225bb72 100755 --- a/src/output/Makefile.am +++ b/src/output/Makefile.am @@ -4,9 +4,9 @@ AM_CPPFLAGS = -I.. -I../include -I../libnffile -I../libnfdump -I../inline $(DEPS noinst_LIBRARIES = liboutput.a liboutput_a_SOURCES = output.c output.h \ - output_util.c output_util.h output_csv.c output_csv.h \ - output_fmt.c output_fmt.h output_json.c output_json.h \ - output_raw.c output_raw.h + output_util.c output_util.h output_raw.c output_raw.h \ + output_csv.c output_csv.h output_csv_fast.c \ + output_fmt.c output_fmt.h output_json.c output_json.h EXTRA_DIST = itoa.c diff --git a/src/output/output.c b/src/output/output.c index 833edbed..6a9138a2 100644 --- a/src/output/output.c +++ b/src/output/output.c @@ -105,6 +105,7 @@ static struct printmap_s { {"json", MODE_JSON, NULL, "json output"}, {"json-log", MODE_JSON_LOG, NULL, "json output for logging"}, {"csv", MODE_CSV, FORMAT_CSV, "csv predefined"}, + {"csv-fast", MODE_CSV_FAST, NULL, "csv fast predefined"}, {"null", MODE_NULL, NULL, "do not print any output"}, // This is always the last line @@ -119,6 +120,7 @@ static struct printerFunc_s { [MODE_FMT] = {fmt_record, fmt_prolog, fmt_epilog}, [MODE_RAW] = {raw_record, raw_prolog, raw_epilog}, [MODE_CSV] = {csv_record, csv_prolog, csv_epilog}, + [MODE_CSV_FAST] = {csv_record_fast, csv_prolog_fast, csv_epilog_fast}, [MODE_JSON] = {flow_record_to_json_human, json_prolog, json_epilog}, [MODE_JSON_LOG] = {flow_record_to_json_log, json_prolog, json_epilog}}; diff --git a/src/output/output.h b/src/output/output.h index 7eb15c0d..ed5036d3 100644 --- a/src/output/output.h +++ b/src/output/output.h @@ -40,7 +40,7 @@ typedef void (*RecordPrinter_t)(FILE *, recordHandle_t *, int); typedef void (*PrologPrinter_t)(void); typedef void (*EpilogPrinter_t)(void); -typedef enum { MODE_NULL = 0, MODE_RAW, MODE_FMT, MODE_CSV, MODE_JSON, MODE_JSON_LOG } outputMode_t; +typedef enum { MODE_NULL = 0, MODE_RAW, MODE_FMT, MODE_CSV, MODE_CSV_FAST, MODE_JSON, MODE_JSON_LOG } outputMode_t; typedef struct outputParams_s { bool printPlain; diff --git a/src/output/output_csv.h b/src/output/output_csv.h index aca09080..aa570394 100644 --- a/src/output/output_csv.h +++ b/src/output/output_csv.h @@ -44,4 +44,10 @@ void csv_epilog(void); void csv_record(FILE *stream, recordHandle_t *recordHandle, int tag); +void csv_prolog_fast(void); + +void csv_epilog_fast(void); + +void csv_record_fast(FILE *stream, recordHandle_t *recordHandle, int tag); + #endif // _OUTPUT_CSV_H diff --git a/src/output/output_csv_fast.c b/src/output/output_csv_fast.c new file mode 100644 index 00000000..1b609c0d --- /dev/null +++ b/src/output/output_csv_fast.c @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2024, Peter Haag + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the author nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfxV3.h" +#include "output_csv.h" +#include "output_util.h" +#include "util.h" + +#define IP_STRING_LEN (INET6_ADDRSTRLEN) + +// record counter +static uint32_t recordCount; + +void csv_prolog_fast(void) { + // empty prolog + recordCount = 0; + printf("cnt,af,firstSeen,lastSeen,proto,srcAddr,srcPort,dstAddr,dstPort,srcAS,dstAS,input,output,flags,srcTos,packets,bytes\n"); +} // End of csv_prolog_fast + +void csv_epilog_fast(void) { + // empty epilog +} // End of csv_epilog_fast + +void csv_record_fast(FILE *stream, recordHandle_t *recordHandle, int tag) { + EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; + EXipv4Flow_t *ipv4Flow = (EXipv4Flow_t *)recordHandle->extensionList[EXipv4FlowID]; + EXipv6Flow_t *ipv6Flow = (EXipv6Flow_t *)recordHandle->extensionList[EXipv6FlowID]; + EXflowMisc_t *flowMisc = (EXflowMisc_t *)recordHandle->extensionList[EXflowMiscID]; + EXasRouting_t *asRouting = (EXasRouting_t *)recordHandle->extensionList[EXasRoutingID]; + + EXgenericFlow_t genericNull = {0}; + if (!genericFlow) genericFlow = &genericNull; + + EXflowMisc_t miscNull = {0}; + if (!flowMisc) flowMisc = &miscNull; + + EXasRouting_t asNULL = {0}; + if (!asRouting) asRouting = &asNULL; + + int af = 0; + char sa[IP_STRING_LEN], da[IP_STRING_LEN]; + if (ipv4Flow) { + af = PF_INET; + uint32_t src = htonl(ipv4Flow->srcAddr); + uint32_t dst = htonl(ipv4Flow->dstAddr); + + inet_ntop(AF_INET, &src, sa, sizeof(sa)); + inet_ntop(AF_INET, &dst, da, sizeof(da)); + } + + if (ipv6Flow) { + af = PF_INET6; + uint64_t src[2], dst[2]; + src[0] = htonll(ipv6Flow->srcAddr[0]); + src[1] = htonll(ipv6Flow->srcAddr[1]); + dst[0] = htonll(ipv6Flow->dstAddr[0]); + dst[1] = htonll(ipv6Flow->dstAddr[1]); + + inet_ntop(AF_INET6, &src, sa, sizeof(sa)); + inet_ntop(AF_INET6, &dst, da, sizeof(da)); + } + + fprintf(stream, "%u,%i,%" PRIu64 ",%" PRIu64 ",%u,%s,%u,%s,%u,%u,%u,%u,%u,%s,%u,%" PRIu64 ",%" PRIu64 "\n", ++recordCount, af, + genericFlow->msecFirst, genericFlow->msecLast, genericFlow->proto, sa, genericFlow->srcPort, da, genericFlow->dstPort, asRouting->srcAS, + asRouting->dstAS, flowMisc->input, flowMisc->output, FlagsString(genericFlow->proto == IPPROTO_TCP ? genericFlow->tcpFlags : 0), + genericFlow->srcTos, genericFlow->inPackets, genericFlow->inBytes); + +} // End of csv_record_fast