diff --git a/configure.ac b/configure.ac index 6ffca601..4ef02521 100644 --- a/configure.ac +++ b/configure.ac @@ -78,10 +78,30 @@ if test "x$ac_cv_prog_LEX" = "xflex"; then LFLAGS=-i fi +AC_ARG_WITH(lz4path, +[ --with-lz4path=PATH Expect liblz4 installed in PATH; default /usr/local], +if test "x$with_lz4path" != "xno" ; then + CPPFLAGS="${CPPFLAGS} -I${with_lz4path}/include" + LDFLAGS="${LDFLAGS} -L${with_lz4path}/lib" +fi +, +) + AC_ARG_WITH(zstdpath, [ --with-zstdpath=PATH Expect libzstd installed in PATH; default /usr/local], +if test "x$with_zstdpath" != "xno" ; then CPPFLAGS="${CPPFLAGS} -I${with_zstdpath}/include" LDFLAGS="${LDFLAGS} -L${with_zstdpath}/lib" +fi +, +) + +AC_ARG_WITH(bz2path, +[ --with-bz2path=PATH Expect libbz2 installed in PATH; default /usr/local], +if test "x$with_bz2path" != "xno" ; then + CPPFLAGS="${CPPFLAGS} -I${with_bz2path}/include" + LDFLAGS="${LDFLAGS} -L${with_bz2path}/lib" +fi , ) @@ -141,18 +161,15 @@ if test -d "$WHERE_FTPATH"; then else AC_MSG_ERROR(flow-tools directory '$WHERE_FTPATH' does not exists. Use --with-ftpath=PATH) fi -AM_CONDITIONAL(FT2NFDUMP, true) -, -AM_CONDITIONAL(FT2NFDUMP, false) +build_ftconv="yes" , build_ftconv="no" ) +AM_CONDITIONAL([FT2NFDUMP], [test "x$build_ftconv" = "xyes"]) AC_ARG_ENABLE(maxmind, [ --enable-maxmind Build geolookup for MaxMind GeoDB; default is NO], -AM_CONDITIONAL(MAXMIND, true) -, -AM_CONDITIONAL(MAXMIND, false) +use_maxmind="yes" , use_maxmind="no" ) - +AM_CONDITIONAL([MAXMIND], [test "x$use_maxmind" = "xyes"]) #Needs tidy AC_ARG_ENABLE(nfprofile, @@ -168,7 +185,7 @@ AC_SUBST(RRD_LIBS) , AC_MSG_ERROR(Can not link librrd. Please specify --with-rrdpath=.. configure failed! )) AC_CHECK_HEADERS([rrd.h]) if test "$ac_cv_header_rrd_h" = yes; then - AM_CONDITIONAL(NFPROFILE, true) + build_nfprofile="yes" else AC_MSG_ERROR(Required rrd.h header file not found!) fi @@ -185,8 +202,9 @@ AC_SUBST(RRD_LIBS) ] , -AM_CONDITIONAL(NFPROFILE, false) +build_nfprofile="no" ) +AM_CONDITIONAL([NFPROFILE], [test "x$build_nfprofile" = "xyes"]) AC_ARG_ENABLE(influxdb, [AS_HELP_STRING([--enable-influxdb], [enable stats to influxdb (default is no)])], [ influxdb=${enableval} ], [influxdb=no]) @@ -244,7 +262,9 @@ AM_CONDITIONAL(NFTRACK, false) ) AC_ARG_ENABLE(sflow, -[ --enable-sflow Build sflow collector sfcpad; default is NO]) +[ --enable-sflow Build sflow collector sfcpad; default is NO], +build_sflow="yes", build_sflow="no" +) AM_CONDITIONAL(SFLOW, test "$enable_sflow" = yes) AC_ARG_ENABLE(readpcap, @@ -265,7 +285,7 @@ AC_SUBST(PCAP_LIBS) AC_CHECK_LIB(pcap, pcap_dump_open_append, AM_CONDITIONAL(HAVEPCAPAPPEND, true), AM_CONDITIONAL(HAVEPCAPAPPEND, false)) AC_CHECK_HEADERS([pcap.h]) if test "$ac_cv_header_pcap_h" = yes; then - AM_CONDITIONAL(BUILDNFPCAPD, true) + build_nfpcapd="yes" else AC_MSG_ERROR(Required pcap.h header file not found!) fi @@ -282,9 +302,10 @@ AC_SUBST(PCAP_LIBS) ] , -AM_CONDITIONAL(BUILDNFPCAPD, false) AM_CONDITIONAL(HAVEPCAPAPPEND, false) +build_nfpcapd="no" ) +AM_CONDITIONAL(BUILDNFPCAPD, test "$build_nfpcapd" = yes) ## Check for BSD socket or TPACKET_V3 AM_COND_IF([BUILDNFPCAPD], @@ -383,12 +404,68 @@ AC_CHECK_HEADERS(sys/types.h netinet/in.h arpa/nameser.h arpa/nameser_compat.h n #endif]]) -AC_CHECK_HEADERS([bzlib.h]) -if test "$ac_cv_header_bzlib_h" = no; then - AC_MSG_ERROR(Required bzlib.h header file not found!) +if test "x$with_lz4path" != "xno"; then +AC_CHECK_HEADER(lz4.h, [ + AC_CHECK_LIB(lz4, LZ4_compress, [ + AC_DEFINE(HAVE_LZ4, 1, [Define if you have lz4 library]) + LIBS="$LIBS -llz4" + AM_CONDITIONAL(LZ4EMBEDDED, false) + use_lz4="yes" + ], [ + AM_CONDITIONAL(LZ4EMBEDDED, true) + use_lz4="embedded" + ]) + ], [ + AM_CONDITIONAL(LZ4EMBEDDED, true) + use_lz4="embedded" + ] +) +else + AM_CONDITIONAL(LZ4EMBEDDED, true) + use_lz4="embedded" +fi + +if test "x$with_bz2path" != "xno"; then +AC_CHECK_HEADER(bzlib.h, [ + AC_CHECK_LIB(bz2, BZ2_bzCompressInit, [ + AC_DEFINE(HAVE_BZIP2, 1, [Define if you have bz2 library]) + LIBS="$LIBS -lbz2" + AM_CONDITIONAL(HAVE_BZIP2, true) + use_bzip2="yes" + ], [ + AM_CONDITIONAL(HAVE_BZIP2, false) + use_bzip2="no" + ]) + ], [ + AM_CONDITIONAL(HAVE_BZIP2, false) + use_bzip2="no" + ] +) +else + AM_CONDITIONAL(HAVE_BZIP2, false) + use_bzip2="disabled" fi -AC_CHECK_HEADERS([zstd.h]) +if test "x$with_zstdpath" != "xno"; then +AC_CHECK_HEADER(zstd.h, [ + AC_CHECK_LIB(zstd, ZSTD_decompress, [ + AC_DEFINE(HAVE_ZSTD, 1, [Define if you have zstd library]) + LIBS="$LIBS -lzstd" + AM_CONDITIONAL(HAVE_ZSTD, true) + use_zstd="yes" + ], [ + AM_CONDITIONAL(HAVE_ZSTD, false) + use_zstd="no" + ]) + ], [ + AM_CONDITIONAL(HAVE_ZSTD, false) + use_zstd="no" + ] +) +else + AM_CONDITIONAL(HAVE_ZSTD, false) + use_zstd="disabled" +fi if test "$ac_cv_header_fts_h" != yes; then FTS_OBJ=fts_compat.o @@ -438,17 +515,6 @@ AC_CHECK_LIB(socket, res_search, [ ]) ]) -if test "$ac_cv_header_zstd_h" = "yes"; then -AC_CHECK_LIB(zstd, ZSTD_decompress, [ - LIBS="$LIBS -lzstd" - AC_DEFINE(HAVE_ZSTDLIB,1,[ ]) - ], []) -fi - -AC_CHECK_LIB(bz2, BZ2_bzCompressInit, [ - LIBS="$LIBS -lbz2" - ], []) - # lzo compression requirements AC_CHECK_TYPE(ptrdiff_t, long) AC_TYPE_SIZE_T @@ -578,12 +644,14 @@ echo " CFLAGS = $AM_CFLAGS $CFLAGS" echo " CPPFLAGS = $AM_CPPFLAGS $CPPFLAGS" echo " LDFLAGS = $AM_LDFLAGS $LDFLAGS" echo " LIBS = $LIBS" -echo " Enable libzstd = $ac_cv_lib_zstd_ZSTD_decompress" -echo " Build geolookup = $enable_maxmind" -echo " Build sflow = $enable_sflow" -echo " Build nfpcapd = $enable_nfpcapd" -echo " Build flowtools conv = $enable_ftconv" -echo " Build nfprofile = $enable_nfprofile" +echo " Enable liblz4 = $use_lz4" +echo " Enable libbz2 = $use_bzip2" +echo " Enable libzstd = $use_zstd" +echo " Build geolookup = $use_maxmind" +echo " Build sflow = $build_sflow" +echo " Build nfpcapd = $build_nfpcapd" +echo " Build flowtools conv = $build_ftconv" +echo " Build nfprofile = $build_nfprofile" echo "----------------------------------" echo "" echo " You can run ./make now." diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am index c529e426..4c08f808 100755 --- a/src/lib/Makefile.am +++ b/src/lib/Makefile.am @@ -11,7 +11,9 @@ LDADD = $(DEPS_LIBS) util = util.c util.h pidfile = pidfile.c pidfile.h compress = compress/minilzo.c compress/minilzo.h compress/lzoconf.h compress/lzodefs.h +if LZ4EMBEDDED compress += compress/lz4.c compress/lz4.h compress/lz4hc.c compress/lz4hc.h +endif nffile = nffile.c nffile.h nffileV2.h queue.c queue.h nfx.c nfx.h nfxV3.h nfxV3.c id.h conf = conf/nfconf.c conf/nfconf.h conf/toml.c conf/toml.h diff --git a/src/lib/nffile.c b/src/lib/nffile.c index d7008cee..9076fab4 100644 --- a/src/lib/nffile.c +++ b/src/lib/nffile.c @@ -27,17 +27,21 @@ * POSSIBILITY OF SUCH DAMAGE. * */ - #include "nffile.h" #include #include -#ifdef HAVE_BZLIB_H + +#ifdef HAVE_BZIP2 #include #endif #ifdef HAVE_ZSTDLIB #include #endif +#ifdef HAVE_LZ4 +#include +#include +#endif #include #include #include @@ -57,10 +61,11 @@ #include #include -#include "config.h" #include "flist.h" +#ifndef HAVE_LZ4 #include "lz4.h" #include "lz4hc.h" +#endif #include "minilzo.h" #include "nfconf.h" #include "nfdump.h" @@ -87,11 +92,7 @@ static int LZ4_initialize(void); static int BZ2_initialize(void); -#ifdef HAVE_ZSTDLIB static int ZSTD_initialize(void); -#endif - -static void BZ2_prep_stream(bz_stream *); static int Compress_Block_LZO(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size); @@ -103,11 +104,9 @@ static int Uncompress_Block_LZ4(dataBlock_t *in_block, dataBlock_t *out_block, s static int Compress_Block_BZ2(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size); -#ifdef HAVE_ZSTDLIB static int Compress_Block_ZSTD(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size, int level); static int Uncompress_Block_ZSTD(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size); -#endif static int Uncompress_Block_BZ2(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size); @@ -155,12 +154,10 @@ int Init_nffile(int workers, queue_t *fileList) { LogError("Failed to initialize BZ2"); return 0; } -#ifdef HAVE_ZSTDLIB if (!ZSTD_initialize()) { LogError("Failed to initialize ZSTD"); return 0; } -#endif atomic_init(&blocksInUse, 0); @@ -236,9 +233,19 @@ int ParseCompression(char *arg) { return -1; } } - if (strcmp(arg, "bz2") == 0 || strcmp(arg, "bzip2") == 0 || strcmp(arg, "2") == 0) return BZ2_COMPRESSED; -#ifdef HAVE_ZSTDLIB + + if (strcmp(arg, "bz2") == 0 || strcmp(arg, "bzip2") == 0 || strcmp(arg, "2") == 0) { +#ifdef HAVE_BZIP2 + return BZ2_COMPRESSED; + } +#else + LogError("BZIP2 compression not compiled in"); + return -1; + } +#endif + if (strcmp(arg, "zstd") == 0 || strcmp(arg, "4") == 0) { +#ifdef HAVE_ZSTDLIB if (level <= ZSTD_maxCLevel()) { return (level << 16) | ZSTD_COMPRESSED; } else { @@ -247,8 +254,7 @@ int ParseCompression(char *arg) { } } #else - if (strcmp(arg, "zstd") == 0) { - LogError("ZSTD compression not enabled"); + LogError("ZSTD compression not compiled in"); return -1; } #endif @@ -287,23 +293,18 @@ static int LZ4_initialize(void) { static int BZ2_initialize(void) { return 1; } // End of BZ2_initialize -static void BZ2_prep_stream(bz_stream *bs) { - bs->bzalloc = NULL; - bs->bzfree = NULL; - bs->opaque = NULL; -} // End of BZ2_prep_stream - -#ifdef HAVE_ZSTDLIB static int ZSTD_initialize(void) { +#ifdef HAVE_ZSTDLIB size_t const cBuffSize = ZSTD_compressBound(WRITE_BUFFSIZE); if (cBuffSize > (BUFFSIZE - sizeof(dataBlock_t))) { LogError("LZSTD_compressBound() error in %s line %d: Buffer too small", __FILE__, __LINE__); return 0; } return 1; - -} // End of ZSTD_initialize +#else + return 1; #endif +} // End of ZSTD_initialize static int Compress_Block_LZO(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size) { unsigned char __LZO_MMODEL *in; @@ -412,11 +413,10 @@ static int Uncompress_Block_LZ4(dataBlock_t *in_block, dataBlock_t *out_block, s } // End of Uncompress_Block_LZ4 -#ifdef HAVE_BZLIB_H static int Compress_Block_BZ2(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size) { - bz_stream bs; +#ifdef HAVE_BZIP2 + bz_stream bs = {0}; - BZ2_prep_stream(&bs); BZ2_bzCompressInit(&bs, 9, 0, 0); bs.next_in = (char *)((void *)in_block + sizeof(dataBlock_t)); @@ -441,13 +441,15 @@ static int Compress_Block_BZ2(dataBlock_t *in_block, dataBlock_t *out_block, siz BZ2_bzCompressEnd(&bs); return 1; - +#else + return 0; +#endif } // End of Compress_Block_BZ2 static int Uncompress_Block_BZ2(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size) { - bz_stream bs; +#ifdef HAVE_BZIP2 + bz_stream bs = {0}; - BZ2_prep_stream(&bs); BZ2_bzDecompressInit(&bs, 0, 0); bs.next_in = (char *)((void *)in_block + sizeof(dataBlock_t)); @@ -474,12 +476,14 @@ static int Uncompress_Block_BZ2(dataBlock_t *in_block, dataBlock_t *out_block, s BZ2_bzDecompressEnd(&bs); return 1; +#else + return 0; +#endif } // End of Uncompress_Block_BZ2 -#endif -#ifdef HAVE_ZSTDLIB static int Compress_Block_ZSTD(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size, int level) { +#ifdef HAVE_ZSTDLIB const char *in = (const char *)((void *)in_block + sizeof(dataBlock_t)); char *out = (char *)((void *)out_block + sizeof(dataBlock_t)); int in_len = in_block->size; @@ -497,10 +501,14 @@ static int Compress_Block_ZSTD(dataBlock_t *in_block, dataBlock_t *out_block, si out_block->size = out_len; return 1; +#else + return 0; +#endif } // End of Compress_Block_ZSTD static int Uncompress_Block_ZSTD(dataBlock_t *in_block, dataBlock_t *out_block, size_t block_size) { +#ifdef HAVE_ZSTDLIB const char *in = (const char *)((void *)in_block + sizeof(dataBlock_t)); char *out = (char *)((void *)out_block + sizeof(dataBlock_t)); int in_len = in_block->size; @@ -516,8 +524,10 @@ static int Uncompress_Block_ZSTD(dataBlock_t *in_block, dataBlock_t *out_block, out_block->size = out_len; return 1; -} // End of Uncompress_Block_ZSTD +#else + return 0; #endif +} // End of Uncompress_Block_ZSTD static dataBlock_t *NewDataBlock(void) { dataBlock_t *dataBlock = malloc(BUFFSIZE); @@ -854,7 +864,15 @@ static nffile_t *OpenFileStatic(char *filename, nffile_t *nffile) { #ifndef HAVE_ZSTDLIB if (nffile->file_header->compression == ZSTD_COMPRESSED) { - LogError("ZSTD compression not enabled. Skip file: %s", filename); + LogError("ZSTD compression not compiled in. Skip file: %s", filename); + CloseFile(nffile); + return NULL; + } +#endif + +#ifndef HAVE_BZIP2 + if (nffile->file_header->compression == BZ2_COMPRESSED) { + LogError("BZIP2 compression not compiled in. Skip file: %s", filename); CloseFile(nffile); return NULL; } @@ -912,7 +930,15 @@ nffile_t *OpenNewFile(char *filename, nffile_t *nffile, int creator, int compres #ifndef HAVE_ZSTDLIB if ((compress & 0xFFFF) == ZSTD_COMPRESSED) { - LogError("Open file %s: ZSTD compressionnot enabled"); + LogError("Open file %s: ZSTD compression not compiled in"); + CloseFile(nffile); + return NULL; + } +#endif + +#ifndef HAVE_BZIP2 + if ((compress & 0xFFFF) == BZ2_COMPRESSED) { + LogError("Open file %s: BZIP2 compression not compiled in"); CloseFile(nffile); return NULL; } @@ -1306,13 +1332,11 @@ static dataBlock_t *nfread(nffile_t *nffile) { if (Uncompress_Block_BZ2(buff, block_header, nffile->buff_size) < 0) failed = 1; FreeDataBlock(buff); break; -#ifdef HAVE_ZSTDLIB case ZSTD_COMPRESSED: block_header = NewDataBlock(); if (Uncompress_Block_ZSTD(buff, block_header, nffile->buff_size) < 0) failed = 1; FreeDataBlock(buff); break; -#endif } if (failed) { @@ -1432,13 +1456,11 @@ static int nfwrite(nffile_t *nffile, dataBlock_t *block_header) { if (Compress_Block_BZ2(block_header, buff, nffile->buff_size) < 0) failed = 1; wptr = buff; break; -#ifdef HAVE_ZSTDLIB case ZSTD_COMPRESSED: buff = NewDataBlock(); if (Compress_Block_ZSTD(block_header, buff, nffile->buff_size, level) < 0) failed = 1; wptr = buff; break; -#endif } if (failed) { // error @@ -1743,11 +1765,19 @@ int QueryFile(char *filename, int verbose) { #ifndef HAVE_ZSTDLIB if (fileHeader.compression == ZSTD_COMPRESSED) { - LogError("ZSTD compression not enabled. Skip checking."); + LogError("ZSTD compression not compiled in. Skip checking."); close(fd); return 0; } #endif +#ifndef HAVE_BZIP2 + if (fileHeader.compression == BZ2_COMPRESSED) { + LogError("BZIP2 compression not compiled in. Skip checking."); + close(fd); + return 0; + } +#endif + // first check ok - abstract nffile level nffile_t *nffile = NewFile(NULL); if (!nffile) { @@ -1886,7 +1916,6 @@ int QueryFile(char *filename, int verbose) { } } break; case ZSTD_COMPRESSED: { -#ifdef HAVE_ZSTDLIB dataBlock_t *b = nffile->block_header; nffile->block_header = buff; buff = b; @@ -1894,9 +1923,6 @@ int QueryFile(char *filename, int verbose) { LogError("Zstd decompress failed"); failed = 1; } -#else - LogError("Zstd compression not enabled"); -#endif } break; default: LogError("Unknown compression: %d", compression); diff --git a/src/test/Makefile.am b/src/test/Makefile.am index 82f6ace0..81f7e83e 100755 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -1,6 +1,24 @@ check_PROGRAMS = nftest nfgen -TESTS = nftest runtest.sh +TESTS = nftest runprepare.sh runlzo.sh runlz4.sh + +if HAVE_BZIP2 +TEST_BZIP2=yes +TESTS += runbz2.sh +endif + +if HAVE_ZSTD +TEST_ZSTD=yes +TESTS += runzstd.sh +endif + +TESTS += runtest.sh + +AM_TESTS_ENVIRONMENT = \ +## Set enabled compression +TEST_BZIP2="$(TEST_BZIP2)"; export TEST_BZIP2 \ +TEST_ZSTD="$(TEST_ZSTD)"; export TEST_ZSTD \ +; AM_CPPFLAGS = -I.. -I../include -I../lib -I../inline -I../netflow -I../collector $(DEPS_CFLAGS) AM_CFLAGS = -ggdb diff --git a/src/test/runbz2.sh b/src/test/runbz2.sh new file mode 100755 index 00000000..35ca53d6 --- /dev/null +++ b/src/test/runbz2.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# This file is part of the nfdump project. +# +# Copyright (c) 2023, Peter Haag +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Peter Haag nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +set -e +TZ=MET +export TZ + +# prevent any default goelookup for testing +NFDUMP="../nfdump/nfdump -G none" + +# bzip2 compression tests +$NFDUMP -J 2 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J 1 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J bzip2 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J 0 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null diff --git a/src/test/runlz4.sh b/src/test/runlz4.sh new file mode 100755 index 00000000..09960db5 --- /dev/null +++ b/src/test/runlz4.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# This file is part of the nfdump project. +# +# Copyright (c) 2023, Peter Haag +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Peter Haag nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +set -e +TZ=MET +export TZ + +# prevent any default goelookup for testing +NFDUMP="../nfdump/nfdump -G none" + +# lz4 compression tests +$NFDUMP -J 3 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J 1 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J lz4:5 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J lz4:9 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J 0 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null diff --git a/src/test/runlzo.sh b/src/test/runlzo.sh new file mode 100755 index 00000000..295971a4 --- /dev/null +++ b/src/test/runlzo.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# This file is part of the nfdump project. +# +# Copyright (c) 2023, Peter Haag +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Peter Haag nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +set -e +TZ=MET +export TZ + +# prevent any default goelookup for testing +NFDUMP="../nfdump/nfdump -G none" + +# lzo compression tests +$NFDUMP -J 0 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J 1 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J 0 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J lzo -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null diff --git a/src/test/runprepare.sh b/src/test/runprepare.sh new file mode 100755 index 00000000..7eb0d991 --- /dev/null +++ b/src/test/runprepare.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# This file is part of the nfdump project. +# +# Copyright (c) 2023, Peter Haag +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Peter Haag nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +set -e +TZ=MET +export TZ + +# Check for correct output +rm -f test.* +./nfgen + +# prevent any default goelookup for testing +NFDUMP="../nfdump/nfdump -G none" + +# verify test +$NFDUMP -v test.flows.nf + +# read test +rm -f test1.out +$NFDUMP -r test.flows.nf -q -o raw >test.1.out +diff -u test.1.out nftest.1.out +rm -f test.1.out diff --git a/src/test/runtest.sh b/src/test/runtest.sh index 614e2649..e506ad7f 100755 --- a/src/test/runtest.sh +++ b/src/test/runtest.sh @@ -1,8 +1,7 @@ #!/bin/sh # This file is part of the nfdump project. # -# Copyright (c) 2009-2023, Peter Haag -# Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung +# Copyright (c) 2023, Peter Haag # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -13,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of SWITCH nor the names of its contributors may be +# * Neither the name of Peter Haag nor the names of its contributors may be # used to endorse or promote products derived from this software without # specific prior written permission. # @@ -34,36 +33,9 @@ set -e TZ=MET export TZ -# Check for correct output -rm -f test.* -./nfgen - # prevent any default goelookup for testing NFDUMP="../nfdump/nfdump -G none" -# verify test -$NFDUMP -v test.flows.nf - -# read test -rm -f test1.out -$NFDUMP -r test.flows.nf -q -o raw >test.1.out -diff -u test.1.out nftest.1.out - -# compression tests -$NFDUMP -J 0 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J 1 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J 2 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J 3 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J 2 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J 1 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J 0 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J lzo -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J lz4:5 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J bzip2 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J lz4:9 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null -$NFDUMP -J 0 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null - -rm -f test.1.out $NFDUMP -r test.flows.nf -q -o raw >test.1.out diff -u test.1.out nftest.1.out diff --git a/src/test/runzstd.sh b/src/test/runzstd.sh new file mode 100755 index 00000000..bf37c06a --- /dev/null +++ b/src/test/runzstd.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# This file is part of the nfdump project. +# +# Copyright (c) 2023, Peter Haag +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Peter Haag nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +set -e +TZ=MET +export TZ + +# prevent any default goelookup for testing +NFDUMP="../nfdump/nfdump -G none" + +# zstd compression tests +$NFDUMP -J 4 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J 1 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J zstd -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J zstd:5 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J zstd:9 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J 0 -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null +$NFDUMP -J lzo -r test.flows.nf && $NFDUMP -v test.flows.nf >/dev/null