From 5b4fe1e74c98554e1186201220f5aea5e89caa46 Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Thu, 29 Aug 2024 15:44:16 +0200 Subject: [PATCH] Add: gzip file decompression stream & tests for compressutils This adds a method for using a gzip compressed file as a standard input stream and some tests for compressutils in general. This will allow services like gvmd to load data from files that are compressed to save storage space. --- CMakeLists.txt | 3 +- util/CMakeLists.txt | 18 +++++++ util/compressutils.c | 66 ++++++++++++++++++++++++ util/compressutils.h | 5 ++ util/compressutils_tests.c | 101 +++++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 util/compressutils_tests.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 100da742..153bf08e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,7 +229,8 @@ if (BUILD_TESTS AND NOT SKIP_SRC) add_custom_target (tests DEPENDS array-test alivedetection-test boreas_error-test boreas_io-test - cli-test cpeutils-test cvss-test ping-test sniffer-test util-test networking-test + cli-test compressutils-test cpeutils-test cvss-test ping-test + sniffer-test util-test networking-test passwordbasedauthentication-test xmlutils-test version-test osp-test nvti-test hosts-test) diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt index 95c94235..09c7f4d5 100644 --- a/util/CMakeLists.txt +++ b/util/CMakeLists.txt @@ -161,6 +161,24 @@ if (BUILD_TESTS) add_custom_target (tests-passwordbasedauthentication DEPENDS passwordbasedauthentication-test) + add_executable (compressutils-test + EXCLUDE_FROM_ALL + compressutils_tests.c) + + add_test (compressutils-test compressutils-test) + + target_include_directories (compressutils-test PRIVATE ${CGREEN_INCLUDE_DIRS}) + + target_link_libraries (compressutils-test ${CGREEN_LIBRARIES} + ${GLIB_LDFLAGS} ${GIO_LDFLAGS} ${GPGME_LDFLAGS} ${ZLIB_LDFLAGS} + ${RADIUS_LDFLAGS} ${LIBSSH_LDFLAGS} ${GNUTLS_LDFLAGS} + ${GCRYPT_LDFLAGS} ${LDAP_LDFLAGS} ${REDIS_LDFLAGS} + ${LIBXML2_LDFLAGS} ${UUID_LDFLAGS} + ${LINKER_HARDENING_FLAGS}) + + add_custom_target (tests-compressutils + DEPENDS compressutils-test) + add_executable (cpeutils-test EXCLUDE_FROM_ALL cpeutils_tests.c) diff --git a/util/compressutils.c b/util/compressutils.c index a75f928d..baed77bb 100644 --- a/util/compressutils.c +++ b/util/compressutils.c @@ -15,6 +15,8 @@ #define ZLIB_CONST #endif +#define _GNU_SOURCE + #include "compressutils.h" #include /* for g_free, g_malloc0 */ @@ -237,3 +239,67 @@ gvm_compress_gzipheader (const void *src, unsigned long srclen, } } } + +/** + * @brief Read decompressed data from a gzip file. + * + * @param[in] cookie The gzFile to read from. + * @param[in] buffer The buffer to output decompressed data to. + * @param[in] buffer_size The size of the buffer. + * + * @return The number of bytes read into the buffer. + */ +static ssize_t +gz_file_read (void *cookie, char *buffer, size_t buffer_size) +{ + gzFile gz_file = cookie; + + return gzread (gz_file, buffer, buffer_size); +} + +/** + * @brief Close a gzip file. + * + * @param[in] cookie The gzFile to close. + * + * @return 0 on success, other values on error (see gzclose() from zlib). + */ +static int +gz_file_close (void *cookie) +{ + gzFile gz_file = cookie; + + return gzclose (gz_file);; +} + +/** + * @brief Opens a gzip file as a FILE* stream for reading and decompression. + * + * @param[in] path Path to the gzip file to open. + * + * @return The FILE* on success, NULL otherwise. + */ +FILE * +gvm_gzip_open_file_reader (const char *path) +{ + static cookie_io_functions_t io_functions = { + .read = gz_file_read, + .write = NULL, + .seek = NULL, + .close = gz_file_close, + }; + + if (path == NULL) + { + return NULL; + } + + gzFile gz_file = gzopen (path, "r"); + if (gz_file == NULL) + { + return NULL; + } + + FILE *file = fopencookie (gz_file, "r", io_functions); + return file; +} diff --git a/util/compressutils.h b/util/compressutils.h index 039f0a56..f319952f 100644 --- a/util/compressutils.h +++ b/util/compressutils.h @@ -11,6 +11,8 @@ #ifndef _GVM_COMPRESSUTILS_H #define _GVM_COMPRESSUTILS_H +#include + void * gvm_compress (const void *, unsigned long, unsigned long *); @@ -20,4 +22,7 @@ gvm_compress_gzipheader (const void *, unsigned long, unsigned long *); void * gvm_uncompress (const void *, unsigned long, unsigned long *); +FILE * +gvm_gzip_open_file_reader (const char *); + #endif /* not _GVM_COMPRESSUTILS_H */ diff --git a/util/compressutils_tests.c b/util/compressutils_tests.c new file mode 100644 index 00000000..ddc35d62 --- /dev/null +++ b/util/compressutils_tests.c @@ -0,0 +1,101 @@ +/* SPDX-FileCopyrightText: 2019-2023 Greenbone AG + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "compressutils.c" + +#include +#include + +Describe (compressutils); +BeforeEach (compressutils) +{ +} + +AfterEach (compressutils) +{ +} + +Ensure(compressutils, can_compress_and_uncompress_without_header) +{ + const char *testdata = "TEST-12345-12345-TEST"; + + size_t compressed_len; + char *compressed + = gvm_compress (testdata, strlen(testdata) + 1, &compressed_len); + assert_that (compressed_len, is_greater_than (0)); + assert_that (compressed, is_not_null); + assert_that (compressed, is_not_equal_to_string (testdata)); + + size_t uncompressed_len; + char *uncompressed + = gvm_uncompress (compressed, compressed_len, &uncompressed_len); + assert_that (uncompressed_len, is_equal_to (strlen(testdata) + 1)); + assert_that (uncompressed, is_equal_to_string (testdata)); +} + +Ensure(compressutils, can_compress_and_uncompress_with_header) +{ + const char *testdata = "TEST-12345-12345-TEST"; + + size_t compressed_len; + char *compressed + = gvm_compress_gzipheader (testdata, strlen(testdata) + 1, &compressed_len); + assert_that (compressed_len, is_greater_than (0)); + assert_that (compressed, is_not_null); + assert_that (compressed, is_not_equal_to_string (testdata)); + // Check for gzip magic number and deflate compression mode byte + assert_that (compressed[0], is_equal_to((char)0x1f)); + assert_that (compressed[1], is_equal_to((char)0x8b)); + assert_that (compressed[2], is_equal_to(8)); + + size_t uncompressed_len; + char *uncompressed + = gvm_uncompress (compressed, compressed_len, &uncompressed_len); + assert_that (uncompressed_len, is_equal_to (strlen(testdata) + 1)); + assert_that (uncompressed, is_equal_to_string (testdata)); +} + +Ensure(compressutils, can_uncompress_using_reader) +{ + const char *testdata = "TEST-12345-12345-TEST"; + size_t compressed_len; + char *compressed + = gvm_compress_gzipheader (testdata, strlen(testdata) + 1, &compressed_len); + + char compressed_filename[35] = "/tmp/gvm_gzip_test_XXXXXX"; + int compressed_fd = mkstemp (compressed_filename); + write (compressed_fd, compressed, compressed_len); + close (compressed_fd); + + FILE *stream = gvm_gzip_open_file_reader (compressed_filename); + assert_that (stream, is_not_null); + + gchar *uncompressed = g_malloc0 (30); + fread (uncompressed, 1, 30, stream); + assert_that (uncompressed, is_equal_to_string (testdata)); + + assert_that (fclose (stream), is_equal_to (0)); +} + +/* Test suite. */ +int +main (int argc, char **argv) +{ + TestSuite *suite; + + suite = create_test_suite (); + + add_test_with_context (suite, compressutils, + can_compress_and_uncompress_without_header); + add_test_with_context (suite, compressutils, + can_compress_and_uncompress_with_header); + add_test_with_context (suite, compressutils, + can_uncompress_using_reader); + + if (argc > 1) + return run_single_test (suite, argv[1], create_text_reporter ()); + + return run_test_suite (suite, create_text_reporter ()); +} \ No newline at end of file