diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index 59bc3a57c8e..a00ac48bf55 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -225,6 +225,7 @@ struct flb_config { int storage_del_bad_chunks; /* delete irrecoverable chunks */ char *storage_bl_mem_limit; /* storage backlog memory limit */ struct flb_storage_metrics *storage_metrics_ctx; /* storage metrics context */ + int storage_trim_files; /* enable/disable file trimming */ /* Embedded SQL Database support (SQLite3) */ #ifdef FLB_HAVE_SQLDB @@ -355,6 +356,7 @@ enum conf_type { #define FLB_CONF_STORAGE_MAX_CHUNKS_UP "storage.max_chunks_up" #define FLB_CONF_STORAGE_DELETE_IRRECOVERABLE_CHUNKS \ "storage.delete_irrecoverable_chunks" +#define FLB_CONF_STORAGE_TRIM_FILES "storage.trim_files" /* Coroutines */ #define FLB_CONF_STR_CORO_STACK_SIZE "Coro_Stack_Size" diff --git a/lib/chunkio/CMakeLists.txt b/lib/chunkio/CMakeLists.txt index 8bfe5199d19..f49d01d9095 100644 --- a/lib/chunkio/CMakeLists.txt +++ b/lib/chunkio/CMakeLists.txt @@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.0) project(chunk-io C) set(CIO_VERSION_MAJOR 1) -set(CIO_VERSION_MINOR 4) -set(CIO_VERSION_PATCH 0) +set(CIO_VERSION_MINOR 5) +set(CIO_VERSION_PATCH 1) set(CIO_VERSION_STR "${CIO_VERSION_MAJOR}.${CIO_VERSION_MINOR}.${CIO_VERSION_PATCH}") # CFLAGS @@ -15,11 +15,7 @@ else() endif() # Set __FILENAME__ -if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__FILENAME__='\"$(subst ${CMAKE_SOURCE_DIR}/,,$(abspath $<))\"'") -else() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__FILENAME__=__FILE__") -endif() +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__FILENAME__=__FILE__") include(cmake/macros.cmake) @@ -91,6 +87,18 @@ if(CIO_HAVE_FALLOCATE) CIO_DEFINITION(CIO_HAVE_FALLOCATE) endif() +# posix_fallocate(2) support +check_c_source_compiles(" + #include + int main() { + posix_fallocate(0,0,0); + return 0; + }" CIO_HAVE_POSIX_FALLOCATE) + +if(CIO_HAVE_POSIX_FALLOCATE) + CIO_DEFINITION(CIO_HAVE_POSIX_FALLOCATE) +endif() + configure_file( "${PROJECT_SOURCE_DIR}/include/chunkio/cio_info.h.in" "${PROJECT_BINARY_DIR}/include/chunkio/cio_info.h" @@ -105,6 +113,7 @@ include_directories( include deps/ deps/monkey/include + ${PROJECT_BINARY_DIR}/include/ ) add_subdirectory(deps/crc32) diff --git a/lib/chunkio/cmake/sanitizers-cmake/CMakeLists.txt b/lib/chunkio/cmake/sanitizers-cmake/CMakeLists.txt index d249a4db5cc..a1928525587 100644 --- a/lib/chunkio/cmake/sanitizers-cmake/CMakeLists.txt +++ b/lib/chunkio/cmake/sanitizers-cmake/CMakeLists.txt @@ -30,7 +30,7 @@ # # minimum required cmake version -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.8.12) # project name project("CMake-sanitizers") diff --git a/lib/chunkio/cmake/sanitizers-cmake/README.md b/lib/chunkio/cmake/sanitizers-cmake/README.md index 8df8d175c21..b4ca621c31e 100644 --- a/lib/chunkio/cmake/sanitizers-cmake/README.md +++ b/lib/chunkio/cmake/sanitizers-cmake/README.md @@ -11,7 +11,7 @@ CMake module to enable sanitizers for binary targets. To use [FindSanitizers.cmake](cmake/FindSanitizers.cmake), simply add this repository as git submodule into your own repository ```Shell mkdir externals -git submodule add git://github.com/arsenm/sanitizers-cmake.git externals/sanitizers-cmake +git submodule add git@github.com:arsenm/sanitizers-cmake.git externals/sanitizers-cmake ``` and adding ```externals/sanitizers-cmake/cmake``` to your ```CMAKE_MODULE_PATH``` ```CMake diff --git a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindASan.cmake b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindASan.cmake index 98ea7cb311c..4548e46a887 100644 --- a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindASan.cmake +++ b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindASan.cmake @@ -25,6 +25,9 @@ option(SANITIZE_ADDRESS "Enable AddressSanitizer for sanitized targets." Off) set(FLAG_CANDIDATES + # MSVC uses + "/fsanitize=address" + # Clang 3.2+ use this version. The no-omit-frame-pointer option is optional. "-g -fsanitize=address -fno-omit-frame-pointer" "-g -fsanitize=address" diff --git a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindMSan.cmake b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindMSan.cmake index 22d0050e97c..d744c34be81 100644 --- a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindMSan.cmake +++ b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindMSan.cmake @@ -25,6 +25,9 @@ option(SANITIZE_MEMORY "Enable MemorySanitizer for sanitized targets." Off) set(FLAG_CANDIDATES + # MSVC uses + "/fsanitize=memory" + # GNU/Clang "-g -fsanitize=memory" ) diff --git a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindSanitizers.cmake b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindSanitizers.cmake old mode 100644 new mode 100755 index a11809b0bb4..d9b438c0f6c --- a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindSanitizers.cmake +++ b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindSanitizers.cmake @@ -26,8 +26,8 @@ # link against the sanitizers. option(SANITIZE_LINK_STATIC "Try to link static against sanitizers." Off) - - +# Highlight this module has been loaded. +set(Sanitizers_FOUND TRUE) set(FIND_QUIETLY_FLAG "") if (DEFINED Sanitizers_FIND_QUIETLY) @@ -39,9 +39,6 @@ find_package(TSan ${FIND_QUIETLY_FLAG}) find_package(MSan ${FIND_QUIETLY_FLAG}) find_package(UBSan ${FIND_QUIETLY_FLAG}) - - - function(sanitizer_add_blacklist_file FILE) if(NOT IS_ABSOLUTE ${FILE}) set(FILE "${CMAKE_CURRENT_SOURCE_DIR}/${FILE}") @@ -52,7 +49,7 @@ function(sanitizer_add_blacklist_file FILE) "SanitizerBlacklist" "SanBlist") endfunction() -function(add_sanitizers ...) +function(add_sanitizers) # If no sanitizer is enabled, return immediately. if (NOT (SANITIZE_ADDRESS OR SANITIZE_MEMORY OR SANITIZE_THREAD OR SANITIZE_UNDEFINED)) @@ -77,12 +74,12 @@ function(add_sanitizers ...) "Target will be compiled without sanitizers.") return() - # If the target is compiled by no known compiler, ignore it. elseif (NUM_COMPILERS EQUAL 0) - message(WARNING "Can't use any sanitizers for target ${TARGET}, " - "because it uses an unknown compiler. Target will be " - "compiled without sanitizers.") - return() + # If the target is compiled by no known compiler, give a warning. + message(WARNING "Sanitizers for target ${TARGET} may not be" + " usable, because it uses no or an unknown compiler. " + "This is a false warning for targets using only " + "object lib(s) as input.") endif () # Add sanitizers for target. @@ -90,5 +87,5 @@ function(add_sanitizers ...) add_sanitize_thread(${TARGET}) add_sanitize_memory(${TARGET}) add_sanitize_undefined(${TARGET}) - endforeach () + endforeach () endfunction(add_sanitizers) diff --git a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindTSan.cmake b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindTSan.cmake index 3cba3c03b6b..efb2e9525b8 100644 --- a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindTSan.cmake +++ b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindTSan.cmake @@ -25,6 +25,9 @@ option(SANITIZE_THREAD "Enable ThreadSanitizer for sanitized targets." Off) set(FLAG_CANDIDATES + # MSVC uses + "/fsanitize=thread" + # GNU/Clang "-g -fsanitize=thread" ) diff --git a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindUBSan.cmake b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindUBSan.cmake index ae103f71725..4354c2e4d63 100644 --- a/lib/chunkio/cmake/sanitizers-cmake/cmake/FindUBSan.cmake +++ b/lib/chunkio/cmake/sanitizers-cmake/cmake/FindUBSan.cmake @@ -26,6 +26,9 @@ option(SANITIZE_UNDEFINED "Enable UndefinedBehaviorSanitizer for sanitized targets." Off) set(FLAG_CANDIDATES + # MSVC uses + "/fsanitize=undefined" + # GNU/Clang "-g -fsanitize=undefined" ) diff --git a/lib/chunkio/cmake/sanitizers-cmake/cmake/sanitize-helpers.cmake b/lib/chunkio/cmake/sanitizers-cmake/cmake/sanitize-helpers.cmake old mode 100644 new mode 100755 index 610f8957420..efc325ce309 --- a/lib/chunkio/cmake/sanitizers-cmake/cmake/sanitize-helpers.cmake +++ b/lib/chunkio/cmake/sanitizers-cmake/cmake/sanitize-helpers.cmake @@ -25,6 +25,11 @@ # Helper function to get the language of a source file. function (sanitizer_lang_of_source FILE RETURN_VAR) get_filename_component(LONGEST_EXT "${FILE}" EXT) + # If extension is empty return. This can happen for extensionless headers + if("${LONGEST_EXT}" STREQUAL "") + set(${RETURN_VAR} "" PARENT_SCOPE) + return() + endif() # Get shortest extension as some files can have dot in their names string(REGEX REPLACE "^.*(\\.[^.]+)$" "\\1" FILE_EXT ${LONGEST_EXT}) string(TOLOWER "${FILE_EXT}" FILE_EXT) @@ -70,6 +75,7 @@ endfunction () # Helper function to check compiler flags for language compiler. function (sanitizer_check_compiler_flag FLAG LANG VARIABLE) + if (${LANG} STREQUAL "C") include(CheckCCompilerFlag) check_c_compiler_flag("${FLAG}" ${VARIABLE}) @@ -92,6 +98,7 @@ function (sanitizer_check_compiler_flag FLAG LANG VARIABLE) " - Failed (Check not supported)") endif () endif() + endfunction () @@ -105,7 +112,7 @@ function (sanitizer_check_compiler_flags FLAG_CANDIDATES NAME PREFIX) # So instead of searching flags foreach language, search flags foreach # compiler used. set(COMPILER ${CMAKE_${LANG}_COMPILER_ID}) - if (NOT DEFINED ${PREFIX}_${COMPILER}_FLAGS) + if (COMPILER AND NOT DEFINED ${PREFIX}_${COMPILER}_FLAGS) foreach (FLAG ${FLAG_CANDIDATES}) if(NOT CMAKE_REQUIRED_QUIET) message(STATUS "Try ${COMPILER} ${NAME} flag = [${FLAG}]") @@ -162,11 +169,10 @@ function (sanitizer_add_flags TARGET NAME PREFIX) return() endif() - # Set compile- and link-flags for target. - set_property(TARGET ${TARGET} APPEND_STRING - PROPERTY COMPILE_FLAGS " ${${PREFIX}_${TARGET_COMPILER}_FLAGS}") - set_property(TARGET ${TARGET} APPEND_STRING - PROPERTY COMPILE_FLAGS " ${SanBlist_${TARGET_COMPILER}_FLAGS}") - set_property(TARGET ${TARGET} APPEND_STRING - PROPERTY LINK_FLAGS " ${${PREFIX}_${TARGET_COMPILER}_FLAGS}") + separate_arguments(flags_list UNIX_COMMAND "${${PREFIX}_${TARGET_COMPILER}_FLAGS} ${SanBlist_${TARGET_COMPILER}_FLAGS}") + target_compile_options(${TARGET} PUBLIC ${flags_list}) + + separate_arguments(flags_list UNIX_COMMAND "${${PREFIX}_${TARGET_COMPILER}_FLAGS}") + target_link_options(${TARGET} PUBLIC ${flags_list}) + endfunction () diff --git a/lib/chunkio/include/chunkio/chunkio.h b/lib/chunkio/include/chunkio/chunkio.h index 6dff9edb08d..c60ddacce1d 100644 --- a/lib/chunkio/include/chunkio/chunkio.h +++ b/lib/chunkio/include/chunkio/chunkio.h @@ -44,6 +44,7 @@ #define CIO_CHECKSUM 4 /* enable checksum verification (crc32) */ #define CIO_FULL_SYNC 8 /* force sync to fs through MAP_SYNC */ #define CIO_DELETE_IRRECOVERABLE 16 /* delete irrecoverable chunks from disk */ +#define CIO_TRIM_FILES 32 /* trim files to their required size */ /* Return status */ #define CIO_CORRUPTED -3 /* Indicate that a chunk is corrupted */ @@ -51,13 +52,25 @@ #define CIO_ERROR -1 /* Generic error */ #define CIO_OK 0 /* OK */ +/* Configuration limits */ +/* The file minimum growth factor is 8 memory pages and + * the file maximum growth factor is 8 megabytes + */ +#define CIO_REALLOC_HINT_MIN (cio_getpagesize() * 8) +#define CIO_REALLOC_HINT_MAX (8 * 1000 * 1000) /* defaults */ -#define CIO_MAX_CHUNKS_UP 64 /* default limit for cio_ctx->max_chunks_up */ +#define CIO_MAX_CHUNKS_UP 64 /* default limit for cio_ctx->max_chunks_up */ +#define CIO_DISABLE_REALLOC_HINT -1 /* default value of size of realloc hint */ +#define CIO_DEFAULT_REALLOC_HINT CIO_REALLOC_HINT_MIN +#define CIO_INITIALIZED 1337 struct cio_ctx; struct cio_options { + /* this bool flag sets if the options has been initialized, that's a mandatory step */ + int initialized; + int flags; char *root_path; @@ -68,10 +81,14 @@ struct cio_options { char *user; char *group; char *chmod; + + /* chunk handlings */ + int realloc_size_hint; }; struct cio_ctx { int page_size; + int realloc_size_hint; struct cio_options options; void *processed_user; @@ -104,7 +121,7 @@ struct cio_ctx { #include #include - +void cio_options_init(struct cio_options *options); struct cio_ctx *cio_create(struct cio_options *options); void cio_destroy(struct cio_ctx *ctx); int cio_load(struct cio_ctx *ctx, char *chunk_extension); @@ -113,6 +130,10 @@ int cio_qsort(struct cio_ctx *ctx, int (*compar)(const void *, const void *)); void cio_set_log_callback(struct cio_ctx *ctx, void (*log_cb)); int cio_set_log_level(struct cio_ctx *ctx, int level); int cio_set_max_chunks_up(struct cio_ctx *ctx, int n); +int cio_set_realloc_size_hint(struct cio_ctx *ctx, size_t realloc_size_hint); + +void cio_enable_file_trimming(struct cio_ctx *ctx); +void cio_disable_file_trimming(struct cio_ctx *ctx); int cio_meta_write(struct cio_chunk *ch, char *buf, size_t size); int cio_meta_cmp(struct cio_chunk *ch, char *meta_buf, int meta_len); diff --git a/lib/chunkio/include/chunkio/cio_file.h b/lib/chunkio/include/chunkio/cio_file.h index d75b14c3a30..7d447492902 100644 --- a/lib/chunkio/include/chunkio/cio_file.h +++ b/lib/chunkio/include/chunkio/cio_file.h @@ -35,6 +35,7 @@ struct cio_file { int allocate_strategy; /* linux-only: fallocate strategy */ size_t fs_size; /* original size in the file system */ size_t data_size; /* number of bytes used */ + size_t page_size; /* curent page size */ size_t alloc_size; /* allocated size */ size_t realloc_size; /* chunk size to increase alloc */ char *path; /* root path + stream */ diff --git a/lib/chunkio/include/chunkio/cio_file_st.h b/lib/chunkio/include/chunkio/cio_file_st.h index a6b4f46cd60..4b1552b612a 100644 --- a/lib/chunkio/include/chunkio/cio_file_st.h +++ b/lib/chunkio/include/chunkio/cio_file_st.h @@ -36,7 +36,10 @@ * +--------------+----------------+ * | 0xC1 | 0x00 +--> Header 2 bytes * +--------------+----------------+ - * | 4 BYTES CRC32 + 16 BYTES +--> CRC32(Content) + Padding + * | 4 BYTES +--> CRC32(Content) + * | 4 BYTES +--> CRC32(Padding) + * | 4 BYTES +--> Content length + * | 8 BYTES +--> Padding * +-------------------------------+ * | Content | * | +-------------------------+ | @@ -55,11 +58,14 @@ * +-------------------------------+ */ -#define CIO_FILE_ID_00 0xc1 /* header: first byte */ -#define CIO_FILE_ID_01 0x00 /* header: second byte */ -#define CIO_FILE_HEADER_MIN 24 /* 24 bytes for the header */ -#define CIO_FILE_CONTENT_OFFSET 22 - +#define CIO_FILE_ID_00 0xc1 /* header: first byte */ +#define CIO_FILE_ID_01 0x00 /* header: second byte */ +#define CIO_FILE_HEADER_MIN 24 /* 24 bytes for the header */ +#define CIO_FILE_CONTENT_OFFSET 22 +#define CIO_FILE_CONTENT_LENGTH_OFFSET 10 /* We store the content length + * right after the checksum in + * what used to be padding + */ /* Return pointer to hash position */ static inline char *cio_file_st_get_hash(char *map) { @@ -94,22 +100,71 @@ static inline char *cio_file_st_get_content(char *map) return map + CIO_FILE_HEADER_MIN + len; } -static inline ssize_t cio_file_st_get_content_size(char *map, size_t size) +/* Infer content length when not available */ +static inline ssize_t cio_file_st_infer_content_len(char *map, size_t size) +{ + size_t content_length; + + content_length = size; + content_length -= CIO_FILE_HEADER_MIN; + content_length -= cio_file_st_get_meta_len(map); + + return content_length; +} + +/* Get content length */ +static inline ssize_t cio_file_st_get_content_len(char *map, size_t size, + size_t page_size) { - int meta_len; - size_t s; + uint8_t *content_length_buffer; + ssize_t content_length; if (size < CIO_FILE_HEADER_MIN) { return -1; } - meta_len = cio_file_st_get_meta_len(map); - s = (size - CIO_FILE_HEADER_MIN) - meta_len; - if (s < size) { - return s; + content_length_buffer = (uint8_t *) &map[CIO_FILE_CONTENT_LENGTH_OFFSET]; + + content_length = (ssize_t) (((uint32_t) content_length_buffer[0]) << 24) | + (((uint32_t) content_length_buffer[1]) << 16) | + (((uint32_t) content_length_buffer[2]) << 8) | + (((uint32_t) content_length_buffer[3]) << 0); + + /* This is required in order to be able to load chunk files generated by + * previous versions of chunkio that didn't include the content length + * as part of the headers. + * + * The reason why we need to ensure that the file size is larger than 4096 + * is that this is the minimal expected page size which is the unit used + * to initialize chunk files when they are created. + * + * In doing so, we effectively avoid returning bogus results when loading + * newly created, non trimmed files while at the same time retaining the + * capability of loading legacy files (that don't have a content size) + * that are larger than 4096 bytes. + * + * The only caveat is that trimmed files + */ + if (content_length == 0 && + size > 0 && + size != page_size) { + content_length = cio_file_st_infer_content_len(map, size); } - return -1; + return content_length; +} + +/* Set content length */ +static inline void cio_file_st_set_content_len(char *map, uint32_t len) +{ + uint8_t *content_length_buffer; + + content_length_buffer = (uint8_t *) &map[CIO_FILE_CONTENT_LENGTH_OFFSET]; + + content_length_buffer[0] = (uint8_t) ((len & 0xFF000000) >> 24); + content_length_buffer[1] = (uint8_t) ((len & 0x00FF0000) >> 16); + content_length_buffer[2] = (uint8_t) ((len & 0x0000FF00) >> 8); + content_length_buffer[3] = (uint8_t) ((len & 0x000000FF) >> 0); } #endif diff --git a/lib/chunkio/src/chunkio.c b/lib/chunkio/src/chunkio.c index 89e76b39da3..a69325cfe9e 100644 --- a/lib/chunkio/src/chunkio.c +++ b/lib/chunkio/src/chunkio.c @@ -65,25 +65,44 @@ static int check_root_path(struct cio_ctx *ctx, const char *root_path) return access(root_path, W_OK); } +void cio_options_init(struct cio_options *options) +{ + memset(options, 0, sizeof(struct cio_options)); + + options->initialized = CIO_INITIALIZED; + + options->root_path = NULL; + options->user = NULL; + options->group = NULL; + options->chmod = NULL; + options->log_cb = NULL; + options->log_level = CIO_LOG_INFO; + options->flags = CIO_OPEN_RW; + options->realloc_size_hint = CIO_DISABLE_REALLOC_HINT; +} + struct cio_ctx *cio_create(struct cio_options *options) { int ret; struct cio_ctx *ctx; struct cio_options default_options; - memset(&default_options, 0, sizeof(default_options)); - - default_options.root_path = NULL; - default_options.user = NULL; - default_options.group = NULL; - default_options.chmod = NULL; - default_options.log_cb = NULL; - default_options.log_level = CIO_LOG_INFO; - default_options.flags = 0; - if (options == NULL) { + cio_options_init(&default_options); options = &default_options; } + else { + if (options->initialized != CIO_INITIALIZED) { + /* the caller 'must' call cio_options_init() or pass NULL before creating a context */ + fprintf(stderr, "[cio] 'options' has not been initialized properly\n"); + return NULL; + } + } + + /* sanitize chunk open flags */ + if (!(options->flags & CIO_OPEN_RW) && !(options->flags & CIO_OPEN_RD)) { + options->flags |= CIO_OPEN_RW; + } if (options->log_level < CIO_LOG_ERROR || options->log_level > CIO_LOG_TRACE) { @@ -107,6 +126,7 @@ struct cio_ctx *cio_create(struct cio_options *options) ctx->page_size = cio_getpagesize(); ctx->max_chunks_up = CIO_MAX_CHUNKS_UP; ctx->options.flags = options->flags; + ctx->realloc_size_hint = CIO_DISABLE_REALLOC_HINT; if (options->user != NULL) { ctx->options.user = strdup(options->user); @@ -171,6 +191,18 @@ struct cio_ctx *cio_create(struct cio_options *options) ctx->processed_group = NULL; } + if (options->realloc_size_hint > 0) { + ret = cio_set_realloc_size_hint(ctx, options->realloc_size_hint); + if (ret == -1) { + cio_log_error(ctx, + "[chunkio] cannot initialize with realloc size hint %d\n", + options->realloc_size_hint); + cio_destroy(ctx); + + return NULL; + } + } + return ctx; } @@ -305,3 +337,33 @@ int cio_set_max_chunks_up(struct cio_ctx *ctx, int n) ctx->max_chunks_up = n; return 0; } + +int cio_set_realloc_size_hint(struct cio_ctx *ctx, size_t realloc_size_hint) +{ + if (realloc_size_hint < CIO_REALLOC_HINT_MIN) { + cio_log_error(ctx, + "[chunkio] cannot specify less than %zu bytes\n", + CIO_REALLOC_HINT_MIN); + return -1; + } + else if (realloc_size_hint > CIO_REALLOC_HINT_MAX) { + cio_log_error(ctx, + "[chunkio] cannot specify more than %zu bytes\n", + CIO_REALLOC_HINT_MAX); + return -1; + } + + ctx->realloc_size_hint = realloc_size_hint; + + return 0; +} + +void cio_enable_file_trimming(struct cio_ctx *ctx) +{ + ctx->options.flags |= CIO_TRIM_FILES; +} + +void cio_disable_file_trimming(struct cio_ctx *ctx) +{ + ctx->options.flags &= ~CIO_TRIM_FILES; +} \ No newline at end of file diff --git a/lib/chunkio/src/cio_file.c b/lib/chunkio/src/cio_file.c index 786a022e927..019baa89048 100644 --- a/lib/chunkio/src/cio_file.c +++ b/lib/chunkio/src/cio_file.c @@ -40,6 +40,8 @@ #include #include +size_t scio_file_page_size = 0; + char cio_file_init_bytes[] = { /* file type (2 bytes) */ CIO_FILE_ID_00, CIO_FILE_ID_01, @@ -59,25 +61,31 @@ char cio_file_init_bytes[] = { #define ROUND_UP(N, S) ((((N) + (S) - 1) / (S)) * (S)) -/* Get the number of bytes in the Content section */ -static size_t content_len(struct cio_file *cf) -{ - int meta; - size_t len; - - meta = cio_file_st_get_meta_len(cf->map); - len = 2 + meta + cf->data_size; - return len; -} /* Calculate content checksum in a variable */ void cio_file_calculate_checksum(struct cio_file *cf, crc_t *out) { crc_t val; size_t len; + ssize_t content_length; unsigned char *in_data; - len = content_len(cf); + if (cf->fs_size == 0) { + cio_file_update_size(cf); + } + + /* Metadata length header + metadata length + content length */ + len = 2; + len += cio_file_st_get_meta_len(cf->map); + + content_length = cio_file_st_get_content_len(cf->map, + cf->fs_size, + cf->page_size); + + if (content_length > 0) { + len += content_length; + } + in_data = (unsigned char *) cf->map + CIO_FILE_CONTENT_OFFSET; val = cio_crc32_update(cf->crc_cur, in_data, len); *out = val; @@ -109,6 +117,7 @@ static void finalize_checksum(struct cio_file *cf) crc = cio_crc32_finalize(cf->crc_cur); crc = htonl(crc); + memcpy(cf->map + 2, &crc, sizeof(crc)); } @@ -146,21 +155,26 @@ static void write_init_header(struct cio_chunk *ch, struct cio_file *cf) cf->map[4] = 0; cf->map[5] = 0; } + + cio_file_st_set_content_len(cf->map, 0); } /* Return the available size in the file map to write data */ static size_t get_available_size(struct cio_file *cf, int *meta_len) { size_t av; - int len; + int metadata_len; /* Get metadata length */ - len = cio_file_st_get_meta_len(cf->map); + metadata_len = cio_file_st_get_meta_len(cf->map); + + av = cf->alloc_size; + av -= CIO_FILE_HEADER_MIN; + av -= metadata_len; + av -= cf->data_size; - av = cf->alloc_size - cf->data_size; - av -= (CIO_FILE_HEADER_MIN + len); + *meta_len = metadata_len; - *meta_len = len; return av; } @@ -171,6 +185,9 @@ static size_t get_available_size(struct cio_file *cf, int *meta_len) static int cio_file_format_check(struct cio_chunk *ch, struct cio_file *cf, int flags) { + size_t metadata_length; + ssize_t content_length; + ssize_t logical_length; unsigned char *p; crc_t crc_check; crc_t crc; @@ -186,6 +203,7 @@ static int cio_file_format_check(struct cio_chunk *ch, cio_log_warn(ch->ctx, "[cio file] cannot initialize chunk (read-only)"); cio_error_set(ch, CIO_ERR_PERMISSION); + return -1; } @@ -193,6 +211,7 @@ static int cio_file_format_check(struct cio_chunk *ch, if (cf->alloc_size < CIO_FILE_HEADER_MIN) { cio_log_warn(ch->ctx, "[cio file] cannot initialize chunk"); cio_error_set(ch, CIO_ERR_BAD_LAYOUT); + return -1; } @@ -210,6 +229,34 @@ static int cio_file_format_check(struct cio_chunk *ch, cio_log_debug(ch->ctx, "[cio file] invalid header at %s", ch->name); cio_error_set(ch, CIO_ERR_BAD_LAYOUT); + + return -1; + } + + /* Expected / logical file size verification */ + content_length = cio_file_st_get_content_len(cf->map, + cf->fs_size, + cf->page_size); + + if (content_length == -1) { + cio_log_debug(ch->ctx, "[cio file] truncated header (%zu / %zu) %s", + cf->fs_size, CIO_FILE_HEADER_MIN, ch->name); + cio_error_set(ch, CIO_ERR_BAD_FILE_SIZE); + + return -1; + } + + metadata_length = cio_file_st_get_meta_len(cf->map); + + logical_length = CIO_FILE_HEADER_MIN + + metadata_length + + content_length; + + if (logical_length > cf->fs_size) { + cio_log_debug(ch->ctx, "[cio file] truncated file (%zd / %zd) %s", + cf->fs_size, logical_length, ch->name); + cio_error_set(ch, CIO_ERR_BAD_FILE_SIZE); + return -1; } @@ -227,12 +274,15 @@ static int cio_file_format_check(struct cio_chunk *ch, /* Compare */ crc_check = cio_crc32_finalize(crc); crc_check = htonl(crc_check); + if (memcmp(p, &crc_check, sizeof(crc_check)) != 0) { - cio_log_debug(ch->ctx, "[cio file] invalid crc32 at %s/%s", + cio_log_info(ch->ctx, "[cio file] invalid crc32 at %s/%s", ch->name, cf->path); cio_error_set(ch, CIO_ERR_BAD_CHECKSUM); + return -1; } + cf->crc_cur = crc; } } @@ -368,7 +418,9 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) /* check content data size */ if (fs_size > 0) { - content_size = cio_file_st_get_content_size(cf->map, fs_size); + content_size = cio_file_st_get_content_len(cf->map, + fs_size, + cf->page_size); if (content_size == -1) { cio_error_set(ch, CIO_ERR_BAD_FILE_SIZE); @@ -383,6 +435,7 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) return CIO_CORRUPTED; } + cf->data_size = content_size; cf->fs_size = fs_size; } @@ -591,7 +644,15 @@ struct cio_file *cio_file_open(struct cio_ctx *ctx, cf->fd = -1; cf->flags = flags; - cf->realloc_size = cio_getpagesize() * 8; + cf->page_size = cio_getpagesize(); + + if (ctx->realloc_size_hint > 0) { + cf->realloc_size = ctx->realloc_size_hint; + } + else { + cf->realloc_size = CIO_REALLOC_HINT_MIN; + } + cf->st_content = NULL; cf->crc_cur = cio_crc32_init(); cf->path = path; @@ -699,14 +760,14 @@ int cio_file_delete(struct cio_ctx *ctx, struct cio_stream *st, const char *name char *path; int ret; - ret = cio_file_native_filename_check(name); + ret = cio_file_native_filename_check((char *) name); if (ret != CIO_OK) { cio_log_error(ctx, "[cio file] invalid file name"); return CIO_ERROR; } - path = cio_file_native_compose_path(ctx->options.root_path, st->name, name); + path = cio_file_native_compose_path(ctx->options.root_path, st->name, (char *) name); if (path == NULL) { return CIO_ERROR; } @@ -890,6 +951,7 @@ void cio_file_close(struct cio_chunk *ch, int delete) free(cf); } + int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) { int ret; @@ -946,6 +1008,14 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) old_size, new_size); } + /* If crc_reset was toggled we know that data_size was + * modified by cio_chunk_write_at which means we need + * to update the header before we recalculate the checksum + */ + if (cf->crc_reset) { + cio_file_st_set_content_len(cf->map, cf->data_size); + } + if (ch->ctx->options.flags & CIO_CHECKSUM) { update_checksum(cf, (unsigned char *) buf, count); } @@ -956,6 +1026,8 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) cf->data_size += count; cf->synced = CIO_FALSE; + cio_file_st_set_content_len(cf->map, cf->data_size); + return 0; } @@ -1069,28 +1141,41 @@ int cio_file_sync(struct cio_chunk *ch) return -1; } - /* If there are extra space, truncate the file size */ - av_size = get_available_size(cf, &meta_len); + /* File trimming has been made opt-in because it causes + * performance degradation and excessive fragmentation + * in XFS. + */ + if ((ch->ctx->options.flags & CIO_TRIM_FILES) != 0) { + /* If there are extra space, truncate the file size */ + av_size = get_available_size(cf, &meta_len); - if (av_size > 0) { - desired_size = cf->alloc_size - av_size; - } - else if (cf->alloc_size > file_size) { - desired_size = cf->alloc_size; - } - else { - desired_size = file_size; - } + if (av_size > 0) { + desired_size = cf->alloc_size - av_size; + } + else if (cf->alloc_size > file_size) { + desired_size = cf->alloc_size; + } + else { + desired_size = file_size; + } - if (desired_size != file_size) { - ret = cio_file_resize(cf, desired_size); + if (desired_size != file_size) { + /* When file trimming is enabled we still round the file size up + * to the memory page size because even though not explicitly + * stated there seems to be a performance degradation issue that + * correlates with sub-page mapping. + */ + desired_size = ROUND_UP(desired_size, ch->ctx->page_size); - if (ret != CIO_OK) { - cio_log_error(ch->ctx, - "[cio file sync] error adjusting size at: " - " %s/%s", ch->st->name, ch->name); + ret = cio_file_resize(cf, desired_size); + + if (ret != CIO_OK) { + cio_log_error(ch->ctx, + "[cio file sync] error adjusting size at: " + " %s/%s", ch->st->name, ch->name); - return ret; + return ret; + } } } diff --git a/lib/chunkio/src/cio_file_unix.c b/lib/chunkio/src/cio_file_unix.c index a9ef6056fbe..72d49312dc1 100644 --- a/lib/chunkio/src/cio_file_unix.c +++ b/lib/chunkio/src/cio_file_unix.c @@ -497,45 +497,65 @@ int cio_file_native_sync(struct cio_file *cf, int sync_mode) int cio_file_native_resize(struct cio_file *cf, size_t new_size) { + int fallocate_available; int result; result = -1; +#if defined(CIO_HAVE_FALLOCATE) || defined(CIO_HAVE_POSIX_FALLOCATE) + fallocate_available = CIO_TRUE; +#else + fallocate_available = CIO_FALSE; +#endif + /* * fallocate() is not portable an Linux only. Since macOS does not have * fallocate() we use ftruncate(). */ -#if defined(CIO_HAVE_FALLOCATE) - if (new_size > cf->alloc_size) { + if (fallocate_available && new_size > cf->fs_size) { retry: - if (cf->allocate_strategy == CIO_FILE_LINUX_FALLOCATE) { - /* - * To increase the file size we use fallocate() since this option - * will send a proper ENOSPC error if the file system ran out of - * space. ftruncate() will not fail and upon memcpy() over the - * mmap area it will trigger a 'Bus Error' crashing the program. - * - * fallocate() is not portable, Linux only. - */ - result = fallocate(cf->fd, 0, 0, new_size); - if (result == -1 && errno == EOPNOTSUPP) { - /* - * If fallocate fails with an EOPNOTSUPP try operation using - * posix_fallocate. Required since some filesystems do not support - * the fallocate operation e.g. ext3 and reiserfs. - */ - cf->allocate_strategy = CIO_FILE_LINUX_POSIX_FALLOCATE; - goto retry; - } - } - else if (cf->allocate_strategy == CIO_FILE_LINUX_POSIX_FALLOCATE) { + if (cf->allocate_strategy == CIO_FILE_LINUX_FALLOCATE) { + /* + * To increase the file size we use fallocate() since this option + * will send a proper ENOSPC error if the file system ran out of + * space. ftruncate() will not fail and upon memcpy() over the + * mmap area it will trigger a 'Bus Error' crashing the program. + * + * fallocate() is not portable, Linux only. + */ +#if defined(CIO_HAVE_FALLOCATE) + result = fallocate(cf->fd, 0, 0, new_size); + +#elif defined(CIO_HAVE_POSIX_FALLOCATE) + result = -1; + errno = EOPNOTSUPP; +#endif + + if (result == -1 && errno == EOPNOTSUPP) { + /* + * If fallocate fails with an EOPNOTSUPP try operation using + * posix_fallocate. Required since some filesystems do not support + * the fallocate operation e.g. ext3 and reiserfs. + */ + cf->allocate_strategy = CIO_FILE_LINUX_POSIX_FALLOCATE; + goto retry; + } + } + else if (cf->allocate_strategy == CIO_FILE_LINUX_POSIX_FALLOCATE) { +#if defined(CIO_HAVE_POSIX_FALLOCATE) result = posix_fallocate(cf->fd, 0, new_size); - } +#else + goto fallback; +#endif + } } else -#endif { +#if !defined(CIO_HAVE_POSIX_FALLOCATE) + fallback: +#endif + result = ftruncate(cf->fd, new_size); } diff --git a/lib/chunkio/src/cio_memfs.c b/lib/chunkio/src/cio_memfs.c index d114355c97a..8cd0f6c10eb 100644 --- a/lib/chunkio/src/cio_memfs.c +++ b/lib/chunkio/src/cio_memfs.c @@ -54,7 +54,11 @@ struct cio_memfs *cio_memfs_open(struct cio_ctx *ctx, struct cio_stream *st, } mf->buf_size = size; mf->buf_len = 0; - mf->realloc_size = cio_getpagesize() * 8; + if (ctx->realloc_size_hint > 0) { + mf->realloc_size = ctx->realloc_size_hint; + } else { + mf->realloc_size = cio_getpagesize() * 8; + } return mf; } diff --git a/lib/chunkio/tests/CMakeLists.txt b/lib/chunkio/tests/CMakeLists.txt index 063d2fa8e8c..7cee7e6c936 100644 --- a/lib/chunkio/tests/CMakeLists.txt +++ b/lib/chunkio/tests/CMakeLists.txt @@ -34,3 +34,24 @@ foreach(source_file ${UNIT_TESTS_FILES}) add_test(${source_file_we} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${source_file_we}) endforeach() + +# Perf tests for dev purposes: note these tests are not registered, they need to +# be executed manually +set(UNIT_PERF_TESTS + fs_perf.c + fs_fragmentation.c + ) +foreach(source_file ${UNIT_PERF_TESTS}) + get_filename_component(source_file_we ${source_file} NAME_WE) + set(source_file_we cio-${source_file_we}) + add_executable( + ${source_file_we} + ${source_file} + ) + target_link_libraries(${source_file_we} chunkio-static) + + if (CIO_SANITIZE_ADDRESS) + add_sanitizers(${source_file_we}) + endif() + #add_test(${source_file_we} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${source_file_we}) +endforeach() diff --git a/lib/chunkio/tests/context.c b/lib/chunkio/tests/context.c index c64b94ced93..7b7d9ea4865 100644 --- a/lib/chunkio/tests/context.c +++ b/lib/chunkio/tests/context.c @@ -46,10 +46,8 @@ static void test_context() flags = CIO_CHECKSUM; - memset(&cio_opts, 0, sizeof(cio_opts)); - + cio_options_init(&cio_opts); cio_opts.flags = flags; - cio_opts.log_cb = NULL; /* Invalid path */ cio_opts.root_path = ""; @@ -97,10 +95,7 @@ static void test_log_level() struct cio_ctx *ctx; struct cio_options cio_opts; - memset(&cio_opts, 0, sizeof(cio_opts)); - - cio_opts.flags = 0; - cio_opts.log_cb = NULL; + cio_options_init(&cio_opts); /* Logging with unset callback at creation, but set later */ log_check = 0; @@ -159,8 +154,29 @@ static void test_log_level() cio_destroy(ctx); } +static void test_open_flags() +{ + struct cio_ctx *ctx; + struct cio_options cio_opts; + + cio_options_init(&cio_opts); + TEST_CHECK(cio_opts.flags & CIO_OPEN_RW); + + /* reset flags */ + cio_opts.flags = 0; + + /* check that after context creation a default has been set */ + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + TEST_CHECK(cio_opts.flags & CIO_OPEN_RW); + + /* destroy context */ + cio_destroy(ctx); +} + TEST_LIST = { {"context", test_context}, {"log_level", test_log_level}, + {"open_flags", test_open_flags}, { 0 } }; diff --git a/lib/chunkio/tests/fs.c b/lib/chunkio/tests/fs.c index 2de052617e8..a976f46d103 100644 --- a/lib/chunkio/tests/fs.c +++ b/lib/chunkio/tests/fs.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "cio_tests_internal.h" @@ -73,11 +74,10 @@ static void test_fs_write() flags = CIO_CHECKSUM; - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; - cio_opts.log_level = CIO_LOG_INFO; cio_opts.flags = flags; /* cleanup environment */ @@ -160,7 +160,7 @@ static void test_fs_write() free(in_data); cio_destroy(ctx); - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; @@ -221,11 +221,10 @@ static void test_fs_checksum() /* cleanup environment */ cio_utils_recursive_delete(CIO_ENV); - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; - cio_opts.log_level = CIO_LOG_INFO; cio_opts.flags = flags; ctx = cio_create(&cio_opts); @@ -336,7 +335,7 @@ static void test_fs_up_down() /* cleanup environment */ cio_utils_recursive_delete(CIO_ENV); - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; @@ -442,7 +441,7 @@ static void test_issue_51() struct cio_options cio_opts; /* Create a temporal storage */ - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = "tmp"; cio_opts.log_cb = log_cb; @@ -495,7 +494,7 @@ static void test_issue_flb_2025() cio_utils_recursive_delete("tmp"); /* Create a temporal storage */ - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = "tmp"; cio_opts.log_cb = log_cb; @@ -547,7 +546,7 @@ void test_fs_size_chunks_up() flags = CIO_CHECKSUM; - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; @@ -646,7 +645,7 @@ void test_issue_write_at() cio_utils_recursive_delete(CIO_ENV); /* create Chunk I/O context */ - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; @@ -740,7 +739,7 @@ void test_fs_up_down_up_append() cio_utils_recursive_delete(CIO_ENV); /* create Chunk I/O context */ - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; @@ -818,7 +817,7 @@ static void test_deep_hierarchy() cio_utils_recursive_delete("tmp"); /* Create a temporal storage */ - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = "tmp/deep/log/dir"; cio_opts.log_cb = log_cb; @@ -849,6 +848,122 @@ static void test_deep_hierarchy() cio_destroy(ctx); } +static void truncate_file(struct cio_file *chunk_file, + size_t new_file_size, + int remove_content_length) +{ + int result; + + result = cio_file_native_open(chunk_file); + TEST_CHECK(result == CIO_OK); + + result = cio_file_native_map(chunk_file, + chunk_file->page_size); + TEST_CHECK(result == CIO_OK); + + if (remove_content_length) { + chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 0] = 0; + chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 1] = 0; + chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 2] = 0; + chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 3] = 0; + } + + result = cio_file_native_unmap(chunk_file); + TEST_CHECK(result == CIO_OK); + + result = cio_file_native_resize(chunk_file, new_file_size); + TEST_CHECK(result == 0); + + result = cio_file_native_close(chunk_file); + TEST_CHECK(result == CIO_OK); +} + +static void test_legacy_core(int trigger_checksum_error) +{ + struct cio_options cio_opts; + char *in_data; + size_t in_size; + struct cio_stream *stream; + struct cio_chunk *chunk; + size_t delta; + struct cio_ctx *ctx; + int ret; + + /* delete any previous temporary content directory */ + cio_utils_recursive_delete(CIO_ENV); + /* + * Load sample data file and with the same content through multiple write + * operations generating other files. + */ + ret = cio_utils_read_file(CIO_FILE_400KB, &in_data, &in_size); + TEST_CHECK(ret == 0); + if (ret == -1) { + exit(EXIT_FAILURE); + } + + /* create Chunk I/O context */ + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_DEBUG; + cio_opts.flags = CIO_CHECKSUM; + + /* Create a temporal storage */ + ctx = cio_create(&cio_opts); + + stream = cio_stream_create(ctx, "test-legacy", CIO_STORE_FS); + + /* do not force a maximum of chunks up, we want to test writing overhead */ + cio_set_max_chunks_up(ctx, 1); + + chunk = cio_chunk_open(ctx, + stream, + "test_chunk", + CIO_OPEN, + 1000, + &ret); + + ret = cio_chunk_write(chunk, in_data, 128); + TEST_CHECK(ret == 0); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + delta = CIO_FILE_HEADER_MIN; + + if (trigger_checksum_error) { + delta++; + } + + truncate_file((struct cio_file *) chunk->backend, + 128 + delta, + CIO_TRUE); + + ret = cio_chunk_up(chunk); + + if (trigger_checksum_error) { + TEST_CHECK(ret != CIO_OK); + } + else { + TEST_CHECK(ret == CIO_OK); + } + + cio_destroy(ctx); + + free(in_data); +} + +void test_legacy_success() +{ + test_legacy_core(CIO_FALSE); +} + +void test_legacy_failure() +{ + test_legacy_core(CIO_TRUE); +} + TEST_LIST = { {"fs_write", test_fs_write}, {"fs_checksum", test_fs_checksum}, @@ -859,5 +974,7 @@ TEST_LIST = { {"issue_write_at", test_issue_write_at}, {"fs_up_down_up_append", test_fs_up_down_up_append}, {"fs_deep_hierachy", test_deep_hierarchy}, + {"legacy_success", test_legacy_success}, + {"legacy_failure", test_legacy_failure}, { 0 } }; diff --git a/lib/chunkio/tests/memfs.c b/lib/chunkio/tests/memfs.c index e10d0205710..0c79f0097e4 100644 --- a/lib/chunkio/tests/memfs.c +++ b/lib/chunkio/tests/memfs.c @@ -93,11 +93,10 @@ static void test_memfs_write() flags = CIO_CHECKSUM; - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); + cio_opts.flags = flags; - cio_opts.root_path = NULL; cio_opts.log_cb = log_cb; - cio_opts.log_level = CIO_LOG_INFO; cio_opts.flags = flags; /* Create main context */ diff --git a/lib/chunkio/tests/stream.c b/lib/chunkio/tests/stream.c index 1b47a8a2d9a..0a86c1437b2 100644 --- a/lib/chunkio/tests/stream.c +++ b/lib/chunkio/tests/stream.c @@ -54,7 +54,7 @@ static void test_stream_delete() cio_utils_recursive_delete("tmp"); - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = "tmp"; cio_opts.log_cb = log_cb; diff --git a/lib/chunkio/tools/cio.c b/lib/chunkio/tools/cio.c index 9ac27129766..a278276534e 100644 --- a/lib/chunkio/tools/cio.c +++ b/lib/chunkio/tools/cio.c @@ -591,7 +591,7 @@ int main(int argc, char **argv) verbose = 0; } - memset(&cio_opts, 0, sizeof(cio_opts)); + cio_options_init(&cio_opts); cio_opts.root_path = root_path; cio_opts.flags = flags; diff --git a/src/flb_config.c b/src/flb_config.c index 5f5289d0186..34a4e727b0e 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -147,6 +147,9 @@ struct flb_service_config service_configs[] = { {FLB_CONF_STORAGE_DELETE_IRRECOVERABLE_CHUNKS, FLB_CONF_TYPE_BOOL, offsetof(struct flb_config, storage_del_bad_chunks)}, + {FLB_CONF_STORAGE_TRIM_FILES, + FLB_CONF_TYPE_BOOL, + offsetof(struct flb_config, storage_trim_files)}, /* Coroutines */ {FLB_CONF_STR_CORO_STACK_SIZE, diff --git a/src/flb_fstore.c b/src/flb_fstore.c index 40980986d75..03bcc99dba6 100644 --- a/src/flb_fstore.c +++ b/src/flb_fstore.c @@ -461,6 +461,8 @@ struct flb_fstore *flb_fstore_create(char *path, int store_type) flags = CIO_OPEN; /* Create Chunk I/O context */ + cio_options_init(&opts); + opts.root_path = path; opts.log_cb = log_cb; opts.flags = flags; diff --git a/src/flb_storage.c b/src/flb_storage.c index 3535fae3a4c..a89a4c4b558 100644 --- a/src/flb_storage.c +++ b/src/flb_storage.c @@ -622,7 +622,14 @@ int flb_storage_create(struct flb_config *ctx) flags |= CIO_CHECKSUM; } + /* file trimming */ + if (ctx->storage_trim_files == FLB_TRUE) { + flags |= CIO_TRIM_FILES; + } + /* chunkio options */ + cio_options_init(&opts); + opts.root_path = ctx->storage_path; opts.flags = flags; opts.log_cb = log_cb; diff --git a/tests/internal/input_chunk.c b/tests/internal/input_chunk.c index f9ded1ed189..757e3b4cc0c 100644 --- a/tests/internal/input_chunk.c +++ b/tests/internal/input_chunk.c @@ -424,6 +424,8 @@ void flb_test_input_chunk_fs_chunks_size_real() i_ins = flb_input_new(cfg, "dummy", NULL, FLB_TRUE); i_ins->storage_type = CIO_STORE_FS; + cio_options_init(&opts); + opts.root_path = "/tmp/input-chunk-fs_chunks-size_real"; opts.log_cb = log_cb; opts.log_level = CIO_LOG_DEBUG;