lib: chunkio: upgrade to v1.5.0

include/fluent-bit/flb_config.h

@@ -225,6 +225,7 @@ struct flb_config {
     int   storage_del_bad_chunks;   /* delete irrecoverable chunks */
     char *storage_bl_mem_limit;     /* storage backlog memory limit */
     struct flb_storage_metrics *storage_metrics_ctx; /* storage metrics context */
+    int   storage_trim_files;       /* enable/disable file trimming */
 
     /* Embedded SQL Database support (SQLite3) */
 #ifdef FLB_HAVE_SQLDB
@@ -355,6 +356,7 @@ enum conf_type {
 #define FLB_CONF_STORAGE_MAX_CHUNKS_UP "storage.max_chunks_up"
 #define FLB_CONF_STORAGE_DELETE_IRRECOVERABLE_CHUNKS \
                                        "storage.delete_irrecoverable_chunks"
+#define FLB_CONF_STORAGE_TRIM_FILES    "storage.trim_files"
 
 /* Coroutines */
 #define FLB_CONF_STR_CORO_STACK_SIZE "Coro_Stack_Size"

lib/chunkio/CMakeLists.txt

@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.0)
 project(chunk-io C)
 
 set(CIO_VERSION_MAJOR  1)
-set(CIO_VERSION_MINOR  4)
-set(CIO_VERSION_PATCH  0)
+set(CIO_VERSION_MINOR  5)
+set(CIO_VERSION_PATCH  1)
 set(CIO_VERSION_STR "${CIO_VERSION_MAJOR}.${CIO_VERSION_MINOR}.${CIO_VERSION_PATCH}")
 
 # CFLAGS
@@ -15,11 +15,7 @@ else()
 endif()
 
 # Set __FILENAME__
-if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__FILENAME__='\"$(subst ${CMAKE_SOURCE_DIR}/,,$(abspath $<))\"'")
-else()
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__FILENAME__=__FILE__")
-endif()
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__FILENAME__=__FILE__")
 
 include(cmake/macros.cmake)
 
@@ -91,6 +87,18 @@ if(CIO_HAVE_FALLOCATE)
   CIO_DEFINITION(CIO_HAVE_FALLOCATE)
 endif()
 
+# posix_fallocate(2) support
+check_c_source_compiles("
+  #include <fcntl.h>
+  int main() {
+     posix_fallocate(0,0,0);
+     return 0;
+  }" CIO_HAVE_POSIX_FALLOCATE)
+
+if(CIO_HAVE_POSIX_FALLOCATE)
+  CIO_DEFINITION(CIO_HAVE_POSIX_FALLOCATE)
+endif()
+
 configure_file(
   "${PROJECT_SOURCE_DIR}/include/chunkio/cio_info.h.in"
   "${PROJECT_BINARY_DIR}/include/chunkio/cio_info.h"
@@ -105,6 +113,7 @@ include_directories(
   include
   deps/
   deps/monkey/include
+  ${PROJECT_BINARY_DIR}/include/
   )
 
 add_subdirectory(deps/crc32)

lib/chunkio/cmake/sanitizers-cmake/CMakeLists.txt

@@ -30,7 +30,7 @@
 #
 
 # minimum required cmake version
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 2.8.12)
 
 # project name
 project("CMake-sanitizers")

lib/chunkio/cmake/sanitizers-cmake/README.md

@@ -11,7 +11,7 @@ CMake module to enable sanitizers for binary targets.
 To use [FindSanitizers.cmake](cmake/FindSanitizers.cmake), simply add this repository as git submodule into your own repository
 ```Shell
 mkdir externals
-git submodule add git://github.com/arsenm/sanitizers-cmake.git externals/sanitizers-cmake
+git submodule add git@github.com:arsenm/sanitizers-cmake.git externals/sanitizers-cmake
 ```
 and adding ```externals/sanitizers-cmake/cmake``` to your ```CMAKE_MODULE_PATH```
 ```CMake

lib/chunkio/cmake/sanitizers-cmake/cmake/FindASan.cmake

@@ -25,6 +25,9 @@
 option(SANITIZE_ADDRESS "Enable AddressSanitizer for sanitized targets." Off)
 
 set(FLAG_CANDIDATES
+    # MSVC uses
+    "/fsanitize=address"
+
     # Clang 3.2+ use this version. The no-omit-frame-pointer option is optional.
     "-g -fsanitize=address -fno-omit-frame-pointer"
     "-g -fsanitize=address"

lib/chunkio/cmake/sanitizers-cmake/cmake/FindMSan.cmake

@@ -25,6 +25,9 @@
 option(SANITIZE_MEMORY "Enable MemorySanitizer for sanitized targets." Off)
 
 set(FLAG_CANDIDATES
+    # MSVC uses
+    "/fsanitize=memory"
+    # GNU/Clang
     "-g -fsanitize=memory"
 )
 

lib/chunkio/cmake/sanitizers-cmake/cmake/FindSanitizers.cmake

@@ -26,8 +26,8 @@
 # link against the sanitizers.
 option(SANITIZE_LINK_STATIC "Try to link static against sanitizers." Off)
 
-
-
+# Highlight this module has been loaded.
+set(Sanitizers_FOUND TRUE)
 
 set(FIND_QUIETLY_FLAG "")
 if (DEFINED Sanitizers_FIND_QUIETLY)
@@ -39,9 +39,6 @@ find_package(TSan ${FIND_QUIETLY_FLAG})
 find_package(MSan ${FIND_QUIETLY_FLAG})
 find_package(UBSan ${FIND_QUIETLY_FLAG})
 
-
-
-
 function(sanitizer_add_blacklist_file FILE)
     if(NOT IS_ABSOLUTE ${FILE})
         set(FILE "${CMAKE_CURRENT_SOURCE_DIR}/${FILE}")
@@ -52,7 +49,7 @@ function(sanitizer_add_blacklist_file FILE)
         "SanitizerBlacklist" "SanBlist")
 endfunction()
 
-function(add_sanitizers ...)
+function(add_sanitizers)
     # If no sanitizer is enabled, return immediately.
     if (NOT (SANITIZE_ADDRESS OR SANITIZE_MEMORY OR SANITIZE_THREAD OR
         SANITIZE_UNDEFINED))
@@ -77,18 +74,18 @@ function(add_sanitizers ...)
                     "Target will be compiled without sanitizers.")
             return()
 
-        # If the target is compiled by no known compiler, ignore it.
         elseif (NUM_COMPILERS EQUAL 0)
-            message(WARNING "Can't use any sanitizers for target ${TARGET}, "
-                    "because it uses an unknown compiler. Target will be "
-                    "compiled without sanitizers.")
-            return()
+            # If the target is compiled by no known compiler, give a warning.
+            message(WARNING "Sanitizers for target ${TARGET} may not be"
+                    " usable, because it uses no or an unknown compiler. "
+                    "This is a false warning for targets using only "
+                    "object lib(s) as input.")
         endif ()
 
         # Add sanitizers for target.
         add_sanitize_address(${TARGET})
         add_sanitize_thread(${TARGET})
         add_sanitize_memory(${TARGET})
         add_sanitize_undefined(${TARGET})
-	endforeach ()
+    endforeach ()
 endfunction(add_sanitizers)

lib/chunkio/cmake/sanitizers-cmake/cmake/FindTSan.cmake

@@ -25,6 +25,9 @@
 option(SANITIZE_THREAD "Enable ThreadSanitizer for sanitized targets." Off)
 
 set(FLAG_CANDIDATES
+    # MSVC uses
+    "/fsanitize=thread"
+    # GNU/Clang
     "-g -fsanitize=thread"
 )
 

lib/chunkio/cmake/sanitizers-cmake/cmake/FindUBSan.cmake

@@ -26,6 +26,9 @@ option(SANITIZE_UNDEFINED
     "Enable UndefinedBehaviorSanitizer for sanitized targets." Off)
 
 set(FLAG_CANDIDATES
+    # MSVC uses
+    "/fsanitize=undefined"
+    # GNU/Clang
     "-g -fsanitize=undefined"
 )
 

lib/chunkio/cmake/sanitizers-cmake/cmake/sanitize-helpers.cmake

@@ -25,6 +25,11 @@
 # Helper function to get the language of a source file.
 function (sanitizer_lang_of_source FILE RETURN_VAR)
     get_filename_component(LONGEST_EXT "${FILE}" EXT)
+    # If extension is empty return. This can happen for extensionless headers
+    if("${LONGEST_EXT}" STREQUAL "")
+       set(${RETURN_VAR} "" PARENT_SCOPE)
+       return()
+    endif()
     # Get shortest extension as some files can have dot in their names
     string(REGEX REPLACE "^.*(\\.[^.]+)$" "\\1" FILE_EXT ${LONGEST_EXT})
     string(TOLOWER "${FILE_EXT}" FILE_EXT)
@@ -70,6 +75,7 @@ endfunction ()
 
 # Helper function to check compiler flags for language compiler.
 function (sanitizer_check_compiler_flag FLAG LANG VARIABLE)
+
     if (${LANG} STREQUAL "C")
         include(CheckCCompilerFlag)
         check_c_compiler_flag("${FLAG}" ${VARIABLE})
@@ -92,6 +98,7 @@ function (sanitizer_check_compiler_flag FLAG LANG VARIABLE)
                 " - Failed (Check not supported)")
         endif ()
     endif()
+
 endfunction ()
 
 
@@ -105,7 +112,7 @@ function (sanitizer_check_compiler_flags FLAG_CANDIDATES NAME PREFIX)
         # So instead of searching flags foreach language, search flags foreach
         # compiler used.
         set(COMPILER ${CMAKE_${LANG}_COMPILER_ID})
-        if (NOT DEFINED ${PREFIX}_${COMPILER}_FLAGS)
+        if (COMPILER AND NOT DEFINED ${PREFIX}_${COMPILER}_FLAGS)
             foreach (FLAG ${FLAG_CANDIDATES})
                 if(NOT CMAKE_REQUIRED_QUIET)
                     message(STATUS "Try ${COMPILER} ${NAME} flag = [${FLAG}]")
@@ -162,11 +169,10 @@ function (sanitizer_add_flags TARGET NAME PREFIX)
         return()
     endif()
 
-    # Set compile- and link-flags for target.
-    set_property(TARGET ${TARGET} APPEND_STRING
-        PROPERTY COMPILE_FLAGS " ${${PREFIX}_${TARGET_COMPILER}_FLAGS}")
-    set_property(TARGET ${TARGET} APPEND_STRING
-        PROPERTY COMPILE_FLAGS " ${SanBlist_${TARGET_COMPILER}_FLAGS}")
-    set_property(TARGET ${TARGET} APPEND_STRING
-        PROPERTY LINK_FLAGS " ${${PREFIX}_${TARGET_COMPILER}_FLAGS}")
+    separate_arguments(flags_list UNIX_COMMAND "${${PREFIX}_${TARGET_COMPILER}_FLAGS} ${SanBlist_${TARGET_COMPILER}_FLAGS}")
+    target_compile_options(${TARGET} PUBLIC ${flags_list})
+
+    separate_arguments(flags_list UNIX_COMMAND "${${PREFIX}_${TARGET_COMPILER}_FLAGS}")
+    target_link_options(${TARGET} PUBLIC ${flags_list})
+
 endfunction ()

lib/chunkio/include/chunkio/chunkio.h

@@ -44,20 +44,33 @@
 #define CIO_CHECKSUM             4         /* enable checksum verification (crc32) */
 #define CIO_FULL_SYNC            8         /* force sync to fs through MAP_SYNC */
 #define CIO_DELETE_IRRECOVERABLE 16        /* delete irrecoverable chunks from disk */
+#define CIO_TRIM_FILES           32        /* trim files to their required size */
 
 /* Return status */
 #define CIO_CORRUPTED      -3         /* Indicate that a chunk is corrupted */
 #define CIO_RETRY          -2         /* The operations needs to be retried */
 #define CIO_ERROR          -1         /* Generic error */
 #define CIO_OK              0         /* OK */
 
+/* Configuration limits */
+/* The file minimum growth factor is 8 memory pages and
+ * the file maximum growth factor is 8 megabytes
+ */
+#define CIO_REALLOC_HINT_MIN            (cio_getpagesize() * 8)
+#define CIO_REALLOC_HINT_MAX            (8 * 1000 * 1000)
 
 /* defaults */
-#define CIO_MAX_CHUNKS_UP  64   /* default limit for cio_ctx->max_chunks_up */
+#define CIO_MAX_CHUNKS_UP               64  /* default limit for cio_ctx->max_chunks_up */
+#define CIO_DISABLE_REALLOC_HINT        -1  /* default value of size of realloc hint */
+#define CIO_DEFAULT_REALLOC_HINT        CIO_REALLOC_HINT_MIN
+#define CIO_INITIALIZED                 1337
 
 struct cio_ctx;
 
 struct cio_options {
+    /* this bool flag sets if the options has been initialized, that's a mandatory step */
+    int initialized;
+
     int flags;
     char *root_path;
 
@@ -68,10 +81,14 @@ struct cio_options {
     char *user;
     char *group;
     char *chmod;
+
+    /* chunk handlings */
+    int realloc_size_hint;
 };
 
 struct cio_ctx {
     int page_size;
+    int realloc_size_hint;
     struct cio_options options;
 
     void *processed_user;
@@ -104,7 +121,7 @@ struct cio_ctx {
 #include <chunkio/cio_stream.h>
 #include <chunkio/cio_chunk.h>
 
-
+void cio_options_init(struct cio_options *options);
 struct cio_ctx *cio_create(struct cio_options *options);
 void cio_destroy(struct cio_ctx *ctx);
 int cio_load(struct cio_ctx *ctx, char *chunk_extension);
@@ -113,6 +130,10 @@ int cio_qsort(struct cio_ctx *ctx, int (*compar)(const void *, const void *));
 void cio_set_log_callback(struct cio_ctx *ctx, void (*log_cb));
 int cio_set_log_level(struct cio_ctx *ctx, int level);
 int cio_set_max_chunks_up(struct cio_ctx *ctx, int n);
+int cio_set_realloc_size_hint(struct cio_ctx *ctx, size_t realloc_size_hint);
+
+void cio_enable_file_trimming(struct cio_ctx *ctx);
+void cio_disable_file_trimming(struct cio_ctx *ctx);
 
 int cio_meta_write(struct cio_chunk *ch, char *buf, size_t size);
 int cio_meta_cmp(struct cio_chunk *ch, char *meta_buf, int meta_len);

lib/chunkio/include/chunkio/cio_file.h

@@ -35,6 +35,7 @@ struct cio_file {
     int allocate_strategy;    /* linux-only: fallocate strategy */
     size_t fs_size;           /* original size in the file system */
     size_t data_size;         /* number of bytes used */
+    size_t page_size;         /* curent page size */
     size_t alloc_size;        /* allocated size       */
     size_t realloc_size;      /* chunk size to increase alloc */
     char *path;               /* root path + stream   */