From 049a0e0024d2d812bc56b156ebebb9d5bedb3532 Mon Sep 17 00:00:00 2001 From: Ji Bin Date: Sun, 14 Jan 2024 00:52:10 +0800 Subject: [PATCH] v2.3.4 Signed-off-by: Ji Bin --- .github/workflows/main.yml | 2 +- milvus_binary/env.sh | 2 +- ...001-link-with-CoreServices-for-macos.patch | 29 +++++++++++ .../milvus-v2.3.4/0002-fix-for-gettid.patch | 25 +++++++++ src/milvus/__init__.py | 2 +- src/milvus/data/config.yaml.template | 52 ++++++++++++++++--- 6 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 milvus_binary/patches/milvus-v2.3.4/0001-link-with-CoreServices-for-macos.patch create mode 100644 milvus_binary/patches/milvus-v2.3.4/0002-fix-for-gettid.patch diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9db412c..e07b697 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,7 +79,7 @@ jobs: CCACHE_COMPRESS: 1 CCACHE_COMPRESSLEVEL: 5 CCACHE_MAXSIZE: 2G - timeout-minutes: 360 + timeout-minutes: 900 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/milvus_binary/env.sh b/milvus_binary/env.sh index 8df7035..3639401 100644 --- a/milvus_binary/env.sh +++ b/milvus_binary/env.sh @@ -1,5 +1,5 @@ MILVUS_REPO="https://github.com/milvus-io/milvus.git" -MILVUS_VERSION="v2.3.3" +MILVUS_VERSION="v2.3.4" BUILD_PROXY= BUILD_FORCE=NO diff --git a/milvus_binary/patches/milvus-v2.3.4/0001-link-with-CoreServices-for-macos.patch b/milvus_binary/patches/milvus-v2.3.4/0001-link-with-CoreServices-for-macos.patch new file mode 100644 index 0000000..207a8fe --- /dev/null +++ b/milvus_binary/patches/milvus-v2.3.4/0001-link-with-CoreServices-for-macos.patch @@ -0,0 +1,29 @@ +From 3ef7229943e2a879d845467d31b616aabe1bcf53 Mon Sep 17 00:00:00 2001 +From: Ji Bin +Date: Mon, 13 Nov 2023 23:20:16 +0800 +Subject: [PATCH 1/2] link with CoreServices for macos + +Signed-off-by: Ji Bin +--- + internal/core/src/storage/azure-blob-storage/CMakeLists.txt | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/internal/core/src/storage/azure-blob-storage/CMakeLists.txt b/internal/core/src/storage/azure-blob-storage/CMakeLists.txt +index 7f27d5838..bde7f4457 100644 +--- a/internal/core/src/storage/azure-blob-storage/CMakeLists.txt ++++ b/internal/core/src/storage/azure-blob-storage/CMakeLists.txt +@@ -25,6 +25,11 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-return-type - + add_library(blob-chunk-manager SHARED AzureBlobChunkManager.cpp) + target_link_libraries(blob-chunk-manager PUBLIC Azure::azure-identity Azure::azure-storage-blobs) + ++if (APPLE) ++ find_library(CORESERVICES_LIBRARY CoreServices) ++ target_link_libraries(blob-chunk-manager PRIVATE ${CORESERVICES_LIBRARY}) ++endif (APPLE) ++ + install(TARGETS blob-chunk-manager DESTINATION "${CMAKE_INSTALL_LIBDIR}") + + if ( BUILD_UNIT_TEST STREQUAL "ON" ) +-- +2.43.0 + diff --git a/milvus_binary/patches/milvus-v2.3.4/0002-fix-for-gettid.patch b/milvus_binary/patches/milvus-v2.3.4/0002-fix-for-gettid.patch new file mode 100644 index 0000000..790337d --- /dev/null +++ b/milvus_binary/patches/milvus-v2.3.4/0002-fix-for-gettid.patch @@ -0,0 +1,25 @@ +From e5556b236823272485da41e3fb092957e4d7387a Mon Sep 17 00:00:00 2001 +From: Ji Bin +Date: Sun, 5 Nov 2023 18:49:29 +0800 +Subject: [PATCH 2/2] fix for gettid + +Signed-off-by: Ji Bin +--- + internal/core/thirdparty/knowhere/CMakeLists.txt | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt +index 80781717c..c46b50783 100644 +--- a/internal/core/thirdparty/knowhere/CMakeLists.txt ++++ b/internal/core/thirdparty/knowhere/CMakeLists.txt +@@ -41,6 +41,7 @@ FetchContent_Declare( + GIT_TAG ${KNOWHERE_VERSION} + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/knowhere-src + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/knowhere-build ++ PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/../../../patches/knowhere-v2.2.2.patch + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) + + FetchContent_GetProperties( knowhere ) +-- +2.43.0 + diff --git a/src/milvus/__init__.py b/src/milvus/__init__.py index 4174847..54a9150 100644 --- a/src/milvus/__init__.py +++ b/src/milvus/__init__.py @@ -21,7 +21,7 @@ import json import hashlib -__version__ = '2.3.3' +__version__ = '2.3.4' LOGGERS = {} diff --git a/src/milvus/data/config.yaml.template b/src/milvus/data/config.yaml.template index ea0ad52..3e79436 100644 --- a/src/milvus/data/config.yaml.template +++ b/src/milvus/data/config.yaml.template @@ -201,9 +201,31 @@ proxy: maxTaskNum: 1024 # max task number of proxy task queue accessLog: enable: false - filename: milvus_access_log.log # Log filename, leave empty to disable file log. - localPath: {{ system_log_path }} - # maxSize: 64 # max log file size of singal log file to trigger rotate. + # Log filename, set as "" to use stdout. + # filename: "" + # define formatters for access log by XXX:{format: XXX, method:[XXX,XXX]} + formatters: + # "base" formatter could not set methods + # all method will use "base" formatter default + base: + # will not print access log if set as "" + format: "[$time_now] [ACCESS] <$user_name: $user_addr> $method_name [status: $method_status] [code: $error_code] [sdk: $sdk_version] [msg: $error_msg] [traceID: $trace_id] [timeCost: $time_cost]" + query: + format: "[$time_now] [ACCESS] <$user_name: $user_addr> $method_name [status: $method_status] [code: $error_code] [sdk: $sdk_version] [msg: $error_msg] [traceID: $trace_id] [timeCost: $time_cost] [database: $database_name] [collection: $collection_name] [partitions: $partition_name] [expr: $method_expr]" + # set formatter owners by method name(method was all milvus external interface) + # all method will use base formatter default + # one method only could use one formatter + # if set a method formatter mutiple times, will use random fomatter. + methods: ["Query", "Search", "Delete"] + # localPath: /tmp/milvus_accesslog // log file rootpath + # maxSize: 64 # max log file size(MB) of singal log file, mean close when time <= 0. + # rotatedTime: 0 # max time range of singal log file, mean close when time <= 0; + # maxBackups: 8 # num of reserved backups. will rotate and crate a new backup when access log file trigger maxSize or rotatedTime. + # cacheSize: 10240 # write cache of accesslog in Byte + + # minioEnable: false # update backups to milvus minio when minioEnable is true. + # remotePath: "access_log/" # file path when update backups to minio + # remoteMaxTime: 0 # max time range(in Hour) of backups in minio, 0 means close time retention. http: enabled: true # Whether to enable the http server debug_mode: false # Whether to enable http server debug mode @@ -264,11 +286,12 @@ queryNode: # Use more threads to make better use of SSD throughput in disk index. # This parameter is only useful when enable-disk = true. # And this value should be a number greater than 1 and less than 32. - chunkRows: 1024 # The number of vectors in a chunk. - growing: # growing a vector index for growing segment to accelerate search + chunkRows: 128 # The number of vectors in a chunk. + interimIndex: # build a vector temperate index for growing segment or binlog to accelerate search enableIndex: true - nlist: 128 # growing segment index nlist - nprobe: 16 # nprobe to search growing segment, based on your accuracy requirement, must smaller than nlist + nlist: 128 # segment index nlist + nprobe: 16 # nprobe to search segment, based on your accuracy requirement, must smaller than nlist + memExpansionRate: 1.15 # the ratio of building interim index memory usage to raw data loadMemoryUsageFactor: 1 # The multiply factor of calculating the memory usage while loading segments enableDisk: false # enable querynode load disk index, and search on disk index maxDiskUsagePercentage: 95 @@ -402,6 +425,12 @@ dataNode: maxQueueLength: 16 # Maximum length of task queue in flowgraph maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph maxParallelSyncTaskNum: 6 # Maximum number of sync tasks executed in parallel in each flush manager + skipMode: + # when there are only timetick msg in flowgraph for a while (longer than coldTime), + # flowGraph will turn on skip mode to skip most timeticks to reduce cost, especially there are a lot of channels + enable: true + skipNum: 4 + coldTime: 60 segment: insertBufSize: 16777216 # Max buffer size to flush for a single segment. deleteBufBytes: 67108864 # Max buffer size to flush del for a single channel @@ -427,6 +456,10 @@ dataNode: # if this parameter <= 0, will set it as the maximum number of CPUs that can be executing # suggest to set it bigger on large collection numbers to avoid blocking workPoolSize: -1 + # specify the size of global work pool for channel checkpoint updating + # if this parameter <= 0, will set it as 1000 + # suggest to set it bigger on large collection numbers to avoid blocking + updateChannelCheckpointMaxParallel: 1000 # Configures the system log output. log: @@ -662,12 +695,15 @@ quotaAndLimits: trace: # trace exporter type, default is stdout, - # optional values: ['stdout', 'jaeger'] + # optional values: ['stdout', 'jaeger', 'otlp'] exporter: stdout # fraction of traceID based sampler, # optional values: [0, 1] # Fractions >= 1 will always sample. Fractions < 0 are treated as zero. sampleFraction: 0 + otlp: + endpoint: # "127.0.0.1:4318" + secure: true jaeger: url: # "http://127.0.0.1:14268/api/traces" # when exporter is jaeger should set the jaeger's URL