v2.2.14 (#68)

Signed-off-by: Ji Bin <[email protected]>

milvus_binary/env.sh

@@ -1,5 +1,5 @@
 
 MILVUS_REPO="https://github.com/milvus-io/milvus.git"
-MILVUS_VERSION="v2.2.13"
+MILVUS_VERSION="v2.2.14"
 BUILD_PROXY=
 BUILD_FORCE=NO

milvus_binary/patches/milvus-v2.2.14/0002-workaround-for-sys-mman.h-for-mignw.patch

@@ -0,0 +1,31 @@
+From 060a5eefb4d8afaa3e1e2a48116e861715e17c3c Mon Sep 17 00:00:00 2001
+From: Ji Bin <[email protected]>
+Date: Thu, 8 Jun 2023 19:34:14 +0800
+Subject: [PATCH 2/4] workaround for sys/mman.h for mignw
+
+Signed-off-by: Ji Bin <[email protected]>
+---
+ internal/core/src/common/Column.h | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/internal/core/src/common/Column.h b/internal/core/src/common/Column.h
+index 4a8f39457..b1d59635c 100644
+--- a/internal/core/src/common/Column.h
++++ b/internal/core/src/common/Column.h
+@@ -10,8 +10,12 @@
+ // or implied. See the License for the specific language governing permissions and limitations under the License
+
+ #pragma once
+-
++#ifndef __MINGW64__
+ #include <sys/mman.h>
++#else
++// mman for win32, we have it in annoy
++#include <annoy/src/mman.h>
++#endif
+
+ #include <cstddef>
+ #include <ostream>
+-- 
+2.42.0
+

milvus_binary/patches/milvus-v2.2.14/0003-workaround-for-querynode-indexnode-folder.patch

@@ -0,0 +1,65 @@
+From 4f1beae860789b041489efeb5ead5045a098f40a Mon Sep 17 00:00:00 2001
+From: Ji Bin <[email protected]>
+Date: Thu, 8 Jun 2023 19:01:18 +0800
+Subject: [PATCH 3/4] workaround for querynode/indexnode folder
+
+Signed-off-by: Ji Bin <[email protected]>
+---
+ internal/core/src/storage/Util.cpp | 27 +++++++++++++++++++++++++++
+ 1 file changed, 27 insertions(+)
+
+diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp
+index 659307570..5d4335037 100644
+--- a/internal/core/src/storage/Util.cpp
++++ b/internal/core/src/storage/Util.cpp
+@@ -33,6 +33,29 @@
+ #include "storage/DiskFileManagerImpl.h"
+ #endif
+
++namespace  {
++std::string patch_index_files_path(const std::string& path) {
++    auto file = path;
++    auto index = file.find("querynode/index_files");
++    if (index != std::string::npos) {
++        file.replace(index, strlen("querynode/index_files"), "index_files");
++    }
++    index = file.find("querynode\\index_files");
++    if (index != std::string::npos) {
++        file.replace(index, strlen("querynode\\index_files"), "index_files");
++    }
++    index = file.find("indexnode/index_files");
++    if (index != std::string::npos) {
++        file.replace(index, strlen("indexnode/index_files"), "index_files");
++    }
++    index = file.find("indexnode\\index_files");
++    if (index != std::string::npos) {
++        file.replace(index, strlen("indexnode\\index_files"), "index_files");
++    }
++    return file;
++}
++}
++
+ namespace milvus::storage {
+
+ std::map<std::string, ChunkManagerType> ChunkManagerType_Map = {{"local", ChunkManagerType::Local},
+@@ -360,6 +383,8 @@ DownloadAndDecodeRemoteFile(RemoteChunkManager* remote_chunk_manager, std::strin
+     ChunkManager *chunk_manager = remote_chunk_manager;
+     if (! chunk_manager) {
+         chunk_manager = &LocalChunkManager::GetInstance();
++        // workaround: replace any /querynode/ /indexnode/ => /
++        file = patch_index_files_path(file);
+     }
+     auto fileSize = chunk_manager->Size(file);
+     auto buf = std::shared_ptr<uint8_t[]>(new uint8_t[fileSize]);
+@@ -386,6 +411,8 @@ EncodeAndUploadIndexSlice(RemoteChunkManager* remote_chunk_manager,
+     ChunkManager *chunk_manager = remote_chunk_manager;
+     if (! chunk_manager) {
+         chunk_manager = &LocalChunkManager::GetInstance();
++        // workaround: replace any /querynode/ /indexnode/ => /
++        object_key = patch_index_files_path(object_key);
+     }
+     chunk_manager->Write(object_key, serialized_index_data.data(), serialized_index_size);
+     return std::pair<std::string, size_t>(object_key, serialized_index_size);
+-- 
+2.42.0
+

milvus_binary/patches/milvus-v2.2.14/0004-patch-for-v2.2.10.patch

@@ -0,0 +1,49 @@
+From 5dee8df08eadaeafaac2ad48ee0089cac1e7d4b4 Mon Sep 17 00:00:00 2001
+From: Ji Bin <[email protected]>
+Date: Fri, 16 Jun 2023 11:06:10 +0800
+Subject: [PATCH 4/4] patch for v2.2.10
+
+Signed-off-by: Ji Bin <[email protected]>
+---
+ internal/core/src/storage/MinioChunkManager.cpp | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/internal/core/src/storage/MinioChunkManager.cpp b/internal/core/src/storage/MinioChunkManager.cpp
+index d60eb038c..f481c5a11 100644
+--- a/internal/core/src/storage/MinioChunkManager.cpp
++++ b/internal/core/src/storage/MinioChunkManager.cpp
+@@ -56,6 +56,7 @@ namespace milvus::storage {
+ std::atomic<size_t> MinioChunkManager::init_count_(0);
+ std::mutex MinioChunkManager::client_mutex_;
+
++#ifndef WIN32
+ static void
+ SwallowHandler(int signal) {
+ #pragma GCC diagnostic push
+@@ -71,6 +72,7 @@ SwallowHandler(int signal) {
+     }
+ #pragma GCC diagnostic pop
+ }
++#endif
+
+ /**
+  * @brief convert std::string to Aws::String
+@@ -105,6 +107,7 @@ MinioChunkManager::InitSDKAPI(RemoteStorageType type, bool useIAM, const std::st
+     std::scoped_lock lock{client_mutex_};
+     const size_t initCount = init_count_++;
+     if (initCount == 0) {
++#ifndef WIN32
+         // sdk_options_.httpOptions.installSigPipeHandler = true;
+         struct sigaction psa;
+         memset(&psa, 0, sizeof psa);
+@@ -114,6 +117,7 @@ MinioChunkManager::InitSDKAPI(RemoteStorageType type, bool useIAM, const std::st
+         sigemptyset(&psa.sa_mask);
+         sigaddset(&psa.sa_mask, SIGPIPE);
+         sigaction(SIGPIPE, &psa, 0);
++#endif
+ #ifdef BUILD_GCP
+         if (type == RemoteStorageType::GOOGLE_CLOUD && useIAM) {
+             sdk_options_.httpOptions.httpClientFactory_create_fn = []() {
+-- 
+2.42.0
+

src/milvus/__init__.py

@@ -20,7 +20,7 @@
 import urllib.request
 import hashlib
 
-__version__ = '2.2.13-3'
+__version__ = '2.2.14'
 
 LOGGERS = {}
 

src/milvus/data/config.yaml.template

@@ -173,7 +173,7 @@ proxy:
   # So adjust at your risk!
   maxDimension: 32768 # Maximum dimension of a vector
   # It's strongly DISCOURAGED to set `maxShardNum` > 64.
-  maxShardNum: 64 # Maximum number of shards in a collection
+  maxShardNum: 16 # Maximum number of shards in a collection
   maxTaskNum: 1024 # max task number of proxy task queue
   # please adjust in embedded Milvus: false
   ginLogging: false # Whether to produce gin logs.
@@ -216,7 +216,6 @@ queryNode:
   loadMemoryUsageFactor: 3 # The multiply factor of calculating the memory usage while loading segments
   enableDisk: true # enable querynode load disk index, and search on disk index
   maxDiskUsagePercentage: 95
-  gracefulStopTimeout: 30
 
   stats:
     publishInterval: 1000 # Interval for querynode to report node information (milliseconds)
@@ -258,13 +257,13 @@ queryNode:
     # the valid value is [3600, infinite)
     maxTimestampLag: 86400
     # read task schedule policy: fifo(by default), user-task-polling.
-    scheduleReadPolicy: 
+    scheduleReadPolicy:
       # fifo: A FIFO queue support the schedule.
-      # user-task-polling: 
-      #     The user's tasks will be polled one by one and scheduled. 
+      # user-task-polling:
+      #     The user's tasks will be polled one by one and scheduled.
       #     Scheduling is fair on task granularity.
       #     The policy is based on the username for authentication.
-      #     And an empty username is considered the same user. 
+      #     And an empty username is considered the same user.
       #     When there are no multi-users, the policy decay into FIFO
       name: fifo
       # user-task-polling configure:
@@ -298,7 +297,6 @@ indexNode:
   port: {{ index_node_port(int) }}
   enableDisk: true # enable index node build disk vector index
   maxDiskUsagePercentage: 95
-  gracefulStopTimeout: 30
 
   scheduler:
     buildParallel: 1
@@ -351,8 +349,8 @@ dataCoord:
 
   gc:
     interval: 3600 # gc interval in seconds
-    missingTolerance: 86400 # file meta missing tolerance duration in seconds, 60*24
-    dropTolerance: 3600 # file belongs to dropped entity tolerance duration in seconds
+    missingTolerance: 3600 # file meta missing tolerance duration in seconds, 60*24
+    dropTolerance: 10800 # file belongs to dropped entity tolerance duration in seconds
 
 
 dataNode:
@@ -407,7 +405,7 @@ grpc:
     keepAliveTimeout: 20000
     maxMaxAttempts: 5
     initialBackOff: 1.0
-    maxBackoff: 60.0
+    maxBackoff: 10.0
     backoffMultiplier: 2.0
   server:
     retryTimes: 5 # retry times when receiving a grpc return value with a failure and retryable state code
@@ -451,7 +449,7 @@ common:
   entityExpiration: -1     # Entity expiration in seconds, CAUTION make sure entityExpiration >= retentionDuration and -1 means never expire
 
   gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency.
-  gracefulStopTimeout: 30 # seconds. it will force quit the server if the graceful stop process is not completed during this time.
+  gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time.
 
   # Default value: auto
   # Valid values: [auto, avx512, avx2, avx, sse4_2]
@@ -576,9 +574,9 @@ quotaAndLimits:
       queryNodeMemoryLowWaterLevel: 0.85 # (0, 1], memoryLowWaterLevel in QueryNodes
       queryNodeMemoryHighWaterLevel: 0.95 # (0, 1], memoryHighWaterLevel in QueryNodes
     growingSegmentsSizeProtection:
-      # 1. No action will be taken if the ratio of growing segments size is less than the low water level.
-      # 2. The DML rate will be reduced if the ratio of growing segments size is greater than the low water level and less than the high water level.
-      # 3. All DML requests will be rejected if the ratio of growing segments size is greater than the high water level.
+      # No action will be taken if the growing segments size is less than the low watermark.
+      # When the growing segments size exceeds the low watermark, the dml rate will be reduced,
+      # but the rate will not be lower than `minRateRatio * dmlRate`.
       enabled: false
       minRateRatio: 0.5
       lowWaterLevel: 0.2