Merge remote-tracking branch 'upstream/master' into vdb-bench-pgo

shaoting-huang · May 27, 2024 · 1122974 · 1122974
2 parents bd5419a + 066c8ea
commit 1122974
Show file tree

Hide file tree

Showing 169 changed files with 6,559 additions and 5,901 deletions.
diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml
@@ -56,7 +56,7 @@ jobs:
       - name: Setup Go environment
         uses: actions/[email protected]
         with:
-          go-version: '~1.20.7'
+          go-version: '~1.21.10'
       - name: Mac Cache Go Mod Volumes
         uses: actions/cache@v3
         with:

diff --git a/Makefile b/Makefile
@@ -480,6 +480,7 @@ generate-mockery-datanode: getdeps
 	$(INSTALL_PATH)/mockery --name=BinlogIO --dir=$(PWD)/internal/datanode/io --output=$(PWD)/internal/datanode/io --filename=mock_binlogio.go --with-expecter --structname=MockBinlogIO --outpkg=io --inpackage
 	$(INSTALL_PATH)/mockery --name=FlowgraphManager --dir=$(PWD)/internal/datanode --output=$(PWD)/internal/datanode --filename=mock_fgmanager.go --with-expecter --structname=MockFlowgraphManager --outpkg=datanode --inpackage
 	$(INSTALL_PATH)/mockery --name=ChannelManager --dir=$(PWD)/internal/datanode --output=$(PWD)/internal/datanode --filename=mock_channelmanager.go --with-expecter --structname=MockChannelManager --outpkg=datanode --inpackage
+	$(INSTALL_PATH)/mockery --name=Compactor --dir=$(PWD)/internal/datanode/compaction --output=$(PWD)/internal/datanode/compaction --filename=mock_compactor.go --with-expecter --structname=MockCompactor --outpkg=compaction --inpackage
 
 generate-mockery-metastore: getdeps
 	$(INSTALL_PATH)/mockery --name=RootCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_rootcoord_catalog.go --with-expecter --structname=RootCoordCatalog --outpkg=mocks
@@ -522,4 +523,4 @@ mmap-migration:
 	@source $(PWD)/scripts/setenv.sh && \
     		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
     		GO111MODULE=on $(GO) build -pgo=$(PGO_PATH)/default.pgo -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
-    		-tags dynamic -o $(INSTALL_PATH)/mmap-migration $(MMAP_MIGRATION_PATH)/main.go 1>/dev/null
+    		-tags dynamic -o $(INSTALL_PATH)/mmap-migration $(MMAP_MIGRATION_PATH)/main.go 1>/dev/null
diff --git a/client/.golangci.yml b/client/.golangci.yml
@@ -1,5 +1,5 @@
 run:
-  go: "1.20"
+  go: "1.21"
   skip-dirs:
     - build
     - configs

diff --git a/client/go.mod b/client/go.mod
@@ -1,6 +1,6 @@
 module github.com/milvus-io/milvus/client/v2
 
-go 1.20
+go 1.21
 
 require (
 	github.com/blang/semver/v4 v4.0.0

diff --git a/configs/milvus.yaml b/configs/milvus.yaml
@@ -335,7 +335,7 @@ queryNode:
     # chunk cache during the load process. This approach has the potential to substantially reduce query/search latency
     # for a specific duration post-load, albeit accompanied by a concurrent increase in disk usage;
     # 2. If set to "off," original vector data will only be loaded into the chunk cache during search/query.
-    warmup: async
+    warmup: off
   mmap:
     mmapEnabled: false # Enable mmap for loading data
   lazyload:
@@ -494,7 +494,7 @@ dataNode:
       coldTime: 60 # Turn on skip mode after there are only timetick msg for x seconds
   segment:
     insertBufSize: 16777216 # Max buffer size to flush for a single segment.
-    deleteBufBytes: 67108864 # Max buffer size in bytes to flush del for a single channel, default as 16MB
+    deleteBufBytes: 16777216 # Max buffer size in bytes to flush del for a single channel, default as 16MB
     syncPeriod: 600 # The period to sync segments if buffer is not empty.
   memory:
     forceSyncEnable: true # Set true to force sync if memory usage is too high
@@ -612,7 +612,7 @@ common:
   ttMsgEnabled: true # Whether the instance disable sending ts messages
   traceLogMode: 0 # trace request info
   bloomFilterSize: 100000 # bloom filter initial size
-  maxBloomFalsePositive: 0.05 # max false positive rate for bloom filter
+  maxBloomFalsePositive: 0.001 # max false positive rate for bloom filter
 
 # QuotaConfig, configurations of Milvus quota and limits.
 # By default, we enable:
@@ -631,6 +631,14 @@ quotaAndLimits:
   # collects metrics from Proxies, Query cluster and Data cluster.
   # seconds, (0 ~ 65536)
   quotaCenterCollectInterval: 3
+  limits:
+    allocRetryTimes: 15 # retry times when delete alloc forward data from rate limit failed
+    allocWaitInterval: 1000 # retry wait duration when delete alloc forward data rate failed, in millisecond
+    complexDeleteLimitEnable: false # whether complex delete check forward data by limiter
+    maxCollectionNum: 65536
+    maxCollectionNumPerDB: 65536
+    maxInsertSize: -1 # maximum size of a single insert request, in bytes, -1 means no limit
+    maxResourceGroupNumOfQueryNode: 1024 # maximum number of resource groups of query nodes
   ddl:
     enabled: false
     collectionRate: -1 # qps, default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection
@@ -711,11 +719,6 @@ quotaAndLimits:
         max: -1 # qps, default no limit
       partition:
         max: -1 # qps, default no limit
-  limits:
-    maxCollectionNum: 65536
-    maxCollectionNumPerDB: 65536
-    maxInsertSize: -1 # maximum size of a single insert request, in bytes, -1 means no limit
-    maxResourceGroupNumOfQueryNode: 1024 # maximum number of resource groups of query nodes
   limitWriting:
     # forceDeny false means dml requests are allowed (except for some
     # specific conditions, such as memory of nodes to water marker), true means always reject all dml requests.

diff --git a/go.sum b/go.sum
@@ -290,6 +290,7 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -516,6 +517,7 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb
 github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
 github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
 github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
+github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

diff --git a/internal/core/src/common/Channel.h b/internal/core/src/common/Channel.h
@@ -14,6 +14,7 @@
 #include <atomic>
 #include <exception>
 #include <optional>
+#include "Exception.h"
 
 namespace milvus {
 template <typename T>
@@ -55,7 +56,7 @@ class Channel {
     }
 
     void
-    close(std::optional<std::exception> ex = std::nullopt) {
+    close(std::optional<MilvusException> ex = std::nullopt) {
         if (ex.has_value()) {
             ex_ = std::move(ex);
         }
@@ -64,6 +65,6 @@ class Channel {
 
  private:
     oneapi::tbb::concurrent_bounded_queue<std::optional<T>> inner_{};
-    std::optional<std::exception> ex_{};
+    std::optional<MilvusException> ex_{};
 };
 }  // namespace milvus
diff --git a/internal/core/src/common/Exception.h b/internal/core/src/common/Exception.h
@@ -20,6 +20,22 @@
 
 namespace milvus {
 
+class MilvusException : public std::exception {
+ public:
+    explicit MilvusException(const std::string& msg)
+        : std::exception(), exception_message_(msg) {
+    }
+    const char*
+    what() const noexcept {
+        return exception_message_.c_str();
+    }
+    virtual ~MilvusException() {
+    }
+
+ private:
+    std::string exception_message_;
+};
+
 class NotImplementedException : public std::exception {
  public:
     explicit NotImplementedException(const std::string& msg)

diff --git a/internal/core/src/config/ConfigKnowhere.h b/internal/core/src/config/ConfigKnowhere.h
@@ -15,6 +15,7 @@
 // limitations under the License.
 
 #pragma once
+#include <cstdint>
 #include <string>
 
 namespace milvus::config {

diff --git a/internal/core/src/index/Index.h b/internal/core/src/index/Index.h
@@ -81,7 +81,10 @@ class IndexBase {
                index_type_ == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 ||
                index_type_ == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT ||
                index_type_ == knowhere::IndexEnum::INDEX_FAISS_IDMAP ||
-               index_type_ == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP;
+               index_type_ == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP ||
+               index_type_ ==
+                   knowhere::IndexEnum::INDEX_SPARSE_INVERTED_INDEX ||
+               index_type_ == knowhere::IndexEnum::INDEX_SPARSE_WAND;
     }
 
     const IndexType&