From b980f36c383f3a32de9dbfbee0b4978d6d5a622a Mon Sep 17 00:00:00 2001 From: Ji Bin Date: Sun, 3 Dec 2023 16:45:45 +0800 Subject: [PATCH] version v2.2.16 Signed-off-by: Ji Bin --- milvus_binary/env.sh | 2 +- .../0001-patch-for-milvus-lite.patch | 895 ++++++++++++++++++ src/milvus/__init__.py | 2 +- src/milvus/data/config.yaml.template | 21 +- 4 files changed, 902 insertions(+), 18 deletions(-) create mode 100644 milvus_binary/patches/milvus-v2.2.16/0001-patch-for-milvus-lite.patch diff --git a/milvus_binary/env.sh b/milvus_binary/env.sh index 54ca421..a000d56 100644 --- a/milvus_binary/env.sh +++ b/milvus_binary/env.sh @@ -1,5 +1,5 @@ MILVUS_REPO="https://github.com/milvus-io/milvus.git" -MILVUS_VERSION="v2.2.15" +MILVUS_VERSION="v2.2.16" BUILD_PROXY= BUILD_FORCE=NO diff --git a/milvus_binary/patches/milvus-v2.2.16/0001-patch-for-milvus-lite.patch b/milvus_binary/patches/milvus-v2.2.16/0001-patch-for-milvus-lite.patch new file mode 100644 index 0000000..43824c1 --- /dev/null +++ b/milvus_binary/patches/milvus-v2.2.16/0001-patch-for-milvus-lite.patch @@ -0,0 +1,895 @@ +From 1031ff06bba53484cc41b9c80aace88992ad77d9 Mon Sep 17 00:00:00 2001 +From: Ji Bin +Date: Wed, 7 Jun 2023 22:10:47 +0800 +Subject: [PATCH] patch for milvus lite + +Signed-off-by: Ji Bin +--- + go.mod | 1 - + internal/core/CMakeLists.txt | 19 ++++++-- + internal/core/src/common/Column.h | 6 ++- + internal/core/src/common/Utils.h | 4 ++ + internal/core/src/exceptions/CMakeLists.txt | 3 ++ + internal/core/src/index/ScalarIndexSort-inl.h | 2 +- + internal/core/src/index/StringIndexMarisa.cpp | 2 +- + internal/core/src/index/Utils.cpp | 16 +++---- + internal/core/src/index/VectorDiskIndex.cpp | 2 +- + internal/core/src/index/VectorMemIndex.cpp | 2 +- + internal/core/src/indexbuilder/CMakeLists.txt | 1 + + internal/core/src/segcore/CMakeLists.txt | 1 + + internal/core/src/segcore/Utils.cpp | 2 +- + internal/core/src/storage/CMakeLists.txt | 7 ++- + internal/core/src/storage/DataCodec.cpp | 2 +- + .../core/src/storage/DiskFileManagerImpl.cpp | 15 +++++-- + internal/core/src/storage/FileManager.h | 12 +++-- + .../core/src/storage/LocalChunkManager.cpp | 4 ++ + .../core/src/storage/MemFileManagerImpl.cpp | 8 ++-- + .../core/src/storage/MinioChunkManager.cpp | 10 +++++ + internal/core/src/storage/MinioChunkManager.h | 2 + + .../src/storage/RemoteChunkManagerFactory.h | 3 +- + internal/core/src/storage/Util.cpp | 44 +++++++++++++++++-- + .../core/thirdparty/aws_sdk/CMakeLists.txt | 17 +++++++ + .../core/thirdparty/boost_ext/CMakeLists.txt | 1 + + .../core/thirdparty/knowhere/CMakeLists.txt | 1 + + internal/util/etcd/etcd_server.go | 16 +++++++ + internal/util/funcutil/func.go | 12 ++++- + .../util/indexcgowrapper/build_index_info.go | 2 +- + internal/util/paramtable/grpc_param.go | 4 +- + scripts/core_build.sh | 9 ++-- + scripts/install_deps.sh | 15 ++----- + scripts/install_deps_msys.sh | 11 +++-- + scripts/setenv.sh | 1 + + 34 files changed, 199 insertions(+), 58 deletions(-) + +diff --git a/go.mod b/go.mod +index 3ac144705..3d6f34e04 100644 +--- a/go.mod ++++ b/go.mod +@@ -17,7 +17,6 @@ require ( + github.com/confluentinc/confluent-kafka-go v1.9.1 + github.com/containerd/cgroups v1.0.2 + github.com/gin-gonic/gin v1.9.1 +- github.com/go-basic/ipv4 v1.0.0 + github.com/gofrs/flock v0.8.1 + github.com/golang/protobuf v1.5.2 + github.com/google/btree v1.0.1 +diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt +index 72c8b00f3..b9672af51 100644 +--- a/internal/core/CMakeLists.txt ++++ b/internal/core/CMakeLists.txt +@@ -19,12 +19,26 @@ cmake_minimum_required( VERSION 3.18 ) + if ( APPLE ) + set( CMAKE_CROSSCOMPILING TRUE ) + set( RUN_HAVE_GNU_POSIX_REGEX 0 ) +- set( CMAKE_C_COMPILER "/usr/local/opt/llvm/bin/clang" ) +- set( CMAKE_CXX_COMPILER "/usr/local/opt/llvm/bin/clang++" ) ++ ++ if ( DEFINED ENV{HOMEBREW_PREFIX} ) ++ set( APPLE_LLVM_PREFIX $ENV{HOMEBREW_PREFIX} ) ++ else() ++ set( APPLE_LLVM_PREFIX "/usr/local" ) ++ endif() ++ include_directories( ++ ${APPLE_LLVM_PREFIX}/opt/libomp/include ++ ${APPLE_LLVM_PREFIX}/opt/boost/include ++ ${APPLE_LLVM_PREFIX}/opt/tbb/include ++ ) + endif () + + add_definitions(-DELPP_THREAD_SAFE) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) ++ ++if ( MSYS ) ++ add_definitions(-DPROTOBUF_USE_DLLS) ++endif () ++ + message( STATUS "Building using CMake version: ${CMAKE_VERSION}" ) + + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") +@@ -33,7 +47,6 @@ project(core) + include(CheckCXXCompilerFlag) + if ( APPLE ) + message(STATUS "==============Darwin Environment==============") +- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/opt/llvm/include -I/usr/local/include -I/usr/local/opt/libomp/include -L/usr/local/opt/libomp/lib") + elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + message(STATUS "==============Linux Environment===============") + set(LINUX TRUE) +diff --git a/internal/core/src/common/Column.h b/internal/core/src/common/Column.h +index 4a8f39457..b1d59635c 100644 +--- a/internal/core/src/common/Column.h ++++ b/internal/core/src/common/Column.h +@@ -10,8 +10,12 @@ + // or implied. See the License for the specific language governing permissions and limitations under the License + + #pragma once +- ++#ifndef __MINGW64__ + #include ++#else ++// mman for win32, we have it in annoy ++#include ++#endif + + #include + #include +diff --git a/internal/core/src/common/Utils.h b/internal/core/src/common/Utils.h +index 2d313194f..765efaab2 100644 +--- a/internal/core/src/common/Utils.h ++++ b/internal/core/src/common/Utils.h +@@ -20,7 +20,11 @@ + #include + + #include ++#ifdef __MINGW64__ ++#include ++#else + #include ++#endif + + #include "common/Consts.h" + #include "common/FieldMeta.h" +diff --git a/internal/core/src/exceptions/CMakeLists.txt b/internal/core/src/exceptions/CMakeLists.txt +index b92906ca8..57eab20f3 100644 +--- a/internal/core/src/exceptions/CMakeLists.txt ++++ b/internal/core/src/exceptions/CMakeLists.txt +@@ -18,6 +18,9 @@ set(exceptions_files + EasyAssert.cpp + ) + ++find_package(Boost REQUIRED) ++ + add_library(milvus_exceptions STATIC ${exceptions_files}) + + target_link_libraries(milvus_exceptions milvus_proto) ++target_include_directories(milvus_exceptions PRIVATE ${Boost_INCLUDE_DIRS}) +diff --git a/internal/core/src/index/ScalarIndexSort-inl.h b/internal/core/src/index/ScalarIndexSort-inl.h +index 8c581a335..1f503f3b4 100644 +--- a/internal/core/src/index/ScalarIndexSort-inl.h ++++ b/internal/core/src/index/ScalarIndexSort-inl.h +@@ -127,7 +127,7 @@ ScalarIndexSort::Upload(const Config& config) { + BinarySet ret; + for (auto& file : remote_paths_to_size) { + auto abs_file_path = file.first; +- ret.Append(abs_file_path.substr(abs_file_path.find_last_of("/") + 1), nullptr, file.second); ++ ret.Append(abs_file_path.substr(abs_file_path.find_last_of("/\\") + 1), nullptr, file.second); + } + + return ret; +diff --git a/internal/core/src/index/StringIndexMarisa.cpp b/internal/core/src/index/StringIndexMarisa.cpp +index c11820896..8d254cd25 100644 +--- a/internal/core/src/index/StringIndexMarisa.cpp ++++ b/internal/core/src/index/StringIndexMarisa.cpp +@@ -149,7 +149,7 @@ StringIndexMarisa::Upload(const Config& config) { + BinarySet ret; + for (auto& file : remote_paths_to_size) { + auto abs_file_path = file.first; +- ret.Append(abs_file_path.substr(abs_file_path.find_last_of("/") + 1), nullptr, file.second); ++ ret.Append(abs_file_path.substr(abs_file_path.find_last_of("/\\") + 1), nullptr, file.second); + } + + return ret; +diff --git a/internal/core/src/index/Utils.cpp b/internal/core/src/index/Utils.cpp +index 5e63b0720..5cf6d9243 100644 +--- a/internal/core/src/index/Utils.cpp ++++ b/internal/core/src/index/Utils.cpp +@@ -149,22 +149,22 @@ GetFieldDataMetaFromConfig(const Config& config) { + // set collection id + auto collection_id = index::GetValueFromConfig(config, index::COLLECTION_ID); + AssertInfo(collection_id.has_value(), "collection id not exist in index config"); +- field_data_meta.collection_id = std::stol(collection_id.value()); ++ field_data_meta.collection_id = std::stoll(collection_id.value()); + + // set partition id + auto partition_id = index::GetValueFromConfig(config, index::PARTITION_ID); + AssertInfo(partition_id.has_value(), "partition id not exist in index config"); +- field_data_meta.partition_id = std::stol(partition_id.value()); ++ field_data_meta.partition_id = std::stoll(partition_id.value()); + + // set segment id + auto segment_id = index::GetValueFromConfig(config, index::SEGMENT_ID); + AssertInfo(segment_id.has_value(), "segment id not exist in index config"); +- field_data_meta.segment_id = std::stol(segment_id.value()); ++ field_data_meta.segment_id = std::stoll(segment_id.value()); + + // set field id + auto field_id = index::GetValueFromConfig(config, index::FIELD_ID); + AssertInfo(field_id.has_value(), "field id not exist in index config"); +- field_data_meta.field_id = std::stol(field_id.value()); ++ field_data_meta.field_id = std::stoll(field_id.value()); + + return field_data_meta; + } +@@ -175,22 +175,22 @@ GetIndexMetaFromConfig(const Config& config) { + // set segment id + auto segment_id = index::GetValueFromConfig(config, index::SEGMENT_ID); + AssertInfo(segment_id.has_value(), "segment id not exist in index config"); +- index_meta.segment_id = std::stol(segment_id.value()); ++ index_meta.segment_id = std::stoll(segment_id.value()); + + // set field id + auto field_id = index::GetValueFromConfig(config, index::FIELD_ID); + AssertInfo(field_id.has_value(), "field id not exist in index config"); +- index_meta.field_id = std::stol(field_id.value()); ++ index_meta.field_id = std::stoll(field_id.value()); + + // set index version + auto index_version = index::GetValueFromConfig(config, index::INDEX_VERSION); + AssertInfo(index_version.has_value(), "index_version id not exist in index config"); +- index_meta.index_version = std::stol(index_version.value()); ++ index_meta.index_version = std::stoll(index_version.value()); + + // set index id + auto build_id = index::GetValueFromConfig(config, index::INDEX_BUILD_ID); + AssertInfo(build_id.has_value(), "build id not exist in index config"); +- index_meta.build_id = std::stol(build_id.value()); ++ index_meta.build_id = std::stoll(build_id.value()); + + return index_meta; + } +diff --git a/internal/core/src/index/VectorDiskIndex.cpp b/internal/core/src/index/VectorDiskIndex.cpp +index c02badcde..08677d7ec 100644 +--- a/internal/core/src/index/VectorDiskIndex.cpp ++++ b/internal/core/src/index/VectorDiskIndex.cpp +@@ -79,7 +79,7 @@ VectorDiskAnnIndex::Upload(const Config& config) { + BinarySet ret; + for (auto& file : remote_paths_to_size) { + auto abs_file_path = file.first; +- ret.Append(abs_file_path.substr(abs_file_path.find_last_of("/") + 1), nullptr, file.second); ++ ret.Append(abs_file_path.substr(abs_file_path.find_last_of("/\\") + 1), nullptr, file.second); + } + + return ret; +diff --git a/internal/core/src/index/VectorMemIndex.cpp b/internal/core/src/index/VectorMemIndex.cpp +index 0c95e3234..bc9c5bbc9 100644 +--- a/internal/core/src/index/VectorMemIndex.cpp ++++ b/internal/core/src/index/VectorMemIndex.cpp +@@ -62,7 +62,7 @@ VectorMemIndex::Upload(const Config& config) { + BinarySet ret; + for (auto& file : remote_paths_to_size) { + auto abs_file_path = file.first; +- ret.Append(abs_file_path.substr(abs_file_path.find_last_of("/") + 1), nullptr, file.second); ++ ret.Append(abs_file_path.substr(abs_file_path.find_last_of("/\\") + 1), nullptr, file.second); + } + + return ret; +diff --git a/internal/core/src/indexbuilder/CMakeLists.txt b/internal/core/src/indexbuilder/CMakeLists.txt +index cae7415c9..29cef2aa0 100644 +--- a/internal/core/src/indexbuilder/CMakeLists.txt ++++ b/internal/core/src/indexbuilder/CMakeLists.txt +@@ -23,6 +23,7 @@ add_library(milvus_indexbuilder SHARED ${INDEXBUILDER_FILES}) + find_library(TBB NAMES tbb) + set(PLATFORM_LIBS dl) + if (MSYS) ++find_library(TBB NAMES tbb12) + set(PLATFORM_LIBS -Wl,--allow-multiple-definition) + endif () + +diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt +index 15d6209b6..db5f081d9 100644 +--- a/internal/core/src/segcore/CMakeLists.txt ++++ b/internal/core/src/segcore/CMakeLists.txt +@@ -42,6 +42,7 @@ add_library(milvus_segcore SHARED ${SEGCORE_FILES}) + find_library(TBB NAMES tbb) + set(PLATFORM_LIBS dl) + if (MSYS) ++find_library(TBB NAMES tbb12) + set(PLATFORM_LIBS ) + endif () + +diff --git a/internal/core/src/segcore/Utils.cpp b/internal/core/src/segcore/Utils.cpp +index 329fd86ed..2279c7851 100644 +--- a/internal/core/src/segcore/Utils.cpp ++++ b/internal/core/src/segcore/Utils.cpp +@@ -552,7 +552,7 @@ std::vector + LoadFieldDatasFromRemote(std::vector& remote_files) { + auto rcm = storage::RemoteChunkManagerFactory::GetInstance().GetRemoteChunkManager(); + std::sort(remote_files.begin(), remote_files.end(), [](const std::string& a, const std::string& b) { +- return std::stol(a.substr(a.find_last_of("/") + 1)) < std::stol(b.substr(b.find_last_of("/") + 1)); ++ return std::stoll(a.substr(a.find_last_of("/\\") + 1)) < std::stoll(b.substr(b.find_last_of("/\\") + 1)); + }); + + auto parallel_degree = uint64_t(DEFAULT_FIELD_MAX_MEMORY_LIMIT / FILE_SLICE_SIZE); +diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt +index 46dd20e3a..a4c20cf27 100644 +--- a/internal/core/src/storage/CMakeLists.txt ++++ b/internal/core/src/storage/CMakeLists.txt +@@ -44,6 +44,11 @@ set(STORAGE_FILES + AliyunCredentialsProvider.cpp + MemFileManagerImpl.cpp) + ++if (MSYS) ++ list(REMOVE_ITEM STORAGE_FILES ++ DiskFileManagerImpl.cpp) ++endif (MSYS) ++ + add_library(milvus_storage SHARED ${STORAGE_FILES}) + + find_package(Boost REQUIRED COMPONENTS filesystem) +@@ -51,7 +56,7 @@ find_package(Boost REQUIRED COMPONENTS filesystem) + if ( LINUX OR APPLE ) + target_link_libraries(milvus_storage PUBLIC milvus_common Boost::filesystem aws-cpp-sdk-core aws-cpp-sdk-s3 google_cloud_cpp_storage google_cloud_cpp_common google_cloud_cpp_rest_internal pthread) + elseif ( MSYS ) +- message(FATAL_ERROR "Unsupported platform!" ) ++ target_link_libraries(milvus_storage PUBLIC milvus_common Boost::filesystem aws-cpp-sdk-core aws-cpp-sdk-s3 aws-crt-cpp pthread) + endif () + + if(NOT CMAKE_INSTALL_PREFIX) +diff --git a/internal/core/src/storage/DataCodec.cpp b/internal/core/src/storage/DataCodec.cpp +index 703383afb..c09319d90 100644 +--- a/internal/core/src/storage/DataCodec.cpp ++++ b/internal/core/src/storage/DataCodec.cpp +@@ -65,7 +65,7 @@ DeserializeRemoteFileData(BinlogReaderPtr reader) { + index_meta.field_id = data_meta.field_id; + auto& extras = descriptor_event.event_data.extras; + AssertInfo(extras.find(INDEX_BUILD_ID_KEY) != extras.end(), "index build id not exist"); +- index_meta.build_id = std::stol(extras[INDEX_BUILD_ID_KEY]); ++ index_meta.build_id = std::stoll(extras[INDEX_BUILD_ID_KEY]); + index_data->set_index_meta(index_meta); + index_data->SetTimestamps(index_event_data.start_timestamp, index_event_data.end_timestamp); + return index_data; +diff --git a/internal/core/src/storage/DiskFileManagerImpl.cpp b/internal/core/src/storage/DiskFileManagerImpl.cpp +index 96126d7f4..2209817e1 100644 +--- a/internal/core/src/storage/DiskFileManagerImpl.cpp ++++ b/internal/core/src/storage/DiskFileManagerImpl.cpp +@@ -35,7 +35,9 @@ DiskFileManagerImpl::DiskFileManagerImpl(const FieldDataMeta& field_mata, + const IndexMeta& index_meta, + const StorageConfig& storage_config) + : FileManagerImpl(field_mata, index_meta) { +- rcm_ = std::make_unique(storage_config); ++ if (storage_config.storage_type == "minio") { ++ rcm_ = std::make_unique(storage_config); ++ } + } + + DiskFileManagerImpl::DiskFileManagerImpl(const FieldDataMeta& field_mata, +@@ -159,13 +161,18 @@ DiskFileManagerImpl::CacheIndexToDisk(std::vector remote_files) { + } + + auto EstimateParalleDegree = [&](const std::string& file) -> uint64_t { +- auto fileSize = rcm_->Size(file); ++ size_t fileSize = 0; ++ if (rcm_) { ++ fileSize = rcm_->Size(file); ++ } else { ++ fileSize = LocalChunkManager::GetInstance().Size(file); ++ } + return uint64_t(DEFAULT_FIELD_MAX_MEMORY_LIMIT / fileSize); + }; + + for (auto& slices : index_slices) { + auto prefix = slices.first; +- auto local_index_file_name = GetLocalIndexObjectPrefix() + prefix.substr(prefix.find_last_of("/") + 1); ++ auto local_index_file_name = GetLocalIndexObjectPrefix() + prefix.substr(prefix.find_last_of("/\\") + 1); + local_chunk_manager.CreateFile(local_index_file_name); + int64_t offset = 0; + std::vector batch_remote_files; +@@ -216,7 +223,7 @@ DiskFileManagerImpl::CacheBatchIndexFilesToDisk(const std::vector& + std::string + DiskFileManagerImpl::CacheRawDataToDisk(std::vector remote_files) { + std::sort(remote_files.begin(), remote_files.end(), [](const std::string& a, const std::string& b) { +- return std::stol(a.substr(a.find_last_of("/") + 1)) < std::stol(b.substr(b.find_last_of("/") + 1)); ++ return std::stoll(a.substr(a.find_last_of("/\\") + 1)) < std::stoll(b.substr(b.find_last_of("/\\") + 1)); + }); + + auto segment_id = GetFieldDataMeta().segment_id; +diff --git a/internal/core/src/storage/FileManager.h b/internal/core/src/storage/FileManager.h +index 7f458a3d5..a1031ab6f 100644 +--- a/internal/core/src/storage/FileManager.h ++++ b/internal/core/src/storage/FileManager.h +@@ -21,6 +21,7 @@ + #include + + #include "common/Consts.h" ++#include "config/ConfigChunkManager.h" + #include "knowhere/common/FileManager.h" + #include "storage/ChunkManager.h" + #include "storage/Types.h" +@@ -110,9 +111,14 @@ class FileManagerImpl : public knowhere::FileManager { + + virtual std::string + GetRemoteIndexObjectPrefix() const { +- return rcm_->GetRemoteRootPath() + "/" + std::string(INDEX_ROOT_PATH) + "/" + +- std::to_string(index_meta_.build_id) + "/" + std::to_string(index_meta_.index_version) + "/" + +- std::to_string(field_meta_.partition_id) + "/" + std::to_string(field_meta_.segment_id); ++ if (rcm_) { ++ return rcm_->GetRemoteRootPath() + "/" + std::string(INDEX_ROOT_PATH) + "/" + ++ std::to_string(index_meta_.build_id) + "/" + std::to_string(index_meta_.index_version) + "/" + ++ std::to_string(field_meta_.partition_id) + "/" + std::to_string(field_meta_.segment_id); ++ } ++ return ChunkMangerConfig::GetLocalRootPath() + "/" + std::string(INDEX_ROOT_PATH) + "/" + ++ std::to_string(index_meta_.build_id) + "/" + std::to_string(index_meta_.index_version) + "/" + ++ std::to_string(field_meta_.partition_id) + "/" + std::to_string(field_meta_.segment_id); + } + + protected: +diff --git a/internal/core/src/storage/LocalChunkManager.cpp b/internal/core/src/storage/LocalChunkManager.cpp +index 4842e5d9b..f26947fad 100644 +--- a/internal/core/src/storage/LocalChunkManager.cpp ++++ b/internal/core/src/storage/LocalChunkManager.cpp +@@ -97,6 +97,8 @@ LocalChunkManager::Read(const std::string& filepath, uint64_t offset, void* buf, + void + LocalChunkManager::Write(const std::string& absPathStr, void* buf, uint64_t size) { + std::ofstream outfile; ++ boost::filesystem::path absPath(absPathStr); ++ boost::filesystem::create_directories(absPath.parent_path()); + outfile.open(absPathStr, std::ios_base::binary); + if (outfile.fail()) { + std::stringstream err_msg; +@@ -113,6 +115,8 @@ LocalChunkManager::Write(const std::string& absPathStr, void* buf, uint64_t size + void + LocalChunkManager::Write(const std::string& absPathStr, uint64_t offset, void* buf, uint64_t size) { + std::ofstream outfile; ++ boost::filesystem::path absPath(absPathStr); ++ boost::filesystem::create_directories(absPath.parent_path()); + outfile.open(absPathStr, std::ios_base::in | std::ios_base::out | std::ios_base::binary); + if (outfile.fail()) { + std::stringstream err_msg; +diff --git a/internal/core/src/storage/MemFileManagerImpl.cpp b/internal/core/src/storage/MemFileManagerImpl.cpp +index cd7a91a0f..2fa6157d7 100644 +--- a/internal/core/src/storage/MemFileManagerImpl.cpp ++++ b/internal/core/src/storage/MemFileManagerImpl.cpp +@@ -26,7 +26,9 @@ MemFileManagerImpl::MemFileManagerImpl(const FieldDataMeta& field_mata, + const IndexMeta& index_meta, + const StorageConfig& storage_config) + : FileManagerImpl(field_mata, index_meta) { +- rcm_ = std::make_unique(storage_config); ++ if (storage_config.storage_type == "minio") { ++ rcm_ = std::make_unique(storage_config); ++ } + } + + MemFileManagerImpl::MemFileManagerImpl(const FieldDataMeta& field_mata, +@@ -92,7 +94,7 @@ MemFileManagerImpl::LoadIndexToMemory(std::vector remote_files) { + auto LoadBatchIndexFiles = [&]() { + auto index_datas = GetObjectData(rcm_.get(), batch_files); + for (size_t idx = 0; idx < batch_files.size(); ++idx) { +- auto file_name = batch_files[idx].substr(batch_files[idx].find_last_of("/") + 1); ++ auto file_name = batch_files[idx].substr(batch_files[idx].find_last_of("/\\") + 1); + file_to_index_data[file_name] = index_datas[idx]; + } + }; +@@ -118,7 +120,7 @@ MemFileManagerImpl::LoadIndexToMemory(std::vector remote_files) { + std::vector + MemFileManagerImpl::CacheRawDataToMemory(std::vector remote_files) { + std::sort(remote_files.begin(), remote_files.end(), [](const std::string& a, const std::string& b) { +- return std::stol(a.substr(a.find_last_of("/") + 1)) < std::stol(b.substr(b.find_last_of("/") + 1)); ++ return std::stoll(a.substr(a.find_last_of("/\\") + 1)) < std::stoll(b.substr(b.find_last_of("/\\") + 1)); + }); + + auto parallel_degree = uint64_t(DEFAULT_FIELD_MAX_MEMORY_LIMIT / FILE_SLICE_SIZE); +diff --git a/internal/core/src/storage/MinioChunkManager.cpp b/internal/core/src/storage/MinioChunkManager.cpp +index 667d20dc5..f481c5a11 100644 +--- a/internal/core/src/storage/MinioChunkManager.cpp ++++ b/internal/core/src/storage/MinioChunkManager.cpp +@@ -35,6 +35,12 @@ + #include "signal.h" + #include "log/Log.h" + ++ ++#if __MINGW32__ ++#undef GetMessage ++#undef GetObject ++#endif ++ + #define THROWS3ERROR(FUNCTION) \ + do { \ + auto& err = outcome.GetError(); \ +@@ -50,6 +56,7 @@ namespace milvus::storage { + std::atomic MinioChunkManager::init_count_(0); + std::mutex MinioChunkManager::client_mutex_; + ++#ifndef WIN32 + static void + SwallowHandler(int signal) { + #pragma GCC diagnostic push +@@ -65,6 +72,7 @@ SwallowHandler(int signal) { + } + #pragma GCC diagnostic pop + } ++#endif + + /** + * @brief convert std::string to Aws::String +@@ -99,6 +107,7 @@ MinioChunkManager::InitSDKAPI(RemoteStorageType type, bool useIAM, const std::st + std::scoped_lock lock{client_mutex_}; + const size_t initCount = init_count_++; + if (initCount == 0) { ++#ifndef WIN32 + // sdk_options_.httpOptions.installSigPipeHandler = true; + struct sigaction psa; + memset(&psa, 0, sizeof psa); +@@ -108,6 +117,7 @@ MinioChunkManager::InitSDKAPI(RemoteStorageType type, bool useIAM, const std::st + sigemptyset(&psa.sa_mask); + sigaddset(&psa.sa_mask, SIGPIPE); + sigaction(SIGPIPE, &psa, 0); ++#endif + #ifdef BUILD_GCP + if (type == RemoteStorageType::GOOGLE_CLOUD && useIAM) { + sdk_options_.httpOptions.httpClientFactory_create_fn = []() { +diff --git a/internal/core/src/storage/MinioChunkManager.h b/internal/core/src/storage/MinioChunkManager.h +index 2a10e28ae..bbd82006d 100644 +--- a/internal/core/src/storage/MinioChunkManager.h ++++ b/internal/core/src/storage/MinioChunkManager.h +@@ -26,8 +26,10 @@ + #include + #include + #include ++#ifdef BUILD_GCP + #include + #include ++#endif // BUILD_GCP + #include + #include + #include +diff --git a/internal/core/src/storage/RemoteChunkManagerFactory.h b/internal/core/src/storage/RemoteChunkManagerFactory.h +index 47e9f21e9..a59558698 100644 +--- a/internal/core/src/storage/RemoteChunkManagerFactory.h ++++ b/internal/core/src/storage/RemoteChunkManagerFactory.h +@@ -47,7 +47,8 @@ class RemoteChunkManagerFactory { + if (rcm_ == nullptr) { + switch (storage_type) { + case ChunkManagerType::Local: { +- PanicInfo("unsupported"); // TODO :: ++ // PanicInfo("unsupported"); // TODO :: ++ break; + } + case ChunkManagerType::Minio: { + rcm_ = std::make_shared(storage_config); +diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp +index fac421fac..3df3abb9f 100644 +--- a/internal/core/src/storage/Util.cpp ++++ b/internal/core/src/storage/Util.cpp +@@ -22,6 +22,9 @@ + #include "config/ConfigChunkManager.h" + #include "storage/FieldData.h" + #include "storage/parquet_c.h" ++#include "storage/FieldDataFactory.h" ++#include "storage/LocalChunkManager.h" ++#include "storage/MemFileManagerImpl.h" + #include "storage/ThreadPool.h" + #include "storage/MemFileManagerImpl.h" + #include "storage/FieldDataFactory.h" +@@ -30,6 +33,29 @@ + #include "storage/DiskFileManagerImpl.h" + #endif + ++namespace { ++std::string patch_index_files_path(const std::string& path) { ++ auto file = path; ++ auto index = file.find("querynode/index_files"); ++ if (index != std::string::npos) { ++ file.replace(index, strlen("querynode/index_files"), "index_files"); ++ } ++ index = file.find("querynode\\index_files"); ++ if (index != std::string::npos) { ++ file.replace(index, strlen("querynode\\index_files"), "index_files"); ++ } ++ index = file.find("indexnode/index_files"); ++ if (index != std::string::npos) { ++ file.replace(index, strlen("indexnode/index_files"), "index_files"); ++ } ++ index = file.find("indexnode\\index_files"); ++ if (index != std::string::npos) { ++ file.replace(index, strlen("indexnode\\index_files"), "index_files"); ++ } ++ return file; ++} ++} ++ + namespace milvus::storage { + + std::map ChunkManagerType_Map = {{"local", ChunkManagerType::Local}, +@@ -354,9 +380,15 @@ CreateFileManager(IndexType index_type, + + std::unique_ptr + DownloadAndDecodeRemoteFile(RemoteChunkManager* remote_chunk_manager, std::string file) { +- auto fileSize = remote_chunk_manager->Size(file); ++ ChunkManager *chunk_manager = remote_chunk_manager; ++ if (! chunk_manager) { ++ chunk_manager = &LocalChunkManager::GetInstance(); ++ // workaround: replace any /querynode/ /indexnode/ => / ++ file = patch_index_files_path(file); ++ } ++ auto fileSize = chunk_manager->Size(file); + auto buf = std::shared_ptr(new uint8_t[fileSize]); +- remote_chunk_manager->Read(file, buf.get(), fileSize); ++ chunk_manager->Read(file, buf.get(), fileSize); + + return DeserializeFileData(buf, fileSize); + } +@@ -376,7 +408,13 @@ EncodeAndUploadIndexSlice(RemoteChunkManager* remote_chunk_manager, + auto serialized_index_data = indexData->serialize_to_remote_file(); + auto serialized_index_size = serialized_index_data.size(); + +- remote_chunk_manager->Write(object_key, serialized_index_data.data(), serialized_index_size); ++ ChunkManager *chunk_manager = remote_chunk_manager; ++ if (! chunk_manager) { ++ chunk_manager = &LocalChunkManager::GetInstance(); ++ // workaround: replace any /querynode/ /indexnode/ => / ++ object_key = patch_index_files_path(object_key); ++ } ++ chunk_manager->Write(object_key, serialized_index_data.data(), serialized_index_size); + return std::pair(object_key, serialized_index_size); + } + +diff --git a/internal/core/thirdparty/aws_sdk/CMakeLists.txt b/internal/core/thirdparty/aws_sdk/CMakeLists.txt +index fc6b5fc5d..fe89a94a2 100644 +--- a/internal/core/thirdparty/aws_sdk/CMakeLists.txt ++++ b/internal/core/thirdparty/aws_sdk/CMakeLists.txt +@@ -44,6 +44,11 @@ macro(build_aws_sdk_s3) + list(APPEND AWS_SDK_S3_CMAKE_ARGS "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}" "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}" "-DCMAKE_OSX_ARCHITECTURES=${CPU_ARCH}") + endif() + ++ if ( MSYS ) ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DAWS_IO_EXPORTS -DAWS_CRT_CPP_EXPORTS") ++ set(AWS_SDK_S3_CMAKE_ARGS ${AWS_SDK_S3_CMAKE_ARGS} "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}") ++ endif() ++ + ExternalProject_Add(aws_sdk_s3_ep + + GIT_REPOSITORY https://github.com/aws/aws-sdk-cpp.git +@@ -58,6 +63,8 @@ macro(build_aws_sdk_s3) + BUILD_COMMAND ${AWS_SDK_BUILD_COMMAND} + INSTALL_COMMAND ${AWS_SDK_INSTALL_COMMAND} + CMAKE_ARGS ${AWS_SDK_S3_CMAKE_ARGS} ++ # If want build with mingw, need to patch the source code ++ # PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/../../../patches/aws-sdk-1.9.234.patch + ) + + add_library(aws-cpp-sdk-core SHARED IMPORTED) +@@ -66,6 +73,11 @@ macro(build_aws_sdk_s3) + IMPORTED_GLOBAL TRUE + IMPORTED_LOCATION ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_SHARED_LIBRARY_SUFFIX} + INTERFACE_INCLUDE_DIRECTORIES ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) ++ if ( MSYS ) ++ set_target_properties(aws-cpp-sdk-core ++ PROPERTIES ++ IMPORTED_IMPLIB ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_SHARED_LIBRARY_SUFFIX}.a) ++ endif () + add_dependencies(aws-cpp-sdk-core aws_sdk_s3_ep) + + add_library(aws-cpp-sdk-s3 SHARED IMPORTED) +@@ -74,6 +86,11 @@ macro(build_aws_sdk_s3) + IMPORTED_GLOBAL TRUE + IMPORTED_LOCATION ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_SHARED_LIBRARY_SUFFIX} + INTERFACE_INCLUDE_DIRECTORIES ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) ++ if ( MSYS ) ++ set_target_properties(aws-cpp-sdk-s3 ++ PROPERTIES ++ IMPORTED_IMPLIB ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_SHARED_LIBRARY_SUFFIX}.a) ++ endif () + add_dependencies(aws-cpp-sdk-s3 aws_sdk_s3_ep) + + # add_library(aws-cpp-sdk-sts SHARED IMPORTED) +diff --git a/internal/core/thirdparty/boost_ext/CMakeLists.txt b/internal/core/thirdparty/boost_ext/CMakeLists.txt +index e28099e7c..03a29634b 100644 +--- a/internal/core/thirdparty/boost_ext/CMakeLists.txt ++++ b/internal/core/thirdparty/boost_ext/CMakeLists.txt +@@ -1,2 +1,3 @@ + find_package(Boost REQUIRED) ++include_directories(${Boost_INCLUDE_DIRS}) + add_library(boost_bitset_ext dynamic_bitset_ext.cpp) +diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt +index 561d84cb9..4dbb5bab3 100644 +--- a/internal/core/thirdparty/knowhere/CMakeLists.txt ++++ b/internal/core/thirdparty/knowhere/CMakeLists.txt +@@ -52,6 +52,7 @@ macro(build_knowhere) + PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/knowhere-subbuild + BINARY_DIR knowhere-bin + INSTALL_DIR ${KNOWHERE_INSTALL_PREFIX} ++ PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/../../../patches/knowhere-v1.3.19.patch + ) + + ExternalProject_Get_Property(knowhere_ep INSTALL_DIR) +diff --git a/internal/util/etcd/etcd_server.go b/internal/util/etcd/etcd_server.go +index 75f81c43e..ffd758812 100644 +--- a/internal/util/etcd/etcd_server.go ++++ b/internal/util/etcd/etcd_server.go +@@ -1,6 +1,9 @@ + package etcd + + import ( ++ "net/url" ++ "os" ++ "runtime" + "sync" + + "github.com/milvus-io/milvus/internal/log" +@@ -45,6 +48,12 @@ func InitEtcdServer( + } else { + cfg = embed.NewConfig() + } ++ if runtime.GOOS == "windows" { ++ err := zap.RegisterSink("winfile", newWinFileSink) ++ if err != nil { ++ initError = err ++ } ++ } + cfg.Dir = dataDir + cfg.LogOutputs = []string{logPath} + cfg.LogLevel = logLevel +@@ -73,3 +82,10 @@ func StopEtcdServer() { + }) + } + } ++ ++// special file sink for zap, as etcd using zap as Logger ++// See: https://github.com/uber-go/zap/issues/621 ++func newWinFileSink(u *url.URL) (zap.Sink, error) { ++ // e.g. winfile:///D:/test/ -> D:/test/ ++ return os.OpenFile(u.Path[1:], os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) ++} +diff --git a/internal/util/funcutil/func.go b/internal/util/funcutil/func.go +index fc30a7c07..bd348fdd0 100644 +--- a/internal/util/funcutil/func.go ++++ b/internal/util/funcutil/func.go +@@ -31,7 +31,6 @@ import ( + "strings" + "time" + +- "github.com/go-basic/ipv4" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" +@@ -58,7 +57,16 @@ func CheckGrpcReady(ctx context.Context, targetCh chan error) { + + // GetLocalIP return the local ip address + func GetLocalIP() string { +- return ipv4.LocalIP() ++ addrs, err := net.InterfaceAddrs() ++ if err == nil { ++ for _, addr := range addrs { ++ ipaddr, ok := addr.(*net.IPNet) ++ if ok && ipaddr.IP.IsGlobalUnicast() && ipaddr.IP.To4() != nil { ++ return ipaddr.IP.String() ++ } ++ } ++ } ++ return "127.0.0.1" + } + + // WaitForComponentStates wait for component's state to be one of the specific states +diff --git a/internal/util/indexcgowrapper/build_index_info.go b/internal/util/indexcgowrapper/build_index_info.go +index c93197200..ab8d012f5 100644 +--- a/internal/util/indexcgowrapper/build_index_info.go ++++ b/internal/util/indexcgowrapper/build_index_info.go +@@ -44,7 +44,7 @@ func NewBuildIndexInfo(config *indexpb.StorageConfig) (*BuildIndexInfo, error) { + var cBuildIndexInfo C.CBuildIndexInfo + + // TODO::xige-16 support embedded milvus +- storageType := "minio" ++ storageType := config.StorageType + cAddress := C.CString(config.Address) + cBucketName := C.CString(config.BucketName) + cAccessKey := C.CString(config.AccessKeyID) +diff --git a/internal/util/paramtable/grpc_param.go b/internal/util/paramtable/grpc_param.go +index 5a6ae88d5..1869b2f9c 100644 +--- a/internal/util/paramtable/grpc_param.go ++++ b/internal/util/paramtable/grpc_param.go +@@ -17,8 +17,8 @@ import ( + "sync" + "time" + +- "github.com/go-basic/ipv4" + "github.com/milvus-io/milvus/internal/log" ++ "github.com/milvus-io/milvus/internal/util/funcutil" + "go.uber.org/zap" + ) + +@@ -82,7 +82,7 @@ func (p *grpcConfig) init(domain string) { + + // LoadFromEnv is used to initialize configuration items from env. + func (p *grpcConfig) LoadFromEnv() { +- p.IP = ipv4.LocalIP() ++ p.IP = funcutil.GetLocalIP() + } + + // LoadFromArgs is used to initialize configuration items from args. +diff --git a/scripts/core_build.sh b/scripts/core_build.sh +index db41d3f62..980985d6f 100755 +--- a/scripts/core_build.sh ++++ b/scripts/core_build.sh +@@ -243,20 +243,21 @@ case "${unameOut}" in + fi + done + if [ "${llvm_version}" = "NOT_FOUND" ] ; then +- echo "valid llvm(14 or 15) not installed" ++ echo "llvm@14~15 is not installed" + exit 1 + fi + llvm_prefix="$(brew --prefix llvm@${llvm_version})" + export CLANG_TOOLS_PATH="${llvm_prefix}/bin" ++ export PATH=${CLANG_TOOLS_PATH}:${PATH} + export CC="${llvm_prefix}/bin/clang" + export CXX="${llvm_prefix}/bin/clang++" +- export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include" ++ export CFLAGS="-Wno-deprecated-declarations -I${llvm_prefix}/include -I/usr/local/include -I$(brew --prefix libomp)/include -I$(brew --prefix boost)/include -I$(brew --prefix tbb)/include" + export CXXFLAGS=${CFLAGS} +- export LDFLAGS="-L$(brew --prefix libomp)/lib" ++ export LDFLAGS="-L${llvm_prefix}/lib -L$(brew --prefix libomp)/lib -L$(brew --prefix boost)/lib -L$(brew --prefix tbb)/lib" + ;; + Linux*) + ;; +- *) ++ *) + echo "Cannot build on windows" + ;; + esac +diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh +index a5eed150b..bb7f00a65 100755 +--- a/scripts/install_deps.sh ++++ b/scripts/install_deps.sh +@@ -29,7 +29,7 @@ function install_linux_deps() { + sudo yum install -y git make lcov libtool m4 autoconf automake ccache openssl-devel zlib-devel libzstd-devel \ + libcurl-devel python3-devel \ + devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran \ +- llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra libuuid-devel pulseaudio-libs-devel ++ llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra libuuid-devel pulseaudio-libs-devel + + echo "source scl_source enable devtoolset-7" | sudo tee -a /etc/profile.d/devtoolset-7.sh + echo "source scl_source enable llvm-toolset-7.0" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh +@@ -56,16 +56,9 @@ function install_linux_deps() { + } + + function install_mac_deps() { +- sudo xcode-select --install > /dev/null 2>&1 +- brew install libomp ninja cmake llvm@15 ccache grep +- export PATH="/usr/local/opt/grep/libexec/gnubin:$PATH" +- brew update && brew upgrade && brew cleanup +- +- if [[ $(arch) == 'arm64' ]]; then +- brew install openssl +- brew install librdkafka +- brew install pkg-config +- fi ++ sudo xcode-select --install > /dev/null 2>&1 ++ brew install boost libomp ninja tbb openblas cmake llvm@15 ccache pkg-config zstd openssl librdkafka ++ brew install grep + } + + if ! command -v go &> /dev/null +diff --git a/scripts/install_deps_msys.sh b/scripts/install_deps_msys.sh +index a08296010..c1d8bde66 100644 +--- a/scripts/install_deps_msys.sh ++++ b/scripts/install_deps_msys.sh +@@ -11,6 +11,10 @@ pacmanInstall() + { + pacman -S --noconfirm --needed \ + git make tar dos2unix zip unzip patch \ ++ mingw-w64-x86_64-arrow \ ++ mingw-w64-x86_64-aws-c-http \ ++ mingw-w64-x86_64-aws-c-s3 \ ++ mingw-w64-x86_64-aws-sdk-cpp \ + mingw-w64-x86_64-toolchain \ + mingw-w64-x86_64-make \ + mingw-w64-x86_64-ccache \ +@@ -22,11 +26,10 @@ pacmanInstall() + mingw-w64-x86_64-clang-tools-extra \ + mingw-w64-x86_64-python2 \ + mingw-w64-x86_64-diffutils \ +- mingw-w64-x86_64-arrow \ +- mingw-w64-x86_64-go \ + mingw-w64-x86_64-zstd +- pacman -U --noconfirm \ +- https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-rocksdb-6.26.1-1-any.pkg.tar.zst ++ pacman -U --noconfirm --needed \ ++ https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-rocksdb-6.26.1-1-any.pkg.tar.zst \ ++ https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-go-1.18-2-any.pkg.tar.zst + } + + updateKey() +diff --git a/scripts/setenv.sh b/scripts/setenv.sh +index 577683dfb..08a80abe3 100644 +--- a/scripts/setenv.sh ++++ b/scripts/setenv.sh +@@ -42,6 +42,7 @@ case "${unameOut}" in + export RPATH=$LD_LIBRARY_PATH;; + Darwin*) + export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig" ++ export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$(brew --prefix openssl)/lib/pkgconfig" + export DYLD_LIBRARY_PATH=$ROOT_DIR/internal/core/output/lib + export RPATH=$DYLD_LIBRARY_PATH;; + MINGW*) +-- +2.43.0 + diff --git a/src/milvus/__init__.py b/src/milvus/__init__.py index 55f5b9d..5361d59 100644 --- a/src/milvus/__init__.py +++ b/src/milvus/__init__.py @@ -20,7 +20,7 @@ import urllib.request import hashlib -__version__ = '2.2.15' +__version__ = '2.2.16' LOGGERS = {} diff --git a/src/milvus/data/config.yaml.template b/src/milvus/data/config.yaml.template index 9337931..59ca668 100644 --- a/src/milvus/data/config.yaml.template +++ b/src/milvus/data/config.yaml.template @@ -52,17 +52,6 @@ etcd: metastore: type: etcd -# Related configuration of mysql, used to store Milvus metadata. -mysql: - username: root - password: 123456 - address: localhost - port: 3306 - dbName: milvus_meta - driverName: mysql - maxOpenConns: 20 - maxIdleConns: 5 - # please adjust in embedded Milvus: /tmp/milvus/data/ localStorage: path: {{ local_storage_dir }} @@ -81,13 +70,13 @@ minio: # For more infomation, refer to # aws: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html # gcp: https://cloud.google.com/storage/docs/access-control/iam - # aliyun (ack): https://www.alibabacloud.com/help/en/container-service-for-kubernetes/latest/use-rrsa-to-enforce-access-control + # aliyun (ack): https://www.alibabacloud.com/help/en/container-service-for-kubernetes/latest/use-rrsa-to-enforce-access-control # aliyun (ecs): https://www.alibabacloud.com/help/en/elastic-compute-service/latest/attach-an-instance-ram-role useIAM: false - # Cloud Provider of S3. Supports: "aws", "gcp", "aliyun". + # Cloud Provider of S3. Supports: "aws", "gcp", "aliyun". # You can use "aws" for other cloud provider supports S3 API with signature v4, e.g.: minio # You can use "gcp" for other cloud provider supports S3 API with signature v2 - # You can use "aliyun" for other cloud provider uses virtual host style bucket + # You can use "aliyun" for other cloud provider uses virtual host style bucket # When useIAM enabled, only "aws", "gcp", "aliyun" is supported for now cloudProvider: aws # Custom endpoint for fetch IAM role credentials. when useIAM is true & cloudProvider is "aws". @@ -114,6 +103,7 @@ pulsar: maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar. tenant: public namespace: default + requestTimeout: 60 # pulsar client global request timeout in seconds # If you want to enable kafka, needs to comment the pulsar configs kafka: @@ -178,10 +168,9 @@ proxy: # please adjust in embedded Milvus: false ginLogging: false # Whether to produce gin logs. grpc: + # change to smaller value to limit client request and response serverMaxRecvSize: 67108864 # 64M serverMaxSendSize: 67108864 # 64M - clientMaxRecvSize: 104857600 # 100 MB, 100 * 1024 * 1024 - clientMaxSendSize: 104857600 # 100 MB, 100 * 1024 * 1024