Merge remote-tracking branch 'up_stream/main'

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,10 @@
+---
+name: Feature request
+title: '[Feature Request]: '
+about: Suggest an idea for Infinity
+labels: ''
+---
+
+**Summary**
+
+Description for this feature.

.github/ISSUE_TEMPLATE/subtask.md

@@ -0,0 +1,10 @@
+---
+name: Subtask
+title: '[Subtask]: '
+about: Subtask of an issue
+labels: ''
+---
+
+**Summary**
+
+Description for this subtask.

.github/workflows/nightly.yml

@@ -2,7 +2,7 @@ name: Nightly Build
 
 on:
   schedule:
-    - cron: '0 0 * * *'  # This schedule runs every night at midnight UTC
+    - cron: '*/30 * * * *'  # This schedule runs every night at midnight UTC
 
 jobs:
   nightly:
@@ -12,10 +12,15 @@ jobs:
       run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
 
     - name: Check out code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - name: Show working copy revision
-      run: git branch --show-current && git rev-parse HEAD
+    - name: Prepare release body
+      run: |
+        RELEASE_DATETIME=$(date --rfc-3339=seconds)
+        cat <<EOF > release_template.md
+        Release created at $RELEASE_DATETIME
+        EOF
+        envsubst < release_template.md > release_body.md
 
     - name: Start builder container
       run: |
@@ -28,41 +33,20 @@ jobs:
     - name: Build RPM and DEB
       run: sudo docker exec infinity_build bash -c "cd /infinity/cmake-build-release && cpack"
 
-    - name: Delete Existing nightly build if it exists
-      run: |
-        TAG_NAME=nightly
-        RELEASE_ID=$(curl -s -H "Authorization: Bearer ${{ secrets.MY_GITHUB_TOKEN }}" \
-                      -H "Accept: application/vnd.github.v3+json" \
-                      https://api.github.com/repos/${{ github.repository }}/releases/tags/$TAG_NAME \
-                      | jq -r '.id')
-        if [ -n "$RELEASE_ID ]; then
-          curl -X DELETE -H "Authorization: Bearer ${{ secrets.MY_GITHUB_TOKEN }}" \
-             -H "Accept: application/vnd.github.v3+json" \
-             https://api.github.com/repos/${{ github.repository }}/releases/$RELEASE_ID
-        fi
+    - name: Remove existing tag
+      # https://github.com/softprops/action-gh-release/issues/171
+      run: git push origin :refs/tags/nightly || true
 
-    - name: Create GitHub Release
-      id: create_release
-      uses: actions/create-release@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+    - name: Create or overwrite a releae
+      # https://github.com/actions/upload-release-asset has been replaced by https://github.com/softprops/action-gh-release
+      uses: softprops/action-gh-release@v1
       with:
+        token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+        prerelease: true
         tag_name: nightly
-        release_name: Nightly build $(date -d @${{ env.GITHUB_RUN_STARTED }} '+%Y-%m-%d %H:%M:%S')
-        body: |
-          # Nightly build $(date -d @${{ env.GITHUB_RUN_STARTED }} '+%Y-%m-%d %H:%M:%S') from ${{ github.ref }}
-        draft: true
-        prerelease: false
-
-    - name: Upload Binary
-      id: upload-release-asset
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: |
+        # The body field does not support environment variable substitution directly.
+        body_path: release_body.md
+        files: |
           cmake-build-release/infinity-*.deb
           cmake-build-release/infinity-*.rpm
           cmake-build-release/infinity-*.tar.gz
-        asset_content_type: application/octet-stream

.github/workflows/tests.yml

@@ -3,8 +3,14 @@ name: tests
 on:
   push:
     branches: [ main ]
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
   pull_request:
     types: [ opened, synchronize, reopened, edited ]
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
 
 jobs:
   tests:

README.md

@@ -26,8 +26,6 @@ Infinity comes with high performance, flexibility, ease-of-use, and many feature
 - Achieves 0.1 milliseconds query latency on million-scale vector datasets.
 - Up to 10K QPS on million-scale vector datasets.
 
-> See the [Benchmark](docs/benchmark.md).
-
 
 ### 🔮 Fused search
 

benchmark/local_infinity/fulltext/fulltext_import_benchmark.cpp

@@ -102,9 +102,10 @@ int main() {
     }
 
     auto r = table->CreateIndex(index_name, index_info_list, CreateIndexOptions());
-    if (!r.IsOk()) {
-        std::cout << "Create index failed: " << r.ToString() << std::endl;
-        return 1;
+    if (r.IsOk()) {
+        r = infinity->Flush();
+    } else {
+        std::cout << "Fail to create index." << r.ToString() << std::endl;
     }
     std::cout << "Create index cost: " << profiler.ElapsedToString();
     profiler.End();

src/parser/type/complex/embedding_type.h

@@ -142,22 +142,20 @@ struct EmbeddingType {
         return ss.str();
     }
 
-//    template <>
-//    inline std::string Embedding2StringInternal<float>(const EmbeddingType &embedding, size_t dimension) {
-//        char buf[6 * dimension * 2];
-//        std::stringstream ss(buf, std::ios::in | std::ios::out | std::ios::binary);
-//        for (size_t i = 0; i < dimension - 1; ++i) {
-//            char buffer[20];
-//            auto [ptr, ec] = std::to_chars(buffer, buffer + sizeof(buffer), ((float *)(embedding.ptr))[i], std::chars_format::general, 6);
-//            ss.write((const char *)buffer, ptr - buffer);
-//            ss.put(',');
-//        }
-//        char buffer[20];
-//        auto [ptr, ec] = std::to_chars(buffer, buffer + sizeof(buffer), ((float *)(embedding.ptr))[dimension - 1], std::chars_format::general, 6);
-//        ss.write((const char *)buffer, ptr - buffer);
-//        return ss.str();
-
-//    }
+    template <>
+    inline std::string Embedding2StringInternal<float>(const EmbeddingType &embedding, size_t dimension) {
+        std::stringstream ss;
+        for (size_t i = 0; i < dimension - 1; ++i) {
+            char buffer[20];
+            auto [ptr, ec] = std::to_chars(buffer, buffer + sizeof(buffer), ((float *)(embedding.ptr))[i], std::chars_format::general, 6);
+            ss.write((const char *)buffer, ptr - buffer);
+            ss.put(',');
+        }
+        char buffer[20];
+        auto [ptr, ec] = std::to_chars(buffer, buffer + sizeof(buffer), ((float *)(embedding.ptr))[dimension - 1], std::chars_format::general, 6);
+        ss.write((const char *)buffer, ptr - buffer);
+        return ss.str();
+    }
 
     [[nodiscard]] static inline std::string BitmapEmbedding2StringInternal(const EmbeddingType &embedding, size_t dimension) {
         // TODO:  This is for bitmap, and high-performance implementation is needed here.

src/storage/buffer/buffer_obj.cpp

@@ -128,30 +128,32 @@ bool BufferObj::Free() {
 }
 
 bool BufferObj::Save() {
-    rw_locker_.lock(); // This lock will be released in CloseFile()
-    if (type_ == BufferType::kPersistent) {
-        // No need to save because of no change happens
-        rw_locker_.unlock();
-        return false;
-    }
-    switch (status_) {
-        case BufferStatus::kLoaded:
-        case BufferStatus::kUnloaded: {
-            file_worker_->WriteToFile(false);
-            break;
-        }
-        case BufferStatus::kFreed: {
-            file_worker_->MoveFile();
-            break;
-        }
-        default: {
-            UniquePtr<String> err_msg = MakeUnique<String>("Invalid buffer status.");
-            LOG_ERROR(*err_msg);
-            Error<StorageException>(*err_msg);
+    bool write = false;
+    rw_locker_.lock(); // This lock will be released on return if not write, otherwise released by CloseFile()
+    if (type_ != BufferType::kPersistent) {
+        switch (status_) {
+            case BufferStatus::kLoaded:
+            case BufferStatus::kUnloaded: {
+                file_worker_->WriteToFile(false);
+                write = true;
+                break;
+            }
+            case BufferStatus::kFreed: {
+                file_worker_->MoveFile();
+                break;
+            }
+            default: {
+                UniquePtr<String> err_msg = MakeUnique<String>("Invalid buffer status.");
+                LOG_ERROR(*err_msg);
+                Error<StorageException>(*err_msg);
+            }
         }
+        type_ = BufferType::kPersistent;
     }
-    type_ = BufferType::kPersistent;
-    return true;
+    if (!write) {
+        rw_locker_.unlock();
+    }
+    return write;
 }
 
 void BufferObj::Sync() { file_worker_->Sync(); }
@@ -190,4 +192,4 @@ void BufferObj::CheckState() const {
     }
 }
 
-} // namespace infinity
+} // namespace infinity

src/unit_test/storage/buffer/buffer_obj.cpp

@@ -25,6 +25,8 @@ import data_file_worker;
 import global_resource_usage;
 import infinity_context;
 
+using namespace infinity;
+
 class BufferObjTest : public BaseTest {
     void SetUp() override {
         BaseTest::SetUp();
@@ -41,13 +43,19 @@ class BufferObjTest : public BaseTest {
         infinity::GlobalResourceUsage::UnInit();
         BaseTest::TearDown();
     }
+
+public:
+    void SaveBufferObj(BufferObj *buffer_obj) {
+        if (buffer_obj->Save()) {
+            buffer_obj->Sync();
+            buffer_obj->CloseFile();
+        }
+    };
 };
 
 // Test status transfer of buffer handle.
 // ?? status transfer in all
 TEST_F(BufferObjTest, test1) {
-    using namespace infinity;
-
     SizeT memory_limit = 1024;
     auto temp_dir = MakeShared<String>("/tmp/infinity/spill");
     auto base_dir = MakeShared<String>("/tmp/infinity/data");
@@ -117,7 +125,7 @@ TEST_F(BufferObjTest, test1) {
     // kUnloaded, kTemp -> kFreed, kTemp
     EXPECT_EQ(buf1->status(), BufferStatus::kFreed);
 
-    buf1->Save();
+    SaveBufferObj(buf1);
     // kFreed, kTemp -> kFreed, kPersistent
     EXPECT_EQ(buf1->type(), BufferType::kPersistent);
 
@@ -148,7 +156,7 @@ TEST_F(BufferObjTest, test1) {
         EXPECT_EQ(buf1->type(), BufferType::kEphemeral);
     }
 
-    buf1->Save();
+    SaveBufferObj(buf1);
     // kUnloadedModified, kEphemeral -> kUnloaded, kPersistent
     EXPECT_EQ(buf1->status(), BufferStatus::kUnloaded);
     EXPECT_EQ(buf1->type(), BufferType::kPersistent);
@@ -160,7 +168,7 @@ TEST_F(BufferObjTest, test1) {
         EXPECT_EQ(buf1->status(), BufferStatus::kLoaded);
         EXPECT_EQ(buf1->type(), BufferType::kEphemeral);
 
-        buf1->Save();
+        SaveBufferObj(buf1);
         // kLoaded, kEphemeral -> kLoaded, kPersistent
         EXPECT_EQ(buf1->status(), BufferStatus::kLoaded);
         EXPECT_EQ(buf1->type(), BufferType::kPersistent);
@@ -173,7 +181,7 @@ TEST_F(BufferObjTest, test1) {
 
     {
         auto handle1 = buf1->Load();
-        buf1->Save();
+        SaveBufferObj(buf1);
         // kLoaded, kEphemeral -> kLoaded, kPersistent
         EXPECT_EQ(buf1->status(), BufferStatus::kLoaded);
         EXPECT_EQ(buf1->type(), BufferType::kPersistent);
@@ -186,7 +194,7 @@ TEST_F(BufferObjTest, test1) {
     { auto handle2 = buf2->Load(); }
     {
         auto handle1 = buf1->Load();
-        buf1->Save();
+        SaveBufferObj(buf1);
         // kLoaded, kPersistent -> kLoaded, kPersistent
         EXPECT_EQ(buf1->status(), BufferStatus::kLoaded);
         EXPECT_EQ(buf1->type(), BufferType::kPersistent);
@@ -197,7 +205,7 @@ TEST_F(BufferObjTest, test1) {
     }
     { auto handle2 = buf2->Load(); }
     { auto handle1 = buf1->Load(); }
-    buf1->Save();
+    SaveBufferObj(buf1);
     // kUnloaded, kPersistent -> kUnloaded, kPersistent
     EXPECT_EQ(buf1->status(), BufferStatus::kUnloaded);
     EXPECT_EQ(buf1->type(), BufferType::kPersistent);