infiniflow · vsian · Dec 2, 2024 · Dec 2, 2024 · Dec 3, 2024 · Dec 3, 2024
diff --git a/.github/workflows/slow_test.yml b/.github/workflows/slow_test.yml
@@ -126,7 +126,7 @@ jobs:
           # Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086
           old_value=$(sudo sysctl -n vm.mmap_rnd_bits)
           sudo sysctl -w vm.mmap_rnd_bits=28
-          sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY}"
+          sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
           sudo sysctl -w vm.mmap_rnd_bits=$old_value
 
       - name: Collect thread sanitizer output in cluster test

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -84,7 +84,7 @@ jobs:
         id: run_cluster_test
         run: |
           sudo docker exec ${TESTER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
-          sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY}"
+          sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
 
       - name: Collect cluster test output
         if: ${{ !cancelled() }}
@@ -123,7 +123,7 @@ jobs:
           MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
           echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
           echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
-          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9001" && sleep 5s
+          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
 
       - name: Start infinity debug version with minio
         if: ${{ !cancelled() && !failure() }}
@@ -290,7 +290,7 @@ jobs:
           MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
           echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
           echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
-          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9001" && sleep 5s
+          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
 
       - name: Start infinity release version with minio
         if: ${{ !cancelled() && !failure() }}

diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 <div align="center">
-  <img width="187" src="https://github.com/infiniflow/infinity/assets/7248/015e1f02-1f7f-4b09-a0c2-9d261cd4858b"/>
+  <img width="187" src="https://github.com/infiniflow/infinity/assets/7248/015e1f02-1f7f-4b09-a0c2-9d261cd4858b" alt="Infinity logo"/>
 </div>
 
 
@@ -26,7 +26,7 @@ Infinity is a cutting-edge AI-native database that provides a wide range of sear
 ## ⚡️ Performance
 
 <div class="column" align="middle">
-  <img src="https://github.com/user-attachments/assets/c4c98e23-62ac-4d1a-82e5-614bca96fe0a"/>
+  <img src="https://github.com/user-attachments/assets/c4c98e23-62ac-4d1a-82e5-614bca96fe0a" alt="Infinity performance comparison"/>
 </div>
 
 ## 🌟 Key Features
@@ -60,9 +60,9 @@ Supports a wide range of data types including strings, numerics, vectors, and mo
 Infinity supports two working modes, embedded mode and client-server mode. Infinity's embedded mode enables you to quickly embed Infinity into your Python applications, without the need to connect to a separate backend server. The following shows how to operate in embedded mode:
 
    ```bash
-   pip install infinity-embedded-sdk==0.5.0.dev5
+   pip install infinity-embedded-sdk==0.5.0.dev6
    ```
-1. Use Infinity to conduct a dense vector search:
+   Use Infinity to conduct a dense vector search:
    ```python
    import infinity_embedded
 

diff --git a/benchmark/local_infinity/fulltext/fulltext_benchmark.cpp b/benchmark/local_infinity/fulltext/fulltext_benchmark.cpp
@@ -254,7 +254,7 @@ void BenchmarkQuery(SharedPtr<Infinity> infinity, const String &db_name, const S
             output_columns->emplace_back(select_rowid_expr);
             output_columns->emplace_back(select_score_expr);
         }
-        infinity->Search(db_name, table_name, search_expr, nullptr, nullptr, nullptr, output_columns, nullptr, nullptr, nullptr);
+        infinity->Search(db_name, table_name, search_expr, nullptr, nullptr, nullptr, output_columns, nullptr, nullptr, nullptr, false);
         /*
         auto result = infinity->Search(db_name, table_name, search_expr, nullptr, output_columns);
         {

diff --git a/benchmark/local_infinity/infinity_benchmark.cpp b/benchmark/local_infinity/infinity_benchmark.cpp
@@ -226,7 +226,8 @@ int main() {
                                                                          output_columns,
                                                                          nullptr,
                                                                          nullptr,
-                                                                         nullptr);
+                                                                         nullptr,
+                                                                         false);
                     });
                 results.push_back(fmt::format("-> Select QPS: {}", total_times / tims_costing_second));
             }

diff --git a/benchmark/local_infinity/knn/knn_query_benchmark.cpp b/benchmark/local_infinity/knn/knn_query_benchmark.cpp
@@ -220,7 +220,7 @@ int main(int argc, char *argv[]) {
             auto select_rowid_expr = new FunctionExpr();
             select_rowid_expr->func_name_ = "row_id";
             output_columns->emplace_back(select_rowid_expr);
-            auto result = infinity->Search(db_name, table_name, search_expr, nullptr, nullptr, nullptr, output_columns, nullptr, nullptr, nullptr);
+            auto result = infinity->Search(db_name, table_name, search_expr, nullptr, nullptr, nullptr, output_columns, nullptr, nullptr, nullptr, false);
             {
                 auto &cv = result.result_table_->GetDataBlockById(0)->column_vectors;
                 auto &column = *cv[0];

diff --git a/benchmark/remote_infinity/remote_query_benchmark.cpp b/benchmark/remote_infinity/remote_query_benchmark.cpp
@@ -51,7 +51,7 @@ struct InfinityClient {
         transport->open();
         CommonResponse response;
         ConnectRequest request;
-        request.__set_client_version(26); // 0.5.0.dev5
+        request.__set_client_version(27); // 0.5.0.dev6
         client->Connect(response, request);
         session_id = response.session_id;
     }

diff --git a/client/cpp/infinity_client.cpp b/client/cpp/infinity_client.cpp
@@ -25,7 +25,7 @@ Client Client::Connect(const std::string &ip_address, uint16_t port) {
     transport->open();
     CommonResponse response;
     ConnectRequest request;
-    request.__set_client_version(26); // 0.5.0.dev5
+    request.__set_client_version(27); // 0.5.0.dev6
     client->Connect(response, request);
     return {socket, transport, protocol, std::move(client), response.session_id};
 }

diff --git a/conf/follower.toml b/conf/follower.toml
@@ -1,7 +1,7 @@
 [general]
 version                  = "0.5.0"
 time_zone                = "utc-8"
-server_mode              = "cluster"
+server_mode              = "admin" # "standalone"
 
 [network]
 server_address           = "0.0.0.0"
@@ -46,7 +46,7 @@ mem_index_capacity       = 1048576
 storage_type             = "minio"
 
 [storage.object_storage]
-url                      = "127.0.0.1:9000"
+url                      = "127.0.0.1:9005"
 bucket_name              = "infinity"
 access_key               = "minioadmin"
 secret_key               = "minioadmin"

diff --git a/conf/infinity_conf.toml b/conf/infinity_conf.toml
@@ -37,7 +37,7 @@ mem_index_capacity       = 1048576
 
 # S3 storage config example:
 # [storage.object_storage]
-# url                      = "127.0.0.1:9000"
+# url                      = "127.0.0.1:9005"
 # bucket_name              = "infinity"
 # access_key               = "minioadmin"
 # secret_key               = "minioadmin"

diff --git a/conf/infinity_minio_conf.toml b/conf/infinity_minio_conf.toml
@@ -0,0 +1,72 @@
+[general]
+version                  = "0.5.0"
+time_zone                = "utc-8"
+
+[network]
+server_address           = "0.0.0.0"
+postgres_port            = 5432
+http_port                = 23820
+client_port              = 23817
+connection_pool_size     = 128
+
+[log]
+log_filename             = "infinity.log"
+log_dir                  = "/var/infinity/log"
+log_to_stdout            = false
+log_file_max_size        = "10GB"
+log_file_rotate_count    = 10
+
+# trace/debug/info/warning/error/critical 6 log levels, default: info
+log_level               = "info"
+
+[storage]
+persistence_dir         = "/var/infinity/persistence"
+data_dir                = "/var/infinity/data"
+# periodically activates garbage collection:
+# 0 means real-time,
+# s means seconds, for example "60s", 60 seconds
+# m means minutes, for example "60m", 60 minutes
+# h means hours, for example "1h", 1 hour
+optimize_interval        = "10s"
+cleanup_interval         = "60s"
+compact_interval         = "120s"
+storage_type             = "minio"
+# dump memory index entry when it reachs the capacity
+mem_index_capacity       = 1048576
+
+[storage.object_storage]
+url                      = "127.0.0.1:9005"
+bucket_name              = "infinity"
+access_key               = "minioadmin"
+secret_key               = "minioadmin"
+enable_https             = false
+
+# S3 storage config example:
+# [storage.object_storage]
+# url                      = "127.0.0.1:9005"
+# bucket_name              = "infinity"
+# access_key               = "minioadmin"
+# secret_key               = "minioadmin"
+# enable_https             = false
+
+[buffer]
+buffer_manager_size      = "4GB"
+lru_num                  = 7
+temp_dir                 = "/var/infinity/tmp"
+result_cache             = "off"
+memindex_memory_quota    = "1GB"
+
+[wal]
+wal_dir                       = "/var/infinity/wal"
+full_checkpoint_interval      = "86400s"
+delta_checkpoint_interval     = "60s"
+# delta_checkpoint_threshold  = 1000000000
+wal_compact_threshold         = "1GB"
+
+# flush_at_once: write and flush log each commit
+# only_write: write log, OS control when to flush the log, default
+# flush_per_second: logs are written after each commit and flushed to disk per second.
+wal_flush                     = "only_write"
+
+[resource]
+resource_dir                  = "/var/infinity/resource"
diff --git a/conf/leader.toml b/conf/leader.toml
@@ -1,7 +1,7 @@
 [general]
 version                  = "0.5.0"
 time_zone                = "utc-8"
-server_mode              = "cluster"
+server_mode              = "admin" # "standalone"
 
 [network]
 server_address           = "0.0.0.0"
@@ -46,7 +46,7 @@ mem_index_capacity       = 1048576
 storage_type             = "minio"
 
 [storage.object_storage]
-url                      = "127.0.0.1:9000"
+url                      = "127.0.0.1:9005"
 bucket_name              = "infinity"
 access_key               = "minioadmin"
 secret_key               = "minioadmin"

diff --git a/conf/learner.toml b/conf/learner.toml
@@ -1,7 +1,7 @@
 [general]
 version                  = "0.5.0"
 time_zone                = "utc-8"
-server_mode              = "cluster"
+server_mode              = "admin" # "standalone"
 
 [network]
 server_address           = "0.0.0.0"
@@ -46,7 +46,7 @@ mem_index_capacity       = 1048576
 storage_type             = "minio"
 
 [storage.object_storage]
-url                      = "127.0.0.1:9000"
+url                      = "127.0.0.1:9005"
 bucket_name              = "infinity"
 access_key               = "minioadmin"
 secret_key               = "minioadmin"

diff --git a/conf/learner2.toml b/conf/learner2.toml
@@ -1,7 +1,7 @@
 [general]
 version                  = "0.5.0"
 time_zone                = "utc-8"
-server_mode              = "cluster"
+server_mode              = "admin" # "standalone"
 
 [network]
 server_address           = "0.0.0.0"
@@ -46,7 +46,7 @@ mem_index_capacity       = 1048576
 storage_type             = "minio"
 
 [storage.object_storage]
-url                      = "127.0.0.1:9000"
+url                      = "127.0.0.1:9005"
 bucket_name              = "infinity"
 access_key               = "minioadmin"
 secret_key               = "minioadmin"

diff --git a/conf/pytest_parallel_infinity_conf.toml b/conf/pytest_parallel_infinity_conf.toml
@@ -16,6 +16,8 @@ log_level               = "trace"
 
 [storage]
 persistence_dir          = "/var/infinity/persistence"
+compact_interval         = "10s"
+cleanup_interval         = "0s"
 
 [buffer]
 buffer_manager_size      = "8GB"

diff --git a/conf/pytest_parallel_infinity_follower.toml b/conf/pytest_parallel_infinity_follower.toml
@@ -1,7 +1,7 @@
 [general]
 version                  = "0.5.0"
 time_zone                = "utc-8"
-server_mode              = "cluster"
+server_mode              = "admin" # "standalone"
 
 [network]
 server_address           = "0.0.0.0"
@@ -40,7 +40,7 @@ mem_index_capacity       = 1048576
 storage_type             = "minio"
 
 [storage.object_storage]
-url                      = "127.0.0.1:9000"
+url                      = "127.0.0.1:9005"
 bucket_name              = "infinity"
 access_key               = "pk9s2oJFX1qXLYObwIcz"
 secret_key               = "ho1G9xh2iKup4Xj9Ja3eRgg8bfwMyDv4fvkQGcZl"

diff --git a/conf/pytest_parallel_infinity_leader.toml b/conf/pytest_parallel_infinity_leader.toml
@@ -1,7 +1,7 @@
 [general]
 version                  = "0.5.0"
 time_zone                = "utc-8"
-server_mode              = "cluster"
+server_mode              = "admin" # "standalone"
 
 [network]
 server_address           = "0.0.0.0"
@@ -40,7 +40,7 @@ mem_index_capacity       = 1048576
 storage_type             = "minio"
 
 [storage.object_storage]
-url                      = "127.0.0.1:9000"
+url                      = "127.0.0.1:9005"
 bucket_name              = "infinity"
 access_key               = "minioadmin"
 secret_key               = "minioadmin"

diff --git a/conf/pytest_parallel_infinity_minio.toml b/conf/pytest_parallel_infinity_minio.toml
@@ -18,7 +18,7 @@ persistence_dir          = "/var/infinity/persistence"
 storage_type             = "minio"
 
 [storage.object_storage]
-url                      = "127.0.0.1:9000"
+url                      = "127.0.0.1:9005"
 bucket_name              = "infinity"
 access_key               = "minioadmin"
 secret_key               = "minioadmin"

diff --git a/docs/getstarted/build_from_source.mdx b/docs/getstarted/build_from_source.mdx
@@ -7,6 +7,10 @@ slug: /build_from_source
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
+Build Infinity from source, build and run unit/functional tests.
+
+---
+
 This document provides instructions for building Infinity from source, as well as building and running unit and functional tests. 
 
 :::tip NOTE
@@ -260,7 +264,7 @@ cmake --build . -t test_main
 
 2. Install Python sdk of infinity:
    ```bash
-   pip install infinity-sdk==0.5.0.dev5
+   pip install infinity-sdk==0.5.0.dev6
    ```
 
 3. Run the functional tests:
@@ -282,7 +286,7 @@ cmake --build . -t test_main
 
 2. Install Python sdk of infinity:
    ```bash
-   pip install infinity-sdk==0.5.0.dev5
+   pip install infinity-sdk==0.5.0.dev6
    ```
 
 3. Run the functional tests: 
@@ -305,7 +309,7 @@ cmake --build . -t test_main
 
 2. Install Python sdk of infinity:
    ```bash
-   pip install infinity-sdk==0.5.0.dev5
+   pip install infinity-sdk==0.5.0.dev6
    ```
 
 3. Run the functional tests: