Run slow test (#1998)

### What problem does this PR solve? Slow test: Now a pr with "invalid" label will trigger slow test. slow test also triggered in schedule. CI: 1. Add slow test ci trigger. 2. Add testcases that marked as slow in "restart test"/"pysdk test" to slow test. 3. Collect log and FULL catalog of infinity if restart test failed. 4. Fix some test script. Fix: 1. sparse vector mem index dump & insert conflict. [c83703a](c83703a) 2. column entry replay [f2ff2c8](f2ff2c8), [8b34b9e](8b34b9e) 4. chunk index entry replay [1d7f308](1d7f308), [8c4e438](8c4e438), [188fe24](188fe24) 6. handle exception and decrease ref count in persistent manager. [b6a917b](b6a917b) 7. Add dump index wal when merge fulltext index. [8b34b9e](8b34b9e) 8. Init compaction alg after restart. [a59136b](a59136b) ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] Test cases
infiniflow · Oct 14, 2024 · e140330 · e140330
1 parent aca0a69
commit e140330
Show file tree

Hide file tree

Showing 37 changed files with 669 additions and 259 deletions.
diff --git a/.github/workflows/slow_test.yml b/.github/workflows/slow_test.yml
@@ -1,12 +1,21 @@
 name: slow_tests
 
 on:
+  # Schedule the workflow to run at 00:30 UTC+8 every day
+  # https://docs.github.com/zh/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#schedule
+  schedule:
+    - cron: '30 16 * * *' # 00:30 - 8 = 16:30
   # The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags.
   create:
     tags:
       - "v*.*.*"                  # normal release
       - "nightly"                 # mutable tag
       - "slow-test"               # mutable tag
+  pull_request:
+    types: [ opened, synchronize, reopened, labeled ]
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
 
 # https://docs.github.com/en/actions/using-jobs/using-concurrency
 concurrency:
@@ -16,6 +25,7 @@ concurrency:
 jobs:
   slow_tests:
     name: run slow test
+    if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'invalid') }}
     runs-on: ["self-hosted", "slow-test" ]
     steps:
 
@@ -47,32 +57,62 @@ jobs:
 
       - name: Build release version
         if: ${{ !cancelled() && !failure() }}
-        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-release && mkdir -p cmake-build-release && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-release && cmake --build /infinity/cmake-build-release --target infinity test_main knn_import_benchmark knn_query_benchmark"
+        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-release && mkdir -p cmake-build-release && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-release && cmake --build /infinity/cmake-build-release --target infinity"
 
       - name: Install pysdk for Python 3.10
         if: ${{ !cancelled() && !failure() }}
-        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk && cd python/infinity_sdk/ && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
+        run: |
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk && cd python/infinity_sdk/ && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
 
-      - name: Start infinity release version
+      - name: Prepare restart test data
+        if: ${{ !cancelled() && !failure() }}
+        run: |
+          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
+          echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV
+          sudo python3 scripts/prepare_restart_test_data.py --from_dir=${RUNNER_WORKSPACE_PREFIX} --op=add
+
+      - name: Run restart test
         if: ${{ !cancelled() && !failure() }}
+        id: run_restart_test
+        run : |
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_restart_test_continuously.py --infinity_path=cmake-build-release/src/infinity"
+        #   sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_data[infinity_runner0-columns5-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity"
+        #   sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_index[infinity_runner0-columns2-indexes2-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity"
+
+      - name: Collect restart test output
+        if: ${{ !cancelled() }} # always run this step even if previous steps failed
+          # remove symbolic link
+          # find all log file like [debug.log.*] in directory, and cat to stdout
+        run: |
+          sudo python3 scripts/prepare_restart_test_data.py --op=remove
+          failure="${{ steps.run_restart_test.outcome == 'failure'}}"
+          sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
+
+      - name: Start infinity release version
+        if: ${{ !cancelled() && !failure()}}
+        # && !contains(github.event.pull_request.labels.*.name, 'invalid')
         run: |
           # Run a command in the background
           sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-release/src/infinity --config=conf/pytest_parallel_continuous_conf.toml > release.log 2> release_error.log" &
 
       - name: Run pysdk remote infinity & parallel & http_api & sqllogic test release version continously
-        if: ${{ !cancelled() && !failure() }}
+        if: ${{ !cancelled() && !failure()}}
+        # && !contains(github.event.pull_request.labels.*.name, 'invalid')
         id: run_py_tests
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_pytest_parallel_continuous.py" && sleep 1s
 
       - name: Stop infinity release
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && !failure()}}
+        # && !contains(github.event.pull_request.labels.*.name, 'invalid')
         id: stop_py_tests
         run: |
           pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-release/src/infinity | xargs echo)
           sudo chmod +x scripts/timeout_kill.sh
           sudo docker exec ${BUILDER_CONTAINER} bash -c "/infinity/scripts/timeout_kill.sh 10 ${pids}"
           if [ $? -ne 0 ]; then
-            echo "Failed to kill infinity debug version"
+            echo "Failed to kill infinity release version"
             exit 1
           fi
 
@@ -82,29 +122,6 @@ jobs:
           failure="${{ steps.run_py_tests.outcome == 'failure' || steps.stop_py_tests.outcome == 'failure' }}"
           sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=release.log --stderror_path=release_error.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
-      - name: Prepare restart test data
-        if: ${{ !cancelled() && !failure() }}
-        run: |
-          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
-          echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV
-          touch tmp.txt && echo "${RUNNER_WORKSPACE_PREFIX}" > tmp.txt
-          sudo mkdir -p test/data/benchmark && sudo ln -sf ${RUNNER_WORKSPACE_PREFIX}/benchmark/enwiki/enwiki-10w.csv test/data/benchmark/enwiki-10w.csv
-    
-      - name: Run restart test
-        if: ${{ !cancelled() && !failure() }}
-        run : |
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_data[infinity_runner0-columns5-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity"
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_index[infinity_runner0-columns2-indexes2-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity"
-
-      - name: Collect restart test output
-        if: ${{ !cancelled() }} # always run this step even if previous steps failed
-        # remove symbolic link
-        # find all log file like [debug.log.*] in directory, and cat to stdout
-        run: |
-          sudo rm -f test/data/benchmark/enwiki-10w.csv
-          find . -name "restart_test.log.*" -exec cat {} \;
-
       - name: Destroy builder container
         if: always()  # always run this step even if previous steps failed
         run: |

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -69,20 +69,17 @@ jobs:
         run: |
           RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
           echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV
-          touch tmp.txt && echo "${RUNNER_WORKSPACE_PREFIX}" > tmp.txt
-          sudo mkdir -p test/data/benchmark && sudo ln -sf ${RUNNER_WORKSPACE_PREFIX}/benchmark/enwiki/enwiki-10w.csv test/data/benchmark/enwiki-10w.csv
 
       - name: Run restart test
         if: ${{ !cancelled() && !failure() }}
+        id: run_restart_test
         run : sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_restart_test.py --infinity_path=cmake-build-debug/src/infinity"
 
       - name: Collect restart test output
         if: ${{ !cancelled() }} # always run this step even if previous steps failed
-        # remove symbolic link
-        # find all log file like [debug.log.*] in directory, and cat to stdout
         run: |
-          sudo rm -f test/data/benchmark/enwiki-10w.csv
-          find . -name "restart_test.log.*" -exec cat {} \;
+          failure="${{ steps.run_restart_test.outcome == 'failure'}}"
+          sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Start infinity debug version
         if: ${{ !cancelled() && !failure() }}
@@ -208,20 +205,17 @@ jobs:
         run: |
           RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
           echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV
-          touch tmp.txt && echo "${RUNNER_WORKSPACE_PREFIX}" > tmp.txt
-          sudo mkdir -p test/data/benchmark && sudo ln -sf ${RUNNER_WORKSPACE_PREFIX}/benchmark/enwiki/enwiki-10w.csv test/data/benchmark/enwiki-10w.csv
     
       - name: Run restart test
         if: ${{ !cancelled() && !failure() }}
+        id: run_restart_test
         run : sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && python3 tools/run_restart_test.py --infinity_path=cmake-build-release/src/infinity"
 
       - name: Collect restart test output
         if: ${{ !cancelled() }} # always run this step even if previous steps failed
-        # remove symbolic link
-        # find all log file like [debug.log.*] in directory, and cat to stdout
         run: |
-          sudo rm -f test/data/benchmark/enwiki-10w.csv
-          find . -name "restart_test.log.*" -exec cat {} \;
+          failure="${{ steps.run_restart_test.outcome == 'failure'}}"
+          sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Test embedded infinity for Python 3.10
         if: ${{ !cancelled() && !failure() }}

diff --git a/python/restart_test/conftest.py b/python/restart_test/conftest.py
@@ -1,4 +1,5 @@
 from infinity_runner import InfinityRunner
+from restart_timeout import *
 
 
 def pytest_addoption(parser):
@@ -19,3 +20,10 @@ def pytest_generate_tests(metafunc):
         runner = InfinityRunner(infinity_path)
         metafunc.parametrize("infinity_runner", [runner])
         pass
+
+
+# def pytest_collection_modifyitems(config, items):
+#     for item in items:
+#         # Apply the decorator to each test function
+#         test_name = item.name
+#         item.obj = my_timeout(test_name)(item.obj)
diff --git a/python/restart_test/infinity_runner.py b/python/restart_test/infinity_runner.py
@@ -13,6 +13,10 @@ def __init__(self, infinity_path: str):
         self.default_config_path = "./conf/infinity_conf.toml"
         self.script_path = "./scripts/timeout_kill.sh"
         self.infinity_path = infinity_path
+
+        if not os.access(self.infinity_path, os.X_OK):
+            raise Exception(f"{self.infinity_path} is not executable.")
+
         self.i = 0
 
     def clear(self):
@@ -21,15 +25,23 @@ def clear(self):
         )
         os.system(f"rm -rf restart_test.log.*")
         print(f"clear {self.data_dir}")
+        self.i = 0
 
     def init(self, config_path: str | None = None):
+        init_timeout = 60
         if config_path is None:
             config_path = self.default_config_path
         cmd = f"{self.infinity_path} --config={config_path} > restart_test.log.{self.i} 2>&1"
 
-        pids = [proc.pid for proc in psutil.process_iter(['pid', 'name']) if "infinity" in proc.info['name']]
+        pids = [
+            proc.pid
+            for proc in psutil.process_iter(["pid", "name"])
+            if "infinity" in proc.info["name"]
+        ]
         if len(pids) > 0:
-            ret = os.system(f"bash {self.script_path} 30 {' '.join(map(str, pids))}")
+            ret = os.system(
+                f"bash {self.script_path} {init_timeout} {' '.join(map(str, pids))}"
+            )
             if ret != 0:
                 raise Exception("An error occurred.")
 
@@ -41,7 +53,9 @@ def init(self, config_path: str | None = None):
         self.i += 1
 
     def uninit(self):
-        timeout = 30
+        if self.process is None:
+            return
+        timeout = 60
         pids = []
         for child in psutil.Process(self.process.pid).children(recursive=True):
             pids.append(child.pid)
@@ -50,6 +64,10 @@ def uninit(self):
         ret = os.system(f"bash {self.script_path} {timeout} {' '.join(map(str, pids))}")
         if ret != 0:
             raise Exception("An error occurred.")
+        self.process = None
+
+    def connected(self):
+        return self.process is not None
 
     @staticmethod
     def connect(uri: str):
@@ -71,7 +89,7 @@ def decorator(f):
         def wrapper(*args, **kwargs):
             infinity_runner.init(config_path)
             infinity_obj = InfinityRunner.connect(uri)
-            try :
+            try:
                 f(infinity_obj, *args, **kwargs)
             finally:
                 infinity_obj.disconnect()

diff --git a/python/restart_test/requirements.txt b/python/restart_test/requirements.txt
@@ -1 +1,2 @@
-psutil~=6.0.0
+psutil~=6.0.0
+timeout-decorator~=0.5.0
diff --git a/python/restart_test/restart_timeout.py b/python/restart_test/restart_timeout.py
@@ -0,0 +1,31 @@
+import timeout_decorator
+
+# Description: use timeout_decorator as backup solution if a test not finish in time
+
+ONE_TEST_TIMEOUT = 60 * 60  # 1 hour
+
+
+class TimeoutException(Exception):
+    def __init__(self, name):
+        super().__init__(name)
+        print(f"Timeout decorator for {name}")
+        self.timeout_name = name
+
+    def __str__(self):
+        return f"Timeout: run {self.timeout_name} exceed {ONE_TEST_TIMEOUT} seconds"
+
+
+# wrap timeout_decorator with time and exception
+def my_timeout(timeout_name):
+    def wrapper(func):
+        def inner(*args, **kwargs):
+            try:
+                return timeout_decorator.timeout(ONE_TEST_TIMEOUT, use_signals=False)(
+                    func
+                )(*args, **kwargs)
+            except timeout_decorator.timeout_decorator.TimeoutError:
+                raise TimeoutException(timeout_name)
+
+        return inner
+
+    return wrapper