Docker cluster (infiniflow#2118)

### What problem does this PR solve? Add docker cluster test. ### Type of change - [x] Test cases
cike8899 · Oct 31, 2024 · 520372f · 520372f
1 parent 2291af7
commit 520372f
Show file tree

Hide file tree

Showing 16 changed files with 881 additions and 269 deletions.
diff --git a/.github/workflows/slow_test.yml b/.github/workflows/slow_test.yml
@@ -65,22 +65,27 @@ jobs:
           sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk && cd python/infinity_sdk/ && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
           sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
 
-      - name: Prepare cluster test
-        if: ${{ !cancelled() && !failure() }}
-        run : |
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/test_cluster/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "yum install sudo iproute2 bridge-utils -y"
-
-      - name: Run cluster test
-        if: ${{ !cancelled() && !failure() }}
-        id: run_cluster_test
-        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity"
-
-      - name: Collect cluster test output
-        if: ${{ !cancelled() }}
-        run: |
-          failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
-          sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-release/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
+    #   - name: Prepare cluster test
+    #     if: ${{ !cancelled() && !failure() }}
+    #     run : |
+    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/test_cluster/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
+    #     #   sudo docker exec ${BUILDER_CONTAINER} bash -c "yum install sudo iproute2 bridge-utils -y"
+    #     #   sudo apt install virtualenv -y
+    #     #   virtualenv -p python3 .venv
+    #     #   source .venv/bin/activate && pip3 install pytest && pip3 install -r python/test_cluster/requirements.txt
+
+    #   - name: Run cluster test
+    #     if: ${{ !cancelled() && !failure() }}
+    #     id: run_cluster_test
+    #     run: |
+    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity"
+    #     #   source .venv/bin/activate && sudo python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity --docker
+
+    #   - name: Collect cluster test output
+    #     if: ${{ !cancelled() }}
+    #     run: |
+    #       failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
+    #       sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-release/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Prepare restart test data
         if: ${{ !cancelled() && !failure() }}

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -64,22 +64,27 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='Debug' --config-settings=build-dir='cmake-build-debug' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
 
-      - name: Prepare cluster test
-        if: ${{ !cancelled() && !failure() }}
-        run : |
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/test_cluster/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "yum install sudo iproute2 bridge-utils -y"
+    #   - name: Prepare cluster test
+    #     if: ${{ !cancelled() && !failure() }}
+    #     run : |
+    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/test_cluster/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
+    #     #   sudo docker exec ${BUILDER_CONTAINER} bash -c "yum install sudo iproute2 bridge-utils -y"
+    #     #   sudo apt install virtualenv -y
+    #     #   virtualenv -p python3 .venv
+    #     #   source .venv/bin/activate && pip3 install pytest && pip3 install -r python/test_cluster/requirements.txt
 
-      - name: Run cluster test
-        if: ${{ !cancelled() && !failure() }}
-        id: run_cluster_test
-        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity"
-
-      - name: Collect cluster test output
-        if: ${{ !cancelled() }}
-        run: |
-          failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
-          sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-debug/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
+    #   - name: Run cluster test
+    #     if: ${{ !cancelled() && !failure() }}
+    #     id: run_cluster_test
+    #     run: |
+    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity"
+    #     #   source .venv/bin/activate && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --docker
+
+    #   - name: Collect cluster test output
+    #     if: ${{ !cancelled() }}
+    #     run: |
+    #       failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
+    #       sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-debug/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Prepare restart test data
         if: ${{ !cancelled() && !failure() }}
@@ -237,22 +242,27 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
 
-      - name: Prepare cluster test
-        if: ${{ !cancelled() && !failure() }}
-        run : |
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/test_cluster/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "yum install sudo iproute2 bridge-utils -y"
-
-      - name: Run cluster test
-        if: ${{ !cancelled() && !failure() }}
-        id: run_cluster_test
-        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity"
-
-      - name: Collect cluster test output
-        if: ${{ !cancelled() }}
-        run: |
-          failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
-          sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-release/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
+    #   - name: Prepare cluster test
+    #     if: ${{ !cancelled() && !failure() }}
+    #     run : |
+    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/test_cluster/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
+    #     #   sudo docker exec ${BUILDER_CONTAINER} bash -c "yum install sudo iproute2 bridge-utils -y"
+    #     #   sudo apt install virtualenv -y
+    #     #   virtualenv -p python3 .venv
+    #     #   source .venv/bin/activate && pip3 install pytest && pip3 install -r python/test_cluster/requirements.txt
+
+    #   - name: Run cluster test
+    #     if: ${{ !cancelled() && !failure() }}
+    #     id: run_cluster_test
+    #     run:
+    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity"
+    #     #   source .venv/bin/activate && python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity --docker
+
+    #   - name: Collect cluster test output
+    #     if: ${{ !cancelled() }}
+    #     run: |
+    #       failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
+    #       sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-release/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Prepare restart test data
         if: ${{ !cancelled() && !failure() }}

diff --git a/.gitignore b/.gitignore
@@ -138,3 +138,4 @@ tmp*/
 
 dist/
 python/infinity_sdk/infinity_sdk.egg-info/
+minio/