-
Notifications
You must be signed in to change notification settings - Fork 278
239 lines (210 loc) · 16.2 KB
/
slow_test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
name: slow_tests
on:
# Schedule the workflow to run at 00:30 UTC+8 every day
# https://docs.github.com/zh/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#schedule
schedule:
- cron: '30 10,18,2 * * *' # utc-8: 18:30, 2:30, 10:30
# The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags.
create:
tags:
- "v*.*.*" # normal release
- "nightly" # mutable tag
- "slow-test" # mutable tag
pull_request:
types: [ opened, synchronize, reopened, labeled ]
paths-ignore:
- 'docs/**'
- '*.md'
# https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
slow_tests:
name: run slow test
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'slow-test') }}
runs-on: ["self-hosted", "slow-test" ]
steps:
- name: Choose test type
id: choose_test_type
run: |
echo "RUN_TSAN_TEST=true" >> $GITHUB_OUTPUT
echo "RUN_RESTART_TEST=true" >> $GITHUB_OUTPUT
echo "RUN_PYTEST_PARALLEL=true" >> $GITHUB_OUTPUT
- name: Show PR labels
if: ${{ !cancelled() && !failure() }}
run: |
echo "Workflow triggered by ${{ github.event_name }}"
if [[ ${{ github.event_name }} == 'pull_request' ]]; then
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}"
fi
- name: Ensure workspace ownership
if: ${{ !cancelled() && !failure() }}
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
- name: Check out code
if: ${{ !cancelled() && !failure() }}
uses: actions/checkout@v4
- name: Start builder container
if: ${{ !cancelled() && !failure() }}
run: |
BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
CPUS=${CPUS:-$(nproc)}
echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV
echo "CPUS=${CPUS}" >> $GITHUB_ENV
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
- name: Create Test image & Start tester container
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
run: |
TESTER_IMAGE_NAME=infiniflow/infinity_tester:centos7_clang18
sudo python3 scripts/build_tester_image.py --image_name=${TESTER_IMAGE_NAME} --bin_path=${RUNNER_WORKSPACE_PREFIX}/build_tester_image
TESTER_CONTAINER=infinity_tester_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
CPUS=${CPUS:-$(nproc)}
echo "TESTER_CONTAINER=${TESTER_CONTAINER}" >> $GITHUB_ENV
echo "CPUS=${CPUS}" >> $GITHUB_ENV
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
INF_DIRECTORY=$PWD
echo "${INF_DIRECTORY}" | sudo tee .tester_env
sudo docker rm -f -v ${TESTER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${TESTER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} ${TESTER_IMAGE_NAME}
sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY}"
- name: Build release version
if: ${{ !cancelled() && !failure() }}
run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-release && mkdir -p cmake-build-release && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-release && cmake --build /infinity/cmake-build-release --target infinity"
- name: Install pysdk for Python 3.10
if: ${{ !cancelled() && !failure() }}
run: |
sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk && cd python/infinity_sdk/ && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
- name: Build debug version with thread sanitizer
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DENABLE_SANITIZER_THREAD=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-debug && cmake --build /infinity/cmake-build-debug --target infinity"
- name: Prepare restart test data
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_RESTART_TEST == 'true' }}
run: |
RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV
sudo python3 scripts/prepare_restart_test_data.py --from_dir=${RUNNER_WORKSPACE_PREFIX} --op=add
- name: Run restart test with thread sanitizer
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
id: run_restart_test_sanitizer
run : |
sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
# Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086
old_value=$(sudo sysctl -n vm.mmap_rnd_bits)
sudo sysctl -w vm.mmap_rnd_bits=28
sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so TSAN_OPTIONS=log_path=restart_tsan.log ASAN_OPTIONS=detect_leaks=0 python3 tools/run_restart_test.py --infinity_path=cmake-build-debug/src/infinity"
sudo sysctl -w vm.mmap_rnd_bits=$old_value
- name: Collect restart test output with thread sanitizer
if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} # always run this step even if previous steps failed
# remove symbolic link
# find all log file like [debug.log.*] in directory, and cat to stdout
run: |
failure="${{ steps.run_restart_test_sanitizer.outcome == 'failure'}}"
sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} --tsan_log=restart_tsan.log
sudo rm restart_tsan.log*
- name: Run cluster test
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
id: run_cluster_test
run: |
sudo docker exec ${TESTER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
# Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086
old_value=$(sudo sysctl -n vm.mmap_rnd_bits)
sudo sysctl -w vm.mmap_rnd_bits=28
sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
sudo sysctl -w vm.mmap_rnd_bits=$old_value
- name: Collect thread sanitizer output in cluster test
if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
run: |
failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-debug/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} --tsan_log=cluster_tsan.log
sudo rm -f cluster_tsan.log*
- name: Start infinity debug version with thread sanitizer
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
id: start_tests_debug
run: |
# Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086
old_value=$(sudo sysctl -n vm.mmap_rnd_bits)
echo "old_mmmap_rnd_bits=${old_value}" >> "$GITHUB_OUTPUT"
sudo sysctl -w vm.mmap_rnd_bits=28
sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && TSAN_OPTIONS=log_path=tsan.log cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > debug.log 2> debug_error.log &" && sleep 1s
if ! sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-debug/src/infinity > /dev/null; then
echo "Failed to start infinity debug version"
cat debug_error.log
exit 1
fi
- name: Run pytest parallel with thread sanitizer
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
id: run_tests_debug
run: |
sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_pytest_parallel.py" && sleep 1s
- name: Stop infinity pysdk & http_api & sqllogictest debug
if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
id: stop_tests_debug
env:
old_value: ${{ steps.start_tests_debug.outputs.old_mmmap_rnd_bits }}
run: |
sudo chmod +x scripts/timeout_kill.sh
pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-debug/src/infinity | xargs echo)
sudo docker exec ${BUILDER_CONTAINER} bash -c "/infinity/scripts/timeout_kill.sh 15 ${pids}"
if [ $? -ne 0 ]; then
echo "Failed to kill infinity debug version"
exit 1
fi
sudo sysctl -w vm.mmap_rnd_bits=$old_value
- name: Collect thread sanitizer output in pytest parallel
if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
# GitHub Actions interprets output lines starting with "Error" as error messages, and it automatically sets the step status to failed when such lines are detected.
run: |
failure="${{ steps.run_tests_debug.outcome == 'failure' || steps.stop_tests_debug.outcome == 'failure' }}"
sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=debug.log --stderror_path=debug_error.log --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} --tsan_log=tsan.log
sudo rm -f tsan.log*
- name: Run restart test
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_RESTART_TEST == 'true' }}
id: run_restart_test
run : |
sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_restart_test_continuously.py --infinity_path=cmake-build-release/src/infinity"
# sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_data[infinity_runner0-columns5-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity"
# sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_index[infinity_runner0-columns2-indexes2-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity"
- name: Collect restart test output
if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_RESTART_TEST == 'true' }} # always run this step even if previous steps failed
# remove symbolic link
# find all log file like [debug.log.*] in directory, and cat to stdout
run: |
sudo python3 scripts/prepare_restart_test_data.py --op=remove
failure="${{ steps.run_restart_test.outcome == 'failure'}}"
sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
- name: Start infinity release version
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_PYTEST_PARALLEL == 'true' }}
# && !contains(github.event.pull_request.labels.*.name, 'invalid')
run: |
# Run a command in the background
sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-release/src/infinity --config=conf/pytest_parallel_continuous_conf.toml > release.log 2> release_error.log" &
- name: Run pysdk remote infinity & parallel & http_api & sqllogic test release version continously
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_PYTEST_PARALLEL == 'true' }}
# && !contains(github.event.pull_request.labels.*.name, 'invalid')
id: run_py_tests
run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_pytest_parallel_continuous.py" && sleep 1s
- name: Stop infinity release
if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_PYTEST_PARALLEL == 'true' }}
# && !contains(github.event.pull_request.labels.*.name, 'invalid')
id: stop_py_tests
run: |
sudo chmod +x scripts/timeout_kill.sh
pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-release/src/infinity | xargs echo)
sudo docker exec ${BUILDER_CONTAINER} bash -c "/infinity/scripts/timeout_kill.sh 15 ${pids}"
if [ $? -ne 0 ]; then
echo "Failed to kill infinity release version"
exit 1
fi
- name: Collect infinity release output
if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_PYTEST_PARALLEL == 'true' }} # always run this step even if previous steps failed
run: |
failure="${{ steps.run_py_tests.outcome == 'failure' || steps.stop_py_tests.outcome == 'failure' }}"
sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=release.log --stderror_path=release_error.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
- name: Destroy builder container
if: always() # always run this step even if previous steps failed
run: |
if [ -n "${BUILDER_CONTAINER}" ]; then
sudo docker rm -f -v ${BUILDER_CONTAINER}
fi