From 912bd2cf3a20417ce0c3a3b5254229285deec1d7 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Fri, 30 Aug 2024 19:20:22 +0900 Subject: [PATCH 1/8] workflows: aarch64: linux: Compile and execute internal test on QEMU aarch64 Linux Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 61355f4a977..2ce789a2bf4 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -110,6 +110,52 @@ jobs: CXX: g++ FLB_OPT: ${{ matrix.flb_option }} + + run-qemu-aarch64-unit-tests: + # We chain this after Linux one as there are CPU time costs for QEMU emulation + needs: + - run-ubuntu-unit-tests + runs-on: ubuntu-20.04 + timeout-minutes: 60 + strategy: + fail-fast: false + steps: + - name: Checkout Fluent Bit code + uses: actions/checkout@v4 + + - name: Prepare and build with QEMU aarch64 + uses: uraimo/run-on-arch-action@v2 + id: aarch64-build-on-qemu-aarch64 + with: + arch: aarch64 + distro: ubuntu20.04 + shell: /bin/bash + dockerRunArgs: | + --volume "/var/lib/dbus/machine-id:/var/lib/dbus/machine-id" + --volume "/etc/machine-id:/etc/machine-id" + install: | + apt-get update + apt-get install -y gcc-7 g++-7 clang-6.0 libyaml-dev cmake flex bison libssl-dev #libsystemd-dev + ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true + + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90 + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90 + run: | + cd build + export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) )) + export FLB_OPTION="-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1" + export GLOBAL_OPTION="-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off" + export FLB_UNIT_TEST_OPTION="-DFLB_TESTS_INTERNAL=On" + export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION}" + + echo "CC = gcc, CXX = gcc, FLB_OPT = $FLB_OPT" + + cmake ${FLB_OPT} ../ + make -j $nparallel + ctest -j $nparallel --build-run-dir . --output-on-failure + + # Required check looks at this so do not remove run-all-unit-tests: if: always() @@ -120,6 +166,7 @@ jobs: needs: - run-macos-unit-tests - run-ubuntu-unit-tests + - run-qemu-aarch64-unit-tests steps: - name: Check build matrix status # Ignore MacOS failures From 6d7ae8f05c99a91aa5b86ecb18897e60654ec557 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Mon, 2 Sep 2024 12:14:26 +0900 Subject: [PATCH 2/8] workflows: aarch64: Omit to execute flb-it-utils and flb-it-pack tests Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 2ce789a2bf4..9d4dcefb0c2 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -145,9 +145,10 @@ jobs: cd build export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) )) export FLB_OPTION="-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1" + export FLB_OMIT_OPTION="-DFLB_WITHOUT_flb-it-utils=1 -DFLB_WITHOUT_flb-it-pack=1" export GLOBAL_OPTION="-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off" export FLB_UNIT_TEST_OPTION="-DFLB_TESTS_INTERNAL=On" - export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION}" + export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}" echo "CC = gcc, CXX = gcc, FLB_OPT = $FLB_OPT" From 79cf99b614644a7326c1c8aefcf3a22e954e2ca0 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 3 Sep 2024 14:28:07 +0900 Subject: [PATCH 3/8] workflows: aarch64: Tweak for easily to extend for other non-amd64 platforms Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 9d4dcefb0c2..24cb430c62e 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -111,7 +111,7 @@ jobs: FLB_OPT: ${{ matrix.flb_option }} - run-qemu-aarch64-unit-tests: + run-qemu-ubuntu-unit-tests: # We chain this after Linux one as there are CPU time costs for QEMU emulation needs: - run-ubuntu-unit-tests @@ -119,15 +119,18 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false + matrix: + arch: + - aarch64 steps: - name: Checkout Fluent Bit code uses: actions/checkout@v4 - - name: Prepare and build with QEMU aarch64 + - name: Prepare and build with QEMU ${{ matrix.arch }} uses: uraimo/run-on-arch-action@v2 - id: aarch64-build-on-qemu-aarch64 + id: build-and-test-on-qemu with: - arch: aarch64 + arch: ${{ matrix.arch }} distro: ubuntu20.04 shell: /bin/bash dockerRunArgs: | @@ -167,7 +170,7 @@ jobs: needs: - run-macos-unit-tests - run-ubuntu-unit-tests - - run-qemu-aarch64-unit-tests + - run-qemu-ubuntu-unit-tests steps: - name: Check build matrix status # Ignore MacOS failures From 177c89e5f5c3da0ad7de6b9bd971c4d7a56f2980 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 4 Sep 2024 15:06:36 +0900 Subject: [PATCH 4/8] workflows: aarch64: Add actuated runner settings for aarch64 Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 24cb430c62e..14362d4252e 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -110,6 +110,63 @@ jobs: CXX: g++ FLB_OPT: ${{ matrix.flb_option }} + run-aarch64-unit-tests: + # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage. + runs-on: ${{(github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb' || 'ubuntu-latest' }} + permissions: + contents: read + # We chain this after Linux one as there are costs for actuated workers + needs: + - run-ubuntu-unit-tests + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + config: + - name: "Aarch64 actuated testing" + flb_option: "-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1" + omit_option: "-DFLB_WITHOUT_flb-it-utils=1 -DFLB_WITHOUT_flb-it-pack=1" + global_option: "-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off" + unit_test_option: "-DFLB_TESTS_INTERNAL=On" + compiler: gcc + steps: + - name: Checkout Fluent Bit code + uses: actions/checkout@v4 + + - name: Set up Actuated mirror + if: github.repository == 'fluent/fluent-bit' + uses: self-actuated/hub-mirror@master + + - name: Setup environment + if: github.repository == 'fluent/fluent-bit' + run: | + sudo apt-get update + sudo apt-get install -y gcc-9 g++-9 clang-12 cmake flex bison libsystemd-dev gcovr libyaml-dev + sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true + + - name: Build and test with actuated runners + if: github.repository == 'fluent/fluent-bit' + run: | + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 90 + + export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) )) + export FLB_OPTION="${{ matrix.config.flb_option }}" + export FLB_OMIT_OPTION="${{ matrix.config.omit_option }}" + export GLOBAL_OPTION="${{ matrix.config.global_option }}" + export FLB_UNIT_TEST_OPTION="${{ matrix.config.unit_test_option }}" + export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}" + + echo "CC = ${{ matrix.config.compiler }}, CXX = ${{ matrix.config.compiler }}, FLB_OPT = $FLB_OPT" + + cmake ${FLB_OPT} ../ + make -j $nparallel + ctest -j $nparallel --build-run-dir . --output-on-failure + working-directory: build + env: + CC: ${{ matrix.config.compiler }} + CXX: ${{ matrix.config.compiler }} run-qemu-ubuntu-unit-tests: # We chain this after Linux one as there are CPU time costs for QEMU emulation @@ -170,6 +227,7 @@ jobs: needs: - run-macos-unit-tests - run-ubuntu-unit-tests + - run-aarch64-unit-tests - run-qemu-ubuntu-unit-tests steps: - name: Check build matrix status From a85a6fe69b524e08f2436258527e20b77dbf274e Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 4 Sep 2024 15:07:00 +0900 Subject: [PATCH 5/8] workflows: Reuse qemu settings for s390x CI task Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 14362d4252e..0779427ffec 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -178,7 +178,7 @@ jobs: fail-fast: false matrix: arch: - - aarch64 + - s390x steps: - name: Checkout Fluent Bit code uses: actions/checkout@v4 @@ -209,8 +209,10 @@ jobs: export GLOBAL_OPTION="-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off" export FLB_UNIT_TEST_OPTION="-DFLB_TESTS_INTERNAL=On" export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}" + export CC=gcc + export CXX=gcc - echo "CC = gcc, CXX = gcc, FLB_OPT = $FLB_OPT" + echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT" cmake ${FLB_OPT} ../ make -j $nparallel From 0e155c6789fb3c2e62995381d6a8e5095bd8729a Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 4 Sep 2024 16:45:42 +0900 Subject: [PATCH 6/8] workflows: s390x: Remove omit option to execute pack and utils internal tests Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 0779427ffec..79777986473 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -205,7 +205,7 @@ jobs: cd build export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) )) export FLB_OPTION="-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1" - export FLB_OMIT_OPTION="-DFLB_WITHOUT_flb-it-utils=1 -DFLB_WITHOUT_flb-it-pack=1" + export FLB_OMIT_OPTION="" export GLOBAL_OPTION="-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off" export FLB_UNIT_TEST_OPTION="-DFLB_TESTS_INTERNAL=On" export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}" From 70303027ef809cd7211dfc9a379d9e8375dd9ded Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Wed, 4 Sep 2024 21:13:13 +0900 Subject: [PATCH 7/8] workflows: Remove needless conditions Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 79777986473..4a970980e74 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -138,14 +138,12 @@ jobs: uses: self-actuated/hub-mirror@master - name: Setup environment - if: github.repository == 'fluent/fluent-bit' run: | sudo apt-get update sudo apt-get install -y gcc-9 g++-9 clang-12 cmake flex bison libsystemd-dev gcovr libyaml-dev sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true - name: Build and test with actuated runners - if: github.repository == 'fluent/fluent-bit' run: | sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90 sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90 From ab9f9d6b3f4141e3943e182a78685b1422542a27 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 5 Sep 2024 11:33:52 +0900 Subject: [PATCH 8/8] workflows: aarch64: Shorten the timeout from 60mins. to 10mins. Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 4a970980e74..0fa50f11890 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -118,7 +118,7 @@ jobs: # We chain this after Linux one as there are costs for actuated workers needs: - run-ubuntu-unit-tests - timeout-minutes: 60 + timeout-minutes: 10 strategy: fail-fast: false matrix: