From 35931162e9ca38b28c09568a7c4f81be7667f7e7 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 10 Nov 2023 14:45:41 +0100 Subject: [PATCH 1/7] Update version of Github action "checkout" to v4. --- .github/workflows/publish.yml | 6 +++--- .github/workflows/test_build.yml | 6 +++--- .github/workflows/test_format.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1cf163d5f7..bfbe1cc4fe 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build package mlrl-testbed run: | python3 -m pip install --upgrade pip @@ -40,7 +40,7 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install OpenCL if: matrix.os == 'ubuntu-latest' run: | @@ -108,7 +108,7 @@ jobs: os: [ubuntu-latest] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU if: runner.os == 'Linux' uses: docker/setup-qemu-action@v2 diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 2e45064a9c..2dde0d6afe 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install OpenCL run: | sudo apt update @@ -49,7 +49,7 @@ jobs: runs-on: macos-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install OpenMP run: | brew install libomp @@ -65,7 +65,7 @@ jobs: runs-on: windows-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1 - name: Remove conflicting link.exe diff --git a/.github/workflows/test_format.yml b/.github/workflows/test_format.yml index b736bfeddc..c6dfc0b817 100644 --- a/.github/workflows/test_format.yml +++ b/.github/workflows/test_format.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check C++ code style run: | ./build test_format_cpp From c9a1b8860632ef61a623b2ceb52d206a276b07e6 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 10 Nov 2023 14:49:40 +0100 Subject: [PATCH 2/7] Update version of Github action "setup-qemu-action" to v3. --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bfbe1cc4fe..dbcedfba12 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -111,7 +111,7 @@ jobs: uses: actions/checkout@v4 - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Build package mlrl-common From 4b370bb3bf007473bf22894d979c29e88a4fbeba Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 10 Nov 2023 23:59:05 +0100 Subject: [PATCH 3/7] Update conditions for running Github workflows. --- .github/workflows/publish.yml | 2 +- .github/workflows/test_build.yml | 2 +- .github/workflows/test_format.yml | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dbcedfba12..c774227951 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: 'Publish packages' on: release: types: [ published ] - branches: [ master ] + branches: [ main ] jobs: build_pure_wheels: name: Build pure wheels diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 2dde0d6afe..8c2b942ffb 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -8,7 +8,7 @@ on: - '**/*.pyx' - '**/*.py' - '**/*.build' - - 'build' + - 'scons/**' - '**/requirements.txt' - 'doc/**' - 'python/subprojects/testbed/tests/**' diff --git a/.github/workflows/test_format.yml b/.github/workflows/test_format.yml index c6dfc0b817..77ef004099 100644 --- a/.github/workflows/test_format.yml +++ b/.github/workflows/test_format.yml @@ -5,9 +5,10 @@ on: - '**/*.hpp' - '**/*.cpp' - '**/*.py' - - 'build' + - 'scons/**' - '.clang-format' - '.isort.cfg' + - '.pylintrc' - '.style.yapf' - '.github/workflows/test_format.yml' jobs: From a4a33fe6b3e35917422fcb5da27f2d9d3606ba3b Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 11 Nov 2023 00:02:07 +0100 Subject: [PATCH 4/7] Format Github workflow definitions. --- .github/workflows/publish.yml | 3 +-- .github/workflows/test_build.yml | 24 ++++++++---------------- .github/workflows/test_format.yml | 6 ++---- 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c774227951..471939e972 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -53,8 +53,7 @@ jobs: brew link libomp --force - name: Install OpenCL if: matrix.os == 'macos-latest' - run: | - brew install opencl-clhpp-headers + run: brew install opencl-clhpp-headers - name: Prepare MSVC if: matrix.os == 'windows-latest' uses: ilammy/msvc-dev-cmd@v1 diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 8c2b942ffb..823e971a43 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -25,25 +25,19 @@ jobs: sudo apt update sudo apt install -y opencl-headers ocl-icd-opencl-dev - name: Install GoogleTest - run: | - sudo apt install -y googletest + run: sudo apt install -y googletest - name: Compile via GCC - run: | - ./build compile + run: ./build compile - name: Test C++ code - run: | - ./build tests_cpp + run: ./build tests_cpp - name: Test Python code - run: | - ./build tests_python + run: ./build tests_python - name: Install Doxygen uses: ssciwr/doxygen-install@v1 - name: Install Roboto font - run: | - sudo apt install -y fonts-roboto + run: sudo apt install -y fonts-roboto - name: Generate Documentation - run: | - ./build doc + run: ./build doc macos_build: name: Test MacOS build runs-on: macos-latest @@ -55,11 +49,9 @@ jobs: brew install libomp brew link libomp --force - name: Install OpenCL - run: | - brew install opencl-clhpp-headers + run: brew install opencl-clhpp-headers - name: Compile via Clang - run: | - TEST_SUPPORT=disabled CPLUS_INCLUDE_PATH=/usr/local/opt/opencl-clhpp-headers/include ./build compile + run: TEST_SUPPORT=disabled CPLUS_INCLUDE_PATH=/usr/local/opt/opencl-clhpp-headers/include ./build compile windows_build: name: Test Windows build runs-on: windows-latest diff --git a/.github/workflows/test_format.yml b/.github/workflows/test_format.yml index 77ef004099..03b1404e9c 100644 --- a/.github/workflows/test_format.yml +++ b/.github/workflows/test_format.yml @@ -19,8 +19,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Check C++ code style - run: | - ./build test_format_cpp + run: ./build test_format_cpp - name: Check Python code style - run: | - ./build test_format_python + run: ./build test_format_python From a5fe468e0bc04921903903a0a9b9e2822152e1de Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 11 Nov 2023 00:15:39 +0100 Subject: [PATCH 5/7] Add conditions for running individual steps of Github workflows. --- .github/workflows/test_build.yml | 68 ++++++++++++++++++++++++++----- .github/workflows/test_format.yml | 22 +++++++++- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 823e971a43..13344a493d 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -2,43 +2,87 @@ name: 'Test build' on: push: paths: - - '**/*.hpp' - - '**/*.cpp' - - '**/*.pxd' - - '**/*.pyx' - - '**/*.py' - - '**/*.build' + - '.github/workflows/test_build.yml' - 'scons/**' - - '**/requirements.txt' + - 'cpp/**' + - 'python/**' - 'doc/**' - - 'python/subprojects/testbed/tests/**' - - '.github/workflows/test_build.yml' jobs: + changes: + name: Detect changes + runs-on: ubuntu-latest + outputs: + cpp: ${{ steps.filter.outputs.cpp }} + cpp_tests: ${{ steps.filter.outputs.cpp_tests }} + python: ${{ steps.filter.outputs.python }} + python_tests: ${{ steps.filter.outputs.python_tests }} + doc: ${{ steps.filter.outputs.doc }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + cpp: &cpp + - 'scons/**' + - 'cpp/**/include/**' + - 'cpp/**/src/**' + - '**/*.pxd' + - '**/*.pyx' + - '**/meson.build' + cpp_tests: + - *cpp + - 'cpp/**/test/**' + python: &python + - 'scons/**' + - 'python/requirements.txt' + - 'python/**/mlrl/**' + python_tests: + - *python + - python/**/tests/** + doc: + - 'scons/**' + - 'doc/**' linux_build: + needs: changes name: Test Linux build runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Install OpenCL + if: ${{ needs.changes.outputs.cpp_tests == 'true' || needs.changes.outputs.python_tests == 'true' || needs.changes.outputs.doc == 'true' }} run: | sudo apt update sudo apt install -y opencl-headers ocl-icd-opencl-dev - name: Install GoogleTest + if: ${{ needs.changes.outputs.cpp_tests == 'true' || needs.changes.outputs.python_tests == 'true' || needs.changes.outputs.doc == 'true' }} run: sudo apt install -y googletest - - name: Compile via GCC - run: ./build compile - name: Test C++ code + if: ${{ needs.changes.outputs.cpp_tests == 'true' }} run: ./build tests_cpp - name: Test Python code + if: ${{ needs.changes.outputs.cpp == 'true' || needs.changes.outputs.python_tests == 'true' }} run: ./build tests_python - name: Install Doxygen + if: ${{ needs.changes.outputs.cpp == 'true' || needs.changes.outputs.doc == 'true' }} uses: ssciwr/doxygen-install@v1 + - name: Generate C++ Apidoc + if: ${{ needs.changes.outputs.cpp == 'true' }} + run: ./build apidoc_cpp + - name: Generate Python Apidoc + if: ${{ needs.changes.outputs.python == 'true' }} + run: ./build apidoc_python - name: Install Roboto font + if: ${{ needs.changes.outputs.doc == 'true' }} run: sudo apt install -y fonts-roboto - name: Generate Documentation + if: ${{ needs.changes.outputs.doc == 'true' }} run: ./build doc macos_build: + needs: changes + if: ${{ needs.changes.outputs.cpp == 'true' }} name: Test MacOS build runs-on: macos-latest steps: @@ -53,6 +97,8 @@ jobs: - name: Compile via Clang run: TEST_SUPPORT=disabled CPLUS_INCLUDE_PATH=/usr/local/opt/opencl-clhpp-headers/include ./build compile windows_build: + needs: changes + if: ${{ needs.changes.outputs.cpp == 'true' }} name: Test Windows build runs-on: windows-latest steps: diff --git a/.github/workflows/test_format.yml b/.github/workflows/test_format.yml index 03b1404e9c..64778232ba 100644 --- a/.github/workflows/test_format.yml +++ b/.github/workflows/test_format.yml @@ -2,15 +2,15 @@ name: 'Check code style' on: push: paths: + - '.github/workflows/test_format.yml' + - 'scons/**' - '**/*.hpp' - '**/*.cpp' - '**/*.py' - - 'scons/**' - '.clang-format' - '.isort.cfg' - '.pylintrc' - '.style.yapf' - - '.github/workflows/test_format.yml' jobs: test_format: name: Check code style @@ -18,7 +18,25 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Detect changes + uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + cpp: + - 'scons/**' + - '**/*.hpp' + - '**/*.cpp' + - '.clang-format' + python: + - 'scons/**' + - '**/*.py' + - '.isort.cfg' + - '.pylintrc' + - '.style.yapf' - name: Check C++ code style + if: steps.filter.outputs.cpp == 'true' run: ./build test_format_cpp - name: Check Python code style + if: steps.filter.outputs.python == 'true' run: ./build test_format_python From 205179a2e84765fd40b9c384e9604375579a2ac3 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 11 Nov 2023 23:40:58 +0100 Subject: [PATCH 6/7] Cache the git repository in Github workflows. --- .github/workflows/test_build.yml | 36 +++++++++++++++++++++++++++++++ .github/workflows/test_format.yml | 14 ++++++++++++ 2 files changed, 50 insertions(+) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 13344a493d..8c51f7dace 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -18,11 +18,19 @@ jobs: python_tests: ${{ steps.filter.outputs.python_tests }} doc: ${{ steps.filter.outputs.doc }} steps: + - name: Look up Git repository in cache + uses: actions/cache/restore@v3 + with: + path: Boomer/.git/ + key: test-build-git - name: Checkout uses: actions/checkout@v4 + with: + path: Boomer/ - uses: dorny/paths-filter@v2 id: filter with: + working-directory: Boomer/ filters: | cpp: &cpp - 'scons/**' @@ -44,13 +52,26 @@ jobs: doc: - 'scons/**' - 'doc/**' + - name: Save Git repository to cache + uses: actions/cache/save@v3 + if: always() + with: + path: Boomer/.git/ + key: test-build-git linux_build: needs: changes name: Test Linux build runs-on: ubuntu-latest steps: + - name: Look up Git repository in cache + uses: actions/cache/restore@v3 + with: + path: Boomer/.git/ + key: test-build-git - name: Checkout uses: actions/checkout@v4 + with: + path: Boomer/ - name: Install OpenCL if: ${{ needs.changes.outputs.cpp_tests == 'true' || needs.changes.outputs.python_tests == 'true' || needs.changes.outputs.doc == 'true' }} run: | @@ -86,8 +107,15 @@ jobs: name: Test MacOS build runs-on: macos-latest steps: + - name: Look up Git repository in cache + uses: actions/cache/restore@v3 + with: + path: Boomer/.git/ + key: test-build-git - name: Checkout uses: actions/checkout@v4 + with: + path: Boomer/ - name: Install OpenMP run: | brew install libomp @@ -102,8 +130,16 @@ jobs: name: Test Windows build runs-on: windows-latest steps: + - name: Look up Git repository in cache + uses: actions/cache/restore@v3 + with: + enableCrossOsArchive: true + path: Boomer/.git/ + key: test-build-git - name: Checkout uses: actions/checkout@v4 + with: + path: Boomer/ - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1 - name: Remove conflicting link.exe diff --git a/.github/workflows/test_format.yml b/.github/workflows/test_format.yml index 64778232ba..37c4f58473 100644 --- a/.github/workflows/test_format.yml +++ b/.github/workflows/test_format.yml @@ -16,12 +16,20 @@ jobs: name: Check code style runs-on: ubuntu-latest steps: + - name: Look up Git repository in cache + uses: actions/cache/restore@v3 + with: + path: Boomer/.git/ + key: test-format-git - name: Checkout uses: actions/checkout@v4 + with: + path: Boomer/ - name: Detect changes uses: dorny/paths-filter@v2 id: filter with: + working-directory: Boomer/ filters: | cpp: - 'scons/**' @@ -40,3 +48,9 @@ jobs: - name: Check Python code style if: steps.filter.outputs.python == 'true' run: ./build test_format_python + - name: Save Git repository to cache + uses: actions/cache/save@v3 + if: always() + with: + path: Boomer/.git/ + key: test-format-git From 4cf6d64b3965f5a41ff4b9b1126fec24ad107396 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 12 Nov 2023 00:23:53 +0100 Subject: [PATCH 7/7] Cache build files in Github workflows. --- .github/workflows/test_build.yml | 81 ++++++++++++++++++++++++++++--- .github/workflows/test_format.yml | 25 ++++++++-- 2 files changed, 97 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 8c51f7dace..9624b79a35 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -17,6 +17,7 @@ jobs: python: ${{ steps.filter.outputs.python }} python_tests: ${{ steps.filter.outputs.python_tests }} doc: ${{ steps.filter.outputs.doc }} + any: ${{ steps.filter.outputs.any }} steps: - name: Look up Git repository in cache uses: actions/cache/restore@v3 @@ -39,27 +40,32 @@ jobs: - '**/*.pxd' - '**/*.pyx' - '**/meson.build' - cpp_tests: + cpp_tests: &cpp_tests - *cpp - 'cpp/**/test/**' python: &python - 'scons/**' - 'python/requirements.txt' - 'python/**/mlrl/**' - python_tests: + python_tests: &python_tests - *python - python/**/tests/** - doc: + doc: &doc - 'scons/**' - 'doc/**' + any: + - *cpp_tests + - *python_tests + - *doc - name: Save Git repository to cache uses: actions/cache/save@v3 - if: always() + if: success() || failure() with: path: Boomer/.git/ key: test-build-git linux_build: needs: changes + if: ${{ needs.changes.outputs.any == 'true' }} name: Test Linux build runs-on: ubuntu-latest steps: @@ -73,13 +79,26 @@ jobs: with: path: Boomer/ - name: Install OpenCL - if: ${{ needs.changes.outputs.cpp_tests == 'true' || needs.changes.outputs.python_tests == 'true' || needs.changes.outputs.doc == 'true' }} run: | sudo apt update sudo apt install -y opencl-headers ocl-icd-opencl-dev - name: Install GoogleTest - if: ${{ needs.changes.outputs.cpp_tests == 'true' || needs.changes.outputs.python_tests == 'true' || needs.changes.outputs.doc == 'true' }} run: sudo apt install -y googletest + - name: Look up build files in cache + uses: actions/cache/restore@v3 + with: + path: | + - Boomer/venv/ + - Boomer/scons/build/ + - Boomer/cpp/build/ + - Boomer/python/**/cython/*.so* + - Boomer/python/**/build/ + - Boomer/python/**/dist/ + - Boomer/python/**/*egg-info/ + - Boomer/doc/_build/ + - Boomer/doc/_extra/ + - Boomer/doc/development/api/python/ + key: ${{ runner.os }}-test-build-tmp-files - name: Test C++ code if: ${{ needs.changes.outputs.cpp_tests == 'true' }} run: ./build tests_cpp @@ -101,6 +120,22 @@ jobs: - name: Generate Documentation if: ${{ needs.changes.outputs.doc == 'true' }} run: ./build doc + - name: Save build files to cache + uses: actions/cache/save@v3 + if: success() || failure() + with: + path: | + - Boomer/venv/ + - Boomer/scons/build/ + - Boomer/cpp/build/ + - Boomer/python/**/cython/*.so* + - Boomer/python/**/build/ + - Boomer/python/**/dist/ + - Boomer/python/**/*egg-info/ + - Boomer/doc/_build/ + - Boomer/doc/_extra/ + - Boomer/doc/development/api/python/ + key: ${{ runner.os }}-test-build-tmp-files macos_build: needs: changes if: ${{ needs.changes.outputs.cpp == 'true' }} @@ -122,8 +157,25 @@ jobs: brew link libomp --force - name: Install OpenCL run: brew install opencl-clhpp-headers + - name: Look up build files in cache + uses: actions/cache/restore@v3 + with: + path: | + - Boomer/venv/ + - Boomer/scons/build/ + - Boomer/cpp/build/ + key: ${{ runner.os }}-test-build-tmp-files - name: Compile via Clang run: TEST_SUPPORT=disabled CPLUS_INCLUDE_PATH=/usr/local/opt/opencl-clhpp-headers/include ./build compile + - name: Save build files to cache + uses: actions/cache/save@v3 + if: success() || failure() + with: + path: | + - Boomer/venv/ + - Boomer/scons/build/ + - Boomer/cpp/build/ + key: ${{ runner.os }}-test-build-tmp-files windows_build: needs: changes if: ${{ needs.changes.outputs.cpp == 'true' }} @@ -152,9 +204,26 @@ jobs: pkgs: opencl triplet: x64-windows token: ${{ github.token }} + - name: Look up build files in cache + uses: actions/cache/restore@v3 + with: + path: | + - Boomer/venv/ + - Boomer/scons/build/ + - Boomer/cpp/build/ + key: ${{ runner.os }}-test-build-tmp-files - name: Compile via MSVC run: | $env:TEST_SUPPORT = "disabled" $env:INCLUDE += ";$($pwd.Path)\vcpkg\packages\opencl_x64-windows\include" $env:LIB += ";$($pwd.Path)\vcpkg\packages\opencl_x64-windows\lib" ./build.bat compile + - name: Save build files to cache + uses: actions/cache/save@v3 + if: success() || failure() + with: + path: | + - Boomer/venv/ + - Boomer/scons/build/ + - Boomer/cpp/build/ + key: ${{ runner.os }}-test-build-tmp-files diff --git a/.github/workflows/test_format.yml b/.github/workflows/test_format.yml index 37c4f58473..f54219f85f 100644 --- a/.github/workflows/test_format.yml +++ b/.github/workflows/test_format.yml @@ -31,26 +31,45 @@ jobs: with: working-directory: Boomer/ filters: | - cpp: + cpp: &cpp - 'scons/**' - '**/*.hpp' - '**/*.cpp' - '.clang-format' - python: + python: &python - 'scons/**' - '**/*.py' - '.isort.cfg' - '.pylintrc' - '.style.yapf' + any: + - *cpp + - *python + - name: Look up build files in cache + uses: actions/cache/restore@v3 + if: steps.filter.outputs.any == 'true' + with: + path: | + - Boomer/venv/ + - Boomer/scons/build/ + key: test-format-tmp-files - name: Check C++ code style if: steps.filter.outputs.cpp == 'true' run: ./build test_format_cpp - name: Check Python code style if: steps.filter.outputs.python == 'true' run: ./build test_format_python + - name: Save build files to cache + uses: actions/cache/save@v3 + if: (success() || failure()) && steps.filter.outputs.any == 'true' + with: + path: | + - Boomer/venv/ + - Boomer/scons/build/ + key: test-format-tmp-files - name: Save Git repository to cache uses: actions/cache/save@v3 - if: always() + if: success() || failure() with: path: Boomer/.git/ key: test-format-git