From 2a91ad80af81cb3dec94e258dbfaa26a4f29fff6 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 2 Jul 2024 08:17:20 -0700 Subject: [PATCH] CI updates (#243) * update developer.yml * update Intel.yml * update Linux.yml * fix Linux.yml * update MacOS.yml * update Spack.yml * fix Spack.yml * fix developer.yml * fix Spack.yml * (re)install openblas in Spack.yml * update MacOS.yml * fix Spack.yml (grib-util +tests) * fix Spack.yml * --parallel 2 for MacOS.yml --- .github/workflows/Intel.yml | 35 ++++++--------------- .github/workflows/Linux.yml | 15 +++------ .github/workflows/MacOS.yml | 15 +++------ .github/workflows/Spack.yml | 56 ++++++++++----------------------- .github/workflows/developer.yml | 25 ++++++--------- 5 files changed, 45 insertions(+), 101 deletions(-) diff --git a/.github/workflows/Intel.yml b/.github/workflows/Intel.yml index 2c08edeb..78b962f3 100644 --- a/.github/workflows/Intel.yml +++ b/.github/workflows/Intel.yml @@ -12,49 +12,32 @@ on: branches: - develop -# Use custom shell with -l so .bash_profile is sourced which loads intel/oneapi/setvars.sh -# without having to do it in manually every step -defaults: - run: - shell: bash -leo pipefail {0} - jobs: Intel: runs-on: ubuntu-latest strategy: matrix: openmp: [ ON, OFF ] - compilers: ["CC=icc FC=ifort", "CC=icx FC=ifx"] + compilers: ["oneapi", "classic"] steps: - # See https://software.intel.com/content/www/us/en/develop/articles/oneapi-repo-instructions.html - - name: install-intel - run: | - cd /tmp - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list - sudo apt-get update - sudo apt-get install intel-oneapi-openmp intel-oneapi-compiler-fortran-2023.2.1 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.1 intel-oneapi-mkl-devel-2023.2.0 - echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile + - name: "Install Intel" + uses: NOAA-EMC/ci-install-intel-toolkit@develop + with: + compiler-setup: ${{ matrix.compilers }} - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: path: ip - name: build run: | - cd ip - mkdir build - cd build - ${{ matrix.compilers }} cmake -DOPENMP=${{ matrix.openmp }} -DBUILD_SHARED_LIBS=OFF -DBUILD_8=ON .. - make -j2 VERBOSE=1 + cmake -S ip -B ip/build -DOPENMP=${{ matrix.openmp }} -DBUILD_SHARED_LIBS=OFF -DBUILD_8=ON + cmake --build ip/build --parallel 2 --verbose - name: test run: | ulimit -s unlimited - cd $GITHUB_WORKSPACE/ip/build - ctest --verbose --output-on-failure --rerun-failed + ctest --test-dir ip/build --verbose --output-on-failure --rerun-failed diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index 95baece0..e1ec6781 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -27,19 +27,16 @@ jobs: steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: path: ip - name: build run: | sudo apt install libopenblas-serial-dev - cd ip - mkdir build - cd build - cmake -DCMAKE_PREFIX_PATH="~/" -DOPENMP=${{ matrix.openmp }} ${{ matrix.options }} -DCMAKE_INSTALL_PREFIX=~/install -DBLA_VENDOR=OpenBLAS .. - make -j2 VERBOSE=1 - make install + cmake -S ip -B ip/build -DCMAKE_PREFIX_PATH="~/" -DOPENMP=${{ matrix.openmp }} ${{ matrix.options }} -DCMAKE_INSTALL_PREFIX=~/install -DBLA_VENDOR=OpenBLAS + cmake --build ip/build --parallel 2 --verbose + cmake --install ip/build # Ensure that manual setting of '-DBLA_VENDOR=...' is reflected in output CMake config if [ $(grep -c "BLA_VENDOR OpenBLAS" ~/install/lib/cmake/ip/ip-config.cmake) -eq 0 ]; then echo "OpenBLAS not set as BLA_VENDOR in ip-config.cmake!" @@ -47,6 +44,4 @@ jobs: fi - name: test - run: | - cd $GITHUB_WORKSPACE/ip/build - ctest --verbose --output-on-failure --rerun-failed + run: ctest --test-dir ip/build --verbose --output-on-failure --rerun-failed diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml index 9a6efc9c..1fffd233 100644 --- a/.github/workflows/MacOS.yml +++ b/.github/workflows/MacOS.yml @@ -26,22 +26,17 @@ jobs: steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: path: ip - name: build run: | - cd ip - mkdir build - cd build - cmake -DOPENMP=${{ matrix.openmp }} -DBUILD_SHARED_LIBS=${{ matrix.sharedlibs }} -DCMAKE_INSTALL_PREFIX=~/install -DBUILD_8=ON .. - make -j2 VERBOSE=2 - make install + cmake -S ip -B ip/build -DOPENMP=${{ matrix.openmp }} -DBUILD_SHARED_LIBS=${{ matrix.sharedlibs }} -DCMAKE_INSTALL_PREFIX=~/install -DBUILD_8=ON -DCMAKE_BUILD_TYPE=Debug + cmake --build ip/build --parallel 2 --verbose + cmake --install ip/build ls -l ~/install ls -l ~/install/lib - name: test - run: | - cd $GITHUB_WORKSPACE/ip/build - ctest --verbose --output-on-failure --rerun-failed + run: ctest --test-dir ip/build --parallel 2 --verbose --output-on-failure --rerun-failed diff --git a/.github/workflows/Spack.yml b/.github/workflows/Spack.yml index 7b9577d8..d9c2ceae 100644 --- a/.github/workflows/Spack.yml +++ b/.github/workflows/Spack.yml @@ -25,51 +25,29 @@ jobs: runs-on: ${{ matrix.os }} steps: - - - name: checkout-ip - uses: actions/checkout@v4 - with: - path: ip - - name: spack-build-and-test - run: | - sudo apt install libopenblas-serial-dev - git clone -c feature.manyFiles=true https://github.com/spack/spack - . spack/share/spack/setup-env.sh - spack env create ip-env - spack env activate ip-env - cp $GITHUB_WORKSPACE/ip/spack/package.py $SPACK_ROOT/var/spack/repos/builtin/packages/ip/package.py - spack develop --no-clone --path $GITHUB_WORKSPACE/ip ip@develop - spack add ip@develop%gcc@11 ${{ matrix.variants }} target=x86_64 - precision=$(echo ${{ matrix.variants }} | grep -oP " precision=\K[4d8]") - if [ "$precision" == "d" ]; then spack add grib-util@develop ; fi - spack external find cmake gmake - spack external find --path /usr/lib/x86_64-linux-gnu/openblas-serial openblas - spack config add "packages:lapack:buildable:false" - spack concretize - # Run installation and run CTest suite - spack install --fail-fast --test root - # Run 'spack load' and check that key build options were respected - spack load ip - if [[ "${{ matrix.variants }}" =~ "+shared" ]]; then suffix="so" ; else suffix="a"; fi - libvar=IP_LIB${precision} - ls ${!libvar} | grep -cE "/libip_${precision}\."$suffix'$' + - name: "Install OpenBLAS" + run: sudo apt install libopenblas-serial-dev + + - name: "Build Spack package" + uses: NOAA-EMC/ci-test-spack-package@develop + with: + package-name: ip + package-variants: ${{ matrix.variants }} ${{ matrix.variants == '+openmp +shared +pic precision=d' && 'grib-util@develop +tests ^g2c@develop +utils +build_v2_api' || '' }} + custom-recipe: spack/package.py + spack-compiler: gcc + spack-externals: gmake cmake openblas + repo-cache-key-suffix: ${{ matrix.os }}-${{ matrix.variants }}-1 # This job validates the Spack recipe by making sure each cmake build option is represented recipe-check: runs-on: ubuntu-latest steps: - - - name: checkout-ip - uses: actions/checkout@v4 - with: - path: ip - name: recipe-check - run: | - echo "If this jobs fails, look at the most recently output CMake option below and make sure that option appears in spack/package.py" - for opt in $(grep -ioP '^option\(\K(?!(ENABLE_DOCS|TEST_TIME_LIMIT))[^ ]+' $GITHUB_WORKSPACE/ip/CMakeLists.txt) ; do - echo "Checking for presence of '$opt' CMake option in package.py" - grep -cP "define.+\b${opt}\b" $GITHUB_WORKSPACE/ip/spack/package.py - done + uses: NOAA-EMC/ci-check-spack-recipe@develop + with: + recipe-file: package/spack/package.py + cmakelists-txt: package/CMakeLists.txt + ignore-list: ENABLE_DOCS,TEST_TIME_LIMIT diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index cd802787..852d4bf1 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -29,37 +29,30 @@ jobs: python3 -m pip install gcovr - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: path: ip - name: build run: | - cd ip - mkdir build - cd build - cmake -DENABLE_DOCS=YES -DOPENMP=ON -DCMAKE_Fortran_FLAGS="-g -fprofile-abs-path -fprofile-arcs -ftest-coverage -O0 -fsanitize=address" -DCMAKE_C_FLAGS="-g -fprofile-abs-path -fprofile-arcs -ftest-coverage -O0 -fsanitize=address" -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DBUILD_8=ON .. - make -j2 VERBOSE=1 + cmake -S ip -B ip/build -DENABLE_DOCS=YES -DOPENMP=ON -DCMAKE_Fortran_FLAGS="-g -fprofile-abs-path -fprofile-arcs -ftest-coverage -O0 -fsanitize=address" -DCMAKE_C_FLAGS="-g -fprofile-abs-path -fprofile-arcs -ftest-coverage -O0 -fsanitize=address" -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DBUILD_8=ON + cmake --build ip/build --parallel 2 --verbose - name: test - run: | - cd $GITHUB_WORKSPACE/ip/build - ctest -j2 --verbose --output-on-failure --rerun-failed + run: ctest --test-dir ip/build -j2 --verbose --output-on-failure --rerun-failed - name: run-gcovr - run: | - cd $GITHUB_WORKSPACE/ip/build - gcovr -r .. -v --html-details --exclude ../tests --exclude CMakeFiles --print-summary -o test-coverage.html + run: gcovr -r ip -v --html-details --exclude ip/tests --exclude ip/build/CMakeFiles --print-summary -o test-coverage.html - name: upload-test-coverage - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: ip-test-coverage path: | - ip/build/*.html - ip/build/*.css + *.html + *.css - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: docs path: |