ci.yml

name: CI

on: [pull_request]

# Cancel "duplicated" workflows triggered by pushes to internal
# branches with associated PRs.
concurrency:
  group: ${{ github.ref }}-${{ github.head_ref }}-CI
  cancel-in-progress: true

env:
  CTEST_OUTPUT_ON_FAILURE: 1
  CMAKE_GENERATOR: Ninja
  CMAKE_BUILD_PARALLEL_LEVEL: 8 # num threads for build
  MACHINE_CFG: external/parthenon/cmake/machinecfg/CI.cmake
  OMPI_MCA_mpi_common_cuda_event_max: 1000
  # disable CUDA IPC within container entirely as it's causing issue for
  # unknown reasons, see https://github.com/parthenon-hpc-lab/athenapk/pull/114
  OMPI_MCA_btl_smcuda_use_cuda_ipc: 0

jobs:
  regression:
    strategy:
      matrix:
        parallel: ['serial', 'mpi']
    runs-on: [self-hosted, A100]
    container:
      image: ghcr.io/parthenon-hpc-lab/cuda11.6-noascent
      # map to local user id on CI  machine to allow writing to build cache
      options: --user 1001 --cap-add CAP_SYS_PTRACE --shm-size="8g" --ulimit memlock=134217728
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: 'true'
      - name: Configure
        run: |
          cmake -B build -DMACHINE_VARIANT=cuda-${{ matrix.parallel }}
      - name: Build
        run: cmake --build build -t athenaPK
      - name: Test
        run: |
          cd build
          # Pick GPU with most available memory
          export CUDA_VISIBLE_DEVICES=$(nvidia-smi --query-gpu=memory.free,index --format=csv,nounits,noheader | sort -nr | head -1 | awk '{ print $NF }')
          ctest -L ${{ matrix.parallel }} --timeout 3600
      - uses: actions/upload-artifact@v3
        if: ${{ always() }}
        with:
          name: regression-output
          path: |
            build/CMakeFiles/CMakeOutput.log
            build/CMakeFiles/CMakeOutput.log
            build/tst/regression/outputs/convergence
            build/tst/regression/outputs/mhd_convergence
            build/tst/regression/outputs/performance
            build/tst/regression/outputs/cluster_hse/analytic_comparison.png
            build/tst/regression/outputs/cluster_tabular_cooling/convergence.png
            build/tst/regression/outputs/aniso_therm_cond_ring_conv/ring_convergence.png
            build/tst/regression/outputs/aniso_therm_cond_gauss_conv/cond.png
            build/tst/regression/outputs/diffusion/ohm.png
            build/tst/regression/outputs/diffusion/visc.png
            build/tst/regression/outputs/field_loop/field_loop.png
            build/tst/regression/outputs/riemann_hydro/shock_tube.png
            build/tst/regression/outputs/turbulence/parthenon.out1.hst
          retention-days: 3