Merge remote-tracking branch 'origin/development' into meb_kappa_sandbox

Still need to check regtests

 Conflicts:
	src/convection/incflo_compute_advection_term.cpp
	src/diffusion/DiffusionScalarOp.cpp
	src/diffusion/DiffusionTensorOp.cpp
	src/incflo.H
	src/incflo.cpp
	src/incflo_redistribute.cpp

.github/workflows/cleanup-cache-postpr.yml

@@ -0,0 +1,37 @@
+name: CleanUpCachePostPR
+
+on:
+ workflow_run:
+ workflows: [PostPR]
+ types:
+ - completed
+
+jobs:
+ CleanUpCcacheCachePostPR:
+ name: Clean Up Ccahe Cache Post PR
+ runs-on: ubuntu-latest
+ permissions:
+ actions: write
+ contents: read
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - uses: actions/checkout@v3
+ - name: Clean up ccahe
+ run: |
+ gh extension install actions/gh-actions-cache
+
+ REPO=${{ github.repository }}
+
+ gh run download ${{ github.event.workflow_run.id }} -n pr_number
+ pr_number=`cat pr_number.txt`
+ BRANCH=refs/pull/${pr_number}/merge
+
+ # Setting this to not fail the workflow while deleting cache keys.
+ set +e
+
+ keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1)
+ for k in $keys
+ do
+ gh actions-cache delete $k -R $REPO -B $BRANCH --confirm
+ done

.github/workflows/cleanup-cache.yml

@@ -0,0 +1,60 @@
+name: CleanUpCache
+
+on:
+ workflow_run:
+ workflows: [GCC,CUDA,HIP,SYCL]
+ types:
+ - completed
+
+jobs:
+ CleanUpCcacheCache:
+ name: Clean Up Ccahe Cache for ${{ github.event.workflow_run.name }}
+ runs-on: ubuntu-latest
+ permissions:
+ actions: write
+ contents: read
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - uses: actions/checkout@v3
+ - name: Clean up ccahe
+ run: |
+ gh extension install actions/gh-actions-cache
+
+ REPO=${{ github.repository }}
+
+ # push or pull_request or schedule or ...
+ EVENT=${{ github.event.workflow_run.event }}
+
+ # Triggering workflow run name (e.g., LinuxClang)
+ WORKFLOW_NAME=${{ github.event.workflow_run.name }}
+
+ if [[ $EVENT == "pull_request" ]]; then
+ gh run download ${{ github.event.workflow_run.id }} -n pr_number
+ pr_number=`cat pr_number.txt`
+ BRANCH=refs/pull/${pr_number}/merge
+ else
+ BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }}
+ fi
+
+ # Setting this to not fail the workflow while deleting cache keys.
+ set +e
+
+ # In our cache keys, substring after `-git-` is git hash, substring
+ # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`).
+ # The goal is to keep the last used key of each job and delete all others.
+
+ # something like ccache-LinuxClang-
+ keyprefix=ccache-${WORKFLOW_NAME}-
+
+ cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key $keyprefix | awk -F '-git-' '{print $1}' | sort | uniq)
+
+ # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d".
+ for j in $cached_jobs
+ do
+ old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key ${j}-git- --sort last-used | cut -f 1 | tail -n +2)
+ for k in $old_keys
+ do
+ gh actions-cache delete $k -R $REPO -B $BRANCH --confirm
+ done
+ done

.github/workflows/cuda.yml

@@ -0,0 +1,158 @@
+---
+name: CUDA
+
+on:
+ push:
+ pull_request:
+ schedule:
+ - cron: "13 8 * * *"
+
+concurrency:
+ group: ${{ github.ref }}-${{ github.head_ref }}-CUDA
+ cancel-in-progress: true
+
+jobs:
+ cuda_eb_3d:
+ name: CUDA EB 3D
+ runs-on: ubuntu-20.04
+ env: {CXXFLAGS: "-Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs"}
+ steps:
+ - name: Checkout incflo
+ uses: actions/checkout@v3
+ with:
+ path: incflo
+ - name: Checkout AMReX-Hydro
+ uses: actions/checkout@v3
+ with:
+ repository: AMReX-Codes/AMReX-Hydro
+ path: AMReX-Hydro
+ - name: Checkout AMReX
+ uses: actions/checkout@v3
+ with:
+ repository: AMReX-Codes/amrex
+ path: amrex
+ - name: Set Up Cache
+ uses: actions/cache@v3
+ with:
+ path: ~/.cache/ccache
+ key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+ restore-keys: |
+ ccache-${{ github.workflow }}-${{ github.job }}-git-
+ - name: Load Dependencies
+ run: |
+ amrex/.github/workflows/dependencies/dependencies_nvcc.sh 11.2
+ amrex/.github/workflows/dependencies/dependencies_ccache.sh
+ - name: Build
+ run: |
+ export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
+ export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
+ nvcc --version
+ g++ --version
+
+ export CCACHE_COMPRESS=1
+ export CCACHE_COMPRESSLEVEL=10
+ export CCACHE_MAXSIZE=500M
+ ccache -z
+
+ cd incflo
+ mkdir build
+ cd build
+ cmake .. \
+ -DINCFLO_DIM=3 \
+ -DINCFLO_MPI=ON \
+ -DINCFLO_OMP=OFF \
+ -DINCFLO_EB=ON \
+ -DINCFLO_CUDA=ON \
+ -DAMReX_GPU_BACKEND=CUDA \
+ -DAMReX_CUDA_ARCH=70 \
+ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
+ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON \
+ -DAMREX_HOME=${{ github.workspace }}/amrex \
+ -DAMREX_HYDRO_HOME=${{ github.workspace }}/AMReX-Hydro \
+ -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+ make -j 2
+
+ ccache -s
+ du -hs ~/.cache/ccache
+
+ cuda_no_eb_2d:
+ name: CUDA NO EB 2D
+ runs-on: ubuntu-20.04
+ env: {CXXFLAGS: "-Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs"}
+ steps:
+ - name: Checkout incflo
+ uses: actions/checkout@v3
+ with:
+ path: incflo
+ - name: Checkout AMReX-Hydro
+ uses: actions/checkout@v3
+ with:
+ repository: AMReX-Codes/AMReX-Hydro
+ path: AMReX-Hydro
+ - name: Checkout AMReX
+ uses: actions/checkout@v3
+ with:
+ repository: AMReX-Codes/amrex
+ path: amrex
+ - name: Set Up Cache
+ uses: actions/cache@v3
+ with:
+ path: ~/.cache/ccache
+ key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+ restore-keys: |
+ ccache-${{ github.workflow }}-${{ github.job }}-git-
+ - name: Load Dependencies
+ run: |
+ amrex/.github/workflows/dependencies/dependencies_nvcc.sh 11.2
+ amrex/.github/workflows/dependencies/dependencies_ccache.sh
+ - name: Build
+ run: |
+ export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
+ export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
+ nvcc --version
+ g++ --version
+
+ export CCACHE_COMPRESS=1
+ export CCACHE_COMPRESSLEVEL=10
+ export CCACHE_MAXSIZE=400M
+ ccache -z
+
+ cd incflo
+ mkdir build
+ cd build
+ cmake .. \
+ -DINCFLO_DIM=2 \
+ -DINCFLO_MPI=ON \
+ -DINCFLO_OMP=OFF \
+ -DINCFLO_EB=OFF \
+ -DINCFLO_CUDA=ON \
+ -DAMReX_GPU_BACKEND=CUDA \
+ -DAMReX_CUDA_ARCH=70 \
+ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
+ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON \
+ -DAMREX_HOME=${{ github.workspace }}/amrex \
+ -DAMREX_HYDRO_HOME=${{ github.workspace }}/AMReX-Hydro \
+ -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+ make -j 2
+
+ ccache -s
+ du -hs ~/.cache/ccache
+
+ save_pr_number:
+ if: github.event_name == 'pull_request'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Save PR number
+ env:
+ PR_NUMBER: ${{ github.event.number }}
+ run: |
+ echo $PR_NUMBER > pr_number.txt
+ - uses: actions/upload-artifact@v3
+ with:
+ name: pr_number
+ path: pr_number.txt
+ retention-days: 1