Merge branch 'main' into exacb

.devcontainer/devcontainer.json

@@ -0,0 +1,40 @@
+// devcontainer.json
+    {
+      "name": "athenapk-dev",
+      "image": "ghcr.io/parthenon-hpc-lab/cuda11.6-mpi-hdf5-ascent",
+      // disable Dockerfile for now
+      //"build": {
+      //  // Path is relative to the devcontainer.json file.
+      //  "dockerfile": "Dockerfile"
+      //},
+      "hostRequirements": {
+        "cpus": 4
+      },
+      "customizations": {
+        "vscode": {
+          "settings": {},
+          "extensions": [
+            "-ms-vscode.cpptools",
+            "llvm-vs-code-extensions.vscode-clangd",
+            "github.vscode-pull-request-github",
+            "ms-python.python",
+            "ms-toolsai.jupyter",
+            "ms-vscode.live-server",
+            "ms-azuretools.vscode-docker",
+            "swyddfa.esbonio",
+            "tomoki1207.pdf",
+            "ms-vscode.cmake-tools",
+            "ms-vsliveshare.vsliveshare"
+        ]
+        }
+      },
+      "remoteEnv": {
+        "PATH": "${containerEnv:PATH}:/usr/local/hdf5/parallel/bin",
+        "OMPI_MCA_opal_warn_on_missing_libcuda": "0"
+      },
+      //"remoteUser": "ubuntu",
+      // we need to manually checkout the submodules,
+      // but VSCode may try to configure CMake before they are fully checked-out.
+      // workaround TBD
+      "postCreateCommand": "git submodule update --init"
+    }

.github/workflows/ci.yml

@@ -14,6 +14,9 @@ env:
   CMAKE_BUILD_PARALLEL_LEVEL: 8 # num threads for build
   MACHINE_CFG: external/parthenon/cmake/machinecfg/CI.cmake
   OMPI_MCA_mpi_common_cuda_event_max: 1000
+  # disable CUDA IPC within container entirely as it's causing issue for
+  # unknown reasons, see https://github.com/parthenon-hpc-lab/athenapk/pull/114
+  OMPI_MCA_btl_smcuda_use_cuda_ipc: 0
 
 jobs:
   regression:
@@ -22,9 +25,9 @@ jobs:
         parallel: ['serial', 'mpi']
     runs-on: [self-hosted, A100]
     container:
-      image: ghcr.io/parthenon-hpc-lab/cuda11.6-mpi-hdf5-ascent
+      image: ghcr.io/parthenon-hpc-lab/cuda11.6-noascent
       # map to local user id on CI  machine to allow writing to build cache
-      options: --user 1001
+      options: --user 1001 --cap-add CAP_SYS_PTRACE --shm-size="8g" --ulimit memlock=134217728
     steps:
       - uses: actions/checkout@v3
         with:
@@ -53,8 +56,9 @@ jobs:
             build/tst/regression/outputs/cluster_hse/analytic_comparison.png
             build/tst/regression/outputs/cluster_tabular_cooling/convergence.png
             build/tst/regression/outputs/aniso_therm_cond_ring_conv/ring_convergence.png
+            build/tst/regression/outputs/aniso_therm_cond_gauss_conv/cond.png
             build/tst/regression/outputs/field_loop/field_loop.png
             build/tst/regression/outputs/riemann_hydro/shock_tube.png
-            build/tst/regression/outputs/turbulence/parthenon.hst
+            build/tst/regression/outputs/turbulence/parthenon.out1.hst
           retention-days: 3
 

CHANGELOG.md

@@ -3,19 +3,35 @@
 ## Current develop (i.e., `main` branch)
 
 ### Added (new features/APIs/variables/...)
+- [[PR 1]](https://github.com/parthenon-hpc-lab/athenapk/pull/1) Add isotropic thermal conduction and RKL2 supertimestepping
 
 ### Changed (changing behavior/API/variables/...)
+- [[PR 97]](https://github.com/parthenon-hpc-lab/athenapk/pull/97) Fixed Schure cooling curve. Removed SD one. Added description of cooling function conventions.
 - [[PR 84]](https://github.com/parthenon-hpc-lab/athenapk/pull/84) Bump Parthenon to latest develop (2024-02-15)
 
 ### Fixed (not changing behavior/API/variables/...)
 
 ### Infrastructure
+- [[PR 114]](https://github.com/parthenon-hpc-lab/athenapk/pull/114) Bump Parthenon 24.08 and Kokkos to 4.4.00
+- [[PR 112]](https://github.com/parthenon-hpc-lab/athenapk/pull/112) Add dev container configuration
 - [[PR 105]](https://github.com/parthenon-hpc-lab/athenapk/pull/105) Bump Parthenon to latest develop (2024-03-13)
 - [[PR 84]](https://github.com/parthenon-hpc-lab/athenapk/pull/84) Added `CHANGELOG.md`
 
 ### Removed (removing behavior/API/varaibles/...)
 
 ### Incompatibilities (i.e. breaking changes)
+- [[PR 114]](https://github.com/parthenon-hpc-lab/athenapk/pull/114) Bump Parthenon 24.08 and Kokkos to 4.4.00
+  - Changed signature of `UserWorkBeforeOutput` to include `SimTime` as last paramter
+  - Fixes bitwise idential restarts for AMR simulations (the derefinement counter is now included)
+  - Order of operations in flux-correction has changed (expect round-off error differences to previous results for AMR sims)
+  - History outputs now carry the output block number, i.e., a file previously called parthenon.hst might now be called parthenon.out1.hst
+  - History outputs now contain two additional columns (cycle number and meshblock counts), which changes/shifts the column indices (hint: use the column headers to parse the contents and do not rely on fixed indices as they may also vary between different pgen due to custom/pgen-dependent content in the history file)
+  - Given the introduction of a forest of tree (rather than a single tree), the logical locations are each meshblock (`pmb->loc`) are now local to the tree and not global any more. To recover the original global index use `auto loc = pmb->pmy_mesh->Forest().GetLegacyTreeLocation(pmb->loc);`
+- [[PR 97]](https://github.com/parthenon-hpc-lab/athenapk/pull/97)
+  - Removes original `schure.cooling` cooling curve as it had unknown origin.
+  - To avoid confusion, only cooling table for a single solar metallicity are supported
+    from now on (i.e., the parameters to specify temperature and lambda columns have been removed).
+  - Added `schure.cooling_#Z` curves (and associated notebook to calculate it from the paper tables).
 - [[PR 84]](https://github.com/parthenon-hpc-lab/athenapk/pull/84) Bump Parthenon to latest develop (2024-02-15)
   - Updated access to block dimension: `pmb->block_size.nx1` -> `pmb->block_size.nx(X1DIR)` (and similarly x2 and x3)
   - Update access to mesh size: `pmesh->mesh_size.x1max` -> `pmesh->mesh_size.xmax(X1DIR)` (and similarly x2, x3, and min)

README.md

@@ -14,7 +14,8 @@ Current features include
   - HLLE (hydro and MHD), HLLC (hydro), and HLLD (MHD) Riemann solvers
   - adiabatic equation of state
   - MHD based on hyperbolic divergence cleaning following Dedner+ 2002
-  - anisotropic thermal conduction
+  - isotropic and anisotropic thermal conduction
+  - operator-split, second-order RKL2 supertimestepping for diffusive terms
   - optically thin cooling based on tabulated cooling tables with either Townsend 2009 exact integration or operator-split subcycling
 - static and adaptive mesh refinement
 - problem generators for