merge commit

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   Formatting:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Clone
         uses: actions/checkout@v4
@@ -27,7 +27,7 @@ jobs:
 
   Codespell:
     needs: Formatting
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Clone
         uses: actions/checkout@v4
@@ -46,7 +46,7 @@ jobs:
         run: codespell
 
   CEPTR:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -81,7 +81,7 @@ jobs:
 
   Transport-EOS-Reactions:
     needs: Formatting
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         comp: [gnu, llvm, cuda, hip, sycl]
@@ -131,9 +131,9 @@ jobs:
           echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
           echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
           if [ "${{matrix.comp}}" == 'cuda' ]; then \
-            echo "CUDA_HOME=/usr/local/cuda-11.2" >> $GITHUB_ENV; \
-            echo "LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV; \
-            echo "/usr/local/cuda-11.2/bin" >> $GITHUB_PATH; \
+            echo "CUDA_HOME=/usr/local/cuda-12.6" >> $GITHUB_ENV; \
+            echo "LD_LIBRARY_PATH=/usr/local/cuda-12.6/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV; \
+            echo "/usr/local/cuda-12.6/bin" >> $GITHUB_PATH; \
           fi
       - name: Install Ccache
         run: |
@@ -150,6 +150,7 @@ jobs:
       - name: Dependencies
         working-directory: ${{env.TRANSPORT_WORKING_DIRECTORY}}
         run: |
+          ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh
           if [ "${{matrix.comp}}" == 'cuda' ]; then \
             ${{github.workspace}}/PelePhysics-${{matrix.comp}}/${{matrix.dependency_cmds}}; \
             echo "CUDA_HOME=${CUDA_HOME}"; \

.github/workflows/cleanup-cache-postpr.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   CleanUpCcacheCachePostPR:
     name: Clean Up Ccahe Cache Post PR
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       actions: write
       contents: read

.github/workflows/cleanup-cache.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   CleanUpCcacheCache:
     name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       actions: write
       contents: read

.github/workflows/dependencies/dependencies_cuda.sh

@@ -1,21 +1,11 @@
 #!/usr/bin/env bash
-#
-# Copyright 2020 Axel Huebl
-#
-# License: BSD-3-Clause-LBNL
-
-# search recursive inside a folder if a file contains tabs
-#
-# @result 0 if no files are found, else 1
-#
 
 set -eu -o pipefail
 
-CUDA_VERSION=11-2
+CUDA_VERSION=12-6
 
-sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
-echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" \
-    | sudo tee /etc/apt/sources.list.d/cuda.list
+wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
+sudo dpkg -i cuda-keyring_1.1-1_all.deb
 sudo apt-get update
 sudo apt-get install -y \
     cuda-command-line-tools-${CUDA_VERSION} \
@@ -28,8 +18,9 @@ sudo apt-get install -y \
     libcusolver-dev-${CUDA_VERSION}         \
     libcusparse-dev-${CUDA_VERSION}         \
     libcublas-dev-${CUDA_VERSION}           \
-    libcurand-dev-${CUDA_VERSION}
+    libcurand-dev-${CUDA_VERSION}           \
+    libnvjitlink-${CUDA_VERSION}
 
-export PATH=/usr/local/cuda-11.2/bin:${PATH}
+export PATH=/usr/local/nvidia/bin:/usr/local/cuda-12.6/bin:${PATH}
 which nvcc
 nvcc --version

.github/workflows/dependencies/dependencies_dpcpp.sh

@@ -1,18 +1,8 @@
 #!/usr/bin/env bash
-#
-# Copyright 2020 The AMReX Community
-#
-# License: BSD-3-Clause-LBNL
-# Authors: Axel Huebl
-
 set -eu -o pipefail
 
 sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-echo "deb https://apt.repos.intel.com/oneapi all main" \
-  | sudo tee /etc/apt/sources.list.d/oneAPI.list
-
+echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
 sudo apt-get update
-sudo apt-get install -y \
-    intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 \
-    intel-oneapi-mkl-devel=2024.2.2-15
+sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 intel-oneapi-mkl-devel=2024.2.2-15

.github/workflows/dependencies/dependencies_hip.sh

@@ -1,30 +1,18 @@
 #!/usr/bin/env bash
-#
-# Copyright 2020 The AMReX Community
-#
-# License: BSD-3-Clause-LBNL
-# Authors: Axel Huebl
-
-# search recursive inside a folder if a file contains tabs
-#
-# @result 0 if no files are found, else 1
-#
-
 set -eu -o pipefail
 
-sudo wget https://repo.radeon.com/rocm/rocm.gpg.key
-sudo apt-key add rocm.gpg.key
-echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/5.6.1 ubuntu main' \
-  | sudo tee /etc/apt/sources.list.d/rocm.list
-echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
-  | sudo tee -a /etc/profile.d/rocm.sh
-
+sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.2.1 noble main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh
 sudo apt-get update
-sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev
+sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev hiprand-dev
 
 source /etc/profile.d/rocm.sh
 which hipcc
 which clang
 which clang++
 which flang
 hipcc --version
+hipconfig --full

.github/workflows/docs.yml

@@ -16,7 +16,7 @@ on:
 
 jobs:
   Docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Clone
         uses: actions/checkout@v4

.github/workflows/post-pr.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   cleanup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Save PR number
         env:

Source/Spray/SprayIO.cpp

@@ -146,10 +146,12 @@ SprayParticleContainer::PostInitRestart(const std::string& dir)
       Vector<Real> fluentdpm_inj_init_dpm_time(in_numjets);
       Vector<Real> fluentdpm_inj_nxt_flw_time(in_numjets);
       Vector<Real> fluentdpm_inj_nxt_dpm_time(in_numjets);
+
       for (int i = 0; i < in_numjets; ++i) {
         JetDataFile >> in_jet_names[i] >> in_inj_ppp[i] >> in_inj_mass[i] >>
           in_inj_time[i] >> in_min_parcel[i] >> in_total_mass[i] >>
           in_total_time[i];
+
         std::string in_name = in_jet_names[i];
 
         bool jet_is_fluent_dpm_based = false;
@@ -171,6 +173,7 @@ SprayParticleContainer::PostInitRestart(const std::string& dir)
             break;
           }
         }
+
       }
       for (int ijets = 0; ijets < in_numjets; ++ijets) {
         std::string in_name = in_jet_names[ijets];

Source/Spray/SprayJet.H

@@ -40,33 +40,34 @@ public:
 
   virtual ~SprayJet() = default;
 
-  inline const amrex::RealVect& jet_norm() const { return m_norm; }
-  inline const amrex::RealVect& jet_cent() const { return m_cent; }
-  inline amrex::Real jet_vel(const amrex::Real time) const
+  const amrex::RealVect& jet_norm() const { return m_norm; }
+  const amrex::RealVect& jet_cent() const { return m_cent; }
+  amrex::Real jet_vel(const amrex::Real time) const
+
   {
     if (m_useROI) {
       return interpolateVel(time);
     }
     return m_jetVel;
   }
-  inline const amrex::Real& max_jet_vel() const { return m_maxJetVel; }
-  inline const amrex::Real& jet_dia() const { return m_jetDia; }
-  inline const amrex::Real& spread_angle() const { return m_spreadAngle; }
-  inline amrex::Real mass_flow_rate(const amrex::Real time) const
+  const amrex::Real& max_jet_vel() const { return m_maxJetVel; }
+  const amrex::Real& jet_dia() const { return m_jetDia; }
+  const amrex::Real& spread_angle() const { return m_spreadAngle; }
+  amrex::Real mass_flow_rate(const amrex::Real time) const
 
   {
     if (m_useROI) {
       return interpolateMdot(time);
     }
     return m_massFlow;
   }
-  inline const amrex::Real& Y(const int spf) const { return m_jetY[spf]; }
-  inline const amrex::Real& T() const { return m_jetT; }
-  inline const amrex::Real& start_time() const { return m_startTime; }
-  inline const amrex::Real& end_time() const { return m_endTime; }
-  inline const amrex::Real& num_ppp() const { return m_numPPP; }
-  inline const std::string& jet_name() const { return m_jetName; }
-  inline int Proc() const { return m_proc; }
+  const amrex::Real& Y(const int spf) const { return m_jetY[spf]; }
+  const amrex::Real& T() const { return m_jetT; }
+  const amrex::Real& start_time() const { return m_startTime; }
+  const amrex::Real& end_time() const { return m_endTime; }
+  const amrex::Real& num_ppp() const { return m_numPPP; }
+  const std::string& jet_name() const { return m_jetName; }
+  int Proc() const { return m_proc; }
 
   // Call this before using spray jet
   bool jet_active(const amrex::Real time) const
@@ -88,7 +89,8 @@ public:
     return check;
   }
 
-  inline bool hollow_spray() const { return m_hollowSpray; }
+
+  bool hollow_spray() const { return m_hollowSpray; }
 
   /// Returns the average values given for the jet
   amrex::Real get_avg_dia() const { return m_avgDia; }
@@ -123,6 +125,7 @@ public:
     }
   }
 
+
   void set_jet_vel(amrex::Real jet_vel) { m_jetVel = jet_vel; }
 
   void set_mass_flow(amrex::Real mass_flow) { m_massFlow = mass_flow; }
@@ -197,6 +200,7 @@ public:
     return m_override_inj_plane_loc;
   }
 
+
   void set_start_time(amrex::Real start_time) { m_startTime = start_time; }
   void set_end_time(amrex::Real end_time) { m_endTime = end_time; }
   void set_num_ppp(amrex::Real num_ppp) { m_numPPP = num_ppp; }
@@ -220,7 +224,6 @@ public:
      @param[out] T_part Droplet temperature
      @param[out] Y_part Droplet mass fractions
   */
-
   virtual bool get_new_particle(
     const amrex::Real time,
     const amrex::Real& phi_radial,
@@ -333,6 +336,7 @@ public:
     }
   }
 
+
   amrex::Real interpolateMdot(const amrex::Real time) const
   {
     return interpolateROI(time, inject_mass.dataPtr());
@@ -352,6 +356,7 @@ public:
   amrex::Real m_totalInjMass = 0.;
   amrex::Real m_totalInjTime = 0.;
 
+
   bool m_use_Fluentdpmfile;
   bool m_rstrt_Fltdpmsim_from_nonFltDPMChckPointFile = false;
   std::string m_FluentDPMFile;

Source/Spray/SprayParticles.H

@@ -110,7 +110,6 @@ public:
   amrex::Real estTimestep(int level) const;
 
   /// \brief Reset the particle ID in case we need to reinitialize the particles
-
   static void resetID(const int id) { ParticleType::NextID(id); }
 
   /// \brief Returns the number of ghost cells for making ghost particles. This
@@ -122,7 +121,6 @@ public:
   /// @param cfl Particle CFL number on level N
   /// @param depos_width Number of cells adjacent to the cell containing the
   /// particle needed to interpolate the particle source term to the mesh
-
   static int getGhostPartCells(
     const int level,
     const int finest_level,