Redesigned "caar loop pre-boundary exchange", tuned for Frontier

cime_config/machines/config_machines.xml

@@ -1441,22 +1441,19 @@
       <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
       <modules compiler="crayclang-scream">
         <command name="reset"></command>
-        <command name="load">PrgEnv-cray</command>
+        <command name="load">cpe/23.12</command>
         <command name="load">craype-accel-amd-gfx90a</command>
-        <command name="load">rocm/5.4.0</command>
+        <command name="load">rocm/5.7.1</command>
         <command name="load">libunwind/1.6.2</command>
       </modules>
       <modules>
-        <command name="load">cce/15.0.1</command>
-        <command name="switch">craype craype/2.7.20</command>
-        <command name="switch">cray-mpich cray-mpich/8.1.26</command>
         <command name="load">cray-python/3.9.13.1</command>
         <command name="load">subversion/1.14.1</command>
         <command name="load">git/2.36.1</command>
         <command name="load">cmake/3.21.3</command>
-        <command name="load">cray-hdf5-parallel/1.12.2.1</command>
-        <command name="load">cray-netcdf-hdf5parallel/4.9.0.1</command>
-        <command name="load">cray-parallel-netcdf/1.12.3.1</command>
+        <command name="load">cray-hdf5-parallel</command>
+        <command name="load">cray-netcdf-hdf5parallel</command>
+        <command name="load">cray-parallel-netcdf</command>
         <command name="unload">darshan-runtime</command>
       </modules>
     </module_system>

components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake

@@ -6,4 +6,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/hip.cmake)
 set(SCREAM_MPIRUN_EXE "srun" CACHE STRING "")
 set(SCREAM_MACHINE "frontier-scream-gpu" CACHE STRING "")
 
-set(CMAKE_CXX_FLAGS "--amdgpu-target=gfx90a -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE)
+set(CMAKE_CXX_FLAGS "--offload-arch=gfx90a -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE)

components/homme/cmake/machineFiles/frontier.cmake

@@ -0,0 +1,55 @@
+SET (HOMME_MACHINE "frontier" CACHE STRING "")
+
+SET (HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING  "")
+
+SET (NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "")
+SET (HDF5_DIR $ENV{CRAY_HDF5_PARALLEL_PREFIX} CACHE FILEPATH "")
+SET (CPRNC_DIR /ccs/proj/cli115/software/frontier/cprnc CACHE FILEPATH "")
+
+SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
+
+SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
+
+SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
+
+SET(USE_QUEUING FALSE CACHE BOOL "")
+
+SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
+SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
+
+SET (HOMMEXX_BFB_TESTING FALSE CACHE BOOL "")
+
+SET(USE_TRILINOS OFF CACHE BOOL "")
+
+SET(HIP_BUILD TRUE CACHE BOOL "")
+
+SET(Kokkos_ENABLE_SERIAL ON CACHE BOOL "")
+SET(Kokkos_ENABLE_DEBUG OFF CACHE BOOL "")
+SET(Kokkos_ARCH_VEGA90A ON CACHE BOOL "")
+SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "")
+SET(Kokkos_ENABLE_HIP ON CACHE BOOL "")
+SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
+
+SET(CMAKE_C_COMPILER "cc" CACHE STRING "")
+SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "")
+#SET(CMAKE_Fortran_FLAGS "--gcc-toolchain=$ENV{MEMBERWORK}/cli115/workaround" CACHE STRING "")
+SET(CMAKE_CXX_COMPILER "hipcc" CACHE STRING "")
+
+SET(Extrae_LIBRARY "-I$ENV{CRAY_MPICH_DIR}/include -L$ENV{CRAY_MPICH_DIR}/lib -lmpi $ENV{PE_MPICH_GTL_DIR_amd_gfx90a} $ENV{PE_MPICH_GTL_LIBS_amd_gfx90a}" CACHE STRING "")
+
+#SET(ADD_Fortran_FLAGS "-G2 --gcc-toolchain=$ENV{MEMBERWORK}/cli115/workaround ${Extrae_LIBRARY}" CACHE STRING "")
+SET(ADD_C_FLAGS "-g -O ${Extrae_LIBRARY}" CACHE STRING "")
+SET(ADD_CXX_FLAGS "-g -std=c++14 -O3 -munsafe-fp-atomics --offload-arch=gfx90a -fno-gpu-rdc -Wno-unused-command-line-argument -Wno-unsupported-floating-point-opt -Wno-#pragma-messages ${Extrae_LIBRARY}" CACHE STRING "")
+SET(ADD_LINKER_FLAGS "${Extrae_LIBRARY}" CACHE STRING "")
+
+
+set (ENABLE_OPENMP OFF CACHE BOOL "")
+set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "")
+set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
+
+set (HOMME_TESTING_PROFILE "short" CACHE STRING "")
+SET (BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
+
+set (USE_NUM_PROCS 8 CACHE STRING "")
+
+SET (USE_MPI_OPTIONS "-c7 --gpu-bind=closest --gpus-per-task=1" CACHE FILEPATH "")

components/homme/src/preqx_kokkos/cxx/HyperviscosityFunctorImpl.hpp

@@ -297,13 +297,8 @@ class HyperviscosityFunctorImpl
   SphereOperators       m_sphere_ops;
 
   // Policies
-#ifndef NDEBUG
-  template<typename Tag>
-  using TeamPolicyType = Kokkos::TeamPolicy<ExecSpace,Kokkos::LaunchBounds<512,1>,Tag>;
-#else
   template<typename Tag>
   using TeamPolicyType = Kokkos::TeamPolicy<ExecSpace,Tag>;
-#endif
   Kokkos::RangePolicy<ExecSpace,TagUpdateStates>    m_policy_update_states;
   TeamPolicyType<TagFirstLaplaceHV>                 m_policy_first_laplace;
   TeamPolicyType<TagHyperPreExchange>               m_policy_pre_exchange;

components/homme/src/share/compose/compose_homme.hpp

@@ -200,7 +200,7 @@ struct HommeFormatArray {
   std::vector<T*> ie_data_ptr;
   const Int nlev, qsized, ntimelev;
 
-#ifdef COMPOSE_ENABLE_GPU
+#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__
   COMPOSE_INLINE_FUNCTION static T& unused () {
     static T unused = 0;
     assert(0);

components/homme/src/share/cxx/ExecSpaceDefs.cpp

@@ -6,7 +6,6 @@
 
 #include <cassert>
 
-#include <sstream>
 #include <vector>
 
 #include "ExecSpaceDefs.hpp"
@@ -35,32 +34,9 @@ void initialize_kokkos () {
   // provides, as that algorithm is hardcoded in Kokkos::initialize(int& narg,
   // char* arg[]). Once the behavior is exposed in the InitArguments version of
   // initialize, we can remove this string code.
-  //   If for some reason we're running on a GPU platform, have Cuda enabled,
-  // but are using a different execution space, this initialization is still
-  // OK. The rank gets a GPU assigned and simply will ignore it.
-#ifdef KOKKOS_ENABLE_CUDA
-  int nd;
-  const auto ret = cudaGetDeviceCount(&nd);
-  if (ret != cudaSuccess) {
-    // It isn't a big deal if we can't get the device count.
-    nd = 1;
-  }
-#elif defined(KOKKOS_ENABLE_HIP)
-  int nd;
-  const auto ret = hipGetDeviceCount(&nd);
-  if (ret != hipSuccess) {
-    // It isn't a big deal if we can't get the device count.
-    nd = 1;
-  }
-#endif
 #ifdef HOMMEXX_ENABLE_GPU  
-  std::stringstream ss;
-  ss << "--kokkos-num-devices=" << nd;
-  const auto key = ss.str();
-  std::vector<char> str(key.size()+1);
-  std::copy(key.begin(), key.end(), str.begin());
-  str.back() = 0;
-  args.push_back(const_cast<char*>(str.data()));
+  const char *const map_device = "--kokkos-map-device-id-by=mpi_rank";
+  args.push_back(const_cast<char*>(map_device));
 #endif