Merge branch 'bartgol/point-to-ekat-master' into master (PR #6421)

Point to ekat master branch, and fix usage of deprecated kokkos code in Homme

We just merged the kokkos-4.2 branch in ekat/master, so now we can
make e3sm point directly to the ekat/master branch, which avoids
conflicts when doing a downstream merge into scream repo.

Also, fix usage of deprecated kokkos code in Homme. The tests on e3sm
repo are set so that Kokkos allows using deprecated code (with
warnings), but it caused some errors downstream in eamxx. Using the
non-deprecated code is a simple fix

[BFB]

cime_config/machines/cmake_macros/gnugpu_pm-gpu.cmake

@@ -6,7 +6,7 @@ if (COMP_NAME STREQUAL gptl)
 endif()
 string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
 string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
-string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_AMPERE80=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=Off")
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_AMPERE80=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=Off -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=Off")
 set(CMAKE_CUDA_ARCHITECTURES "80")
 string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")

components/homme/src/share/compose/cedr_kokkos.hpp

@@ -123,7 +123,7 @@ struct ExeSpaceUtils {
 #ifdef COMPOSE_MIMIC_GPU
     const int max_threads =
 #ifdef KOKKOS_ENABLE_OPENMP
-      ExeSpace::concurrency()
+      ExeSpace().concurrency()
 #else
       1
 #endif

components/homme/src/share/cxx/ExecSpaceDefs.hpp

@@ -179,17 +179,14 @@ static
 typename std::enable_if<!OnGpu<ExecSpaceType>::value,int>::type
 get_num_concurrent_teams (const Kokkos::TeamPolicy<ExecSpaceType,Tags...>& policy) {
   const int team_size = policy.team_size();
-  const int concurrency = ExecSpaceType::concurrency();
+  const int concurrency = ExecSpaceType().concurrency();
   return (concurrency + team_size - 1) / team_size;
 }
 
 template<typename ExecSpaceType, typename... Tags>
 static
 typename std::enable_if<OnGpu<ExecSpaceType>::value,int>::type
 get_num_concurrent_teams (const Kokkos::TeamPolicy<ExecSpaceType,Tags...>& policy) {
-  // const int team_size = policy.team_size() * policy.vector_length();
-  // const int concurrency = ExecSpaceType::concurrency();
-  // return (concurrency + team_size - 1) / team_size;
   return policy.league_size();
 }
 

components/homme/src/share/cxx/kokkos_utils.hpp

@@ -35,7 +35,7 @@ class _TeamUtilsCommonBase
   template <typename TeamPolicy>
   _TeamUtilsCommonBase(const TeamPolicy& policy)
   {
-    _max_threads = ExeSpace::concurrency() / ( OnGpu<ExeSpace>::value ? 2 : 1);
+    _max_threads = ExeSpace().concurrency() / ( OnGpu<ExeSpace>::value ? 2 : 1);
     const int team_size = policy.team_size();
     _num_teams = _max_threads / team_size;
     _team_size = _max_threads / _num_teams;