Update ALCF Sunspot and Aurora machine configs

cime_config/allactive/config_pesall.xml

@@ -1472,6 +1472,21 @@
         </rootpe>
       </pes>
     </mach>
+    <mach name="sunspot|aurora">
+      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+SGLC_SWAV_SIAC_SESP_BGC.*" pesize="any">
+        <comment>sunspot|aurora: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 2 nodes pure-MPI</comment>
+        <ntasks>
+          <ntasks_atm>-2</ntasks_atm>
+          <ntasks_lnd>-2</ntasks_lnd>
+          <ntasks_rof>-2</ntasks_rof>
+          <ntasks_ice>-2</ntasks_ice>
+          <ntasks_ocn>-2</ntasks_ocn>
+          <ntasks_cpl>-2</ntasks_cpl>
+          <ntasks_glc>-2</ntasks_glc>
+          <ntasks_wav>-2</ntasks_wav>
+        </ntasks>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%ne30np4.pg2_l%r05_oi%IcosXISC30E3r7_r%r05_.+">
     <mach name="chrysalis">
@@ -1744,6 +1759,21 @@
         </nthrds>
       </pes>
     </mach>
+    <mach name="sunspot|aurora">
+      <pes compset="any" pesize="any">
+        <comment>allactive+sunspot|aurora: default, 96 mpi x 1 omp @ root 0</comment>
+        <ntasks>
+          <ntasks_atm>96</ntasks_atm>
+          <ntasks_lnd>96</ntasks_lnd>
+          <ntasks_rof>96</ntasks_rof>
+          <ntasks_ice>96</ntasks_ice>
+          <ntasks_ocn>96</ntasks_ocn>
+          <ntasks_glc>96</ntasks_glc>
+          <ntasks_wav>96</ntasks_wav>
+          <ntasks_cpl>96</ntasks_cpl>
+        </ntasks>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%ne30np4">
     <mach name="pm-gpu|muller-gpu">

cime_config/machines/Depends.oneapi-ifxgpu.cmake

@@ -1,14 +1,5 @@
 
-set(CPPDEFS "${CPPDEFS} -DMPAS_OPENMP_OFFLOAD")
-list(APPEND MPAS_ADD_ACC_FLAGS
-  ${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_mesh_pool.f90
-  ${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_velocity_solver_variational.f90
-  ${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_velocity_solver.f90
-)
-
-foreach(ITEM IN LISTS MPAS_ADD_ACC_FLAGS)
-  e3sm_add_flags("${ITEM}" "-fiopenmp -fopenmp-targets=spir64")
-endforeach()
-
 # compile mpas_seaice_core_interface.f90 with ifort, not ifx
-e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort")
+if (NOT MPILIB STREQUAL "openmpi")
+  e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort")
+endif()

cime_config/machines/cmake_macros/oneapi-ifx.cmake

@@ -7,7 +7,7 @@ endif()
 string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
 string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -fpe0")
 string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
 string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99")

cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake

@@ -7,7 +7,7 @@ endif()
 string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
 string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -fpe0")
 string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
 string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99")

cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake

@@ -3,5 +3,5 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c
 if (compile_threaded)
   string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
 endif()
-string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
 string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
+string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")

cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake

@@ -0,0 +1,7 @@
+
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
+if (compile_threaded)
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
+endif()
+string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
+string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")

cime_config/machines/config_batch.xml

@@ -562,7 +562,7 @@
     </batch_system>
 
     <batch_system MACH="aurora" type="pbspro">
-      <batch_submit>/lus/gecko/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
+      <batch_submit>/lus/flare/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
       <queues>
         <queue walltimemax="00:59:00" jobmin="1" jobmax="2560" default="true">EarlyAppAccess</queue>
         <queue walltimemax="00:59:00" jobmin="1" jobmax="10624">workq-route</queue>

cime_config/machines/config_machines.xml

@@ -3365,7 +3365,7 @@
     <NODENAME_REGEX>uan-.*</NODENAME_REGEX>
     <OS>LINUX</OS>
     <COMPILERS>oneapi-ifx,oneapi-ifxgpu,gnu</COMPILERS>
-    <MPILIBS>mpich,impi,openmpi</MPILIBS>
+    <MPILIBS>mpich</MPILIBS>
     <CHARGE_ACCOUNT>CSC249ADSE15_CNDA</CHARGE_ACCOUNT>
     <SAVE_TIMING_DIR>/gila/CSC249ADSE15_CNDA/performance_archive</SAVE_TIMING_DIR>
     <SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
@@ -3385,7 +3385,7 @@
     <MAX_TASKS_PER_NODE compiler="oneapi-ifxgpu">104</MAX_TASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>104</MAX_MPITASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE compiler="oneapi-ifx">104</MAX_MPITASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">12</MAX_MPITASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">48</MAX_MPITASKS_PER_NODE>
     <PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
        <executable>mpiexec</executable>
@@ -3398,13 +3398,6 @@
           <arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
        </arguments>
     </mpirun>
-    <mpirun mpilib="openmpi">
-        <executable>mpirun</executable>
-        <arguments>
-           <arg name="num_tasks">--tag-output -n {{ total_tasks }}</arg>
-           <arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread</arg>
-        </arguments>
-    </mpirun>
     <module_system type="module" allow_error="true">
        <init_path lang="sh">/soft/packaging/lmod/lmod/init/sh</init_path>
        <init_path lang="csh">/soft/packaging/lmod/lmod/init/csh</init_path>
@@ -3413,14 +3406,13 @@
        <cmd_path lang="csh">module</cmd_path>
        <cmd_path lang="python">/soft/packaging/lmod/lmod/libexec/lmod python</cmd_path>
        <modules>
-          <command name="purge"></command>
-          <command name="use">/soft/modulefiles</command>
-          <command name="load">spack cmake/3.26.3-gcc-11.2.0-vnn7ncx</command>
-          <command name="load">prepend-deps/default</command>
+          <command name="load">cmake</command>
         </modules>
         <modules compiler="!gnu">
-          <command name="unload">gcc</command>
-          <command name="load">oneapi/eng-compiler/2023.05.15.007</command>
+          <command name="load">oneapi/eng-compiler/2024.04.15.002</command>
+        </modules>
+        <modules compiler="oneapi-ifxgpu">
+          <command name="load">kokkos/git.7ff87a5-omp-sycl</command>
         </modules>
         <modules compiler="gnu">
            <command name="unload">spack cmake</command>
@@ -3438,12 +3430,6 @@
      <environment_variables mpilib="mpich" DEBUG="TRUE">
         <env name="HYDRA_TOPO_DEBUG">1</env>
      </environment_variables>
-     <environment_variables mpilib="impi">
-        <env name="I_MPI_DEBUG">10</env>
-        <env name="I_MPI_PIN_DOMAIN">omp</env>
-        <env name="I_MPI_PIN_ORDER">spread</env>
-        <env name="I_MPI_PIN_CELL">unit</env>
-     </environment_variables>
      <environment_variables compiler="oneapi-ifxgpu">
         <env name="ONEAPI_DEVICE_SELECTOR">level_zero:gpu</env>
         <env name="ONEAPI_MPICH_GPU">NO_GPU</env>
@@ -3456,6 +3442,9 @@
         <env name="GPU_TILE_COMPACT">/soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
         <env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
         <env name="FI_CXI_CQ_FILL_PERCENT">20</env>
+        <env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>
+        <env name="ZES_ENABLE_SYSMAN">1</env>
+        <env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4</env>
     </environment_variables>
     <environment_variables compiler="oneapi-ifx">
         <env name="LIBOMPTARGET_DEBUG">0</env><!--default 0, max 5 -->
@@ -3485,14 +3474,14 @@
     <COMPILERS>oneapi-ifx,oneapi-ifxgpu,gnu</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <CHARGE_ACCOUNT>CSC249ADSE15_CNDA</CHARGE_ACCOUNT>
-    <SAVE_TIMING_DIR>/lus/gecko/projects/CSC249ADSE15_CNDA/performance_archive</SAVE_TIMING_DIR>
+    <SAVE_TIMING_DIR>/lus/flare/projects/CSC249ADSE15_CNDA/performance_archive</SAVE_TIMING_DIR>
     <SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
-    <CIME_OUTPUT_ROOT>/lus/gecko/projects/CSC249ADSE15_CNDA/$USER/scratch</CIME_OUTPUT_ROOT>
-    <DIN_LOC_ROOT>/lus/gecko/projects/CSC249ADSE15_CNDA/inputdata</DIN_LOC_ROOT>
-    <DIN_LOC_ROOT_CLMFORC>/lus/gecko/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+    <CIME_OUTPUT_ROOT>/lus/flare/projects/CSC249ADSE15_CNDA/$USER/scratch</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>/lus/flare/projects/CSC249ADSE15_CNDA/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>/lus/flare/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
     <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
-    <BASELINE_ROOT>/lus/gecko/projects/CSC249ADSE15_CNDA/baselines/$COMPILER</BASELINE_ROOT>
-    <CCSM_CPRNC>/lus/gecko/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc</CCSM_CPRNC>
+    <BASELINE_ROOT>/lus/flare/projects/CSC249ADSE15_CNDA/baselines/$COMPILER</BASELINE_ROOT>
+    <CCSM_CPRNC>/lus/flare/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc</CCSM_CPRNC>
     <GMAKE_J>16</GMAKE_J>
     <TESTS>e3sm_developer</TESTS>
     <NTEST_PARALLEL_JOBS>4</NTEST_PARALLEL_JOBS>
@@ -3501,54 +3490,48 @@
     <MAX_TASKS_PER_NODE>208</MAX_TASKS_PER_NODE>
     <MAX_TASKS_PER_NODE compiler="oneapi-ifxgpu">104</MAX_TASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>104</MAX_MPITASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">12</MAX_MPITASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">48</MAX_MPITASKS_PER_NODE>
     <PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED> 
     <mpirun mpilib="default">
        <executable>mpiexec</executable>
        <!--executable>numactl -m 2-3 mpiexec</executable--><!--for HBM runs-->
        <arguments>
           <arg name="total_num_tasks">-np {{ total_tasks }} --label</arg>
           <arg name="ranks_per_node">-ppn {{ tasks_per_node }}</arg>
-          <arg name="ranks_bind">--cpu-bind $ENV{RANKS_BIND} -envall</arg>
+          <arg name="ranks_bind">--cpu-bind depth -envall</arg>
           <arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS}</arg>
           <arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
        </arguments>
     </mpirun>
     <module_system type="module" allow_error="true">
-       <init_path lang="sh">/lus/gecko/projects/CSC249ADSE15_CNDA/modules/lmod.sh</init_path>
+       <init_path lang="sh">/lus/flare/projects/CSC249ADSE15_CNDA/modules/lmod.sh</init_path>
        <init_path lang="csh">/soft/sunspot_migrate/soft/packaging/lmod/lmod/init/csh</init_path>
        <init_path lang="python">/soft/sunspot_migrate/soft/packaging/lmod/lmod/init/env_modules_python.py</init_path>
        <cmd_path lang="sh">module</cmd_path>
        <cmd_path lang="csh">module</cmd_path>
        <cmd_path lang="python">/soft/sunspot_migrate/soft/packaging/lmod/lmod/libexec/lmod python</cmd_path>
        <modules>
-          <command name="purge"></command>
-          <command name="use">/soft/modulefiles</command>
-          <command name="use">/soft/restricted/CNDA/updates/modulefiles</command>
-          <command name="load">spack-pe-gcc cmake</command>
+          <command name="load">cmake</command>
         </modules>
         <modules compiler="!gnu">
-          <command name="load">oneapi/eng-compiler/2023.05.15.007</command>
+          <command name="load">oneapi/eng-compiler/2024.04.15.002</command>
+        </modules>
+        <modules compiler="oneapi-ifxgpu">
+          <command name="load">kokkos/git.7ff87a5-omp-sycl</command>
         </modules>
         <modules compiler="gnu">
            <command name="unload">spack-pe-gcc cmake</command>
            <command name="load">gcc/10.3.0</command>
         </modules>
-        <modules>
-          <command name="load">cray-pals</command>
-          <command name="load">libfabric/1.15.2.0</command>
-          <command name="load">cray-libpals/1.3.2</command>
-        </modules>
      </module_system>
      <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
      <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
      <environment_variables>
-        <env name="NETCDF_C_PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007</env>
-        <env name="NETCDF_FORTRAN_PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007</env>
-        <env name="PNETCDF_PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007</env>
-        <env name="LD_LIBRARY_PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007/lib:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007/lib:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007/lib:$ENV{LD_LIBRARY_PATH}</env>
-        <env name="PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007/bin:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007/bin:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007/bin:$ENV{PATH}</env>
-        <env name="RANKS_BIND">list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203</env>
+        <env name="NETCDF_C_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002</env>
+        <env name="NETCDF_FORTRAN_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002</env>
+        <env name="PNETCDF_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002</env>
+        <env name="LD_LIBRARY_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/lib:$ENV{LD_LIBRARY_PATH}</env>
+        <env name="PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/bin:$ENV{PATH}</env>
      </environment_variables>
      <environment_variables DEBUG="TRUE">
         <env name="HYDRA_TOPO_DEBUG">1</env>
@@ -3565,6 +3548,9 @@
         <env name="GPU_TILE_COMPACT">/soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
         <env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
         <env name="FI_CXI_CQ_FILL_PERCENT">20</env>
+        <env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>
+        <env name="ZES_ENABLE_SYSMAN">1</env>
+        <env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4</env>
     </environment_variables>
     <environment_variables compiler="oneapi-ifx">
         <env name="LIBOMPTARGET_DEBUG">0</env><!--default 0, max 5 -->

components/eamxx/cmake/machine-files/aurora.cmake

@@ -0,0 +1,14 @@
+include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
+common_setup()
+
+include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake)
+include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake)
+
+set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE)
+set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE)
+set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "")
+set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "")
+
+set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}")
+set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}")
+set(PNETCDF_PATH "$ENV{PNETCDF_PATH}")

components/eamxx/cmake/machine-files/sunspot.cmake

@@ -0,0 +1,10 @@
+include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
+common_setup()
+
+include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake)
+include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake)
+
+set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE)
+set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE)
+set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "")
+set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "")

components/homme/src/share/compose/cedr_kokkos.hpp

@@ -18,7 +18,7 @@ typedef Kokkos::Experimental::HIPSpace CedrGpuSpace;
 # endif
 # if defined KOKKOS_ENABLE_SYCL
 typedef Kokkos::Experimental::SYCL CedrGpuExeSpace;
-typedef Kokkos::Experimental::SYCL> CedrGpuSpace;
+typedef Kokkos::Experimental::SYCL CedrGpuSpace;
 # endif
 #endif
 

components/homme/src/share/compose/compose_slmm_siqk.cpp

@@ -60,8 +60,10 @@ class TestSphereToRefKernel {
     // tol is on dx, not (a,b), so adjust slightly.
     if ( ! info.success || err > 1e4*tol_) {
       jinfo.nfails++;
+#ifndef KOKKOS_ENABLE_SYCL
       printf("calc_sphere_to_ref ei %d i %d j %d: nits %d re %1.1e\n",
              ei, i, j, info.n_iterations, err);
+#endif
     }
     jinfo.sum_nits += info.n_iterations;
     jinfo.max_nits = max(jinfo.max_nits, info.n_iterations);