Update ALCF Sunspot and Aurora machine configs

cime_config/allactive/config_pesall.xml

@@ -1472,6 +1472,21 @@
         </rootpe>
       </pes>
     </mach>
+    <mach name="sunspot">
+      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+SGLC_SWAV_SIAC_SESP_BGC.*" pesize="any">
+        <comment>sunspot: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 2 nodes pure-MPI</comment>
+        <ntasks>
+          <ntasks_atm>-2</ntasks_atm>
+          <ntasks_lnd>-2</ntasks_lnd>
+          <ntasks_rof>-2</ntasks_rof>
+          <ntasks_ice>-2</ntasks_ice>
+          <ntasks_ocn>-2</ntasks_ocn>
+          <ntasks_cpl>-2</ntasks_cpl>
+          <ntasks_glc>-2</ntasks_glc>
+          <ntasks_wav>-2</ntasks_wav>
+        </ntasks>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%ne30np4.pg2_l%r05_oi%IcosXISC30E3r7_r%r05_.+">
     <mach name="chrysalis">
@@ -1744,6 +1759,21 @@
         </nthrds>
       </pes>
     </mach>
+    <mach name="sunspot">
+      <pes compset="any" pesize="any">
+        <comment>allactive+sunspot: default, 96 mpi x 1 omp @ root 0</comment>
+        <ntasks>
+          <ntasks_atm>96</ntasks_atm>
+          <ntasks_lnd>96</ntasks_lnd>
+          <ntasks_rof>96</ntasks_rof>
+          <ntasks_ice>96</ntasks_ice>
+          <ntasks_ocn>96</ntasks_ocn>
+          <ntasks_glc>96</ntasks_glc>
+          <ntasks_wav>96</ntasks_wav>
+          <ntasks_cpl>96</ntasks_cpl>
+        </ntasks>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%ne30np4">
     <mach name="pm-gpu|muller-gpu">

cime_config/machines/cmake_macros/oneapi-ifx.cmake

@@ -7,7 +7,7 @@ endif()
 string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
 string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -fpe0")
 string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
 string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99")

cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake

@@ -7,7 +7,7 @@ endif()
 string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
 string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -fpe0")
 string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
 string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99")

cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake

@@ -5,3 +5,4 @@ if (compile_threaded)
 endif()
 string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
 string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
+string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")

cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake

@@ -0,0 +1,7 @@
+
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
+if (compile_threaded)
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
+endif()
+string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
+string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")

cime_config/machines/config_machines.xml

@@ -3365,7 +3365,7 @@
     <NODENAME_REGEX>uan-.*</NODENAME_REGEX>
     <OS>LINUX</OS>
     <COMPILERS>oneapi-ifx,oneapi-ifxgpu,gnu</COMPILERS>
-    <MPILIBS>mpich,impi,openmpi</MPILIBS>
+    <MPILIBS>mpich</MPILIBS>
     <CHARGE_ACCOUNT>CSC249ADSE15_CNDA</CHARGE_ACCOUNT>
     <SAVE_TIMING_DIR>/gila/CSC249ADSE15_CNDA/performance_archive</SAVE_TIMING_DIR>
     <SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
@@ -3398,13 +3398,6 @@
           <arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
        </arguments>
     </mpirun>
-    <mpirun mpilib="openmpi">
-        <executable>mpirun</executable>
-        <arguments>
-           <arg name="num_tasks">--tag-output -n {{ total_tasks }}</arg>
-           <arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread</arg>
-        </arguments>
-    </mpirun>
     <module_system type="module" allow_error="true">
        <init_path lang="sh">/soft/packaging/lmod/lmod/init/sh</init_path>
        <init_path lang="csh">/soft/packaging/lmod/lmod/init/csh</init_path>
@@ -3413,14 +3406,13 @@
        <cmd_path lang="csh">module</cmd_path>
        <cmd_path lang="python">/soft/packaging/lmod/lmod/libexec/lmod python</cmd_path>
        <modules>
-          <command name="purge"></command>
-          <command name="use">/soft/modulefiles</command>
-          <command name="load">spack cmake/3.26.3-gcc-11.2.0-vnn7ncx</command>
-          <command name="load">prepend-deps/default</command>
+          <command name="load">cmake</command>
         </modules>
         <modules compiler="!gnu">
-          <command name="unload">gcc</command>
-          <command name="load">oneapi/eng-compiler/2023.05.15.007</command>
+          <command name="load">oneapi/eng-compiler/2024.04.15.002</command>
+        </modules>
+        <modules compiler="oneapi-ifxgpu">
+          <command name="load">kokkos/git.7ff87a5-omp-sycl</command>
         </modules>
         <modules compiler="gnu">
            <command name="unload">spack cmake</command>
@@ -3438,12 +3430,6 @@
      <environment_variables mpilib="mpich" DEBUG="TRUE">
         <env name="HYDRA_TOPO_DEBUG">1</env>
      </environment_variables>
-     <environment_variables mpilib="impi">
-        <env name="I_MPI_DEBUG">10</env>
-        <env name="I_MPI_PIN_DOMAIN">omp</env>
-        <env name="I_MPI_PIN_ORDER">spread</env>
-        <env name="I_MPI_PIN_CELL">unit</env>
-     </environment_variables>
      <environment_variables compiler="oneapi-ifxgpu">
         <env name="ONEAPI_DEVICE_SELECTOR">level_zero:gpu</env>
         <env name="ONEAPI_MPICH_GPU">NO_GPU</env>
@@ -3456,6 +3442,8 @@
         <env name="GPU_TILE_COMPACT">/soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
         <env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
         <env name="FI_CXI_CQ_FILL_PERCENT">20</env>
+        <env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>
+        <env name="ZES_ENABLE_SYSMAN">1</env>
     </environment_variables>
     <environment_variables compiler="oneapi-ifx">
         <env name="LIBOMPTARGET_DEBUG">0</env><!--default 0, max 5 -->

components/eamxx/cmake/machine-files/sunspot.cmake

@@ -0,0 +1,10 @@
+include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
+common_setup()
+
+include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake)
+include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake)
+
+set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE)
+set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE)
+set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "")
+set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "")

components/homme/src/share/compose/cedr_kokkos.hpp

@@ -18,7 +18,7 @@ typedef Kokkos::Experimental::HIPSpace CedrGpuSpace;
 # endif
 # if defined KOKKOS_ENABLE_SYCL
 typedef Kokkos::Experimental::SYCL CedrGpuExeSpace;
-typedef Kokkos::Experimental::SYCL> CedrGpuSpace;
+typedef Kokkos::Experimental::SYCL CedrGpuSpace;
 # endif
 #endif
 

components/homme/src/share/compose/compose_slmm_siqk.cpp

@@ -60,8 +60,10 @@ class TestSphereToRefKernel {
     // tol is on dx, not (a,b), so adjust slightly.
     if ( ! info.success || err > 1e4*tol_) {
       jinfo.nfails++;
+#ifndef KOKKOS_ENABLE_SYCL
       printf("calc_sphere_to_ref ei %d i %d j %d: nits %d re %1.1e\n",
              ei, i, j, info.n_iterations, err);
+#endif
     }
     jinfo.sum_nits += info.n_iterations;
     jinfo.max_nits = max(jinfo.max_nits, info.n_iterations);