Aurora config

cime_config/machines/cmake_macros/oneapi-ifx.cmake

@@ -4,15 +4,15 @@ if (compile_threaded)
   string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
   string(APPEND CMAKE_EXE_LINKER_FLAGS  " -qopenmp")
 endif()
-string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
-string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -fpe0")
+string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2 -gline-tables-only -g")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2 -gline-tables-only -g")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -gline-tables-only -g")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
 string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
-string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99")
-string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
-string(APPEND CMAKE_Fortran_FLAGS   " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
+string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99 -gline-tables-only -g")
+string(APPEND CMAKE_CXX_FLAGS " -fp-model precise -gline-tables-only -g")
+string(APPEND CMAKE_Fortran_FLAGS   " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -gline-tables-only -g")
 string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
 string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
 string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
@@ -23,4 +23,6 @@ set(MPICXX "mpicxx")
 set(SCC "icx")
 set(SCXX "icpx")
 set(SFC "ifx")
-set(E3SM_LINK_WITH_FORTRAN "TRUE")
+
+
+#set(E3SM_LINK_WITH_FORTRAN "TRUE")

cime_config/machines/cmake_macros/oneapi-ifx_auroracpu.cmake

@@ -0,0 +1,19 @@
+
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
+if (compile_threaded)
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
+endif()
+
+string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
+
+#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")
+
+
+
+
+
+
+
+
+
+

cime_config/machines/cmake_macros/oneapi-ifx_sunspotcpu.cmake

@@ -0,0 +1,19 @@
+
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
+if (compile_threaded)
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
+endif()
+
+string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
+
+#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")
+
+
+
+
+
+
+
+
+
+

cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake

@@ -4,15 +4,17 @@ if (compile_threaded)
   string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
   string(APPEND CMAKE_EXE_LINKER_FLAGS  " -qopenmp")
 endif()
-string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
-string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -fpe0")
+
+#adding -g here leads to linker internal errors
+string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2 -g -gline-tables-only")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2 -fpscomp logicals -g -gline-tables-only")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -g -gline-tables-only")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -fpscomp logicals -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
 string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
 string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99")
 string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
-string(APPEND CMAKE_Fortran_FLAGS   " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
+string(APPEND CMAKE_Fortran_FLAGS   " -fpscomp logicals -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
 string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
 string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
 string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")

cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake

@@ -1,7 +1,9 @@
 
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=16 -Wl,--no-relax")
 if (compile_threaded)
   string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
 endif()
+string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
 string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
-string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")
+
+set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")

cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot-gen.cmake

@@ -0,0 +1,30 @@
+
+set(CXX_LINKER "CXX")
+
+execute_process(COMMAND $ENV{NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0} -Wl,-rpath -Wl,$ENV{NETCDF_PATH}/lib -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
+
+execute_process(COMMAND $ENV{NETCDF_PATH}/bin/nc-config --libs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0}")
+string(APPEND SLIBS " -fiopenmp -fopenmp-targets=spir64")
+
+set(NETCDF_PATH "$ENV{NETCDF_PATH}")
+set(PNETCDF_PATH "$ENV{PNETCDF_PATH}")
+
+set(USE_SYCL "TRUE")
+
+string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_GEN=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
+
+string(APPEND SYCL_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -fsycl -mlong-double-64 -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
+
+#string(APPEND SYCL_FLAGS " -\-intel -fsycl")
+string(APPEND CXX_LDFLAGS " -Wl,-\-defsym,main=MAIN_\_ -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -fsycl -lsycl -mlong-double-64 -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64")
+
+SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "")
+SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
+SET(CMAKE_FORTRAN_COMPILER "mpifort" CACHE STRING "")
+
+
+

cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot-pvc.cmake

@@ -0,0 +1,21 @@
+
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=16")
+if (compile_threaded)
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
+endif()
+
+if (DEBUG)
+#undefined reference to `__msan....
+#https://community.intel.com/t5/Intel-Fortran-Compiler/Linking-errors-when-using-memory-sanitizer-in-fortran-project/m-p/1521476
+#When you compile with -check uninit (or -check all) you also need to link with that compiler option.
+#  string(APPEND CMAKE_EXE_LINKER_FLAGS " -check uninit")
+endif()
+
+string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off -DCMAKE_CXX_FLAGS='-fsycl-device-code-split=per_kernel'")
+string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
+
+set(SCREAM_MPI_ON_DEVICE ON CACHE STRING "")
+
+
+
+

cime_config/machines/config_batch.xml

@@ -561,6 +561,30 @@
       </queues>
     </batch_system>
 
+    <batch_system MACH="sunspotcpu" type="pbspro">
+      <batch_submit>/lus/gila/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
+      <queues>
+        <queue walltimemax="01:00:00" jobmin="1" jobmax="128" default="true">workq</queue>
+        <queue walltimemax="01:00:00" jobmin="1" jobmax="128">debug</queue>
+      </queues>
+    </batch_system>
+
+    <batch_system MACH="sunspot-pvc" type="pbspro">
+      <batch_submit>/lus/gila/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
+      <queues>
+        <queue walltimemax="01:00:00" jobmin="1" jobmax="128" default="true">workq</queue>
+        <queue walltimemax="01:00:00" jobmin="1" jobmax="128">debug</queue>
+      </queues>
+    </batch_system>
+
+    <batch_system MACH="sunspot-gen" type="pbspro">
+      <batch_submit>/lus/gila/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
+      <queues>
+        <queue walltimemax="01:00:00" jobmin="1" jobmax="128" default="true">workq</queue>
+        <queue walltimemax="01:00:00" jobmin="1" jobmax="128">debug</queue>
+      </queues>
+    </batch_system>
+
     <batch_system MACH="aurora" type="pbspro">
       <batch_submit>/lus/flare/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
       <queues>
@@ -569,6 +593,15 @@
         <queue walltimemax="00:59:00" jobmin="1" jobmax="10624">workq</queue>
       </queues>
     </batch_system>
+
+    <batch_system MACH="auroracpu" type="pbspro">
+      <batch_submit>/lus/flare/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
+      <queues>
+        <queue walltimemax="00:59:00" jobmin="1" jobmax="2560" default="true">EarlyAppAccess</queue>
+        <queue walltimemax="00:59:00" jobmin="1" jobmax="10624">workq-route</queue>
+        <queue walltimemax="00:59:00" jobmin="1" jobmax="10624">workq</queue>
+      </queues>
+    </batch_system>
 
     <batch_system MACH="polaris" type="pbspro">
       <batch_submit>/grand/E3SMinput/soft/qsub/throttle</batch_submit>