Update Frontier machine/compilers following system update

- Retain current software modules instead of updating to the latest versions.
- Add linker options to use GCC 12.2, addressing linker errors.
- Utilize Fortran linker to resolve additional linker errors.
- Replace hipcc with mpicxx for MPICXX macro in the GPU compiler definitions.
- Adjust compiler priority to prioritize reliability over performance.
- Temporarily comment out ADIOS2 configurations

cime_config/machines/cmake_macros/amdclang_frontier.cmake

@@ -9,4 +9,7 @@ string(APPEND CPPDEFS " -DLINUX")
 if (COMP_NAME STREQUAL gptl)
     string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
 endif()
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++")
+
+# to support Fortran specific compiler intrinsic functions
+set(E3SM_LINK_WITH_FORTRAN "TRUE")

cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake

@@ -1,8 +1,8 @@
 set(MPICC "cc")
-set(MPICXX "CC")
+set(MPICXX "mpicxx")
 set(MPIFC "ftn")
 set(SCC "cc")
-set(SCXX "CC")
+set(SCXX "hipcc")
 set(SFC "ftn")
 
 string(APPEND CPPDEFS " -DLINUX")
@@ -14,13 +14,12 @@ string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
 
 string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")
 
-set(MPICXX "hipcc")
-set(SCXX "hipcc")
-string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
-string(APPEND CMAKE_EXE_LINKER_FLAGS    " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
+string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++")
 string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
 
 set(USE_HIP "TRUE")
 string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip")
+set(E3SM_LINK_WITH_FORTRAN "TRUE")

cime_config/machines/cmake_macros/crayclang_frontier.cmake

@@ -16,5 +16,7 @@ string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero")
 # Scorpio installs
 string(APPEND CMAKE_Fortran_FLAGS " -em -ef")
 
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++")
+
 # to support Fortran specific compiler intrinsic functions
 set(E3SM_LINK_WITH_FORTRAN "TRUE")

cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake

@@ -1,5 +1,6 @@
 set(MPICC "cc")
-set(MPICXX "hipcc")
+set(MPICXX "mpicxx")
+#set(MPICXX "CC")
 set(MPIFC "ftn")
 set(SCC "cc")
 set(SCXX "hipcc")
@@ -33,7 +34,7 @@ set(HAS_F2008_CONTIGUOUS "TRUE")
 # -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue:
 # /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock<std::mutex>&)@GLIBCXX_3.4.30'
 # AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain.
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined")
+#string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined")
 
 # Switching to O3 for performance benchmarking
 # Will revisit any failing tests
@@ -53,11 +54,13 @@ string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc")
 
 string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
 
-string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
-string(APPEND CMAKE_EXE_LINKER_FLAGS    " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
+string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
 string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++")
 
 string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
 
 set(USE_HIP "TRUE")
 string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip")
+set(E3SM_LINK_WITH_FORTRAN "TRUE")

cime_config/machines/cmake_macros/gnu_frontier.cmake

@@ -13,3 +13,8 @@ string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface")
 
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
 string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
+
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++")
+
+# to support Fortran specific compiler intrinsic functions
+set(E3SM_LINK_WITH_FORTRAN "TRUE")

cime_config/machines/cmake_macros/gnugpu_frontier.cmake

@@ -1,6 +1,6 @@
 set(MPICC "cc")
-set(MPICXX "hipcc") # Needs MPICH_CXX to use hipcc
-set(MPIFC "ftn") # Linker needs to be the Cray wrapper ftn, not mpif90
+set(MPICXX "mpicxx")
+set(MPIFC "ftn")
 set(SCC "cc")
 set(SCXX "hipcc")
 set(SFC "ftn")
@@ -16,11 +16,17 @@ string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
 string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
 string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
 
-set(E3SM_LINK_WITH_FORTRAN "TRUE")
-string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
-string(APPEND CMAKE_EXE_LINKER_FLAGS    " -L/opt/cray/pe/gcc/11.2.0/snos/lib64/ -lgfortran -L/opt/rocm-5.4.0/lib -lhsa-runtime64 -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ")
+string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++")
+#
+#string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/rocm-5.4.0/lib -lhsa-runtime64 -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ")
 
 string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off")
 
 set(USE_HIP "TRUE")
 string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip")
+
+
+set(E3SM_LINK_WITH_FORTRAN "TRUE")

cime_config/machines/config_machines.xml

@@ -980,7 +980,7 @@
     <DESC>Frontier exascale supercomputer at ORNL. 9408 nodes, Node: 4 AMD MI250X GPUs (2 GCDs) ~ 8 GPUs, 512 GB HDB2E, AMD EPYC 64 cores, 512GB DDR4 </DESC>
     <NODENAME_REGEX>.*frontier.*</NODENAME_REGEX>
     <OS>Linux</OS>
-    <COMPILERS>crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu</COMPILERS>
+    <COMPILERS>gnu,amdclang,crayclang,gnugpu,amdclanggpu,crayclanggpu</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>cli115</PROJECT>
     <SAVE_TIMING_DIR>/lustre/orion/cli115/world-shared/frontier</SAVE_TIMING_DIR>
@@ -1042,15 +1042,15 @@
       <modules compiler="gnu.*">
         <command name="reset"></command>
         <command name="switch">PrgEnv-cray PrgEnv-gnu/8.3.3</command>
-        <!-- TODO: gcc/12.2.0 is default, need to check -->
-        <command name="switch">gcc gcc/11.2.0</command>
+        <command name="switch">gcc gcc/12.2.0</command>
       </modules>
       <modules compiler="gnugpu">
         <command name="load">craype-accel-amd-gfx90a</command>
         <command name="load">rocm/5.4.0</command>
       </modules>
       <modules>
         <command name="load">cray-python/3.9.13.1</command>
+        <command name="load">cray-libsci</command>
         <command name="load">subversion/1.14.1</command>
         <command name="load">git/2.36.1</command>
         <command name="load">cmake/3.21.3</command>
@@ -1066,14 +1066,12 @@
     <TEST_MEMLEAK_TOLERANCE>0.25</TEST_MEMLEAK_TOLERANCE>
     <MAX_GB_OLD_TEST_DATA>0</MAX_GB_OLD_TEST_DATA>
     <environment_variables>
+      <env name="PKG_CONFIG_PATH">$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}</env>
       <env name="NETCDF_PATH">$ENV{NETCDF_DIR}</env>
       <env name="PNETCDF_PATH">$ENV{PNETCDF_DIR}</env>
       <env name="NTASKS_PER_GPU"> </env>
       <env name="GPU_BIND_ARGS"> </env>
     </environment_variables>
-    <environment_variables compiler="amdclang">
-      <env name="LD_LIBRARY_PATH">$ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH}</env>
-    </environment_variables>
     <environment_variables compiler=".*gpu.*">
       <env name="NTASKS_PER_GPU">--ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc}</env>
       <env name="GPU_BIND_ARGS">--gpu-bind=closest</env>
@@ -1090,6 +1088,9 @@
       <env name="OMP_PROC_BIND">spread</env>
       <env name="OMP_PLACES">threads</env>
     </environment_variables>
+<!--
+commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" issue is resolved.
+
     <environment_variables compiler="gnu.*" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.23/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi}</env>
     </environment_variables>
@@ -1099,6 +1100,7 @@
     <environment_variables compiler="amdclang.*" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.23/amdclang-15.0.0; else echo "$ADIOS2_ROOT"; fi}</env>
     </environment_variables>
+-->
   </machine>
 
   <machine MACH="crusher">