diff --git a/cime_config/machines/cmake_macros/amdclang_frontier.cmake b/cime_config/machines/cmake_macros/amdclang_frontier.cmake index 2df5074d11f3..3727164933dc 100644 --- a/cime_config/machines/cmake_macros/amdclang_frontier.cmake +++ b/cime_config/machines/cmake_macros/amdclang_frontier.cmake @@ -9,4 +9,7 @@ string(APPEND CPPDEFS " -DLINUX") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") + +# to support Fortran specific compiler intrinsic functions +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake index 6f3c0074798e..4412ea0de7b3 100644 --- a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake @@ -1,8 +1,8 @@ set(MPICC "cc") -set(MPICXX "CC") +set(MPICXX "mpicxx") set(MPIFC "ftn") set(SCC "cc") -set(SCXX "CC") +set(SCXX "hipcc") set(SFC "ftn") string(APPEND CPPDEFS " -DLINUX") @@ -14,13 +14,12 @@ string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") -set(MPICXX "hipcc") -set(SCXX "hipcc") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") +string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") set(USE_HIP "TRUE") string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/crayclang_frontier.cmake b/cime_config/machines/cmake_macros/crayclang_frontier.cmake index 6c8d4164cb28..76ec41bdbe72 100644 --- a/cime_config/machines/cmake_macros/crayclang_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclang_frontier.cmake @@ -16,5 +16,7 @@ string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") # Scorpio installs string(APPEND CMAKE_Fortran_FLAGS " -em -ef") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") + # to support Fortran specific compiler intrinsic functions set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake index 32f125947b5c..a37ccde439e5 100644 --- a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake @@ -1,5 +1,6 @@ set(MPICC "cc") -set(MPICXX "hipcc") +set(MPICXX "mpicxx") +#set(MPICXX "CC") set(MPIFC "ftn") set(SCC "cc") set(SCXX "hipcc") @@ -33,7 +34,7 @@ set(HAS_F2008_CONTIGUOUS "TRUE") # -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue: # /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock&)@GLIBCXX_3.4.30' # AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") +#string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") # Switching to O3 for performance benchmarking # Will revisit any failing tests @@ -53,11 +54,13 @@ string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") +string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") set(USE_HIP "TRUE") string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/gnu_frontier.cmake b/cime_config/machines/cmake_macros/gnu_frontier.cmake index 9b242243989b..ee44a2ca08f5 100644 --- a/cime_config/machines/cmake_macros/gnu_frontier.cmake +++ b/cime_config/machines/cmake_macros/gnu_frontier.cmake @@ -13,3 +13,8 @@ string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") + +# to support Fortran specific compiler intrinsic functions +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake index 174f8207d0d0..6ca4b83d9c2f 100644 --- a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake @@ -1,6 +1,6 @@ set(MPICC "cc") -set(MPICXX "hipcc") # Needs MPICH_CXX to use hipcc -set(MPIFC "ftn") # Linker needs to be the Cray wrapper ftn, not mpif90 +set(MPICXX "mpicxx") +set(MPIFC "ftn") set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") @@ -16,11 +16,17 @@ string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -set(E3SM_LINK_WITH_FORTRAN "TRUE") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/11.2.0/snos/lib64/ -lgfortran -L/opt/rocm-5.4.0/lib -lhsa-runtime64 -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") +string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") +# +#string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/rocm-5.4.0/lib -lhsa-runtime64 -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") set(USE_HIP "TRUE") string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") + + +set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 6cd9d585a25b..2841df7200c7 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -980,7 +980,7 @@ Frontier exascale supercomputer at ORNL. 9408 nodes, Node: 4 AMD MI250X GPUs (2 GCDs) ~ 8 GPUs, 512 GB HDB2E, AMD EPYC 64 cores, 512GB DDR4 .*frontier.* Linux - crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu + gnu,amdclang,crayclang,gnugpu,amdclanggpu,crayclanggpu mpich cli115 /lustre/orion/cli115/world-shared/frontier @@ -1042,8 +1042,7 @@ PrgEnv-cray PrgEnv-gnu/8.3.3 - - gcc gcc/11.2.0 + gcc gcc/12.2.0 craype-accel-amd-gfx90a @@ -1051,6 +1050,7 @@ cray-python/3.9.13.1 + cray-libsci subversion/1.14.1 git/2.36.1 cmake/3.21.3 @@ -1066,14 +1066,12 @@ 0.25 0 + $ENV{CRAY_LIBSCI_PREFIX_DIR}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH} $ENV{NETCDF_DIR} $ENV{PNETCDF_DIR} - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - --ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc} --gpu-bind=closest @@ -1090,6 +1088,9 @@ spread threads +