Skip to content

Commit

Permalink
Merge branch 'azamat/machines/sunspot20240816' (PR #6553)
Browse files Browse the repository at this point in the history 
Update ALCF Sunspot and Aurora machine configs.

[BFB]
  • Loading branch information
@amametjanov
amametjanov committed Sep 6, 2024
2 parents ed06ec6 + 4b5cf35 commit 0ce7fcf
Show file tree
Hide file tree
Showing 12 changed files with 103 additions and 63 deletions.
30 changes: 30 additions & 0 deletions cime_config/allactive/config_pesall.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1472,6 +1472,21 @@
</rootpe>
</pes>
</mach>
<mach name="sunspot|aurora">
<pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+SGLC_SWAV_SIAC_SESP_BGC.*" pesize="any">
<comment>sunspot|aurora: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 2 nodes pure-MPI</comment>
<ntasks>
<ntasks_atm>-2</ntasks_atm>
<ntasks_lnd>-2</ntasks_lnd>
<ntasks_rof>-2</ntasks_rof>
<ntasks_ice>-2</ntasks_ice>
<ntasks_ocn>-2</ntasks_ocn>
<ntasks_cpl>-2</ntasks_cpl>
<ntasks_glc>-2</ntasks_glc>
<ntasks_wav>-2</ntasks_wav>
</ntasks>
</pes>
</mach>
</grid>
<grid name="a%ne30np4.pg2_l%r05_oi%IcosXISC30E3r7_r%r05_.+">
<mach name="chrysalis">
Expand Down Expand Up @@ -1744,6 +1759,21 @@
</nthrds>
</pes>
</mach>
<mach name="sunspot|aurora">
<pes compset="any" pesize="any">
<comment>allactive+sunspot|aurora: default, 96 mpi x 1 omp @ root 0</comment>
<ntasks>
<ntasks_atm>96</ntasks_atm>
<ntasks_lnd>96</ntasks_lnd>
<ntasks_rof>96</ntasks_rof>
<ntasks_ice>96</ntasks_ice>
<ntasks_ocn>96</ntasks_ocn>
<ntasks_glc>96</ntasks_glc>
<ntasks_wav>96</ntasks_wav>
<ntasks_cpl>96</ntasks_cpl>
</ntasks>
</pes>
</mach>
</grid>
<grid name="a%ne30np4">
<mach name="pm-gpu|muller-gpu">
Expand Down
15 changes: 3 additions & 12 deletions cime_config/machines/Depends.oneapi-ifxgpu.cmake
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,5 @@

set(CPPDEFS "${CPPDEFS} -DMPAS_OPENMP_OFFLOAD")
list(APPEND MPAS_ADD_ACC_FLAGS
${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_mesh_pool.f90
${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_velocity_solver_variational.f90
${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_velocity_solver.f90
)

foreach(ITEM IN LISTS MPAS_ADD_ACC_FLAGS)
e3sm_add_flags("${ITEM}" "-fiopenmp -fopenmp-targets=spir64")
endforeach()

# compile mpas_seaice_core_interface.f90 with ifort, not ifx
e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort")
if (NOT MPILIB STREQUAL "openmpi")
e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort")
endif()
2 changes: 1 addition & 1 deletion cime_config/machines/cmake_macros/oneapi-ifx.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ endif()
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0")
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99")
Expand Down
2 changes: 1 addition & 1 deletion cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ endif()
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0")
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99")
Expand Down
2 changes: 1 addition & 1 deletion cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c
if (compile_threaded)
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
endif()
string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")
7 changes: 7 additions & 0 deletions cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@

string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
if (compile_threaded)
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
endif()
string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")
2 changes: 1 addition & 1 deletion cime_config/machines/config_batch.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,7 +562,7 @@
</batch_system>

<batch_system MACH="aurora" type="pbspro">
<batch_submit>/lus/gecko/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
<batch_submit>/lus/flare/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
<queues>
<queue walltimemax="00:59:00" jobmin="1" jobmax="2560" default="true">EarlyAppAccess</queue>
<queue walltimemax="00:59:00" jobmin="1" jobmax="10624">workq-route</queue>
Expand Down
78 changes: 32 additions & 46 deletions cime_config/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3367,7 +3367,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<NODENAME_REGEX>uan-.*</NODENAME_REGEX>
<OS>LINUX</OS>
<COMPILERS>oneapi-ifx,oneapi-ifxgpu,gnu</COMPILERS>
<MPILIBS>mpich,impi,openmpi</MPILIBS>
<MPILIBS>mpich</MPILIBS>
<CHARGE_ACCOUNT>CSC249ADSE15_CNDA</CHARGE_ACCOUNT>
<SAVE_TIMING_DIR>/gila/CSC249ADSE15_CNDA/performance_archive</SAVE_TIMING_DIR>
<SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
Expand All @@ -3387,7 +3387,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<MAX_TASKS_PER_NODE compiler="oneapi-ifxgpu">104</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>104</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="oneapi-ifx">104</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">12</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">48</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>mpiexec</executable>
Expand All @@ -3400,13 +3400,6 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
</arguments>
</mpirun>
<mpirun mpilib="openmpi">
<executable>mpirun</executable>
<arguments>
<arg name="num_tasks">--tag-output -n {{ total_tasks }}</arg>
<arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="sh">/soft/packaging/lmod/lmod/init/sh</init_path>
<init_path lang="csh">/soft/packaging/lmod/lmod/init/csh</init_path>
Expand All @@ -3415,14 +3408,13 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<cmd_path lang="csh">module</cmd_path>
<cmd_path lang="python">/soft/packaging/lmod/lmod/libexec/lmod python</cmd_path>
<modules>
<command name="purge"></command>
<command name="use">/soft/modulefiles</command>
<command name="load">spack cmake/3.26.3-gcc-11.2.0-vnn7ncx</command>
<command name="load">prepend-deps/default</command>
<command name="load">cmake</command>
</modules>
<modules compiler="!gnu">
<command name="unload">gcc</command>
<command name="load">oneapi/eng-compiler/2023.05.15.007</command>
<command name="load">oneapi/eng-compiler/2024.04.15.002</command>
</modules>
<modules compiler="oneapi-ifxgpu">
<command name="load">kokkos/git.7ff87a5-omp-sycl</command>
</modules>
<modules compiler="gnu">
<command name="unload">spack cmake</command>
Expand All @@ -3440,12 +3432,6 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<environment_variables mpilib="mpich" DEBUG="TRUE">
<env name="HYDRA_TOPO_DEBUG">1</env>
</environment_variables>
<environment_variables mpilib="impi">
<env name="I_MPI_DEBUG">10</env>
<env name="I_MPI_PIN_DOMAIN">omp</env>
<env name="I_MPI_PIN_ORDER">spread</env>
<env name="I_MPI_PIN_CELL">unit</env>
</environment_variables>
<environment_variables compiler="oneapi-ifxgpu">
<env name="ONEAPI_DEVICE_SELECTOR">level_zero:gpu</env>
<env name="ONEAPI_MPICH_GPU">NO_GPU</env>
Expand All @@ -3458,6 +3444,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<env name="GPU_TILE_COMPACT">/soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
<env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
<env name="FI_CXI_CQ_FILL_PERCENT">20</env>
<env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>
<env name="ZES_ENABLE_SYSMAN">1</env>
<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4</env>
</environment_variables>
<environment_variables compiler="oneapi-ifx">
<env name="LIBOMPTARGET_DEBUG">0</env><!--default 0, max 5 -->
Expand Down Expand Up @@ -3487,14 +3476,14 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<COMPILERS>oneapi-ifx,oneapi-ifxgpu,gnu</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<CHARGE_ACCOUNT>CSC249ADSE15_CNDA</CHARGE_ACCOUNT>
<SAVE_TIMING_DIR>/lus/gecko/projects/CSC249ADSE15_CNDA/performance_archive</SAVE_TIMING_DIR>
<SAVE_TIMING_DIR>/lus/flare/projects/CSC249ADSE15_CNDA/performance_archive</SAVE_TIMING_DIR>
<SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
<CIME_OUTPUT_ROOT>/lus/gecko/projects/CSC249ADSE15_CNDA/$USER/scratch</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/lus/gecko/projects/CSC249ADSE15_CNDA/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/lus/gecko/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<CIME_OUTPUT_ROOT>/lus/flare/projects/CSC249ADSE15_CNDA/$USER/scratch</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/lus/flare/projects/CSC249ADSE15_CNDA/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/lus/flare/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/lus/gecko/projects/CSC249ADSE15_CNDA/baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/lus/gecko/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc</CCSM_CPRNC>
<BASELINE_ROOT>/lus/flare/projects/CSC249ADSE15_CNDA/baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/lus/flare/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc</CCSM_CPRNC>
<GMAKE_J>16</GMAKE_J>
<TESTS>e3sm_developer</TESTS>
<NTEST_PARALLEL_JOBS>4</NTEST_PARALLEL_JOBS>
Expand All @@ -3503,54 +3492,48 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<MAX_TASKS_PER_NODE>208</MAX_TASKS_PER_NODE>
<MAX_TASKS_PER_NODE compiler="oneapi-ifxgpu">104</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>104</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">12</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">48</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>mpiexec</executable>
<!--executable>numactl -m 2-3 mpiexec</executable--><!--for HBM runs-->
<arguments>
<arg name="total_num_tasks">-np {{ total_tasks }} --label</arg>
<arg name="ranks_per_node">-ppn {{ tasks_per_node }}</arg>
<arg name="ranks_bind">--cpu-bind $ENV{RANKS_BIND} -envall</arg>
<arg name="ranks_bind">-envall</arg>
<arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS}</arg>
<arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="sh">/lus/gecko/projects/CSC249ADSE15_CNDA/modules/lmod.sh</init_path>
<init_path lang="sh">/lus/flare/projects/CSC249ADSE15_CNDA/modules/lmod.sh</init_path>
<init_path lang="csh">/soft/sunspot_migrate/soft/packaging/lmod/lmod/init/csh</init_path>
<init_path lang="python">/soft/sunspot_migrate/soft/packaging/lmod/lmod/init/env_modules_python.py</init_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<cmd_path lang="python">/soft/sunspot_migrate/soft/packaging/lmod/lmod/libexec/lmod python</cmd_path>
<modules>
<command name="purge"></command>
<command name="use">/soft/modulefiles</command>
<command name="use">/soft/restricted/CNDA/updates/modulefiles</command>
<command name="load">spack-pe-gcc cmake</command>
<command name="load">cmake</command>
</modules>
<modules compiler="!gnu">
<command name="load">oneapi/eng-compiler/2023.05.15.007</command>
<command name="load">oneapi/eng-compiler/2024.04.15.002</command>
</modules>
<modules compiler="oneapi-ifxgpu">
<command name="load">kokkos/git.7ff87a5-omp-sycl</command>
</modules>
<modules compiler="gnu">
<command name="unload">spack-pe-gcc cmake</command>
<command name="load">gcc/10.3.0</command>
</modules>
<modules>
<command name="load">cray-pals</command>
<command name="load">libfabric/1.15.2.0</command>
<command name="load">cray-libpals/1.3.2</command>
</modules>
</module_system>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables>
<env name="NETCDF_C_PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007</env>
<env name="NETCDF_FORTRAN_PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007</env>
<env name="PNETCDF_PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007</env>
<env name="LD_LIBRARY_PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007/lib:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007/lib:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007/lib:$ENV{LD_LIBRARY_PATH}</env>
<env name="PATH">/lus/gecko/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007/bin:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007/bin:/lus/gecko/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007/bin:$ENV{PATH}</env>
<env name="RANKS_BIND">list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203</env>
<env name="NETCDF_C_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002</env>
<env name="NETCDF_FORTRAN_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002</env>
<env name="PNETCDF_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002</env>
<env name="LD_LIBRARY_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/lib:$ENV{LD_LIBRARY_PATH}</env>
<env name="PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/bin:$ENV{PATH}</env>
</environment_variables>
<environment_variables DEBUG="TRUE">
<env name="HYDRA_TOPO_DEBUG">1</env>
Expand All @@ -3567,6 +3550,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<env name="GPU_TILE_COMPACT">/soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
<env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
<env name="FI_CXI_CQ_FILL_PERCENT">20</env>
<env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>
<env name="ZES_ENABLE_SYSMAN">1</env>
<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4</env>
</environment_variables>
<environment_variables compiler="oneapi-ifx">
<env name="LIBOMPTARGET_DEBUG">0</env><!--default 0, max 5 -->
Expand Down
14 changes: 14 additions & 0 deletions components/eamxx/cmake/machine-files/aurora.cmake
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
common_setup()

include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake)
include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake)

set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE)
set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE)
set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "")
set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "")

set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}")
set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}")
set(PNETCDF_PATH "$ENV{PNETCDF_PATH}")
10 changes: 10 additions & 0 deletions components/eamxx/cmake/machine-files/sunspot.cmake
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
common_setup()

include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake)
include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake)

set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE)
set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE)
set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "")
set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "")
2 changes: 1 addition & 1 deletion components/homme/src/share/compose/cedr_kokkos.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ typedef Kokkos::Experimental::HIPSpace CedrGpuSpace;
# endif
# if defined KOKKOS_ENABLE_SYCL
typedef Kokkos::Experimental::SYCL CedrGpuExeSpace;
typedef Kokkos::Experimental::SYCL> CedrGpuSpace;
typedef Kokkos::Experimental::SYCL CedrGpuSpace;
# endif
#endif

Expand Down
2 changes: 2 additions & 0 deletions components/homme/src/share/compose/compose_slmm_siqk.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,10 @@ class TestSphereToRefKernel {
// tol is on dx, not (a,b), so adjust slightly.
if ( ! info.success || err > 1e4*tol_) {
jinfo.nfails++;
#ifndef KOKKOS_ENABLE_SYCL
printf("calc_sphere_to_ref ei %d i %d j %d: nits %d re %1.1e\n",
ei, i, j, info.n_iterations, err);
#endif
}
jinfo.sum_nits += info.n_iterations;
jinfo.max_nits = max(jinfo.max_nits, info.n_iterations);
Expand Down

0 comments on commit 0ce7fcf

Please sign in to comment.