Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

hommexx: fix templating #6449

Closed
wants to merge 2 commits into from
Closed

hommexx: fix templating #6449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
268539e
cmake files
oksanaguba May 28, 2024
5abd740
fix gpu templating for column ops
oksanaguba May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions components/homme/cmake/machineFiles/hops-bfb.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
## config that worked for this file:
#module purge
#
#module load cmake/3.25.2 aue/gcc/10.3.0 aue/hdf5/1.14.2-gcc-10.3.0-openmpi-4.1.6 aue/netcdf-c/4.9.2-gcc-10.3.0-openmpi-4.1.6
#module load aue/binutils/2.41 aue/cuda/11.8.0-gcc-10.3.0 aue/parallel-netcdf/1.12.3-gcc-10.3.0-openmpi-4.1.6 aue/openmpi/4.1.6-gcc-10.3.0

#EXECUTE_PROCESS(COMMAND nf-config --prefix
# RESULT_VARIABLE NFCONFIG_RESULT
# OUTPUT_VARIABLE NFCONFIG_OUTPUT
# ERROR_VARIABLE NFCONFIG_ERROR
# OUTPUT_STRIP_TRAILING_WHITESPACE
#)
#SET (NetCDF_Fortran_PATH "${NFCONFIG_OUTPUT}" CACHE STRING "")
#SET (NetCDF_C_PATH "${NCCONFIG_OUTPUT}" CACHE STRING "")

#SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")

# TPL settings
#set(HDF5_DIR $ENV{HDF5_ROOT} CACHE FILEPATH "")
#set(ZLIB_DIR $ENV{ZLIB_ROOT} CACHE FILEPATH "")
#set(CURL_ROOT $ENV{CURL_ROOT} CACHE FILEPATH "")
#set(CURL_LIBRARY -L$ENV{CURL_ROOT}/lib -lcurl CACHE LIST "")

# Flag tweaks
set(CMAKE_C_FLAGS "-w" CACHE STRING "")
set(ADD_CXX_FLAGS "-Xcudafe --diag_suppress=esa_on_defaulted_function_ignored -Wno-unknown-pragmas --fmad=false -O0 -I/projects/aue/hpc/builds/x86_64/rhel8/ompiflux/openmpi-4.1.6-gcc-10.3.0/install/linux-rhel8-x86_64/gcc-10.3.0/openmpi-4.1.6-67j5mxl/include/" CACHE STRING "")
set(ADD_Fortran_FLAGS " -ffp-contract=off -O0 -I/projects/aue/hpc/builds/x86_64/rhel8/ompiflux/openmpi-4.1.6-gcc-10.3.0/install/linux-rhel8-x86_64/gcc-10.3.0/openmpi-4.1.6-67j5mxl/include/" CACHE STRING "")
set(CMAKE_EXE_LINKER_FLAGS "-ldl" CACHE STRING "")
set(OPT_FLAGS "-O0" CACHE STRING "")
#set(DEBUG_FLAGS "-ffp-contract=off -g"CACHE STRING "")

# Homme settings
set(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")

set(HOMMEXX_VECTOR_SIZE 1 CACHE STRING "")
set(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
set(USE_QUEUING FALSE CACHE BOOL "")
set(USE_TRILINOS OFF CACHE BOOL "")
#set(WITH_PNETCDF FALSE CACHE FILEPATH "")
set(USE_NUM_PROCS 4 CACHE STRING "Num mpiprocs to use")
set(HAVE_EXTRAE TRUE CACHE BOOL "")
set(ENABLE_COLUMN_OPENMP OFF CACHE BOOL "")
set(ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
set(HOMMEXX_EXEC_SPACE "CUDA" CACHE STRING "")
set(HOMMEXX_CUDA_MAX_WARP_PER_TEAM 8 CACHE STRING "")

SET(BUILD_HOMME_PREQX_KOKKOS FALSE CACHE BOOL "")
SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
#sET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "")

set(HOMMEXX_BFB_TESTING TRUE CACHE STRING ON)

# Kokkos settings
set(ENABLE_OPENMP FALSE CACHE BOOL "")
set(Kokkos_ENABLE_DEBUG FALSE CACHE BOOL "")
set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION FALSE CACHE BOOL "")
set(Kokkos_ENABLE_CUDA TRUE CACHE BOOL "")
set(Kokkos_ENABLE_CUDA_LAMBDA TRUE CACHE BOOL "")
set(Kokkos_ARCH_HOPPER90 TRUE CACHE BOOL "")
set(Kokkos_ENABLE_DEPRECATED_CODE FALSE CACHE BOOL "")
set(Kokkos_ENABLE_EXPLICIT_INSTANTIATION FALSE CACHE BOOL "")

# Compilers
set(CMAKE_C_COMPILER mpicc CACHE STRING "")
set(CMAKE_Fortran_COMPILER mpif90 CACHE STRING "")
######## CHANGE THAT
set(CMAKE_CXX_COMPILER "/home/onguba/acme-b2/externals/ekat/extern/kokkos/bin/nvcc_wrapper" CACHE STRING "")
68 changes: 68 additions & 0 deletions components/homme/cmake/machineFiles/hops.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
## config that worked for this file:
#
#module purge
#
#module load cmake/3.25.2 aue/gcc/10.3.0 aue/hdf5/1.14.2-gcc-10.3.0-openmpi-4.1.6 aue/netcdf-c/4.9.2-gcc-10.3.0-openmpi-4.1.6
#module load aue/binutils/2.41 aue/cuda/11.8.0-gcc-10.3.0 aue/parallel-netcdf/1.12.3-gcc-10.3.0-openmpi-4.1.6 aue/openmpi/4.1.6-gcc-10.3.0

#EXECUTE_PROCESS(COMMAND nf-config --prefix
# RESULT_VARIABLE NFCONFIG_RESULT
# OUTPUT_VARIABLE NFCONFIG_OUTPUT
# ERROR_VARIABLE NFCONFIG_ERROR
# OUTPUT_STRIP_TRAILING_WHITESPACE
#)
#SET (NetCDF_Fortran_PATH "${NFCONFIG_OUTPUT}" CACHE STRING "")
#SET (NetCDF_C_PATH "${NCCONFIG_OUTPUT}" CACHE STRING "")

#SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")

# TPL settings
#set(HDF5_DIR $ENV{HDF5_ROOT} CACHE FILEPATH "")
#set(ZLIB_DIR $ENV{ZLIB_ROOT} CACHE FILEPATH "")
#set(CURL_ROOT $ENV{CURL_ROOT} CACHE FILEPATH "")
#set(CURL_LIBRARY -L$ENV{CURL_ROOT}/lib -lcurl CACHE LIST "")

# Flag tweaks
set(CMAKE_C_FLAGS "-w" CACHE STRING "")
set(ADD_CXX_FLAGS "-Xcudafe --diag_suppress=esa_on_defaulted_function_ignored -Wno-unknown-pragmas -I/projects/aue/hpc/builds/x86_64/rhel8/ompiflux/openmpi-4.1.6-gcc-10.3.0/install/linux-rhel8-x86_64/gcc-10.3.0/openmpi-4.1.6-67j5mxl/include/" CACHE STRING "")
set(ADD_Fortran_FLAGS "-I/projects/aue/hpc/builds/x86_64/rhel8/ompiflux/openmpi-4.1.6-gcc-10.3.0/install/linux-rhel8-x86_64/gcc-10.3.0/openmpi-4.1.6-67j5mxl/include/" CACHE STRING "")
set(CMAKE_EXE_LINKER_FLAGS "-ldl" CACHE STRING "")
set(OPT_FLAGS "-O3" CACHE STRING "")
#set(DEBUG_FLAGS "-ffp-contract=off -g"CACHE STRING "")

# Homme settings
set(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")

set(HOMMEXX_VECTOR_SIZE 1 CACHE STRING "")
set(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
set(USE_QUEUING FALSE CACHE BOOL "")
set(USE_TRILINOS OFF CACHE BOOL "")
#set(WITH_PNETCDF FALSE CACHE FILEPATH "")
set(USE_NUM_PROCS 4 CACHE STRING "Num mpiprocs to use")
set(HAVE_EXTRAE TRUE CACHE BOOL "")
set(ENABLE_COLUMN_OPENMP OFF CACHE BOOL "")
set(ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
set(HOMMEXX_EXEC_SPACE "CUDA" CACHE STRING "")
set(HOMMEXX_CUDA_MAX_WARP_PER_TEAM 8 CACHE STRING "")

SET(BUILD_HOMME_PREQX_KOKKOS FALSE CACHE BOOL "")
SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
#sET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "")

#set(HOMMEXX_BFB_TESTING TRUE CACHE STRING ON)

# Kokkos settings
set(ENABLE_OPENMP FALSE CACHE BOOL "")
set(Kokkos_ENABLE_DEBUG FALSE CACHE BOOL "")
set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION FALSE CACHE BOOL "")
set(Kokkos_ENABLE_CUDA TRUE CACHE BOOL "")
set(Kokkos_ENABLE_CUDA_LAMBDA TRUE CACHE BOOL "")
set(Kokkos_ARCH_HOPPER90 TRUE CACHE BOOL "")
set(Kokkos_ENABLE_DEPRECATED_CODE FALSE CACHE BOOL "")
set(Kokkos_ENABLE_EXPLICIT_INSTANTIATION FALSE CACHE BOOL "")

# Compilers
set(CMAKE_C_COMPILER mpicc CACHE STRING "")
set(CMAKE_Fortran_COMPILER mpif90 CACHE STRING "")
######## CHANGE THAT
set(CMAKE_CXX_COMPILER "/home/onguba/acme-b2/externals/ekat/extern/kokkos/bin/nvcc_wrapper" CACHE STRING "")
8 changes: 8 additions & 0 deletions components/homme/src/share/cxx/ExecSpaceDefs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,11 +383,19 @@ struct Dispatch<HommexxGPU> {
// serialize parallel scans.

// Detect the value type
#ifdef HOMMEXX_ENABLE_GPU
using value_type =
typename Kokkos::Impl::FunctorAnalysis
< Kokkos::Impl::FunctorPatternInterface::SCAN
, void
, Lambda, void >::value_type ;
#else
using value_type =
typename Kokkos::Impl::FunctorAnalysis
< Kokkos::Impl::FunctorPatternInterface::SCAN
, void
, Lambda >::value_type ;
Copy link
Contributor

@trey-ornl trey-ornl Aug 27, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So adding the extra void template argument doesn't work for the non-GPU case, thus requiring the #ifdef?

#endif

// All threads init result.
value_type accumulator = Kokkos::reduction_identity<value_type>::sum();
Expand Down
Loading