Skip to content

Commit

Permalink
Merge pull request #3357 from ye-luo/separate-omptarget-math
Browse files Browse the repository at this point in the history
Adjust scalar/vector math function handling
  • Loading branch information
ye-luo authored Aug 13, 2021
2 parents 5716f3e + da9e4f1 commit af4fce1
Show file tree
Hide file tree
Showing 39 changed files with 304 additions and 240 deletions.
4 changes: 2 additions & 2 deletions CMake/CheckSincos.cmake
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
include(${CMAKE_ROOT}/Modules/CheckCXXSourceCompiles.cmake)
include(CheckCXXSourceCompiles)

set(CMAKE_REQUIRED_LIBRARIES Math::scalar_vector_functions)

set(SINCOS_TEST_SRC
"#include \"${SINCOS_INCLUDE}\"
int main(void) {
Expand All @@ -12,3 +11,4 @@ sincos(input,&outsin, &outcos);
")

check_cxx_source_compiles("${SINCOS_TEST_SRC}" HAVE_SINCOS)
unset(CMAKE_REQUIRED_LIBRARIES)
55 changes: 55 additions & 0 deletions CMake/FindAMDLibM.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Output: HAVE_AMD_LIBM

message(STATUS "Setting up AMD LibM libraries")
message(WARNING "In limited benchmarks, AMD LibM slows down QMCPACK. "
"Please check carefully before using it in production runs.")

include(CheckCXXSourceCompiles)

find_path(
AMD_LIBM_INCLUDE_DIR NAME amdlibm.h
HINTS ${AMD_LIBM_ROOT} ${AOCL_ROOT}
PATH_SUFFIXES include)

if(NOT AMD_LIBM_INCLUDE_DIR)
message(FATAL_ERROR "AMD_LIBM_INCLUDE_DIR not set. Header file amdlib.h not found!")
else()
message(STATUS "Header file amdlib.h found at ${AMD_LIBM_INCLUDE_DIR}")
endif()

find_library(
AMD_LIBM_LIBRARY NAME amdlibm
HINTS ${AMD_LIBM_ROOT} ${AOCL_ROOT}
PATH_SUFFIXES lib)

if(NOT AMD_LIBM_LIBRARY)
message(FATAL_ERROR "AMD_LIBM_LIBRARY not set. Library file amdlibm not found!")
else()
message(STATUS "Library file amdlibm found as ${AMD_LIBM_LIBRARY}")
endif()

set(CMAKE_REQUIRED_INCLUDES "${AMD_LIBM_INCLUDE_DIR}")
set(CMAKE_REQUIRED_LIBRARIES "${AMD_LIBM_LIBRARY}")
check_cxx_source_compiles(
"
#include <amdlibm.h>
int main() {
double d_in(0), d_sin, d_cos;
amd_sincos(d_in, &d_sin, &d_cos);
float s_in(0), s_sin, s_cos;
amd_sincosf(s_in, &s_sin, &s_cos);
return 0;
}
"
HAVE_AMD_LIBM)
unset(CMAKE_REQUIRED_INCLUDES)
unset(CMAKE_REQUIRED_LIBRARIES)

if(HAVE_AMD_LIBM)
target_compile_definitions(Math::scalar_vector_functions INTERFACE "HAVE_AMD_LIBM")
target_include_directories(Math::scalar_vector_functions INTERFACE "${AMD_LIBM_INCLUDE_DIR}")
target_link_libraries(Math::scalar_vector_functions INTERFACE "${AMD_LIBM_LIBRARY}")
message(STATUS "AMD LIBM found")
else()
message(FATAL_ERROR "AMD LIBM not found")
endif()
9 changes: 3 additions & 6 deletions CMake/FindFFTW.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,9 @@ if(FFTW_INCLUDE_DIR AND FFTW_LIBRARIES)
message(STATUS "FFTW_INCLUDE_DIR=${FFTW_INCLUDE_DIR}")
message(STATUS "FFTW_LIBRARIES=${FFTW_LIBRARIES}")
set(FFTW_FOUND TRUE)
#create FFTW3 target
add_library(Math::FFTW3 INTERFACE IMPORTED)
set_target_properties(
Math::FFTW3
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIR}" INTERFACE_COMPILE_DEFINITIONS "HAVE_LIBFFTW"
INTERFACE_LINK_LIBRARIES "${FFTW_LIBRARIES}")
target_include_directories(Math::FFTW3 INTERFACE "${FFTW_INCLUDE_DIR}")
target_compile_definitions(Math::FFTW3 INTERFACE "HAVE_LIBFFTW")
target_link_libraries(Math::FFTW3 INTERFACE "${FFTW_LIBRARIES}")
endif()

mark_as_advanced(FFTW_INCLUDE_DIR FFTW_LIBRARIES FFTW_FOUND)
19 changes: 7 additions & 12 deletions CMake/FindIBMMASS.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -81,23 +81,18 @@ if(HAVE_MASS OR HAVE_MASSV)
set(MASS_FOUND TRUE)
message(STATUS "MASS found: HAVE_MASS=${HAVE_MASS}, HAVE_MASSV=${HAVE_MASSV}")

#create scalar_vector_functions target
add_library(Math::scalar_vector_functions INTERFACE IMPORTED)
set_target_properties(
Math::scalar_vector_functions PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${MASS_INCLUDE_DIRECTORIES}"
INTERFACE_LINK_OPTIONS "${MASS_LINKER_FLAGS}")
target_include_directories(Math::scalar_vector_functions INTERFACE "${MASS_INCLUDE_DIRECTORIES}")
target_link_options(Math::scalar_vector_functions INTERFACE "${MASS_LINKER_FLAGS}")

if(HAVE_MASS)
set_property(
TARGET Math::scalar_vector_functions
APPEND
PROPERTY INTERFACE_COMPILE_DEFINITIONS "HAVE_MASS")
set_property(TARGET Math::scalar_vector_functions APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${MASS_LIBRARY}")
target_compile_definitions(Math::scalar_vector_functions INTERFACE "HAVE_MASS")
target_link_libraries(Math::scalar_vector_functions INTERFACE "${MASS_LIBRARY}")
set(SINCOS_INCLUDE mass.h)
endif()

if(HAVE_MASSV)
set_property(TARGET Math::scalar_vector_functions APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS "HAVE_MASSV")
set_property(TARGET Math::scalar_vector_functions APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${MASSV_LIBRARY}")
target_compile_definitions(Math::scalar_vector_functions INTERFACE "HAVE_MASSV")
target_link_libraries(Math::scalar_vector_functions INTERFACE "${MASSV_LIBRARY}")
endif()

else()
Expand Down
53 changes: 16 additions & 37 deletions CMake/FindMKL.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,20 @@ set(MKL_LIBRARIES ${LAPACK_LINKER_FLAGS} ${LAPACK_LIBRARIES})

message(STATUS "Looking for Intel MKL library header files")

# Finding and setting the MKL_INCLUDE_DIRECTORIES based on MKL_ROOT, $ENV{MKLROOT}, $ENV{MKL_ROOT}, $ENV{MKL_HOME}
# Finding and setting the MKL_INCLUDE based on MKL_ROOT, $ENV{MKLROOT}, $ENV{MKL_ROOT}, $ENV{MKL_HOME}
# Extremely Basic Support of common mkl module environment variables
find_path(
MKL_INCLUDE_DIRECTORIES "mkl.h"
MKL_INCLUDE "mkl.h"
HINTS ${MKL_ROOT} $ENV{MKLROOT} $ENV{MKL_ROOT} $ENV{MKL_HOME}
PATH_SUFFIXES include)
if(NOT MKL_INCLUDE_DIRECTORIES)
if(NOT MKL_INCLUDE)
# Finding MKL headers in the system
find_path(MKL_INCLUDE_DIRECTORIES "mkl.h" PATH_SUFFIXES mkl)
find_path(MKL_INCLUDE "mkl.h" PATH_SUFFIXES mkl)
endif()

if(MKL_INCLUDE_DIRECTORIES)
message(STATUS "MKL_INCLUDE_DIRECTORIES: ${MKL_INCLUDE_DIRECTORIES}")
else(MKL_INCLUDE_DIRECTORIES)
if(MKL_INCLUDE)
message(STATUS "MKL_INCLUDE: ${MKL_INCLUDE}")
else(MKL_INCLUDE)
message(STATUS "mkl.h cannot be found")
if(COMPILER MATCHES "Intel")
message(
Expand All @@ -30,32 +30,24 @@ else(MKL_INCLUDE_DIRECTORIES)
else(COMPILER MATCHES "Intel")
message(FATAL_ERROR "Pass mkl root directory to cmake via MKL_ROOT.")
endif(COMPILER MATCHES "Intel")
endif(MKL_INCLUDE_DIRECTORIES)
endif(MKL_INCLUDE)

# Check for mkl.h
file(WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl.cxx"
"#include <iostream>\n #include <mkl.h>\n int main() { return 0; }\n")
try_compile(
HAVE_MKL ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl.cxx
CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${MKL_INCLUDE_DIRECTORIES} "
CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${MKL_INCLUDE} "
LINK_LIBRARIES "${MKL_LIBRARIES}"
OUTPUT_VARIABLE MKL_OUT)
if(NOT HAVE_MKL)
message("${MKL_OUT}")
endif(NOT HAVE_MKL)

# Check for mkl_vml_functions.h
file(WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_vml.cxx"
"#include <iostream>\n #include <mkl_vml_functions.h>\n int main() { return 0; }\n")
try_compile(
HAVE_MKL_VML ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_vml.cxx
CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${MKL_INCLUDE_DIRECTORIES} "
OUTPUT_VARIABLE MKL_OUT)

# Check for fftw3
find_path(
MKL_FFTW3 "fftw3.h"
HINTS ${MKL_INCLUDE_DIRECTORIES}
HINTS ${MKL_INCLUDE}
PATH_SUFFIXES fftw)
if(MKL_FFTW3)
file(WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_fftw3.cxx"
Expand All @@ -71,29 +63,16 @@ endif(MKL_FFTW3)

if(HAVE_MKL)
set(MKL_FOUND TRUE)
message(STATUS "MKL found: HAVE_MKL=${HAVE_MKL}, HAVE_MKL_VML=${HAVE_MKL_VML}, HAVE_MKL_FFTW3=${HAVE_MKL_FFTW3}")
message(STATUS "MKL found: HAVE_MKL=${HAVE_MKL}, HAVE_MKL_FFTW3=${HAVE_MKL_FFTW3}")

#Add BLAS_LAPACK header
set_target_properties(Math::BLAS_LAPACK PROPERTIES INTERFACE_COMPILE_DEFINITIONS "HAVE_MKL"
INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIRECTORIES}")

if(HAVE_MKL_VML)
#create scalar_vector_functions target
add_library(Math::scalar_vector_functions INTERFACE IMPORTED)
set_target_properties(
Math::scalar_vector_functions
PROPERTIES INTERFACE_COMPILE_DEFINITIONS "HAVE_MKL;HAVE_MKL_VML" INTERFACE_INCLUDE_DIRECTORIES
"${MKL_INCLUDE_DIRECTORIES}"
INTERFACE_LINK_LIBRARIES "${MKL_LIBRARIES}")
endif(HAVE_MKL_VML)
target_compile_definitions(Math::BLAS_LAPACK INTERFACE "HAVE_MKL")
target_include_directories(Math::BLAS_LAPACK INTERFACE "${MKL_INCLUDE}")

if(HAVE_MKL_FFTW3)
#create FFTW3 target
add_library(Math::FFTW3 INTERFACE IMPORTED)
set_target_properties(
Math::FFTW3
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${MKL_FFTW3}" INTERFACE_COMPILE_DEFINITIONS "HAVE_MKL;HAVE_LIBFFTW"
INTERFACE_LINK_LIBRARIES "${MKL_LIBRARIES}")
target_compile_definitions(Math::FFTW3 INTERFACE "HAVE_MKL;HAVE_LIBFFTW")
target_include_directories(Math::FFTW3 INTERFACE "${MKL_FFTW3}")
target_link_libraries(Math::FFTW3 INTERFACE "${MKL_LIBRARIES}")
endif(HAVE_MKL_FFTW3)
else(HAVE_MKL)
set(MKL_FOUND FALSE)
Expand Down
25 changes: 25 additions & 0 deletions CMake/FindMKLVML.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Simple file to verify MKL VML
# Input: MKL_FOUND, MKL_INCLUDE, MKL_LIBRARIES
# Output: HAVE_MKL_VML

message(STATUS "Setting up Intel MKL Vector Math Library (VML)")

if(NOT MKL_FOUND)
message(FATAL_ERROR "MKL was not found. Cannot proceed with Vector Math Library (VML) searching!")
endif()

include(CheckIncludeFileCXX)

# Check for mkl_vml_functions.h
set(CMAKE_REQUIRED_INCLUDES "${MKL_INCLUDE}")
check_include_file_cxx(mkl_vml_functions.h HAVE_MKL_VML)
unset(CMAKE_REQUIRED_INCLUDES)

if(HAVE_MKL_VML)
target_compile_definitions(Math::scalar_vector_functions INTERFACE "HAVE_MKL;HAVE_MKL_VML")
target_include_directories(Math::scalar_vector_functions INTERFACE "${MKL_INCLUDE}")
target_link_libraries(Math::scalar_vector_functions INTERFACE "${MKL_LIBRARIES}")
message(STATUS "MKL VML found")
else()
message(FATAL_ERROR "MKL VML not found")
endif(HAVE_MKL_VML)
39 changes: 29 additions & 10 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -579,11 +579,20 @@ message(STATUS "LAPACK libraries: ${LAPACK_LIBRARIES}")
set_target_properties(Math::BLAS_LAPACK PROPERTIES INTERFACE_LINK_LIBRARIES
"${LAPACK_LINKER_FLAGS};${LAPACK_LIBRARIES}")

#create scalar_vector_functions target
add_library(Math::scalar_vector_functions INTERFACE IMPORTED)
#create fftw3 target
add_library(Math::FFTW3 INTERFACE IMPORTED)

# Detects MKL header files and other components.
if(LAPACK_LIBRARIES MATCHES "mkl_core")
include(CMake/FindMKL.cmake)
endif()

if(MKL_FOUND)
set(MATH_VENDOR_GUESS "INTEL_VML")
endif()

# AFQMC requires MKL sparse for good performance (roughly a factor of 2x)
if(BUILD_AFQMC AND NOT MKL_FOUND)
message(
Expand All @@ -595,19 +604,29 @@ endif()
#-------------------------------------------------------------------
# set up scalar/vector math libraries
#-------------------------------------------------------------------
set(ENABLE_MASS
FALSE
CACHE BOOL "ENABLE MASS scalar and vector math libraries for Power architecture")
set(VALID_MATH_VENDORS "GENERIC" "INTEL_VML" "IBM_MASS" "AMD_LIBM")
if(NOT DEFINED MATH_VENDOR_GUESS)
set(MATH_VENDOR_GUESS "GENERIC")
endif()
set(QMC_MATH_VENDOR
${MATH_VENDOR_GUESS}
CACHE STRING "Vendor optimized libraries for scalar and vector math functions")

# Perform math vendor option check
if(NOT QMC_MATH_VENDOR IN_LIST VALID_MATH_VENDORS)
message(FATAL_ERROR "Invalid vendor math library ${QMC_MATH_VENDOR}, value must be one of ${VALID_MATH_VENDORS}")
else()
message(STATUS "Selected vendor math library ${QMC_MATH_VENDOR}")
endif()

# This needs to go before HAVE_SINCOS
if(ENABLE_MASS)
if(QMC_MATH_VENDOR STREQUAL "INTEL_VML")
include(CMake/FindMKLVML.cmake)
elseif(QMC_MATH_VENDOR STREQUAL "IBM_MASS")
include(CMake/FindIBMMASS.cmake)
endif(ENABLE_MASS)

if(NOT MASS_FOUND AND NOT HAVE_MKL_VML)
message(STATUS "No usable vector math library detected.")
add_library(Math::scalar_vector_functions INTERFACE IMPORTED)
endif(NOT MASS_FOUND AND NOT HAVE_MKL_VML)
elseif(QMC_MATH_VENDOR STREQUAL "AMD_LIBM")
include(CMake/FindAMDLibM.cmake)
endif()

# CheckSincos relies on SINCOS_INCLUDE which may be modified based on vendor libraries
if(NOT SINCOS_INCLUDE)
Expand Down
4 changes: 2 additions & 2 deletions config/build_olcf_summit_Clang.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ if [[ ! -d /ccs/proj/mat151/opt/modules ]] ; then
exit 1
fi
module use /ccs/proj/mat151/opt/modules
module load llvm/master-latest
module load llvm/main-20210811

#the XL built fftw is buggy, use the gcc version
#module load fftw
Expand All @@ -39,7 +39,7 @@ for name in offload_cuda_real offload_cuda_real_MP offload_cuda_cplx offload_cud
cpu_real cpu_real_MP cpu_cplx cpu_cplx_MP
do

CMAKE_FLAGS="-D CMAKE_BUILD_TYPE=$TYPE -D ENABLE_MASS=1 -D MASS_ROOT=/sw/summit/xl/16.1.1-5/xlmass/9.1.1 -D MPIEXEC_EXECUTABLE=`which jsrun` -D MPIEXEC_NUMPROC_FLAG='-n' -D MPIEXEC_PREFLAGS='-c;16;-g;1;-b;packed:16;--smpiargs=off'"
CMAKE_FLAGS="-D CMAKE_BUILD_TYPE=$TYPE -D QMC_MATH_VENDOR=IBM_MASS -D MASS_ROOT=/sw/summit/xl/16.1.1-5/xlmass/9.1.1 -D MPIEXEC_EXECUTABLE=`which jsrun` -D MPIEXEC_NUMPROC_FLAG='-n' -D MPIEXEC_PREFLAGS='-c;16;-g;1;-b;packed:16;--smpiargs=off'"

if [[ $name == *"cplx"* ]]; then
CMAKE_FLAGS="$CMAKE_FLAGS -D QMC_COMPLEX=1"
Expand Down
66 changes: 0 additions & 66 deletions config/build_olcf_summit_XL.sh

This file was deleted.

Loading

0 comments on commit af4fce1

Please sign in to comment.