-
Notifications
You must be signed in to change notification settings - Fork 141
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'develop' into adds-comment
- Loading branch information
Showing
43 changed files
with
1,885 additions
and
182 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| #!/bin/bash | ||
| # This recipe is intended for NERSC Perlmutter https://docs.nersc.gov/systems/perlmutter | ||
| # It builds all the varaints of QMCPACK in the current directory | ||
| # last revision: Aug 12th 2023 | ||
| # | ||
| # How to invoke this script? | ||
| # build_nersc_perlmutter_Clang.sh # build all the variants assuming the current directory is the source directory. | ||
| # build_nersc_perlmutter_Clang.sh <source_dir> # build all the variants with a given source directory <source_dir> | ||
| # build_nersc_perlmutter_Clang.sh <source_dir> <install_dir> # build all the variants with a given source directory <source_dir> and install to <install_dir> | ||
|
|
||
| module load PrgEnv-gnu | ||
| module load cray-libsci | ||
| CRAY_LIBSCI_LIB=$CRAY_LIBSCI_PREFIX_DIR/lib/libsci_gnu_mp.so | ||
|
|
||
| module load PrgEnv-llvm/0.1 llvm/16 | ||
| module load cray-fftw/3.3.10.3 | ||
| module load cray-hdf5-parallel/1.12.2.3 | ||
| module load cmake/3.24.3 | ||
|
|
||
|
|
||
| echo "**********************************" | ||
| echo '$ clang -v' | ||
| clang -v | ||
| echo "**********************************" | ||
|
|
||
| TYPE=Release | ||
| Machine=perlmutter | ||
| Compiler=Clang16 | ||
|
|
||
| if [[ $# -eq 0 ]]; then | ||
| source_folder=`pwd` | ||
| elif [[ $# -eq 1 ]]; then | ||
| source_folder=$1 | ||
| else | ||
| source_folder=$1 | ||
| install_folder=$2 | ||
| fi | ||
|
|
||
| if [[ -f $source_folder/CMakeLists.txt ]]; then | ||
| echo Using QMCPACK source directory $source_folder | ||
| else | ||
| echo "Source directory $source_folder doesn't contain CMakeLists.txt. Pass QMCPACK source directory as the first argument." | ||
| exit | ||
| fi | ||
|
|
||
| for name in offload_cuda_real_MP offload_cuda_real offload_cuda_cplx_MP offload_cuda_cplx \ | ||
| cpu_real_MP cpu_real cpu_cplx_MP cpu_cplx | ||
| do | ||
|
|
||
| CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=$TYPE -DBLAS_LIBRARIES=$CRAY_LIBSCI_LIB" | ||
|
|
||
| if [[ $name == *"cplx"* ]]; then | ||
| CMAKE_FLAGS="$CMAKE_FLAGS -DQMC_COMPLEX=ON" | ||
| fi | ||
|
|
||
| if [[ $name == *"_MP"* ]]; then | ||
| CMAKE_FLAGS="$CMAKE_FLAGS -DQMC_MIXED_PRECISION=ON" | ||
| fi | ||
|
|
||
| if [[ $name == *"offload"* || $name == *"cuda"* ]]; then | ||
| CMAKE_FLAGS="$CMAKE_FLAGS -DQMC_GPU_ARCHS=sm_80" | ||
| fi | ||
|
|
||
| if [[ $name == *"offload"* ]]; then | ||
| CMAKE_FLAGS="$CMAKE_FLAGS -DENABLE_OFFLOAD=ON" | ||
| fi | ||
|
|
||
| if [[ $name == *"cuda"* ]]; then | ||
| CMAKE_FLAGS="$CMAKE_FLAGS -DENABLE_CUDA=ON" | ||
| fi | ||
|
|
||
| folder=build_${Machine}_${Compiler}_${name} | ||
|
|
||
| if [[ -v install_folder ]]; then | ||
| CMAKE_FLAGS="$CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX=$install_folder/$folder" | ||
| fi | ||
|
|
||
| echo "**********************************" | ||
| echo "$folder" | ||
| echo "$CMAKE_FLAGS" | ||
| echo "**********************************" | ||
|
|
||
| mkdir $folder | ||
| cd $folder | ||
|
|
||
| if [ ! -f CMakeCache.txt ] ; then | ||
| cmake $CMAKE_FLAGS -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx $source_folder | ||
| fi | ||
|
|
||
| if [[ -v install_folder ]]; then | ||
| make -j16 install && chmod -R -w $install_folder/$folder | ||
| else | ||
| make -j16 | ||
| fi | ||
|
|
||
| cd .. | ||
|
|
||
| echo | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,12 +9,14 @@ | |
| #// File created by: Ye Luo, [email protected], Argonne National Laboratory | ||
| #////////////////////////////////////////////////////////////////////////////////////// | ||
|
|
||
| set(CUDA_RT_SRCS CUDAfill.cpp CUDAallocator.cpp CUDAruntime.cpp) | ||
| set(CUDA_RT_SRCS CUDAfill.cpp CUDAallocator.cpp CUDAruntime.cpp CUDADeviceManager.cpp) | ||
| set(CUDA_LA_SRCS cuBLAS_missing_functions.cu) | ||
|
|
||
| add_library(platform_cuda_runtime ${CUDA_RT_SRCS}) | ||
| add_library(platform_cuda_LA ${CUDA_LA_SRCS}) | ||
|
|
||
| target_link_libraries(platform_cuda_runtime PRIVATE platform_host_runtime) | ||
|
|
||
| if(NOT QMC_CUDA2HIP) | ||
| target_link_libraries(platform_cuda_runtime PUBLIC CUDA::cudart) | ||
| target_link_libraries(platform_cuda_LA PUBLIC CUDA::cublas CUDA::cusolver) | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| ////////////////////////////////////////////////////////////////////////////////////// | ||
| // This file is distributed under the University of Illinois/NCSA Open Source License. | ||
| // See LICENSE file in top directory for details. | ||
| // | ||
| // Copyright (c) 2023 QMCPACK developers. | ||
| // | ||
| // File developed by: Ye Luo, [email protected], Argonne National Laboratory | ||
| // | ||
| // File created by: Ye Luo, [email protected], Argonne National Laboratory | ||
| // | ||
| ////////////////////////////////////////////////////////////////////////////////////// | ||
|
|
||
|
|
||
| #include "CUDADeviceManager.h" | ||
| #include <stdexcept> | ||
| #include "CUDAruntime.hpp" | ||
| #include "OutputManager.h" | ||
| #include "determineDefaultDeviceNum.h" | ||
|
|
||
| namespace qmcplusplus | ||
| { | ||
| CUDADeviceManager::CUDADeviceManager(int& default_device_num, int& num_devices, int local_rank, int local_size) | ||
| : cuda_default_device_num(-1), cuda_device_count(0) | ||
| { | ||
| cudaErrorCheck(cudaGetDeviceCount(&cuda_device_count), "cudaGetDeviceCount failed!"); | ||
| if (num_devices == 0) | ||
| num_devices = cuda_device_count; | ||
| else if (num_devices != cuda_device_count) | ||
| throw std::runtime_error("Inconsistent number of CUDA devices with the previous record!"); | ||
| if (cuda_device_count > local_size) | ||
| app_warning() << "More CUDA devices than the number of MPI ranks. " | ||
| << "Some devices will be left idle.\n" | ||
| << "There is potential performance issue with the GPU affinity. " | ||
| << "Use CUDA_VISIBLE_DEVICE or MPI launcher to expose desired devices.\n"; | ||
| if (num_devices > 0) | ||
| { | ||
| cuda_default_device_num = determineDefaultDeviceNum(cuda_device_count, local_rank, local_size); | ||
| if (default_device_num < 0) | ||
| default_device_num = cuda_default_device_num; | ||
| else if (default_device_num != cuda_default_device_num) | ||
| throw std::runtime_error("Inconsistent assigned CUDA devices with the previous record!"); | ||
|
|
||
| #pragma omp parallel | ||
| { | ||
| cudaErrorCheck(cudaSetDevice(cuda_default_device_num), "cudaSetDevice failed!"); | ||
| cudaErrorCheck(cudaFree(0), "cudaFree failed!"); | ||
| } | ||
| } | ||
| } | ||
| } // namespace qmcplusplus |
Oops, something went wrong.