Merge remote-tracking branch 'williamfgc/ref-add-SPOSetT' into update…

…_RotatedSPOsT_with_mw
williamfgc · Nov 10, 2023 · 6f2a7f1 · 6f2a7f1
2 parents 6e4910d + c3298e8
commit 6f2a7f1
Show file tree

Hide file tree

Showing 474 changed files with 31,618 additions and 47,353 deletions.
diff --git a/CMake/DetermineDeviceArchitectures.cmake b/CMake/DetermineDeviceArchitectures.cmake
@@ -63,6 +63,7 @@ function(verifyNVIDIAGPUconsistency)
 endfunction()
 
 # auto detect QMC_GPU_ARCHS if not set by user and GPU features are enabled.
+# CMAKE_CUDA/HIP_ARCHITECTURES are used as hints
 if(NOT QMC_GPU_ARCHS AND ENABLE_CUDA)
   if(QMC_CUDA2HIP)
     detectAMDGPU()
@@ -93,3 +94,13 @@ endif()
 set(QMC_GPU_ARCHS
     ${QMC_GPU_ARCHS}
     CACHE STRING "Accelerator device architectures" FORCE)
+
+# QMC_GPU_ARCHS is the single source of truth and thus overwrite CMAKE_CUDA/HIP_ARCHITECTURES
+if(ENABLE_CUDA)
+  if(QMC_CUDA2HIP)
+    set(CMAKE_HIP_ARCHITECTURES ${QMC_GPU_ARCHS} CACHE STRING "HIP architectures" FORCE)
+  else()
+    string(REPLACE "sm_" "" CUDA_ARCH_NUMBERS "${QMC_GPU_ARCHS}")
+    set(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_NUMBERS} CACHE STRING "CUDA architectures" FORCE)
+  endif()
+endif()
diff --git a/CMake/FindMKL.cmake b/CMake/FindMKL.cmake
@@ -85,7 +85,7 @@ endif(HAVE_MKL)
 if(HAVE_MKL AND ENABLE_SYCL)
   find_library(MKL_SYCL mkl_sycl
     HINTS ${MKL_ROOT} $ENV{MKLROOT} $ENV{MKL_ROOT} $ENV{MKL_HOME}
-    PATH_SUFFIXES lib/intel64
+    PATH_SUFFIXES lib/intel64 lib
     REQUIRED
   )
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -368,32 +368,34 @@ if(alignment_remainder)
 endif()
 message(STATUS "QMC_SIMD_ALIGNMENT is set to ${QMC_SIMD_ALIGNMENT}")
 
-#---------------------------------------------------------
-# Determine if OpenMP taskloop works with the CXX compiler
-#---------------------------------------------------------
-include(TestOpenMPtaskloop)
-option(ENABLE_OMP_TASKLOOP "Enable OpenMP taskloop" ${OMP_TASKLOOP_OKAY})
-message(STATUS "ENABLE_OMP_TASKLOOP is set to ${ENABLE_OMP_TASKLOOP}")
-
-#---------------------------------------------------------
-# Set up OpenMP offload compile options
-#---------------------------------------------------------
-set(QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT OFF)
-if(ENABLE_OFFLOAD AND DEFINED OPENMP_OFFLOAD_COMPILE_OPTIONS)
-  message(STATUS "OpenMP offload CXX flags: ${OPENMP_OFFLOAD_COMPILE_OPTIONS}")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPENMP_OFFLOAD_COMPILE_OPTIONS}")
-  if(${COMPILER} MATCHES "Clang"
-     AND OPENMP_OFFLOAD_COMPILE_OPTIONS MATCHES "gfx"
-     AND QMC_CUDA2HIP)
-    # As of 11/2021, QMC_OFFLOAD_MEM_ASSOCIATED=ON is needed for AMD and mainline LLVM compilers
-    # when using OpenMP offload to AMD GPU together with HIP.
-    set(QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT ON)
+if(QMC_OMP)
+  #---------------------------------------------------------
+  # Determine if OpenMP taskloop works with the CXX compiler
+  #---------------------------------------------------------
+  include(TestOpenMPtaskloop)
+  option(ENABLE_OMP_TASKLOOP "Enable OpenMP taskloop" ${OMP_TASKLOOP_OKAY})
+  message(STATUS "ENABLE_OMP_TASKLOOP is set to ${ENABLE_OMP_TASKLOOP}")
+
+  #---------------------------------------------------------
+  # Set up OpenMP offload compile options
+  #---------------------------------------------------------
+  set(QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT OFF)
+  if(ENABLE_OFFLOAD AND DEFINED OPENMP_OFFLOAD_COMPILE_OPTIONS)
+    message(STATUS "OpenMP offload CXX flags: ${OPENMP_OFFLOAD_COMPILE_OPTIONS}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPENMP_OFFLOAD_COMPILE_OPTIONS}")
+    if(${COMPILER} MATCHES "Clang"
+	AND OPENMP_OFFLOAD_COMPILE_OPTIONS MATCHES "gfx"
+	AND QMC_CUDA2HIP)
+      # As of 11/2021, QMC_OFFLOAD_MEM_ASSOCIATED=ON is needed for AMD and mainline LLVM compilers
+      # when using OpenMP offload to AMD GPU together with HIP.
+      set(QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT ON)
+    endif()
   endif()
+  # Some OpenMP offload runtime libraries have composibility issue with a vendor native runtime.
+  # A workaround is making the vendor native runtime responsible for memory allocations and OpenMP associate/disassocate them.
+  cmake_dependent_option(QMC_OFFLOAD_MEM_ASSOCIATED "Manage OpenMP memory allocations via the vendor runtime"
+    ${QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT} "ENABLE_OFFLOAD;ENABLE_CUDA" OFF)
 endif()
-# Some OpenMP offload runtime libraries have composibility issue with a vendor native runtime.
-# A workaround is making the vendor native runtime responsible for memory allocations and OpenMP associate/disassocate them.
-cmake_dependent_option(QMC_OFFLOAD_MEM_ASSOCIATED "Manage OpenMP memory allocations via the vendor runtime"
-                       ${QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT} "ENABLE_OFFLOAD;ENABLE_CUDA" OFF)
 
 #-------------------------------------------------------------------------------------
 # consider making this always on if OpenMP is no longer UB with Thread Support Library

diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -69,7 +69,7 @@ sphinxcontrib-qthelp==1.0.3
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
-urllib3==2.0.6
+urllib3==2.0.7
     # via requests
 
 # The following packages are considered to be unsafe in a requirements file:

diff --git a/external_codes/mpi_wrapper/mpi3/.gitlab-ci.yml b/external_codes/mpi_wrapper/mpi3/.gitlab-ci.yml
@@ -1,15 +1,15 @@
 # -*-indent-tabs-mode:nil;c-basic-offset:2;tab-width:4;-*-
 # Copyright 2020-2023 Alfredo A. Correa
 
-image: correaadock/gnudev:v2
+image: debian:testing
 
 variables:
   GIT_SUBMODULE_STRATEGY: recursive
 
 openmpi:
   stage: build
   script:
-    - apt update -qq && apt install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make g++ git libboost-serialization-dev
+    - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make g++ git libboost-serialization-dev
     - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3
     - cd test
     - mkdir build && cd build
@@ -18,12 +18,43 @@ openmpi:
     - cmake --build . --parallel 2 || make VERBOSE=1
     - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure
 
+exampi:
+  allow_failure: true
+  stage: build
+  script:
+    - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates cmake g++ git libboost-serialization-dev make python3 ssh strace  #  ca-certificates libopenmpi-dev openmpi-bin cmake make g++ git libboost-serialization-dev
+    - git clone https://correaa:${EXATOKEN}@github.com/tonyskjellum/ExaMPI.git
+    - cd ExaMPI
+    - mkdir build && cd build
+    - cmake .. --install-prefix=$HOME/exa
+    - make -j 4
+    - make install
+    - export PATH=$HOME/exa/bin:$PATH
+    - export PATH=$HOME/exa/runtime:$PATH
+    - export LD_LIBRARY_PATH=$HOME/exa/lib:$LD_LIBRARY_PATH
+    - export MPI_PATH=$HOME/exa/bin
+    - export MPI_LIB=$HOME/exa/lib
+    - export MPI_INC=$HOME/exa/include
+    - export MPI_HOME=$HOME/exa
+    - which mpicxx
+    - which mpirun
+    - strace mpirun -n 4 tests/integration_tests/allreduce
+    - strace mpirun -n 4 tests/integration_tests/alltoall
+    - ctest --output-on-failure
+    - cd ../..
+    - mkdir build && cd build
+    - which mpicxx
+    - mpicxx --version
+    - cmake .. -DCMAKE_BUILD_TYPE=Debug -DUSE_EXAMPI=1 -DMPI_HOME=$HOME/exa
+    - make -j 2 || make VERBOSE=1
+    - ls
+    - ctest --output-on-failure
+
 icpc-intelmpi:
   stage: build
   image: intel/oneapi-hpckit:latest
-  allow_failure: true
   script:
-    - apt-get update && apt-get install --no-install-recommends -y --quiet ca-certificates cmake curl g++ git make libboost-test-dev libboost-serialization-dev
+    - apt-get update && apt-get install --no-install-recommends -y --quiet bash ca-certificates cmake curl g++ git make libboost-test-dev libboost-serialization-dev
     - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3
     - cd test
     - mkdir build && cd build
@@ -50,8 +81,9 @@ icpx-intelmpi:
 
 openmpi-clang:
   stage: build
+  image: debian:testing
   script:
-    - apt update -qq && apt install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make clang g++ git libstdc++-12-dev libboost-serialization-dev
+    - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make clang g++ git libstdc++-12-dev libboost-serialization-dev
     - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3
     - cd test
     - mkdir build && cd build
@@ -63,10 +95,11 @@ openmpi-clang:
     - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure
   needs: ["openmpi"]
 
-openmpi-clang20:
+"openmpi-clang20":
   stage: build
+  image: debian:stable
   script:
-    - apt update -qq && apt install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make clang git libstdc++-12-dev libboost-serialization-dev
+    - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make clang git libstdc++-12-dev libboost-serialization-dev
     - cd test
     - mkdir build && cd build
     - export MPI_OVERSUBSCRIBE="--oversubscribe"
@@ -80,7 +113,7 @@ openmpi-clang20:
 openmpi-clang-tidy:
   stage: build
   script:
-    - apt update -qq && apt install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin clang libstdc++-12-dev clang-tidy cmake git make libboost-serialization-dev
+    - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin clang libstdc++-12-dev clang-tidy cmake git make libboost-serialization-dev
     - mkdir build && cd build
     - export MPI_OVERSUBSCRIBE="--oversubscribe"
     - clang++ --version
@@ -93,7 +126,7 @@ openmpi-clang-tidy:
 openmpi-cppcheck:
   stage: build
   script:
-    - apt update -qq && apt install -qq -y --no-install-recommends libopenmpi-dev openmpi-bin g++ libstdc++-12-dev ca-certificates cmake cppcheck git make libboost-serialization-dev
+    - apt-get update -qq && apt-get install -qq -y --no-install-recommends libopenmpi-dev openmpi-bin g++ libstdc++-12-dev ca-certificates cmake cppcheck git make libboost-serialization-dev
     - mkdir build && cd build
     - export MPI_OVERSUBSCRIBE="--oversubscribe"
     - g++ --version
@@ -106,10 +139,12 @@ openmpi-cppcheck:
 mpich-debug:
   stage: build
   script:
-    - apt update -qq && apt install -qq -y --no-install-recommends libmpich-dev mpich
+    - apt-get update -qq && apt-get install -qq -y --no-install-recommends libopenmpi-dev openmpi-bin g++ libstdc++-12-dev ca-certificates cmake cppcheck git make libboost-serialization-dev
     - cd test
     - mkdir build && cd build
     - export MPI_OVERSUBSCRIBE=""
+    - export OMPI_ALLOW_RUN_AS_ROOT=1
+    - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 
     - cmake .. -DCMAKE_BUILD_TYPE=Debug
     - cmake --build . --parallel 2 || cmake --build . --verbose
     - ctest --output-on-failure
@@ -118,7 +153,7 @@ mpich-valgrind:
   stage: build
   allow_failure: true
   script:
-    - apt update -qq && apt-get install -qq -y --no-install-recommends libmpich-dev mpich
+    - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates cmake git libboost-test-dev libboost-serialization-dev libmpich-dev make mpich valgrind
     - mpirun --version
     - mkdir build && cd build
     - export MPI_OVERSUBSCRIBE=""
@@ -132,6 +167,7 @@ mpich-valgrind:
 
 qmcpack-openmpi:
   stage: test
+  image: debian:testing
   script:
     - apt-get -qq update && apt-get -qq install --no-install-recommends -y libblas-dev liblapack-dev libfftw3-dev libboost-serialization-dev libopenmpi-dev gfortran g++ cmake make git ca-certificates numdiff python3 python3-numpy python3-h5py python3-mpi4py python3-scipy libxml2-dev libhdf5-dev
     - git clone https://github.com/QMCPACK/qmcpack.git
@@ -146,7 +182,7 @@ qmcpack-openmpi:
   needs: ["openmpi"]
 
 qmcpack-cuda-runner:
-  allow_failure: true
+  allow_failure: false
   image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04
   tags:
     - nvidia-docker
@@ -161,25 +197,26 @@ qmcpack-cuda-runner:
     - git subtree add --squash -P external_codes/mpi3 $CI_REPOSITORY_URL $CI_COMMIT_BRANCH # e.g. https://gitlab.com/correaa/boost-multi.git
     - cd ../qmcpack
     - cd build
-    - CUDACXX=/usr/local/cuda/bin/nvcc cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DBUILD_AFQMC=1 -DQMC_CXX_STANDARD=17 -DENABLE_CUDA=1 -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_HOST_COMPILER=g++ -DCMAKE_CXX_FLAGS="-Wno-deprecated -Wno-deprecated-declarations" ..
+    - CUDACXX=/usr/local/cuda/bin/nvcc cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DBUILD_AFQMC=1 -DQMC_CXX_STANDARD=17 -DENABLE_CUDA=1 -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DQMC_GPU_ARCHS=sm_61 -DCMAKE_CUDA_HOST_COMPILER=g++ -DCMAKE_CXX_FLAGS="-Wno-deprecated -Wno-deprecated-declarations" ..
     - make -j4 afqmc test_afqmc_matrix test_afqmc_numerics test_afqmc_slaterdeterminantoperations test_afqmc_walkers test_afqmc_hamiltonians test_afqmc_hamiltonian_operations test_afqmc_phmsd test_afqmc_wfn_factory test_afqmc_prop_factory test_afqmc_estimators qmc-afqmc-performance
     - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest -R afqmc --output-on-failure
   needs: ["openmpi-cuda-11", "qmcpack-openmpi"]
 
 inq-openmpi:
   stage: test
+  image: debian:testing
   tags:
     - cpu
   script:
-    - apt-get update && apt-get install --no-install-recommends -y --quiet libblas-dev liblapack-dev libfftw3-dev libboost-filesystem-dev libboost-serialization-dev libopenmpi-dev libhdf5-dev gfortran g++ cmake pkg-config make git ca-certificates wget
+    - apt-get update && apt-get install --no-install-recommends -y --quiet libblas-dev liblapack-dev libfftw3-dev libboost-filesystem-dev libboost-serialization-dev libboost-iostreams-dev libopenmpi-dev libhdf5-dev gfortran g++ cmake pkg-config python3-dev make git ca-certificates wget
     - cmake --version
     - git clone https://gitlab.com/npneq/inq.git --recurse-submodules
     - cd inq
     - cd external_libs/mpi3
-    - git checkout $CI_COMMIT_BRANCH
+    - git checkout $CI_COMMIT_BRANCH  # check that multi repo is mirrored correctly from this repo to the submodule repo (npneq)
     - cd ../..
     - mkdir build && cd build
-    - ../configure --prefix=$HOME --disable-debug
+    - cmake .. --install-prefix=$HOME -DCMAKE_BUILD_TYPE=Release
     - make --jobs=2 || make VERBOSE=1
     - make install
     - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure
@@ -222,7 +259,7 @@ inq-cuda-11-openmpi-compileonly:
   tags:
     - nvidia-docker
   script:
-    - apt-get update && apt-get install --no-install-recommends -y --quiet cmake libblas-dev liblapack-dev libfftw3-dev libboost-filesystem-dev libboost-serialization-dev libopenmpi-dev libhdf5-dev gfortran g++ pkg-config make git ca-certificates wget
+    - apt-get update && apt-get install --no-install-recommends -y --quiet cmake git libblas-dev liblapack-dev libfftw3-dev libboost-filesystem-dev libboost-iostreams-dev libboost-serialization-dev libopenmpi-dev libhdf5-dev ca-certificatesgfortran g++ make pkg-config python3-dev wget
     - cmake --version
     - git clone https://gitlab.com/npneq/inq.git --recurse-submodules
     - cd inq
@@ -231,7 +268,7 @@ inq-cuda-11-openmpi-compileonly:
     - cd ../..
     - mkdir build && cd build
     - /usr/local/cuda-11/bin/nvcc -V
-    - CUDA_ARCH_OVERRIDE=1 ../configure --prefix=$HOME --enable-cuda --with-cuda-prefix=/usr/local/cuda --pass-thru -DCMAKE_CUDA_COMPILER=/usr/local/cuda-11/bin/nvcc -DCMAKE_CUDA_ARCHITECTURES=61
+    - CUDACXX=/usr/local/cuda/bin/nvcc cmake .. --install-prefix=$HOME -DENABLE_CUDA=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_ARCHITECTURES=61
     - make silicon --jobs=2
     - make install
     - ctest -R silicon

diff --git a/external_codes/mpi_wrapper/mpi3/CMakeLists.txt b/external_codes/mpi_wrapper/mpi3/CMakeLists.txt
@@ -2,27 +2,37 @@
 cmake_minimum_required(VERSION 3.16)
 
 project(
-	mpi3
+	bmpi3
 	VERSION 0.79.0
 	DESCRIPTION "B-MPI3 is a C++ library wrapper for version 3.1 of the MPI standard interface that simplifies the utilization and maintenance of MPI code."
 	HOMEPAGE_URL "https://gitlab.com/correaa/boost-mpi3"
 	LANGUAGES CXX
 )
 
-include(GNUInstallDirs)
+find_package(MPI REQUIRED)  # might need to `module load mpi`
 
 add_library(${PROJECT_NAME} INTERFACE)
 
+target_compile_features(${PROJECT_NAME} INTERFACE cxx_std_17)
 target_include_directories(${PROJECT_NAME} INTERFACE $<BUILD_INTERFACE:${${PROJECT_NAME}_SOURCE_DIR}/include> $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${${PROJECT_NAME}_SOURCE_DIR}/include/mpi3/dummy>)
+target_link_libraries(${PROJECT_NAME} INTERFACE MPI::MPI_CXX)
 
-target_compile_features(${PROJECT_NAME} INTERFACE cxx_std_17)
-#set_target_properties(${PROJECT_NAME} PROPERTIES CXX_EXTENSIONS OFF)
+# to use this project directly from CMake
+# FetchContent_Declare(
+#     bmpi3
+#     GIT_REPOSITORY         [email protected]:correaa/boost-mpi3.git  # https://gitlab.com/correaa/boost-mpi3.git
+#     GIT_TAG                master)
+# FetchContent_MakeAvailable(bmpi3)
+# add_executable(main main.cpp)
+# target_link_libraries(main PUBLIC bmpi3)
 
 # this makes CM FetchContent friendly https://www.foonathan.net/2022/06/cmake-fetchcontent/
 if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
-  	return()
+	return()
 endif()
 
+include(GNUInstallDirs)
+
 include(CTest)
 
 enable_testing()