Parallelize (MPI, optional) two-center integration

dftbplus · Aug 16, 2024 · d9ffa81 · d9ffa81
1 parent 2f14d28
commit d9ffa81
Show file tree

Hide file tree

Showing 33 changed files with 1,486 additions and 64 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -16,6 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
+        mpi: [nompi, openmpi]
         config: [Debug]
         version: [13]
 
@@ -38,19 +39,39 @@ jobs:
           --slave /usr/bin/gfortran gfortran /usr/bin/gfortran-${{ matrix.version }} \
           --slave /usr/bin/gcov gcov /usr/bin/gcov-${{ matrix.version }}
 
+    - name: Enable MPI build
+      if: contains(matrix.mpi, 'openmpi') || contains(matrix.mpi, 'mpich')
+      run: echo "WITH_MPI=true" >> $GITHUB_ENV
+
+    - name: Disable MPI build
+      if: contains(matrix.mpi, 'nompi')
+      run: echo "WITH_MPI=false" >> $GITHUB_ENV
+
     - name: Set Compiler
       run: |
         echo "FC=gfortran" >> $GITHUB_ENV
         echo "CC=gcc" >> $GITHUB_ENV
 
+    - name: Install OpenMPI
+      if: contains(matrix.mpi, 'openmpi')
+      run: |
+        sudo apt-get update
+        sudo apt-get install libopenmpi-dev
+
+    - name: Install MPICH
+      if: contains(matrix.mpi, 'mpich')
+      run: |
+        sudo apt-get update
+        sudo apt-get install mpich
+
     - name: Compile and Install libXC
       run: |
         git clone https://gitlab.com/libxc/libxc.git
         cd libxc/
         git checkout 6.2.2
         cmake -DCMAKE_INSTALL_PREFIX=${PWD}/${BUILD_DIR}/${INSTALL_DIR} -DENABLE_FORTRAN=True -B ${BUILD_DIR} .
         cd ${BUILD_DIR}
-        make -j 2
+        make -j2
         make install
         cd ../../
 
@@ -65,16 +86,30 @@ jobs:
         pip3 install cmake fypp numpy scipy
 
     - name: Configure build
-      run: |
-        cmake -B ${BUILD_DIR} -DCMAKE_INSTALL_PREFIX=${PWD}/${BUILD_DIR}/${INSTALL_DIR} -DCMAKE_BUILD_TYPE=Debug .
+      run: >-
+        cmake -B ${BUILD_DIR} -DWITH_MPI=${WITH_MPI}
+	-DCMAKE_INSTALL_PREFIX=${PWD}/${BUILD_DIR}/${INSTALL_DIR}
+	-DCMAKE_BUILD_TYPE=${{ matrix.config }} .
 
     - name: Build project
       run: cmake --build ${BUILD_DIR}
 
     - name: Run regression tests
       run: |
         pushd ${BUILD_DIR}
-        ctest -j 2 --output-on-failure
+        ctest -j2 --output-on-failure
+        popd
+
+    - name: Configure build (TEST_MPI_PROCS=2)
+      if: contains(matrix.mpi, 'openmpi') || contains(matrix.mpi, 'mpich')
+      run: |
+        cmake -B ${BUILD_DIR} -DTEST_MPI_PROCS=2 .
+
+    - name: Run regression tests (TEST_MPI_PROCS=2)
+      if: contains(matrix.mpi, 'openmpi') || contains(matrix.mpi, 'mpich')
+      run: |
+        pushd ${BUILD_DIR}
+        ctest -j1 --output-on-failure
         popd
 
     - name: Install project
@@ -96,6 +131,7 @@ jobs:
       CMAKE_OPTIONS: >-
         -DCMAKE_BUILD_TYPE=${{ matrix.config }}
         -DFYPP_FLAGS='-DTRAVIS'
+	-DWITH_MPI=false
 
     steps:
     - name: Checkout code
@@ -128,7 +164,7 @@ jobs:
         git checkout 6.2.2
         cmake -DCMAKE_INSTALL_PREFIX=${PWD}/${BUILD_DIR}/${INSTALL_DIR} -DENABLE_FORTRAN=True -B ${BUILD_DIR} .
         cd ${BUILD_DIR}
-        make -j 2
+        make -j2
         make install
         cd ../../
 
@@ -156,7 +192,7 @@ jobs:
     - name: Run regression tests
       run: |
         pushd ${BUILD_DIR}
-        ctest -j 2 --output-on-failure
+        ctest -j2 --output-on-failure
         popd
 
     - name: Install project

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -30,6 +30,16 @@ endif()
 
 find_package(Libxc QUIET)
 
+if(WITH_MPI)
+  find_package(MPI REQUIRED)
+  if(NOT MPI_FORTRAN_FOUND)
+    message(FATAL_ERROR "Compiler ${CMAKE_Fortran_COMPILER} is not MPI capable but is specified "
+      "for a WITH_MPI=TRUE build")
+  endif()
+  list(FILTER MPI_Fortran_COMPILE_OPTIONS EXCLUDE REGEX  "-fallow-argument-mismatch")
+  message(STATUS "MPI_Fortran_COMPILE_OPTIONS: ${MPI_Fortran_COMPILE_OPTIONS}")
+endif()
+
 if (Libxc_FOUND AND Libxc_VERSION VERSION_LESS "6.0.0")
   message(FATAL_ERROR "SkProgs requires libXC version >=6.0.0 to work properly.")
 endif()
@@ -72,6 +82,19 @@ set(FYPP_BUILD_FLAGS "${FYPP_FLAGS}" "--file-var-root=${CMAKE_SOURCE_DIR}"
 set(PYTHON_INTERPRETER "python3" CACHE STRING
   "Python interpreter to use for installing and test python components")
 
+############## External components (optional) #################
+
+# Note 1 : GIT_TAG hashes below must be updated with the utils/test/check_submodule_commits script!
+# Note 2 : GIT_REPOSITORY URLs should be to https// addresses for the CI pipelines to see them
+
+if(WITH_MPI)
+  set(MPIFX_GIT_REPOSITORY "https://github.com/dftbplus/mpifx.git")
+  set(MPIFX_GIT_TAG "aca7a212f68778aacb33ed33925d45f83dab91ca")  # do not change manually!
+  skprogs_config_hybrid_dependency(MpiFx MpiFx::MpiFx "${HYBRID_CONFIG_METHODS}" "QUIET"
+    external/mpifx "${exclude}" "${MPIFX_GIT_REPOSITORY}" "${MPIFX_GIT_TAG}")
+  list(APPEND PKG_CONFIG_REQUIRES mpifx)
+endif()
+
 #################### Add source components ####################
 
 add_subdirectory(common)

diff --git a/README.rst b/README.rst
@@ -27,7 +27,9 @@ Prerequisites
 
 * LAPACK/BLAS libraries (or compatible equivalents)
 
-* libXC library with f03 interface (version >=6.0.0)
+* libXC library with f03 interface (>=6.0.0)
+
+* MpiFx (>=1.5, MPI-enabled build only)
 
 
 Obtaining via Conda
@@ -65,7 +67,11 @@ Follow the usual CMake build workflow:
   the install location (i.e. path stored in ``YOUR_SKPROGS_INSTALL_FOLDER``,
   e.g. ``$HOME/opt/skprogs``) and the build directory (e.g. ``_build``)::
 
-    FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=YOUR_SKPROGS_INSTALL_FOLDER -B _build .
+    FC=gfortran cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=YOUR_SKPROGS_INSTALL_FOLDER -B _build .
+
+  An MPI enabled build is obtained by additionally setting ``-DWITH_MPI=1``
+  (default: ``-DWITH_MPI=0``). At the moment only the two-center integration
+  code ``sktwocnt`` is MPI parallelized and benefits from multiple processors.
 
   If libXC is installed in a non-standard location, you may need to specify
   either the ``CMAKE_PREFIX_PATH`` environment variable (if libXC was built with
@@ -86,6 +92,18 @@ Follow the usual CMake build workflow:
     ctest -j
     popd
 
+* If you want to test the MPI enabled binary with more than one MPI-process, you
+  should set the ``TEST_MPI_PROCS`` variable in ``config.cmake`` accordingly,
+  e.g.::
+
+    set(TEST_MPI_PROCS "2" CACHE STRING "Nr. of processes used for testing")
+
+  The ``TEST_MPI_PROCS`` cache variable can be updated or changed also after the
+  compilation by invoking CMake with the appropriate ``-D`` option, e.g.::
+
+    cmake -B _build -DTEST_MPI_PROCS=2 .
+    pushd _build; ctest; popd
+
 * If the tests were successful, install the package via ::
 
     cmake --install _build
@@ -177,6 +195,11 @@ follows:
 
     skgen -o slateratom -t sktwocnt sktable -d C,H,O C,H,O
 
+  For an MPI enabled binary, make sure to prepend any required information to
+  the two-center binary, e.g.::
+
+    skgen -o slateratom -t "mpirun -np 2 sktwocnt" sktable -d C C |& tee output
+
   The SK-files will be created in the current folder. See the help (e.g. ``skgen
   -h``) for additional options.
 

diff --git a/cmake/SkProgsUtils.cmake b/cmake/SkProgsUtils.cmake
@@ -50,6 +50,11 @@ endfunction()
 function (skprogs_add_fypp_defines fyppflags)
 
   set(_fyppflags "${${fyppflags}}")
+
+  if(WITH_MPI)
+    list(APPEND _fyppflags -DWITH_MPI)
+  endif()
+
   set(${fyppflags} ${_fyppflags} PARENT_SCOPE)
 
 endfunction()
@@ -195,6 +200,130 @@ macro(skprogs_setup_global_compiler_flags)
 endmacro()
 
 
+# Handles a hybrid dependency.
+#
+# Depending on the list items in the config_methods variable, it will try to:
+#
+# - checkout the source as a submodule within the origin sub-folder ("Submodule")
+# - find the package as external dependency ("Find")
+# - fetch the source from a git repository ("Fetch") into the build folder
+#
+# The methods are tried in the order of their appearance until success the first eligible one.
+#
+# The methods "Submodule" and "Fetch" would call the passed sub-directory with add_subdirectory()
+# passing two variables with the source and binary directory.
+#
+# Args:
+#     package [in]: Name of the dependency to look for.
+#     config_methods [in]: Config methods to try
+#     target [in]: Name of the target, which must be exported after the configuration.
+#     findpkgopts [in]: Options to pass to find_package()
+#     subdir [in]: Subdirectory with CMakeFiles.txt for integrating package source.
+#     subdiropts [in]: Options to pass to the add_subdir() command.
+#     git_repository [in]: Git repository to fetch the package from.
+#     git_tag [in]: Git tag to use when fetching the source.
+#
+# Variables:
+#     <UPPER_PACKAGE_NAME>_SOURCE_DIR, <UPPER_PACKAGE_NAME>_BINARY_DIR:
+#         Source and binary directories for the build (to pass to add_subdirectory())
+#
+macro(skprogs_config_hybrid_dependency package target config_methods findpkgopts subdir subdiropts
+    git_repository git_tag)
+
+  set(_allowed_methods "submodule;find;fetch;pkgconf")
+  string(TOLOWER "${package}" _package_lower)
+  string(TOUPPER "${package}" _package_upper)
+
+  foreach(_config_method IN ITEMS ${config_methods})
+
+    string(TOLOWER "${_config_method}" _config_lower)
+    if(NOT ${_config_lower} IN_LIST _allowed_methods)
+      message(FATAL_ERROR "${package}: Unknown configuration method '${_config_method}'")
+    endif()
+
+    if("${_config_lower}" STREQUAL "find")
+
+      message(STATUS "${package}: Trying to find installed package")
+      find_package(${package} ${findpkgopts})
+      if(${package}_FOUND)
+        message(STATUS "${package}: Installed package found")
+        break()
+      else()
+        message(STATUS "${package}: Installed package could not be found")
+      endif()
+
+    elseif("${_config_lower}" STREQUAL "pkgconf")
+      message(STATUS "${package}: Trying to find installed package (pkg-config)")
+
+      find_package(PkgConfig QUIET)
+      if(PkgConfig_FOUND)
+        pkg_check_modules("${_package_upper}" QUIET "${package}")
+        if("${${_package_upper}_FOUND}")
+          message(STATUS "${package}: Installed package found (pkg-config)")
+          add_library("${target}" INTERFACE IMPORTED)
+          target_link_libraries(
+            "${target}"
+            INTERFACE
+            "${${_package_upper}_LINK_LIBRARIES}"
+          )
+          target_include_directories(
+            "${target}"
+            INTERFACE
+            "${${_package_upper}_INCLUDE_DIRS}"
+          )
+          break()
+        else()
+          message(STATUS "${package}: Installed package could not be found (pkg-config)")
+        endif()
+      endif()
+
+    elseif("${_config_lower}" STREQUAL "submodule")
+
+      if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${subdir}/origin/CMakeLists.txt
+          AND GIT_WORKING_COPY)
+        message(STATUS "${package}: Downloading via git submodule update")
+        execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init ${subdir}/origin
+          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+      endif()
+
+      if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${subdir}/origin/CMakeLists.txt)
+        message(STATUS "${package}: Using source in ${subdir}/origin")
+        set(${_package_upper}_SOURCE_DIR "origin")
+        set(${_package_upper}_BINARY_DIR)
+        add_subdirectory(${subdir} ${subdiropts})
+        break()
+      endif()
+
+    elseif("${_config_lower}" STREQUAL "fetch")
+
+      message(STATUS "${package}: Fetching from repository ${git_repository}@${git_tag}")
+      FetchContent_Declare(${_package_lower} GIT_REPOSITORY ${git_repository} GIT_TAG ${git_tag})
+      FetchContent_GetProperties(${_package_lower})
+      if(NOT ${_package_lower}_POPULATED)
+        FetchContent_Populate(${_package_lower})
+      endif()
+      set(${_package_upper}_SOURCE_DIR "${${_package_lower}_SOURCE_DIR}")
+      set(${_package_upper}_BINARY_DIR "${${_package_lower}_BINARY_DIR}")
+      add_subdirectory(${subdir} ${subdiropts})
+      break()
+
+    endif()
+
+  endforeach()
+
+  if(NOT TARGET ${target})
+    message(FATAL_ERROR "Could not configure ${package} to export target '${target}'")
+  endif()
+
+  unset(_allowed_methods)
+  unset(_package_lower)
+  unset(_package_upper)
+  unset(_config_method)
+  unset(_config_lower)
+
+endmacro()
+
+
 # Checks whether the current compiler fullfills minimal version requirements.
 #
 #

diff --git a/common/include/error.fypp b/common/include/error.fypp
@@ -15,6 +15,27 @@
 #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 
+#! Macro to return an error flag if return variable available or throw
+#! an error and shut down otherwise
+#:def ERROR_HANDLING(errVar, errNumber, msg)
+  block
+    use common_accuracy, only : lc
+    use common_message
+    !> Error handling string
+    character(lc) :: stringTmp
+
+    write(stringTmp,"(A)")${msg}$
+    if (present(${errVar}$)) then
+      ${errVar}$ = ${errNumber}$
+      call warning(stringTmp)
+      return
+    else
+      call error(stringTmp)
+    end if
+  end block
+#:enddef ERROR_HANDLING
+
+
 #! Propagation of error handling, for now it just returns when in error
 #:def HANDLE_ERROR(errVar)
   if (present(${errVar}$)) then

diff --git a/common/lib/CMakeLists.txt b/common/lib/CMakeLists.txt
@@ -31,12 +31,26 @@ set(sources-f90
   utils.F90)
 
 set(sources-fpp
-  assert.F90)
+  assert.F90
+  environment.F90
+  globalenv.F90
+  mpifx.F90
+  schedule.F90)
+
+if(WITH_MPI)
+  list(APPEND sources-fpp
+    mpienv.F90)
+endif()
 
 skprogs_preprocess("${FYPP}" "${fypp_flags}" "F90" "f90" "${sources-fpp}" sources-f90-preproc)
 
 add_library(skprogs-common ${sources-f90} ${sources-f90-preproc})
 
+if(WITH_MPI)
+  target_link_libraries(skprogs-common MPI::MPI_Fortran)
+  target_link_libraries(skprogs-common MpiFx::MpiFx)
+endif()
+
 set(moddir ${CMAKE_CURRENT_BINARY_DIR}/modfiles)
 set_target_properties(skprogs-common PROPERTIES Fortran_MODULE_DIRECTORY ${moddir})
 target_include_directories(skprogs-common PUBLIC