From f404a0cb895b33545cc7a630feac97b4aa148b80 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Sat, 13 Jun 2020 12:27:16 -0700 Subject: [PATCH] Release process (#357) * Remove error if mpi4py found at install time but not at run time. * Only error on missing lapack if we are not on readthedocs. * Fix typo in CMakeLists.txt when checking for static build. * Fix typo * Another small fix for the case of mpi4py found at build time but not at run time. * Remove stale MPI files. Remove MPI information from the Environment class in the compiled extension so that this information is only checked at run time. This allows installing toast-cmb and mpi4py in any order and things always work based on the current environment. * Move wheel tests back to a daily job. Update release to rc4 --- .github/workflows/deploy.yml | 121 ++++----- .github/workflows/test.yml | 12 + .github/workflows/wheels.yml | 117 ++++----- CMakeLists.txt | 28 ++- cmake/FindMPI4PY.cmake | 28 --- cmake/test_mpi4py.py | 13 - src/CMakeLists.txt | 4 - .../include/toast/sys_environment.hpp | 8 - src/libtoast/src/toast_sys_environment.cpp | 115 --------- src/libtoast_mpi/CMakeLists.txt | 83 ------- src/libtoast_mpi/include/toast/mpi_shmem.hpp | 229 ------------------ src/libtoast_mpi/include/toast/mpi_test.hpp | 14 -- src/libtoast_mpi/include/toast_mpi.hpp | 23 -- .../include/toast_mpi_internal.hpp | 16 -- src/libtoast_mpi/src/toast_mpi.cpp | 31 --- src/libtoast_mpi/tests/toast_mpi_test.hpp | 29 --- .../tests/toast_mpi_test_runner.cpp | 35 --- .../tests/toast_mpi_test_shmem.cpp | 53 ---- src/libtoast_mpi/toast_mpi.cpp | 6 - src/libtoast_mpi/toast_mpi_test.cpp | 12 - src/toast/CMakeLists.txt | 47 ---- src/toast/RELEASE | 2 +- src/toast/_libtoast_mpi.cpp | 196 --------------- src/toast/_libtoast_mpi.hpp | 13 - src/toast/_libtoast_sys.cpp | 10 - src/toast/mpi.py | 46 ++-- src/toast/tests/runner.py | 4 +- src/toast/todmap/conviqt.py | 4 +- src/toast/todmap/madam.py | 4 +- src/toast/utils.py | 6 +- 30 files changed, 186 insertions(+), 1123 deletions(-) delete mode 100644 cmake/FindMPI4PY.cmake delete mode 100644 cmake/test_mpi4py.py delete mode 100644 src/libtoast_mpi/CMakeLists.txt delete mode 100644 src/libtoast_mpi/include/toast/mpi_shmem.hpp delete mode 100644 src/libtoast_mpi/include/toast/mpi_test.hpp delete mode 100644 src/libtoast_mpi/include/toast_mpi.hpp delete mode 100644 src/libtoast_mpi/include/toast_mpi_internal.hpp delete mode 100644 src/libtoast_mpi/src/toast_mpi.cpp delete mode 100644 src/libtoast_mpi/tests/toast_mpi_test.hpp delete mode 100644 src/libtoast_mpi/tests/toast_mpi_test_runner.cpp delete mode 100644 src/libtoast_mpi/tests/toast_mpi_test_shmem.cpp delete mode 100644 src/libtoast_mpi/toast_mpi.cpp delete mode 100644 src/libtoast_mpi/toast_mpi_test.cpp delete mode 100644 src/toast/_libtoast_mpi.cpp delete mode 100644 src/toast/_libtoast_mpi.hpp diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 170ae14fb..6805704a5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -16,23 +16,23 @@ jobs: name: Python source dist runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Install Dependencies - run: pip install twine - - name: Pull Dependency Image - run: docker pull hpc4cmb/toast-deps-py37:latest - - name: Create dist directory - run: mkdir -p dist && rm -f dist/* - - name: Build source package - run: docker run -v "$(pwd)":/home/toast hpc4cmb/toast-deps-py37:latest /home/toast/wheels/build_sdist.sh - - name: Upload to PyPI - run: | - python -m twine upload dist/toast*.tar.gz + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install Dependencies + run: pip install twine + - name: Pull Dependency Image + run: docker pull hpc4cmb/toast-deps-py37:latest + - name: Create dist directory + run: mkdir -p dist && rm -f dist/* + - name: Build source package + run: docker run -v "$(pwd)":/home/toast hpc4cmb/toast-deps-py37:latest /home/toast/wheels/build_sdist.sh + - name: Upload to PyPI + run: | + python -m twine upload dist/toast*.tar.gz wheels-36: name: Python 3.6 wheels for ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -51,20 +51,21 @@ jobs: CIBW_BEFORE_TEST: pip3 install numpy && pip3 install mpi4py CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()' steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' - - name: Install cibuildwheel - run: | - python -m pip install twine cibuildwheel==1.4.2 - - name: Build wheel - run: | - python -m cibuildwheel --output-dir wheelhouse - - name: Upload to PyPI - run: | - python -m twine upload wheelhouse/toast*.whl + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.7' + - name: Install cibuildwheel + run: | + python -m pip install twine cibuildwheel==1.4.2 + - name: Build wheel + run: | + python -m cibuildwheel --output-dir wheelhouse + - name: Upload to PyPI + run: | + python -m twine upload wheelhouse/toast*.whl wheels-37: name: Python 3.7 wheels for ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -83,20 +84,21 @@ jobs: CIBW_BEFORE_TEST: pip3 install numpy && pip3 install mpi4py CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()' steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' - - name: Install cibuildwheel - run: | - python -m pip install twine cibuildwheel==1.4.2 - - name: Build wheel - run: | - python -m cibuildwheel --output-dir wheelhouse - - name: Upload to PyPI - run: | - python -m twine upload wheelhouse/toast*.whl + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.7' + - name: Install cibuildwheel + run: | + python -m pip install twine cibuildwheel==1.4.2 + - name: Build wheel + run: | + python -m cibuildwheel --output-dir wheelhouse + - name: Upload to PyPI + run: | + python -m twine upload wheelhouse/toast*.whl wheels-38: name: Python 3.8 wheels for ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -115,17 +117,18 @@ jobs: CIBW_BEFORE_TEST: pip3 install numpy && pip3 install mpi4py CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()' steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' - - name: Install cibuildwheel - run: | - python -m pip install twine cibuildwheel==1.4.2 - - name: Build wheel - run: | - python -m cibuildwheel --output-dir wheelhouse - - name: Upload to PyPI - run: | - python -m twine upload wheelhouse/toast*.whl + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.7' + - name: Install cibuildwheel + run: | + python -m pip install twine cibuildwheel==1.4.2 + - name: Build wheel + run: | + python -m cibuildwheel --output-dir wheelhouse + - name: Upload to PyPI + run: | + python -m twine upload wheelhouse/toast*.whl diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6802364a8..35a3b1c67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,10 @@ jobs: name: Python 3.6 runs-on: ubuntu-latest steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.4.0 + with: + access_token: ${{ github.token }} - name: Checkout uses: actions/checkout@v2 - name: Pull Dependency Image @@ -29,6 +33,10 @@ jobs: name: Python 3.7 runs-on: ubuntu-latest steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.4.0 + with: + access_token: ${{ github.token }} - name: Checkout uses: actions/checkout@v2 - name: Pull Dependency Image @@ -45,6 +53,10 @@ jobs: name: Python 3.8 runs-on: ubuntu-latest steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.4.0 + with: + access_token: ${{ github.token }} - name: Checkout uses: actions/checkout@v2 - name: Pull Dependency Image diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 08dedecfe..d0a96ddef 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -18,18 +18,18 @@ jobs: name: Python source dist runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Pull Dependency Image - run: docker pull hpc4cmb/toast-deps-py37:latest - - name: Create dist directory - run: mkdir -p dist && rm -f dist/* - - name: Build source package - run: docker run -v "$(pwd)":/home/toast hpc4cmb/toast-deps-py37:latest /home/toast/wheels/build_sdist.sh - - uses: actions/upload-artifact@v2 - with: - name: sdist - path: ./dist/toast*.gz + - name: Checkout + uses: actions/checkout@v2 + - name: Pull Dependency Image + run: docker pull hpc4cmb/toast-deps-py37:latest + - name: Create dist directory + run: mkdir -p dist && rm -f dist/* + - name: Build source package + run: docker run -v "$(pwd)":/home/toast hpc4cmb/toast-deps-py37:latest /home/toast/wheels/build_sdist.sh + - uses: actions/upload-artifact@v2 + with: + name: sdist + path: ./dist/toast*.gz wheels-py36: name: Python 3.6 wheels for ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -48,21 +48,22 @@ jobs: CIBW_BEFORE_TEST: pip3 install numpy && pip3 install mpi4py CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()' steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' - - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel==1.4.2 - - name: Build wheel - run: | - python -m cibuildwheel --output-dir wheelhouse - - uses: actions/upload-artifact@v2 - with: - name: wheels - path: ./wheelhouse/toast*.whl + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.7' + - name: Install cibuildwheel + run: | + python -m pip install cibuildwheel==1.4.2 + - name: Build wheel + run: | + python -m cibuildwheel --output-dir wheelhouse + - uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/toast*.whl wheels-py37: name: Python 3.7 wheels for ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -81,21 +82,22 @@ jobs: CIBW_BEFORE_TEST: pip3 install numpy && pip3 install mpi4py CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()' steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' - - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel==1.4.2 - - name: Build wheel - run: | - python -m cibuildwheel --output-dir wheelhouse - - uses: actions/upload-artifact@v2 - with: - name: wheels - path: ./wheelhouse/toast*.whl + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.7' + - name: Install cibuildwheel + run: | + python -m pip install cibuildwheel==1.4.2 + - name: Build wheel + run: | + python -m cibuildwheel --output-dir wheelhouse + - uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/toast*.whl wheels-py38: name: Python 3.8 wheels for ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -114,18 +116,19 @@ jobs: CIBW_BEFORE_TEST: pip3 install numpy && pip3 install mpi4py CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()' steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' - - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel==1.4.2 - - name: Build wheel - run: | - python -m cibuildwheel --output-dir wheelhouse - - uses: actions/upload-artifact@v2 - with: - name: wheels - path: ./wheelhouse/toast*.whl + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.7' + - name: Install cibuildwheel + run: | + python -m pip install cibuildwheel==1.4.2 + - name: Build wheel + run: | + python -m cibuildwheel --output-dir wheelhouse + - uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/toast*.whl diff --git a/CMakeLists.txt b/CMakeLists.txt index 5116bea0b..38b77e133 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,10 @@ foreach(policy endif() endforeach() -project(toast VERSION 2.3.0 LANGUAGES C CXX) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/src/toast/RELEASE REL_VERSION) +string(REGEX REPLACE "^([0-9]+\\.[0-9]+)\\..*" "\\1" MAJMIN_VERSION "${REL_VERSION}") + +project(toast VERSION ${MAJMIN_VERSION} LANGUAGES C CXX) # Force C++11 set(CMAKE_CXX_STANDARD 11) @@ -42,7 +45,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) # extensions. We check a variable here, and if set, we look for static versions of # our dependencies. # -if(NOT TOAST_STATIC_DEPS AND ENV{TOAST_STATIC_DEPS}) +if(NOT TOAST_STATIC_DEPS AND NOT $ENV{TOAST_STATIC_DEPS} STREQUAL "") set(TOAST_STATIC_DEPS $ENV{TOAST_STATIC_DEPS}) endif() @@ -61,12 +64,16 @@ if(BLAS_FOUND) find_package(LAPACK) if(LAPACK_FOUND) find_package(LAPACKnames) - else(LAPACK_FOUND) - message(FATAL_ERROR "Could not find a working LAPACK installation") - endif(LAPACK_FOUND) -else(BLAS_FOUND) - message(FATAL_ERROR "Could not find a working BLAS installation") -endif(BLAS_FOUND) + else() + if($ENV{READTHEDOCS} STREQUAL "") + message(FATAL_ERROR "Could not find a working LAPACK installation") + endif() + endif() +else() + if($ENV{READTHEDOCS} STREQUAL "") + message(FATAL_ERROR "Could not find a working BLAS installation") + endif() +endif() find_package(FFTW) @@ -74,13 +81,8 @@ find_package(AATM) find_package(SuiteSparse) -# Re-enable if we ever add MPI compiled extensions. -# find_package(MPI) - find_package(PythonInterp REQUIRED) -find_package(MPI4PY) - # Internal products enable_testing() diff --git a/cmake/FindMPI4PY.cmake b/cmake/FindMPI4PY.cmake deleted file mode 100644 index 26e889011..000000000 --- a/cmake/FindMPI4PY.cmake +++ /dev/null @@ -1,28 +0,0 @@ -# - FindMPI4PY -# Find mpi4py includes and library -# This module defines: -# MPI4PY_INCLUDE_DIR, where to find mpi4py.h, etc. -# MPI4PY_FOUND - -# Roughly based on a file in a branch of the Dolfin project. - -if(NOT MPI4PY_INCLUDE_DIR) - execute_process( - COMMAND "${PYTHON_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/cmake/test_mpi4py.py" - OUTPUT_VARIABLE MPI4PY_INCLUDE_DIR - RESULT_VARIABLE MPI4PY_COMMAND_RESULT - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - if(MPI4PY_COMMAND_RESULT STREQUAL "0") - message("-- Found mpi4py include: ${MPI4PY_INCLUDE_DIR}") - set(MPI4PY_FOUND TRUE) - set(MPI4PY_INCLUDE_DIR ${MPI4PY_INCLUDE_DIR} - CACHE STRING "mpi4py include path") - else(MPI4PY_COMMAND_RESULT STREQUAL "0") - message("-- mpi4py NOT found") - set(MPI4PY_FOUND FALSE) - endif(MPI4PY_COMMAND_RESULT STREQUAL "0") -else(NOT MPI4PY_INCLUDE_DIR) - message("-- Using mpi4py include: ${MPI4PY_INCLUDE_DIR}") - set(MPI4PY_FOUND TRUE) -endif(NOT MPI4PY_INCLUDE_DIR) diff --git a/cmake/test_mpi4py.py b/cmake/test_mpi4py.py deleted file mode 100644 index 2de9178f5..000000000 --- a/cmake/test_mpi4py.py +++ /dev/null @@ -1,13 +0,0 @@ -# Test whether we can import mpi4py -from __future__ import (absolute_import, division, print_function, - unicode_literals) - -import sys - -try: - import mpi4py - print(mpi4py.get_include()) - sys.exit(0) -except ImportError: - print("") - sys.exit(1) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d9f5e5c32..518501954 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,8 +6,4 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) add_subdirectory(libtoast) -if(MPI_FOUND) - add_subdirectory(libtoast_mpi) -endif(MPI_FOUND) - add_subdirectory(toast) diff --git a/src/libtoast/include/toast/sys_environment.hpp b/src/libtoast/include/toast/sys_environment.hpp index 886d35983..b44145cee 100644 --- a/src/libtoast/include/toast/sys_environment.hpp +++ b/src/libtoast/include/toast/sys_environment.hpp @@ -28,8 +28,6 @@ class Environment { std::vector signals() const; std::vector info() const; void print() const; - bool use_mpi() const; - bool use_mpi4py() const; bool function_timers() const; int max_threads() const; int current_threads() const; @@ -46,13 +44,7 @@ class Environment { std::vector signals_avail_; std::map signals_value_; std::map signals_enabled_; - bool have_mpi_; - bool have_mpi4py_; - bool use_mpi_; - bool use_mpi4py_; bool func_timers_; - bool at_nersc_; - bool in_slurm_; int max_threads_; int cur_threads_; std::string git_version_; diff --git a/src/libtoast/src/toast_sys_environment.cpp b/src/libtoast/src/toast_sys_environment.cpp index abb3744ad..2bfee807b 100644 --- a/src/libtoast/src/toast_sys_environment.cpp +++ b/src/libtoast/src/toast_sys_environment.cpp @@ -153,82 +153,6 @@ toast::Environment::Environment() { max_threads_ = omp_get_max_threads(); #endif // ifdef _OPENMP - // Was toast configured to use MPI? We put this setting here in the - // non-MPI library so that we can always access it before trying load - // the MPI library. - use_mpi_ = true; - use_mpi4py_ = true; - - have_mpi_ = false; - have_mpi4py_ = false; - - #ifdef HAVE_MPI - - // Build system found MPI - have_mpi_ = true; - #endif // ifdef HAVE_MPI - - #ifdef HAVE_MPI4PY - - // Build system found mpi4py - have_mpi4py_ = true; - #endif // ifdef HAVE_MPI4PY - - // Handle special case of running on a NERSC login node, where MPI is - // used for compilation, but cannot be used at runtime. - envval = ::getenv("NERSC_HOST"); - if (envval == NULL) { - at_nersc_ = false; - } else { - at_nersc_ = true; - } - envval = ::getenv("SLURM_JOB_NAME"); - if (envval == NULL) { - in_slurm_ = false; - } else { - in_slurm_ = true; - } - - // See if the user explicitly disabled MPI in the runtime environment. - bool disabled_mpi_ = false; - envval = ::getenv("TOAST_MPI_DISABLE"); - if (envval != NULL) { - disabled_mpi_ = true; - } - bool disabled_mpi4py_ = false; - envval = ::getenv("TOAST_MPI4PY_DISABLE"); - if (envval != NULL) { - disabled_mpi4py_ = true; - } - - // First determine if we are using mpi4py. - if (!have_mpi4py_) { - // we don't have mpi4py - use_mpi4py_ = false; - } - if (disabled_mpi4py_) { - // the user disabled mpi4py - use_mpi4py_ = false; - } - if (at_nersc_ && !in_slurm_) { - // we are on a login node... - use_mpi4py_ = false; - } - - // Now see if we have MPI compiled extensions. This requires mpi4py as well. - if (!use_mpi4py_) { - // No mpi4py - use_mpi_ = false; - } - if (!have_mpi_) { - // we don't have MPI compilers - use_mpi_ = false; - } - if (disabled_mpi_) { - // the user disabled compiled MPI extensions - use_mpi_ = false; - } - git_version_ = GIT_VERSION; release_version_ = RELEASE_VERSION; @@ -271,14 +195,6 @@ void toast::Environment::set_log_level(char const * level) { loglvl_ = std::string(level); } -bool toast::Environment::use_mpi() const { - return use_mpi_; -} - -bool toast::Environment::use_mpi4py() const { - return use_mpi4py_; -} - bool toast::Environment::function_timers() const { return func_timers_; } @@ -351,37 +267,6 @@ std::vector toast::Environment::info() const { o << "Max threads = " << max_threads_; ret.push_back(o.str()); - o.str(""); - if (have_mpi_) { - o << "MPI build enabled"; - } else { - o << "MPI build disabled"; - } - ret.push_back(o.str()); - - o.str(""); - if (have_mpi4py_) { - o << "mpi4py found at install time"; - } else { - o << "mpi4py disabled at install time"; - } - ret.push_back(o.str()); - - o.str(""); - if (use_mpi_) { - o << "MPI compiled extensions and mpi4py enabled"; - } else if (use_mpi4py_) { - o << "MPI compiled extensions disabled, but mpi4py enabled"; - } else { - o << "MPI compiled extensions and mpi4py disabled"; - if (at_nersc_ && !in_slurm_) { - ret.push_back(o.str()); - o.str(""); - o << "Cannot use MPI on NERSC login nodes"; - } - } - ret.push_back(o.str()); - return ret; } diff --git a/src/libtoast_mpi/CMakeLists.txt b/src/libtoast_mpi/CMakeLists.txt deleted file mode 100644 index 7ac1d9667..000000000 --- a/src/libtoast_mpi/CMakeLists.txt +++ /dev/null @@ -1,83 +0,0 @@ - -# MPI compile flags- strip whitespace -string(STRIP "${MPI_CXX_COMPILE_FLAGS}" mpi_comp_flags) -string(STRIP "${MPI_CXX_LINK_FLAGS}" mpi_link_flags) - - -# Define the sources - -set(toast_mpi_SOURCES - toast_mpi.cpp - src/toast_mpi.cpp - tests/toast_mpi_test_runner.cpp - tests/toast_mpi_test_shmem.cpp -) - -add_library(toast_mpi ${toast_mpi_SOURCES}) - -target_include_directories(toast_mpi BEFORE PRIVATE - "${CMAKE_CURRENT_SOURCE_DIR}/../libtoast" - "${CMAKE_CURRENT_SOURCE_DIR}/../libtoast/include" - "${CMAKE_CURRENT_SOURCE_DIR}/../libtoast/tests" - "${CMAKE_CURRENT_SOURCE_DIR}/../libtoast/gtest/googletest/include" - "${CMAKE_CURRENT_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/tests" -) - -target_include_directories(toast_mpi BEFORE PUBLIC - $ - $ -) - -target_include_directories(toast_mpi PRIVATE "${MPI_CXX_INCLUDE_PATH}") - -target_compile_options(toast_mpi PRIVATE "${mpi_comp_flags}") - -set_target_properties(toast_mpi PROPERTIES LINK_FLAGS "${mpi_link_flags}") - -target_link_libraries(toast_mpi toast "${MPI_CXX_LIBRARIES}") - -if(OpenMP_CXX_FOUND) - target_compile_options(toast_mpi PRIVATE "${OpenMP_CXX_FLAGS}") - set_target_properties(toast_mpi PROPERTIES LINK_FLAGS "${OpenMP_CXX_FLAGS}") - target_link_libraries(toast_mpi "${OpenMP_CXX_LIBRARIES}") -endif(OpenMP_CXX_FOUND) - -# Public headers -install(DIRECTORY include/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") - -# Hardcode this to "lib" for now, since lib32/lib64 is a pain -# to use on many systems. -#install(TARGETS toast_mpi DESTINATION "${CMAKE_INSTALL_LIBDIR}") -install(TARGETS toast_mpi DESTINATION lib) - - -# MPI unit test executable - -add_executable(toast_mpi_test - toast_mpi_test.cpp -) - -if(OpenMP_CXX_FOUND) - target_compile_options(toast_mpi_test PRIVATE "${OpenMP_CXX_FLAGS}") - set_target_properties(toast_mpi_test PROPERTIES LINK_FLAGS "${OpenMP_CXX_FLAGS}") -endif(OpenMP_CXX_FOUND) - -target_include_directories(toast_mpi_test BEFORE PRIVATE - "${CMAKE_CURRENT_SOURCE_DIR}/../libtoast" - "${CMAKE_CURRENT_SOURCE_DIR}/../libtoast/include" - "${CMAKE_CURRENT_SOURCE_DIR}/../libtoast/tests" - "${CMAKE_CURRENT_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/include" - "${CMAKE_CURRENT_SOURCE_DIR}/tests" -) - -target_include_directories(toast_mpi_test PRIVATE "${MPI_CXX_INCLUDE_PATH}") - -target_compile_options(toast_mpi_test PRIVATE "${mpi_comp_flags}") - -target_link_libraries(toast_mpi_test toast_mpi) - -install(TARGETS toast_mpi_test DESTINATION ${CMAKE_INSTALL_BINDIR}) - -add_test(NAME mpi_tests COMMAND toast_mpi_test) diff --git a/src/libtoast_mpi/include/toast/mpi_shmem.hpp b/src/libtoast_mpi/include/toast/mpi_shmem.hpp deleted file mode 100644 index b00a633ba..000000000 --- a/src/libtoast_mpi/include/toast/mpi_shmem.hpp +++ /dev/null @@ -1,229 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#ifndef TOAST_MPI_SHMEM_HPP -#define TOAST_MPI_SHMEM_HPP - -#include - -#include - -#include - - -namespace toast { -template -class mpi_shmem { - public: - - mpi_shmem(MPI_Comm comm = MPI_COMM_WORLD) : comm_(comm) { - int ret = MPI_Comm_rank(comm, &world_rank_); - if (ret != MPI_SUCCESS) { - auto here = TOAST_HERE(); - auto log = toast::Logger::get(); - std::string msg("Failed to get rank from input comm"); - log.error(msg.c_str(), here); - throw std::runtime_error(msg.c_str()); - } - - // Split the provided communicator into groups that share - // memory (are on the same node). - - ret = MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, - MPI_INFO_NULL, &shmcomm_); - if (ret != MPI_SUCCESS) { - if (world_rank_ == 0) { - auto here = TOAST_HERE(); - auto log = toast::Logger::get(); - std::string msg("Failed to split communicator by node."); - log.error(msg.c_str(), here); - throw std::runtime_error(msg.c_str()); - } - } - - ret = MPI_Comm_size(shmcomm_, &ntasks_); - if (ret != MPI_SUCCESS) { - if (world_rank_ == 0) { - auto here = TOAST_HERE(); - auto log = toast::Logger::get(); - std::string msg("Failed to get node communicator size"); - log.error(msg.c_str(), here); - throw std::runtime_error(msg.c_str()); - } - } - - ret = MPI_Comm_rank(shmcomm_, &rank_); - if (ret != MPI_SUCCESS) { - if (world_rank_ == 0) { - auto here = TOAST_HERE(); - auto log = toast::Logger::get(); - std::string msg("Failed to get node communicator rank"); - log.error(msg.c_str(), here); - throw std::runtime_error(msg.c_str()); - } - } - } - - mpi_shmem(size_t n, MPI_Comm comm = MPI_COMM_WORLD) : mpi_shmem(comm) { - allocate(n); - } - - T operator[](int i) const { - return global_[i]; - } - - T & operator[](int i) { - return global_[i]; - } - - T * allocate(size_t n) { - // Determine the number of elements each process should offer - // for the shared allocation - - nlocal_ = n / ntasks_; - - if (nlocal_ * ntasks_ < n) nlocal_ += 1; - if (nlocal_ * (rank_ + 1) > n) { - nlocal_ = n - nlocal_ * rank_; - } - if (nlocal_ < 0) nlocal_ = 0; - - // Allocate the shared memory - - int ret = MPI_Win_allocate_shared( - nlocal_ * sizeof(T), sizeof(T), MPI_INFO_NULL, shmcomm_, - &local_, &win_); - if (ret != MPI_SUCCESS) { - if (rank_ == 0) { - auto here = TOAST_HERE(); - auto log = toast::Logger::get(); - std::ostringstream o; - o << " Failed to allocate " << n / 1024. / 1024. - << " MB of shared memory with " << ntasks_ - << " tasks."; - log.error(o.str().c_str(), here); - throw std::runtime_error(o.str().c_str()); - } - } - n_ = n; - - // Get a pointer to the beginning of the shared memory - // on rank # 0 - - MPI_Aint nn; - int disp; - - ret = MPI_Win_shared_query(win_, 0, &nn, &disp, &global_); - if (ret != MPI_SUCCESS) { - if (rank_ == 0) { - auto here = TOAST_HERE(); - auto log = toast::Logger::get(); - std::string msg("Failed to query shared memory address."); - log.error(msg.c_str(), here); - throw std::runtime_error(msg.c_str()); - } - } - - return global_; - } - - void free() { - if (global_) { - int ret = MPI_Win_free(&win_); - if (ret != MPI_SUCCESS) { - if (rank_ == 0) { - auto here = TOAST_HERE(); - auto log = toast::Logger::get(); - std::string msg("Failed to free shared memory."); - log.error(msg.c_str(), here); - throw std::runtime_error(msg.c_str()); - } - } - local_ = NULL; - global_ = NULL; - n_ = 0; - nlocal_ = 0; - } - return; - } - - void set(T val) { - for (int i = 0; i < nlocal_; ++i) { - local_[i] = val; - } - return; - } - - T * resize(size_t n) { - // If there is memory already allocated, preserve its - // contents. - - size_t n_copy = 0; - toast::AlignedVector temp; - - if (n < n_) { - // We are shrinking the memory - n_copy = n; - } else if (n_ > 0) { - // We are expanding the memory AND - // we currently have a non-zero memory size - n_copy = n_; - } - - if ((n_copy > 0) && (rank_ == 0)) { - temp.resize(n_copy); - std::copy(global_, global_ + n_copy, temp.begin()); - } - - free(); - allocate(n); - - if ((n_copy > 0) && (rank_ == 0)) { - std::copy(temp.begin(), temp.end(), global_); - } - - return global_; - } - - T * data() { - return global_; - } - - size_t size() { - return n_; - } - - int rank() { - return rank_; - } - - int ntasks() { - return ntasks_; - } - - ~mpi_shmem() { - free(); - if (shmcomm_ != MPI_COMM_NULL) { - MPI_Comm_free(&shmcomm_); - shmcomm_ = MPI_COMM_NULL; - } - } - - private: - - T * local_ = NULL; - T * global_ = NULL; - size_t n_ = 0; - size_t nlocal_ = 0; - MPI_Comm comm_ = MPI_COMM_NULL; - MPI_Comm shmcomm_ = MPI_COMM_NULL; - MPI_Win win_ = MPI_WIN_NULL; - int ntasks_; - int rank_; - int world_rank_; -}; -} - -#endif // ifndef TOAST_MPI_SHMEM_HPP diff --git a/src/libtoast_mpi/include/toast/mpi_test.hpp b/src/libtoast_mpi/include/toast/mpi_test.hpp deleted file mode 100644 index 95667e4d4..000000000 --- a/src/libtoast_mpi/include/toast/mpi_test.hpp +++ /dev/null @@ -1,14 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#ifndef TOAST_MPI_TEST_HPP -#define TOAST_MPI_TEST_HPP - - -namespace toast { -int test_mpi_runner(int argc, char * argv[]); -} - -#endif // ifndef TOAST_TEST_HPP diff --git a/src/libtoast_mpi/include/toast_mpi.hpp b/src/libtoast_mpi/include/toast_mpi.hpp deleted file mode 100644 index a36735df8..000000000 --- a/src/libtoast_mpi/include/toast_mpi.hpp +++ /dev/null @@ -1,23 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#ifndef TOAST_MPI_HPP -#define TOAST_MPI_HPP - -#include - -#include - -#include -#include - - -namespace toast { -void mpi_init(int argc, char * argv[]); - -void mpi_finalize(); -} - -#endif // ifndef TOAST_MPI_HPP diff --git a/src/libtoast_mpi/include/toast_mpi_internal.hpp b/src/libtoast_mpi/include/toast_mpi_internal.hpp deleted file mode 100644 index 16cbd9f3e..000000000 --- a/src/libtoast_mpi/include/toast_mpi_internal.hpp +++ /dev/null @@ -1,16 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#ifndef TOAST_MPI_INTERNAL_HPP -#define TOAST_MPI_INTERNAL_HPP - -#include - -// This header declares code which is conditionally built based on -// available external packages. If the external dependencies are -// ever made into requirements, move these includes into toast.hpp. - - -#endif // ifndef TOAST_MPI_HPP diff --git a/src/libtoast_mpi/src/toast_mpi.cpp b/src/libtoast_mpi/src/toast_mpi.cpp deleted file mode 100644 index 62915c140..000000000 --- a/src/libtoast_mpi/src/toast_mpi.cpp +++ /dev/null @@ -1,31 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#include - - -void toast::mpi_init(int argc, char * argv[]) { - // If MPI is not yet initialized (by mpi4py or some other place), - // then initialize it here. - - int ret; - int initialized; - int threadprovided; - int rank; - - ret = MPI_Initialized(&initialized); - - if (!initialized) { - ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, - &threadprovided); - } - - return; -} - -void toast::mpi_finalize() { - int ret = MPI_Finalize(); - return; -} diff --git a/src/libtoast_mpi/tests/toast_mpi_test.hpp b/src/libtoast_mpi/tests/toast_mpi_test.hpp deleted file mode 100644 index 97995d334..000000000 --- a/src/libtoast_mpi/tests/toast_mpi_test.hpp +++ /dev/null @@ -1,29 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#ifndef TOAST_MPI_TEST_TEST_HPP -#define TOAST_MPI_TEST_TEST_HPP - -#include - -#include - - -class MPITOASTShmemTest : public ::testing::Test { - public: - - MPITOASTShmemTest() {} - - ~MPITOASTShmemTest() {} - - virtual void SetUp() {} - - virtual void TearDown() {} - - static const size_t n; -}; - - -#endif // ifndef TOAST_MPI_TEST_HPP diff --git a/src/libtoast_mpi/tests/toast_mpi_test_runner.cpp b/src/libtoast_mpi/tests/toast_mpi_test_runner.cpp deleted file mode 100644 index 1dd75e605..000000000 --- a/src/libtoast_mpi/tests/toast_mpi_test_runner.cpp +++ /dev/null @@ -1,35 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#include - - -int toast::test_mpi_runner(int argc, char * argv[]) { - ::testing::GTEST_FLAG(filter) = std::string("MPITOAST*"); - ::testing::InitGoogleTest(&argc, argv); - - toast::mpi_init(argc, argv); - - // Disable result printing from all processes except the root one. - - MPI_Comm comm = MPI_COMM_WORLD; - int rank; - MPI_Comm_rank(comm, &rank); - - ::testing::TestEventListeners & listeners = - ::testing::UnitTest::GetInstance()->listeners(); - - if (rank != 0) { - delete listeners.Release(listeners.default_result_printer()); - } - - // FIXME: rank 0 of MPI_COMM_WORLD should create the test - // output directory here if it does not exist. Currently none of the - // C++ unit tests write or read data, so this is not yet an issue. - - int result = RUN_ALL_TESTS(); - - return result; -} diff --git a/src/libtoast_mpi/tests/toast_mpi_test_shmem.cpp b/src/libtoast_mpi/tests/toast_mpi_test_shmem.cpp deleted file mode 100644 index b117e8c20..000000000 --- a/src/libtoast_mpi/tests/toast_mpi_test_shmem.cpp +++ /dev/null @@ -1,53 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#include - -#include - - -const size_t MPITOASTShmemTest::n = 10; - - -TEST_F(MPITOASTShmemTest, instantiate) { - toast::mpi_shmem shmem; - shmem.allocate(n); - - toast::mpi_shmem shmem2(n); - - size_t sz = shmem.size(); - - EXPECT_EQ(shmem2.size(), n); - - EXPECT_EQ(shmem2.size(), shmem.size()); -} - -TEST_F(MPITOASTShmemTest, access) { - MPI_Comm comm = MPI_COMM_WORLD; - int rank; - MPI_Comm_rank(comm, &rank); - - toast::mpi_shmem shmem(n); - - shmem.set(20); - - int ret = MPI_Barrier(comm); - - shmem.resize(2 * n); - - ret = MPI_Barrier(comm); - - double * p = shmem.data(); - - if (rank == 0) { - shmem[n - 1] = 10; - } - - ret = MPI_Barrier(comm); - - EXPECT_FLOAT_EQ(shmem[n - 2], 20); - EXPECT_FLOAT_EQ(shmem[n - 1], 10); - EXPECT_EQ(shmem[n - 1], p[n - 1]); -} diff --git a/src/libtoast_mpi/toast_mpi.cpp b/src/libtoast_mpi/toast_mpi.cpp deleted file mode 100644 index b49e25c06..000000000 --- a/src/libtoast_mpi/toast_mpi.cpp +++ /dev/null @@ -1,6 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#include diff --git a/src/libtoast_mpi/toast_mpi_test.cpp b/src/libtoast_mpi/toast_mpi_test.cpp deleted file mode 100644 index 2f075ecb1..000000000 --- a/src/libtoast_mpi/toast_mpi_test.cpp +++ /dev/null @@ -1,12 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#include - - -int main(int argc, char * argv[]) { - int ret = toast::test_mpi_runner(argc, argv); - return ret; -} diff --git a/src/toast/CMakeLists.txt b/src/toast/CMakeLists.txt index 1348f382a..d3ae5a908 100644 --- a/src/toast/CMakeLists.txt +++ b/src/toast/CMakeLists.txt @@ -95,53 +95,6 @@ install(FILES DESTINATION ${PYTHON_SITE}/toast ) -# Optionally create a module for the MPI library - -if(MPI_FOUND AND MPI4PY_FOUND) - pybind11_add_module(_libtoast_mpi MODULE - _libtoast_common.cpp - _libtoast_mpi.cpp - ) - - string(STRIP "${MPI_CXX_COMPILE_FLAGS}" mpi_comp_flags) - target_compile_options(_libtoast_mpi PRIVATE "${mpi_comp_flags}") - - string(STRIP "${MPI_CXX_LINK_FLAGS}" mpi_link_flags) - set_target_properties(_libtoast_mpi - PROPERTIES LINK_FLAGS "${mpi_link_flags}" - ) - - target_include_directories(_libtoast_mpi BEFORE PRIVATE - "${CMAKE_CURRENT_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/../libtoast/include" - "${CMAKE_CURRENT_SOURCE_DIR}/../libtoast_mpi/include" - ${MPI_CXX_INCLUDE_PATH} - ${MPI4PY_INCLUDE_DIR} - ) - - if(OpenMP_CXX_FOUND) - target_compile_options(_libtoast_mpi PRIVATE "${OpenMP_CXX_FLAGS}") - set_target_properties(_libtoast_mpi PROPERTIES LINK_FLAGS "${OpenMP_CXX_FLAGS}") - endif(OpenMP_CXX_FOUND) - - if(AATM_FOUND) - target_compile_definitions(_libtoast_mpi PRIVATE HAVE_AATM=1) - target_include_directories(_libtoast_mpi PUBLIC "${AATM_INCLUDE_DIRS}") - endif(AATM_FOUND) - - if(CHOLMOD_FOUND) - target_compile_definitions(_libtoast_mpi PRIVATE HAVE_CHOLMOD=1) - target_include_directories(_libtoast_mpi PUBLIC "${CHOLMOD_INCLUDE_DIR}") - endif(CHOLMOD_FOUND) - - target_link_libraries(_libtoast_mpi - PRIVATE toast_mpi "${MPI_CXX_LIBRARIES}" "${OMP_LIBS}" - ) - - install(TARGETS _libtoast_mpi DESTINATION ${PYTHON_SITE}/toast) - -endif(MPI_FOUND AND MPI4PY_FOUND) - # Process the sub directories add_subdirectory(tests) add_subdirectory(tod) diff --git a/src/toast/RELEASE b/src/toast/RELEASE index 2ceab935c..b1f90fd66 100644 --- a/src/toast/RELEASE +++ b/src/toast/RELEASE @@ -1 +1 @@ -2.3.7rc3 +2.3.7rc4 diff --git a/src/toast/_libtoast_mpi.cpp b/src/toast/_libtoast_mpi.cpp deleted file mode 100644 index d0b589582..000000000 --- a/src/toast/_libtoast_mpi.cpp +++ /dev/null @@ -1,196 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#include <_libtoast_mpi.hpp> - -#include - - -MPI_Comm toast_mpi_extract_comm(py::object & pycomm) { - PyObject * pp = pycomm.ptr(); - - // For some reason this official way segfaults. So we access the - // structure directly. Although an internal detail, this has been - // consistent in mpi4py since version 1.2.2. - // MPI_Comm * comm = PyMPIComm_Get(pp); - MPI_Comm * comm = &(((struct PyMPICommObject *)pp)->ob_mpi); - - if (import_mpi4py() < 0) { - throw std::runtime_error("could not import mpi4py\n"); - } - if (comm == nullptr) { - throw std::runtime_error("mpi4py Comm pointer is null\n"); - } - MPI_Comm newcomm; - int ret = MPI_Comm_dup((*comm), &newcomm); - return newcomm; -} - -// Currently the only compiled code that uses MPI and needs to be bound to python is -// the atmosphere simulation code. If the number of things increases, we should split -// this file into multiple files. - -void init_mpi_atm(py::module & m) { -#ifdef HAVE_CHOLMOD - py::class_ ( - m, "AtmSimMPI", - R"( - Class representing a single atmosphere simulation. - - This simulation consists of a particular realization of a "slab" of the - atmosphere that moves with a constant wind speed and can be observed by - individual detectors. - - Args: - azmin (float): The minimum of the azimuth range. - azmax (float): The maximum of the azimuth range. - elmin (float): The minimum of the elevation range. - elmax (float): The maximum of the elevation range. - tmin (float): The minimum of the time range. - tmax (float): The maximum of the time range. - lmin_center (float): Center point of the distribution of the dissipation - scale of the Kolmogorov turbulence. - lmin_sigma (float): Width of the distribution of the dissipation - scale of the Kolmogorov turbulence. - lmax_center (float): Center point of the distribution of the injection - scale of the Kolmogorov turbulence. - lmax_sigma (float): Width of the distribution of the injection - scale of the Kolmogorov turbulence. - w_center (float): Center point of the distribution of wind speed (m/s). - w_sigma (float): Width of the distribution of wind speed. - wdir_center (float): Center point of the distribution of wind direction - (radians). - wdir_sigma (float): Width of the distribution of wind direction. - z0_center (float): Center point of the distribution of the water vapor (m). - z0_sigma (float): Width of the distribution of the water vapor. - T0_center (float): Center point of the distribution of ground temperature - (Kelvin). - T0_sigma (float): Width of the distribution of ground temperature. - zatm (float): Atmosphere extent for temperature profile. - zmax (float): Water vaport extent for integration. - xstep (float): Size of volume element in the X direction. - ystep (float): Size of volume element in the Y direction. - zstep (float): Size of volume element in the Z direction. - nelem_sim_max (int): Size of the simulation slices. - verbosity (int): Controls logging. - comm (mpi4py.MPI.Comm): The MPI communicator. - key1 (uint64): Streamed RNG key 1. - key2 (uint64): Streamed RNG key 2. - counterval1 (uint64): Streamed RNG counter 1. - counterval2 (uint64): Streamed RNG counter 2. - cachedir (str): The location of the cached simulation. - rmin (float): Minimum line of sight observing distance. - rmax (float): Maximum line of sight observing distance. - - )") - .def( - py::init( - [](double azmin, double azmax, double elmin, double elmax, - double tmin, double tmax, double lmin_center, double lmin_sigma, - double lmax_center, double lmax_sigma, double w_center, double w_sigma, - double wdir_center, double wdir_sigma, double z0_center, double z0_sigma, - double T0_center, double T0_sigma, double zatm, double zmax, - double xstep, double ystep, double zstep, long nelem_sim_max, - int verbosity, py::object & pycomm, uint64_t key1, uint64_t key2, - uint64_t counterval1, uint64_t counterval2, std::string cachedir, - double rmin, double rmax) { - MPI_Comm comm = toast_mpi_extract_comm(pycomm); - return new toast::mpi_atm_sim( - azmin, azmax, elmin, elmax, tmin, tmax, lmin_center, lmin_sigma, - lmax_center, lmax_sigma, w_center, w_sigma, wdir_center, - wdir_sigma, z0_center, z0_sigma, T0_center, T0_sigma, zatm, - zmax, xstep, ystep, zstep, nelem_sim_max, verbosity, comm, key1, - key2, counterval1, counterval2, cachedir, rmin, rmax); - }), py::arg("azmin"), py::arg("azmax"), py::arg("elmin"), py::arg("elmax"), - py::arg("tmin"), py::arg("tmax"), py::arg("lmin_center"), - py::arg("lmin_sigma"), py::arg("lmax_center"), py::arg("lmax_sigma"), - py::arg("w_center"), py::arg("w_sigma"), py::arg("wdir_center"), - py::arg("wdir_sigma"), py::arg("z0_center"), py::arg("z0_sigma"), - py::arg("T0_center"), py::arg("T0_sigma"), py::arg("zatm"), - py::arg("zmax"), py::arg("xstep"), py::arg("ystep"), py::arg("zstep"), - py::arg("nelem_sim_max"), py::arg("verbosity"), py::arg("comm"), - py::arg("key1"), py::arg("key2"), py::arg("counterval1"), - py::arg("counterval2"), py::arg("cachedir"), py::arg("rmin"), - py::arg("rmax") - ) - .def("simulate", &toast::mpi_atm_sim::simulate, py::arg( - "use_cache"), R"( - Perform the simulation. - - Args: - use_cache (bool): If True, use the disk cache for save / load. - - Returns: - (int): A status value (zero == good). - - )") - .def("observe", [](toast::mpi_atm_sim & self, py::buffer times, - py::buffer az, py::buffer el, py::buffer tod, double fixed_r) { - pybuffer_check_1D (times); - pybuffer_check_1D (az); - pybuffer_check_1D (el); - pybuffer_check_1D (tod); - py::buffer_info info_times = times.request(); - py::buffer_info info_az = az.request(); - py::buffer_info info_el = el.request(); - py::buffer_info info_tod = tod.request(); - size_t nsamp = info_times.size; - if ((info_az.size != nsamp) || - (info_el.size != nsamp) || - (info_tod.size != nsamp)) { - auto log = toast::Logger::get(); - std::ostringstream o; - o << "Buffer sizes are not consistent."; - log.error(o.str().c_str()); - throw std::runtime_error(o.str().c_str()); - } - double * rawtimes = reinterpret_cast (info_times.ptr); - double * rawaz = reinterpret_cast (info_az.ptr); - double * rawel = reinterpret_cast (info_el.ptr); - double * rawtod = reinterpret_cast (info_tod.ptr); - auto status = self.observe(rawtimes, rawaz, rawel, rawtod, nsamp, fixed_r); - return status; - }, py::arg("times"), py::arg("az"), py::arg("el"), py::arg("tod"), py::arg( - "fixed_r") = -1.0, R"( - Observe the atmosphere with a detector. - - The timestamps and Azimuth / Elevation pointing are provided. The TOD - buffer is filled with the integrated atmospheric signal. - - For each sample, integrate along the line of sight by summing the - atmosphere values. See Church (1995) Section 2.2, first equation. - We omit the optical depth factor which is close to unity. - - Args: - times (array_like): Detector timestamps. - az (array like): Azimuth values. - el (array_like): Elevation values. - tod (array_like): The output buffer to fill. - fixed_r (float): If greater than zero, use this single radial value. - - Returns: - (int): A status value (zero == good). - - )") - .def("__repr__", - [](toast::mpi_atm_sim const & self) { - std::ostringstream o; - o << ""; - return o.str(); - }); -#endif // ifdef HAVE_CHOLMOD - return; -} - -PYBIND11_MODULE(_libtoast_mpi, m) { - m.doc() = R"( - Interface to C++ TOAST MPI library. - - )"; - - init_mpi_atm(m); -} diff --git a/src/toast/_libtoast_mpi.hpp b/src/toast/_libtoast_mpi.hpp deleted file mode 100644 index 40dcfba33..000000000 --- a/src/toast/_libtoast_mpi.hpp +++ /dev/null @@ -1,13 +0,0 @@ - -// Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#ifndef LIBTOAST_MPI_HPP -#define LIBTOAST_MPI_HPP - -#include <_libtoast_common.hpp> - -#include - -#endif // ifndef LIBTOAST_MPI_HPP diff --git a/src/toast/_libtoast_sys.cpp b/src/toast/_libtoast_sys.cpp index df6253fde..9c5952ebb 100644 --- a/src/toast/_libtoast_sys.cpp +++ b/src/toast/_libtoast_sys.cpp @@ -50,16 +50,6 @@ void init_sys(py::module & m) { R"( Return a list of the currently available signals. )") - .def("use_mpi", &toast::Environment::use_mpi, - R"( - Return True if TOAST was compiled with MPI support **and** MPI - is supported in the current runtime environment. - )") - .def("use_mpi4py", &toast::Environment::use_mpi4py, - R"( - Return True if mpi4py was detected at install time **and** mpi4py - is supported in the current runtime environment. - )") .def("function_timers", &toast::Environment::function_timers, R"( Return True if function timing has been enabled. diff --git a/src/toast/mpi.py b/src/toast/mpi.py index ab6ae03cc..ad1761ed4 100644 --- a/src/toast/mpi.py +++ b/src/toast/mpi.py @@ -2,31 +2,39 @@ # All rights reserved. Use of this source code is governed by # a BSD-style license that can be found in the LICENSE file. +import os import sys import itertools import numpy as np -from .utils import Environment, Logger +from ._libtoast import Logger from .pshmem import MPIShared, MPILock -env = Environment.get() - -use_mpi = env.use_mpi() -use_mpi4py = env.use_mpi4py() - +use_mpi = None MPI = None -if use_mpi4py and (MPI is None): - try: - import mpi4py.MPI as MPI - except ImportError: - raise ImportError( - "TOAST built with MPI + mpi4py support, but mpi4py " - "not found at run time. Is mpi4py currently in " - "your python search path?" - ) +if use_mpi is None: + # Special handling for running on a NERSC login node. This is for convenience. + # The same behavior could be implemented with environment variables set in a + # shell resource file. + at_nersc = False + if "NERSC_HOST" in os.environ: + at_nersc = True + in_slurm = False + if "SLURM_JOB_NAME" in os.environ: + in_slurm = True + if not at_nersc or in_slurm: + try: + import mpi4py.MPI as MPI + + use_mpi = True + except: + # There could be many possible exceptions raised... + log = Logger.get() + log.info("mpi4py not found- using serial operations only") + use_mpi = False def get_world(): @@ -43,7 +51,7 @@ def get_world(): rank = 0 procs = 1 world = None - if use_mpi4py: + if use_mpi: world = MPI.COMM_WORLD rank = world.rank procs = world.size @@ -73,14 +81,14 @@ class Comm(object): def __init__(self, world=None, groupsize=0): log = Logger.get() if world is None: - if use_mpi4py: + if use_mpi: # Default is COMM_WORLD world = MPI.COMM_WORLD else: # MPI is disabled, leave world as None. pass else: - if use_mpi4py: + if use_mpi: # We were passed a communicator to use. Check that it is # actually a communicator, otherwise fall back to COMM_WORLD. if not isinstance(world, MPI.Comm): @@ -138,7 +146,7 @@ def __init__(self, world=None, groupsize=0): if self._ngroups == 1: # We just have one group with all processes. self._gcomm = self._wcomm - if use_mpi4py: + if use_mpi: self._rcomm = MPI.COMM_SELF else: self._rcomm = None diff --git a/src/toast/tests/runner.py b/src/toast/tests/runner.py index 226596e54..b219d183e 100644 --- a/src/toast/tests/runner.py +++ b/src/toast/tests/runner.py @@ -2,7 +2,7 @@ # All rights reserved. Use of this source code is governed by # a BSD-style license that can be found in the LICENSE file. -from ..mpi import MPI, use_mpi4py +from ..mpi import MPI, use_mpi import os import sys @@ -80,7 +80,7 @@ def test(name=None, verbosity=2): # We run tests with COMM_WORLD if available comm = None rank = 0 - if use_mpi4py: + if use_mpi: comm = MPI.COMM_WORLD rank = comm.rank diff --git a/src/toast/todmap/conviqt.py b/src/toast/todmap/conviqt.py index 15bf5d914..c1066ba94 100644 --- a/src/toast/todmap/conviqt.py +++ b/src/toast/todmap/conviqt.py @@ -2,7 +2,7 @@ # All rights reserved. Use of this source code is governed by # a BSD-style license that can be found in the LICENSE file. -from ..mpi import use_mpi4py +from ..mpi import use_mpi import numpy as np @@ -14,7 +14,7 @@ conviqt = None -if use_mpi4py: +if use_mpi: try: import libconviqt_wrapper as conviqt except ImportError: diff --git a/src/toast/todmap/madam.py b/src/toast/todmap/madam.py index 5b84e14b8..f3a71e263 100644 --- a/src/toast/todmap/madam.py +++ b/src/toast/todmap/madam.py @@ -2,7 +2,7 @@ # All rights reserved. Use of this source code is governed by # a BSD-style license that can be found in the LICENSE file. -from ..mpi import MPI, use_mpi4py +from ..mpi import MPI, use_mpi import os @@ -15,7 +15,7 @@ from ..utils import Logger, memreport madam = None -if use_mpi4py: +if use_mpi: try: import libmadam_wrapper as madam except ImportError: diff --git a/src/toast/utils.py b/src/toast/utils.py index 74ad8eb02..92b82b9ef 100644 --- a/src/toast/utils.py +++ b/src/toast/utils.py @@ -36,6 +36,8 @@ vfast_erfinv, ) +from .mpi import MPI, use_mpi + # This function sets the numba threading layer to (hopefully) be compatible with TOAST. # The TOAST threading concurrency is used to attempt to set the numba threading. We @@ -76,9 +78,7 @@ def set_numba_threading(): print("max toast threads = ", toastthreads, flush=True) rank = 0 - if env.use_mpi4py(): - from .mpi import MPI - + if use_mpi: rank = MPI.COMM_WORLD.rank threading = "default"