From 4893d52b3e06b7dca9008b40dab7155407d41ebb Mon Sep 17 00:00:00 2001 From: Eduard Valeyev Date: Thu, 2 Nov 2023 17:23:05 -0400 Subject: [PATCH] MADWorld/PaRSEC and TTG/PaRSEC can coexist by having TA initialize PaRSEC context and feeding it to both bumps TTG and MAD tags to pull in https://github.com/m-a-d-n-e-s-s/madness/pull/509 --- INSTALL.md | 4 +- external/versions.cmake | 8 ++-- src/TiledArray/tiledarray.cpp | 82 ++++++++++++++++++++++++++++++----- 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 1181e2d570..705ef1e0be 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -42,7 +42,7 @@ Both methods are supported. However, for most users we _strongly_ recommend to b - Boost.Range: header-only, *only used for unit testing* - [BTAS](http://github.com/ValeevGroup/BTAS), tag bf0c376d5cdd6f668174b2a4c67b19634d1c0da7 . If usable BTAS installation is not found, TiledArray will download and compile BTAS from source. *This is the recommended way to compile BTAS for all users*. -- [MADNESS](https://github.com/m-a-d-n-e-s-s/madness), tag 03c82cf2780d9e96298cc9140ac128c73eacd3b1 . +- [MADNESS](https://github.com/m-a-d-n-e-s-s/madness), tag 41375b2b9f4a877ee4c33c67f9162c9c2efe1629 . Only the MADworld runtime and BLAS/LAPACK C API component of MADNESS is used by TiledArray. If usable MADNESS installation is not found, TiledArray will download and compile MADNESS from source. *This is the recommended way to compile MADNESS for all users*. @@ -75,7 +75,7 @@ Optional prerequisites: - [blacspp](https://github.com/wavefunction91/blacspp.git) -- a modern C++ (C++17) wrapper for BLACS - Python3 interpreter -- to test (optionally-built) Python bindings - [Range-V3](https://github.com/ericniebler/range-v3.git) -- a Ranges library that served as the basis for Ranges component of C++20; only used for some unit testing of the functionality anticipated to be supported by future C++ standards. -- [TTG](https://github.com/TESSEorg/ttg.git) -- C++ implementation of the Template Task Graph programming model for fine-grained flow-graph composition of distributed memory programs (tag 3fe4a06dbf4b05091269488aab38223da1f8cb8e). +- [TTG](https://github.com/TESSEorg/ttg.git) -- C++ implementation of the Template Task Graph programming model for fine-grained flow-graph composition of distributed memory programs (tag 6b1cc1eaab54641b6885ccb66b620d8057899245). Many of these dependencies can be installed with a package manager, such as Homebrew on OS X or apt-get on Debian Linux distributions; diff --git a/external/versions.cmake b/external/versions.cmake index b5c6309e6f..db360bc5c3 100644 --- a/external/versions.cmake +++ b/external/versions.cmake @@ -19,8 +19,8 @@ set(TA_INSTALL_EIGEN_PREVIOUS_VERSION 3.3.7) set(TA_INSTALL_EIGEN_URL_HASH SHA256=b4c198460eba6f28d34894e3a5710998818515104d6e74e5cc331ce31e46e626) set(TA_INSTALL_EIGEN_PREVIOUS_URL_HASH MD5=b9e98a200d2455f06db9c661c5610496) -set(TA_TRACKED_MADNESS_TAG 03c82cf2780d9e96298cc9140ac128c73eacd3b1) -set(TA_TRACKED_MADNESS_PREVIOUS_TAG 4f7d30b0a738621037b96bb5b820029835753667) +set(TA_TRACKED_MADNESS_TAG 41375b2b9f4a877ee4c33c67f9162c9c2efe1629) +set(TA_TRACKED_MADNESS_PREVIOUS_TAG 03c82cf2780d9e96298cc9140ac128c73eacd3b1) set(TA_TRACKED_MADNESS_VERSION 0.10.1) set(TA_TRACKED_MADNESS_PREVIOUS_VERSION 0.10.1) @@ -40,5 +40,5 @@ set(TA_TRACKED_RANGEV3_TAG 2e0591c57fce2aca6073ad6e4fdc50d841827864) set(TA_TRACKED_RANGEV3_PREVIOUS_TAG dbdaa247a25a0daa24c68f1286a5693c72ea0006) set(TA_TRACKED_TTG_URL https://github.com/TESSEorg/ttg) -set(TA_TRACKED_TTG_TAG 3fe4a06dbf4b05091269488aab38223da1f8cb8e) -set(TA_TRACKED_TTG_PREVIOUS_TAG 26da9b40872660b864794658d4fdeee1a95cb4d6) +set(TA_TRACKED_TTG_TAG 6b1cc1eaab54641b6885ccb66b620d8057899245) +set(TA_TRACKED_TTG_PREVIOUS_TAG 3fe4a06dbf4b05091269488aab38223da1f8cb8e) diff --git a/src/TiledArray/tiledarray.cpp b/src/TiledArray/tiledarray.cpp index 7d58434979..97687efcb8 100644 --- a/src/TiledArray/tiledarray.cpp +++ b/src/TiledArray/tiledarray.cpp @@ -62,6 +62,18 @@ inline bool& finalized_accessor() { return flag; } +inline bool& initialized_mpi_accessor() { + static bool flag = false; + return flag; +} + +#if defined(TILEDARRAY_HAS_TTG) || defined(HAVE_PARSEC) +inline parsec_context_t*& initialized_parsec_ctx_accessor() { + static parsec_context_t* ctx = nullptr; + return ctx; +} +#endif + inline bool& quiet_accessor() { static bool quiet = false; return quiet; @@ -93,6 +105,49 @@ TiledArray::World& TiledArray::initialize(int& argc, char**& argv, "MADWorld initialized before TiledArray::initialize(argc, argv, " "comm), but not initialized with comm"); } + + // if need to use TTG, and MADNESS uses PaRSEC *also* + // must initialize PaRSEC here so that both MADNESS and TTG use + // the same context +#if defined(TILEDARRAY_HAS_TTG) && defined(HAVE_PARSEC) + if (!ttg::initialized()) { + // preconditions: + // - MADWorld is NOT initialized, or initialized and used PaRSEC context + // using same communicator as comm + if (madness::initialized()) { + auto parsec_comm = reinterpret_cast( + madness::ParsecRuntime::context()->comm_ctx); + int parsec_comm_equiv_comm = 0; + MADNESS_MPI_TEST(MPI_Comm_compare(comm.Get_mpi_comm(), parsec_comm, + &parsec_comm_equiv_comm)); + if (!parsec_comm_equiv_comm) + throw std::runtime_error( + "TiledArray::initialize(): existing MADWorld's PaRSEC context " + "must use same communicator as given to TA::initialize()"); + } else { + // if MPI is not initialized yet, initialize it here, and remember we + // did that + if (!SafeMPI::Is_initialized()) { + SafeMPI::Init_thread(argc, argv, MPI_THREAD_MULTIPLE); + initialized_mpi_accessor() = true; + } + + // create a PaRSEC context using the given communicator + const auto nthreads = madness::ThreadPool::default_nthread(); + const int num_parsec_threads = nthreads + 1; + auto* ctx = parsec_init(num_parsec_threads, &argc, &argv); + parsec_remote_dep_set_ctx(ctx, (intptr_t)comm.Get_mpi_comm()); + + // tell MADWorld to use this context + madness::ParsecRuntime::initialize_with_existing_context(ctx); + + // remember the context and free it in finalize() + initialized_parsec_ctx_accessor() = ctx; + } + } +#endif + + // initialize MADWorld, if not yet initialized auto& default_world = initialized_madworld() ? madness::initialize(argc, argv, comm, quiet) : *madness::World::find_instance(comm); @@ -107,17 +162,14 @@ TiledArray::World& TiledArray::initialize(int& argc, char**& argv, quiet_accessor() = quiet; // if have TTG, initialize it also -#if TILEDARRAY_HAS_TTG - // MADNESS/PaRSEC creates PaRSEC context that uses MPI_COMM_SELF to avoid - // creation of a PaRSEC comm thread to be able to use TTG/PaRSEC need to - // tell PaRSEC context to use the full communicator - if (madness::ParsecRuntime::context()->nb_nodes != default_world.size()) { - auto default_world_comm = default_world.mpi.comm().Get_mpi_comm(); - parsec_remote_dep_set_ctx(madness::ParsecRuntime::context(), - (intptr_t)default_world_comm); - } +#if defined(TILEDARRAY_HAS_TTG) + // use same PaRSEC context as used by MADWorld if it uses PaRSEC +#if defined(HAVE_PARSEC) ttg::initialize(argc, argv, -1, madness::ParsecRuntime::context()); -#endif +#else // defined(HAVE_PARSEC) + ttg::initialize(argc, argv, -1); +#endif // defined(HAVE_PARSEC) +#endif // defined(TILEDARRAY_HAS_TTG) // check if user specified linear algebra backend + params auto* linalg_backend_cstr = std::getenv("TA_LINALG_BACKEND"); @@ -173,6 +225,16 @@ void TiledArray::finalize() { madness::finalize(); } TiledArray::reset_default_world(); +#if defined(TILEDARRAY_HAS_TTG) && defined(HAVE_PARSEC) + if (initialized_parsec_ctx_accessor()) { + parsec_context_wait(initialized_parsec_ctx_accessor()); + parsec_fini(&initialized_parsec_ctx_accessor()); + initialized_parsec_ctx_accessor() = nullptr; + } + if (initialized_mpi_accessor()) { + SafeMPI::Finalize(); + } +#endif // defined(TILEDARRAY_HAS_TTG) && defined(HAVE_PARSEC) initialized_accessor() = false; finalized_accessor() = true; }