MADWorld/PaRSEC and TTG/PaRSEC can coexist by having TA initialize Pa…

…RSEC context and feeding it to both

bumps TTG and MAD tags to pull in m-a-d-n-e-s-s/madness#509

INSTALL.md

@@ -42,7 +42,7 @@ Both methods are supported. However, for most users we _strongly_ recommend to b
   - Boost.Range: header-only, *only used for unit testing*
 - [BTAS](http://github.com/ValeevGroup/BTAS), tag bf0c376d5cdd6f668174b2a4c67b19634d1c0da7 . If usable BTAS installation is not found, TiledArray will download and compile
   BTAS from source. *This is the recommended way to compile BTAS for all users*.
-- [MADNESS](https://github.com/m-a-d-n-e-s-s/madness), tag 03c82cf2780d9e96298cc9140ac128c73eacd3b1 .
+- [MADNESS](https://github.com/m-a-d-n-e-s-s/madness), tag 41375b2b9f4a877ee4c33c67f9162c9c2efe1629 .
   Only the MADworld runtime and BLAS/LAPACK C API component of MADNESS is used by TiledArray.
   If usable MADNESS installation is not found, TiledArray will download and compile
   MADNESS from source. *This is the recommended way to compile MADNESS for all users*.
@@ -75,7 +75,7 @@ Optional prerequisites:
     - [blacspp](https://github.com/wavefunction91/blacspp.git) -- a modern C++ (C++17) wrapper for BLACS
 - Python3 interpreter -- to test (optionally-built) Python bindings
 - [Range-V3](https://github.com/ericniebler/range-v3.git) -- a Ranges library that served as the basis for Ranges component of C++20; only used for some unit testing of the functionality anticipated to be supported by future C++ standards.
-- [TTG](https://github.com/TESSEorg/ttg.git) -- C++ implementation of the Template Task Graph programming model for fine-grained flow-graph composition of distributed memory programs (tag 3fe4a06dbf4b05091269488aab38223da1f8cb8e).
+- [TTG](https://github.com/TESSEorg/ttg.git) -- C++ implementation of the Template Task Graph programming model for fine-grained flow-graph composition of distributed memory programs (tag 6b1cc1eaab54641b6885ccb66b620d8057899245).
 
 Many of these dependencies can be installed with a package manager,
 such as Homebrew on OS X or apt-get on Debian Linux distributions;

external/versions.cmake

@@ -19,8 +19,8 @@ set(TA_INSTALL_EIGEN_PREVIOUS_VERSION 3.3.7)
 set(TA_INSTALL_EIGEN_URL_HASH SHA256=b4c198460eba6f28d34894e3a5710998818515104d6e74e5cc331ce31e46e626)
 set(TA_INSTALL_EIGEN_PREVIOUS_URL_HASH MD5=b9e98a200d2455f06db9c661c5610496)
 
-set(TA_TRACKED_MADNESS_TAG 03c82cf2780d9e96298cc9140ac128c73eacd3b1)
-set(TA_TRACKED_MADNESS_PREVIOUS_TAG 4f7d30b0a738621037b96bb5b820029835753667)
+set(TA_TRACKED_MADNESS_TAG 41375b2b9f4a877ee4c33c67f9162c9c2efe1629)
+set(TA_TRACKED_MADNESS_PREVIOUS_TAG 03c82cf2780d9e96298cc9140ac128c73eacd3b1)
 set(TA_TRACKED_MADNESS_VERSION 0.10.1)
 set(TA_TRACKED_MADNESS_PREVIOUS_VERSION 0.10.1)
 
@@ -40,5 +40,5 @@ set(TA_TRACKED_RANGEV3_TAG 2e0591c57fce2aca6073ad6e4fdc50d841827864)
 set(TA_TRACKED_RANGEV3_PREVIOUS_TAG dbdaa247a25a0daa24c68f1286a5693c72ea0006)
 
 set(TA_TRACKED_TTG_URL https://github.com/TESSEorg/ttg)
-set(TA_TRACKED_TTG_TAG 3fe4a06dbf4b05091269488aab38223da1f8cb8e)
-set(TA_TRACKED_TTG_PREVIOUS_TAG 26da9b40872660b864794658d4fdeee1a95cb4d6)
+set(TA_TRACKED_TTG_TAG 6b1cc1eaab54641b6885ccb66b620d8057899245)
+set(TA_TRACKED_TTG_PREVIOUS_TAG 3fe4a06dbf4b05091269488aab38223da1f8cb8e)

src/TiledArray/tiledarray.cpp

@@ -62,6 +62,18 @@ inline bool& finalized_accessor() {
   return flag;
 }
 
+inline bool& initialized_mpi_accessor() {
+  static bool flag = false;
+  return flag;
+}
+
+#if defined(TILEDARRAY_HAS_TTG) || defined(HAVE_PARSEC)
+inline parsec_context_t*& initialized_parsec_ctx_accessor() {
+  static parsec_context_t* ctx = nullptr;
+  return ctx;
+}
+#endif
+
 inline bool& quiet_accessor() {
   static bool quiet = false;
   return quiet;
@@ -93,6 +105,49 @@ TiledArray::World& TiledArray::initialize(int& argc, char**& argv,
             "MADWorld initialized before TiledArray::initialize(argc, argv, "
             "comm), but not initialized with comm");
     }
+
+    // if need to use TTG, and MADNESS uses PaRSEC *also*
+    // must initialize PaRSEC here so that both MADNESS and TTG use
+    // the same context
+#if defined(TILEDARRAY_HAS_TTG) && defined(HAVE_PARSEC)
+    if (!ttg::initialized()) {
+      // preconditions:
+      // - MADWorld is NOT initialized, or initialized and used PaRSEC context
+      // using same communicator as comm
+      if (madness::initialized()) {
+        auto parsec_comm = reinterpret_cast<MPI_Comm>(
+            madness::ParsecRuntime::context()->comm_ctx);
+        int parsec_comm_equiv_comm = 0;
+        MADNESS_MPI_TEST(MPI_Comm_compare(comm.Get_mpi_comm(), parsec_comm,
+                                          &parsec_comm_equiv_comm));
+        if (!parsec_comm_equiv_comm)
+          throw std::runtime_error(
+              "TiledArray::initialize(): existing MADWorld's PaRSEC context "
+              "must use same communicator as given to TA::initialize()");
+      } else {
+        // if MPI is not initialized yet, initialize it here, and remember we
+        // did that
+        if (!SafeMPI::Is_initialized()) {
+          SafeMPI::Init_thread(argc, argv, MPI_THREAD_MULTIPLE);
+          initialized_mpi_accessor() = true;
+        }
+
+        // create a PaRSEC context using the given communicator
+        const auto nthreads = madness::ThreadPool::default_nthread();
+        const int num_parsec_threads = nthreads + 1;
+        auto* ctx = parsec_init(num_parsec_threads, &argc, &argv);
+        parsec_remote_dep_set_ctx(ctx, (intptr_t)comm.Get_mpi_comm());
+
+        // tell MADWorld to use this context
+        madness::ParsecRuntime::initialize_with_existing_context(ctx);
+
+        // remember the context and free it in finalize()
+        initialized_parsec_ctx_accessor() = ctx;
+      }
+    }
+#endif
+
+    // initialize MADWorld, if not yet initialized
     auto& default_world = initialized_madworld()
                               ? madness::initialize(argc, argv, comm, quiet)
                               : *madness::World::find_instance(comm);
@@ -107,17 +162,14 @@ TiledArray::World& TiledArray::initialize(int& argc, char**& argv,
     quiet_accessor() = quiet;
 
     // if have TTG, initialize it also
-#if TILEDARRAY_HAS_TTG
-    // MADNESS/PaRSEC creates PaRSEC context that uses MPI_COMM_SELF to avoid
-    // creation of a PaRSEC comm thread to be able to use TTG/PaRSEC need to
-    // tell PaRSEC context to use the full communicator
-    if (madness::ParsecRuntime::context()->nb_nodes != default_world.size()) {
-      auto default_world_comm = default_world.mpi.comm().Get_mpi_comm();
-      parsec_remote_dep_set_ctx(madness::ParsecRuntime::context(),
-                                (intptr_t)default_world_comm);
-    }
+#if defined(TILEDARRAY_HAS_TTG)
+    // use same PaRSEC context as used by MADWorld if it uses PaRSEC
+#if defined(HAVE_PARSEC)
     ttg::initialize(argc, argv, -1, madness::ParsecRuntime::context());
-#endif
+#else   // defined(HAVE_PARSEC)
+    ttg::initialize(argc, argv, -1);
+#endif  // defined(HAVE_PARSEC)
+#endif  // defined(TILEDARRAY_HAS_TTG)
 
     // check if user specified linear algebra backend + params
     auto* linalg_backend_cstr = std::getenv("TA_LINALG_BACKEND");
@@ -173,6 +225,16 @@ void TiledArray::finalize() {
     madness::finalize();
   }
   TiledArray::reset_default_world();
+#if defined(TILEDARRAY_HAS_TTG) && defined(HAVE_PARSEC)
+  if (initialized_parsec_ctx_accessor()) {
+    parsec_context_wait(initialized_parsec_ctx_accessor());
+    parsec_fini(&initialized_parsec_ctx_accessor());
+    initialized_parsec_ctx_accessor() = nullptr;
+  }
+  if (initialized_mpi_accessor()) {
+    SafeMPI::Finalize();
+  }
+#endif  // defined(TILEDARRAY_HAS_TTG) && defined(HAVE_PARSEC)
   initialized_accessor() = false;
   finalized_accessor() = true;
 }