Merge pull request #2395 from sstsimulator/devel

Automatically Merged using SST Master Branch Merger

src/sst/elements/iris/Makefile.am

@@ -31,6 +31,7 @@ libsumi_la_SOURCES = \
   sumi/gatherv.cc \
   sumi/message.cc \
   sumi/monitor.cc \
+  sumi/rank_mapper.cc \
   sumi/reduce.cc \
   sumi/reduce_scatter.cc \
   sumi/scan.cc \
@@ -66,6 +67,7 @@ nobase_library_include_HEADERS = \
   sumi/network_id.h \
   sumi/null_buffer.h \
   sumi/options.h \
+  sumi/rank_mapper.h \
   sumi/reduce.h \
   sumi/reduce_scatter.h \
   sumi/scan.h \

src/sst/elements/iris/sumi/allgather.h

@@ -56,10 +56,6 @@ namespace SST::Iris::sumi {
 class AllgatherCollective : public DagCollective
 {
  public:
-//FIXME
-//  SPKT_DECLARE_BASE(AllgatherCollective)
-//  SPKT_DECLARE_CTOR(CollectiveEngine*, void*, void*,
-//                    int, int, int, int, Communicator*)
 
  protected:
   AllgatherCollective(CollectiveEngine* engine, void* dst, void* src,
@@ -111,13 +107,6 @@ class BruckAllgatherCollective :
   public AllgatherCollective
 {
  public:
-//FIXME
-//  SPKT_REGISTER_DERIVED(
-//    AllgatherCollective,
-//    BruckAllgatherCollective,
-//    "macro",
-//    "bruck",
-//    "Bruck log(N) allgather collective")
 
   BruckAllgatherCollective(CollectiveEngine* engine, void* dst, void* src,
                   int nelems, int type_size, int tag, int cq_id, Communicator* comm)
@@ -184,13 +173,6 @@ class RingAllgatherCollective :
   public AllgatherCollective
 {
  public:
-//FIXME
-//  SPKT_REGISTER_DERIVED(
-//    AllgatherCollective,
-//    RingAllgatherCollective,
-//    "macro",
-//    "ring",
-//    "O(N) ring allgather collective")
 
   RingAllgatherCollective(CollectiveEngine* engine, void* dst, void* src,
                           int nelems, int type_size, int tag, int cq_id, Communicator* comm)

src/sst/elements/iris/sumi/alltoall.h

@@ -48,17 +48,12 @@ Questions? Contact [email protected]
 #include <iris/sumi/collective_actor.h>
 #include <iris/sumi/collective_message.h>
 #include <iris/sumi/comm_functions.h>
-//#include <mercury/common/factory.h>
 #include <sst/core/eli/elementbuilder.h>
 
 namespace SST::Iris::sumi {
 
 class AllToAllCollective : public DagCollective {
  public:
-//FIXME
-//  SPKT_DECLARE_BASE(AllToAllCollective)
-//  SPKT_DECLARE_CTOR(CollectiveEngine*, void*, void*,
-//                    int, int, int, int, Communicator*)
 
   AllToAllCollective(CollectiveEngine* engine, void* dst, void* src,
                      int nelems, int type_size, int tag, int cq_id,
@@ -107,13 +102,6 @@ class BruckAlltoallCollective :
   public AllToAllCollective
 {
  public:
-//FIXME
-//  SPKT_REGISTER_DERIVED(
-//    AllToAllCollective,
-//    BruckAlltoallCollective,
-//    "macro",
-//    "bruck",
-//    "Bruck log(N) all-to-all collective")
 
   BruckAlltoallCollective(CollectiveEngine* engine, void* dst, void* src,
                             int nelems, int type_size, int tag, int cq_id, Communicator* comm)
@@ -173,13 +161,6 @@ class DirectAlltoallCollective :
   public AllToAllCollective
 {
  public:
-//FIXME
-//  SPKT_REGISTER_DERIVED(
-//    AllToAllCollective,
-//    DirectAlltoallCollective,
-//    "macro",
-//    "direct",
-//    "direct all-to-all collective")
 
     DirectAlltoallCollective(CollectiveEngine* engine, void *dst, void *src, int nelems,
                               int type_size, int tag, int cq_id, Communicator* comm) :

src/sst/elements/iris/sumi/rank_mapper.cc

@@ -0,0 +1,101 @@
+/**
+Copyright 2009-2023 National Technology and Engineering Solutions of Sandia,
+LLC (NTESS).  Under the terms of Contract DE-NA-0003525, the U.S. Government
+retains certain rights in this software.
+
+Sandia National Laboratories is a multimission laboratory managed and operated
+by National Technology and Engineering Solutions of Sandia, LLC., a wholly
+owned subsidiary of Honeywell International, Inc., for the U.S. Department of
+Energy's National Nuclear Security Administration under contract DE-NA0003525.
+
+Copyright (c) 2009-2023, NTESS
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, 
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of the copyright holder nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+#include <iris/sumi/rank_mapper.h>
+#include "common/errors.h"
+#include <mutex>
+#include <string>
+
+namespace SST::Iris::sumi {
+
+std::vector<RankMapping::ptr> RankMapping::app_ids_launched_(1024);
+std::map<std::string, RankMapping::ptr> RankMapping::app_names_launched_;
+std::vector<int> RankMapping::local_refcounts_(1024);
+
+static std::mutex mutex;
+
+RankMapping::ptr
+RankMapping::globalMapping(const std::string& name)
+{
+  std::lock_guard<std::mutex> lk(mutex);
+  auto iter = app_names_launched_.find(name);
+  if (iter == app_names_launched_.end()){
+    std::string err_msg = "cannot find global task mapping for ";
+    err_msg += name.c_str();
+    Hg::abort(err_msg);
+  }
+  auto ret = iter->second;
+  return ret;
+}
+
+const RankMapping::ptr&
+RankMapping::globalMapping(AppId aid)
+{
+  std::lock_guard<std::mutex> lk(mutex);
+  auto& mapping = app_ids_launched_[aid];
+  if (!mapping){
+    std::string err_msg = "No task mapping exists for app ";
+    err_msg += aid;
+    Hg::abort(err_msg);
+  }
+  return mapping;
+}
+
+void
+RankMapping::addGlobalMapping(AppId aid, const std::string &unique_name, const RankMapping::ptr &mapping)
+{
+  std::lock_guard<std::mutex> lk(mutex);
+  app_ids_launched_[aid] = mapping;
+  app_names_launched_[unique_name] = mapping;
+  local_refcounts_[aid]++;
+}
+
+void
+RankMapping::removeGlobalMapping(AppId aid, const std::string& name)
+{
+  std::lock_guard<std::mutex> lk(mutex);
+  local_refcounts_[aid]--;
+  if (local_refcounts_[aid] == 0){
+    app_ids_launched_[aid] = 0;
+    app_names_launched_.erase(name);
+  }
+}
+}

src/sst/elements/iris/sumi/rank_mapper.h

@@ -0,0 +1,124 @@
+/**
+Copyright 2009-2024 National Technology and Engineering Solutions of Sandia,
+LLC (NTESS).  Under the terms of Contract DE-NA-0003525, the U.S. Government
+retains certain rights in this software.
+
+Sandia National Laboratories is a multimission laboratory managed and operated
+by National Technology and Engineering Solutions of Sandia, LLC., a wholly
+owned subsidiary of Honeywell International, Inc., for the U.S. Department of
+Energy's National Nuclear Security Administration under contract DE-NA0003525.
+
+Copyright (c) 2009-2024, NTESS
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of the copyright holder nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+// NOTE: currently the merlin rank mapping capability is used
+//   which allows the mercury environment to just use logical
+//   node IDs. As such, this rank mapper is currently unused
+//   but could be used in the future. -- JPK 8/22/2024
+
+#pragma once
+
+#include <list>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <vector>
+
+#include "sst/core/serialization/serializable.h"
+
+namespace SST::Iris::sumi {
+
+using AppId = int;
+using NodeId = uint32_t;
+
+class RankMapping : public SST::Core::Serialization::serializable{
+public:
+  RankMapping(AppId aid) : aid_(aid) {}
+
+  using ptr = std::shared_ptr<RankMapping>;
+
+  NodeId rankToNode(int rank) const { return rank_to_node_indexing_[rank]; }
+
+  const std::list<int> &nodeToRanks(int node) const {
+    return node_to_rank_indexing_[node];
+  }
+
+  AppId aid() const { return aid_; }
+
+  int numRanks() const { return rank_to_node_indexing_.size(); }
+
+  int nproc() const { return rank_to_node_indexing_.size(); }
+
+  const std::vector<NodeId> &rankToNode() const {
+    return rank_to_node_indexing_;
+  }
+
+  std::vector<NodeId> &rankToNode() { return rank_to_node_indexing_; }
+
+  const std::vector<std::list<int>> &nodeToRank() const {
+    return node_to_rank_indexing_;
+  }
+
+  std::vector<std::list<int>> &nodeToRank() { return node_to_rank_indexing_; }
+
+  static const RankMapping::ptr &globalMapping(AppId aid);
+
+  static RankMapping::ptr globalMapping(const std::string &unique_name);
+
+  static void addGlobalMapping(AppId aid, const std::string &unique_name,
+                               const RankMapping::ptr &mapping);
+
+  static void removeGlobalMapping(AppId aid, const std::string &name);
+
+  /* Do not use.  For serialization only */
+  RankMapping() {}
+
+  void serialize_order(SST::Core::Serialization::serializer& ser) override
+  {
+  }
+
+  ImplementSerializable(RankMapping);
+
+private:
+  AppId aid_;
+  std::vector<NodeId> rank_to_node_indexing_;
+  std::vector<std::list<int>> node_to_rank_indexing_;
+  std::vector<int> core_affinities_;
+
+  static std::vector<int> local_refcounts_;
+  static std::vector<RankMapping::ptr> app_ids_launched_;
+  static std::map<std::string, RankMapping::ptr> app_names_launched_;
+
+  static void deleteStatics();
+};
+
+} // end namespace SST::Iris::sumi