WIP: MeshData-based particle communication

src/bvals/bvals.cpp

@@ -43,8 +43,9 @@
 
 namespace parthenon {
 
-BoundarySwarm::BoundarySwarm(std::weak_ptr<MeshBlock> pmb, const std::string &label)
-    : bswarm_index(), pmy_block(pmb), pmy_mesh_(pmb.lock()->pmy_mesh) {
+BoundarySwarm::BoundarySwarm(std::weak_ptr<MeshBlock> pmb, const std::string &label,
+                             const int swarm_idx)
+    : bswarm_index(swarm_idx), pmy_block(pmb), pmy_mesh_(pmb.lock()->pmy_mesh) {
 #ifdef MPI_PARALLEL
   swarm_comm = pmy_mesh_->GetMPIComm(label);
 #endif

src/bvals/comms/boundary_communication.cpp

@@ -384,6 +384,21 @@ ProlongateBounds<BoundaryType::nonlocal>(std::shared_ptr<MeshData<Real>> &);
 template TaskStatus
 ProlongateBounds<BoundaryType::gmg_prolongate_recv>(std::shared_ptr<MeshData<Real>> &);
 
+TaskID AddSwarmBoundaryExchangeTasks(TaskID dependency, TaskList &tl,
+                                     std::shared_ptr<MeshData<Real>> &md) {
+  auto send = tl.AddTask(dependency, SendSwarmBoundaryBuffers, md);
+  auto recv = tl.AddTask(dependency, ReceiveSwarmBoundaryBuffers, md);
+  return recv;
+}
+
+TaskID AddSwarmBoundaryExchangeTasks(TaskID dependency, TaskList &tl,
+                                     std::shared_ptr<MeshData<Real>> &md,
+                                     std::vector<std::string> swarm_names) {
+  auto send = tl.AddTask(dependency, SendSwarmBoundaryBuffers, md, swarm_names);
+  auto recv = tl.AddTask(dependency, ReceiveSwarmBoundaryBuffers, md, swarm_names);
+  return recv;
+}
+
 // Adds all relevant boundary communication to a single task list
 template <BoundaryType bounds>
 TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl,
@@ -441,4 +456,58 @@ TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl,
   auto receive = tl.AddTask(dependency, ReceiveBoundBufs<BoundaryType::flxcor_recv>, md);
   return tl.AddTask(receive, SetBounds<BoundaryType::flxcor_recv>, md);
 }
+
+TaskStatus SendSwarmBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md) {
+  const int nblocks = md->NumBlocks();
+  for (int n = 0; n < nblocks; n++) {
+    auto &sc = md->GetBlockData(n)->GetSwarmData();
+    sc->ResetCommunication();
+    // TODO(BRR) remove this BoundaryCommSubset argument?
+    sc->Send(BoundaryCommSubset::all);
+  }
+
+  return TaskStatus::complete;
+}
+
+TaskStatus SendSwarmBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md,
+                                    std::vector<std::string> swarm_names) {
+  const int nblocks = md->NumBlocks();
+  for (int n = 0; n < nblocks; n++) {
+    auto &sc = md->GetBlockData(n)->GetSwarmData();
+    sc->ResetCommunication(swarm_names);
+    sc->Send(BoundaryCommSubset::all, swarm_names);
+  }
+
+  return TaskStatus::complete;
+}
+
+TaskStatus ReceiveSwarmBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md) {
+  TaskStatus status = TaskStatus::complete;
+  const int nblocks = md->NumBlocks();
+  for (int n = 0; n < nblocks; n++) {
+    auto &sc = md->GetBlockData(n)->GetSwarmData();
+    auto local_status = sc->Receive(BoundaryCommSubset::all);
+    if (local_status == TaskStatus::incomplete) {
+      status = TaskStatus::incomplete;
+    }
+  }
+
+  return status;
+}
+
+TaskStatus ReceiveSwarmBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md,
+                                       std::vector<std::string> swarm_names) {
+  TaskStatus status = TaskStatus::complete;
+  const int nblocks = md->NumBlocks();
+  for (int n = 0; n < nblocks; n++) {
+    auto &sc = md->GetBlockData(n)->GetSwarmData();
+    auto local_status = sc->Receive(BoundaryCommSubset::all, swarm_names);
+    if (local_status == TaskStatus::incomplete) {
+      status = TaskStatus::incomplete;
+    }
+  }
+
+  return status;
+}
+
 } // namespace parthenon

src/bvals/comms/bvals_in_one.hpp

@@ -79,6 +79,13 @@ static TaskStatus SetFluxCorrections(std::shared_ptr<MeshData<Real>> &md) {
   return SetBounds<BoundaryType::flxcor_recv>(md);
 }
 
+TaskStatus SendSwarmBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md);
+TaskStatus SendSwarmBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md,
+                                    std::vector<std::string> swarm_names);
+TaskStatus ReceiveSwarmBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md);
+TaskStatus ReceiveSwarmBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md,
+                                       std::vector<std::string> swarm_names);
+
 // Adds all relevant boundary communication to a single task list
 template <BoundaryType bounds = BoundaryType::any>
 TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl,
@@ -88,6 +95,13 @@ TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl,
 TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl,
                               std::shared_ptr<MeshData<Real>> &md, bool multilevel);
 
+// Adds all relevant swarm boundary communication to a single task list
+TaskID AddSwarmBoundaryExchangeTasks(TaskID dependency, TaskList &tl,
+                                     std::shared_ptr<MashData<Real>> &md);
+TaskID AddSwarmBoundaryExchangeTasks(TaskID dependency, TaskList &tl,
+                                     std::shared_ptr<MashData<Real>> &md,
+                                     std::vector<std::string> swarm_names);
+
 // These tasks should not be called in down stream code
 TaskStatus BuildBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md);
 TaskStatus BuildGMGBoundaryBuffers(std::shared_ptr<MeshData<Real>> &md);

src/interface/swarm_comms.cpp

@@ -660,13 +660,13 @@ bool Swarm::FinalizeCommunicationIterative() {
   return true;
 }
 
-void Swarm::AllocateComms(std::weak_ptr<MeshBlock> wpmb) {
+void Swarm::AllocateComms(std::weak_ptr<MeshBlock> wpmb, const int swarm_idx) {
   if (wpmb.expired()) return;
 
   std::shared_ptr<MeshBlock> pmb = wpmb.lock();
 
   // Create the boundary object
-  vbswarm = std::make_shared<BoundarySwarm>(pmb, label_);
+  vbswarm = std::make_shared<BoundarySwarm>(pmb, label_, swarm_idx);
 
   // Enroll SwarmVariable object
   vbswarm->bswarm_index = pmb->pbswarm->bswarms.size();

src/interface/swarm_container.cpp

@@ -86,7 +86,8 @@ void SwarmContainer::Add(const std::string &label, const Metadata &metadata) {
 
   auto swarm = std::make_shared<Swarm>(label, metadata);
   swarm->SetBlockPointer(GetBlockPointer());
-  swarm->AllocateComms(GetBlockPointer());
+  const int swarm_idx = swarmVector_.size() - 1;
+  swarm->AllocateComms(GetBlockPointer(), swarm_idx);
   Add(swarm);
 }