From 25816392c645772ad7aaad51e7c3323db2b962e0 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 9 Oct 2024 20:54:11 -0600 Subject: [PATCH 1/5] working --- src/interface/data_collection.hpp | 2 +- src/interface/mesh_data.hpp | 8 +++++ src/interface/meshblock_data.hpp | 49 ++++++++++++++++++++++++++++-- src/interface/state_descriptor.cpp | 1 + src/interface/state_descriptor.hpp | 8 +++++ src/mesh/mesh-amr_loadbalance.cpp | 5 +-- src/mesh/mesh.cpp | 11 ++++--- src/mesh/mesh.hpp | 3 +- 8 files changed, 76 insertions(+), 11 deletions(-) diff --git a/src/interface/data_collection.hpp b/src/interface/data_collection.hpp index 043bf878f8b8..485b642498f9 100644 --- a/src/interface/data_collection.hpp +++ b/src/interface/data_collection.hpp @@ -58,7 +58,7 @@ class DataCollection { auto key = GetKey(name, src); auto it = containers_.find(key); if (it != containers_.end()) { - if (fields.size() && !(it->second)->Contains(fields)) { + if (fields.size() && !(it->second)->CreatedFrom(fields)) { PARTHENON_THROW(key + " already exists in collection but fields do not match."); } return it->second; diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp index 621ffce9763f..14ae3959c32a 100644 --- a/src/interface/mesh_data.hpp +++ b/src/interface/mesh_data.hpp @@ -476,6 +476,14 @@ class MeshData { [this, vars](const auto &b) { return b->ContainsExactly(vars); }); } + // Checks that the same set of variables was requested to create this container + // (which may be different than the set of variables in the container because of fluxes) + template + bool CreatedFrom(const Vars_t &vars) const noexcept { + return std::all_of(block_data_.begin(), block_data_.end(), + [this, vars](const auto &b) { return b->CreatedFrom(vars); }); + } + std::shared_ptr GetSwarmData(int n) { PARTHENON_REQUIRE(n >= 0 && n < block_data_.size(), "MeshData::GetSwarmData requires n within [0, block_data_.size()]"); diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp index 1ead0e5f957d..2e95800c2e79 100644 --- a/src/interface/meshblock_data.hpp +++ b/src/interface/meshblock_data.hpp @@ -141,6 +141,17 @@ class MeshBlockData { resolved_packages = resolved_packages_in; is_shallow_ = shallow_copy; + // Store the list of variables used to create this container + // so we can compare to it when searching the cache + varUidIn_.clear(); + if constexpr (std::is_same_v) { + for (const auto &var : vars) + varUidIn_.insert(Variable::GetUniqueID(var)); + } else { + for (const auto &var : vars) + varUidIn_.insert(var); + } + // clear all variables, maps, and pack caches varVector_.clear(); varMap_.clear(); @@ -185,9 +196,29 @@ class MeshBlockData { if (!found) add_var(src->GetVarPtr(flx_name)); } } + } else if constexpr (std::is_same_v && + std::is_same_v) { + for (const auto &v : vars) { + const auto &vid = resolved_packages->GetFieldVarID(v); + const auto &md = resolved_packages->GetFieldMetadata(v); + AddField(vid.base_name, md, vid.sparse_id); + // Add the associated flux as well if not explicitly + // asked for + if (md.IsSet(Metadata::WithFluxes)) { + auto flx_name = md.GetFluxName(); + bool found = false; + for (const auto &v2 : vars) + if (v2 == flx_name) found = true; + if (!found) { + const auto &vid = resolved_packages->GetFieldVarID(flx_name); + const auto &md = resolved_packages->GetFieldMetadata(flx_name); + AddField(vid.base_name, md, vid.sparse_id); + } + } + } } else { - PARTHENON_FAIL( - "Variable subset selection not yet implemented for MeshBlock input."); + PARTHENON_FAIL("Variable subset selection not yet implemented for MeshBlock " + "input with unique ids."); } } @@ -525,6 +556,18 @@ class MeshBlockData { return Contains(vars) && (vars.size() == varVector_.size()); } + bool CreatedFrom(const std::vector &vars) { + return (vars.size() == varUidIn_.size()) && + std::all_of(vars.begin(), vars.end(), + [this](const auto &v) { return this->varUidIn_.count(v); }); + } + bool CreatedFrom(const std::vector &vars) { + return (vars.size() == varUidIn_.size()) && + std::all_of(vars.begin(), vars.end(), [this](const auto &v) { + return this->varUidIn_.count(Variable::GetUniqueID(v)); + }); + } + void SetAllVariablesToInitialized() { std::for_each(varVector_.begin(), varVector_.end(), [](auto &sp_var) { sp_var->data.initialized = true; }); @@ -561,6 +604,8 @@ class MeshBlockData { VariableVector varVector_; ///< the saved variable array std::map>> varUidMap_; + std::set varUidIn_; // Uid list from which this MeshBlockData was created, + // empty implies all variables were included MapToVars varMap_; MetadataFlagToVariableMap flagsToVars_; diff --git a/src/interface/state_descriptor.cpp b/src/interface/state_descriptor.cpp index 9116298e8bd5..d970887e50ae 100644 --- a/src/interface/state_descriptor.cpp +++ b/src/interface/state_descriptor.cpp @@ -316,6 +316,7 @@ bool StateDescriptor::AddFieldImpl_(const VarID &vid, const Metadata &m_in, AddFieldImpl_(fId, *(m.GetSPtrFluxMetadata()), control_vid); m.SetFluxName(fId.label()); } + labelToVidMap_.insert({vid.label(), vid}); metadataMap_.insert({vid, m}); refinementFuncMaps_.Register(m, vid.label()); allocControllerReverseMap_.insert({vid, control_vid}); diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index 106c3ea6e525..b4c65d87481b 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -203,6 +203,13 @@ class StateDescriptor { // retrieve all swarm names std::vector Swarms() noexcept; + const auto &GetFieldVarID(const std::string &label) const { + return labelToVidMap_.at(label); + } + const auto &GetFieldMetadata(const std::string &label) const { + return metadataMap_.at(labelToVidMap_.at(label)); + } + const auto &GetFieldMetadata(const VarID &id) const { return metadataMap_.at(id); } const auto &AllFields() const noexcept { return metadataMap_; } const auto &AllSparsePools() const noexcept { return sparsePoolMap_; } const auto &AllSwarms() const noexcept { return swarmMetadataMap_; } @@ -397,6 +404,7 @@ class StateDescriptor { const std::string label_; // for each variable label (full label for sparse variables) hold metadata + std::unordered_map labelToVidMap_; std::unordered_map metadataMap_; std::unordered_map allocControllerReverseMap_; std::unordered_map> allocControllerMap_; diff --git a/src/mesh/mesh-amr_loadbalance.cpp b/src/mesh/mesh-amr_loadbalance.cpp index d54167026066..00d768720988 100644 --- a/src/mesh/mesh-amr_loadbalance.cpp +++ b/src/mesh/mesh-amr_loadbalance.cpp @@ -979,8 +979,9 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, ApplicationInput auto &md_noncc = mesh_data.AddShallow(noncc, md, noncc_names); } - CommunicateBoundaries(noncc); // Called to make sure shared values are correct, - // ghosts of non-cell centered vars may get some junk + CommunicateBoundaries( + noncc, noncc_names); // Called to make sure shared values are correct, + // ghosts of non-cell centered vars may get some junk // Now there is the correct data for prolongating on un-shared topological elements // on the new fine blocks if (nprolong > 0) { diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index d513ec6e0b52..c45f4f9679f2 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -644,7 +644,8 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { } } -void Mesh::CommunicateBoundaries(std::string md_name) { +void Mesh::CommunicateBoundaries(std::string md_name, + const std::vector &fields) { const int num_partitions = DefaultNumPartitions(); const int nmb = GetNumMeshBlocksThisRank(Globals::my_rank); constexpr std::int64_t max_it = 1e10; @@ -656,7 +657,7 @@ void Mesh::CommunicateBoundaries(std::string md_name) { do { all_sent = true; for (int i = 0; i < partitions.size(); ++i) { - auto &md = mesh_data.Add(md_name, partitions[i]); + auto &md = mesh_data.Add(md_name, partitions[i], fields); if (!sent[i]) { if (SendBoundaryBuffers(md) != TaskStatus::complete) { all_sent = false; @@ -680,7 +681,7 @@ void Mesh::CommunicateBoundaries(std::string md_name) { do { all_received = true; for (int i = 0; i < partitions.size(); ++i) { - auto &md = mesh_data.Add(md_name, partitions[i]); + auto &md = mesh_data.Add(md_name, partitions[i], fields); if (!received[i]) { if (ReceiveBoundaryBuffers(md) != TaskStatus::complete) { all_received = false; @@ -696,14 +697,14 @@ void Mesh::CommunicateBoundaries(std::string md_name) { "Too many iterations waiting to receive boundary communication buffers."); for (auto &partition : partitions) { - auto &md = mesh_data.Add(md_name, partition); + auto &md = mesh_data.Add(md_name, partition, fields); // unpack FillGhost variables SetBoundaries(md); } // Now do prolongation, compute primitives, apply BCs for (auto &partition : partitions) { - auto &md = mesh_data.Add(md_name, partition); + auto &md = mesh_data.Add(md_name, partition, fields); if (multilevel) { ApplyBoundaryConditionsOnCoarseOrFineMD(md, true); ProlongateBoundaries(md); diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index 684a897aad56..88b99c333ad4 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -329,7 +329,8 @@ class Mesh { void SetupMPIComms(); void BuildTagMapAndBoundaryBuffers(); - void CommunicateBoundaries(std::string md_name = "base"); + void CommunicateBoundaries(std::string md_name = "base", + const std::vector &fields = {}); void PreCommFillDerived(); void FillDerived(); From 20c7f8dadb5bac66652de3d96298c3650a7eb4fe Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 9 Oct 2024 21:05:48 -0600 Subject: [PATCH 2/5] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb7051c780ba..326c1e6c55e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - [[PR 1161]](https://github.com/parthenon-hpc-lab/parthenon/pull/1161) Make flux field Metadata accessible, add Metadata::CellMemAligned flag, small perfomance upgrades ### Changed (changing behavior/API/variables/...) +- [[PR 1187]](https://github.com/parthenon-hpc-lab/parthenon/pull/1187) Make DataCollection::Add safer and generalize MeshBlockData::Initialize - [[PR 1186]](https://github.com/parthenon-hpc-lab/parthenon/pull/1186) Bump Kokkos submodule to 4.4.1 - [[PR 1171]](https://github.com/parthenon-hpc-lab/parthenon/pull/1171) Add PARTHENON_USE_SYSTEM_PACKAGES build option - [[PR 1172]](https://github.com/parthenon-hpc-lab/parthenon/pull/1172) Make parthenon manager robust against external MPI init and finalize calls From 6c385441bea0750b5ed88b296760f41821bbe4a5 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 10 Oct 2024 09:26:18 -0600 Subject: [PATCH 3/5] Make everything work --- src/interface/meshblock_data.hpp | 15 +++++---------- src/interface/state_descriptor.hpp | 6 ++++++ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp index 2e95800c2e79..a6b681baf142 100644 --- a/src/interface/meshblock_data.hpp +++ b/src/interface/meshblock_data.hpp @@ -196,8 +196,7 @@ class MeshBlockData { if (!found) add_var(src->GetVarPtr(flx_name)); } } - } else if constexpr (std::is_same_v && - std::is_same_v) { + } else if constexpr (std::is_same_v) { for (const auto &v : vars) { const auto &vid = resolved_packages->GetFieldVarID(v); const auto &md = resolved_packages->GetFieldMetadata(v); @@ -205,20 +204,16 @@ class MeshBlockData { // Add the associated flux as well if not explicitly // asked for if (md.IsSet(Metadata::WithFluxes)) { - auto flx_name = md.GetFluxName(); + auto flx_vid = resolved_packages->GetFieldVarID(md.GetFluxName()); bool found = false; for (const auto &v2 : vars) - if (v2 == flx_name) found = true; + if (resolved_packages->GetFieldVarID(v2) == flx_vid) found = true; if (!found) { - const auto &vid = resolved_packages->GetFieldVarID(flx_name); - const auto &md = resolved_packages->GetFieldMetadata(flx_name); - AddField(vid.base_name, md, vid.sparse_id); + const auto &flx_md = resolved_packages->GetFieldMetadata(flx_vid); + AddField(flx_vid.base_name, flx_md, flx_vid.sparse_id); } } } - } else { - PARTHENON_FAIL("Variable subset selection not yet implemented for MeshBlock " - "input with unique ids."); } } diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index b4c65d87481b..b4ef1a5892a8 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -203,6 +203,12 @@ class StateDescriptor { // retrieve all swarm names std::vector Swarms() noexcept; + const auto GetFieldVarID(const VarID &id) const { + PARTHENON_REQUIRE_THROWS(metadataMap_.count(id), + "Asking for a variable that is not in this StateDescriptor."); + return id; + } + const auto &GetFieldVarID(const std::string &label) const { return labelToVidMap_.at(label); } From f4f23c910959b784c8433232507c56e023fbb789 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 10 Oct 2024 09:28:45 -0600 Subject: [PATCH 4/5] format --- src/interface/state_descriptor.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/interface/state_descriptor.hpp b/src/interface/state_descriptor.hpp index b4ef1a5892a8..8f8898170f76 100644 --- a/src/interface/state_descriptor.hpp +++ b/src/interface/state_descriptor.hpp @@ -204,8 +204,9 @@ class StateDescriptor { std::vector Swarms() noexcept; const auto GetFieldVarID(const VarID &id) const { - PARTHENON_REQUIRE_THROWS(metadataMap_.count(id), - "Asking for a variable that is not in this StateDescriptor."); + PARTHENON_REQUIRE_THROWS( + metadataMap_.count(id), + "Asking for a variable that is not in this StateDescriptor."); return id; } From 3a15cfeb8a59c8da3095803b3b3b3d7570daeed7 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 10 Oct 2024 10:47:18 -0600 Subject: [PATCH 5/5] maybe fix doc issue? --- .github/workflows/docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3ada901a78ac..4b95511d82f5 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,9 +23,9 @@ jobs: run: export DEBIAN_FRONTEND=noninteractive - name: install dependencies run: | - pip install sphinx - pip install sphinx-rtd-theme - pip install sphinx-multiversion + pip install --break-system-packages sphinx + pip install --break-system-packages sphinx-rtd-theme + pip install --break-system-packages sphinx-multiversion - name: build docs run: | echo "Repo = ${GITHUB_REPOSITORY}"