diff --git a/packages/panzer/dof-mgr/src/Panzer_DOFManager.cpp b/packages/panzer/dof-mgr/src/Panzer_DOFManager.cpp index 228f484d2b61..a1ceef484d7b 100644 --- a/packages/panzer/dof-mgr/src/Panzer_DOFManager.cpp +++ b/packages/panzer/dof-mgr/src/Panzer_DOFManager.cpp @@ -466,7 +466,7 @@ void DOFManager::buildGlobalUnknowns() connMngr_->buildConnectivity(*aggFieldPattern); // using new geometric pattern, build global unknowns - buildGlobalUnknowns(aggFieldPattern); + this->buildGlobalUnknowns(aggFieldPattern); } /////////////////////////////////////////////////////////////////////////////// @@ -512,13 +512,13 @@ void DOFManager::buildGlobalUnknowns(const Teuchos::RCP & ge ElementBlockAccess ownedAccess(true,connMngr_); // INPUT: To the algorithm in the GUN paper - RCP tagged_overlap_mv = buildTaggedMultiVector(ownedAccess); + RCP tagged_overlap_mv = this->buildTaggedMultiVector(ownedAccess); RCP overlap_map = tagged_overlap_mv->getMap(); RCP overlap_mv = Tpetra::createMultiVector(overlap_map,(size_t)numFields_); // call the GUN paper algorithm - auto non_overlap_pair = buildGlobalUnknowns_GUN(*tagged_overlap_mv,*overlap_mv); + auto non_overlap_pair = this->buildGlobalUnknowns_GUN(*tagged_overlap_mv,*overlap_mv); RCP non_overlap_mv = non_overlap_pair.first; RCP tagged_non_overlap_mv = non_overlap_pair.second; RCP non_overlap_map = non_overlap_mv->getMap(); @@ -529,7 +529,7 @@ void DOFManager::buildGlobalUnknowns(const Teuchos::RCP & ge // this bit of code takes the uniquely assigned GIDs and spreads them // out for processing by local element ID - fillGIDsFromOverlappedMV(ownedAccess,elementGIDs_,*overlap_map,*overlap_mv); + this->fillGIDsFromOverlappedMV(ownedAccess,elementGIDs_,*overlap_map,*overlap_mv); // if neighbor unknowns are required, then make sure they are included // in the elementGIDs_ @@ -537,7 +537,7 @@ void DOFManager::buildGlobalUnknowns(const Teuchos::RCP & ge // neighbor processors ElementBlockAccess neighborAccess(false,connMngr_); RCP overlap_map_neighbor = - buildOverlapMapFromElements(neighborAccess); + this->buildOverlapMapFromElements(neighborAccess); // Export e(overlap_map_neighbor,non_overlap_map); Import imp_neighbor(non_overlap_map,overlap_map_neighbor); @@ -549,7 +549,7 @@ void DOFManager::buildGlobalUnknowns(const Teuchos::RCP & ge overlap_mv_neighbor->doImport(*non_overlap_mv, imp_neighbor, Tpetra::REPLACE); - fillGIDsFromOverlappedMV(neighborAccess, elementGIDs_, + this->fillGIDsFromOverlappedMV(neighborAccess, elementGIDs_, *overlap_map_neighbor, *overlap_mv_neighbor); } @@ -794,8 +794,7 @@ DOFManager::buildGlobalUnknowns_GUN(const Tpetra::MultiVectorgetLocalViewDevice(Tpetra::Access::ReadOnly); - auto mv_size = values.extent(0); - Kokkos::parallel_reduce(mv_size,panzer::dof_functors::SumRank2(values),localsum); + panzer::dof_functors::SumRank2{values}.apply(localsum); } /* 11. Create a map using local sums to generate final GIDs. @@ -906,7 +905,7 @@ DOFManager::buildTaggedMultiVector(const ElementBlockAccess & ownedAccess) } } - RCP overlapmap = buildOverlapMapFromElements(ownedAccess); + RCP overlapmap = this->buildOverlapMapFromElements(ownedAccess); // LINE 22: In the GUN paper...the overlap_mv is reused for the tagged multivector. // This is a bit of a practical abuse of the algorithm presented in the paper. diff --git a/packages/panzer/dof-mgr/src/Panzer_DOFManager.hpp b/packages/panzer/dof-mgr/src/Panzer_DOFManager.hpp index 10018d7e1396..e3df16036764 100644 --- a/packages/panzer/dof-mgr/src/Panzer_DOFManager.hpp +++ b/packages/panzer/dof-mgr/src/Panzer_DOFManager.hpp @@ -227,10 +227,10 @@ class DOFManager : public GlobalIndexer { void getElementGIDs(panzer::LocalOrdinal localElementID, std::vector & gids, const std::string & blockIdHint="") const; //! builds the global unknowns array - void buildGlobalUnknowns(); + virtual void buildGlobalUnknowns(); //! builds the global unknowns array - void buildGlobalUnknowns(const Teuchos::RCP & geomPattern); + virtual void buildGlobalUnknowns(const Teuchos::RCP & geomPattern); int getFieldNum(const std::string & string) const; @@ -364,6 +364,7 @@ class DOFManager : public GlobalIndexer { * and the owned element blocks. */ class ElementBlockAccess { + public: bool useOwned_; Teuchos::RCP connMngr_; public: @@ -383,13 +384,13 @@ class DOFManager : public GlobalIndexer { * This map is used to construct the GIDs, and also to communicate the used * GIDs. (this is steps 1 and 2) */ - Teuchos::RCP > + virtual Teuchos::RCP > buildOverlapMapFromElements(const ElementBlockAccess & access) const; /** Build a tagged multivector (as defined in GUN paper) to use in global unknown numbering algorithm. * Note that this is non-const. It does modify the elementBlockGIDCount member variable. */ - Teuchos::RCP > + virtual Teuchos::RCP > buildTaggedMultiVector(const ElementBlockAccess & access); /** Build global unknowns using the algorithm in the Global Unknowns Numbering paper (GUN). This @@ -397,12 +398,12 @@ class DOFManager : public GlobalIndexer { * tagged overlapped multi-vector (overlap_mv) is overwritten with the global IDs. Note * fields on geometric entities that are not assigned a global ID are given an entry of -1. */ - std::pair >, + virtual std::pair >, Teuchos::RCP > > buildGlobalUnknowns_GUN(const Tpetra::MultiVector & tagged_overlap_mv, Tpetra::MultiVector & overlap_mv) const; - void fillGIDsFromOverlappedMV(const ElementBlockAccess & access, + virtual void fillGIDsFromOverlappedMV(const ElementBlockAccess & access, std::vector > & elementGIDs, const Tpetra::Map & overlapmap, const Tpetra::MultiVector & overlap_mv) const; diff --git a/packages/panzer/dof-mgr/src/Panzer_DOFManager_Functors.hpp b/packages/panzer/dof-mgr/src/Panzer_DOFManager_Functors.hpp index 17ecbb4931f9..4d3f75764bfc 100644 --- a/packages/panzer/dof-mgr/src/Panzer_DOFManager_Functors.hpp +++ b/packages/panzer/dof-mgr/src/Panzer_DOFManager_Functors.hpp @@ -43,26 +43,32 @@ #ifndef __Panzer_DOFManager_Functors_hpp__ #define __Panzer_DOFManager_Functors_hpp__ +#include "Kokkos_Core.hpp" #include "Phalanx_KokkosDeviceTypes.hpp" namespace panzer { namespace dof_functors { //! Sums all entries of a Rank 2 Kokkos View -template +template struct SumRank2 { - typedef GO value_type; - typedef typename PHX::Device execution_space; - - ArrayType a_; + using policy_t = Kokkos::MDRangePolicy>; - SumRank2(ArrayType a) : a_(a) {} - - KOKKOS_INLINE_FUNCTION - void operator () (const unsigned int i, GO& lsum) const { - for (unsigned int j=0; j < a_.extent(1); ++j) - lsum += a_(i,j); - } + const view_t values; + + void apply(ReductionDataType& sum) const + { + const auto& values_ref = values; + + Kokkos::parallel_reduce( + policy_t({0, 0}, {values.extent(0), values.extent(1)}), + KOKKOS_LAMBDA(const typename policy_t::index_type indexi, const typename policy_t::index_type indexj, ReductionDataType& local_sum) + { + local_sum += values_ref(indexi, indexj); + }, + Kokkos::Sum(sum) + ); + } }; }