diff --git a/Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptor.H b/Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptor.H index fbd5227824f..14499f606a1 100644 --- a/Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptor.H +++ b/Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptor.H @@ -16,7 +16,7 @@ namespace amrex { -template +template class AmrMeshParticleDataAdaptor : public sensei::DataAdaptor { public: @@ -27,7 +27,7 @@ public: AmrMesh *mesh, const std::vector*> &mesh_states, const std::vector> &mesh_names, - amrex::ParticleContainer * particles, + amrex::ParticleContainer_impl * particles, const std::map> & rStructs = {}, const std::map & iStructs = {}, const std::map> & rArrays = {}, @@ -55,7 +55,7 @@ protected: AmrMeshParticleDataAdaptor() { m_meshAdaptor = AmrMeshDataAdaptor::New(); - m_particleAdaptor = ParticleDataAdaptor::New(); + m_particleAdaptor = ParticleDataAdaptor::New(); } ~AmrMeshParticleDataAdaptor() @@ -65,7 +65,7 @@ protected: } private: - ParticleDataAdaptor* m_particleAdaptor; + ParticleDataAdaptor* m_particleAdaptor; AmrMeshDataAdaptor* m_meshAdaptor; const std::string m_meshName = "mesh"; diff --git a/Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptorI.H b/Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptorI.H index 4cbb53203b6..aaf795f790f 100644 --- a/Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptorI.H +++ b/Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptorI.H @@ -3,22 +3,22 @@ namespace amrex { //----------------------------------------------------------------------------- -template -AmrMeshParticleDataAdaptor* -AmrMeshParticleDataAdaptor::New() +template +AmrMeshParticleDataAdaptor* +AmrMeshParticleDataAdaptor::New() { - auto result = new AmrMeshParticleDataAdaptor; + auto result = new AmrMeshParticleDataAdaptor; result->InitializeObjectBase(); return result; } //----------------------------------------------------------------------------- -template -int AmrMeshParticleDataAdaptor::SetDataSource( +template +int AmrMeshParticleDataAdaptor::SetDataSource( AmrMesh *mesh, const std::vector*> &mesh_states, const std::vector> &mesh_names, - amrex::ParticleContainer * particles, + amrex::ParticleContainer_impl * particles, const std::map> & rStructs, const std::map & iStructs, const std::map> & rArrays, @@ -31,8 +31,8 @@ int AmrMeshParticleDataAdaptor:: // SENSEI API #if SENSEI_VERSION_MAJOR >= 3 -template -int AmrMeshParticleDataAdaptor::GetMeshMetadata( +template +int AmrMeshParticleDataAdaptor::GetMeshMetadata( unsigned int id, sensei::MeshMetadataPtr &metadata) { @@ -49,8 +49,8 @@ int AmrMeshParticleDataAdaptor:: } #else -template -int AmrMeshParticleDataAdaptor::GetMeshName( +template +int AmrMeshParticleDataAdaptor::GetMeshName( unsigned int id, std::string &meshName) { @@ -66,8 +66,8 @@ int AmrMeshParticleDataAdaptor:: return -1; } -template -int AmrMeshParticleDataAdaptor::GetMeshHasGhostNodes( +template +int AmrMeshParticleDataAdaptor::GetMeshHasGhostNodes( const std::string &meshName, int &nLayers) { if(id == 0) @@ -82,8 +82,8 @@ int AmrMeshParticleDataAdaptor:: return -1; } -template -int AmrMeshParticleDataAdaptor::GetMeshHasGhostCells( +template +int AmrMeshParticleDataAdaptor::GetMeshHasGhostCells( const std::string &meshName, int &nLayers) { if(id == 0) @@ -98,8 +98,8 @@ int AmrMeshParticleDataAdaptor:: return -1; } -template -int AmrMeshParticleDataAdaptor::GetNumberOfArrays( +template +int AmrMeshParticleDataAdaptor::GetNumberOfArrays( const std::string &meshName, int association, unsigned int &numberOfArrays) @@ -116,8 +116,8 @@ int AmrMeshParticleDataAdaptor:: return -1; } -template -int AmrMeshParticleDataAdaptor::GetArrayName( +template +int AmrMeshParticleDataAdaptor::GetArrayName( const std::string &meshName, int association, unsigned int index, @@ -136,16 +136,16 @@ int AmrMeshParticleDataAdaptor:: } #endif -template -int AmrMeshParticleDataAdaptor::GetNumberOfMeshes( +template +int AmrMeshParticleDataAdaptor::GetNumberOfMeshes( unsigned int &numMeshes) { numMeshes = 2; return 0; } -template -int AmrMeshParticleDataAdaptor::GetMesh( +template +int AmrMeshParticleDataAdaptor::GetMesh( const std::string &meshName, bool structureOnly, svtkDataObject *&mesh) @@ -162,8 +162,8 @@ int AmrMeshParticleDataAdaptor:: return -1; } -template -int AmrMeshParticleDataAdaptor::AddGhostNodesArray( +template +int AmrMeshParticleDataAdaptor::AddGhostNodesArray( svtkDataObject* mesh, const std::string &meshName) { @@ -179,8 +179,8 @@ int AmrMeshParticleDataAdaptor:: return -1; } -template -int AmrMeshParticleDataAdaptor::AddGhostCellsArray( +template +int AmrMeshParticleDataAdaptor::AddGhostCellsArray( svtkDataObject* mesh, const std::string &meshName) { @@ -196,8 +196,8 @@ int AmrMeshParticleDataAdaptor:: return -1; } -template -int AmrMeshParticleDataAdaptor::AddArray( +template +int AmrMeshParticleDataAdaptor::AddArray( svtkDataObject* mesh, const std::string &meshName, int association, @@ -215,8 +215,8 @@ int AmrMeshParticleDataAdaptor:: return -1; } -template -int AmrMeshParticleDataAdaptor::ReleaseData() +template +int AmrMeshParticleDataAdaptor::ReleaseData() { this->m_particleAdaptor->ReleaseData(); this->m_meshAdaptor->ReleaseData(); diff --git a/Src/Extern/SENSEI/AMReX_AmrMeshParticleInSituBridge.H b/Src/Extern/SENSEI/AMReX_AmrMeshParticleInSituBridge.H index 9208c8a753b..0a85cb7efa9 100644 --- a/Src/Extern/SENSEI/AMReX_AmrMeshParticleInSituBridge.H +++ b/Src/Extern/SENSEI/AMReX_AmrMeshParticleInSituBridge.H @@ -33,26 +33,26 @@ public: // invoke the in situ analysis with data from an AmrMesh and ParticleContainer // instance. - template + template int update( long step, double time, amrex::AmrMesh *mesh, const std::vector*> &mesh_states, const std::vector> &mesh_names, - amrex::ParticleContainer * particles, + amrex::ParticleContainer_impl * particles, const std::map> & particles_rStructs = {}, const std::map & particles_iStructs = {}, const std::map> & particles_rArrays = {}, const std::map & particles_iArrays = {}); }; -template +template int AmrMeshParticleInSituBridge::update( long step, double time, amrex::AmrMesh *mesh, const std::vector*> &mesh_states, const std::vector> &mesh_names, - amrex::ParticleContainer * particles, + amrex::ParticleContainer_impl * particles, const std::map> & particles_rStructs, const std::map & particles_iStructs, const std::map> & particles_rArrays, @@ -73,8 +73,8 @@ int AmrMeshParticleInSituBridge::update( return -1; } - amrex::AmrMeshParticleDataAdaptor *data_adaptor - = amrex::AmrMeshParticleDataAdaptor::New(); + amrex::AmrMeshParticleDataAdaptor *data_adaptor + = amrex::AmrMeshParticleDataAdaptor::New(); if (comm != MPI_COMM_NULL) data_adaptor->SetCommunicator(comm); diff --git a/Src/Extern/SENSEI/AMReX_AmrParticleDataAdaptor.H b/Src/Extern/SENSEI/AMReX_AmrParticleDataAdaptor.H index 3f7a945e019..b54326fb5b3 100644 --- a/Src/Extern/SENSEI/AMReX_AmrParticleDataAdaptor.H +++ b/Src/Extern/SENSEI/AMReX_AmrParticleDataAdaptor.H @@ -16,7 +16,7 @@ namespace amrex { -template +template class AmrParticleDataAdaptor : public sensei::DataAdaptor { public: @@ -25,7 +25,7 @@ public: int SetDataSource( Amr *amr, - amrex::ParticleContainer * particles, + amrex::ParticleContainer_impl * particles, const std::map> & rStructs = {}, const std::map & iStructs = {}, const std::map> & rArrays = {}, @@ -53,7 +53,7 @@ protected: AmrParticleDataAdaptor() { m_meshAdaptor = AmrDataAdaptor::New(); - m_particleAdaptor = ParticleDataAdaptor::New(); + m_particleAdaptor = ParticleDataAdaptor::New(); } ~AmrParticleDataAdaptor() @@ -63,7 +63,7 @@ protected: } private: - ParticleDataAdaptor* m_particleAdaptor; + ParticleDataAdaptor* m_particleAdaptor; AmrDataAdaptor* m_meshAdaptor; const std::string m_meshName = "mesh"; diff --git a/Src/Extern/SENSEI/AMReX_AmrParticleDataAdaptorI.H b/Src/Extern/SENSEI/AMReX_AmrParticleDataAdaptorI.H index 9035cd0c39c..1a01b0972f1 100644 --- a/Src/Extern/SENSEI/AMReX_AmrParticleDataAdaptorI.H +++ b/Src/Extern/SENSEI/AMReX_AmrParticleDataAdaptorI.H @@ -3,20 +3,20 @@ namespace amrex { //----------------------------------------------------------------------------- -template -AmrParticleDataAdaptor* -AmrParticleDataAdaptor::New() +template +AmrParticleDataAdaptor* +AmrParticleDataAdaptor::New() { - auto result = new AmrParticleDataAdaptor; + auto result = new AmrParticleDataAdaptor; result->InitializeObjectBase(); return result; } //----------------------------------------------------------------------------- -template -int AmrParticleDataAdaptor::SetDataSource( +template +int AmrParticleDataAdaptor::SetDataSource( Amr *amr, - amrex::ParticleContainer * particles, + amrex::ParticleContainer_impl * particles, const std::map> & rStructs, const std::map & iStructs, const std::map> & rArrays, @@ -29,8 +29,8 @@ int AmrParticleDataAdaptor::SetD // SENSEI API #if SENSEI_VERSION_MAJOR >= 3 -template -int AmrParticleDataAdaptor::GetMeshMetadata( +template +int AmrParticleDataAdaptor::GetMeshMetadata( unsigned int id, sensei::MeshMetadataPtr &metadata) { @@ -47,8 +47,8 @@ int AmrParticleDataAdaptor::GetM } #else -template -int AmrParticleDataAdaptor::GetMeshName( +template +int AmrParticleDataAdaptor::GetMeshName( unsigned int id, std::string &meshName) { @@ -64,8 +64,8 @@ int AmrParticleDataAdaptor::GetM return -1; } -template -int AmrParticleDataAdaptor::GetMeshHasGhostNodes( +template +int AmrParticleDataAdaptor::GetMeshHasGhostNodes( const std::string &meshName, int &nLayers) { if(id == 0) @@ -80,8 +80,8 @@ int AmrParticleDataAdaptor::GetM return -1; } -template -int AmrParticleDataAdaptor::GetMeshHasGhostCells( +template +int AmrParticleDataAdaptor::GetMeshHasGhostCells( const std::string &meshName, int &nLayers) { if(id == 0) @@ -96,8 +96,8 @@ int AmrParticleDataAdaptor::GetM return -1; } -template -int AmrParticleDataAdaptor::GetNumberOfArrays( +template +int AmrParticleDataAdaptor::GetNumberOfArrays( const std::string &meshName, int association, unsigned int &numberOfArrays) @@ -114,8 +114,8 @@ int AmrParticleDataAdaptor::GetN return -1; } -template -int AmrParticleDataAdaptor::GetArrayName( +template +int AmrParticleDataAdaptor::GetArrayName( const std::string &meshName, int association, unsigned int index, @@ -134,16 +134,16 @@ int AmrParticleDataAdaptor::GetA } #endif -template -int AmrParticleDataAdaptor::GetNumberOfMeshes( +template +int AmrParticleDataAdaptor::GetNumberOfMeshes( unsigned int &numMeshes) { numMeshes = 2; return 0; } -template -int AmrParticleDataAdaptor::GetMesh( +template +int AmrParticleDataAdaptor::GetMesh( const std::string &meshName, bool structureOnly, svtkDataObject *&mesh) @@ -160,8 +160,8 @@ int AmrParticleDataAdaptor::GetM return -1; } -template -int AmrParticleDataAdaptor::AddGhostNodesArray( +template +int AmrParticleDataAdaptor::AddGhostNodesArray( svtkDataObject* mesh, const std::string &meshName) { @@ -177,8 +177,8 @@ int AmrParticleDataAdaptor::AddG return -1; } -template -int AmrParticleDataAdaptor::AddGhostCellsArray( +template +int AmrParticleDataAdaptor::AddGhostCellsArray( svtkDataObject* mesh, const std::string &meshName) { @@ -194,8 +194,8 @@ int AmrParticleDataAdaptor::AddG return -1; } -template -int AmrParticleDataAdaptor::AddArray( +template +int AmrParticleDataAdaptor::AddArray( svtkDataObject* mesh, const std::string &meshName, int association, @@ -213,8 +213,8 @@ int AmrParticleDataAdaptor::AddA return -1; } -template -int AmrParticleDataAdaptor::ReleaseData() +template +int AmrParticleDataAdaptor::ReleaseData() { this->m_particleAdaptor->ReleaseData(); this->m_meshAdaptor->ReleaseData(); diff --git a/Src/Extern/SENSEI/AMReX_AmrParticleInSituBridge.H b/Src/Extern/SENSEI/AMReX_AmrParticleInSituBridge.H index a739e31675a..ee1d4972251 100644 --- a/Src/Extern/SENSEI/AMReX_AmrParticleInSituBridge.H +++ b/Src/Extern/SENSEI/AMReX_AmrParticleInSituBridge.H @@ -30,19 +30,19 @@ public: /** invoke the in situ analysis with data from an AmrMesh and ParticleContainer * instance. */ - template + template int update(amrex::Amr *amr, - amrex::ParticleContainer * particles, + amrex::ParticleContainer_impl * particles, const std::map> & particles_rStructs = {}, const std::map & particles_iStructs = {}, const std::map> & particles_rArrays = {}, const std::map & particles_iArrays = {}); }; -template +template int AmrParticleInSituBridge::update( amrex::Amr *amr, - amrex::ParticleContainer * particles, + amrex::ParticleContainer_impl * particles, const std::map> & particles_rStructs, const std::map & particles_iStructs, const std::map> & particles_rArrays, @@ -63,8 +63,8 @@ int AmrParticleInSituBridge::update( return -1; } - amrex::AmrParticleDataAdaptor *data_adaptor - = amrex::AmrParticleDataAdaptor::New(); + amrex::AmrParticleDataAdaptor *data_adaptor + = amrex::AmrParticleDataAdaptor::New(); if (comm != MPI_COMM_NULL) data_adaptor->SetCommunicator(comm); @@ -74,7 +74,7 @@ int AmrParticleInSituBridge::update( data_adaptor->SetDataTime(amr->cumTime()); data_adaptor->SetDataTimeStep(amr->levelSteps(0)); - ret = analysis_adaptor->Execute(data_adaptor) ? 0 : -1; + ret = analysis_adaptor->Execute(data_adaptor, nullptr) ? 0 : -1; data_adaptor->ReleaseData(); data_adaptor->Delete(); diff --git a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptor.H b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptor.H index f284b15831b..4608b4d032b 100644 --- a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptor.H +++ b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptor.H @@ -13,16 +13,19 @@ class svtkPolyData; namespace amrex { -template +template class ParticleDataAdaptor : public sensei::DataAdaptor { public: + static constexpr int NStructReal = ParticleType::NReal; + static constexpr int NStructInt = ParticleType::NInt; + static ParticleDataAdaptor* New(); senseiTypeMacro(ParticleDataAdaptor, sensei::DataAdaptor); // set the source of the simulation data int SetDataSource( - amrex::ParticleContainer * particles, + amrex::ParticleContainer_impl * particles, const std::map> & rStructs = {}, const std::map & iStructs = {}, const std::map> & rArrays = {}, @@ -83,7 +86,7 @@ private: const std::string m_particlesName = "particles"; - amrex::ParticleContainer* m_particles; + amrex::ParticleContainer_impl* m_particles; using RealDataMapType = std::map>; using IntDataMapType = std::map; diff --git a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H index 8a2d15562d3..bb5a573c11d 100644 --- a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H +++ b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H @@ -18,20 +18,20 @@ namespace amrex { //----------------------------------------------------------------------------- -template -ParticleDataAdaptor* -ParticleDataAdaptor::New() +template +ParticleDataAdaptor* +ParticleDataAdaptor::New() { - auto result = new ParticleDataAdaptor; + auto result = new ParticleDataAdaptor; result->InitializeObjectBase(); return result; } -// senseiNewMacro(ParticleDataAdaptor); +// senseiNewMacro(ParticleDataAdaptor); //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::SetDataSource( - amrex::ParticleContainer * particles, +template +int ParticleDataAdaptor::SetDataSource( + amrex::ParticleContainer_impl * particles, const std::map> & rStructs, const std::map & iStructs, const std::map> & rArrays, @@ -52,21 +52,21 @@ int ParticleDataAdaptor::SetData } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::SetArrayNames( +template +int ParticleDataAdaptor::SetArrayNames( const std::map> & rStructs, const std::map & iStructs, const std::map> & rArrays, const std::map & iArrays) { - if(rStructs.size() <= NStructReal) + if(rStructs.size() <= ParticleType::NReal) { // check that no indices from the rStructs map exceed allowable value for(auto s : rStructs) { for(auto i : s.second) { - if(i >= NStructReal) + if(i >= ParticleType::NReal) { SENSEI_ERROR("rStruct index exceeds internal storage size"); return -1; @@ -81,12 +81,12 @@ int ParticleDataAdaptor::SetArra return -1; } - if(iStructs.size() <= NStructInt) + if(iStructs.size() <= ParticleType::NInt) { // check that no indices from the iStructs map exceed allowable value for(auto s : iStructs) { - if(s.second >= NStructInt) + if(s.second >= ParticleType::NInt) { SENSEI_ERROR("iStructs index exceeds internal storage size"); return -1; @@ -143,8 +143,8 @@ int ParticleDataAdaptor::SetArra return 0; } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::GetNumberOfMeshes(unsigned int &numMeshes) +template +int ParticleDataAdaptor::GetNumberOfMeshes(unsigned int &numMeshes) { numMeshes = 1; return 0; @@ -152,8 +152,8 @@ int ParticleDataAdaptor::GetNumb //----------------------------------------------------------------------------- #if SENSEI_VERSION_MAJOR < 3 -template -int ParticleDataAdaptor::GetMeshName( +template +int ParticleDataAdaptor::GetMeshName( unsigned int id, std::string &meshName) { meshName = m_particlesName; @@ -163,8 +163,8 @@ int ParticleDataAdaptor::GetMesh //----------------------------------------------------------------------------- #if SENSEI_VERSION_MAJOR < 3 -template -int ParticleDataAdaptor::GetMeshHasGhostNodes( +template +int ParticleDataAdaptor::GetMeshHasGhostNodes( const std::string &meshName, int &nLayers) { @@ -175,8 +175,8 @@ int ParticleDataAdaptor::GetMesh //----------------------------------------------------------------------------- #if SENSEI_VERSION_MAJOR < 3 -template -int ParticleDataAdaptor::GetMeshHasGhostCells( +template +int ParticleDataAdaptor::GetMeshHasGhostCells( const std::string &meshName, int &nLayers) { @@ -187,8 +187,8 @@ int ParticleDataAdaptor::GetMesh //----------------------------------------------------------------------------- #if SENSEI_VERSION_MAJOR < 3 -template -int ParticleDataAdaptor::GetNumberOfArrays( +template +int ParticleDataAdaptor::GetNumberOfArrays( const std::string &meshName, int association, unsigned int &numberOfArrays) @@ -206,8 +206,8 @@ int ParticleDataAdaptor::GetNumb #endif //----------------------------------------------------------------------------- #if SENSEI_VERSION_MAJOR < 3 -template -int ParticleDataAdaptor::GetArrayName( +template +int ParticleDataAdaptor::GetArrayName( const std::string &meshName, int association, unsigned int index, @@ -249,8 +249,8 @@ int ParticleDataAdaptor::GetArra #endif //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::GetMesh( +template +int ParticleDataAdaptor::GetMesh( const std::string &meshName, bool structureOnly, svtkDataObject *&mesh) @@ -284,8 +284,8 @@ int ParticleDataAdaptor::GetMesh } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::AddGhostNodesArray( +template +int ParticleDataAdaptor::AddGhostNodesArray( svtkDataObject*, const std::string &meshName) { @@ -298,8 +298,8 @@ int ParticleDataAdaptor::AddGhos } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::AddGhostCellsArray( +template +int ParticleDataAdaptor::AddGhostCellsArray( svtkDataObject*, const std::string &meshName) { @@ -312,8 +312,8 @@ int ParticleDataAdaptor::AddGhos } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::AddArray( +template +int ParticleDataAdaptor::AddArray( svtkDataObject* mesh, const std::string &meshName, int association, @@ -356,8 +356,8 @@ int ParticleDataAdaptor::AddArra } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::ReleaseData() +template +int ParticleDataAdaptor::ReleaseData() { this->m_particles = nullptr; return 0; @@ -365,8 +365,8 @@ int ParticleDataAdaptor::Release //----------------------------------------------------------------------------- #if SENSEI_VERSION_MAJOR >= 3 -template -int ParticleDataAdaptor::GetMeshMetadata( +template +int ParticleDataAdaptor::GetMeshMetadata( unsigned int id, sensei::MeshMetadataPtr &metadata) { @@ -645,8 +645,8 @@ int ParticleDataAdaptor::GetMesh #endif //----------------------------------------------------------------------------- -template -svtkPolyData* ParticleDataAdaptor::BuildParticles() +template +svtkPolyData* ParticleDataAdaptor::BuildParticles() { // return particle data pd svtkPolyData* pd = svtkPolyData::New(); @@ -682,21 +682,21 @@ svtkPolyData* ParticleDataAdaptor; + using MyParIter = ParIter_impl; for (MyParIter pti(*this->m_particles, lev); pti.isValid(); ++pti) { - auto& aos = pti.GetArrayOfStructs(); + auto ptd = pti.GetParticleTile().getParticleTileData(); auto numReal = pti.numParticles(); for (long long i = 0; i < numReal; ++i) { - const auto &part = aos[i]; + const auto& part = make_particle{}(ptd, i); if (part.id() > 0) { // add a vertex type cell vertex->InsertNextCell(1); vertex->InsertCellPoint(ptId); - // copy the partilce coordinates + // copy the particle coordinates #if (AMREX_SPACEDIM == 1) pCoords[0] = part.pos(0); pCoords[1] = amrex_particle_real(0); @@ -729,8 +729,8 @@ svtkPolyData* ParticleDataAdaptor -int ParticleDataAdaptor::AddParticlesIDArray( +template +int ParticleDataAdaptor::AddParticlesIDArray( svtkDataObject* mesh) { auto svtk_particles = dynamic_cast(mesh); @@ -747,17 +747,17 @@ int ParticleDataAdaptor::AddPart int *partIds = idArray->GetPointer(0); // loop over particles and append their cpu value to the list - using MyParIter = ParIter; + using MyParIter = ParIter_impl; long ptId = 0; for (int level = 0; level < particles.size(); ++level) { for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) { - auto& aos = pti.GetArrayOfStructs(); + auto ptd = pti.GetParticleTile().getParticleTileData(); auto numReal = pti.numParticles(); for (long long i = 0; i < numReal; ++i) { - const auto &part = aos[i]; + const auto &part = make_particle{}(ptd, i); if (part.id() > 0) { partIds[i] = part.id(); @@ -774,8 +774,8 @@ int ParticleDataAdaptor::AddPart } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::AddParticlesCPUArray( +template +int ParticleDataAdaptor::AddParticlesCPUArray( svtkDataObject* mesh) { auto svtk_particles = dynamic_cast(mesh); @@ -792,16 +792,16 @@ int ParticleDataAdaptor::AddPart int* partCpu = cpuArray->GetPointer(0); // loop over particles and append their cpu value to the list - using MyParIter = ParIter; + using MyParIter = ParIter_impl; for (int level = 0; level < particles.size(); ++level) { for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) { - auto& aos = pti.GetArrayOfStructs(); + auto ptd = pti.GetParticleTile().getParticleTileData(); auto numReal = pti.numParticles(); for (long long i = 0; i < numReal; ++i) { - const auto &part = aos[i]; + const auto &part = make_particle{}(ptd, i); if (part.id() > 0) { partCpu[i] = part.cpu(); @@ -818,8 +818,8 @@ int ParticleDataAdaptor::AddPart } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::AddParticlesSOARealArray( +template +int ParticleDataAdaptor::AddParticlesSOARealArray( const std::string &arrayName, svtkDataObject* mesh) { @@ -863,7 +863,7 @@ int ParticleDataAdaptor::AddPart double* pData = data->GetPointer(0); #endif - using MyParIter = ParIter; + using MyParIter = ParIter_impl; for (int level = 0; level < this->m_particles->numLevels(); ++level) { for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) @@ -905,8 +905,8 @@ int ParticleDataAdaptor::AddPart } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::AddParticlesSOAIntArray( +template +int ParticleDataAdaptor::AddParticlesSOAIntArray( const std::string &arrayName, svtkDataObject* mesh) { @@ -938,7 +938,7 @@ int ParticleDataAdaptor::AddPart int* pData = data->GetPointer(0); // fill array - using MyParIter = ParIter; + using MyParIter = ParIter_impl; for (int level = 0; level< this->m_particles->numLevels(); level++) { for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) @@ -976,8 +976,8 @@ int ParticleDataAdaptor::AddPart } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::AddParticlesAOSRealArray( +template +int ParticleDataAdaptor::AddParticlesAOSRealArray( const std::string &arrayName, svtkDataObject* mesh) { @@ -1000,7 +1000,7 @@ int ParticleDataAdaptor::AddPart // check that the indices are within the templated storage spec for (auto i : indices) { - if (i >= NStructReal) + if (i >= ParticleType::NReal) { SENSEI_ERROR("Index out of bounds for real AOS named \"" << arrayName << "\""); return -1; @@ -1024,8 +1024,10 @@ int ParticleDataAdaptor::AddPart double *pData = data->GetPointer(0); #endif + if constexpr(!ParticleType::is_soa_particle) { + // copy the data from each level - using MyParIter = ParIter; + using MyParIter = ParIter_impl; for (int level = 0; levelm_particles, level); pti.isValid(); ++pti) @@ -1058,12 +1060,14 @@ int ParticleDataAdaptor::AddPart auto block = dynamic_cast(blocks->GetBlock(rank)); block->GetPointData()->AddArray(data); + } + return 0; } //----------------------------------------------------------------------------- -template -int ParticleDataAdaptor::AddParticlesAOSIntArray( +template +int ParticleDataAdaptor::AddParticlesAOSIntArray( const std::string &arrayName, svtkDataObject* mesh) { @@ -1084,12 +1088,14 @@ int ParticleDataAdaptor::AddPart int index = ait->second; // check that the index are within the templated storage spec - if(index >= NStructInt) + if(index >= ParticleType::NInt) { SENSEI_ERROR("Index out of bounds for int AOS named \"" << arrayName << "\""); return -1; } + if constexpr(!ParticleType::is_soa_particle) { + // allocate svtkArray svtkNew data; data->SetName(arrayName.c_str()); @@ -1097,7 +1103,7 @@ int ParticleDataAdaptor::AddPart data->SetNumberOfValues(nptsOnProc); int* pData = data->GetPointer(0); - using MyParIter = ParIter; + using MyParIter = ParIter_impl; for (int level = 0; levelm_particles, level); pti.isValid(); ++pti) @@ -1126,6 +1132,7 @@ int ParticleDataAdaptor::AddPart auto block = dynamic_cast(blocks->GetBlock(rank)); block->GetPointData()->AddArray(data); + } return 0; } diff --git a/Src/Extern/SENSEI/AMReX_ParticleInSituBridge.H b/Src/Extern/SENSEI/AMReX_ParticleInSituBridge.H index 5177b44db58..a07e433c856 100644 --- a/Src/Extern/SENSEI/AMReX_ParticleInSituBridge.H +++ b/Src/Extern/SENSEI/AMReX_ParticleInSituBridge.H @@ -28,22 +28,22 @@ public: // invoke the in situ analysis with data from an AmrMesh // instance. - template + template int update( const double time, const int time_step, - amrex::ParticleContainer *particles, + amrex::ParticleContainer_impl *particles, const std::map> & particles_rStructs = {}, const std::map & particles_iStructs = {}, const std::map> & particles_rArrays = {}, const std::map & particles_iArrays = {}); }; -template +template int ParticleInSituBridge::update( const double time, const int time_step, - amrex::ParticleContainer *particles, + amrex::ParticleContainer_impl *particles, const std::map> & particles_rStructs, const std::map & particles_iStructs, const std::map> & particles_rArrays, @@ -64,8 +64,8 @@ int ParticleInSituBridge::update( return -1; } - ParticleDataAdaptor *data_adaptor - = ParticleDataAdaptor::New(); + ParticleDataAdaptor *data_adaptor + = ParticleDataAdaptor::New(); if (comm != MPI_COMM_NULL) data_adaptor->SetCommunicator(comm); @@ -75,7 +75,7 @@ int ParticleInSituBridge::update( data_adaptor->SetDataTime(time); data_adaptor->SetDataTimeStep(time_step); - ret = analysis_adaptor->Execute(data_adaptor) ? 0 : -1; + ret = analysis_adaptor->Execute(data_adaptor, nullptr) ? 0 : -1; data_adaptor->ReleaseData(); data_adaptor->Delete(); diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 3cb752bfa88..73a5dfe69f8 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -120,7 +120,7 @@ else() set( AMREX_TESTS_SUBDIRS AsyncOut MultiBlock Reinit Amr CLZ Parser Parser2 CTOParFor RoundoffDomain) if (AMReX_PARTICLES) - list(APPEND AMREX_TESTS_SUBDIRS Particles) + list(APPEND AMREX_TESTS_SUBDIRS Particles) endif () if (AMReX_EB) diff --git a/Tests/Particles/SENSEI_Insitu_SOA/CMakeLists.txt b/Tests/Particles/SENSEI_Insitu_SOA/CMakeLists.txt new file mode 100644 index 00000000000..fc4ec2a3af2 --- /dev/null +++ b/Tests/Particles/SENSEI_Insitu_SOA/CMakeLists.txt @@ -0,0 +1,13 @@ +if ( NOT AMReX_SENSEI ) + return () +endif () + +foreach(D IN LISTS AMReX_SPACEDIM) + set(_sources main.cpp) + set(_input_files inputs.rt ) + + setup_test(${D} _sources _input_files NTASKS 2) + + unset(_sources) + unset(_input_files) +endforeach() diff --git a/Tests/Particles/SENSEI_Insitu_SOA/GNUmakefile b/Tests/Particles/SENSEI_Insitu_SOA/GNUmakefile new file mode 100644 index 00000000000..653290723f3 --- /dev/null +++ b/Tests/Particles/SENSEI_Insitu_SOA/GNUmakefile @@ -0,0 +1,24 @@ +AMREX_HOME = ../../../ + +DEBUG = FALSE + +DIM = 3 + +COMP = gcc + +USE_MPI = TRUE +USE_OMP = FALSE +USE_CUDA = FALSE + +TINY_PROFILE = TRUE +USE_PARTICLES = TRUE +USE_SENSEI_INSITU = TRUE + +include $(AMREX_HOME)/Tools/GNUMake/Make.defs + +include ./Make.package +include $(AMREX_HOME)/Src/Base/Make.package +include $(AMREX_HOME)/Src/Particle/Make.package +include $(AMREX_HOME)/Src/Extern/SENSEI/Make.package + +include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Tests/Particles/SENSEI_Insitu_SOA/inputs.rt b/Tests/Particles/SENSEI_Insitu_SOA/inputs.rt new file mode 100644 index 00000000000..98b6748591e --- /dev/null +++ b/Tests/Particles/SENSEI_Insitu_SOA/inputs.rt @@ -0,0 +1,10 @@ +insitu.size = (32, 64, 64) +insitu.max_grid_size = 32 +insitu.is_periodic = 1 +insitu.num_ppc = 1 +insitu.nlevs = 1 + +insitu.num_runtime_real = 0 +insitu.num_runtime_int = 0 + +particles.do_tiling=1 diff --git a/Tests/Particles/SENSEI_Insitu_SOA/main.cpp b/Tests/Particles/SENSEI_Insitu_SOA/main.cpp new file mode 100644 index 00000000000..8f9e8f7f7e6 --- /dev/null +++ b/Tests/Particles/SENSEI_Insitu_SOA/main.cpp @@ -0,0 +1,274 @@ +#include +#include +#include +#include + +#if !defined(AMREX_PARTICLES) || !defined(AMREX_USE_SENSEI_INSITU) +#error Incompatible AMReX library configuration! This tutorial requires AMREX_PARTICLES and AMREX_USE_SENSEI_INSITU +#endif +#include + +using namespace amrex; + +static constexpr int NR = 7; +static constexpr int NI = 4; + +int num_runtime_real = 0; +int num_runtime_int = 0; + +bool remove_negative = true; + +void get_position_unit_cell (Real* r, const IntVect& nppc, int i_part) +{ + int nx = nppc[0]; +#if AMREX_SPACEDIM > 1 + int ny = nppc[1]; +#else + int ny = 1; +#endif +#if AMREX_SPACEDIM > 2 + int nz = nppc[2]; +#else + int nz = 1; +#endif + + int ix_part = i_part/(ny * nz); + int iy_part = (i_part % (ny * nz)) % ny; + int iz_part = (i_part % (ny * nz)) / ny; + + r[0] = (0.5+ix_part)/nx; + r[1] = (0.5+iy_part)/ny; + r[2] = (0.5+iz_part)/nz; +} + +class TestParticleContainer + : public amrex::ParticleContainerPureSoA +{ + +public: + + TestParticleContainer (const Vector & a_geom, + const Vector & a_dmap, + const Vector & a_ba, + const Vector & a_rr) + : amrex::ParticleContainerPureSoA(a_geom, a_dmap, a_ba, a_rr) + { + for (int i = 0; i < num_runtime_real; ++i) + { + AddRealComp(true); + } + for (int i = 0; i < num_runtime_int; ++i) + { + AddIntComp(true); + } + } + + void InitParticles (const amrex::IntVect& a_num_particles_per_cell) + { + BL_PROFILE("InitParticles"); + + const int lev = 0; // only add particles on level 0 + const Real* dx = Geom(lev).CellSize(); + const Real* plo = Geom(lev).ProbLo(); + + const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], + *a_num_particles_per_cell[1], + *a_num_particles_per_cell[2]); + + for(MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) + { + const Box& tile_box = mfi.tilebox(); + + std::array, NR> host_real; + std::array, NI> host_int; + + std::vector > host_runtime_real(NumRuntimeRealComps()); + std::vector > host_runtime_int(NumRuntimeIntComps()); + + for (IntVect iv = tile_box.smallEnd(); iv <= tile_box.bigEnd(); tile_box.next(iv)) + { + for (int i_part=0; i_part(id)); + host_int[1].push_back(ParallelDescriptor::MyProc()); + host_real[0].push_back(static_cast (plo[0] + (iv[0] + r[0])*dx[0])); +#if AMREX_SPACEDIM > 1 + host_real[1].push_back(static_cast (plo[1] + (iv[1] + r[1])*dx[1])); +#endif +#if AMREX_SPACEDIM > 2 + host_real[2].push_back(static_cast (plo[2] + (iv[2] + r[2])*dx[2])); +#endif + + for (int i = AMREX_SPACEDIM; i < NR; ++i) + host_real[i].push_back(static_cast(id)); + for (int i = 2; i < NI; ++i) + host_int[i].push_back(static_cast(id)); + for (int i = 0; i < NumRuntimeRealComps(); ++i) + host_runtime_real[i].push_back(static_cast(id)); + for (int i = 0; i < NumRuntimeIntComps(); ++i) + host_runtime_int[i].push_back(static_cast(id)); + } + } + + auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex()); + auto old_size = particle_tile.GetArrayOfStructs().size(); + auto new_size = old_size + host_real[0].size(); + particle_tile.resize(new_size); + + auto& soa = particle_tile.GetStructOfArrays(); + for (int i = 0; i < NR; ++i) + { + Gpu::copyAsync(Gpu::hostToDevice, + host_real[i].begin(), + host_real[i].end(), + soa.GetRealData(i).begin() + old_size); + } + + for (int i = 0; i < NI; ++i) + { + Gpu::copyAsync(Gpu::hostToDevice, + host_int[i].begin(), + host_int[i].end(), + soa.GetIntData(i).begin() + old_size); + } + for (int i = 0; i < NumRuntimeRealComps(); ++i) + { + Gpu::copyAsync(Gpu::hostToDevice, + host_runtime_real[i].begin(), + host_runtime_real[i].end(), + soa.GetRealData(NR+i).begin() + old_size); + } + + for (int i = 0; i < NumRuntimeIntComps(); ++i) + { + Gpu::copyAsync(Gpu::hostToDevice, + host_runtime_int[i].begin(), + host_runtime_int[i].end(), + soa.GetIntData(NI+i).begin() + old_size); + } + + Gpu::streamSynchronize(); + } + + Redistribute(); + } +}; + +struct TestParams +{ + IntVect size; + int max_grid_size; + int num_ppc; + int is_periodic; + int nlevs; +}; + +void testRedistribute(); + +int main (int argc, char* argv[]) +{ + amrex::Initialize(argc,argv); + + amrex::Print() << "Running redistribute test \n"; + testRedistribute(); + + amrex::Finalize(); +} + +void get_test_params(TestParams& params, const std::string& prefix) +{ + ParmParse pp(prefix); + pp.get("size", params.size); + pp.get("max_grid_size", params.max_grid_size); + pp.get("num_ppc", params.num_ppc); + pp.get("is_periodic", params.is_periodic); + pp.get("nlevs", params.nlevs); + pp.query("num_runtime_real", num_runtime_real); + pp.query("num_runtime_int", num_runtime_int); +} + +void testRedistribute () +{ + BL_PROFILE("testSENSEI_insitu"); + TestParams params; + get_test_params(params, "insitu"); + + int is_per[BL_SPACEDIM]; + for (int & d : is_per) + d = params.is_periodic; + + Vector rr(params.nlevs-1); + for (int lev = 1; lev < params.nlevs; lev++) + rr[lev-1] = IntVect(AMREX_D_DECL(2,2,2)); + + RealBox real_box; + for (int n = 0; n < BL_SPACEDIM; n++) + { + real_box.setLo(n, 0.0); + real_box.setHi(n, params.size[n]); + } + + IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); + IntVect domain_hi(AMREX_D_DECL(params.size[0]-1,params.size[1]-1,params.size[2]-1)); + const Box base_domain(domain_lo, domain_hi); + + Vector geom(params.nlevs); + geom[0].define(base_domain, &real_box, CoordSys::cartesian, is_per); + for (int lev = 1; lev < params.nlevs; lev++) { + geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), + &real_box, CoordSys::cartesian, is_per); + } + + Vector ba(params.nlevs); + Vector dm(params.nlevs); + auto lo = IntVect(AMREX_D_DECL(0, 0, 0)); + IntVect size = params.size; + for (int lev = 0; lev < params.nlevs; ++lev) + { + ba[lev].define(Box(lo, lo+params.size-1)); + ba[lev].maxSize(params.max_grid_size); + dm[lev].define(ba[lev]); + lo += size/2; + size *= 2; + } + + TestParticleContainer pc(geom, dm, ba, rr); + + int npc = params.num_ppc; + auto nppc = IntVect(AMREX_D_DECL(npc, npc, npc)); + + amrex::Print() << "About to initialize particles \n"; + + pc.InitParticles(nppc); + + auto *insitu_bridge = new ParticleInSituBridge; + + if (insitu_bridge->initialize()) { + amrex::ErrorStream() << "Failed to initialize the in situ bridge." << std::endl; + amrex::Abort(); + } + + // define specifications for fields on particles + std::map> rStructs; + std::map iStructs; + std::map> rArrays; + std::map iArrays; + + if (insitu_bridge->update(0.0, 0, &pc, rStructs)) { + amrex::ErrorStream() << "Failed to update the in situ bridge." << std::endl; + amrex::Abort(); + } + + if (insitu_bridge->finalize()) { + amrex::ErrorStream() << "Failed to finalize the in situ bridge." << std::endl; + } + + delete insitu_bridge; + + // the way this test is set up, if we make it here we pass + amrex::Print() << "pass \n"; +}