Skip to content

Commit

Permalink
SENSEI: SoA Particle Support (#3349)
Browse files Browse the repository at this point in the history 
## Summary

Add support for pure SoA layouted particle containers for SENSEI.

## Additional background

Follow-up to #2878.

## Checklist

The proposed changes:
- [ ] fix a bug or incorrect behavior in AMReX
- [x] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX
users
- [ ] include documentation in the code and/or rst files, if appropriate

---------

Co-authored-by: Andrew Myers <[email protected]>
  • Loading branch information
@ax3l @atmyers
ax3l and atmyers authored Jul 28, 2023
1 parent 2628ec8 commit d9df80b
Show file tree
Hide file tree
Showing 14 changed files with 493 additions and 162 deletions.
8 changes: 4 additions & 4 deletions Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptor.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

namespace amrex
{
template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
template<typename ParticleType, int NArrayReal, int NArrayInt>
class AmrMeshParticleDataAdaptor : public sensei::DataAdaptor
{
public:
Expand All @@ -27,7 +27,7 @@ public:
AmrMesh *mesh,
const std::vector<amrex::Vector<amrex::MultiFab>*> &mesh_states,
const std::vector<std::vector<std::string>> &mesh_names,
amrex::ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> * particles,
amrex::ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt> * particles,
const std::map<std::string, std::vector<int>> & rStructs = {},
const std::map<std::string, int> & iStructs = {},
const std::map<std::string, std::vector<int>> & rArrays = {},
Expand Down Expand Up @@ -55,7 +55,7 @@ protected:
AmrMeshParticleDataAdaptor()
{
m_meshAdaptor = AmrMeshDataAdaptor::New();
m_particleAdaptor = ParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::New();
m_particleAdaptor = ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::New();
}

~AmrMeshParticleDataAdaptor()
Expand All @@ -65,7 +65,7 @@ protected:
}

private:
ParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>* m_particleAdaptor;
ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>* m_particleAdaptor;
AmrMeshDataAdaptor* m_meshAdaptor;

const std::string m_meshName = "mesh";
Expand Down
62 changes: 31 additions & 31 deletions Src/Extern/SENSEI/AMReX_AmrMeshParticleDataAdaptorI.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,22 @@
namespace amrex
{
//-----------------------------------------------------------------------------
template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>*
AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::New()
template <typename ParticleType, int NArrayReal, int NArrayInt>
AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>*
AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::New()
{
auto result = new AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>;
auto result = new AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>;
result->InitializeObjectBase();
return result;
}

//-----------------------------------------------------------------------------
template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::SetDataSource(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::SetDataSource(
AmrMesh *mesh,
const std::vector<amrex::Vector<amrex::MultiFab>*> &mesh_states,
const std::vector<std::vector<std::string>> &mesh_names,
amrex::ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> * particles,
amrex::ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt> * particles,
const std::map<std::string, std::vector<int>> & rStructs,
const std::map<std::string, int> & iStructs,
const std::map<std::string, std::vector<int>> & rArrays,
Expand All @@ -31,8 +31,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::

// SENSEI API
#if SENSEI_VERSION_MAJOR >= 3
template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::GetMeshMetadata(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetMeshMetadata(
unsigned int id,
sensei::MeshMetadataPtr &metadata)
{
Expand All @@ -49,8 +49,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
}

#else
template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::GetMeshName(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetMeshName(
unsigned int id,
std::string &meshName)
{
Expand All @@ -66,8 +66,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
return -1;
}

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::GetMeshHasGhostNodes(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetMeshHasGhostNodes(
const std::string &meshName, int &nLayers)
{
if(id == 0)
Expand All @@ -82,8 +82,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
return -1;
}

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::GetMeshHasGhostCells(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetMeshHasGhostCells(
const std::string &meshName, int &nLayers)
{
if(id == 0)
Expand All @@ -98,8 +98,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
return -1;
}

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::GetNumberOfArrays(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetNumberOfArrays(
const std::string &meshName,
int association,
unsigned int &numberOfArrays)
Expand All @@ -116,8 +116,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
return -1;
}

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::GetArrayName(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetArrayName(
const std::string &meshName,
int association,
unsigned int index,
Expand All @@ -136,16 +136,16 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
}
#endif

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::GetNumberOfMeshes(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetNumberOfMeshes(
unsigned int &numMeshes)
{
numMeshes = 2;
return 0;
}

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::GetMesh(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetMesh(
const std::string &meshName,
bool structureOnly,
svtkDataObject *&mesh)
Expand All @@ -162,8 +162,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
return -1;
}

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::AddGhostNodesArray(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddGhostNodesArray(
svtkDataObject* mesh,
const std::string &meshName)
{
Expand All @@ -179,8 +179,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
return -1;
}

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::AddGhostCellsArray(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddGhostCellsArray(
svtkDataObject* mesh,
const std::string &meshName)
{
Expand All @@ -196,8 +196,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
return -1;
}

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::AddArray(
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddArray(
svtkDataObject* mesh,
const std::string &meshName,
int association,
Expand All @@ -215,8 +215,8 @@ int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::
return -1;
}

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::ReleaseData()
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::ReleaseData()
{
this->m_particleAdaptor->ReleaseData();
this->m_meshAdaptor->ReleaseData();
Expand Down
12 changes: 6 additions & 6 deletions Src/Extern/SENSEI/AMReX_AmrMeshParticleInSituBridge.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,26 +33,26 @@ public:

// invoke the in situ analysis with data from an AmrMesh and ParticleContainer
// instance.
template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
template<typename ParticleType, int NArrayReal, int NArrayInt>
int update(
long step, double time,
amrex::AmrMesh *mesh,
const std::vector<amrex::Vector<amrex::MultiFab>*> &mesh_states,
const std::vector<std::vector<std::string>> &mesh_names,
amrex::ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> * particles,
amrex::ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt> * particles,
const std::map<std::string, std::vector<int>> & particles_rStructs = {},
const std::map<std::string, int> & particles_iStructs = {},
const std::map<std::string, std::vector<int>> & particles_rArrays = {},
const std::map<std::string, int> & particles_iArrays = {});
};

template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
template<typename ParticleType, int NArrayReal, int NArrayInt>
int AmrMeshParticleInSituBridge::update(
long step, double time,
amrex::AmrMesh *mesh,
const std::vector<amrex::Vector<amrex::MultiFab>*> &mesh_states,
const std::vector<std::vector<std::string>> &mesh_names,
amrex::ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> * particles,
amrex::ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt> * particles,
const std::map<std::string, std::vector<int>> & particles_rStructs,
const std::map<std::string, int> & particles_iStructs,
const std::map<std::string, std::vector<int>> & particles_rArrays,
Expand All @@ -73,8 +73,8 @@ int AmrMeshParticleInSituBridge::update(
return -1;
}

amrex::AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt> *data_adaptor
= amrex::AmrMeshParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::New();
amrex::AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt> *data_adaptor
= amrex::AmrMeshParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::New();

if (comm != MPI_COMM_NULL)
data_adaptor->SetCommunicator(comm);
Expand Down
8 changes: 4 additions & 4 deletions Src/Extern/SENSEI/AMReX_AmrParticleDataAdaptor.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

namespace amrex
{
template<int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
template<typename ParticleType, int NArrayReal, int NArrayInt>
class AmrParticleDataAdaptor : public sensei::DataAdaptor
{
public:
Expand All @@ -25,7 +25,7 @@ public:

int SetDataSource(
Amr *amr,
amrex::ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> * particles,
amrex::ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt> * particles,
const std::map<std::string, std::vector<int>> & rStructs = {},
const std::map<std::string, int> & iStructs = {},
const std::map<std::string, std::vector<int>> & rArrays = {},
Expand Down Expand Up @@ -53,7 +53,7 @@ protected:
AmrParticleDataAdaptor()
{
m_meshAdaptor = AmrDataAdaptor::New();
m_particleAdaptor = ParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>::New();
m_particleAdaptor = ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::New();
}

~AmrParticleDataAdaptor()
Expand All @@ -63,7 +63,7 @@ protected:
}

private:
ParticleDataAdaptor<NStructReal, NStructInt, NArrayReal, NArrayInt>* m_particleAdaptor;
ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>* m_particleAdaptor;
AmrDataAdaptor* m_meshAdaptor;

const std::string m_meshName = "mesh";
Expand Down
Loading

0 comments on commit d9df80b

Please sign in to comment.