Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PureSoA: Disable AoS Access #3290

Merged
merged 17 commits into from
Sep 22, 2023
Merged

PureSoA: Disable AoS Access #3290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
ef535be
More SoA Fixes
AlexanderSinn May 2, 2023
45e458e
More PureSoA FixesMore SoA Fixes
AlexanderSinn May 2, 2023
69215fc
[FIXME] Comment Unusued correctCellVectors Hook
ax3l May 9, 2023
d330a95
test: avoid AoS access
ax3l May 9, 2023
9367bc5
Merge development, fix empty() and likely mess up git
AlexanderSinn May 9, 2023
566e4d4
Merge branch 'development' into more_soa_fixes
AlexanderSinn May 15, 2023
cfe90df
Merge branch 'development' into more_soa_fixes
AlexanderSinn May 24, 2023
ad143a1
Merge branch 'development' into more_soa_fixes
AlexanderSinn Jun 16, 2023
8e190a1
Merge branch 'development' into more_soa_fixes
AlexanderSinn Jun 19, 2023
7cb63ed
Merge branch 'development' into more_soa_fixes
AlexanderSinn Jul 17, 2023
9196be0
Merge branch 'development' into more_soa_fixes
AlexanderSinn Jul 22, 2023
50b37a9
Merge branch 'development' into more_soa_fixes
AlexanderSinn Sep 20, 2023
e952a7a
fix correctCellVectors for PureSoA
AlexanderSinn Sep 20, 2023
d48c90d
fix issue
AlexanderSinn Sep 20, 2023
eaffb62
try fix SENSEI
AlexanderSinn Sep 20, 2023
072cc46
fix SENSEI
AlexanderSinn Sep 20, 2023
cfdb37f
Fix ParticleContainerI
ax3l Sep 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -581,14 +581,14 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetMeshMetadata(
for (const auto& kv : pmap)
{
// loop over particles in ParticleTile
const auto& aos = kv.second.GetArrayOfStructs();
const auto &parts = aos();
auto& particle_tile = kv.second;
auto ptd = particle_tile.getConstParticleTileData();

// visit only the "real" particles, skip the "neighbor" particles.
long long numReal = aos.numRealParticles();
long long numReal = particle_tile.numRealParticles();
for (long long i = 0; i < numReal; ++i)
{
const auto &part = parts[i];
const auto &part = ptd[i];
if (part.id() > 0)
{
#if (AMREX_SPACEDIM == 1)
Expand Down Expand Up @@ -869,7 +869,7 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddParticlesSOARea
for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti)
{
auto& particle_attributes = pti.GetStructOfArrays();
auto& aos = pti.GetArrayOfStructs();
auto ptd = pti.GetParticleTile().getParticleTileData();

auto numReal = pti.numParticles();
// shuffle from the AMReX component order
Expand All @@ -881,7 +881,7 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddParticlesSOARea

for (long long i = 0; i < numReal; ++i)
{
const auto &part = aos[i];
const auto &part = ptd[i];
if (part.id() > 0)
{
pData[i*nComps + j] = realData[i];
Expand Down Expand Up @@ -944,7 +944,7 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddParticlesSOAInt
for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti)
{
auto& particle_attributes = pti.GetStructOfArrays();
auto& aos = pti.GetArrayOfStructs();
auto ptd = pti.GetParticleTile().getParticleTileData();

auto numReal = pti.numParticles();
// shuffle from the AMReX component order
Expand All @@ -953,7 +953,7 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddParticlesSOAInt

for (long long i = 0; i < numReal; ++i)
{
const auto &part = aos[i];
const auto &part = ptd[i];
if (part.id() > 0)
{
pData[i] = intData[i];
Expand Down
18 changes: 10 additions & 8 deletions Src/Particle/AMReX_ParticleContainerI.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1653,8 +1653,8 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
if (npart != 0) {
Long last = npart - 1;
Long pindex = 0;
auto ptd = particle_tile->getParticleTileData();
while (pindex <= last) {
auto ptd = particle_tile->getParticleTileData();
ParticleType p(ptd,pindex);

if ((remove_negative == false) && (p.id() < 0)) {
Expand All @@ -1669,7 +1669,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
for (int comp = 0; comp < NumIntComps(); comp++) {
soa.GetIntData(comp)[pindex] = soa.GetIntData(comp)[last];
}
correctCellVectors(last, pindex, grid, aos[pindex]);
correctCellVectors(last, pindex, grid, ptd[pindex]);
--last;
continue;
}
Expand All @@ -1685,7 +1685,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
for (int comp = 0; comp < NumIntComps(); comp++) {
soa.GetIntData(comp)[pindex] = soa.GetIntData(comp)[last];
}
correctCellVectors(last, pindex, grid, aos[pindex]);
correctCellVectors(last, pindex, grid, ptd[pindex]);
--last;
continue;
}
Expand Down Expand Up @@ -1739,7 +1739,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
for (int comp = 0; comp < NumIntComps(); comp++) {
soa.GetIntData(comp)[pindex] = soa.GetIntData(comp)[last];
}
correctCellVectors(last, pindex, grid, aos[pindex]);
correctCellVectors(last, pindex, grid, ptd[pindex]);
--last;
continue;
}
Expand Down Expand Up @@ -2206,13 +2206,15 @@ RedistributeMPI (std::map<int, Vector<char> >& not_ours,
const auto& src_tile = kv.second;

auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)];
auto old_size = dst_tile.GetArrayOfStructs().size();
auto old_size = dst_tile.size();
auto new_size = old_size + src_tile.size();
dst_tile.resize(new_size);

Gpu::copyAsync(Gpu::hostToDevice,
src_tile.begin(), src_tile.end(),
dst_tile.GetArrayOfStructs().begin() + old_size);
if constexpr(! ParticleType::is_soa_particle) {
Gpu::copyAsync(Gpu::hostToDevice,
src_tile.begin(), src_tile.end(),
dst_tile.GetArrayOfStructs().begin() + old_size);
}

for (int i = 0; i < NumRealComps(); ++i) {
Gpu::copyAsync(Gpu::hostToDevice,
Expand Down
4 changes: 2 additions & 2 deletions Src/Particle/AMReX_ParticleInit.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1143,7 +1143,7 @@ InitRandom (Long icount,
const auto& src_tile = kv.second;

auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)];
auto old_size = dst_tile.GetArrayOfStructs().size();
auto old_size = dst_tile.size();
auto new_size = old_size;
if constexpr(!ParticleType::is_soa_particle)
{
Expand Down Expand Up @@ -1286,7 +1286,7 @@ InitRandom (Long icount,
const auto& src_tile = kv.second;

auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)];
auto old_size = dst_tile.GetArrayOfStructs().size();
auto old_size = dst_tile.size();
auto new_size = old_size;
if constexpr(!ParticleType::is_soa_particle)
{
Expand Down
11 changes: 10 additions & 1 deletion Src/Particle/AMReX_ParticleTile.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,12 @@ struct ConstParticleTileData
}
};

struct ThisParticleTileHasNoParticleVector {};

struct ThisParticleTileHasNoAoS {
using ParticleVector = ThisParticleTileHasNoParticleVector;
};

template <typename T_ParticleType, int NArrayReal, int NArrayInt,
template<class> class Allocator=DefaultAllocator>
struct ParticleTile
Expand All @@ -651,7 +657,10 @@ struct ParticleTile

using SuperParticleType = Particle<NStructReal + NArrayReal, NStructInt + NArrayInt>;

using AoS = ArrayOfStructs<ParticleType, Allocator>;
using AoS = typename std::conditional<
ParticleType::is_soa_particle,
ThisParticleTileHasNoAoS,
ArrayOfStructs<ParticleType, Allocator>>::type;
//using ParticleVector = typename AoS::ParticleVector;

using SoA = StructOfArrays<NArrayReal, NArrayInt, Allocator>;
Expand Down
2 changes: 1 addition & 1 deletion Tests/Particles/RedistributeSOA/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ class TestParticleContainer
}

auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex());
auto old_size = particle_tile.GetArrayOfStructs().size();
auto old_size = particle_tile.size();
auto new_size = old_size + host_real[0].size();
particle_tile.resize(new_size);

Expand Down
2 changes: 1 addition & 1 deletion Tests/Particles/SENSEI_Insitu_SOA/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ class TestParticleContainer
}

auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex());
auto old_size = particle_tile.GetArrayOfStructs().size();
auto old_size = particle_tile.size();
auto new_size = old_size + host_real[0].size();
particle_tile.resize(new_size);

Expand Down
Loading