diff --git a/Src/Base/AMReX_PODVector.H b/Src/Base/AMReX_PODVector.H index e3e631b6b93..6980f32f1ee 100644 --- a/Src/Base/AMReX_PODVector.H +++ b/Src/Base/AMReX_PODVector.H @@ -262,32 +262,40 @@ namespace amrex explicit PODVector (size_type a_size) noexcept : m_data(nullptr), m_size(a_size) { - AllocateBuffer(GetNewCapacity(a_size)); + if (a_size != 0) { + AllocateBuffer(GetNewCapacity(a_size)); + } } PODVector (size_type a_size, const value_type& a_value, const allocator_type& a_allocator = Allocator()) noexcept : Allocator(a_allocator), m_data(nullptr), m_size(a_size) { - AllocateBuffer(GetNewCapacity(a_size)); - detail::uninitializedFillNImpl(m_data, a_size, a_value, *this); + if (a_size != 0) { + AllocateBuffer(GetNewCapacity(a_size)); + detail::uninitializedFillNImpl(m_data, a_size, a_value, *this); + } } PODVector (std::initializer_list a_initializer_list, const allocator_type& a_allocator = Allocator()) noexcept : Allocator(a_allocator), m_data(nullptr), m_size(a_initializer_list.size()) { - AllocateBuffer(GetNewCapacity(m_size)); - detail::initFromListImpl(m_data, a_initializer_list, *this); + if (a_initializer_list.size() != 0) { + AllocateBuffer(GetNewCapacity(m_size)); + detail::initFromListImpl(m_data, a_initializer_list, *this); + } } PODVector (const PODVector& a_vector) noexcept : Allocator(a_vector), m_data(nullptr), m_size(a_vector.size()) { using namespace detail; - AllocateBuffer(a_vector.capacity()); - auto r = memCopyImpl(m_data, a_vector.m_data, a_vector.size() * sizeof(T), *this); - if (r) { Gpu::streamSynchronize(); } + if (a_vector.size() != 0) { + AllocateBuffer(a_vector.capacity()); + auto r = memCopyImpl(m_data, a_vector.m_data, a_vector.size() * sizeof(T), *this); + if (r) { Gpu::streamSynchronize(); } + } } PODVector (PODVector&& a_vector) noexcept diff --git a/Src/F_Interfaces/Particle/AMReX_particlecontainer_fi.cpp b/Src/F_Interfaces/Particle/AMReX_particlecontainer_fi.cpp index 3e80eeb86ff..0f996a7b88a 100644 --- a/Src/F_Interfaces/Particle/AMReX_particlecontainer_fi.cpp +++ b/Src/F_Interfaces/Particle/AMReX_particlecontainer_fi.cpp @@ -69,7 +69,8 @@ extern "C" { } void amrex_fi_get_particles_mfi(FParticleContainer* particlecontainer, - int lev, MFIter* mfi, ParticleReal*& dp, Long& np) + int lev, MFIter* mfi, + FParticleContainer::ParticleType*& dp, Long& np) { const int grid = mfi->index(); const int tile = mfi->LocalTileIndex(); @@ -116,7 +117,8 @@ extern "C" { } void amrex_fi_get_particles_i(FParticleContainer* particlecontainer, - int lev, int grid, int tile, ParticleReal*& dp, Long& np) + int lev, int grid, int tile, + FParticleContainer::ParticleType*& dp, Long& np) { auto& particle_level = particlecontainer->GetParticles(lev); auto search = particle_level.find(std::make_pair(grid, tile)); diff --git a/Src/Particle/AMReX_ArrayOfStructs.H b/Src/Particle/AMReX_ArrayOfStructs.H index 43c84bfd014..121ea6c7dd8 100644 --- a/Src/Particle/AMReX_ArrayOfStructs.H +++ b/Src/Particle/AMReX_ArrayOfStructs.H @@ -62,15 +62,15 @@ public: resize(nrp + num_neighbors); } - [[nodiscard]] int getNumNeighbors () { return m_num_neighbor_particles; } + [[nodiscard]] int getNumNeighbors () const { return m_num_neighbor_particles; } [[nodiscard]] bool empty () const { return m_data.empty(); } - [[nodiscard]] const RealType* data () const { return &(m_data[0].m_pos[0]); } - [[nodiscard]] RealType* data () { return &(m_data[0].m_pos[0]); } + [[nodiscard]] const ParticleType* data () const { return m_data.data(); } + [[nodiscard]] ParticleType* data () { return m_data.data(); } - [[nodiscard]] const RealType* dataPtr () const { return data(); } - [[nodiscard]] RealType* dataPtr () { return data(); } + [[nodiscard]] const ParticleType* dataPtr () const { return data(); } + [[nodiscard]] ParticleType* dataPtr () { return data(); } [[nodiscard]] std::pair dataShape () const { return std::make_pair(SizeInReal, static_cast(m_data.size())); diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 50bbdeab8b3..bf0fd38cd26 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -1090,9 +1090,8 @@ ParticleContainer_impl ::ReorderParticles (int lev, const MFIter& mfi, const index_type* permutations) { auto& ptile = ParticlesAt(lev, mfi); - auto& aos = ptile.GetArrayOfStructs(); - const size_t np = aos.numParticles(); - const size_t np_total = np + aos.numNeighborParticles(); + const size_t np = ptile.numParticles(); + const size_t np_total = np + ptile.numNeighborParticles(); if (memEfficientSort) { if constexpr(!ParticleType::is_soa_particle) { diff --git a/Src/Particle/AMReX_ParticleTile.H b/Src/Particle/AMReX_ParticleTile.H index 7a8fd4cd2e2..9f1fe712c64 100644 --- a/Src/Particle/AMReX_ParticleTile.H +++ b/Src/Particle/AMReX_ParticleTile.H @@ -51,47 +51,28 @@ struct ParticleTileData ParticleReal* AMREX_RESTRICT * AMREX_RESTRICT m_runtime_rdata; int* AMREX_RESTRICT * AMREX_RESTRICT m_runtime_idata; - // AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - // ParticleReal pos (const int dir, const int index) const & - // { - // if constexpr(!ParticleType::is_soa_particle) { - // return this->m_aos[index].pos(dir); - // } else { - // return this->m_rdata[dir][index]; - // } - // } - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE ParticleReal& pos (const int dir, const int index) const & { if constexpr(!ParticleType::is_soa_particle) { return this->m_aos[index].pos(dir); - } - return this->m_rdata[dir][index]; - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto id (const int index) const & - { - if constexpr(!ParticleType::is_soa_particle) { - return this->m_aos[index].id(); } else { - return this->m_idata[0][index]; + return this->m_rdata[dir][index]; } } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto& id (const int index) & + decltype(auto) id (const int index) const & { if constexpr(!ParticleType::is_soa_particle) { - return this->m_aos[index].id(); + return this->m_aos[index].id(); } else { return this->m_idata[0][index]; } } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto cpu (const int index) const & + decltype(auto) cpu (const int index) const & { if constexpr(!ParticleType::is_soa_particle) { return this->m_aos[index].cpu(); @@ -101,23 +82,13 @@ struct ParticleTileData } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto& cpu (const int index) & - { - if constexpr(!ParticleType::is_soa_particle) { - return this->m_aos[index].cpu(); - } else { - return this->m_idata[1][index]; - } - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto * rdata (const int attribute_index) const + ParticleReal * rdata (const int attribute_index) const { return this->m_rdata[attribute_index]; } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto * idata (const int attribute_index) const + int * idata (const int attribute_index) const { return this->m_idata[attribute_index]; } @@ -138,8 +109,10 @@ struct ParticleTileData { AMREX_ASSERT(src_index < m_size); auto dst = buffer + dst_offset; - memcpy(dst, m_aos + src_index, sizeof(ParticleType)); - dst += sizeof(ParticleType); + if constexpr (!ParticleType::is_soa_particle) { + memcpy(dst, m_aos + src_index, sizeof(ParticleType)); + dst += sizeof(ParticleType); + } int array_start_index = AMREX_SPACEDIM + NStructReal; for (int i = 0; i < NAR; ++i) { @@ -184,8 +157,10 @@ struct ParticleTileData { AMREX_ASSERT(dst_index < m_size); auto src = buffer + src_offset; - memcpy(m_aos + dst_index, src, sizeof(ParticleType)); - src += sizeof(ParticleType); + if constexpr (!ParticleType::is_soa_particle) { + memcpy(m_aos + dst_index, src, sizeof(ParticleType)); + src += sizeof(ParticleType); + } int array_start_index = AMREX_SPACEDIM + NStructReal; for (int i = 0; i < NAR; ++i) { @@ -251,21 +226,14 @@ struct ParticleTileData { AMREX_ASSERT(index < m_size); SuperParticleType sp; - for (int i = 0; i < AMREX_SPACEDIM; ++i) - sp.pos(i) = m_rdata[i][index]; - for (int i = 0; i < NStructReal; ++i) - sp.rdata(i) = m_rdata[i][index]; for (int i = 0; i < NAR; ++i) - sp.rdata(NStructReal+i) = m_rdata[i][index]; - sp.id() = m_idata[0][index]; - sp.cpu() = m_idata[1][index]; - for (int i = 0; i < NStructInt; ++i) - sp.idata(i) = m_idata[i][index]; + sp.rdata(i) = m_rdata[i][index]; for (int i = 0; i < NAI; ++i) - sp.idata(NStructInt+i) = m_idata[i][index]; + sp.idata(i) = m_idata[i][index]; return sp; } + template ::type = 0> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void setSuperParticle (const SuperParticleType& sp, int index) const noexcept { @@ -282,6 +250,16 @@ struct ParticleTileData for (int i = 0; i < NAI; ++i) m_idata[i][index] = sp.idata(NStructInt+i); } + + template ::type = 0> + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void setSuperParticle (const SuperParticleType& sp, int index) const noexcept + { + for (int i = 0; i < NAR; ++i) + m_rdata[i][index] = sp.rdata(i); + for (int i = 0; i < NAI; ++i) + m_idata[i][index] = sp.idata(i); + } }; // SOA Particle Structure @@ -484,7 +462,17 @@ struct ConstParticleTileData GpuArray m_idata; AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto id (const int index) const & + const ParticleReal& pos (const int dir, const int index) const & + { + if constexpr(!ParticleType::is_soa_particle) { + return this->m_aos[index].pos(dir); + } else { + return this->m_rdata[dir][index]; + } + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + decltype(auto) id (const int index) const & { if constexpr(!ParticleType::is_soa_particle) { return this->m_aos[index].id(); @@ -494,13 +482,23 @@ struct ConstParticleTileData } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto const* rdata (const int attribute_index) const + decltype(auto) cpu (const int index) const & + { + if constexpr(!ParticleType::is_soa_particle) { + return this->m_aos[index].cpu(); + } else { + return this->m_idata[1][index]; + } + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + const ParticleReal * rdata (const int attribute_index) const { return this->m_rdata[attribute_index]; } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto const* idata (const int attribute_index) const + const int * idata (const int attribute_index) const { return this->m_idata[attribute_index]; } @@ -526,8 +524,10 @@ struct ConstParticleTileData { AMREX_ASSERT(src_index < m_size); auto dst = buffer + dst_offset; - memcpy(dst, m_aos + src_index, sizeof(ParticleType)); - dst += sizeof(ParticleType); + if constexpr (!ParticleType::is_soa_particle) { + memcpy(dst, m_aos + src_index, sizeof(ParticleType)); + dst += sizeof(ParticleType); + } int array_start_index = AMREX_SPACEDIM + NStructReal; for (int i = 0; i < NArrayReal; ++i) { @@ -595,18 +595,10 @@ struct ConstParticleTileData { AMREX_ASSERT(index < m_size); SuperParticleType sp; - for (int i = 0; i < AMREX_SPACEDIM; ++i) - sp.pos(i) = m_rdata[i][index]; - for (int i = 0; i < NStructReal; ++i) - sp.rdata(i) = m_rdata[i][index]; for (int i = 0; i < NAR; ++i) - sp.rdata(NStructReal+i) = m_rdata[i][index]; - sp.id() = m_idata[0][index]; - sp.cpu() = m_idata[1][index]; - for (int i = 0; i < NStructInt; ++i) - sp.idata(i) = m_idata[i][index]; + sp.rdata(i) = m_rdata[i][index]; for (int i = 0; i < NAI; ++i) - sp.idata(NStructInt+i) = m_idata[i][index]; + sp.idata(i) = m_idata[i][index]; return sp; } }; @@ -653,105 +645,66 @@ struct ParticleTile m_runtime_i_cptrs.resize(a_num_runtime_int); } - // Get cpu data - - // AoS - template ::type = 0> - ParticleCPUWrapper cpu (int index) & { - ParticleType p(this->getParticleTileData(), index); - return p.cpu(); - } - - // const - template ::type = 0> - ConstParticleCPUWrapper cpu (int index) const & { - using ConstParticleType = typename ParticleType::ConstType; - ConstParticleType p(this->getConstParticleTileData(), index); - return p.cpu(); - } - - // SoA - template ::type = 0> - ParticleCPUWrapper cpu (int index) & { - ParticleType& p = m_aos_tile().dataPtr()[index]; - return p.cpu(); - } - - // const - - template ::type = 0> - ConstParticleCPUWrapper cpu (int index) const & { - ParticleType& p = m_aos_tile().dataPtr()[index]; - return p.cpu(); - } - // Get id data - - // AoS - template ::type = 0> - ParticleIDWrapper id (int index) & { - ParticleType p(this->getParticleTileData(), index); - return p.id(); + decltype(auto) id (int index) & { + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile[index].id(); + } else { + return m_soa_tile.GetIntData(0)[index]; + } } // const - template ::type = 0> - ConstParticleIDWrapper id (int index) const & { - using ConstParticleType = typename ParticleType::ConstType; - ConstParticleType p(this->getConstParticleTileData(), index); - return p.id(); + decltype(auto) id (int index) const & { + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile[index].id(); + } else { + return m_soa_tile.GetIntData(0)[index]; + } } - // SoA - template ::type = 0> - ParticleIDWrapper id (int index) & { - ParticleType& p = m_aos_tile().dataPtr()[index]; - return p.id(); + // Get cpu data + decltype(auto) cpu (int index) & { + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile[index].cpu(); + } else { + return m_soa_tile.GetIntData(1)[index]; + } } // const - template ::type = 0> - ConstParticleIDWrapper id (int index) const & { - ParticleType& p = m_aos_tile().dataPtr()[index]; - return p.id(); + decltype(auto) cpu (int index) const & { + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile[index].cpu(); + } else { + return m_soa_tile.GetIntData(1)[index]; + } } // Get positions data - - template ::type = 0> RealType& pos (int index, int position_index) & { - static_assert(NArrayReal == T::PTD::NAR, "ParticleTile mismatch in R"); - static_assert(NArrayInt == T::PTD::NAI, "ParticleTile mismatch in I"); - static_assert(0 == T::StorageParticleType::NReal, "ParticleTile 2 mismatch in R"); - static_assert(0 == T::StorageParticleType::NInt, "ParticleTile 2 mismatch in I"); - ParticleTileDataType x=this->getParticleTileData(); - ParticleType p(x, index); - return p.pos(position_index); - } - - template ::type = 0> - RealType pos (int index, int position_index) const & - { - using ConstParticleType = typename ParticleType::ConstType; - ConstParticleType p(this->getConstParticleTileData(), index); - return p.pos(position_index); - } + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile[index].pos(position_index); + } else { + static_assert(NArrayReal == ParticleType::PTD::NAR, "ParticleTile mismatch in R"); + static_assert(NArrayInt == ParticleType::PTD::NAI, "ParticleTile mismatch in I"); + static_assert(0 == ParticleType::StorageParticleType::NReal, "ParticleTile 2 mismatch in R"); + static_assert(0 == ParticleType::StorageParticleType::NInt, "ParticleTile 2 mismatch in I"); - template ::type = 0> - RealType& pos (int index, int position_index) & { - ParticleType& p = m_aos_tile().dataPtr()[index]; - return p.pos(position_index); + return m_soa_tile.GetRealData(position_index)[index]; + } } - - template ::type = 0> + // const RealType pos (int index, int position_index) const & { - ParticleType& p = m_aos_tile().dataPtr()[index]; - return p.pos(position_index); + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile[index].pos(position_index); + } else { + return m_soa_tile.GetRealData(position_index)[index]; + } } - AoS& GetArrayOfStructs () { return m_aos_tile; } const AoS& GetArrayOfStructs () const { return m_aos_tile; } @@ -764,63 +717,83 @@ struct ParticleTile * \brief Returns the total number of particles (real and neighbor) * */ - - template ::type = 0> - std::size_t size () const { return m_aos_tile.size(); } - - template ::type = 0> - std::size_t size () const { return m_soa_tile.size(); } + std::size_t size () const + { + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile.size(); + } else { + return m_soa_tile.size(); + } + } /** * \brief Returns the number of real particles (excluding neighbors) * */ - template ::type = 0> - int numParticles () const { return m_aos_tile.numParticles(); } - - template ::type = 0> - int numParticles () const { return m_soa_tile.numParticles(); } + int numParticles () const + { + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile.numParticles(); + } else { + return m_soa_tile.numParticles(); + } + } /** * \brief Returns the number of real particles (excluding neighbors) * */ - template ::type = 0> - int numRealParticles () const { return m_aos_tile.numRealParticles(); } - - template ::type = 0> - int numRealParticles () const { return m_soa_tile.numRealParticles(); } + int numRealParticles () const + { + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile.numRealParticles(); + } else { + return m_soa_tile.numRealParticles(); + } + } /** * \brief Returns the number of neighbor particles (excluding reals) * */ - template ::type = 0> - int numNeighborParticles () const { return m_aos_tile.numNeighborParticles(); } - - template ::type = 0> - int numNeighborParticles () const { return m_soa_tile.numNeighborParticles(); } + int numNeighborParticles () const + { + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile.numNeighborParticles(); + } else { + return m_soa_tile.numNeighborParticles(); + } + } /** * \brief Returns the total number of particles, real and neighbor * */ - template ::type = 0> - int numTotalParticles () const { return m_aos_tile.numTotalParticles() ; } - - template ::type = 0> - int numTotalParticles () const { return m_soa_tile.numTotalParticles() ; } + int numTotalParticles () const + { + if constexpr (!ParticleType::is_soa_particle) { + return m_aos_tile.numTotalParticles(); + } else { + return m_soa_tile.numTotalParticles(); + } + } void setNumNeighbors (int num_neighbors) { + if constexpr(!ParticleType::is_soa_particle) { + m_aos_tile.setNumNeighbors(num_neighbors); + } m_soa_tile.setNumNeighbors(num_neighbors); - m_aos_tile.setNumNeighbors(num_neighbors); } - int getNumNeighbors () + int getNumNeighbors () const { - AMREX_ASSERT( m_soa_tile.getNumNeighbors() == m_aos_tile.getNumNeighbors() ); - return m_aos_tile.getNumNeighbors(); + if constexpr (!ParticleType::is_soa_particle) { + AMREX_ASSERT( m_soa_tile.getNumNeighbors() == m_aos_tile.getNumNeighbors() ); + return m_aos_tile.getNumNeighbors(); + } else { + return m_soa_tile.getNumNeighbors(); + } } void resize (std::size_t count) @@ -834,6 +807,7 @@ struct ParticleTile /// /// Add one particle to this tile. /// + template ::type = 0> void push_back (const ParticleType& p) { m_aos_tile().push_back(p); } /// @@ -845,22 +819,24 @@ struct ParticleTile { auto np = numParticles(); - m_aos_tile.resize(np+1); + if constexpr (!ParticleType::is_soa_particle) { + m_aos_tile.resize(np+1); + for (int i = 0; i < AMREX_SPACEDIM; ++i) + m_aos_tile[np].pos(i) = sp.pos(i); + for (int i = 0; i < NStructReal; ++i) + m_aos_tile[np].rdata(i) = sp.rdata(i); + m_aos_tile[np].id() = sp.id(); + m_aos_tile[np].cpu() = sp.cpu(); + for (int i = 0; i < NStructInt; ++i) + m_aos_tile[np].idata(i) = sp.idata(i); + } + m_soa_tile.resize(np+1); auto& arr_rdata = m_soa_tile.GetRealData(); auto& arr_idata = m_soa_tile.GetIntData(); - - for (int i = 0; i < AMREX_SPACEDIM; ++i) - m_aos_tile[np].pos(i) = sp.pos(i); - for (int i = 0; i < NStructReal; ++i) - m_aos_tile[np].rdata(i) = sp.rdata(i); for (int i = 0; i < NArrayReal; ++i) arr_rdata[i][np] = sp.rdata(NStructReal+i); - m_aos_tile[np].id() = sp.id(); - m_aos_tile[np].cpu() = sp.cpu(); - for (int i = 0; i < NStructInt; ++i) - m_aos_tile[np].idata(i) = sp.idata(i); for (int i = 0; i < NArrayInt; ++i) arr_idata[i][np] = sp.idata(NStructInt+i); } diff --git a/Src/Particle/AMReX_StructOfArrays.H b/Src/Particle/AMReX_StructOfArrays.H index 172b1e5d807..a35838ab07f 100644 --- a/Src/Particle/AMReX_StructOfArrays.H +++ b/Src/Particle/AMReX_StructOfArrays.H @@ -125,7 +125,7 @@ struct StructOfArrays { resize(nrp + num_neighbors); } - [[nodiscard]] int getNumNeighbors () { return m_num_neighbor_particles; } + [[nodiscard]] int getNumNeighbors () const { return m_num_neighbor_particles; } void resize (size_t count) {