diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index b845d130d5c..9dd1d39b953 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -18,10 +18,10 @@ ParticleContainer_impl; @@ -1530,7 +1530,7 @@ ParticleContainer_implnumParticles(); ParticleLocData pld; - if constexpr(!ParticleType::is_soa_particle){ + if constexpr (!ParticleType::is_soa_particle){ if (npart != 0) { Long last = npart - 1; @@ -1647,7 +1647,7 @@ ParticleContainer_impl >& not_ours, Particle p; - if constexpr (!ParticleType::is_soa_particle) { - std::memcpy(&p, pbuf, sizeof(ParticleType)); - } else { + if constexpr (ParticleType::is_soa_particle) { + std::memcpy(&p.m_idcpu, pbuf, sizeof(uint64_t)); + ParticleReal pos[AMREX_SPACEDIM]; - std::memcpy(&pos[0], pbuf, AMREX_SPACEDIM*sizeof(ParticleReal)); + std::memcpy(&pos[0], pbuf + sizeof(uint64_t), AMREX_SPACEDIM*sizeof(ParticleReal)); AMREX_D_TERM(p.pos(0) = pos[0];, p.pos(1) = pos[1];, p.pos(2) = pos[2]); - - int idcpu[2]; - std::memcpy(&idcpu[0], pbuf + NumRealComps()*sizeof(ParticleReal), 2*sizeof(int)); - - p.id() = idcpu[0]; - p.cpu() = idcpu[1]; + } else { + std::memcpy(&p, pbuf, sizeof(ParticleType)); } bool success = Where(p, pld, lev_min, lev_max, 0); @@ -2097,7 +2114,12 @@ RedistributeMPI (std::map >& not_ours, rcv_tile[ipart])]; char* pbuf = ((char*) &recvdata[offset]) + j*superparticle_size; - if constexpr(! ParticleType::is_soa_particle) { + if constexpr (ParticleType::is_soa_particle) { + uint64_t idcpudata; + std::memcpy(&idcpudata, pbuf, sizeof(uint64_t)); + pbuf += sizeof(uint64_t); + ptile.GetStructOfArrays().GetIdCPUData().push_back(idcpudata); + } else { ParticleType p; std::memcpy(&p, pbuf, sizeof(ParticleType)); pbuf += sizeof(ParticleType); @@ -2146,6 +2168,10 @@ RedistributeMPI (std::map >& not_ours, host_int_attribs.reserve(15); host_int_attribs.resize(finestLevel()+1); + Vector, Gpu::HostVector > > host_idcpu; + host_idcpu.reserve(15); + host_idcpu.resize(finestLevel()+1); + ipart = 0; for (int i = 0; i < nrcvs; ++i) { @@ -2159,7 +2185,15 @@ RedistributeMPI (std::map >& not_ours, char* pbuf = ((char*) &recvdata[offset]) + j*superparticle_size; - if constexpr(! ParticleType::is_soa_particle) { + host_real_attribs[lev][ind].resize(NumRealComps()); + host_int_attribs[lev][ind].resize(NumIntComps()); + + if constexpr (ParticleType::is_soa_particle) { + uint64_t idcpudata; + std::memcpy(&idcpudata, pbuf, sizeof(uint64_t)); + pbuf += sizeof(uint64_t); + host_idcpu[lev][ind].push_back(idcpudata); + } else { ParticleType p; std::memcpy(&p, pbuf, sizeof(ParticleType)); pbuf += sizeof(ParticleType); @@ -2210,7 +2244,12 @@ RedistributeMPI (std::map >& not_ours, auto new_size = old_size + src_tile.size(); dst_tile.resize(new_size); - if constexpr(! ParticleType::is_soa_particle) { + if constexpr (ParticleType::is_soa_particle) { + Gpu::copyAsync(Gpu::hostToDevice, + host_idcpu[host_lev][std::make_pair(grid,tile)].begin(), + host_idcpu[host_lev][std::make_pair(grid,tile)].end(), + dst_tile.GetStructOfArrays().GetIdCPUData().begin() + old_size); + } else { Gpu::copyAsync(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H index e6969c9b1bd..a10f9973a01 100644 --- a/Src/Particle/AMReX_ParticleIO.H +++ b/Src/Particle/AMReX_ParticleIO.H @@ -954,6 +954,10 @@ ParticleContainer_impl, Gpu::HostVector > > host_idcpu; + host_idcpu.reserve(15); + host_idcpu.resize(finestLevel()+1); + for (int i = 0; i < cnt; i++) { // note: for pure SoA particle layouts, we do write the id, cpu and positions as a struct // for backwards compatibility with readers @@ -1021,8 +1025,7 @@ ParticleContainer_impl, Gpu::HostVector > > host_idcpu; + host_idcpu.reserve(15); + host_idcpu.resize(finestLevel()+1); + for (Long j = 0; j < icount; j++) { Particle<0, 0> ptest; @@ -1117,8 +1121,9 @@ InitRandom (Long icount, host_real_attribs[pld.m_lev][ind][i].push_back(pos[j*AMREX_SPACEDIM+i]); } - host_int_attribs[pld.m_lev][ind][0].push_back(ParticleType::NextID()); - host_int_attribs[pld.m_lev][ind][1].push_back(MyProc); + host_idcpu[pld.m_lev][ind].push_back(0); + ParticleIDWrapper(host_idcpu[pld.m_lev][ind].back()) = ParticleType::NextID(); + ParticleCPUWrapper(host_idcpu[pld.m_lev][ind].back()) = ParallelDescriptor::MyProc(); host_particles[pld.m_lev][ind]; @@ -1157,6 +1162,11 @@ InitRandom (Long icount, { Gpu::copyAsync(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); + } else { + Gpu::copyAsync(Gpu::hostToDevice, + host_idcpu[host_lev][std::make_pair(grid,tile)].begin(), + host_idcpu[host_lev][std::make_pair(grid,tile)].end(), + dst_tile.GetStructOfArrays().GetIdCPUData().begin() + old_size); } for (int i = 0; i < NArrayReal; ++i) { // NOLINT(readability-misleading-indentation) @@ -1201,6 +1211,10 @@ InitRandom (Long icount, host_int_attribs.reserve(15); host_int_attribs.resize(finestLevel()+1); + Vector, Gpu::HostVector > > host_idcpu; + host_idcpu.reserve(15); + host_idcpu.resize(finestLevel()+1); + for (Long icnt = 0; icnt < M; icnt++) { Particle<0, 0> ptest; for (int i = 0; i < AMREX_SPACEDIM; i++) { @@ -1261,8 +1275,9 @@ InitRandom (Long icount, host_real_attribs[pld.m_lev][ind][i].push_back(ptest.pos(i)); } - host_int_attribs[pld.m_lev][ind][0].push_back(ptest.id()); - host_int_attribs[pld.m_lev][ind][1].push_back(ptest.cpu()); + host_idcpu[pld.m_lev][ind].push_back(0); + ParticleIDWrapper(host_idcpu[pld.m_lev][ind].back()) = ParticleType::NextID(); + ParticleCPUWrapper(host_idcpu[pld.m_lev][ind].back()) = ParallelDescriptor::MyProc(); host_particles[pld.m_lev][ind]; @@ -1300,6 +1315,11 @@ InitRandom (Long icount, { Gpu::copyAsync(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); + } else { + Gpu::copyAsync(Gpu::hostToDevice, + host_idcpu[host_lev][std::make_pair(grid,tile)].begin(), + host_idcpu[host_lev][std::make_pair(grid,tile)].end(), + dst_tile.GetStructOfArrays().GetIdCPUData().begin() + old_size); } for (int i = 0; i < NArrayReal; ++i) { // NOLINT(readability-misleading-indentation) diff --git a/Src/Particle/AMReX_ParticleTile.H b/Src/Particle/AMReX_ParticleTile.H index e35af847ec4..2b60b37d304 100644 --- a/Src/Particle/AMReX_ParticleTile.H +++ b/Src/Particle/AMReX_ParticleTile.H @@ -43,6 +43,7 @@ struct ParticleTileData ParticleType* AMREX_RESTRICT m_aos; + uint64_t* m_idcpu; GpuArray m_rdata; GpuArray m_idata; @@ -67,7 +68,7 @@ struct ParticleTileData if constexpr(!ParticleType::is_soa_particle) { return this->m_aos[index].id(); } else { - return this->m_idata[0][index]; + return ParticleIDWrapper(this->m_idcpu[index]); } } @@ -77,7 +78,17 @@ struct ParticleTileData if constexpr(!ParticleType::is_soa_particle) { return this->m_aos[index].cpu(); } else { - return this->m_idata[1][index]; + return ParticleCPUWrapper(this->m_idcpu[index]); + } + } + + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + decltype(auto) idcpu (const int index) const & + { + if constexpr(ParticleType::is_soa_particle) { + return this->m_idcpu[index]; + } else { + amrex::Abort("not implemented"); } } @@ -112,6 +123,9 @@ struct ParticleTileData if constexpr (!ParticleType::is_soa_particle) { memcpy(dst, m_aos + src_index, sizeof(ParticleType)); dst += sizeof(ParticleType); + } else { + memcpy(dst, m_idcpu + src_index, sizeof(uint64_t)); + dst += sizeof(uint64_t); } int array_start_index = AMREX_SPACEDIM + NStructReal; for (int i = 0; i < NAR; ++i) @@ -160,6 +174,9 @@ struct ParticleTileData if constexpr (!ParticleType::is_soa_particle) { memcpy(m_aos + dst_index, src, sizeof(ParticleType)); src += sizeof(ParticleType); + } else { + memcpy(m_idcpu + dst_index, src, sizeof(uint64_t)); + src += sizeof(uint64_t); } int array_start_index = AMREX_SPACEDIM + NStructReal; for (int i = 0; i < NAR; ++i) @@ -231,9 +248,8 @@ struct ParticleTileData { AMREX_ASSERT(index < m_size); SuperParticleType sp; + sp.m_idcpu = m_idcpu[index]; for (int i = 0; i < AMREX_SPACEDIM; ++i) {sp.pos(i) = m_rdata[i][index];} - sp.id() = m_idata[0][index]; - sp.cpu() = m_idata[1][index]; for (int i = 0; i < NAR; ++i) { sp.rdata(i) = m_rdata[i][index]; } @@ -270,6 +286,7 @@ struct ParticleTileData AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void setSuperParticle (const SuperParticleType& sp, int index) const noexcept { + m_idcpu[index] = sp.m_idcpu; for (int i = 0; i < NAR; ++i) { m_rdata[i][index] = sp.rdata(i); } @@ -303,10 +320,10 @@ struct ConstSoAParticle : SoAParticleBase //functions to get id and cpu in the SOA data [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - int cpu () const { return this->m_constparticle_tile_data.m_idata[1][m_index]; } + ConstParticleCPUWrapper cpu () const { return this->m_constparticle_tile_data.m_idcpu[m_index]; } [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - int id () const { return this->m_constparticle_tile_data.m_idata[0][m_index]; } + ConstParticleIDWrapper id () const { return this->m_constparticle_tile_data.m_idcpu[m_index]; } //functions to get positions of the particle in the SOA data @@ -366,16 +383,22 @@ struct SoAParticle : SoAParticleBase //functions to get id and cpu in the SOA data [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - int& cpu () & { return this->m_particle_tile_data.m_idata[1][m_index]; } + ParticleCPUWrapper cpu () & { return this->m_particle_tile_data.m_idcpu[m_index]; } + + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + ParticleIDWrapper id () & { return this->m_particle_tile_data.m_idcpu[m_index]; } [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - int& id () & { return this->m_particle_tile_data.m_idata[0][m_index]; } + uint64_t& idcpu () & { return this->m_particle_tile_data.m_idcpu[m_index]; } [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - const int& cpu () const & { return this->m_particle_tile_data.m_idata[1][m_index]; } + ConstParticleCPUWrapper cpu () const & { return this->m_particle_tile_data.m_idcpu[m_index]; } [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - const int& id () const & { return this->m_particle_tile_data.m_idata[0][m_index]; } + ConstParticleIDWrapper id () const & { return this->m_particle_tile_data.m_idcpu[m_index]; } + + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + const uint64_t& idcpu () const & { return this->m_particle_tile_data.m_idcpu[m_index]; } //functions to get positions of the particle in the SOA data @@ -477,6 +500,7 @@ struct ConstParticleTileData Long m_size; const ParticleType* AMREX_RESTRICT m_aos; + const uint64_t* m_idcpu; GpuArray m_rdata; GpuArray m_idata; @@ -496,7 +520,7 @@ struct ConstParticleTileData if constexpr(!ParticleType::is_soa_particle) { return this->m_aos[index].id(); } else { - return this->m_idata[0][index]; + return ConstParticleIDWrapper(this->m_idcpu[index]); } } @@ -506,7 +530,17 @@ struct ConstParticleTileData if constexpr(!ParticleType::is_soa_particle) { return this->m_aos[index].cpu(); } else { - return this->m_idata[1][index]; + return ConstParticleCPUWrapper(this->m_idcpu[index]); + } + } + + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + decltype(auto) idcpu (const int index) const & + { + if constexpr(ParticleType::is_soa_particle) { + return this->m_idcpu[index]; + } else { + amrex::Abort("not implemented"); } } @@ -546,6 +580,9 @@ struct ConstParticleTileData if constexpr (!ParticleType::is_soa_particle) { memcpy(dst, m_aos + src_index, sizeof(ParticleType)); dst += sizeof(ParticleType); + } else { + memcpy(dst, m_idcpu + src_index, sizeof(uint64_t)); + dst += sizeof(uint64_t); } int array_start_index = AMREX_SPACEDIM + NStructReal; for (int i = 0; i < NArrayReal; ++i) @@ -622,8 +659,7 @@ struct ConstParticleTileData AMREX_ASSERT(index < m_size); SuperParticleType sp; for (int i = 0; i < AMREX_SPACEDIM; ++i) {sp.pos(i) = m_rdata[i][index];} - sp.id() = m_idata[0][index]; - sp.cpu() = m_idata[1][index]; + sp.m_idcpu = m_idcpu[index]; for (int i = 0; i < NAR; ++i) { sp.rdata(i) = m_rdata[i][index]; } @@ -663,7 +699,10 @@ struct ParticleTile ArrayOfStructs>::type; //using ParticleVector = typename AoS::ParticleVector; - using SoA = StructOfArrays; + using SoA = typename std::conditional< + ParticleType::is_soa_particle, + StructOfArrays, + StructOfArrays>::type; using RealVector = typename SoA::RealVector; using IntVector = typename SoA::IntVector; using StorageParticleType = typename ParticleType::StorageParticleType; @@ -688,7 +727,7 @@ struct ParticleTile if constexpr (!ParticleType::is_soa_particle) { return m_aos_tile[index].id(); } else { - return m_soa_tile.GetIntData(0)[index]; + return ParticleIDWrapper(m_soa_tile.GetIdCPUData()[index]); } } @@ -697,7 +736,7 @@ struct ParticleTile if constexpr (!ParticleType::is_soa_particle) { return m_aos_tile[index].id(); } else { - return m_soa_tile.GetIntData(0)[index]; + return ConstParticleIDWrapper(m_soa_tile.GetIdCPUData()[index]); } } @@ -706,7 +745,7 @@ struct ParticleTile if constexpr (!ParticleType::is_soa_particle) { return m_aos_tile[index].cpu(); } else { - return m_soa_tile.GetIntData(1)[index]; + return ParticleCPUWrapper(m_soa_tile.GetIdCPUData()[index]); } } @@ -715,7 +754,7 @@ struct ParticleTile if constexpr (!ParticleType::is_soa_particle) { return m_aos_tile[index].cpu(); } else { - return m_soa_tile.GetIntData(1)[index]; + return ConstParticleCPUWrapper(m_soa_tile.GetIdCPUData()[index]); } } @@ -873,7 +912,9 @@ struct ParticleTile } m_soa_tile.resize(np+1); - + if constexpr (!ParticleType::is_soa_particle) { + m_soa_tile.GetIdCPUData()[np] = sp.m_idcpu; + } auto& arr_rdata = m_soa_tile.GetRealData(); auto& arr_idata = m_soa_tile.GetIntData(); for (int i = 0; i < NArrayReal; ++i) { @@ -1092,6 +1133,11 @@ struct ParticleTile } else { ptd.m_aos = nullptr; } + if constexpr (ParticleType::is_soa_particle) { + ptd.m_idcpu = m_soa_tile.GetIdCPUData().dataPtr(); + } else { + ptd.m_idcpu = nullptr; + } if constexpr(NArrayReal > 0) { for (int i = 0; i < NArrayReal; ++i) { ptd.m_rdata[i] = m_soa_tile.GetRealData(i).dataPtr(); @@ -1157,6 +1203,11 @@ struct ParticleTile } else { ptd.m_aos = nullptr; } + if constexpr (ParticleType::is_soa_particle) { + ptd.m_idcpu = m_soa_tile.GetIdCPUData().dataPtr(); + } else { + ptd.m_idcpu = nullptr; + } if constexpr(NArrayReal > 0) { for (int i = 0; i < NArrayReal; ++i) { ptd.m_rdata[i] = m_soa_tile.GetRealData(i).dataPtr(); diff --git a/Src/Particle/AMReX_ParticleTransformation.H b/Src/Particle/AMReX_ParticleTransformation.H index 28ccfa84a97..aa737455ce6 100644 --- a/Src/Particle/AMReX_ParticleTransformation.H +++ b/Src/Particle/AMReX_ParticleTransformation.H @@ -35,7 +35,11 @@ void copyParticle (const ParticleTileData& dst, AMREX_ASSERT(dst.m_num_runtime_real == src.m_num_runtime_real); AMREX_ASSERT(dst.m_num_runtime_int == src.m_num_runtime_int ); - dst.m_aos[dst_i] = src.m_aos[src_i]; + if constexpr(!T_ParticleType::is_soa_particle) { + dst.m_aos[dst_i] = src.m_aos[src_i]; + } else { + dst.m_idcpu[dst_i] = src.m_idcpu[src_i]; + } if constexpr(NAR > 0) { for (int j = 0; j < NAR; ++j) { dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; diff --git a/Src/Particle/AMReX_StructOfArrays.H b/Src/Particle/AMReX_StructOfArrays.H index 0ef3b8ae869..6cd498e20a2 100644 --- a/Src/Particle/AMReX_StructOfArrays.H +++ b/Src/Particle/AMReX_StructOfArrays.H @@ -11,9 +11,11 @@ namespace amrex { template class Allocator=DefaultAllocator> + template class Allocator=DefaultAllocator, + bool use64BitIdCpu=false> struct StructOfArrays { + using IdCPU = amrex::PODVector >; using RealVector = amrex::PODVector >; using IntVector = amrex::PODVector >; @@ -28,9 +30,12 @@ struct StructOfArrays { [[nodiscard]] int NumIntComps () const noexcept { return NInt + m_runtime_idata.size(); } + [[nodiscard]] IdCPU& GetIdCPUData () { return m_idcpu; } [[nodiscard]] std::array& GetRealData () { return m_rdata; } [[nodiscard]] std::array< IntVector, NInt>& GetIntData () { return m_idata; } + /** Get access to the particle id/cpu Arrays */ + [[nodiscard]] const IdCPU& GetIdCPUData () const { return m_idcpu; } /** Get access to the particle Real Arrays (only compile-time components) */ [[nodiscard]] const std::array& GetRealData () const { return m_rdata; } /** Get access to the particle Int Arrays (only compile-time components) */ @@ -119,7 +124,9 @@ struct StructOfArrays { */ [[nodiscard]] std::size_t size () const { - if constexpr (NReal > 0) { + if constexpr (use64BitIdCpu == true) { + return m_idcpu.size(); + } else if constexpr (NReal > 0) { return m_rdata[0].size(); } else if constexpr (NInt > 0) { return m_idata[0].size(); @@ -175,6 +182,9 @@ struct StructOfArrays { void resize (size_t count) { + if constexpr (use64BitIdCpu == true) { + m_idcpu.resize(count); + } if constexpr (NReal > 0) { for (int i = 0; i < NReal; ++i) { m_rdata[i].resize(count); } } @@ -185,6 +195,15 @@ struct StructOfArrays { for (int i = 0; i < int(m_runtime_idata.size()); ++i) { m_runtime_idata[i].resize(count); } } + [[nodiscard]] IdCPU* idcpuarray () { + if constexpr (use64BitIdCpu == true) { + return m_idcpu.dataPtr(); + } else { + return nullptr; + } + + } + [[nodiscard]] GpuArray realarray () { GpuArray arr; @@ -208,6 +227,7 @@ struct StructOfArrays { int m_num_neighbor_particles{0}; private: + IdCPU m_idcpu; std::array m_rdata; std::array< IntVector, NInt> m_idata; diff --git a/Src/Particle/AMReX_WriteBinaryParticleData.H b/Src/Particle/AMReX_WriteBinaryParticleData.H index 31ca7f8df6a..d3cafc5be58 100644 --- a/Src/Particle/AMReX_WriteBinaryParticleData.H +++ b/Src/Particle/AMReX_WriteBinaryParticleData.H @@ -338,9 +338,10 @@ packIOData (Vector& idata, Vector& rdata, const PC& pc, int l else { amrex::ignore_unused(is_checkpoint); // Int: id, cpu - *iptr = soa.GetIntData(0)[pindex]; + uint64_t idcpu = soa.GetIdCPUData()[pindex]; + *iptr = (int) ParticleIDWrapper(idcpu); iptr += 1; - *iptr = soa.GetIntData(1)[pindex]; + *iptr = (int) ParticleCPUWrapper(idcpu); iptr += 1; // Real: position @@ -348,8 +349,8 @@ packIOData (Vector& idata, Vector& rdata, const PC& pc, int l rptr += AMREX_SPACEDIM; } - // extra SoA int data - const int int_start_offset = PC::ParticleType::is_soa_particle ? 2 : 0; // pure SoA: skip id, cpu + // SoA int data + const int int_start_offset = 0; for (int j = int_start_offset; j < pc.NumIntComps(); j++) { if (write_int_comp[PC::NStructInt+j]) { *iptr = soa.GetIntData(j)[pindex]; @@ -1021,14 +1022,15 @@ void WriteBinaryParticleDataAsync (PC const& pc, } else { // Ints: id, cpu - *iptr = soa.GetIntData(0)[pindex]; + uint64_t idcpu = soa.GetIdCPUData()[pindex]; + *iptr = (int) ParticleIDWrapper(idcpu); iptr += 1; - *iptr = soa.GetIntData(1)[pindex]; + *iptr = (int) ParticleCPUWrapper(idcpu); iptr += 1; } // extra SoA Ints - const int int_start_offset = PC::ParticleType::is_soa_particle ? 2 : 0; // pure SoA: skip id, cpu + const int int_start_offset = 0; for (int j = int_start_offset; j < nic; j++) { if (write_int_comp[NStructInt+j]) diff --git a/Tests/Particles/RedistributeSOA/main.cpp b/Tests/Particles/RedistributeSOA/main.cpp index 94715b7d6aa..62da81def81 100644 --- a/Tests/Particles/RedistributeSOA/main.cpp +++ b/Tests/Particles/RedistributeSOA/main.cpp @@ -92,6 +92,7 @@ class TestParticleContainer { const Box& tile_box = mfi.tilebox(); + Gpu::HostVector host_idcpu; std::array, NR> host_real; std::array, NI> host_int; @@ -106,6 +107,10 @@ class TestParticleContainer amrex::Long id = ParticleType::NextID(); + host_idcpu.push_back(0); + ParticleIDWrapper(host_idcpu.back()) = id; + ParticleCPUWrapper(host_idcpu.back()) = ParallelDescriptor::MyProc(); + host_int[0].push_back(static_cast(id)); host_int[1].push_back(ParallelDescriptor::MyProc()); host_real[0].push_back(static_cast (plo[0] + (iv[0] + r[0])*dx[0])); @@ -137,6 +142,13 @@ class TestParticleContainer particle_tile.resize(new_size); auto& soa = particle_tile.GetStructOfArrays(); + { + Gpu::copyAsync(Gpu::hostToDevice, + host_idcpu.begin(), + host_idcpu.end(), + soa.GetIdCPUData().begin() + old_size); + + } for (int i = 0; i < NR; ++i) { Gpu::copyAsync(Gpu::hostToDevice,