From 8df11b69a1169a1b7791a7a5e723feecd121b467 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 9 Oct 2024 10:50:51 -0700 Subject: [PATCH] Named SoA Support (#4163) ## Summary Add support to optionally name SoA Real and Int components, both for compile-time and runtime components. - [x] define and defaults - [x] single tile `Get*Data(std::string)` - [x] refactor out default name generation for reuse, finalize defaults in `AddReal/IntComp` - [x] test ## Additional background Close #3614 ## Checklist The proposed changes: - [ ] fix a bug or incorrect behavior in AMReX - [x] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --------- Co-authored-by: Andrew Myers --- Src/Particle/AMReX_ParticleContainer.H | 39 ++++- Src/Particle/AMReX_ParticleContainerI.H | 40 ++++- Src/Particle/AMReX_ParticleIO.H | 70 +++------ Src/Particle/AMReX_ParticleTile.H | 12 +- Src/Particle/AMReX_ParticleUtil.H | 20 +++ Src/Particle/AMReX_StructOfArrays.H | 74 +++++++++- .../NamedSoAComponents/CMakeLists.txt | 10 ++ .../Particles/NamedSoAComponents/GNUmakefile | 22 +++ .../Particles/NamedSoAComponents/Make.package | 1 + Tests/Particles/NamedSoAComponents/main.cpp | 139 ++++++++++++++++++ 10 files changed, 367 insertions(+), 60 deletions(-) create mode 100644 Tests/Particles/NamedSoAComponents/CMakeLists.txt create mode 100644 Tests/Particles/NamedSoAComponents/GNUmakefile create mode 100644 Tests/Particles/NamedSoAComponents/Make.package create mode 100644 Tests/Particles/NamedSoAComponents/main.cpp diff --git a/Src/Particle/AMReX_ParticleContainer.H b/Src/Particle/AMReX_ParticleContainer.H index 03a2254a10e..4b347d283b8 100644 --- a/Src/Particle/AMReX_ParticleContainer.H +++ b/Src/Particle/AMReX_ParticleContainer.H @@ -60,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -1144,7 +1145,8 @@ public: */ ParticleTileType& DefineAndReturnParticleTile (int lev, int grid, int tile) { - m_particles[lev][std::make_pair(grid, tile)].define(NumRuntimeRealComps(), NumRuntimeIntComps()); + m_particles[lev][std::make_pair(grid, tile)].define(NumRuntimeRealComps(), NumRuntimeIntComps(), &m_soa_rdata_names, &m_soa_idata_names); + return ParticlesAt(lev, grid, tile); } @@ -1247,10 +1249,10 @@ public: Long superParticleSize() const { return superparticle_size; } - template ,int> = 0> - void AddRealComp (T communicate=true) + void AddRealComp (std::string const & name, int communicate=1) { + m_soa_rdata_names.push_back(name); + m_runtime_comps_defined = true; m_num_runtime_real++; h_redistribute_real_comp.push_back(communicate); @@ -1270,10 +1272,15 @@ public: } } - template ,int> = 0> - void AddIntComp (T communicate=true) + void AddRealComp (int communicate=1) { + AddRealComp(getDefaultCompNameReal(NArrayReal+m_num_runtime_real), communicate); + } + + void AddIntComp (std::string const & name, int communicate=1) + { + m_soa_idata_names.push_back(name); + m_runtime_comps_defined = true; m_num_runtime_int++; h_redistribute_int_comp.push_back(communicate); @@ -1293,6 +1300,11 @@ public: } } + void AddIntComp (int communicate=1) + { + AddIntComp(getDefaultCompNameInt(NArrayInt+m_num_runtime_int), communicate); + } + int NumRuntimeRealComps () const { return m_num_runtime_real; } int NumRuntimeIntComps () const { return m_num_runtime_int; } @@ -1403,6 +1415,15 @@ public: #include "AMReX_ParticlesHDF5.H" #endif + /** Overwrite the default names for the compile-time SoA components */ + void SetSoACompileTimeNames (std::vector const & rdata_name, std::vector const & idata_name); + + /** Get the names for the real SoA components **/ + std::vector GetRealSoANames () const {return m_soa_rdata_names;} + + /** Get the names for the int SoA components **/ + std::vector GetIntSoANames () const {return m_soa_idata_names;} + protected: template @@ -1435,6 +1456,10 @@ private: size_t particle_size, superparticle_size; int num_real_comm_comps, num_int_comm_comps; Vector m_particles; + + // names of both compile-time and runtime Real and Int SoA data + std::vector m_soa_rdata_names; + std::vector m_soa_idata_names; }; template class Allocator, class CellAssignor> diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 74e65b792f0..718405f1e66 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -1,6 +1,10 @@ -#include #include +#include +#include +#include + + template class Allocator, class CellAssignor> void @@ -60,10 +64,40 @@ ParticleContainer_impl(i)); + } + for (int i=0; i(i)); + } + initialized = true; } } +template class Allocator, class CellAssignor> +void +ParticleContainer_impl :: SetSoACompileTimeNames ( + std::vector const & rdata_name, std::vector const & idata_name +) +{ + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rdata_name.size() == NArrayReal, "rdata_name must be equal to NArrayReal"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(idata_name.size() == NArrayInt, "idata_name must be equal to NArrayInt"); + + for (int i=0; i class Allocator, class CellAssignor> template @@ -1161,7 +1195,7 @@ ParticleContainer_impl write_real_comp; Vector tmp_real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { write_real_comp.push_back(1); if (real_comp_names.empty()) { - std::stringstream ss; - ss << "real_comp" << i; - tmp_real_comp_names.push_back(ss.str()); + tmp_real_comp_names.push_back(getDefaultCompNameReal(i)); } else { - tmp_real_comp_names.push_back(real_comp_names[i]); + tmp_real_comp_names.push_back(real_comp_names[i-first_rcomp]); } } @@ -75,9 +73,7 @@ ParticleContainer_impl(i)); } else { @@ -98,14 +94,12 @@ ParticleContainer_impl write_real_comp; Vector real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { write_real_comp.push_back(1); - std::stringstream ss; - ss << "real_comp" << i; - real_comp_names.push_back(ss.str()); + real_comp_names.push_back(getDefaultCompNameReal(i)); } Vector write_int_comp; @@ -113,9 +107,7 @@ ParticleContainer_impl(i)); } WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp, @@ -182,9 +174,7 @@ ParticleContainer_impl int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) { - std::stringstream ss; - ss << "int_comp" << i; - int_comp_names.push_back(ss.str()); + int_comp_names.push_back(getDefaultCompNameInt(i)); } WriteBinaryParticleData(dir, name, @@ -211,20 +201,16 @@ ParticleContainer_impl real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { - std::stringstream ss; - ss << "real_comp" << i; - real_comp_names.push_back(ss.str()); + real_comp_names.push_back(getDefaultCompNameReal(i)); } Vector int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) { - std::stringstream ss; - ss << "int_comp" << i; - int_comp_names.push_back(ss.str()); + int_comp_names.push_back(getDefaultCompNameInt(i)); } WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp, @@ -259,14 +245,12 @@ ParticleContainer_impl write_real_comp; Vector real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { write_real_comp.push_back(1); - std::stringstream ss; - ss << "real_comp" << i; - real_comp_names.push_back(ss.str()); + real_comp_names.push_back(getDefaultCompNameReal(i)); } Vector write_int_comp; @@ -274,9 +258,7 @@ ParticleContainer_impl(i)); } WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp, @@ -345,9 +327,7 @@ ParticleContainer_impl int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) { - std::stringstream ss; - ss << "int_comp" << i; - int_comp_names.push_back(ss.str()); + int_comp_names.push_back(getDefaultCompNameInt(i)); } WriteBinaryParticleData(dir, name, @@ -374,20 +354,16 @@ ParticleContainer_impl real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { - std::stringstream ss; - ss << "real_comp" << i; - real_comp_names.push_back(ss.str()); + real_comp_names.push_back(getDefaultCompNameReal(i)); } Vector int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) { - std::stringstream ss; - ss << "int_comp" << i; - int_comp_names.push_back(ss.str()); + int_comp_names.push_back(getDefaultCompNameInt(i)); } WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp, diff --git a/Src/Particle/AMReX_ParticleTile.H b/Src/Particle/AMReX_ParticleTile.H index a645330e04a..7546ff8a216 100644 --- a/Src/Particle/AMReX_ParticleTile.H +++ b/Src/Particle/AMReX_ParticleTile.H @@ -11,7 +11,10 @@ #include #include +#include #include +#include + namespace amrex { @@ -730,10 +733,15 @@ struct ParticleTile ParticleTile& operator= (ParticleTile &&) noexcept = default; #endif - void define (int a_num_runtime_real, int a_num_runtime_int) + void define ( + int a_num_runtime_real, + int a_num_runtime_int, + std::vector* soa_rdata_names=nullptr, + std::vector* soa_idata_names=nullptr + ) { m_defined = true; - GetStructOfArrays().define(a_num_runtime_real, a_num_runtime_int); + GetStructOfArrays().define(a_num_runtime_real, a_num_runtime_int, soa_rdata_names, soa_idata_names); m_runtime_r_ptrs.resize(a_num_runtime_real); m_runtime_i_ptrs.resize(a_num_runtime_int); m_runtime_r_cptrs.resize(a_num_runtime_real); diff --git a/Src/Particle/AMReX_ParticleUtil.H b/Src/Particle/AMReX_ParticleUtil.H index b09f1d3583f..f6f2506ec7f 100644 --- a/Src/Particle/AMReX_ParticleUtil.H +++ b/Src/Particle/AMReX_ParticleUtil.H @@ -883,6 +883,26 @@ void PermutationForDeposition (Gpu::DeviceVector& perm, index_type n }); } +template +std::string getDefaultCompNameReal (const int i) { + int first_r_name = 0; + if constexpr (P::is_soa_particle) { + if (i < AMREX_SPACEDIM) { + constexpr int x_in_ascii = 120; + std::string const name{char(x_in_ascii+i)}; + return name; + } + first_r_name = AMREX_SPACEDIM; + } + std::string const name{("real_comp" + std::to_string(i-first_r_name))}; + return name; +} + +template +std::string getDefaultCompNameInt (const int i) { + std::string const name{("int_comp" + std::to_string(i))}; + return name; +} #ifdef AMREX_USE_HDF5_ASYNC void async_vol_es_wait_particle(); diff --git a/Src/Particle/AMReX_StructOfArrays.H b/Src/Particle/AMReX_StructOfArrays.H index 4de35e085ca..201cf340dc1 100644 --- a/Src/Particle/AMReX_StructOfArrays.H +++ b/Src/Particle/AMReX_StructOfArrays.H @@ -6,7 +6,11 @@ #include #include +#include #include +#include +#include + namespace amrex { @@ -19,11 +23,18 @@ struct StructOfArrays { using RealVector = amrex::PODVector >; using IntVector = amrex::PODVector >; - void define (int a_num_runtime_real, int a_num_runtime_int) + void define ( + int a_num_runtime_real, + int a_num_runtime_int, + std::vector* soa_rdata_names=nullptr, + std::vector* soa_idata_names=nullptr + ) { m_defined = true; m_runtime_rdata.resize(a_num_runtime_real); m_runtime_idata.resize(a_num_runtime_int ); + m_rdata_names = soa_rdata_names; + m_idata_names = soa_idata_names; } [[nodiscard]] int NumRealComps () const noexcept { return NReal + m_runtime_rdata.size(); } @@ -79,6 +90,32 @@ struct StructOfArrays { } } + /** Get access to a particle Real component Array (compile-time and runtime component) + * + * @param name named component component with 0...NReal-1 compile-time and NReal... runtime arguments + */ + [[nodiscard]] RealVector& GetRealData (std::string const & name) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_rdata_names != nullptr, "SoA Real names were not defined."); + auto const pos = std::find(m_rdata_names->begin(), m_rdata_names->end(), name); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_rdata_names->end(), "Soa Real name='" + name + "' was not found components"); + + int const index = std::distance(m_rdata_names->begin(), pos); + return GetRealData(index); + } + + /** Get access to a particle Real component Array (compile-time and runtime component) + * + * @param name named component component with 0...NReal-1 compile-time and NReal... runtime arguments + */ + [[nodiscard]] const RealVector& GetRealData (std::string const & name) const { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_rdata_names != nullptr, "SoA Real names were not defined."); + auto const pos = std::find(m_rdata_names->begin(), m_rdata_names->end(), name); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_rdata_names->end(), "Soa Real name='" + name + "' was not found components"); + + int const index = std::distance(m_rdata_names->begin(), pos); + return GetRealData(index); + } + /** Get access to a particle Int component Array (compile-time and runtime component) * * @param index component with 0...NInt-1 compile-time and NInt... runtime arguments @@ -118,6 +155,34 @@ struct StructOfArrays { } } + /** Get access to a particle Int component Array (compile-time and runtime component) + * + * @param index component with 0...NInt-1 compile-time and NInt... runtime arguments + * @return + */ + [[nodiscard]] IntVector& GetIntData (std::string const & name) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_idata_names != nullptr, "SoA Int names were not defined."); + auto const pos = std::find(m_idata_names->begin(), m_idata_names->end(), name); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_idata_names->end(), "Soa Int name='" + name + "' was not found components"); + + int const index = std::distance(m_idata_names->begin(), pos); + return GetIntData(index); + } + + /** Get access to a particle Int component Array (compile-time and runtime component) + * + * @param index component with 0...NInt-1 compile-time and NInt... runtime arguments + * @return + */ + [[nodiscard]] const IntVector& GetIntData (std::string const & name) const { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_idata_names != nullptr, "SoA Int names were not defined."); + auto const pos = std::find(m_idata_names->begin(), m_idata_names->end(), name); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_idata_names->end(), "Soa Int name='" + name + "' was not found components"); + + int const index = std::distance(m_idata_names->begin(), pos); + return GetIntData(index); + } + /** * \brief Returns the total number of particles (real and neighbor) * @@ -226,13 +291,20 @@ struct StructOfArrays { int m_num_neighbor_particles{0}; private: + // compile-time data IdCPU m_idcpu; std::array m_rdata; std::array< IntVector, NInt> m_idata; + // runtime data std::vector m_runtime_rdata; std::vector m_runtime_idata; + // names of both compile-time and runtime Real and Int data + std::vector* m_rdata_names = nullptr; + std::vector* m_idata_names = nullptr; + + //! whether the runtime components are sized correctly bool m_defined{false}; }; diff --git a/Tests/Particles/NamedSoAComponents/CMakeLists.txt b/Tests/Particles/NamedSoAComponents/CMakeLists.txt new file mode 100644 index 00000000000..e14ddd68975 --- /dev/null +++ b/Tests/Particles/NamedSoAComponents/CMakeLists.txt @@ -0,0 +1,10 @@ +foreach(D IN LISTS AMReX_SPACEDIM) + set(_sources main.cpp) + #set(_input_files) + #set(_input_files inputs) + + setup_test(${D} _sources _input_files NTHREADS 2) + + unset(_sources) + unset(_input_files) +endforeach() diff --git a/Tests/Particles/NamedSoAComponents/GNUmakefile b/Tests/Particles/NamedSoAComponents/GNUmakefile new file mode 100644 index 00000000000..9f49d3ec029 --- /dev/null +++ b/Tests/Particles/NamedSoAComponents/GNUmakefile @@ -0,0 +1,22 @@ +AMREX_HOME = ../../../ + +DEBUG = FALSE + +DIM = 3 + +COMP = gcc + +USE_MPI = TRUE +USE_OMP = FALSE +USE_CUDA = FALSE + +#TINY_PROFILE = TRUE +USE_PARTICLES = TRUE + +include $(AMREX_HOME)/Tools/GNUMake/Make.defs + +include ./Make.package +include $(AMREX_HOME)/Src/Base/Make.package +include $(AMREX_HOME)/Src/Particle/Make.package + +include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Tests/Particles/NamedSoAComponents/Make.package b/Tests/Particles/NamedSoAComponents/Make.package new file mode 100644 index 00000000000..6b4b865e8fc --- /dev/null +++ b/Tests/Particles/NamedSoAComponents/Make.package @@ -0,0 +1 @@ +CEXE_sources += main.cpp diff --git a/Tests/Particles/NamedSoAComponents/main.cpp b/Tests/Particles/NamedSoAComponents/main.cpp new file mode 100644 index 00000000000..39b6a70ff52 --- /dev/null +++ b/Tests/Particles/NamedSoAComponents/main.cpp @@ -0,0 +1,139 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace amrex; + +void addParticles () +{ + using PC = ParticleContainerPureSoA; + int is_per[AMREX_SPACEDIM]; + for (int & d : is_per) { + d = 1; + } + + RealBox real_box; + for (int n = 0; n < AMREX_SPACEDIM; n++) + { + real_box.setLo(n, 0.0); + real_box.setHi(n, 100.0); + } + + IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); + IntVect domain_hi(AMREX_D_DECL(127, 127, 127)); + const Box base_domain(domain_lo, domain_hi); + + Geometry geom(base_domain, &real_box, CoordSys::cartesian, is_per); + BoxArray ba(base_domain); + ba.maxSize(64); + + DistributionMapping dm(ba); + + PC pc(geom, dm, ba); + + amrex::Print() << "Original Real SoA component names are: "; + for (auto& n : pc.GetRealSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "Original Int SoA component names are: "; + for (auto& n : pc.GetIntSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "Adding runtime comps. \n"; + pc.AddRealComp("real_comp1"); + pc.AddRealComp(); // without name - should be real_comp2 + pc.AddIntComp(); // without name - should be int_comp0 + + amrex::Print() << "New Real SoA component names are: "; + for (auto& n : pc.GetRealSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "New Int SoA component names are: "; + for (auto& n : pc.GetIntSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "Reset compile-time SoA names \n"; + pc.SetSoACompileTimeNames({AMREX_D_DECL("x", "y", "z"), "w"}, {"i1", "i2"}); + + amrex::Print() << "New Real SoA component names are: "; + for (auto& n : pc.GetRealSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "New Int SoA component names are: "; + for (auto& n : pc.GetIntSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + int const NArrayReal = PC::NArrayReal; + int const NArrayInt = PC::NArrayInt; + using ParticleType = typename PC::ParticleType; + + const int add_num_particles = 5; + auto& ptile1 = pc.DefineAndReturnParticleTile(0, 0, 0); + ptile1.resize(add_num_particles); + + for (int i = 0; i < add_num_particles; ++i) + { + for (int d = 0; d < AMREX_SPACEDIM; d++) { + ptile1.pos(i, d) = 12.0; + } + ptile1.getParticleTileData().rdata(AMREX_SPACEDIM)[i] = 1.2; // w + + ptile1.push_back_int(0, int(ParticleType::NextID())); + ptile1.push_back_int(1, amrex::ParallelDescriptor::MyProc()); + } + + int lev=0; + using MyParIter = ParIter_impl; + for (MyParIter pti(pc, lev); pti.isValid(); ++pti) { + auto& soa = pti.GetStructOfArrays(); + AMREX_D_TERM( + auto *xp = soa.GetRealData("x").data();, + auto *yp = soa.GetRealData("y").data();, + auto *zp = soa.GetRealData("z").data(); + ); + auto *wp = soa.GetRealData("w").data(); + + const int np = pti.numParticles(); + ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + AMREX_D_TERM( + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(xp[ip] == 12_prt, + "pos attribute expected to be 12");, + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(yp[ip] == 12_prt, + "pos attribute expected to be 12");, + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(zp[ip] == 12_prt, + "pos attribute expected to be 12"); + ); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(wp[ip] == 1.2_prt, + "pos attribute expected to be 1.2"); + }); + } +} + +int main (int argc, char* argv[]) + { + amrex::Initialize(argc,argv); + { + addParticles(); + } + amrex::Finalize(); + }