diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 1806bdb00d1..7e4edc2dc36 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -41,7 +41,7 @@ jobs: WARPX_CI_EB: 'TRUE' # default: 60; maximum: 360 - timeoutInMinutes: 180 + timeoutInMinutes: 240 steps: # set up caches: diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index e7ecf36a034..f16b405973e 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 68244ec91d118b5d4cc21f93376eaae8b56c51eb && cd - + cd ../amrex && git checkout --detach 2230caa24c7d4bd07edb08b54e9368f9c73eae6e && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Docs/source/developers/particles.rst b/Docs/source/developers/particles.rst index a0182f3845f..4b203e483c4 100644 --- a/Docs/source/developers/particles.rst +++ b/Docs/source/developers/particles.rst @@ -36,17 +36,14 @@ A typical loop over particles reads: } } -The innermost step ``[MY INNER LOOP]`` typically calls ``amrex::ParallelFor`` to perform operations on all particles in a portable way. For this reasons, the particle data needs to be converted in plain-old-data structures. The innermost loop in the code snippet above could look like: +The innermost step ``[MY INNER LOOP]`` typically calls ``amrex::ParallelFor`` to perform operations on all particles in a portable way. The innermost loop in the code snippet above could look like: .. code-block:: cpp - // Get Array-Of-Struct particle data, also called data - // (x, y, z, id, cpu) - const auto& particles = pti.GetArrayOfStructs(); // Get Struct-Of-Array particle data, also called attribs - // (ux, uy, uz, w, Exp, Ey, Ez, Bx, By, Bz) + // (x, y, z, ux, uy, uz, w) auto& attribs = pti.GetAttribs(); - auto& Exp = attribs[PIdx::Ex]; + auto& x = attribs[PIdx::x]; // [...] // Number of particles in this box const long np = pti.numParticles(); @@ -66,7 +63,6 @@ On a loop over boxes in a ``MultiFab`` (``MFIter``), it can be useful to access const int tile_id = mfi.LocalTileIndex(); // Get GPU-friendly arrays of particle data auto& ptile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - ParticleType* pp = particle_tile.GetArrayOfStructs()().data(); // Only need attribs (i.e., SoA data) auto& soa = ptile.GetStructOfArrays(); // As an example, let's get the ux momentum diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 2760beed53b..13cbd2c22d5 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 68244ec91d118b5d4cc21f93376eaae8b56c51eb +branch = 2230caa24c7d4bd07edb08b54e9368f9c73eae6e [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 9fb763b683e..1e5e7c79a65 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 68244ec91d118b5d4cc21f93376eaae8b56c51eb +branch = 2230caa24c7d4bd07edb08b54e9368f9c73eae6e [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index f84734308fb..dbac563ca28 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -49,7 +49,7 @@ SmartCopyTag getSmartCopyTag (const NameMap& src, const NameMap& dst) noexcept; * \param num_added the number of particles to set the ids for. */ template -void setNewParticleIDs (PTile& ptile, int old_size, int num_added) +void setNewParticleIDs (PTile& ptile, amrex::Long old_size, amrex::Long num_added) { amrex::Long pid; #ifdef AMREX_USE_OMP @@ -64,8 +64,9 @@ void setNewParticleIDs (PTile& ptile, int old_size, int num_added) auto ptd = ptile.getParticleTileData(); amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { - auto const new_id = ip + old_size; - ptd.m_idcpu[new_id] = amrex::SetParticleIDandCPU(pid+ip, cpuid); + auto const lip = static_cast(ip); + auto const new_id = lip + old_size; + ptd.m_idcpu[new_id] = amrex::SetParticleIDandCPU(pid+lip, cpuid); }); } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 08c784709fa..da1655b9dab 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -51,7 +51,6 @@ #include #include #include -#include #include #include #include @@ -1022,8 +1021,8 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int overlap_realbox.lo(2))}; // count the number of particles that each cell in overlap_box could add - Gpu::DeviceVector counts(overlap_box.numPts(), 0); - Gpu::DeviceVector offset(overlap_box.numPts()); + Gpu::DeviceVector counts(overlap_box.numPts(), 0); + Gpu::DeviceVector offset(overlap_box.numPts()); auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). @@ -1042,7 +1041,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int if (inj_pos->overlapsWith(lo, hi)) { auto index = overlap_box.index(iv); - const int r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv))? + const amrex::Long r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv))? (AMREX_D_TERM(lrrfac[0],*lrrfac[1],*lrrfac[2])) : (1); pcounts[index] = num_ppc*r; // update pcount by checking if cell-corners or cell-center @@ -1080,10 +1079,10 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Max number of new particles. All of them are created, // and invalid ones are then discarded - const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); + const amrex::Long max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - int pid; + amrex::Long pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1092,7 +1091,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, + pid + max_new_particles < LongParticleIds::LastParticleID, "ERROR: overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); @@ -1103,8 +1102,8 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int DefineAndReturnParticleTile(lev, grid_id, tile_id); } - auto old_size = particle_tile.size(); - auto new_size = old_size + max_new_particles; + auto const old_size = static_cast(particle_tile.size()); + auto const new_size = old_size + max_new_particles; particle_tile.resize(new_size); auto& soa = particle_tile.GetStructOfArrays(); @@ -1639,10 +1638,10 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // Max number of new particles. All of them are created, // and invalid ones are then discarded - const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); + const amrex::Long max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - int pid; + amrex::Long pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1651,15 +1650,15 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, + pid + max_new_particles < LongParticleIds::LastParticleID, "overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); auto& particle_tile = tmp_pc.DefineAndReturnParticleTile(0, grid_id, tile_id); - auto old_size = particle_tile.size(); - auto new_size = old_size + max_new_particles; + auto const old_size = static_cast(particle_tile.size()); + auto const new_size = old_size + max_new_particles; particle_tile.resize(new_size); auto& soa = particle_tile.GetStructOfArrays(); diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 58e3450f47d..9f92739a5d5 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -10,7 +10,6 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" -#include #include #include #include diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 9db2b1f1aad..6c4570a4078 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "68244ec91d118b5d4cc21f93376eaae8b56c51eb" +set(WarpX_amrex_branch "2230caa24c7d4bd07edb08b54e9368f9c73eae6e" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 1079735279a..71831357e66 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "5aa700de18a61f933cb435adbe2299d74d794d6b" +set(WarpX_pyamrex_branch "0cbf4b08c9045e1845595c836b99f94bb3c1ac9f" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index a0ba9e056a5..2b53571163a 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 68244ec91d118b5d4cc21f93376eaae8b56c51eb && cd - +cd amrex && git checkout --detach 2230caa24c7d4bd07edb08b54e9368f9c73eae6e && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets