Skip to content

Commit

Permalink
Merge branch 'AMReX-Codes:development' into development
Browse files Browse the repository at this point in the history
  • Loading branch information
@ruohai0925
ruohai0925 authored Jun 5, 2024
2 parents f2b6369 + 0a691f6 commit 6dd5c1e
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 100 deletions.
6 changes: 4 additions & 2 deletions .github/workflows/intel.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ jobs:
tests-oneapi-sycl-eb-nvidia:
name: oneAPI SYCL for Nvidia GPUs [tests w/ EB]
if: 0
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -126,10 +125,13 @@ jobs:
set +e
source /opt/intel/oneapi/setvars.sh --include-intel-llvm
set -e
# Test is disabled due to a compiler issue with some math functions.
# The fix did not make it to the next oneAPI release. So we will
# enable it again after the next next oneAPI release.
cmake -S . -B build \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DAMReX_EB=ON \
-DAMReX_ENABLE_TESTS=ON \
-DAMReX_ENABLE_TESTS=OFF \
-DAMReX_TEST_TYPE=Small \
-DAMReX_GPU_BACKEND=SYCL \
-DCMAKE_C_COMPILER=$(which icx) \
Expand Down
27 changes: 14 additions & 13 deletions Src/Base/AMReX_BC_TYPES.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,23 +60,24 @@ namespace amrex {

namespace PhysBCType {
enum physicalBndryTypes : int {interior=0,inflow,outflow,symmetry,
slipwall,noslipwall};
slipwall,noslipwall,inflowoutflow};
}

namespace BCType {
enum mathematicalBndryTypes : int {
bogus = -666,
reflect_odd = -1,
int_dir = 0,
reflect_even = 1,
foextrap = 2,
ext_dir = 3,
hoextrap = 4,
hoextrapcc = 5,
ext_dir_cc = 6,
user_1 = 1001,
user_2 = 1002,
user_3 = 1003
bogus = -666,
reflect_odd = -1,
int_dir = 0,
reflect_even = 1,
foextrap = 2,
ext_dir = 3,
hoextrap = 4,
hoextrapcc = 5,
ext_dir_cc = 6,
direction_dependent = 7,
user_1 = 1001,
user_2 = 1002,
user_3 = 1003
};
}

Expand Down
118 changes: 37 additions & 81 deletions Src/Particle/AMReX_ParIter.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,29 +62,26 @@ public:

ParIterBase_impl (ContainerRef pc, int level, MFItInfo& info);

#ifdef AMREX_USE_OMP
void operator++ ()
{
if (dynamic) {
#pragma omp atomic capture
m_pariter_index = nextDynamicIndex++;
} else {
++m_pariter_index;
m_particle_current_tile = nullptr;
auto& particles = m_pc->GetParticles(m_level);
while (true) {
MFIter::operator++();
if (isValid()) {
auto key = std::make_pair(index(), LocalTileIndex());
auto f = particles.find(key);
if (f != particles.end() && f->second.numParticles() > 0) {
m_particle_current_tile = &(f->second);
break; // break if isValid() and non-empty particle tile found
}
} else {
break; // break if isValid() returns false
}
}
currentIndex = m_valid_index[m_pariter_index];
}
#else
void operator++ ()
{
++m_pariter_index;
currentIndex = m_valid_index[m_pariter_index];
#ifdef AMREX_USE_GPU
Gpu::Device::setStreamIndex(currentIndex);
#endif
}
#endif

[[nodiscard]] ParticleTileRef GetParticleTile () const { return *m_particle_tiles[m_pariter_index]; }
[[nodiscard]] ParticleTileRef GetParticleTile () const { return *m_particle_current_tile; }

[[nodiscard]] AoSRef GetArrayOfStructs () const { return GetParticleTile().GetArrayOfStructs(); }

Expand All @@ -105,9 +102,7 @@ public:
protected:

int m_level;
int m_pariter_index;
Vector<int> m_valid_index;
Vector<ParticleTilePtr> m_particle_tiles;
ParticleTilePtr m_particle_current_tile = nullptr;
ContainerPtr m_pc;
};

Expand Down Expand Up @@ -172,47 +167,20 @@ ParIterBase_impl<is_const, ParticleType, NArrayReal, NArrayInt, Allocator, CellA
:
MFIter(*pc.m_dummy_mf[level], pc.do_tiling ? info.EnableTiling(pc.tile_size) : info),
m_level(level),
m_pariter_index(0),
m_pc(&pc)
{
auto& particles = pc.GetParticles(level);

int start = dynamic ? 0 : beginIndex;
for (int i = start; i < endIndex; ++i)
{
int grid = (*index_map)[i];
int tile = local_tile_index_map ? (*local_tile_index_map)[i] : 0;
auto key = std::make_pair(grid,tile);
auto f = particles.find(key);
if (f != particles.end() && f->second.numParticles() > 0)
{
m_valid_index.push_back(i);
m_particle_tiles.push_back(&(f->second));
}
}

if (m_valid_index.empty())
{
endIndex = beginIndex;
}
else
{
currentIndex = beginIndex = m_valid_index.front();
if (dynamic) {
#ifdef AMREX_USE_OMP
int ind = omp_get_thread_num();
m_pariter_index += ind;
if (ind < m_valid_index.size()) {
currentIndex = beginIndex = m_valid_index[ind];
} else {
currentIndex = endIndex;
}
for (int i = 0; i < omp_get_num_threads(); ++i) {
m_valid_index.push_back(endIndex);
if (isValid()) {
auto& particles = m_pc->GetParticles(m_level);
while (true) {
auto key = std::make_pair(index(), LocalTileIndex());
auto f = particles.find(key);
if (f != particles.end() && f->second.numParticles() > 0) {
m_particle_current_tile = &(f->second);
break; // break if isValid() and non-empty particle tile found
}
#endif
MFIter::operator++();
if (!isValid()) { break; } // break if isValid() returns false
}
m_valid_index.push_back(endIndex);
}
}

Expand All @@ -224,33 +192,21 @@ ParIterBase_impl<is_const, T_ParticleType, NArrayReal, NArrayInt, Allocator, Cel
MFIter(*pc.m_dummy_mf[level],
pc.do_tiling ? pc.tile_size : IntVect::TheZeroVector()),
m_level(level),
m_pariter_index(0),
m_pc(&pc)
{
auto& particles = pc.GetParticles(level);

for (int i = beginIndex; i < endIndex; ++i)
{
int grid = (*index_map)[i];
int tile = local_tile_index_map ? (*local_tile_index_map)[i] : 0;
auto key = std::make_pair(grid,tile);
auto f = particles.find(key);
if (f != particles.end() && f->second.numParticles() > 0)
{
m_valid_index.push_back(i);
m_particle_tiles.push_back(&(f->second));
if (isValid()) {
auto& particles = m_pc->GetParticles(m_level);
while (true) {
auto key = std::make_pair(index(), LocalTileIndex());
auto f = particles.find(key);
if (f != particles.end() && f->second.numParticles() > 0) {
m_particle_current_tile = &(f->second);
break;
}
MFIter::operator++();
if (!isValid()) { break; }
}
}

if (m_valid_index.empty())
{
endIndex = beginIndex;
}
else
{
currentIndex = beginIndex = m_valid_index.front();
m_valid_index.push_back(endIndex);
}
}

template <bool is_const, int T_NStructReal, int T_NStructInt, int T_NArrayReal=0, int T_NArrayInt=0,
Expand Down
8 changes: 4 additions & 4 deletions Src/Particle/AMReX_ParticleContainerI.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1803,7 +1803,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
// we need to create any missing map entries in serial here
for (pmap_it=tmp_local[lev].begin(); pmap_it != tmp_local[lev].end(); pmap_it++)
{
m_particles[lev][pmap_it->first];
DefineAndReturnParticleTile(lev, pmap_it->first.first, pmap_it->first.second);
grid_tile_ids.push_back(pmap_it->first);
pvec_ptrs.push_back(&(pmap_it->second));
}
Expand All @@ -1814,7 +1814,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
for (int pit = 0; pit < static_cast<int>(pvec_ptrs.size()); ++pit)
{
auto index = grid_tile_ids[pit];
auto& ptile = DefineAndReturnParticleTile(lev, index.first, index.second);
auto& ptile = ParticlesAt(lev, index.first, index.second);
auto& aos = ptile.GetArrayOfStructs();
auto& soa = ptile.GetStructOfArrays();
auto& aos_tmp = *(pvec_ptrs[pit]);
Expand Down Expand Up @@ -1842,7 +1842,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
// we need to create any missing map entries in serial here
for (auto soa_map_it=soa_local[lev].begin(); soa_map_it != soa_local[lev].end(); soa_map_it++)
{
m_particles[lev][soa_map_it->first];
DefineAndReturnParticleTile(lev, soa_map_it->first.first, soa_map_it->first.second);
grid_tile_ids.push_back(soa_map_it->first);
}

Expand All @@ -1852,7 +1852,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
for (int pit = 0; pit < static_cast<int>(grid_tile_ids.size()); ++pit) // NOLINT(modernize-loop-convert)
{
auto index = grid_tile_ids[pit];
auto& ptile = DefineAndReturnParticleTile(lev, index.first, index.second);
auto& ptile = ParticlesAt(lev, index.first, index.second);
auto& soa = ptile.GetStructOfArrays();
auto& soa_tmp = soa_local[lev][index];
for (int i = 0; i < num_threads; ++i) {
Expand Down

0 comments on commit 6dd5c1e

Please sign in to comment.