Skip to content

Commit

Permalink
Merge branch 'AMReX-Codes:development' into development
Browse files Browse the repository at this point in the history
  • Loading branch information
ruohai0925 authored Mar 4, 2024
2 parents 47f539a + 3525b4a commit f2bc520
Show file tree
Hide file tree
Showing 17 changed files with 1,259 additions and 404 deletions.
46 changes: 40 additions & 6 deletions Src/AmrCore/AMReX_AmrMesh.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -763,13 +763,47 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector<BoxArray>&
}
new_bx.Bcast(); // Broadcast the new BoxList to other processes

//
// Refine up to levf.
//
new_bx.refine(ref_ratio[levc]);
BL_ASSERT(new_bx.isDisjoint());
bool odd_ref_ratio = false;
for (auto const& rr : ref_ratio[levc]) {
if (rr != 1 && (rr%2 != 0)) {
odd_ref_ratio = true;
}
}

if (odd_ref_ratio)
{
// This approach imposes max_grid_size (suitably scaled) before
// refining so as to ensure fine grids align with coarse grids

//
// Impose max_grid_size (suitably coarsened)
//
AMREX_ASSERT(max_grid_size[levf].allGE(ref_ratio[levc]));
new_grids[levf] = BoxArray(std::move(new_bx), max_grid_size[levf]/ref_ratio[levc]);

//
// Refine up to levf.
//
new_grids[levf].refine(ref_ratio[levc]);
}
else
{
// This approach imposes max_grid_size after refining.
// For ref_ratio = 3 this can create fine grids that do not correctly divide by 3,
// but we leave it here so as not to change the gridding in
// existing ref_ratio = 2 or 4 applications

new_grids[levf] = BoxArray(std::move(new_bx), max_grid_size[levf]);
//
// Refine up to levf.
//
new_bx.refine(ref_ratio[levc]);

//
// Impose max_grid_size
//
new_grids[levf] = BoxArray(std::move(new_bx), max_grid_size[levf]);
}
BL_ASSERT(new_grids[levf].isDisjoint());
}
}
}
Expand Down
39 changes: 18 additions & 21 deletions Src/AmrCore/AMReX_InterpFaceReg_3D_C.H
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,10 @@ void interp_face_reg (int i, int j, int k, IntVect const& rr, Array4<Real> const
int jc = amrex::coarsen(j,rr[1]);
int kc = amrex::coarsen(k,rr[2]);
if (idim == 0) {
if (jc == domface.smallEnd(1) || jc == domface.bigEnd(1)) {
for (int n = 0; n < ncomp; ++n) {
fine(i,j,k,n+scomp) = crse(ic,jc,kc,n);
}
} else {
for (int n = 0; n < ncomp; ++n) {
fine(i,j,k,n+scomp) = crse(ic,jc,kc,n);
}
if (jc != domface.smallEnd(1) && jc != domface.bigEnd(1) && rr[1] > 1) {
Real sfy = Real(1.0);
for (int n = 0; n < ncomp; ++n) {
Real dc = Real(0.5) * (crse(ic,jc+1,kc,n) - crse(ic,jc-1,kc,n));
Expand All @@ -32,11 +31,11 @@ void interp_face_reg (int i, int j, int k, IntVect const& rr, Array4<Real> const
}
Real yoff = (static_cast<Real>(j - jc*rr[1]) + Real(0.5)) / Real(rr[1]) - Real(0.5);
for (int n = 0; n < ncomp; ++n) {
fine(i,j,k,n+scomp) = crse(ic,jc,kc,n) + yoff * slope(i,j,k,n) * sfy;
fine(i,j,k,n+scomp) += yoff * slope(i,j,k,n) * sfy;
}
}

if (kc != domface.smallEnd(2) && kc != domface.bigEnd(2)) {
if (kc != domface.smallEnd(2) && kc != domface.bigEnd(2) && rr[2] > 1) {
Real sfz = Real(1.0);
for (int n = 0; n < ncomp; ++n) {
Real dc = Real(0.5) * (crse(ic,jc,kc+1,n) - crse(ic,jc,kc-1,n));
Expand All @@ -56,11 +55,10 @@ void interp_face_reg (int i, int j, int k, IntVect const& rr, Array4<Real> const
}
}
} else if (idim == 1) {
if (ic == domface.smallEnd(0) || ic == domface.bigEnd(0)) {
for (int n = 0; n < ncomp; ++n) {
fine(i,j,k,n+scomp) = crse(ic,jc,kc,n);
}
} else {
for (int n = 0; n < ncomp; ++n) {
fine(i,j,k,n+scomp) = crse(ic,jc,kc,n);
}
if (ic != domface.smallEnd(0) && ic != domface.bigEnd(0) && rr[0] > 1) {
Real sfx = Real(1.0);
for (int n = 0; n < ncomp; ++n) {
Real dc = Real(0.5) * (crse(ic+1,jc,kc,n) - crse(ic-1,jc,kc,n));
Expand All @@ -76,11 +74,11 @@ void interp_face_reg (int i, int j, int k, IntVect const& rr, Array4<Real> const
}
Real xoff = (static_cast<Real>(i - ic*rr[0]) + Real(0.5)) / Real(rr[0]) - Real(0.5);
for (int n = 0; n < ncomp; ++n) {
fine(i,j,k,n+scomp) = crse(ic,jc,kc,n) + xoff * slope(i,j,k,n) * sfx;
fine(i,j,k,n+scomp) += xoff * slope(i,j,k,n) * sfx;
}
}

if (kc != domface.smallEnd(2) && kc != domface.bigEnd(2)) {
if (kc != domface.smallEnd(2) && kc != domface.bigEnd(2) && rr[2] > 1) {
Real sfz = Real(1.0);
for (int n = 0; n < ncomp; ++n) {
Real dc = Real(0.5) * (crse(ic,jc,kc+1,n) - crse(ic,jc,kc-1,n));
Expand All @@ -100,11 +98,10 @@ void interp_face_reg (int i, int j, int k, IntVect const& rr, Array4<Real> const
}
}
} else {
if (ic == domface.smallEnd(0) || ic == domface.bigEnd(0)) {
for (int n = 0; n < ncomp; ++n) {
fine(i,j,k,n+scomp) = crse(ic,jc,kc,n);
}
} else {
for (int n = 0; n < ncomp; ++n) {
fine(i,j,k,n+scomp) = crse(ic,jc,kc,n);
}
if (ic != domface.smallEnd(0) && ic != domface.bigEnd(0) && rr[0] > 1) {
Real sfx = Real(1.0);
for (int n = 0; n < ncomp; ++n) {
Real dc = Real(0.5) * (crse(ic+1,jc,kc,n) - crse(ic-1,jc,kc,n));
Expand All @@ -120,11 +117,11 @@ void interp_face_reg (int i, int j, int k, IntVect const& rr, Array4<Real> const
}
Real xoff = (static_cast<Real>(i - ic*rr[0]) + Real(0.5)) / Real(rr[0]) - Real(0.5);
for (int n = 0; n < ncomp; ++n) {
fine(i,j,k,n+scomp) = crse(ic,jc,kc,n) + xoff * slope(i,j,k,n) * sfx;
fine(i,j,k,n+scomp) += xoff * slope(i,j,k,n) * sfx;
}
}

if (jc != domface.smallEnd(1) && jc != domface.bigEnd(1)) {
if (jc != domface.smallEnd(1) && jc != domface.bigEnd(1) && rr[1] > 1) {
Real sfy = Real(1.0);
for (int n = 0; n < ncomp; ++n) {
Real dc = Real(0.5) * (crse(ic,jc+1,kc,n) - crse(ic,jc-1,kc,n));
Expand Down
39 changes: 39 additions & 0 deletions Src/Base/AMReX_FabArrayUtility.H
Original file line number Diff line number Diff line change
Expand Up @@ -1616,6 +1616,13 @@ void setBndry (MF& dst, typename MF::value_type val, int scomp, int ncomp)
dst.setBndry(val, scomp, ncomp);
}

//! dst *= val
template <class MF, std::enable_if_t<IsMultiFabLike_v<MF>,int> = 0>
void Scale (MF& dst, typename MF::value_type val, int scomp, int ncomp, int nghost)
{
dst.mult(val, scomp, ncomp, nghost);
}

//! dst = src
template <class DMF, class SMF,
std::enable_if_t<IsMultiFabLike_v<DMF> &&
Expand Down Expand Up @@ -1650,6 +1657,16 @@ void Xpay (MF& dst, typename MF::value_type a, MF const& src, int scomp, int dco
MF::Xpay(dst, a, src, scomp, dcomp, ncomp, nghost);
}

//! dst = a*src_a + b*src_b
template <class MF, std::enable_if_t<IsMultiFabLike_v<MF>,int> = 0>
void LinComb (MF& dst,
typename MF::value_type a, MF const& src_a, int acomp,
typename MF::value_type b, MF const& src_b, int bcomp,
int dcomp, int ncomp, IntVect const& nghost)
{
MF::LinComb(dst, a, src_a, acomp, b, src_b, bcomp, dcomp, ncomp, nghost);
}

//! dst = src w/ MPI communication
template <class MF, std::enable_if_t<IsMultiFabLike_v<MF>, int> = 0>
void ParallelCopy (MF& dst, MF const& src, int scomp, int dcomp, int ncomp,
Expand Down Expand Up @@ -1686,6 +1703,16 @@ void setBndry (Array<MF,N>& dst, typename MF::value_type val, int scomp, int nco
}
}

//! dst *= val
template <class MF, std::size_t N, std::enable_if_t<IsMultiFabLike_v<MF>,int> = 0>
void Scale (Array<MF,N>& dst, typename MF::value_type val, int scomp, int ncomp,
int nghost)
{
for (auto& mf : dst) {
mf.mult(val, scomp, ncomp, nghost);
}
}

//! dst = src
template <class DMF, class SMF, std::size_t N,
std::enable_if_t<IsMultiFabLike_v<DMF> &&
Expand Down Expand Up @@ -1730,6 +1757,18 @@ void Xpay (Array<MF,N>& dst, typename MF::value_type a,
}
}

//! dst = a*src_a + b*src_b
template <class MF, std::size_t N, std::enable_if_t<IsMultiFabLike_v<MF>,int> = 0>
void LinComb (Array<MF,N>& dst,
typename MF::value_type a, Array<MF,N> const& src_a, int acomp,
typename MF::value_type b, Array<MF,N> const& src_b, int bcomp,
int dcomp, int ncomp, IntVect const& nghost)
{
for (std::size_t i = 0; i < N; ++i) {
MF::LinComb(dst[i], a, src_a[i], acomp, b, src_b[i], bcomp, dcomp, ncomp, nghost);
}
}

//! dst = src w/ MPI communication
template <class MF, std::size_t N, std::enable_if_t<IsMultiFabLike_v<MF>, int> = 0>
void ParallelCopy (Array<MF,N>& dst, Array<MF,N> const& src,
Expand Down
2 changes: 1 addition & 1 deletion Src/Base/AMReX_GpuMemory.H
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ private:

#else

DeviceScalar (T init_val) : d(init_val) {}
DeviceScalar (T const& init_val) : d(init_val) {}
DeviceScalar () = default;
~DeviceScalar () = default;

Expand Down
146 changes: 146 additions & 0 deletions Src/Base/AMReX_LUSolver.H
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
#ifndef AMREX_LU_SOLVER_H_
#define AMREX_LU_SOLVER_H_
#include <AMReX_Config.H>

#include <AMReX_Arena.H>
#include <AMReX_Array.H>
#include <cmath>
#include <limits>

namespace amrex {

// https://en.wikipedia.org/wiki/LU_decomposition

template <int N, typename T>
class LUSolver
{
public:

LUSolver () = default;

LUSolver (Array2D<T, 0, N-1, 0, N-1, Order::C> const& a_mat);

void define (Array2D<T, 0, N-1, 0, N-1, Order::C> const& a_mat);

AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
void operator() (T* AMREX_RESTRICT x, T const* AMREX_RESTRICT b) const
{
for (int i = 0; i < N; ++i) {
x[i] = b[m_piv(i)];
for (int k = 0; k < i; ++k) {
x[i] -= m_mat(i,k) * x[k];
}
}

for (int i = N-1; i >= 0; --i) {
for (int k = i+1; k < N; ++k) {
x[i] -= m_mat(i,k) * x[k];
}
x[i] *= m_mat(i,i);
}
}

[[nodiscard]] AMREX_GPU_HOST_DEVICE
Array2D<T,0,N-1,0,N-1,Order::C> invert () const
{
Array2D<T,0,N-1,0,N-1,Order::C> IA;
for (int j = 0; j < N; ++j) {
for (int i = 0; i < N; ++i) {
IA(i,j) = (m_piv(i) == j) ? T(1.0) : T(0.0);
for (int k = 0; k < i; ++k) {
IA(i,j) -= m_mat(i,k) * IA(k,j);
}
}
for (int i = N-1; i >= 0; --i) {
for (int k = i+1; k < N; ++k) {
IA(i,j) -= m_mat(i,k) * IA(k,j);
}
IA(i,j) *= m_mat(i,i);
}
}
return IA;
}

[[nodiscard]] AMREX_GPU_HOST_DEVICE
T determinant () const
{
T det = m_mat(0,0);
for (int i = 1; i < N; ++i) {
det *= m_mat(i,i);
}
det = T(1.0) / det;
return (m_npivs % 2 == 0) ? det : -det;
}

private:

void define_innard ();

Array2D<T, 0, N-1, 0, N-1, Order::C> m_mat;
Array1D<int, 0, N-1> m_piv;
int m_npivs = 0;
};

template <int N, typename T>
LUSolver<N,T>::LUSolver (Array2D<T, 0, N-1, 0, N-1, Order::C> const& a_mat)
: m_mat(a_mat)
{
define_innard();
}

template <int N, typename T>
void LUSolver<N,T>::define (Array2D<T, 0, N-1, 0, N-1, Order::C> const& a_mat)
{
m_mat = a_mat;
define_innard();
}

template <int N, typename T>
void LUSolver<N,T>::define_innard ()
{
static_assert(N > 1);
static_assert(std::is_floating_point_v<T>);

for (int i = 0; i < N; ++i) { m_piv(i) = i; }
m_npivs = 0;

for (int i = 0; i < N; ++i) {
T maxA = 0;
int imax = i;

for (int k = i; k < N; ++k) {
auto const absA = std::abs(m_mat(k,i));
if (absA > maxA) {
maxA = absA;
imax = k;
}
}

if (maxA < std::numeric_limits<T>::min()) {
amrex::Abort("LUSolver: matrix is degenerate");
}

if (imax != i) {
std::swap(m_piv(i), m_piv(imax));
for (int j = 0; j < N; ++j) {
std::swap(m_mat(i,j), m_mat(imax,j));
}
++m_npivs;
}

for (int j = i+1; j < N; ++j) {
m_mat(j,i) /= m_mat(i,i);
for (int k = i+1; k < N; ++k) {
m_mat(j,k) -= m_mat(j,i) * m_mat(i,k);
}
}
}

for (int i = 0; i < N; ++i) {
m_mat(i,i) = T(1) / m_mat(i,i);
}
}

}

#endif
2 changes: 2 additions & 0 deletions Src/Base/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,8 @@ foreach(D IN LISTS AMReX_SPACEDIM)
Parser/amrex_iparser.tab.cpp
Parser/amrex_iparser.tab.nolint.H
Parser/amrex_iparser.tab.h
# Dense linear algebra solver using LU decomposition
AMReX_LUSolver.H
# AMReX Hydro -----------------------------------------------------
AMReX_Slopes_K.H
# Forward declaration -----------------------------------------------------
Expand Down
3 changes: 3 additions & 0 deletions Src/Base/Make.package
Original file line number Diff line number Diff line change
Expand Up @@ -326,5 +326,8 @@ CEXE_sources += AMReX_IParser_Exe.cpp
CEXE_headers += AMReX_IParser.H
CEXE_sources += AMReX_IParser.cpp

# Dense linear algebra solver using LU decomposition
CEXE_headers += AMReX_LUSolver.H

VPATH_LOCATIONS += $(AMREX_HOME)/Src/Base/Parser
INCLUDE_LOCATIONS += $(AMREX_HOME)/Src/Base/Parser
Loading

0 comments on commit f2bc520

Please sign in to comment.