diff --git a/Src/Boundary/AMReX_EdgeFluxRegister.H b/Src/Boundary/AMReX_EdgeFluxRegister.H index 982137f3b3..a26ec4a164 100644 --- a/Src/Boundary/AMReX_EdgeFluxRegister.H +++ b/Src/Boundary/AMReX_EdgeFluxRegister.H @@ -9,7 +9,7 @@ namespace amrex { /** - * Edfe Flux Register for Constrained Transport + * Edge Flux Register for Constrained Transport * * This Flux Register is useful for solving system like dB/dt + curl E = 0 * on a staggered mesh. (Here d is of course partial derivation.) B is a @@ -19,12 +19,31 @@ namespace amrex { * At the beginning of a coarse step, `reset()` is called. In MFIter for * the coarse level advance, `CrseAdd` is called with coarse flux (i.e., E). * The flux is not scaled. In MFIter for the fine level advance, `FineAdd` - * is called. After the fine level finished its time steps, `Reflux` is - * called to update the coarse level B on the coarse/fine boundary. + * is called. After the fine level finishes its time steps, `Reflux` is + * called to update the coarse level B on the coarse/fine boundary. The user + * is also expected to call this version of average_down_faces from + * AMReX_MultiFabUtil.H to synchronize the coarse level data with the fine + * level. + * + * \vertbatim + template ::value,int>> + void average_down_faces (const Array& fine, + const Array& crse, + const IntVect& ratio, const Geometry& crse_geom) + * \endverbatim * * Note that both CrseAdd and FineAdd are async for GPU builds. That means * it's the user's responsibility to keep the FArrayBox arguments alive or - * call Gpu::streamSynchronize(). + * call Gpu::streamSynchronize() when necessary. + * + * Because stagger grids are used, tiling could be very confusing. To avoid + * confusion, this class assumes that tiling is not enabled for the MFIter + * loop containing calls to CrseAdd and FineAdd. + * + * If the equation has an extra factor due to the choice of units, the + * factor can be absorbed into dt. If we have `v x B` instead of E, the sign + * can also been absorbed into dt. Note that whatever the choice of sign is, + * the dt arguments passed to CrseAdd and FineAdd should have the same sign. * * We try to keep the interface simple by not providing overloads that * specify the component index. If the user's data does not start with @@ -79,7 +98,7 @@ private: // component. For example, at the x-faces, we need to store Ey and Ez. Array,AMREX_SPACEDIM*2> m_E_fine; - // Mask on the coarse level indicating overlapping with m_E_fine + // Mask on the coarse level indicating overlap with m_E_fine Array m_fine_mask; #else diff --git a/Src/Boundary/AMReX_EdgeFluxRegister.cpp b/Src/Boundary/AMReX_EdgeFluxRegister.cpp index 2e36dcffd1..871aeb59fb 100644 --- a/Src/Boundary/AMReX_EdgeFluxRegister.cpp +++ b/Src/Boundary/AMReX_EdgeFluxRegister.cpp @@ -26,6 +26,11 @@ void EdgeFluxRegister::define (const BoxArray& fba, const BoxArray& cba, AMREX_ALWAYS_ASSERT(fgeom.Domain() == amrex::refine(cgeom.Domain(), m_ratio)); m_ncomp = nvar; + m_has_cf.define(cba, cdm); + for (MFIter mfi(m_has_cf, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi) { + m_has_cf[mfi] = 0; + } + #if (AMREX_SPACEDIM == 3) for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -45,11 +50,6 @@ void EdgeFluxRegister::define (const BoxArray& fba, const BoxArray& cba, } } - m_has_cf.define(cba, cdm); - for (MFIter mfi(m_has_cf, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi) { - m_has_cf[mfi] = 0; - } - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { Array fmf; int count = 0; @@ -93,11 +93,6 @@ void EdgeFluxRegister::define (const BoxArray& fba, const BoxArray& cba, m_E_fine[face].define(bba, fdm, nvar, 0); } - m_has_cf.define(cba, cdm); - for (MFIter mfi(m_has_cf, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi) { - m_has_cf[mfi] = 0; - } - for (OrientationIter oit; oit.isValid(); ++oit) { auto face = oit(); LayoutData tmp_has_cf;