Add more comments

Src/Boundary/AMReX_EdgeFluxRegister.H

@@ -9,7 +9,7 @@
 namespace amrex {
 
 /**
- * Edfe Flux Register for Constrained Transport
+ * Edge Flux Register for Constrained Transport
  *
  * This Flux Register is useful for solving system like dB/dt + curl E = 0
  * on a staggered mesh. (Here d is of course partial derivation.) B is a
@@ -19,12 +19,31 @@ namespace amrex {
  * At the beginning of a coarse step, `reset()` is called. In MFIter for
  * the coarse level advance, `CrseAdd` is called with coarse flux (i.e., E).
  * The flux is not scaled. In MFIter for the fine level advance, `FineAdd`
- * is called. After the fine level finished its time steps, `Reflux` is
- * called to update the coarse level B on the coarse/fine boundary.
+ * is called. After the fine level finishes its time steps, `Reflux` is
+ * called to update the coarse level B on the coarse/fine boundary. The user
+ * is also expected to call this version of average_down_faces from
+ * AMReX_MultiFabUtil.H to synchronize the coarse level data with the fine
+ * level.
+ *
+ * \vertbatim
+ template <typename MF, std::enable_if_t<IsFabArray<MF>::value,int>>
+ void average_down_faces (const Array<const MF*,AMREX_SPACEDIM>& fine,
+ const Array<MF*,AMREX_SPACEDIM>& crse,
+ const IntVect& ratio, const Geometry& crse_geom)
+ * \endverbatim
  *
  * Note that both CrseAdd and FineAdd are async for GPU builds. That means
  * it's the user's responsibility to keep the FArrayBox arguments alive or
- * call Gpu::streamSynchronize().
+ * call Gpu::streamSynchronize() when necessary.
+ *
+ * Because stagger grids are used, tiling could be very confusing. To avoid
+ * confusion, this class assumes that tiling is not enabled for the MFIter
+ * loop containing calls to CrseAdd and FineAdd.
+ *
+ * If the equation has an extra factor due to the choice of units, the
+ * factor can be absorbed into dt. If we have `v x B` instead of E, the sign
+ * can also been absorbed into dt. Note that whatever the choice of sign is,
+ * the dt arguments passed to CrseAdd and FineAdd should have the same sign.
  *
  * We try to keep the interface simple by not providing overloads that
  * specify the component index. If the user's data does not start with
@@ -79,7 +98,7 @@ private:
  // component. For example, at the x-faces, we need to store Ey and Ez.
  Array<Array<MultiFab,2>,AMREX_SPACEDIM*2> m_E_fine;
 
- // Mask on the coarse level indicating overlapping with m_E_fine
+ // Mask on the coarse level indicating overlap with m_E_fine
  Array<iMultiFab,AMREX_SPACEDIM> m_fine_mask;
 
 #else

Src/Boundary/AMReX_EdgeFluxRegister.cpp

@@ -26,6 +26,11 @@ void EdgeFluxRegister::define (const BoxArray& fba, const BoxArray& cba,
  AMREX_ALWAYS_ASSERT(fgeom.Domain() == amrex::refine(cgeom.Domain(), m_ratio));
  m_ncomp = nvar;
 
+ m_has_cf.define(cba, cdm);
+ for (MFIter mfi(m_has_cf, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi) {
+ m_has_cf[mfi] = 0;
+ }
+
 #if (AMREX_SPACEDIM == 3)
 
  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
@@ -45,11 +50,6 @@ void EdgeFluxRegister::define (const BoxArray& fba, const BoxArray& cba,
  }
  }
 
- m_has_cf.define(cba, cdm);
- for (MFIter mfi(m_has_cf, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi) {
- m_has_cf[mfi] = 0;
- }
-
  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
  Array<MultiFab const*,4> fmf;
  int count = 0;
@@ -93,11 +93,6 @@ void EdgeFluxRegister::define (const BoxArray& fba, const BoxArray& cba,
  m_E_fine[face].define(bba, fdm, nvar, 0);
  }
 
- m_has_cf.define(cba, cdm);
- for (MFIter mfi(m_has_cf, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi) {
- m_has_cf[mfi] = 0;
- }
-
  for (OrientationIter oit; oit.isValid(); ++oit) {
  auto face = oit();
  LayoutData<int> tmp_has_cf;