From 9b0d63e40412d04e5cf14c0d24efc3a0ec5263db Mon Sep 17 00:00:00 2001
From: Weiqun Zhang <weiqunzhang@lbl.gov>
Date: Thu, 28 Dec 2023 19:17:36 -0800
Subject: [PATCH] Disable optimization for computeRoundOffDomain

It appears dangerous to have optimization on when compiling the
Geometry::computeRoundOffDomain function. In the past, we had to change the
source code for Intel compilers. Recently, there appears to be issues with
ROCm 6.0 when optimization is on. Thus, we disable optimization for that
function.

Inline functions outsideRoundoffDomain and insideRoundoffDomain.
---
 Src/Base/AMReX_Geometry.H   | 18 ++++++++++++--
 Src/Base/AMReX_Geometry.cpp | 47 ++++++++++++++++++++++---------------
 2 files changed, 44 insertions(+), 21 deletions(-)

diff --git a/Src/Base/AMReX_Geometry.H b/Src/Base/AMReX_Geometry.H
index 550b42f2f6a..5932c4cbb36 100644
--- a/Src/Base/AMReX_Geometry.H
+++ b/Src/Base/AMReX_Geometry.H
@@ -408,7 +408,16 @@ public:
     *        are sure to be mapped to cells inside the Domain() box. Note that
     *        the same need not be true for all points inside ProbDomain().
     */
-    [[nodiscard]] bool outsideRoundoffDomain (AMREX_D_DECL(ParticleReal x, ParticleReal y, ParticleReal z)) const;
+    [[nodiscard]] bool outsideRoundoffDomain (AMREX_D_DECL(ParticleReal x, ParticleReal y, ParticleReal z)) const
+    {
+        bool outside = AMREX_D_TERM(x < roundoff_lo[0]
+                                 || x > roundoff_hi[0],
+                                 || y < roundoff_lo[1]
+                                 || y > roundoff_hi[1],
+                                 || z < roundoff_lo[2]
+                                 || z > roundoff_hi[2]);
+        return outside;
+    }
 
     /**
     * \brief Returns true if a point is inside the roundoff domain.
@@ -416,7 +425,10 @@ public:
     *        are sure to be mapped to cells inside the Domain() box. Note that
     *        the same need not be true for all points inside ProbDomain().
     */
-    [[nodiscard]] bool insideRoundoffDomain (AMREX_D_DECL(ParticleReal x, ParticleReal y, ParticleReal z)) const;
+    [[nodiscard]] bool insideRoundoffDomain (AMREX_D_DECL(ParticleReal x, ParticleReal y, ParticleReal z)) const
+    {
+        return !outsideRoundoffDomain(AMREX_D_DECL(x, y, z));
+    }
 
     /**
     * \brief Compute the roundoff domain. Public because it contains an
@@ -426,6 +438,8 @@ public:
 
 private:
     void read_params ();
+    void computeCellSize ();
+    void assertRoundoffDomain () const;
 
     // is_periodic and RealBox used to be static
     bool    is_periodic[AMREX_SPACEDIM] = {AMREX_D_DECL(false,false,false)};
diff --git a/Src/Base/AMReX_Geometry.cpp b/Src/Base/AMReX_Geometry.cpp
index ace1eefa663..7ed12012e4b 100644
--- a/Src/Base/AMReX_Geometry.cpp
+++ b/Src/Base/AMReX_Geometry.cpp
@@ -513,8 +513,7 @@ Geometry::growPeriodicDomain (int ngrow) const noexcept
     return growPeriodicDomain(IntVect(ngrow));
 }
 
-void
-Geometry::computeRoundoffDomain ()
+void Geometry::computeCellSize ()
 {
     for (int k = 0; k < AMREX_SPACEDIM; k++)
     {
@@ -522,6 +521,29 @@ Geometry::computeRoundoffDomain ()
         dx[k] = prob_domain.length(k)/(Real(domain.length(k)));
         inv_dx[k] = 1.0_rt/dx[k];
     }
+}
+
+void Geometry::assertRoundoffDomain () const
+{
+    for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
+        auto const idxlo = int(std::floor((roundoff_lo[idim]-ProbLo(idim)) * InvCellSize(idim)));
+        auto const idxhi = int(std::floor((roundoff_hi[idim]-ProbLo(idim)) * InvCellSize(idim)));
+        AMREX_ALWAYS_ASSERT(idxlo == 0 && idxhi == domain.length(idim)-1);
+    }
+}
+
+#if defined(__clang__)
+[[clang::optnone]]
+#elif defined(__GNUC__)
+__attribute__ ((optimize(0)))
+#elif defined(_WIN32)
+#define AMREX_WIN32_OPTIMIZE_OFF
+#pragma optimize("", off)
+#endif
+void
+Geometry::computeRoundoffDomain ()
+{
+    computeCellSize();
 
     constexpr int maxiters = 200;
 
@@ -684,24 +706,11 @@ Geometry::computeRoundoffDomain ()
                                 && iters < maxiters);
         }
     }
-}
 
-bool
-Geometry::outsideRoundoffDomain (AMREX_D_DECL(ParticleReal x, ParticleReal y, ParticleReal z)) const
-{
-    bool outside = AMREX_D_TERM(x < roundoff_lo[0]
-                             || x > roundoff_hi[0],
-                             || y < roundoff_lo[1]
-                             || y > roundoff_hi[1],
-                             || z < roundoff_lo[2]
-                             || z > roundoff_hi[2]);
-    return outside;
-}
-
-bool
-Geometry::insideRoundoffDomain (AMREX_D_DECL(ParticleReal x, ParticleReal y, ParticleReal z)) const
-{
-    return !outsideRoundoffDomain(AMREX_D_DECL(x, y, z));
+    assertRoundoffDomain();
 }
+#if defined(AMREX_WIN32_OPTIMIZE_OFF)
+#pragma optimize("", on)
+#endif
 
 }