From ecd22852eb64c445d7bf214451704d83a1409c9d Mon Sep 17 00:00:00 2001
From: "Michael T. Campbell" <mtcampbe@illinois.edu>
Date: Fri, 28 Jan 2022 09:19:07 -0800
Subject: [PATCH 1/2] Update for parallel-distributed-lazy

---
 combustor.py | 56 ++++++++++++++++++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 19 deletions(-)

diff --git a/combustor.py b/combustor.py
index 9782132..d64b075 100644
--- a/combustor.py
+++ b/combustor.py
@@ -40,11 +40,9 @@
 from functools import partial
 
 
-from meshmode.array_context import (
-    PyOpenCLArrayContext,
-    SingleGridWorkBalancingPytatoArrayContext as PytatoPyOpenCLArrayContext
-    #PytatoPyOpenCLArrayContext
-)
+from grudge.array_context import MPIPytatoPyOpenCLArrayContext as PytatoPyOpenCLArrayContext
+from meshmode.array_context import PyOpenCLArrayContext
+
 from mirgecom.profiling import PyOpenCLProfilingArrayContext
 from arraycontext import thaw, freeze, flatten, unflatten, to_numpy, from_numpy
 from meshmode.mesh import BTAG_ALL, BTAG_NONE  # noqa
@@ -104,19 +102,19 @@ def filter(self, record):
             return (record.levelno == self.passlevel)
 
 
-h1 = logging.StreamHandler(sys.stdout)
-f1 = SingleLevelFilter(logging.INFO, False)
-h1.addFilter(f1)
-root_logger = logging.getLogger()
-root_logger.addHandler(h1)
-h2 = logging.StreamHandler(sys.stderr)
-f2 = SingleLevelFilter(logging.INFO, True)
-h2.addFilter(f2)
-root_logger.addHandler(h2)
+# h1 = logging.StreamHandler(sys.stdout)
+# f1 = SingleLevelFilter(logging.INFO, False)
+# h1.addFilter(f1)
+#root_logger = logging.getLogger()
+# root_logger.addHandler(h1)
+# h2 = logging.StreamHandler(sys.stderr)
+# f2 = SingleLevelFilter(logging.INFO, True)
+# h2.addFilter(f2)
+# root_logger.addHandler(h2)
 
 logger = logging.getLogger(__name__)
 #logger = logging.getLogger("my.logger")
-logger.setLevel(logging.DEBUG)
+# logger.setLevel(logging.INFO)
 #logger.debug("A DEBUG message")
 #logger.info("An INFO message")
 #logger.warning("A WARNING message")
@@ -798,6 +796,7 @@ def __call__(self, x_vec, *, eos, **kwargs):
 @mpi_entry_point
 def main(ctx_factory=cl.create_some_context, restart_filename=None,
          use_profiling=False, use_logmgr=True, user_input_file=None,
+         use_overintegration=False,
          actx_class=PyOpenCLArrayContext, casename=None):
     """Drive the Y0 example."""
     cl_ctx = ctx_factory()
@@ -824,9 +823,12 @@ def main(ctx_factory=cl.create_some_context, restart_filename=None,
         queue = cl.CommandQueue(cl_ctx)
 
     # main array context for the simulation
-    actx = actx_class(
-        queue,
-        allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
+    if actx_class == PytatoPyOpenCLArrayContext:
+        actx = actx_class(comm, queue, mpi_base_tag=1200)
+    else:
+        actx = actx_class(
+            queue,
+            allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
 
     # an array context for things that just can't lazy
     init_actx = PyOpenCLArrayContext(queue,
@@ -1340,9 +1342,25 @@ def _injection_state_func(discr, btag, gas_model, state_minus, **kwargs):
     if rank == 0:
         logging.info("Making discretization")
 
+    from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD
+    from meshmode.discretization.poly_element import \
+        default_simplex_group_factory, QuadratureSimplexGroupFactory
+
     discr = EagerDGDiscretization(
-        actx, local_mesh, order=order, mpi_communicator=comm
+        actx, local_mesh,
+        discr_tag_to_group_factory={
+            DISCR_TAG_BASE: default_simplex_group_factory(
+                base_dim=local_mesh.dim, order=order),
+            DISCR_TAG_QUAD: QuadratureSimplexGroupFactory(2*order + 1)
+        },
+        mpi_communicator=comm
     )
+
+    if use_overintegration:
+        quadrature_tag = DISCR_TAG_QUAD
+    else:
+        quadrature_tag = None
+
     if rank == 0:
         logging.info("Done making discretization")
 

From 55e18dd9b642f498ed8144feb2f9c0e4059183e7 Mon Sep 17 00:00:00 2001
From: "Michael T. Campbell" <mtcampbe@illinois.edu>
Date: Fri, 28 Jan 2022 10:54:45 -0800
Subject: [PATCH 2/2] Update params to avoid health crash

---
 smoke_test/run_params.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/smoke_test/run_params.yaml b/smoke_test/run_params.yaml
index 550a363..3abe85e 100644
--- a/smoke_test/run_params.yaml
+++ b/smoke_test/run_params.yaml
@@ -3,7 +3,7 @@ nrestart: 10
 nhealth: 1
 nstatus: 1
 current_dt: 1.0e-9
-t_final: 2.e-8
+t_final: 5.e-9
 order:  3
 alpha_sc: 0.01
 s0_sc: -5.0