From 00351c92ba555cee92939f36cb19be7f9731ffbd Mon Sep 17 00:00:00 2001
From: Theodore Kisner <tskisner@users.noreply.github.com>
Date: Thu, 15 Oct 2020 20:57:56 -0700
Subject: [PATCH] Fix a bug in the ground filter triggered when running without
 MPI. (#370)

* Fix a bug in the ground filter triggered when running without MPI.  Backport the toast3 environment variable for disabling MPI at runtime.

* Explicitly disable MPI in serial tests, to catch improper handling of passing None for world communicator.

* Update changelog
---
 .github/workflows/test.yml       |  6 ++---
 docs/changes.rst                 |  7 +++++
 src/toast/RELEASE                |  2 +-
 src/toast/mpi.py                 | 44 ++++++++++++++++++--------------
 src/toast/todmap/groundfilter.py |  6 ++++-
 5 files changed, 41 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 35a3b1c67..c7fb77db2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -26,7 +26,7 @@ jobs:
       - name: Test Documentation Build
         run: docker run -v "$(pwd)":/home/toast test_runner:py36 /home/toast/docs/build_docs.sh
       - name: Run Serial Tests
-        run: docker run test_runner:py36 python -c 'import toast.tests; toast.tests.run()'
+        run: docker run -e MPI_DISABLE=1 test_runner:py36 python -c 'import toast.tests; toast.tests.run()'
       - name: Run MPI Tests
         run: docker run test_runner:py36 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
   py37:
@@ -46,7 +46,7 @@ jobs:
       - name: Test Documentation Build
         run: docker run -v "$(pwd)":/home/toast test_runner:py37 /home/toast/docs/build_docs.sh
       - name: Run Serial Tests
-        run: docker run test_runner:py37 python -c 'import toast.tests; toast.tests.run()'
+        run: docker run -e MPI_DISABLE=1 test_runner:py37 python -c 'import toast.tests; toast.tests.run()'
       - name: Run MPI Tests
         run: docker run test_runner:py37 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
   py38:
@@ -66,6 +66,6 @@ jobs:
       - name: Test Documentation Build
         run: docker run -v "$(pwd)":/home/toast test_runner:py38 /home/toast/docs/build_docs.sh
       - name: Run Serial Tests
-        run: docker run test_runner:py38 python -c 'import toast.tests; toast.tests.run()'
+        run: docker run -e MPI_DISABLE=1 test_runner:py38 python -c 'import toast.tests; toast.tests.run()'
       - name: Run MPI Tests
         run: docker run test_runner:py38 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
diff --git a/docs/changes.rst b/docs/changes.rst
index 19253ec37..8000fc2f4 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -8,6 +8,13 @@ Change Log
 
 * No changes yet.
 
+2.3.9.post1 (2020-10-15)
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Run serial unit tests without MPI.  Fix bug in ground filter (PR `#370`_).
+
+.. _`#370`: https://github.com/hpc4cmb/toast/pull/370
+
 2.3.9 (2020-10-15)
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/src/toast/RELEASE b/src/toast/RELEASE
index c72101f67..dafb2aaff 100644
--- a/src/toast/RELEASE
+++ b/src/toast/RELEASE
@@ -1 +1 @@
-2.3.10a1
+2.3.9.post1
diff --git a/src/toast/mpi.py b/src/toast/mpi.py
index b2d7d4275..22dfbb0b9 100644
--- a/src/toast/mpi.py
+++ b/src/toast/mpi.py
@@ -16,25 +16,31 @@
 MPI = None
 
 if use_mpi is None:
-    # Special handling for running on a NERSC login node.  This is for convenience.
-    # The same behavior could be implemented with environment variables set in a
-    # shell resource file.
-    at_nersc = False
-    if "NERSC_HOST" in os.environ:
-        at_nersc = True
-    in_slurm = False
-    if "SLURM_JOB_NAME" in os.environ:
-        in_slurm = True
-    if not at_nersc or in_slurm:
-        try:
-            import mpi4py.MPI as MPI
-
-            use_mpi = True
-        except:
-            # There could be many possible exceptions raised...
-            log = Logger.get()
-            log.info("mpi4py not found- using serial operations only")
-            use_mpi = False
+    # See if the user has explicitly disabled MPI.
+    if "MPI_DISABLE" in os.environ:
+        use_mpi = False
+    else:
+        # Special handling for running on a NERSC login node.  This is for convenience.
+        # The same behavior could be implemented with environment variables set in a
+        # shell resource file.
+        at_nersc = False
+        if "NERSC_HOST" in os.environ:
+            at_nersc = True
+        in_slurm = False
+        if "SLURM_JOB_NAME" in os.environ:
+            in_slurm = True
+        if (not at_nersc) or in_slurm:
+            try:
+                import mpi4py.MPI as MPI
+
+                use_mpi = True
+            except:
+                # There could be many possible exceptions raised...
+                from ._libtoast import Logger
+
+                log = Logger.get()
+                log.info("mpi4py not found- using serial operations only")
+                use_mpi = False
 
 
 def get_world():
diff --git a/src/toast/todmap/groundfilter.py b/src/toast/todmap/groundfilter.py
index 5fde1f31d..872b23b47 100644
--- a/src/toast/todmap/groundfilter.py
+++ b/src/toast/todmap/groundfilter.py
@@ -167,7 +167,11 @@ def fit_templates(self, tod, det, templates, ref, good):
             ngood = np.sum(good)
         else:
             ngood = 0
-        ngood_tot = comm.allreduce(ngood)
+        ngood_tot = None
+        if comm is None:
+            ngood_tot = ngood
+        else:
+            ngood_tot = comm.allreduce(ngood)
         if ngood_tot == 0:
             return None