Hotfix to handle cases when exposures are not found after being taken

src/gort/enums.py

@@ -46,6 +46,7 @@ class ErrorCode(Enum):
     SCHEDULER_CANNOT_FIND_TILE = 703
     OBSERVER_ERROR = 800
     ACQUISITION_FAILED = 801
+    CALIBRATION_ERROR = 900
     UNKNOWN_ERROR = 9999
 
 

src/gort/exposure.py

@@ -366,24 +366,33 @@ def stop_timer(self):
             self._progress.console.clear_live()
         self._progress = None
 
-    def verify_files(self):
+    async def verify_files(self):
         """Checks that the files have been written and have the right contents."""
 
         config = self.specs.gort.config["specs"]
 
         HEADERS_CRITICAL = config["verification"]["headers"]["critical"]
         HEADERS_WARNING = config["verification"]["headers"]["warning"]
 
-        files = self.get_files()
+        for retry in range(3):
+            # Very infrequently we have cases in which this check fails even if the
+            # files are there. It may be a delay in the NFS disk or a race condition
+            # somewhere. As a hotfile, we try three times with a delay before giving up.
 
-        n_spec = len(self.devices) if self.devices else len(config["devices"])
-        n_files_expected = n_spec * 3
+            files = self.get_files()
 
-        if (n_files_found := len(files)) < n_files_expected:
-            raise RuntimeError(
-                f"Expected {n_files_expected} files but found {n_files_found}. "
-                "Verification failed."
-            )
+            n_spec = len(self.devices) if self.devices else len(config["devices"])
+            n_files_expected = n_spec * 3
+
+            if (n_files_found := len(files)) < n_files_expected:
+                if retry < 2:
+                    await asyncio.sleep(3)
+                    continue
+
+                raise RuntimeError(
+                    f"Expected {n_files_expected} files but found {n_files_found}. "
+                    "Verification failed."
+                )
 
         for file in files:
             header = fits.getheader(str(file))
@@ -434,7 +443,7 @@ async def _done_monitor(self):
             if "ERROR" in reply["status_names"]:
                 self.error = True
 
-        self.verify_files()
+        await self.verify_files()
 
         # Set the Future.
         self.set_result(self)

src/gort/recipes/calibration.py

@@ -14,6 +14,8 @@
 import numpy
 from astropy.time import Time
 
+from gort.enums import ErrorCode, Event
+from gort.pubsub import notify_event
 from gort.tools import get_ephemeris_summary
 
 from .base import BaseRecipe
@@ -254,11 +256,26 @@ async def recipe(
 
             fibre_str = await self.goto_fibre_position(n_fibre, secondary=secondary)
             gort.log.info(f"Taking {fibre_str} exposure with exp_time={exp_time:.2f}.")
-            await gort.specs.expose(
-                exp_time,
-                flavour="flat",
-                header={"CALIBFIB": fibre_str},
-            )
+
+            try:
+                await gort.specs.expose(
+                    exp_time,
+                    flavour="flat",
+                    header={"CALIBFIB": fibre_str},
+                )
+            except Exception as err:
+                gort.log.error(
+                    "Error taking twilight flat exposure. Will ignore since we "
+                    f"are on a schedule here. Error is: {err}"
+                )
+                await notify_event(
+                    Event.ERROR,
+                    payload={
+                        "error": str(err),
+                        "error_code": ErrorCode.CALIBRATION_ERROR.value,
+                        "ignore": True,
+                    },
+                )
 
             n_observed += 1
             if n_observed == 12: