Add Cat-B calibration to OSA

src/osa/configs/sequencer.cfg

@@ -15,9 +15,11 @@ RUN_SUMMARY_DIR: %(MONITORING)s/RunSummary
 RUN_CATALOG: %(MONITORING)s/RunCatalog
 PEDESTAL_FINDER_DIR: %(BASE)s/auxiliary/PedestalFinder
 ANALYSIS_DIR: %(BASE)s/running_analysis
-CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration/Cat-A
-CALIB_DIR: %(CALIB_BASE_DIR)s/calibration
-PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline
+CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration
+CAT_A_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-A
+CAT_A_CALIB_DIR: %(CAT_A_CALIB_BASE)s/calibration
+CAT_A_PEDESTAL_DIR: %(CAT_A_CALIB_BASE)s/drs4_baseline
+CAT_B_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-B
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2

src/osa/paths.py

@@ -45,8 +45,8 @@
 
 
 DATACHECK_WEB_BASEDIR = Path(cfg.get("WEBSERVER", "DATACHECK"))
-CALIB_BASEDIR = Path(cfg.get("LST1", "CALIB_DIR"))
-DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+CALIB_BASEDIR = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
 
 def analysis_path(tel) -> Path:

src/osa/provenance/capture.py

@@ -53,7 +53,7 @@
 PROV_PREFIX = provconfig["PREFIX"]
 SUPPORTED_HASH_METHOD = ["md5"]
 SUPPORTED_HASH_BUFFER = ["content", "path"]
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 # global variables
 traced_entities = {}

src/osa/provenance/config/definition.yaml

@@ -200,6 +200,13 @@ activities:
             #   filepath: /fefs/aswg/data/real/DL1/20200218/v0.4.3_v00/
             #   size: 128
 
+    catB_calibration:
+        description: 
+            "Create Cat-B calibration file for an observation run"
+        parameters:
+        usage:
+        generation:
+
     dl1ab:
         description:
             "Create DL1AB files for an observation run"

src/osa/provenance/utils.py

@@ -10,7 +10,7 @@
 
 __all__ = ["parse_variables", "get_log_config"]
 
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 
 def parse_variables(class_instance):
@@ -43,8 +43,8 @@ def parse_variables(class_instance):
     rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS"))
     dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
     dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
-    calib_dir = Path(cfg.get("LST1", "CALIB_DIR"))
-    pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+    calib_dir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+    pedestal_dir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
     class_instance.SoftwareVersion = get_lstchain_version()
     class_instance.ProcessingConfigFile = str(options.configfile)
@@ -133,6 +133,9 @@ def parse_variables(class_instance):
         class_instance.InterleavedPedestalEventsFile = None
         if class_instance.args[6] is not None:
             class_instance.InterleavedPedestalEventsFile = str(Path(class_instance.args[6]))
+
+    if class_instance.__name__ == "catB_calibration":
+        class_instance.ObservationRun = class_instance.args[0].split(".")[0]
 
     if class_instance.__name__ == "dl1ab":
         # run_str       [0] 02006.0000

src/osa/scripts/calibration_pipeline.py

@@ -44,22 +44,26 @@ def is_calibration_produced(drs4_pedestal_run_id: int, pedcal_run_id: int) -> bo
 def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list:
     """Build the create_drs4_pedestal command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     return [
         "onsite_create_drs4_pedestal_file",
         f"--run_number={drs4_pedestal_run_id}",
         f"--base_dir={base_dir}",
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
 
 
 def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list:
     """Build the create_calibration_file command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     cmd = [
         "onsite_create_calibration_file",
         f"--pedestal_run={drs4_pedestal_run_id}",
         f"--run_number={pedcal_run_id}",
         f"--base_dir={base_dir}",
+        f"--r0-dir={r0_dir}",
     ]
     # In case of problems with trigger tagging:
     if cfg.getboolean("lstchain", "use_ff_heuristic_id"):

src/osa/scripts/datasequence.py

@@ -9,9 +9,11 @@
 from osa.job import historylevel
 from osa.workflow.stages import AnalysisStage
 from osa.provenance.capture import trace
+from osa.paths import get_major_version
 from osa.utils.cliopts import data_sequence_cli_parsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_dir
+from osa.utils.utils import date_to_dir, get_lstchain_version
+
 
 __all__ = ["data_sequence", "r0_to_dl1", "dl1_to_dl2", "dl1ab", "dl1_datacheck"]
 
@@ -50,10 +52,10 @@ def data_sequence(
     history_file = Path(options.directory) / f"sequence_{options.tel_id}_{run_str}.history"
     # Set the starting level and corresponding return code from last analysis step
     # registered in the history file.
-    level, rc = (4, 0) if options.simulate else historylevel(history_file, "DATA")
+    level, rc = (5, 0) if options.simulate else historylevel(history_file, "DATA")
     log.info(f"Going to level {level}")
 
-    if level == 4:
+    if level == 5:
         rc = r0_to_dl1(
             calibration_file,
             pedestal_file,
@@ -67,6 +69,11 @@ def data_sequence(
         level -= 1
         log.info(f"Going to level {level}")
 
+    if level == 4:
+        rc = catB_calibration(run_str)
+        level -= 1
+        log.info(f"Going to level {level}")
+
     if level == 3:
         rc = dl1ab(run_str)
         if cfg.getboolean("lstchain", "store_image_dl1ab"):
@@ -165,6 +172,43 @@ def r0_to_dl1(
     return analysis_step.rc
 
 
+@trace
+def catB_calibration(run_str: str) -> int:
+    """
+    Prepare and launch the lstchain script that creates the 
+    Category B calibration files. It should be executed runwise,
+    so it is only launched for the first subrun of each run.
+
+    Parameters
+    ----------
+    run_str: str
+
+    Returns
+    -------
+    rc: int
+        Return code of the executed command.
+    """
+    if run_str[-4:] != "0000":
+        log.debug(f"{run_str} is not the first subrun of the run, so the script "
+            "onsite_create_cat_B_calibration_file will not be launched for this subrun.")
+        return 0
+
+    base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
+    cmd = [
+        "onsite_create_cat_B_calibration_file",
+        f"--run_number={run_str[:5]}",
+        f"--base_dir={base_dir}",
+        f"--r0-dir={r0_dir}",
+    ]
+    if options.simulate:
+        return 0
+
+    analysis_step = AnalysisStage(run=run_str, command_args=cmd)
+    analysis_step.execute()
+    return analysis_step.rc
+
+
 @trace
 def dl1ab(run_str: str) -> int:
     """
@@ -189,6 +233,10 @@ def dl1ab(run_str: str) -> int:
     input_dl1_datafile = Path(options.directory) / f"dl1_LST-1.Run{run_str}.h5"
     # DL1b output file to be stored in the dl1ab subdirectory
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
+    night_dir = date_to_dir(options.date)
+    calib_prod_id = get_major_version(get_lstchain_version())
+    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / night_dir / calib_prod_id
+    catB_calibration_file = catB_calib_dir / f"cat_B_calibration_filters_{options.filters}.Run{run_str[:5]}.h5"
 
     # Prepare and launch the actual lstchain script
     command = cfg.get("lstchain", "dl1ab")
@@ -197,6 +245,7 @@ def dl1ab(run_str: str) -> int:
         f"--input-file={input_dl1_datafile}",
         f"--output-file={output_dl1_datafile}",
         f"--config={dl1b_config}",
+        f"--catB-calibration-file={catB_calibration_file}",
     ]
     if not cfg.getboolean("lstchain", "store_image_dl1ab"):
         cmd.append("--no-image=True")

src/osa/scripts/tests/test_osa_scripts.py

@@ -298,10 +298,12 @@ def test_drs4_pedestal_cmd(base_test_dir):
     from osa.scripts.calibration_pipeline import drs4_pedestal_command
 
     cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804")
+    r0_dir = base_test_dir / "R0"
     expected_command = [
         "onsite_create_drs4_pedestal_file",
         "--run_number=01804",
         f"--base_dir={base_test_dir}",
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
     assert cmd == expected_command
@@ -311,11 +313,13 @@ def test_calibration_file_cmd(base_test_dir):
     from osa.scripts.calibration_pipeline import calibration_file_command
 
     cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809")
+    r0_dir = base_test_dir / "R0"
     expected_command = [
         "onsite_create_calibration_file",
         "--pedestal_run=01804",
         "--run_number=01809",
         f"--base_dir={base_test_dir}",
+        f"--r0-dir={r0_dir}",
     ]
     assert cmd == expected_command
 

src/osa/workflow/stages.py

@@ -87,7 +87,7 @@ def _clean_up(self):
             self._remove_drs4_baseline()
 
     def _remove_drs4_baseline(self):
-        drs4_pedestal_basedir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+        drs4_pedestal_basedir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
         date = date_to_dir(get_run_date(self.run))
         drs4_pedestal_dir = drs4_pedestal_basedir / date / lstchain.__version__
         file = drs4_pedestal_dir / "drs4_pedestal.Run{self.run}.0000.h5"
@@ -97,7 +97,7 @@ def _remove_drs4_baseline(self):
         drs4_pedestal_dir_pro.unlink(missing_ok=True)
 
     def _remove_calibration(self):
-        calib_basedir = Path(cfg.get("LST1", "CALIB_DIR"))
+        calib_basedir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
         date = date_to_dir(get_run_date(self.run))
         calib_dir = file = calib_basedir / date / lstchain.__version__
         file = calib_dir / f"calibration_filters_{options.filters}.Run{self.run}.0000.h5"