Application of Cat-B calibration (option1)

src/osa/configs/options.py

@@ -15,6 +15,8 @@
 warning = None
 nocheck = None
 no_dl2 = None
+no_dl1ab = None
+no_gainsel = None
 prod_id = None
 dl1_prod_id = None
 dl2_prod_id = None

src/osa/configs/sequencer.cfg

@@ -15,9 +15,11 @@ RUN_SUMMARY_DIR: %(MONITORING)s/RunSummary
 RUN_CATALOG: %(MONITORING)s/RunCatalog
 PEDESTAL_FINDER_DIR: %(BASE)s/auxiliary/PedestalFinder
 ANALYSIS_DIR: %(BASE)s/running_analysis
-CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration/Cat-A
-CALIB_DIR: %(CALIB_BASE_DIR)s/calibration
-PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline
+CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration
+CAT_A_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-A
+CAT_A_CALIB_DIR: %(CAT_A_CALIB_BASE)s/calibration
+CAT_A_PEDESTAL_DIR: %(CAT_A_CALIB_BASE)s/drs4_baseline
+CAT_B_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-B
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2
@@ -42,6 +44,7 @@ DL2_PROD_ID: model2
 # Calibration steps in calibration pipeline script
 drs4_baseline: onsite_create_drs4_pedestal_file
 charge_calibration: onsite_create_calibration_file
+catB_calibration: onsite_create_cat_B_calibration_file
 use_ff_heuristic_id: False
 
 # Data processing steps in datasequence script
@@ -54,6 +57,7 @@ dl1_to_dl2: lstchain_dl1_to_dl2
 dl1a_config: /software/lstchain/data/lstchain_standard_config.json
 store_image_dl1ab: True
 merge_dl1_datacheck: True
+apply_catB_calibration: True
 dl1b_config: /software/lstchain/data/lstchain_standard_config.json
 dl2_config: /software/lstchain/data/lstchain_standard_config.json
 rf_models: /data/models/prod5/zenith_20deg/20201023_v0.6.3

src/osa/job.py

@@ -109,6 +109,12 @@ def are_all_jobs_correctly_finished(sequence_list):
                     f"finished up to DL1ab, but --no-dl2 option selected"
                 )
                 continue
+            if out == 3 and options.no_dl1ab:
+                log.debug(
+                    f"Job {sequence.seq} ({sequence.type}) correctly "
+                    f"finished up to DL1A, but --no-dl1ab option selected"
+                )
+                continue
 
             log.warning(
                 f"Job {sequence.seq} (run {sequence.run}) not correctly finished [level {out}]"
@@ -215,6 +221,8 @@ def historylevel(history_file: Path, data_type: str):
                 # Data sequence
                 elif program == cfg.get("lstchain", "r0_to_dl1"):
                     level = 3 if exit_status == 0 else 4
+                #elif program == cfg.get("lstchain", "catB_calibration"):
+                #    level = 3 if exit_status == 0 else 4
                 elif program == cfg.get("lstchain", "dl1ab"):
                     if (exit_status == 0) and (prod_id == options.dl1_prod_id):
                         log.debug(f"DL1ab prod ID: {options.dl1_prod_id} already produced")
@@ -424,6 +432,8 @@ def data_sequence_job_template(sequence):
         commandargs.extend(("--config", f"{Path(options.configfile).resolve()}"))
     if sequence.type == "DATA" and options.no_dl2:
         commandargs.append("--no-dl2")
+    if sequence.type == "DATA" and options.no_dl1ab:
+        commandargs.append("--no-dl1ab")
 
     commandargs.extend(
         (

src/osa/paths.py

@@ -11,8 +11,7 @@
 import lstchain
 from astropy.table import Table
 from lstchain.onsite import (find_systematics_correction_file,
-                             find_time_calibration_file,
-                             find_filter_wheels)
+                             find_time_calibration_file)
 
 from osa.configs import options
 from osa.configs.config import DEFAULT_CFG, cfg
@@ -45,8 +44,8 @@
 
 
 DATACHECK_WEB_BASEDIR = Path(cfg.get("WEBSERVER", "DATACHECK"))
-CALIB_BASEDIR = Path(cfg.get("LST1", "CALIB_DIR"))
-DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+CALIB_BASEDIR = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
 
 def analysis_path(tel) -> Path:
@@ -136,18 +135,7 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
         return files[-1]  # Get the latest production among the major lstchain version
 
     date = utils.date_to_dir(get_run_date(run_id))
-
-    if options.test:  # Run tests avoiding the access to the database
-        options.filters = 52
-
-    else:
-        mongodb = cfg.get("database", "caco_db")
-        try:
-            # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB
-            options.filters = find_filter_wheels(int(run_id), mongodb)
-        except IOError:
-            log.warning("No filter information found in database. Assuming positions 52.")
-            options.filters = 52
+    options.filters = utils.get_calib_filters(run_id)
 
     return (
         CALIB_BASEDIR
@@ -156,6 +144,15 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
     ).resolve()
 
 
+def get_catB_calibration_filename(run_id: int) -> Path:
+    """Return the Category-B calibration filename of a given run."""
+    date = utils.date_to_dir(options.date)
+    calib_prod_id = utils.get_lstchain_version()
+    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / "calibration" / date / calib_prod_id
+    filters = utils.get_calib_filters(run_id)
+    return catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
+
+
 def pedestal_ids_file_exists(run_id: int) -> bool:
     """Look for the files with pedestal interleaved event identification."""
     pedestal_ids_dir = Path(cfg.get("LST1", "PEDESTAL_FINDER_DIR"))

src/osa/provenance/capture.py

@@ -53,7 +53,7 @@
 PROV_PREFIX = provconfig["PREFIX"]
 SUPPORTED_HASH_METHOD = ["md5"]
 SUPPORTED_HASH_BUFFER = ["content", "path"]
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 # global variables
 traced_entities = {}

src/osa/provenance/config/definition.yaml

@@ -200,6 +200,13 @@ activities:
             #   filepath: /fefs/aswg/data/real/DL1/20200218/v0.4.3_v00/
             #   size: 128
 
+    catB_calibration:
+        description: 
+            "Create Cat-B calibration file for an observation run"
+        parameters:
+        usage:
+        generation:
+
     dl1ab:
         description:
             "Create DL1AB files for an observation run"

src/osa/provenance/utils.py

@@ -10,7 +10,7 @@
 
 __all__ = ["parse_variables", "get_log_config"]
 
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 
 def parse_variables(class_instance):
@@ -43,8 +43,8 @@ def parse_variables(class_instance):
     rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS"))
     dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
     dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
-    calib_dir = Path(cfg.get("LST1", "CALIB_DIR"))
-    pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+    calib_dir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+    pedestal_dir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
     class_instance.SoftwareVersion = get_lstchain_version()
     class_instance.ProcessingConfigFile = str(options.configfile)
@@ -133,6 +133,9 @@ def parse_variables(class_instance):
         class_instance.InterleavedPedestalEventsFile = None
         if class_instance.args[6] is not None:
             class_instance.InterleavedPedestalEventsFile = str(Path(class_instance.args[6]))
+
+    if class_instance.__name__ == "catB_calibration":
+        class_instance.ObservationRun = class_instance.args[0].split(".")[0]
 
     if class_instance.__name__ == "dl1ab":
         # run_str       [0] 02006.0000

src/osa/scripts/calibration_pipeline.py

@@ -44,22 +44,26 @@ def is_calibration_produced(drs4_pedestal_run_id: int, pedcal_run_id: int) -> bo
 def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list:
     """Build the create_drs4_pedestal command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     return [
         "onsite_create_drs4_pedestal_file",
         f"--run_number={drs4_pedestal_run_id}",
         f"--base_dir={base_dir}",
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
 
 
 def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list:
     """Build the create_calibration_file command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     cmd = [
         "onsite_create_calibration_file",
         f"--pedestal_run={drs4_pedestal_run_id}",
         f"--run_number={pedcal_run_id}",
         f"--base_dir={base_dir}",
+        f"--r0-dir={r0_dir}",
     ]
     # In case of problems with trigger tagging:
     if cfg.getboolean("lstchain", "use_ff_heuristic_id"):

src/osa/scripts/datasequence.py

@@ -3,15 +3,19 @@
 import logging
 import sys
 from pathlib import Path
+import time
 
 from osa.configs import options
 from osa.configs.config import cfg
 from osa.job import historylevel
 from osa.workflow.stages import AnalysisStage
 from osa.provenance.capture import trace
+from osa.paths import get_catB_calibration_filename
 from osa.utils.cliopts import data_sequence_cli_parsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_dir
+from osa.utils.utils import date_to_dir, get_calib_filters
+from osa.nightsummary.extract import get_last_pedcalib
+
 
 __all__ = ["data_sequence", "r0_to_dl1", "dl1_to_dl2", "dl1ab", "dl1_datacheck"]
 
@@ -68,13 +72,17 @@ def data_sequence(
         log.info(f"Going to level {level}")
 
     if level == 3:
-        rc = dl1ab(run_str)
-        if cfg.getboolean("lstchain", "store_image_dl1ab"):
-            level -= 1
-            log.info(f"Going to level {level}")
-        else:
-            level -= 2
-            log.info(f"No images stored in dl1ab. Producing DL2. Going to level {level}")
+        if options.no_dl1ab:
+            level = 0
+            log.info(f"No DL1B are going to be produced. Going to level {level}")
+        else: 
+            rc = dl1ab(run_str)
+            if cfg.getboolean("lstchain", "store_image_dl1ab"):
+                level -= 1
+                log.info(f"Going to level {level}")
+            else:
+                level -= 2
+                log.info(f"No images stored in dl1ab. Producing DL2. Going to level {level}")
 
     if level == 2:
         rc = dl1_datacheck(run_str)
@@ -165,6 +173,55 @@ def r0_to_dl1(
     return analysis_step.rc
 
 
+@trace
+def catB_calibration(run_str: str) -> int:
+    """
+    Prepare and launch the lstchain script that creates the 
+    Category B calibration files. It should be executed runwise,
+    so it is only launched for the first subrun of each run.
+
+    Parameters
+    ----------
+    run_str: str
+
+    Returns
+    -------
+    rc: int
+        Return code of the executed command.
+    """
+    if run_str[-4:] != "0000":
+        log.debug(f"{run_str} is not the first subrun of the run, so the script "
+            "onsite_create_cat_B_calibration_file will not be launched for this subrun.")
+
+        catB_calibration_file = get_catB_calibration_filename(int(run_str[:5]))
+        n = 0
+        n_max = 10
+        while not catB_calibration_file.exists() and n<=n_max:
+            time.sleep(120)
+            n += 1
+        return 0
+
+    command = cfg.get("lstchain", "catB_calibration")
+    options.filters = get_calib_filters(int(run_str[:5])) 
+    base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
+    catA_calib_run = get_last_pedcalib(options.date)
+    cmd = [
+        command,
+        f"--run_number={run_str[:5]}",
+        f"--catA_calibration_run={catA_calib_run}",
+        f"--base_dir={base_dir}",
+        f"--r0-dir={r0_dir}",
+        f"--filters={options.filters}",
+    ]
+    if options.simulate:
+        return 0
+
+    analysis_step = AnalysisStage(run=run_str, command_args=cmd)
+    analysis_step.execute()
+    return analysis_step.rc
+
+
 @trace
 def dl1ab(run_str: str) -> int:
     """
@@ -189,7 +246,7 @@ def dl1ab(run_str: str) -> int:
     input_dl1_datafile = Path(options.directory) / f"dl1_LST-1.Run{run_str}.h5"
     # DL1b output file to be stored in the dl1ab subdirectory
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
-
+    
     # Prepare and launch the actual lstchain script
     command = cfg.get("lstchain", "dl1ab")
     cmd = [
@@ -198,9 +255,14 @@ def dl1ab(run_str: str) -> int:
         f"--output-file={output_dl1_datafile}",
         f"--config={dl1b_config}",
     ]
+
     if not cfg.getboolean("lstchain", "store_image_dl1ab"):
         cmd.append("--no-image=True")
 
+    if cfg.getboolean("lstchain", "apply_catB_calibration"):
+        catB_calibration_file = get_catB_calibration_filename(int(run_str[:5]))
+        cmd.append(f"--catB-calibration-file={catB_calibration_file}")
+
     if options.simulate:
         return 0
 

src/osa/scripts/tests/test_osa_scripts.py

@@ -298,10 +298,12 @@ def test_drs4_pedestal_cmd(base_test_dir):
     from osa.scripts.calibration_pipeline import drs4_pedestal_command
 
     cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804")
+    r0_dir = base_test_dir / "R0"
     expected_command = [
         "onsite_create_drs4_pedestal_file",
         "--run_number=01804",
         f"--base_dir={base_test_dir}",
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
     assert cmd == expected_command
@@ -311,11 +313,13 @@ def test_calibration_file_cmd(base_test_dir):
     from osa.scripts.calibration_pipeline import calibration_file_command
 
     cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809")
+    r0_dir = base_test_dir / "R0"
     expected_command = [
         "onsite_create_calibration_file",
         "--pedestal_run=01804",
         "--run_number=01809",
         f"--base_dir={base_test_dir}",
+        f"--r0-dir={r0_dir}",
     ]
     assert cmd == expected_command