From 4915cc1ea7e43b036ad03c4a819b60a2d98e16a0 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Tue, 28 May 2024 16:00:06 +0200
Subject: [PATCH 01/23] Add new step in datasequence to produce the CatB
 calibration files

---
 src/osa/scripts/datasequence.py | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 0a4fe02d..ee224304 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -53,7 +53,7 @@ def data_sequence(
     level, rc = (4, 0) if options.simulate else historylevel(history_file, "DATA")
     log.info(f"Going to level {level}")
 
-    if level == 4:
+    if level == 5:
         rc = r0_to_dl1(
             calibration_file,
             pedestal_file,
@@ -67,6 +67,12 @@ def data_sequence(
         level -= 1
         log.info(f"Going to level {level}")
 
+    if level == 4:
+        rc = catB_calibration(
+        )
+        level -= 1
+        log.info(f"Going to level {level}")
+
     if level == 3:
         rc = dl1ab(run_str)
         if cfg.getboolean("lstchain", "store_image_dl1ab"):
@@ -165,6 +171,27 @@ def r0_to_dl1(
     return analysis_step.rc
 
 
+@trace
+def catB_calibration(run_str: str) -> int:
+    if run_str[-4:] != "0000":  #not first subrun
+        return
+
+    base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
+    cmd = [
+        "onsite_create_cat_B_calibration_file",
+        f"--run_number={run_str[:5]}",
+        f"--base_dir={base_dir}",
+        f"--r0-dir={r0_dir}",
+    ]
+    if options.simulate:
+        return 0
+
+    analysis_step = AnalysisStage(run=run_str, command_args=cmd)
+    analysis_step.execute()
+    return analysis_step.rc
+
+
 @trace
 def dl1ab(run_str: str) -> int:
     """

From 53d57b38a5e402527fc10c0a6c204b9cafbfb4bb Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Tue, 28 May 2024 16:07:54 +0200
Subject: [PATCH 02/23] add r0-dir to the calibration scripts (e.g. in case R0G
 is used)

---
 src/osa/scripts/calibration_pipeline.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/osa/scripts/calibration_pipeline.py b/src/osa/scripts/calibration_pipeline.py
index 51d43080..7736a648 100644
--- a/src/osa/scripts/calibration_pipeline.py
+++ b/src/osa/scripts/calibration_pipeline.py
@@ -44,10 +44,12 @@ def is_calibration_produced(drs4_pedestal_run_id: int, pedcal_run_id: int) -> bo
 def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list:
     """Build the create_drs4_pedestal command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     return [
         "onsite_create_drs4_pedestal_file",
         f"--run_number={drs4_pedestal_run_id}",
         f"--base_dir={base_dir}",
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
 
@@ -55,11 +57,13 @@ def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list:
 def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list:
     """Build the create_calibration_file command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     cmd = [
         "onsite_create_calibration_file",
         f"--pedestal_run={drs4_pedestal_run_id}",
         f"--run_number={pedcal_run_id}",
         f"--base_dir={base_dir}",
+        f"--r0-dir={r0_dir}",
     ]
     # In case of problems with trigger tagging:
     if cfg.getboolean("lstchain", "use_ff_heuristic_id"):

From 0b43696d2ac8f088bf12180243f50cc8e9f98f35 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Tue, 28 May 2024 16:26:55 +0200
Subject: [PATCH 03/23] Add CatB calibration file as input in dl1ab step

---
 src/osa/scripts/datasequence.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index ee224304..058d215f 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -216,6 +216,10 @@ def dl1ab(run_str: str) -> int:
     input_dl1_datafile = Path(options.directory) / f"dl1_LST-1.Run{run_str}.h5"
     # DL1b output file to be stored in the dl1ab subdirectory
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
+    night_dir = date_to_dir(options.date)
+    calib_prod_id = cfg.get("LST1", "CALIB_PROD_ID")
+    catB_calib_dir = Path(cfg.get("LST1", "CATB_CALIB_DIR")) / night_dir / calib_prod_id
+    catB_calibration_file = catB_calib_dir / f"cat_B_calibration_filters_{options.filters}.Run{run_str[:5]}.h5"
 
     # Prepare and launch the actual lstchain script
     command = cfg.get("lstchain", "dl1ab")
@@ -224,6 +228,7 @@ def dl1ab(run_str: str) -> int:
         f"--input-file={input_dl1_datafile}",
         f"--output-file={output_dl1_datafile}",
         f"--config={dl1b_config}",
+        f"--catB-calibration-file={catB_calibration_file}",
     ]
     if not cfg.getboolean("lstchain", "store_image_dl1ab"):
         cmd.append("--no-image=True")

From 95290dad4d435e564a97ba8d93977b1c981f69d4 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Tue, 28 May 2024 16:50:18 +0200
Subject: [PATCH 04/23] adapt configuration file

---
 src/osa/configs/sequencer.cfg | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg
index 6deeae3d..407b7cdf 100644
--- a/src/osa/configs/sequencer.cfg
+++ b/src/osa/configs/sequencer.cfg
@@ -15,9 +15,10 @@ RUN_SUMMARY_DIR: %(MONITORING)s/RunSummary
 RUN_CATALOG: %(MONITORING)s/RunCatalog
 PEDESTAL_FINDER_DIR: %(BASE)s/auxiliary/PedestalFinder
 ANALYSIS_DIR: %(BASE)s/running_analysis
-CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration/Cat-A
-CALIB_DIR: %(CALIB_BASE_DIR)s/calibration
-PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline
+CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration
+CALIB_DIR: %(CALIB_BASE_DIR)s/Cat-A/calibration
+PEDESTAL_DIR: %(CALIB_BASE_DIR)s/Cat-A/drs4_baseline
+CATB_CALIB_DIR: %(CALIB_BASE_DIR)s/Cat-B
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2

From 0828b268ec28d707f7766e95d927f367b1174292 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Tue, 28 May 2024 17:06:17 +0200
Subject: [PATCH 05/23] add docstring

---
 src/osa/scripts/datasequence.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 058d215f..91172452 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -173,7 +173,23 @@ def r0_to_dl1(
 
 @trace
 def catB_calibration(run_str: str) -> int:
-    if run_str[-4:] != "0000":  #not first subrun
+    """
+    Prepare and launch the lstchain script that creates the 
+    Category B calibration files. It should be executed runwise,
+    so it is only launched for the first subrun of each run.
+
+    Parameters
+    ----------
+    run_str: str
+
+    Returns
+    -------
+    rc: int
+        Return code of the executed command.
+    """
+    if run_str[-4:] != "0000":
+        log.debug(f"{run_str} is not the first subrun of the run, so the script "
+            "onsite_create_cat_B_calibration_file will not be launched for this subrun.")
         return
 
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()

From 763ec648070489988780cc4e8b8e80a6ad7a4fea Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Wed, 29 May 2024 14:11:00 +0200
Subject: [PATCH 06/23] adapt tests

---
 src/osa/scripts/tests/test_osa_scripts.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py
index bdfa153f..d6b4fe38 100644
--- a/src/osa/scripts/tests/test_osa_scripts.py
+++ b/src/osa/scripts/tests/test_osa_scripts.py
@@ -298,10 +298,12 @@ def test_drs4_pedestal_cmd(base_test_dir):
     from osa.scripts.calibration_pipeline import drs4_pedestal_command
 
     cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804")
+    r0_dir = base_test_dir / "R0"
     expected_command = [
         "onsite_create_drs4_pedestal_file",
         "--run_number=01804",
         f"--base_dir={base_test_dir}",
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
     assert cmd == expected_command
@@ -311,11 +313,13 @@ def test_calibration_file_cmd(base_test_dir):
     from osa.scripts.calibration_pipeline import calibration_file_command
 
     cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809")
+    r0_dir = base_test_dir / "R0"
     expected_command = [
         "onsite_create_calibration_file",
         "--pedestal_run=01804",
         "--run_number=01809",
         f"--base_dir={base_test_dir}",
+        f"--r0-dir={r0_dir}",
     ]
     assert cmd == expected_command
 

From a3f0e1abc156bfa64faab24abab5790434a6652c Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Wed, 29 May 2024 14:18:18 +0200
Subject: [PATCH 07/23] add argument in catB_calibration function

---
 src/osa/scripts/datasequence.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 91172452..fb9fee89 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -68,8 +68,7 @@ def data_sequence(
         log.info(f"Going to level {level}")
 
     if level == 4:
-        rc = catB_calibration(
-        )
+        rc = catB_calibration(run_str)
         level -= 1
         log.info(f"Going to level {level}")
 

From 9e9193c8d48cfd12af2a57902b08b1e9673b8430 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Wed, 29 May 2024 15:05:56 +0200
Subject: [PATCH 08/23] adapt tests

---
 src/osa/scripts/tests/test_osa_scripts.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py
index d6b4fe38..0c5814cb 100644
--- a/src/osa/scripts/tests/test_osa_scripts.py
+++ b/src/osa/scripts/tests/test_osa_scripts.py
@@ -99,17 +99,17 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 19
-    assert len(dl1["activity"]) == 5
-    assert len(dl1["used"]) == 15
-    assert len(dl1["wasGeneratedBy"]) == 10
+    assert len(dl1["entity"]) == 7
+    assert len(dl1["activity"]) == 2
+    assert len(dl1["used"]) == 3
+    assert len(dl1["wasGeneratedBy"]) == 4
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)
-    assert len(dl2["entity"]) == 25
-    assert len(dl2["activity"]) == 6
-    assert len(dl2["used"]) == 21
-    assert len(dl2["wasGeneratedBy"]) == 12
+    assert len(dl2["entity"]) == 7
+    assert len(dl2["activity"]) == 2
+    assert len(dl2["used"]) == 3
+    assert len(dl2["wasGeneratedBy"]) == 4
 
     rc = run_program("simulate_processing", "-p")
     assert rc.returncode == 0

From 5ac37702ae9164554b404b29e65c5361f5e2a1ce Mon Sep 17 00:00:00 2001
From: marialainez <malainez@ucm.es>
Date: Mon, 3 Jun 2024 13:19:19 +0000
Subject: [PATCH 09/23] fix small issues

---
 src/osa/configs/sequencer.cfg   | 1 +
 src/osa/scripts/datasequence.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg
index 407b7cdf..2ad351b5 100644
--- a/src/osa/configs/sequencer.cfg
+++ b/src/osa/configs/sequencer.cfg
@@ -33,6 +33,7 @@ GAIN_SELECTION_FLAG_DIR: %(OSA_DIR)s/GainSel
 
 # To be set by the user. Using PROD-ID will overcome the automatic
 # fetching of lstchain version. Otherwise leave it empty (and without the colon symbol).
+CALIB_PROD_ID: v0.1.0
 PROD_ID: v0.1.0
 # Change this to produce a different DL1b or DL2 sub-productions.
 # Otherwise, keep it empty to use the common PROD-ID
diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index fb9fee89..9761b111 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -50,7 +50,7 @@ def data_sequence(
     history_file = Path(options.directory) / f"sequence_{options.tel_id}_{run_str}.history"
     # Set the starting level and corresponding return code from last analysis step
     # registered in the history file.
-    level, rc = (4, 0) if options.simulate else historylevel(history_file, "DATA")
+    level, rc = (5, 0) if options.simulate else historylevel(history_file, "DATA")
     log.info(f"Going to level {level}")
 
     if level == 5:
@@ -189,7 +189,7 @@ def catB_calibration(run_str: str) -> int:
     if run_str[-4:] != "0000":
         log.debug(f"{run_str} is not the first subrun of the run, so the script "
             "onsite_create_cat_B_calibration_file will not be launched for this subrun.")
-        return
+        return 0
 
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
     r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()

From 889508cf268b67c8c9da2ed68faf13cfab6d8a02 Mon Sep 17 00:00:00 2001
From: marialainez <malainez@ucm.es>
Date: Mon, 3 Jun 2024 15:05:22 +0000
Subject: [PATCH 10/23] adapt tests

---
 src/osa/provenance/capture.py             |  2 +-
 src/osa/provenance/config/definition.yaml |  7 +++++++
 src/osa/provenance/utils.py               |  5 ++++-
 src/osa/scripts/tests/test_osa_scripts.py | 16 ++++++++--------
 4 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/src/osa/provenance/capture.py b/src/osa/provenance/capture.py
index 464fa597..95fe4992 100644
--- a/src/osa/provenance/capture.py
+++ b/src/osa/provenance/capture.py
@@ -53,7 +53,7 @@
 PROV_PREFIX = provconfig["PREFIX"]
 SUPPORTED_HASH_METHOD = ["md5"]
 SUPPORTED_HASH_BUFFER = ["content", "path"]
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 # global variables
 traced_entities = {}
diff --git a/src/osa/provenance/config/definition.yaml b/src/osa/provenance/config/definition.yaml
index b5ae446a..1b8fa0f8 100644
--- a/src/osa/provenance/config/definition.yaml
+++ b/src/osa/provenance/config/definition.yaml
@@ -200,6 +200,13 @@ activities:
             #   filepath: /fefs/aswg/data/real/DL1/20200218/v0.4.3_v00/
             #   size: 128
 
+    catB_calibration:
+        description: 
+            "Create Cat-B calibration file for an observation run"
+        parameters:
+        usage:
+        generation:
+
     dl1ab:
         description:
             "Create DL1AB files for an observation run"
diff --git a/src/osa/provenance/utils.py b/src/osa/provenance/utils.py
index 92e5cb04..ced43d6a 100644
--- a/src/osa/provenance/utils.py
+++ b/src/osa/provenance/utils.py
@@ -10,7 +10,7 @@
 
 __all__ = ["parse_variables", "get_log_config"]
 
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 
 def parse_variables(class_instance):
@@ -133,6 +133,9 @@ def parse_variables(class_instance):
         class_instance.InterleavedPedestalEventsFile = None
         if class_instance.args[6] is not None:
             class_instance.InterleavedPedestalEventsFile = str(Path(class_instance.args[6]))
+  
+    if class_instance.__name__ == "catB_calibration":
+        class_instance.ObservationRun = class_instance.args[0].split(".")[0]
 
     if class_instance.__name__ == "dl1ab":
         # run_str       [0] 02006.0000
diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py
index 0c5814cb..d6b4fe38 100644
--- a/src/osa/scripts/tests/test_osa_scripts.py
+++ b/src/osa/scripts/tests/test_osa_scripts.py
@@ -99,17 +99,17 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 7
-    assert len(dl1["activity"]) == 2
-    assert len(dl1["used"]) == 3
-    assert len(dl1["wasGeneratedBy"]) == 4
+    assert len(dl1["entity"]) == 19
+    assert len(dl1["activity"]) == 5
+    assert len(dl1["used"]) == 15
+    assert len(dl1["wasGeneratedBy"]) == 10
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)
-    assert len(dl2["entity"]) == 7
-    assert len(dl2["activity"]) == 2
-    assert len(dl2["used"]) == 3
-    assert len(dl2["wasGeneratedBy"]) == 4
+    assert len(dl2["entity"]) == 25
+    assert len(dl2["activity"]) == 6
+    assert len(dl2["used"]) == 21
+    assert len(dl2["wasGeneratedBy"]) == 12
 
     rc = run_program("simulate_processing", "-p")
     assert rc.returncode == 0

From 2d3e2bdb55c2a3ed02ceb758bcc9d9db991f89cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mar=C3=ADa=20L=C3=A1inez?=
 <73472118+marialainez@users.noreply.github.com>
Date: Wed, 26 Jun 2024 12:01:19 +0200
Subject: [PATCH 11/23] Update src/osa/configs/sequencer.cfg

Co-authored-by: Daniel Morcuende <dmorcuende@iaa.es>
---
 src/osa/configs/sequencer.cfg | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg
index 2ad351b5..b3c9dedc 100644
--- a/src/osa/configs/sequencer.cfg
+++ b/src/osa/configs/sequencer.cfg
@@ -16,9 +16,10 @@ RUN_CATALOG: %(MONITORING)s/RunCatalog
 PEDESTAL_FINDER_DIR: %(BASE)s/auxiliary/PedestalFinder
 ANALYSIS_DIR: %(BASE)s/running_analysis
 CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration
-CALIB_DIR: %(CALIB_BASE_DIR)s/Cat-A/calibration
-PEDESTAL_DIR: %(CALIB_BASE_DIR)s/Cat-A/drs4_baseline
-CATB_CALIB_DIR: %(CALIB_BASE_DIR)s/Cat-B
+CAT_A_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-A
+CAT_A_CALIB_DIR: %(CAT_A_CALIB_BASE)s/calibration
+CAT_A_PEDESTAL_DIR: %(CAT_A_CALIB_BASE)s/drs4_baseline
+CAT_B_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-B
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2

From eefba65a3f5d7a4f8a564c26559afb423987205f Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Wed, 26 Jun 2024 12:16:14 +0200
Subject: [PATCH 12/23] adapt name of Cat-A and Cat-B calibration dir to the
 cfg file

---
 src/osa/paths.py                | 4 ++--
 src/osa/provenance/utils.py     | 4 ++--
 src/osa/scripts/datasequence.py | 2 +-
 src/osa/workflow/stages.py      | 4 ++--
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/osa/paths.py b/src/osa/paths.py
index 377a4e6e..f8c89c4a 100644
--- a/src/osa/paths.py
+++ b/src/osa/paths.py
@@ -45,8 +45,8 @@
 
 
 DATACHECK_WEB_BASEDIR = Path(cfg.get("WEBSERVER", "DATACHECK"))
-CALIB_BASEDIR = Path(cfg.get("LST1", "CALIB_DIR"))
-DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+CALIB_BASEDIR = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
 
 def analysis_path(tel) -> Path:
diff --git a/src/osa/provenance/utils.py b/src/osa/provenance/utils.py
index ced43d6a..1fb011ed 100644
--- a/src/osa/provenance/utils.py
+++ b/src/osa/provenance/utils.py
@@ -43,8 +43,8 @@ def parse_variables(class_instance):
     rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS"))
     dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
     dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
-    calib_dir = Path(cfg.get("LST1", "CALIB_DIR"))
-    pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+    calib_dir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+    pedestal_dir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
     class_instance.SoftwareVersion = get_lstchain_version()
     class_instance.ProcessingConfigFile = str(options.configfile)
diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 9761b111..7f667318 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -233,7 +233,7 @@ def dl1ab(run_str: str) -> int:
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
     night_dir = date_to_dir(options.date)
     calib_prod_id = cfg.get("LST1", "CALIB_PROD_ID")
-    catB_calib_dir = Path(cfg.get("LST1", "CATB_CALIB_DIR")) / night_dir / calib_prod_id
+    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / night_dir / calib_prod_id
     catB_calibration_file = catB_calib_dir / f"cat_B_calibration_filters_{options.filters}.Run{run_str[:5]}.h5"
 
     # Prepare and launch the actual lstchain script
diff --git a/src/osa/workflow/stages.py b/src/osa/workflow/stages.py
index d1d7935e..83e20bdc 100644
--- a/src/osa/workflow/stages.py
+++ b/src/osa/workflow/stages.py
@@ -87,7 +87,7 @@ def _clean_up(self):
             self._remove_drs4_baseline()
 
     def _remove_drs4_baseline(self):
-        drs4_pedestal_basedir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+        drs4_pedestal_basedir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
         date = date_to_dir(get_run_date(self.run))
         drs4_pedestal_dir = drs4_pedestal_basedir / date / lstchain.__version__
         file = drs4_pedestal_dir / "drs4_pedestal.Run{self.run}.0000.h5"
@@ -97,7 +97,7 @@ def _remove_drs4_baseline(self):
         drs4_pedestal_dir_pro.unlink(missing_ok=True)
 
     def _remove_calibration(self):
-        calib_basedir = Path(cfg.get("LST1", "CALIB_DIR"))
+        calib_basedir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
         date = date_to_dir(get_run_date(self.run))
         calib_dir = file = calib_basedir / date / lstchain.__version__
         file = calib_dir / f"calibration_filters_{options.filters}.Run{self.run}.0000.h5"

From f483be3ef2ac88cefd4e60a5b6801bcb4c15dc91 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Wed, 26 Jun 2024 12:31:00 +0200
Subject: [PATCH 13/23] use lstchain major version as calib_prod_id

---
 src/osa/configs/sequencer.cfg   | 1 -
 src/osa/scripts/datasequence.py | 6 ++++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg
index b3c9dedc..b1def0d1 100644
--- a/src/osa/configs/sequencer.cfg
+++ b/src/osa/configs/sequencer.cfg
@@ -34,7 +34,6 @@ GAIN_SELECTION_FLAG_DIR: %(OSA_DIR)s/GainSel
 
 # To be set by the user. Using PROD-ID will overcome the automatic
 # fetching of lstchain version. Otherwise leave it empty (and without the colon symbol).
-CALIB_PROD_ID: v0.1.0
 PROD_ID: v0.1.0
 # Change this to produce a different DL1b or DL2 sub-productions.
 # Otherwise, keep it empty to use the common PROD-ID
diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 7f667318..f83c2d7f 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -9,9 +9,11 @@
 from osa.job import historylevel
 from osa.workflow.stages import AnalysisStage
 from osa.provenance.capture import trace
+from osa.paths import get_major_version
 from osa.utils.cliopts import data_sequence_cli_parsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_dir
+from osa.utils.utils import date_to_dir, get_lstchain_version
+
 
 __all__ = ["data_sequence", "r0_to_dl1", "dl1_to_dl2", "dl1ab", "dl1_datacheck"]
 
@@ -232,7 +234,7 @@ def dl1ab(run_str: str) -> int:
     # DL1b output file to be stored in the dl1ab subdirectory
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
     night_dir = date_to_dir(options.date)
-    calib_prod_id = cfg.get("LST1", "CALIB_PROD_ID")
+    calib_prod_id = get_major_version(get_lstchain_version())
     catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / night_dir / calib_prod_id
     catB_calibration_file = catB_calib_dir / f"cat_B_calibration_filters_{options.filters}.Run{run_str[:5]}.h5"
 

From 34c8ab0801abddaa5552a9796dd572853645f78c Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Mon, 5 Aug 2024 18:01:04 +0200
Subject: [PATCH 14/23] add option in the cfg to apply or not catB calibration

---
 src/osa/configs/sequencer.cfg   | 1 +
 src/osa/scripts/datasequence.py | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg
index b1def0d1..0d1119c5 100644
--- a/src/osa/configs/sequencer.cfg
+++ b/src/osa/configs/sequencer.cfg
@@ -56,6 +56,7 @@ dl1_to_dl2: lstchain_dl1_to_dl2
 dl1a_config: /software/lstchain/data/lstchain_standard_config.json
 store_image_dl1ab: True
 merge_dl1_datacheck: True
+apply_catB_calibration: True
 dl1b_config: /software/lstchain/data/lstchain_standard_config.json
 dl2_config: /software/lstchain/data/lstchain_standard_config.json
 rf_models: /data/models/prod5/zenith_20deg/20201023_v0.6.3
diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index f83c2d7f..b4cd7f60 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -245,11 +245,14 @@ def dl1ab(run_str: str) -> int:
         f"--input-file={input_dl1_datafile}",
         f"--output-file={output_dl1_datafile}",
         f"--config={dl1b_config}",
-        f"--catB-calibration-file={catB_calibration_file}",
     ]
+    
     if not cfg.getboolean("lstchain", "store_image_dl1ab"):
         cmd.append("--no-image=True")
 
+    if cfg.getboolean("lstchain", "apply_catB_calibration"):
+        cmd.append(f"--catB-calibration-file={catB_calibration_file}")
+
     if options.simulate:
         return 0
 

From c67cdd54d2bf57f2b2badb4b0f4829d1bc7e7981 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Tue, 6 Aug 2024 11:03:05 +0200
Subject: [PATCH 15/23] add cat A calibration run number

---
 src/osa/scripts/datasequence.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index b4cd7f60..2b9d8161 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -13,6 +13,7 @@
 from osa.utils.cliopts import data_sequence_cli_parsing
 from osa.utils.logging import myLogger
 from osa.utils.utils import date_to_dir, get_lstchain_version
+from osa.nightsummary.extract import get_last_pedcalib
 
 
 __all__ = ["data_sequence", "r0_to_dl1", "dl1_to_dl2", "dl1ab", "dl1_datacheck"]
@@ -195,9 +196,11 @@ def catB_calibration(run_str: str) -> int:
 
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
     r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
+    catA_calib_run = get_last_pedcalib(options.date)
     cmd = [
         "onsite_create_cat_B_calibration_file",
         f"--run_number={run_str[:5]}",
+        f"--catA_calibration_run={catA_calib_run}",
         f"--base_dir={base_dir}",
         f"--r0-dir={r0_dir}",
     ]
@@ -235,9 +238,7 @@ def dl1ab(run_str: str) -> int:
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
     night_dir = date_to_dir(options.date)
     calib_prod_id = get_major_version(get_lstchain_version())
-    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / night_dir / calib_prod_id
-    catB_calibration_file = catB_calib_dir / f"cat_B_calibration_filters_{options.filters}.Run{run_str[:5]}.h5"
-
+    
     # Prepare and launch the actual lstchain script
     command = cfg.get("lstchain", "dl1ab")
     cmd = [
@@ -251,6 +252,8 @@ def dl1ab(run_str: str) -> int:
         cmd.append("--no-image=True")
 
     if cfg.getboolean("lstchain", "apply_catB_calibration"):
+        catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / night_dir / calib_prod_id
+        catB_calibration_file = catB_calib_dir / f"cat_B_calibration_filters_{options.filters}.Run{run_str[:5]}.h5"
         cmd.append(f"--catB-calibration-file={catB_calibration_file}")
 
     if options.simulate:

From c3f8608d401214205be8658d5f295f96bf3aa83b Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Mon, 23 Sep 2024 13:09:29 +0200
Subject: [PATCH 16/23] add filters as an argument of the catB calibration
 command

---
 src/osa/paths.py                | 13 +------------
 src/osa/scripts/datasequence.py |  4 +++-
 src/osa/utils/utils.py          | 16 ++++++++++++++++
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/src/osa/paths.py b/src/osa/paths.py
index f8c89c4a..2448c164 100644
--- a/src/osa/paths.py
+++ b/src/osa/paths.py
@@ -136,18 +136,7 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
         return files[-1]  # Get the latest production among the major lstchain version
 
     date = utils.date_to_dir(get_run_date(run_id))
-
-    if options.test:  # Run tests avoiding the access to the database
-        options.filters = 52
-
-    else:
-        mongodb = cfg.get("database", "caco_db")
-        try:
-            # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB
-            options.filters = find_filter_wheels(int(run_id), mongodb)
-        except IOError:
-            log.warning("No filter information found in database. Assuming positions 52.")
-            options.filters = 52
+    options.filters = utils.get_calib_filters(run_id)
 
     return (
         CALIB_BASEDIR
diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 2b9d8161..259374d6 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -12,7 +12,7 @@
 from osa.paths import get_major_version
 from osa.utils.cliopts import data_sequence_cli_parsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_dir, get_lstchain_version
+from osa.utils.utils import date_to_dir, get_lstchain_version, get_calib_filters
 from osa.nightsummary.extract import get_last_pedcalib
 
 
@@ -194,6 +194,7 @@ def catB_calibration(run_str: str) -> int:
             "onsite_create_cat_B_calibration_file will not be launched for this subrun.")
         return 0
 
+    options.filters = get_calib_filters(int(run_str[:5])) 
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
     r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     catA_calib_run = get_last_pedcalib(options.date)
@@ -203,6 +204,7 @@ def catB_calibration(run_str: str) -> int:
         f"--catA_calibration_run={catA_calib_run}",
         f"--base_dir={base_dir}",
         f"--r0-dir={r0_dir}",
+        f"--filters={options.filters}",
     ]
     if options.simulate:
         return 0
diff --git a/src/osa/utils/utils.py b/src/osa/utils/utils.py
index b4e9adbe..aa867001 100644
--- a/src/osa/utils/utils.py
+++ b/src/osa/utils/utils.py
@@ -8,6 +8,7 @@
 from datetime import datetime, timedelta
 from pathlib import Path
 from socket import gethostname
+from lstchain.onsite import find_filter_wheels
 
 import osa.paths
 from osa.configs import options
@@ -285,3 +286,18 @@ def wait_for_daytime(start=8, end=18):
     while time.localtime().tm_hour <= start or time.localtime().tm_hour >= end:
         log.info("Waiting for sunrise to not interfere with the data-taking. Sleeping.")
         time.sleep(3600)
+
+
+def get_calib_filters(run_id):
+    """Get the filters used for the calibration."""
+    if options.test:  # Run tests avoiding the access to the database
+        return 52
+
+    else:
+        mongodb = cfg.get("database", "caco_db")
+        try:
+            # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB
+            return find_filter_wheels(int(run_id), mongodb)
+        except IOError:
+            log.warning("No filter information found in database. Assuming positions 52.")
+            return 52

From ccf89db60a113d130e01a4ec2c7488a3c8b4c380 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Mon, 23 Sep 2024 13:12:03 +0200
Subject: [PATCH 17/23] correct name of catB_calib_dir

---
 src/osa/scripts/datasequence.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 259374d6..807e0c72 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -239,7 +239,7 @@ def dl1ab(run_str: str) -> int:
     # DL1b output file to be stored in the dl1ab subdirectory
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
     night_dir = date_to_dir(options.date)
-    calib_prod_id = get_major_version(get_lstchain_version())
+    calib_prod_id = get_lstchain_version()
     
     # Prepare and launch the actual lstchain script
     command = cfg.get("lstchain", "dl1ab")
@@ -254,7 +254,7 @@ def dl1ab(run_str: str) -> int:
         cmd.append("--no-image=True")
 
     if cfg.getboolean("lstchain", "apply_catB_calibration"):
-        catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / night_dir / calib_prod_id
+        catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / "calibration" / night_dir / calib_prod_id
         catB_calibration_file = catB_calib_dir / f"cat_B_calibration_filters_{options.filters}.Run{run_str[:5]}.h5"
         cmd.append(f"--catB-calibration-file={catB_calibration_file}")
 

From f8b0eddd09414044615f5b26cf07786544127f24 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Mon, 23 Sep 2024 13:27:02 +0200
Subject: [PATCH 18/23] wait until the catB file is created if subrun != 0000

---
 src/osa/paths.py                |  9 +++++++++
 src/osa/scripts/datasequence.py | 17 +++++++++++------
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/osa/paths.py b/src/osa/paths.py
index 2448c164..65c9525d 100644
--- a/src/osa/paths.py
+++ b/src/osa/paths.py
@@ -145,6 +145,15 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
     ).resolve()
 
 
+def get_catB_calibration_filename(run_id: int) -> Path:
+    """Return the Category-B calibration filename of a given run."""
+    date = utils.date_to_dir(options.date)
+    calib_prod_id = utils.get_lstchain_version()
+    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / date / calib_prod_id
+    filters = utils.get_calib_filters(int(run_str))
+    return catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
+
+
 def pedestal_ids_file_exists(run_id: int) -> bool:
     """Look for the files with pedestal interleaved event identification."""
     pedestal_ids_dir = Path(cfg.get("LST1", "PEDESTAL_FINDER_DIR"))
diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 807e0c72..9ed01640 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -3,16 +3,17 @@
 import logging
 import sys
 from pathlib import Path
+import time
 
 from osa.configs import options
 from osa.configs.config import cfg
 from osa.job import historylevel
 from osa.workflow.stages import AnalysisStage
 from osa.provenance.capture import trace
-from osa.paths import get_major_version
+from osa.paths import get_major_version, get_catB_calibration_filename
 from osa.utils.cliopts import data_sequence_cli_parsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_dir, get_lstchain_version, get_calib_filters
+from osa.utils.utils import date_to_dir, get_calib_filters
 from osa.nightsummary.extract import get_last_pedcalib
 
 
@@ -192,6 +193,13 @@ def catB_calibration(run_str: str) -> int:
     if run_str[-4:] != "0000":
         log.debug(f"{run_str} is not the first subrun of the run, so the script "
             "onsite_create_cat_B_calibration_file will not be launched for this subrun.")
+
+        catB_calibration_file = get_catB_calibration_filename(int(run_str[:5]))
+        n = 0
+        n_max = 10
+        while not catB_calibration_file.exists() and n<=n_max:
+            time.sleep(120)
+            n += 1
         return 0
 
     options.filters = get_calib_filters(int(run_str[:5])) 
@@ -238,8 +246,6 @@ def dl1ab(run_str: str) -> int:
     input_dl1_datafile = Path(options.directory) / f"dl1_LST-1.Run{run_str}.h5"
     # DL1b output file to be stored in the dl1ab subdirectory
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
-    night_dir = date_to_dir(options.date)
-    calib_prod_id = get_lstchain_version()
     
     # Prepare and launch the actual lstchain script
     command = cfg.get("lstchain", "dl1ab")
@@ -254,8 +260,7 @@ def dl1ab(run_str: str) -> int:
         cmd.append("--no-image=True")
 
     if cfg.getboolean("lstchain", "apply_catB_calibration"):
-        catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / "calibration" / night_dir / calib_prod_id
-        catB_calibration_file = catB_calib_dir / f"cat_B_calibration_filters_{options.filters}.Run{run_str[:5]}.h5"
+        catB_calibration_file = get_catB_calibration_filename(int(run_str[:5]))
         cmd.append(f"--catB-calibration-file={catB_calibration_file}")
 
     if options.simulate:

From e94a15acd7f23aea1b16e9460cf85f2723d2f5b5 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Mon, 23 Sep 2024 15:53:27 +0200
Subject: [PATCH 19/23] adapt tests

---
 src/osa/scripts/tests/test_osa_scripts.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py
index d6b4fe38..6bca0165 100644
--- a/src/osa/scripts/tests/test_osa_scripts.py
+++ b/src/osa/scripts/tests/test_osa_scripts.py
@@ -99,17 +99,17 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 19
-    assert len(dl1["activity"]) == 5
-    assert len(dl1["used"]) == 15
-    assert len(dl1["wasGeneratedBy"]) == 10
+    assert len(dl1["entity"]) == 14
+    assert len(dl1["activity"]) == 3
+    assert len(dl1["used"]) == 10
+    assert len(dl1["wasGeneratedBy"]) == 6
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)
-    assert len(dl2["entity"]) == 25
-    assert len(dl2["activity"]) == 6
-    assert len(dl2["used"]) == 21
-    assert len(dl2["wasGeneratedBy"]) == 12
+    assert len(dl2["entity"]) == 14
+    assert len(dl2["activity"]) == 3
+    assert len(dl2["used"]) == 10
+    assert len(dl2["wasGeneratedBy"]) == 6
 
     rc = run_program("simulate_processing", "-p")
     assert rc.returncode == 0

From a9ebfdba71a91867b989cbd6b3313a17e8ec1239 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Mon, 23 Sep 2024 15:56:38 +0200
Subject: [PATCH 20/23] remove unused imports

---
 src/osa/paths.py                | 5 ++---
 src/osa/scripts/datasequence.py | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/osa/paths.py b/src/osa/paths.py
index 65c9525d..64bbc639 100644
--- a/src/osa/paths.py
+++ b/src/osa/paths.py
@@ -11,8 +11,7 @@
 import lstchain
 from astropy.table import Table
 from lstchain.onsite import (find_systematics_correction_file,
-                             find_time_calibration_file,
-                             find_filter_wheels)
+                             find_time_calibration_file)
 
 from osa.configs import options
 from osa.configs.config import DEFAULT_CFG, cfg
@@ -150,7 +149,7 @@ def get_catB_calibration_filename(run_id: int) -> Path:
     date = utils.date_to_dir(options.date)
     calib_prod_id = utils.get_lstchain_version()
     catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / date / calib_prod_id
-    filters = utils.get_calib_filters(int(run_str))
+    filters = utils.get_calib_filters(run_id)
     return catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
 
 
diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 9ed01640..946c1c5a 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -10,7 +10,7 @@
 from osa.job import historylevel
 from osa.workflow.stages import AnalysisStage
 from osa.provenance.capture import trace
-from osa.paths import get_major_version, get_catB_calibration_filename
+from osa.paths import get_catB_calibration_filename
 from osa.utils.cliopts import data_sequence_cli_parsing
 from osa.utils.logging import myLogger
 from osa.utils.utils import date_to_dir, get_calib_filters

From d8df805e7ea9e1e78604a220bcf0e5f834168dfe Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Tue, 24 Sep 2024 14:46:49 +0200
Subject: [PATCH 21/23] adapt tests

---
 src/osa/scripts/tests/test_osa_scripts.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py
index 6bca0165..d6b4fe38 100644
--- a/src/osa/scripts/tests/test_osa_scripts.py
+++ b/src/osa/scripts/tests/test_osa_scripts.py
@@ -99,17 +99,17 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 14
-    assert len(dl1["activity"]) == 3
-    assert len(dl1["used"]) == 10
-    assert len(dl1["wasGeneratedBy"]) == 6
+    assert len(dl1["entity"]) == 19
+    assert len(dl1["activity"]) == 5
+    assert len(dl1["used"]) == 15
+    assert len(dl1["wasGeneratedBy"]) == 10
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)
-    assert len(dl2["entity"]) == 14
-    assert len(dl2["activity"]) == 3
-    assert len(dl2["used"]) == 10
-    assert len(dl2["wasGeneratedBy"]) == 6
+    assert len(dl2["entity"]) == 25
+    assert len(dl2["activity"]) == 6
+    assert len(dl2["used"]) == 21
+    assert len(dl2["wasGeneratedBy"]) == 12
 
     rc = run_program("simulate_processing", "-p")
     assert rc.returncode == 0

From 37062cd618cf5ca43c74736829d35c936476b490 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Wed, 25 Sep 2024 19:02:51 +0200
Subject: [PATCH 22/23] adapt historylevel function

---
 src/osa/configs/sequencer.cfg   | 1 +
 src/osa/job.py                  | 4 +++-
 src/osa/paths.py                | 2 +-
 src/osa/scripts/datasequence.py | 3 ++-
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg
index 0d1119c5..feb10ae6 100644
--- a/src/osa/configs/sequencer.cfg
+++ b/src/osa/configs/sequencer.cfg
@@ -44,6 +44,7 @@ DL2_PROD_ID: model2
 # Calibration steps in calibration pipeline script
 drs4_baseline: onsite_create_drs4_pedestal_file
 charge_calibration: onsite_create_calibration_file
+catB_calibration: onsite_create_cat_B_calibration_file
 use_ff_heuristic_id: False
 
 # Data processing steps in datasequence script
diff --git a/src/osa/job.py b/src/osa/job.py
index b151a69e..860228b2 100644
--- a/src/osa/job.py
+++ b/src/osa/job.py
@@ -188,7 +188,7 @@ def historylevel(history_file: Path, data_type: str):
     #  into account not only the last history line but also the others.
 
     if data_type == "DATA":
-        level = 4
+        level = 5
     elif data_type == "PEDCALIB":
         level = 2
     else:
@@ -214,6 +214,8 @@ def historylevel(history_file: Path, data_type: str):
                     level = 0 if exit_status == 0 else 1
                 # Data sequence
                 elif program == cfg.get("lstchain", "r0_to_dl1"):
+                    level = 4 if exit_status == 0 else 5
+                elif program == cfg.get("lstchain", "catB_calibration"):
                     level = 3 if exit_status == 0 else 4
                 elif program == cfg.get("lstchain", "dl1ab"):
                     if (exit_status == 0) and (prod_id == options.dl1_prod_id):
diff --git a/src/osa/paths.py b/src/osa/paths.py
index 64bbc639..f317f09c 100644
--- a/src/osa/paths.py
+++ b/src/osa/paths.py
@@ -148,7 +148,7 @@ def get_catB_calibration_filename(run_id: int) -> Path:
     """Return the Category-B calibration filename of a given run."""
     date = utils.date_to_dir(options.date)
     calib_prod_id = utils.get_lstchain_version()
-    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / date / calib_prod_id
+    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / "calibration" / date / calib_prod_id
     filters = utils.get_calib_filters(run_id)
     return catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
 
diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index 946c1c5a..f726d9fd 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -202,12 +202,13 @@ def catB_calibration(run_str: str) -> int:
             n += 1
         return 0
 
+    command = cfg.get("lstchain", "catB_calibration")
     options.filters = get_calib_filters(int(run_str[:5])) 
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
     r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     catA_calib_run = get_last_pedcalib(options.date)
     cmd = [
-        "onsite_create_cat_B_calibration_file",
+        command,
         f"--run_number={run_str[:5]}",
         f"--catA_calibration_run={catA_calib_run}",
         f"--base_dir={base_dir}",

From d220c81f8e7f28fe88b53aaa055494dac6946b70 Mon Sep 17 00:00:00 2001
From: Maria Lainez <98marialainez@gmail.com>
Date: Fri, 4 Oct 2024 11:05:53 +0200
Subject: [PATCH 23/23] add --no-dl1ab option to process data with catB
 (sequencer launched twice)

---
 src/osa/configs/options.py      |  2 ++
 src/osa/job.py                  |  8 ++++++++
 src/osa/scripts/datasequence.py | 16 ++++++++++------
 src/osa/utils/cliopts.py        | 14 ++++++++++++++
 4 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/src/osa/configs/options.py b/src/osa/configs/options.py
index 83eec716..057f1616 100644
--- a/src/osa/configs/options.py
+++ b/src/osa/configs/options.py
@@ -15,6 +15,8 @@
 warning = None
 nocheck = None
 no_dl2 = None
+no_dl1ab = None
+no_gainsel = None
 prod_id = None
 dl1_prod_id = None
 dl2_prod_id = None
diff --git a/src/osa/job.py b/src/osa/job.py
index 860228b2..a1fa99eb 100644
--- a/src/osa/job.py
+++ b/src/osa/job.py
@@ -109,6 +109,12 @@ def are_all_jobs_correctly_finished(sequence_list):
                     f"finished up to DL1ab, but --no-dl2 option selected"
                 )
                 continue
+            if out == 3 and options.no_dl1ab:
+                log.debug(
+                    f"Job {sequence.seq} ({sequence.type}) correctly "
+                    f"finished up to DL1A, but --no-dl1ab option selected"
+                )
+                continue
 
             log.warning(
                 f"Job {sequence.seq} (run {sequence.run}) not correctly finished [level {out}]"
@@ -426,6 +432,8 @@ def data_sequence_job_template(sequence):
         commandargs.extend(("--config", f"{Path(options.configfile).resolve()}"))
     if sequence.type == "DATA" and options.no_dl2:
         commandargs.append("--no-dl2")
+    if sequence.type == "DATA" and options.no_dl1ab:
+        commandargs.append("--no-dl1ab")
 
     commandargs.extend(
         (
diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py
index f726d9fd..4a32e840 100644
--- a/src/osa/scripts/datasequence.py
+++ b/src/osa/scripts/datasequence.py
@@ -77,13 +77,17 @@ def data_sequence(
         log.info(f"Going to level {level}")
 
     if level == 3:
-        rc = dl1ab(run_str)
-        if cfg.getboolean("lstchain", "store_image_dl1ab"):
-            level -= 1
-            log.info(f"Going to level {level}")
+        if options.no_dl1ab:
+            level = 0
+            log.info(f"No DL1B are going to be produced. Going to level {level}")
         else:
-            level -= 2
-            log.info(f"No images stored in dl1ab. Producing DL2. Going to level {level}")
+            rc = dl1ab(run_str)
+            if cfg.getboolean("lstchain", "store_image_dl1ab"):
+                level -= 1
+                log.info(f"Going to level {level}")
+            else:
+                level -= 2
+                log.info(f"No images stored in dl1ab. Producing DL2. Going to level {level}")
 
     if level == 2:
         rc = dl1_datacheck(run_str)
diff --git a/src/osa/utils/cliopts.py b/src/osa/utils/cliopts.py
index 95bd41a7..e2ba9722 100644
--- a/src/osa/utils/cliopts.py
+++ b/src/osa/utils/cliopts.py
@@ -189,6 +189,12 @@ def data_sequence_argparser():
         default=False,
         help="Do not produce DL2 files (default False)",
     )
+    parser.add_argument(
+        "--no-dl1ab",
+        action="store_true",
+        default=False,
+        help="Do not launch the script lstchain_dl1ab (default False)",
+    )
     parser.add_argument("--pedcal-file", type=Path, help="Path of the calibration file")
     parser.add_argument("--drs4-pedestal-file", type=Path, help="Path of the DRS4 pedestal file")
     parser.add_argument("--time-calib-file", type=Path, help="Path of the time calibration file")
@@ -227,6 +233,7 @@ def data_sequence_cli_parsing():
     options.simulate = opts.simulate
     options.prod_id = opts.prod_id
     options.no_dl2 = opts.no_dl2
+    options.no_dl1ab = opts.no_dl1ab
     options.tel_id = opts.tel_id
 
     log.debug(f"The options and arguments are {opts}")
@@ -274,6 +281,12 @@ def sequencer_argparser():
         default=False,
         help="Do not produce DL2 files (default False)",
     )
+    parser.add_argument(
+        "--no-dl1ab",
+        action="store_true",
+        default=False,
+        help="Do not launch the script lstchain_dl1ab (default False)",
+    )
     parser.add_argument(
         "--no-gainsel",
         action="store_true",
@@ -298,6 +311,7 @@ def sequencer_cli_parsing():
     options.no_submit = opts.no_submit
     options.no_calib = opts.no_calib
     options.no_dl2 = opts.no_dl2
+    options.no_dl1ab = opts.no_dl1ab
     options.no_gainsel = opts.no_gainsel
 
     log.debug(f"the options are {opts}")