diff --git a/src/osa/configs/options.py b/src/osa/configs/options.py index 83eec716..057f1616 100644 --- a/src/osa/configs/options.py +++ b/src/osa/configs/options.py @@ -15,6 +15,8 @@ warning = None nocheck = None no_dl2 = None +no_dl1ab = None +no_gainsel = None prod_id = None dl1_prod_id = None dl2_prod_id = None diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg index 6deeae3d..feb10ae6 100644 --- a/src/osa/configs/sequencer.cfg +++ b/src/osa/configs/sequencer.cfg @@ -15,9 +15,11 @@ RUN_SUMMARY_DIR: %(MONITORING)s/RunSummary RUN_CATALOG: %(MONITORING)s/RunCatalog PEDESTAL_FINDER_DIR: %(BASE)s/auxiliary/PedestalFinder ANALYSIS_DIR: %(BASE)s/running_analysis -CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration/Cat-A -CALIB_DIR: %(CALIB_BASE_DIR)s/calibration -PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline +CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration +CAT_A_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-A +CAT_A_CALIB_DIR: %(CAT_A_CALIB_BASE)s/calibration +CAT_A_PEDESTAL_DIR: %(CAT_A_CALIB_BASE)s/drs4_baseline +CAT_B_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-B DL1_DIR: %(BASE)s/DL1 DL1AB_DIR: %(BASE)s/DL1 DL2_DIR: %(BASE)s/DL2 @@ -42,6 +44,7 @@ DL2_PROD_ID: model2 # Calibration steps in calibration pipeline script drs4_baseline: onsite_create_drs4_pedestal_file charge_calibration: onsite_create_calibration_file +catB_calibration: onsite_create_cat_B_calibration_file use_ff_heuristic_id: False # Data processing steps in datasequence script @@ -54,6 +57,7 @@ dl1_to_dl2: lstchain_dl1_to_dl2 dl1a_config: /software/lstchain/data/lstchain_standard_config.json store_image_dl1ab: True merge_dl1_datacheck: True +apply_catB_calibration: True dl1b_config: /software/lstchain/data/lstchain_standard_config.json dl2_config: /software/lstchain/data/lstchain_standard_config.json rf_models: /data/models/prod5/zenith_20deg/20201023_v0.6.3 diff --git a/src/osa/job.py b/src/osa/job.py index b151a69e..e7f5a79e 100644 --- a/src/osa/job.py +++ b/src/osa/job.py @@ -109,6 +109,12 @@ def are_all_jobs_correctly_finished(sequence_list): f"finished up to DL1ab, but --no-dl2 option selected" ) continue + if out == 3 and options.no_dl1ab: + log.debug( + f"Job {sequence.seq} ({sequence.type}) correctly " + f"finished up to DL1A, but --no-dl1ab option selected" + ) + continue log.warning( f"Job {sequence.seq} (run {sequence.run}) not correctly finished [level {out}]" @@ -215,6 +221,8 @@ def historylevel(history_file: Path, data_type: str): # Data sequence elif program == cfg.get("lstchain", "r0_to_dl1"): level = 3 if exit_status == 0 else 4 + #elif program == cfg.get("lstchain", "catB_calibration"): + # level = 3 if exit_status == 0 else 4 elif program == cfg.get("lstchain", "dl1ab"): if (exit_status == 0) and (prod_id == options.dl1_prod_id): log.debug(f"DL1ab prod ID: {options.dl1_prod_id} already produced") @@ -424,6 +432,8 @@ def data_sequence_job_template(sequence): commandargs.extend(("--config", f"{Path(options.configfile).resolve()}")) if sequence.type == "DATA" and options.no_dl2: commandargs.append("--no-dl2") + if sequence.type == "DATA" and options.no_dl1ab: + commandargs.append("--no-dl1ab") commandargs.extend( ( diff --git a/src/osa/paths.py b/src/osa/paths.py index 377a4e6e..f317f09c 100644 --- a/src/osa/paths.py +++ b/src/osa/paths.py @@ -11,8 +11,7 @@ import lstchain from astropy.table import Table from lstchain.onsite import (find_systematics_correction_file, - find_time_calibration_file, - find_filter_wheels) + find_time_calibration_file) from osa.configs import options from osa.configs.config import DEFAULT_CFG, cfg @@ -45,8 +44,8 @@ DATACHECK_WEB_BASEDIR = Path(cfg.get("WEBSERVER", "DATACHECK")) -CALIB_BASEDIR = Path(cfg.get("LST1", "CALIB_DIR")) -DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "PEDESTAL_DIR")) +CALIB_BASEDIR = Path(cfg.get("LST1", "CAT_A_CALIB_DIR")) +DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR")) def analysis_path(tel) -> Path: @@ -136,18 +135,7 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path: return files[-1] # Get the latest production among the major lstchain version date = utils.date_to_dir(get_run_date(run_id)) - - if options.test: # Run tests avoiding the access to the database - options.filters = 52 - - else: - mongodb = cfg.get("database", "caco_db") - try: - # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB - options.filters = find_filter_wheels(int(run_id), mongodb) - except IOError: - log.warning("No filter information found in database. Assuming positions 52.") - options.filters = 52 + options.filters = utils.get_calib_filters(run_id) return ( CALIB_BASEDIR @@ -156,6 +144,15 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path: ).resolve() +def get_catB_calibration_filename(run_id: int) -> Path: + """Return the Category-B calibration filename of a given run.""" + date = utils.date_to_dir(options.date) + calib_prod_id = utils.get_lstchain_version() + catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / "calibration" / date / calib_prod_id + filters = utils.get_calib_filters(run_id) + return catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5" + + def pedestal_ids_file_exists(run_id: int) -> bool: """Look for the files with pedestal interleaved event identification.""" pedestal_ids_dir = Path(cfg.get("LST1", "PEDESTAL_FINDER_DIR")) diff --git a/src/osa/provenance/capture.py b/src/osa/provenance/capture.py index 464fa597..95fe4992 100644 --- a/src/osa/provenance/capture.py +++ b/src/osa/provenance/capture.py @@ -53,7 +53,7 @@ PROV_PREFIX = provconfig["PREFIX"] SUPPORTED_HASH_METHOD = ["md5"] SUPPORTED_HASH_BUFFER = ["content", "path"] -REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"] +REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"] # global variables traced_entities = {} diff --git a/src/osa/provenance/config/definition.yaml b/src/osa/provenance/config/definition.yaml index b5ae446a..1b8fa0f8 100644 --- a/src/osa/provenance/config/definition.yaml +++ b/src/osa/provenance/config/definition.yaml @@ -200,6 +200,13 @@ activities: # filepath: /fefs/aswg/data/real/DL1/20200218/v0.4.3_v00/ # size: 128 + catB_calibration: + description: + "Create Cat-B calibration file for an observation run" + parameters: + usage: + generation: + dl1ab: description: "Create DL1AB files for an observation run" diff --git a/src/osa/provenance/utils.py b/src/osa/provenance/utils.py index 92e5cb04..1fb011ed 100644 --- a/src/osa/provenance/utils.py +++ b/src/osa/provenance/utils.py @@ -10,7 +10,7 @@ __all__ = ["parse_variables", "get_log_config"] -REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"] +REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"] def parse_variables(class_instance): @@ -43,8 +43,8 @@ def parse_variables(class_instance): rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS")) dl1_dir = Path(cfg.get("LST1", "DL1_DIR")) dl2_dir = Path(cfg.get("LST1", "DL2_DIR")) - calib_dir = Path(cfg.get("LST1", "CALIB_DIR")) - pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR")) + calib_dir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR")) + pedestal_dir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR")) class_instance.SoftwareVersion = get_lstchain_version() class_instance.ProcessingConfigFile = str(options.configfile) @@ -133,6 +133,9 @@ def parse_variables(class_instance): class_instance.InterleavedPedestalEventsFile = None if class_instance.args[6] is not None: class_instance.InterleavedPedestalEventsFile = str(Path(class_instance.args[6])) + + if class_instance.__name__ == "catB_calibration": + class_instance.ObservationRun = class_instance.args[0].split(".")[0] if class_instance.__name__ == "dl1ab": # run_str [0] 02006.0000 diff --git a/src/osa/scripts/calibration_pipeline.py b/src/osa/scripts/calibration_pipeline.py index 51d43080..7736a648 100644 --- a/src/osa/scripts/calibration_pipeline.py +++ b/src/osa/scripts/calibration_pipeline.py @@ -44,10 +44,12 @@ def is_calibration_produced(drs4_pedestal_run_id: int, pedcal_run_id: int) -> bo def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list: """Build the create_drs4_pedestal command.""" base_dir = Path(cfg.get("LST1", "BASE")).resolve() + r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve() return [ "onsite_create_drs4_pedestal_file", f"--run_number={drs4_pedestal_run_id}", f"--base_dir={base_dir}", + f"--r0-dir={r0_dir}", "--no-progress", ] @@ -55,11 +57,13 @@ def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list: def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list: """Build the create_calibration_file command.""" base_dir = Path(cfg.get("LST1", "BASE")).resolve() + r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve() cmd = [ "onsite_create_calibration_file", f"--pedestal_run={drs4_pedestal_run_id}", f"--run_number={pedcal_run_id}", f"--base_dir={base_dir}", + f"--r0-dir={r0_dir}", ] # In case of problems with trigger tagging: if cfg.getboolean("lstchain", "use_ff_heuristic_id"): diff --git a/src/osa/scripts/datasequence.py b/src/osa/scripts/datasequence.py index 0a4fe02d..f4386cdb 100644 --- a/src/osa/scripts/datasequence.py +++ b/src/osa/scripts/datasequence.py @@ -3,15 +3,19 @@ import logging import sys from pathlib import Path +import time from osa.configs import options from osa.configs.config import cfg from osa.job import historylevel from osa.workflow.stages import AnalysisStage from osa.provenance.capture import trace +from osa.paths import get_catB_calibration_filename from osa.utils.cliopts import data_sequence_cli_parsing from osa.utils.logging import myLogger -from osa.utils.utils import date_to_dir +from osa.utils.utils import date_to_dir, get_calib_filters +from osa.nightsummary.extract import get_last_pedcalib + __all__ = ["data_sequence", "r0_to_dl1", "dl1_to_dl2", "dl1ab", "dl1_datacheck"] @@ -68,13 +72,17 @@ def data_sequence( log.info(f"Going to level {level}") if level == 3: - rc = dl1ab(run_str) - if cfg.getboolean("lstchain", "store_image_dl1ab"): - level -= 1 - log.info(f"Going to level {level}") - else: - level -= 2 - log.info(f"No images stored in dl1ab. Producing DL2. Going to level {level}") + if options.no_dl1ab: + level = 0 + log.info(f"No DL1B are going to be produced. Going to level {level}") + else: + rc = dl1ab(run_str) + if cfg.getboolean("lstchain", "store_image_dl1ab"): + level -= 1 + log.info(f"Going to level {level}") + else: + level -= 2 + log.info(f"No images stored in dl1ab. Producing DL2. Going to level {level}") if level == 2: rc = dl1_datacheck(run_str) @@ -165,6 +173,55 @@ def r0_to_dl1( return analysis_step.rc +@trace +def catB_calibration(run_str: str) -> int: + """ + Prepare and launch the lstchain script that creates the + Category B calibration files. It should be executed runwise, + so it is only launched for the first subrun of each run. + + Parameters + ---------- + run_str: str + + Returns + ------- + rc: int + Return code of the executed command. + """ + if run_str[-4:] != "0000": + log.debug(f"{run_str} is not the first subrun of the run, so the script " + "onsite_create_cat_B_calibration_file will not be launched for this subrun.") + + catB_calibration_file = get_catB_calibration_filename(int(run_str[:5])) + n = 0 + n_max = 10 + while not catB_calibration_file.exists() and n<=n_max: + time.sleep(120) + n += 1 + return 0 + + command = cfg.get("lstchain", "catB_calibration") + options.filters = get_calib_filters(int(run_str[:5])) + base_dir = Path(cfg.get("LST1", "BASE")).resolve() + r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve() + catA_calib_run = get_last_pedcalib(options.date) + cmd = [ + command, + f"--run_number={run_str[:5]}", + f"--catA_calibration_run={catA_calib_run}", + f"--base_dir={base_dir}", + f"--r0-dir={r0_dir}", + f"--filters={options.filters}", + ] + if options.simulate: + return 0 + + analysis_step = AnalysisStage(run=run_str, command_args=cmd) + analysis_step.execute() + return analysis_step.rc + + @trace def dl1ab(run_str: str) -> int: """ @@ -189,7 +246,7 @@ def dl1ab(run_str: str) -> int: input_dl1_datafile = Path(options.directory) / f"dl1_LST-1.Run{run_str}.h5" # DL1b output file to be stored in the dl1ab subdirectory output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5" - + # Prepare and launch the actual lstchain script command = cfg.get("lstchain", "dl1ab") cmd = [ @@ -198,9 +255,14 @@ def dl1ab(run_str: str) -> int: f"--output-file={output_dl1_datafile}", f"--config={dl1b_config}", ] + if not cfg.getboolean("lstchain", "store_image_dl1ab"): cmd.append("--no-image=True") + if cfg.getboolean("lstchain", "apply_catB_calibration"): + catB_calibration_file = get_catB_calibration_filename(int(run_str[:5])) + cmd.append(f"--catB-calibration-file={catB_calibration_file}") + if options.simulate: return 0 diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index bdfa153f..d6b4fe38 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -298,10 +298,12 @@ def test_drs4_pedestal_cmd(base_test_dir): from osa.scripts.calibration_pipeline import drs4_pedestal_command cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804") + r0_dir = base_test_dir / "R0" expected_command = [ "onsite_create_drs4_pedestal_file", "--run_number=01804", f"--base_dir={base_test_dir}", + f"--r0-dir={r0_dir}", "--no-progress", ] assert cmd == expected_command @@ -311,11 +313,13 @@ def test_calibration_file_cmd(base_test_dir): from osa.scripts.calibration_pipeline import calibration_file_command cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809") + r0_dir = base_test_dir / "R0" expected_command = [ "onsite_create_calibration_file", "--pedestal_run=01804", "--run_number=01809", f"--base_dir={base_test_dir}", + f"--r0-dir={r0_dir}", ] assert cmd == expected_command diff --git a/src/osa/utils/cliopts.py b/src/osa/utils/cliopts.py index 95bd41a7..e2ba9722 100644 --- a/src/osa/utils/cliopts.py +++ b/src/osa/utils/cliopts.py @@ -189,6 +189,12 @@ def data_sequence_argparser(): default=False, help="Do not produce DL2 files (default False)", ) + parser.add_argument( + "--no-dl1ab", + action="store_true", + default=False, + help="Do not launch the script lstchain_dl1ab (default False)", + ) parser.add_argument("--pedcal-file", type=Path, help="Path of the calibration file") parser.add_argument("--drs4-pedestal-file", type=Path, help="Path of the DRS4 pedestal file") parser.add_argument("--time-calib-file", type=Path, help="Path of the time calibration file") @@ -227,6 +233,7 @@ def data_sequence_cli_parsing(): options.simulate = opts.simulate options.prod_id = opts.prod_id options.no_dl2 = opts.no_dl2 + options.no_dl1ab = opts.no_dl1ab options.tel_id = opts.tel_id log.debug(f"The options and arguments are {opts}") @@ -274,6 +281,12 @@ def sequencer_argparser(): default=False, help="Do not produce DL2 files (default False)", ) + parser.add_argument( + "--no-dl1ab", + action="store_true", + default=False, + help="Do not launch the script lstchain_dl1ab (default False)", + ) parser.add_argument( "--no-gainsel", action="store_true", @@ -298,6 +311,7 @@ def sequencer_cli_parsing(): options.no_submit = opts.no_submit options.no_calib = opts.no_calib options.no_dl2 = opts.no_dl2 + options.no_dl1ab = opts.no_dl1ab options.no_gainsel = opts.no_gainsel log.debug(f"the options are {opts}") diff --git a/src/osa/utils/utils.py b/src/osa/utils/utils.py index b4e9adbe..aa867001 100644 --- a/src/osa/utils/utils.py +++ b/src/osa/utils/utils.py @@ -8,6 +8,7 @@ from datetime import datetime, timedelta from pathlib import Path from socket import gethostname +from lstchain.onsite import find_filter_wheels import osa.paths from osa.configs import options @@ -285,3 +286,18 @@ def wait_for_daytime(start=8, end=18): while time.localtime().tm_hour <= start or time.localtime().tm_hour >= end: log.info("Waiting for sunrise to not interfere with the data-taking. Sleeping.") time.sleep(3600) + + +def get_calib_filters(run_id): + """Get the filters used for the calibration.""" + if options.test: # Run tests avoiding the access to the database + return 52 + + else: + mongodb = cfg.get("database", "caco_db") + try: + # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB + return find_filter_wheels(int(run_id), mongodb) + except IOError: + log.warning("No filter information found in database. Assuming positions 52.") + return 52 diff --git a/src/osa/workflow/stages.py b/src/osa/workflow/stages.py index d1d7935e..83e20bdc 100644 --- a/src/osa/workflow/stages.py +++ b/src/osa/workflow/stages.py @@ -87,7 +87,7 @@ def _clean_up(self): self._remove_drs4_baseline() def _remove_drs4_baseline(self): - drs4_pedestal_basedir = Path(cfg.get("LST1", "PEDESTAL_DIR")) + drs4_pedestal_basedir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR")) date = date_to_dir(get_run_date(self.run)) drs4_pedestal_dir = drs4_pedestal_basedir / date / lstchain.__version__ file = drs4_pedestal_dir / "drs4_pedestal.Run{self.run}.0000.h5" @@ -97,7 +97,7 @@ def _remove_drs4_baseline(self): drs4_pedestal_dir_pro.unlink(missing_ok=True) def _remove_calibration(self): - calib_basedir = Path(cfg.get("LST1", "CALIB_DIR")) + calib_basedir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR")) date = date_to_dir(get_run_date(self.run)) calib_dir = file = calib_basedir / date / lstchain.__version__ file = calib_dir / f"calibration_filters_{options.filters}.Run{self.run}.0000.h5"