From e270ef0e61a37def4933256c067896ee40bca7a4 Mon Sep 17 00:00:00 2001 From: Piotr Konopka Date: Fri, 6 Dec 2024 15:04:38 +0100 Subject: [PATCH] Add run type and beam type metadata for async and MC QC QC-1188, QC-1002 Seeing no available run type in the surrounding scripts, I naively assume that absolutely all runs in async and MC are considered to be of type PHYSICS. --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- DATA/production/o2dpg_qc_postproc_workflow.py | 1 + MC/bin/o2dpg_qc_finalization_workflow.py | 9 ++++++--- MC/bin/o2dpg_sim_workflow.py | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 025d16ca5..5fa720863 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -713,7 +713,7 @@ if [[ $ALIEN_JDL_QCOFF != "1" ]]; then export WORKFLOW_PARAMETERS="QC,${WORKFLOW_PARAMETERS}" fi -export QC_CONFIG_OVERRIDE+=";qc.config.Activity.number=$RUNNUMBER;qc.config.Activity.passName=$PASS;qc.config.Activity.periodName=$PERIOD;" +export QC_CONFIG_OVERRIDE+=";qc.config.Activity.number=$RUNNUMBER;qc.config.Activity.type=PHYSICS;qc.config.Activity.passName=$PASS;qc.config.Activity.periodName=$PERIOD;qc.config.Activity.beamType=$BEAMTYPE;" export QC_CONFIG_PARAM+=" --local-batch=QC.root " export GEN_TOPO_WORKDIR="./" diff --git a/DATA/production/o2dpg_qc_postproc_workflow.py b/DATA/production/o2dpg_qc_postproc_workflow.py index 89b958a12..679f51777 100755 --- a/DATA/production/o2dpg_qc_postproc_workflow.py +++ b/DATA/production/o2dpg_qc_postproc_workflow.py @@ -69,6 +69,7 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, periodSpec task = createTask(name=taskName, needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000') overrideValues = '--override-values "' overrideValues += f'qc.config.database.host={qcdbUrl};' + overrideValues += f'qc.config.Activity.type=PHYSICS;' overrideValues += f'qc.config.Activity.number={runNumber};' if runSpecific else 'qc.config.Activity.number=0;' overrideValues += f'qc.config.Activity.periodName={periodName};' if periodSpecific else 'qc.config.Activity.periodName=;' overrideValues += f'qc.config.Activity.passName={passName};' if passSpecific else 'qc.config.Activity.passName=;' diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index 58c26bd44..569026aa5 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -33,7 +33,7 @@ def QC_finalize_name(name): return name + "_finalize" qcdir = "QC" -def include_all_QC_finalization(ntimeframes, standalone, run, productionTag, conditionDB, qcdbHost): +def include_all_QC_finalization(ntimeframes, standalone, run, productionTag, conditionDB, qcdbHost, beamType): stages = [] @@ -49,7 +49,7 @@ def add_QC_finalization(taskName, qcConfigPath, needs=None): task = createTask(name=QC_finalize_name(taskName), needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000') task['cmd'] = f'o2-qc --config {qcConfigPath} --remote-batch {taskName}.root' + \ - f' --override-values "qc.config.database.host={qcdbHost};qc.config.Activity.number={run};qc.config.Activity.periodName={productionTag};qc.config.conditionDB.url={conditionDB}"' + \ + f' --override-values "qc.config.database.host={qcdbHost};qc.config.Activity.number={run};qc.config.Activity.type=PHYSICS;qc.config.Activity.periodName={productionTag};qc.config.Activity.beamType={beamType};qc.config.conditionDB.url={conditionDB}"' + \ ' ' + getDPL_global_options() stages.append(task) @@ -66,7 +66,9 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif task = createTask(name=taskName, needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000') overrideValues = '--override-values "' overrideValues += f'qc.config.Activity.number={run};' if runSpecific else 'qc.config.Activity.number=0;' + overrideValues += f'qc.config.Activity.type=PHYSICS;' overrideValues += f'qc.config.Activity.periodName={productionTag};' if prodSpecific else 'qc.config.Activity.periodName=;' + overrideValues += f'qc.config.Activity.beamType={beamType};' overrideValues += f'qc.config.database.host={qcdbHost};qc.config.conditionDB.url={conditionDB}"' task['cmd'] = f'o2-qc --config {qcConfigPath} ' + \ overrideValues + ' ' + getDPL_global_options() @@ -124,6 +126,7 @@ def main() -> int: parser.add_argument('-productionTag',help="Production tag for this MC", default='unknown') parser.add_argument('-conditionDB',help="CCDB url for QC workflows", default='http://alice-ccdb.cern.ch') parser.add_argument('-qcdbHost',help="QCDB url for QC object uploading", default='http://ali-qcdbmc-gpn.cern.ch:8083') + parser.add_argument('-beamType',help="Collision system, e.g. PbPb, pp", default='') args = parser.parse_args() print (args) @@ -145,7 +148,7 @@ def main() -> int: mkdir(qcdir) workflow={} - workflow['stages'] = include_all_QC_finalization(ntimeframes=1, standalone=True, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost) + workflow['stages'] = include_all_QC_finalization(ntimeframes=1, standalone=True, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost, beamType=args.beamType) dump_workflow(workflow["stages"], args.o) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 09e9ecd0b..48f41c8a2 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1302,7 +1302,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): # the --local-batch argument will make QC Tasks store their results in a file and merge with any existing objects task['cmd'] = f'{readerCommand} | o2-qc --config {configFilePath}' + \ f' --local-batch ../{qcdir}/{objectsFile}' + \ - f' --override-values "qc.config.database.host={args.qcdbHost};qc.config.Activity.number={args.run};qc.config.Activity.periodName={args.productionTag};qc.config.Activity.start={args.timestamp};qc.config.conditionDB.url={args.conditionDB}"' + \ + f' --override-values "qc.config.database.host={args.qcdbHost};qc.config.Activity.number={args.run};qc.config.Activity.type=PHYSICS;qc.config.Activity.periodName={args.productionTag};qc.config.Activity.beamType={args.col};qc.config.Activity.start={args.timestamp};qc.config.conditionDB.url={args.conditionDB}"' + \ ' ' + getDPL_global_options(ccdbbackend=False) # Prevents this task from being run for multiple TimeFrames at the same time, thus trying to modify the same file. task['semaphore'] = objectsFile @@ -1572,7 +1572,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): job_merging = False if includeFullQC: - workflow['stages'].extend(include_all_QC_finalization(ntimeframes=NTIMEFRAMES, standalone=False, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost)) + workflow['stages'].extend(include_all_QC_finalization(ntimeframes=NTIMEFRAMES, standalone=False, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost, beamType=args.col)) if includeAnalysis: # include analyses and potentially final QC upload tasks