Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add run type and beam type metadata for async and MC QC #1838

Merged
merged 1 commit into from
Dec 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DATA/production/configurations/asyncReco/setenv_extra.sh
Original file line number Diff line number Diff line change
Expand Up @@ -713,7 +713,7 @@ if [[ $ALIEN_JDL_QCOFF != "1" ]]; then
export WORKFLOW_PARAMETERS="QC,${WORKFLOW_PARAMETERS}"
fi

export QC_CONFIG_OVERRIDE+=";qc.config.Activity.number=$RUNNUMBER;qc.config.Activity.passName=$PASS;qc.config.Activity.periodName=$PERIOD;"
export QC_CONFIG_OVERRIDE+=";qc.config.Activity.number=$RUNNUMBER;qc.config.Activity.type=PHYSICS;qc.config.Activity.passName=$PASS;qc.config.Activity.periodName=$PERIOD;qc.config.Activity.beamType=$BEAMTYPE;"

export QC_CONFIG_PARAM+=" --local-batch=QC.root "
export GEN_TOPO_WORKDIR="./"
Expand Down
1 change: 1 addition & 0 deletions DATA/production/o2dpg_qc_postproc_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, periodSpec
task = createTask(name=taskName, needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000')
overrideValues = '--override-values "'
overrideValues += f'qc.config.database.host={qcdbUrl};'
overrideValues += f'qc.config.Activity.type=PHYSICS;'
overrideValues += f'qc.config.Activity.number={runNumber};' if runSpecific else 'qc.config.Activity.number=0;'
overrideValues += f'qc.config.Activity.periodName={periodName};' if periodSpecific else 'qc.config.Activity.periodName=;'
overrideValues += f'qc.config.Activity.passName={passName};' if passSpecific else 'qc.config.Activity.passName=;'
Expand Down
9 changes: 6 additions & 3 deletions MC/bin/o2dpg_qc_finalization_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def QC_finalize_name(name):
return name + "_finalize"

qcdir = "QC"
def include_all_QC_finalization(ntimeframes, standalone, run, productionTag, conditionDB, qcdbHost):
def include_all_QC_finalization(ntimeframes, standalone, run, productionTag, conditionDB, qcdbHost, beamType):

stages = []

Expand All @@ -49,7 +49,7 @@ def add_QC_finalization(taskName, qcConfigPath, needs=None):

task = createTask(name=QC_finalize_name(taskName), needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000')
task['cmd'] = f'o2-qc --config {qcConfigPath} --remote-batch {taskName}.root' + \
f' --override-values "qc.config.database.host={qcdbHost};qc.config.Activity.number={run};qc.config.Activity.periodName={productionTag};qc.config.conditionDB.url={conditionDB}"' + \
f' --override-values "qc.config.database.host={qcdbHost};qc.config.Activity.number={run};qc.config.Activity.type=PHYSICS;qc.config.Activity.periodName={productionTag};qc.config.Activity.beamType={beamType};qc.config.conditionDB.url={conditionDB}"' + \
' ' + getDPL_global_options()
stages.append(task)

Expand All @@ -66,7 +66,9 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif
task = createTask(name=taskName, needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000')
overrideValues = '--override-values "'
overrideValues += f'qc.config.Activity.number={run};' if runSpecific else 'qc.config.Activity.number=0;'
overrideValues += f'qc.config.Activity.type=PHYSICS;'
overrideValues += f'qc.config.Activity.periodName={productionTag};' if prodSpecific else 'qc.config.Activity.periodName=;'
overrideValues += f'qc.config.Activity.beamType={beamType};'
overrideValues += f'qc.config.database.host={qcdbHost};qc.config.conditionDB.url={conditionDB}"'
task['cmd'] = f'o2-qc --config {qcConfigPath} ' + \
overrideValues + ' ' + getDPL_global_options()
Expand Down Expand Up @@ -124,6 +126,7 @@ def main() -> int:
parser.add_argument('-productionTag',help="Production tag for this MC", default='unknown')
parser.add_argument('-conditionDB',help="CCDB url for QC workflows", default='http://alice-ccdb.cern.ch')
parser.add_argument('-qcdbHost',help="QCDB url for QC object uploading", default='http://ali-qcdbmc-gpn.cern.ch:8083')
parser.add_argument('-beamType',help="Collision system, e.g. PbPb, pp", default='')
args = parser.parse_args()
print (args)

Expand All @@ -145,7 +148,7 @@ def main() -> int:
mkdir(qcdir)

workflow={}
workflow['stages'] = include_all_QC_finalization(ntimeframes=1, standalone=True, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost)
workflow['stages'] = include_all_QC_finalization(ntimeframes=1, standalone=True, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost, beamType=args.beamType)

dump_workflow(workflow["stages"], args.o)

Expand Down
4 changes: 2 additions & 2 deletions MC/bin/o2dpg_sim_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1302,7 +1302,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''):
# the --local-batch argument will make QC Tasks store their results in a file and merge with any existing objects
task['cmd'] = f'{readerCommand} | o2-qc --config {configFilePath}' + \
f' --local-batch ../{qcdir}/{objectsFile}' + \
f' --override-values "qc.config.database.host={args.qcdbHost};qc.config.Activity.number={args.run};qc.config.Activity.periodName={args.productionTag};qc.config.Activity.start={args.timestamp};qc.config.conditionDB.url={args.conditionDB}"' + \
f' --override-values "qc.config.database.host={args.qcdbHost};qc.config.Activity.number={args.run};qc.config.Activity.type=PHYSICS;qc.config.Activity.periodName={args.productionTag};qc.config.Activity.beamType={args.col};qc.config.Activity.start={args.timestamp};qc.config.conditionDB.url={args.conditionDB}"' + \
' ' + getDPL_global_options(ccdbbackend=False)
# Prevents this task from being run for multiple TimeFrames at the same time, thus trying to modify the same file.
task['semaphore'] = objectsFile
Expand Down Expand Up @@ -1572,7 +1572,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''):

job_merging = False
if includeFullQC:
workflow['stages'].extend(include_all_QC_finalization(ntimeframes=NTIMEFRAMES, standalone=False, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost))
workflow['stages'].extend(include_all_QC_finalization(ntimeframes=NTIMEFRAMES, standalone=False, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost, beamType=args.col))

if includeAnalysis:
# include analyses and potentially final QC upload tasks
Expand Down
Loading