fix conflict

DATA/common/gen_topo_helper_functions.sh

@@ -252,6 +252,7 @@ if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then
 else
   GEN_TOPO_QC_CONSUL_SERVER=alio2-cr1-hv-con01.cern.ch
 fi
+GEN_TOPO_QC_APRICOT_SERVER=`curl -s "http://${GEN_TOPO_QC_CONSUL_SERVER}:8500/v1/kv/o2/runtime/aliecs/vars/apricot_endpoint?raw"`
 
 add_QC_from_consul()
 {
@@ -268,4 +269,19 @@ add_QC_from_consul()
   add_W o2-qc "--config $QC_CONFIG_ARG $2"
 }
 
+add_QC_from_apricot()
+{
+  if [[ ! -z ${GEN_TOPO_QC_JSON_FILE:-} ]]; then
+    curl -s -o $GEN_TOPO_QC_JSON_FILE "${GEN_TOPO_QC_APRICOT_SERVER}/${1}?process=true"
+    if [[ $? != 0 ]]; then
+      echo "Error fetching QC JSON $1"
+      exit 1
+    fi
+    QC_CONFIG_ARG="json://${GEN_TOPO_QC_JSON_FILE}"
+  else
+    QC_CONFIG_ARG="apricot://${GEN_TOPO_QC_APRICOT_SERVER}$1"
+  fi
+  add_W o2-qc "--config $QC_CONFIG_ARG $2"
+}
+
 fi # functions.sh sourced

DATA/production/calib/tpc-laser-aggregator.sh

@@ -14,7 +14,7 @@ HOST=localhost
 
 QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn"
 
-QC_CONFIG_CONSUL="/o2/components/qc/ANY/any/tpc-raw-qcmn"
+QC_CONFIG="components/qc/ANY/any/tpc-raw-qcmn"
 max_events=300
 publish_after=440
 min_tracks=0
@@ -50,7 +50,7 @@ add_W o2-dpl-raw-proxy " --proxy-name tpc-laser-input-proxy --dataspec \"$PROXY_
 add_W o2-tpc-calib-laser-tracks "${REMAP} --use-filtered-tracks ${EXTRA_CONFIG_TRACKS} --min-tfs=${min_tracks} "
 add_W o2-tpc-calib-pad-raw " ${EXTRA_CONFIG}" "${CALIB_CONFIG}"
 add_W o2-calibration-ccdb-populator-workflow "--ccdb-path ${CCDB_PATH}" "" 0
-#add_QC_from_consul "${QC_CONFIG_CONSUL}" "--local --host lcoalhost"
+#add_QC_from_apricot "${QC_CONFIG_CONSUL}" "--local --host lcoalhost"
 
 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"
 if [ $WORKFLOWMODE == "print" ]; then
@@ -62,16 +62,4 @@ else
   eval $WORKFLOW
 fi
 
-#o2-dpl-raw-proxy $ARGS_ALL \
-#  --proxy-name tpc-laser-input-proxy \
-#  --dataspec "${PROXY_INSPEC}" \
-#  --network-interface ib0 \
-#  --channel-config "name=tpc-laser-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq" \
-# | o2-tpc-calib-laser-tracks  ${ARGS_ALL} --use-filtered-tracks ${EXTRA_CONFIG_TRACKS} --min-tfs=${min_tracks} \
-# --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser=TPC/Calib/LaserTracks" \
-# | o2-tpc-calib-pad-raw ${ARGS_ALL} \
-# --configKeyValues ${CALIB_CONFIG}  ${EXTRA_CONFIG} \
-# | o2-calibration-ccdb-populator-workflow  ${ARGS_ALL} \
-# --ccdb-path ${CCDB_PATH} \
-# | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE}
 

DATA/production/calib/tpc-laser-filter.sh

@@ -47,7 +47,7 @@ PROXY_OUTSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;D:TPC/CLUSREFS"
 HOST=localhost
 
 QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-laser-calib-qcmn"
-QC_CONFIG_CONSUL=/o2/components/qc/ANY/any/tpc-laser-calib-qcmn
+QC_CONFIG="components/qc/ANY/any/tpc-laser-calib-qcmn"
 
 
 RAWDIGIT_CONFIG="TPCDigitDump.NoiseThreshold=3;TPCDigitDump.LastTimeBin=600;NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR2;NameConf.mDirCollContext=$FILEWORKDIR;NameConf.mDirMatLUT=$FILEWORKDIR"
@@ -74,7 +74,7 @@ add_W o2-tpc-raw-to-digits-workflow "--ignore-grp --input-spec \"$CALIB_INSPEC\"
 add_W o2-tpc-reco-workflow " ${TPC_CORR_SCALING:-} --disable-ctp-lumi-request --input-type digitizer --output-type \"tracks,disable-writer,clusters\" --disable-mc --pipeline tpc-zsEncoder:20,tpc-tracker:8 ${GPU_CONFIG} ${REMAP} " "${RECO_CONFIG}"
 add_W o2-tpc-laser-track-filter "" "" 0
 add_W o2-dpl-output-proxy " --proxy-name tpc-laser-input-proxy --proxy-channel-name tpc-laser-input-proxy --dataspec \"$PROXY_OUTSPEC\" --channel-config \"name=tpc-laser-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" " "" 0
-add_QC_from_consul "${QC_CONFIG_CONSUL}" "--local --host lcoalhost"
+add_QC_from_apricot "${QC_CONFIG}" "--local --host lcoalhost"
 
 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"
 if [ $WORKFLOWMODE == "print" ]; then
@@ -86,30 +86,3 @@ else
   eval $WORKFLOW
 fi
 
-#o2-dpl-raw-proxy ${ARGS_ALL} \
-#    --dataspec "$PROXY_INSPEC" --inject-missing-data \
-#    --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \
-#    | o2-tpc-raw-to-digits-workflow ${ARGS_ALL}  ${LASER_DECODER_ADD} \
-#    --input-spec "$CALIB_INSPEC"  \
-#    --configKeyValues "TPCDigitDump.NoiseThreshold=3;TPCDigitDump.LastTimeBin=600;$ARGS_ALL_CONFIG" \
-#    --pipeline tpc-raw-to-digits-0:20 \
-#    --remove-duplicates \
-#    --send-ce-digits \
-#    | o2-tpc-reco-workflow ${ARGS_ALL}  ${TPC_CORR_SCALING:-} \
-#    --input-type digitizer  \
-#    --output-type "tracks,disable-writer,clusters" \
-#    --disable-ctp-lumi-request \
-#    --disable-mc \
-#    --pipeline tpc-zsEncoder:20,tpc-tracker:8 \
-#    ${GPU_CONFIG} \
-#    --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser=TPC/Calib/LaserTracks" \
-#    --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1;GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;GPU_rec_tpc.clustersShiftTimebinsClusterizer=35;GPU_proc.memoryScalingFactor=2;GPU_proc_param.tpcTriggerHandling=0" \
-#    | o2-tpc-laser-track-filter ${ARGS_ALL}  \
-#    | o2-dpl-output-proxy ${ARGS_ALL} \
-#    --dataspec "$PROXY_OUTSPEC" \
-#    --proxy-name tpc-laser-input-proxy \
-#    --proxy-channel-name tpc-laser-input-proxy \
-#    --channel-config "name=tpc-laser-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0" \
-#    | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \
-#    | o2-dpl-run ${ARGS_ALL}  --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT}
-

DATA/production/calib/tpc-laser.sh

@@ -48,7 +48,7 @@ HOST=localhost
 
 
 QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn"
-QC_CONFIG_CONSUL="/o2/components/qc/ANY/any/tpc-raw-qcmn"
+QC_CONFIG="components/qc/ANY/any/tpc-raw-qcmn"
 
 max_events=300
 publish_after=440
@@ -95,7 +95,7 @@ add_W o2-tpc-laser-track-filter "" "" 0
 add_W o2-tpc-calib-laser-tracks "--use-filtered-tracks ${EXTRA_CONFIG_TRACKS} --min-tfs=${min_tracks}"
 add_W o2-tpc-calib-pad-raw " ${EXTRA_CONFIG}" "${CALIB_CONFIG}"
 add_W o2-calibration-ccdb-populator-workflow "--ccdb-path ${CCDB_PATH}" "" 0
-add_QC_from_consul "${QC_CONFIG_CONSUL}" "--local --host lcoalhost"
+add_QC_from_apricot "${QC_CONFIG_CONSUL}" "--local --host lcoalhost"
 
 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"
 if [ $WORKFLOWMODE == "print" ]; then

DATA/production/calib/tpc-pedestal.sh

@@ -18,7 +18,9 @@ CCDB_PATH="http://o2-ccdb.internal"
 HOST=localhost
 
 #QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-pedestal-calib-qcmn"
-QC_CONFIG="/o2/components/qc/ANY/any/tpc-pedestal-calib-qcmn"
+QC_CONFIG="components/qc/ANY/any/tpc-pedestal-calib-qcmn"
+CALIB_CONFIG="TPCCalibPedestal.ADCMin=20"
+
 
 max_events=50
 publish_after=400
@@ -37,9 +39,9 @@ fi
 
 WORKFLOW=
 add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0
-add_W o2-tpc-calib-pad-raw "--input-spec \"$CALIB_INSPEC\"  --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36" 
+add_W o2-tpc-calib-pad-raw "--input-spec \"$CALIB_INSPEC\" --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36" "${CALIB_CONFIG}"
 add_W o2-calibration-ccdb-populator-workflow "--ccdb-path \"http://o2-ccdb.internal\" " "" 0
-add_QC_from_consul "${QC_CONFIG}" "--local --host localhost"
+add_QC_from_apricot "${QC_CONFIG}" "--local --host localhost"
 
 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"
 

DATA/production/calib/tpc-pulser-long.sh

@@ -18,7 +18,7 @@ CCDB_PATH="http://o2-ccdb.internal"
 HOST=localhost
 
 #QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn"
-QC_CONFIG="/o2/components/qc/ANY/any/tpc-pulser-calib-qcmn"
+QC_CONFIG="components/qc/ANY/any/tpc-pulser-calib-qcmn"
 max_events=1000000
 publish_after=200
 
@@ -38,7 +38,7 @@ WORKFLOW=
 add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0
 add_W o2-tpc-calib-pad-raw "--input-spec \"$CALIB_INSPEC\" --calib-type pulser --reset-after-publish --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36 --check-calib-infos" 
 add_W o2-calibration-ccdb-populator-workflow "--ccdb-path \"http://o2-ccdb.internal\" " "" 0
-add_QC_from_consul "${QC_CONFIG}" "--local --host localhost"
+add_QC_from_apricot "${QC_CONFIG}" "--local --host localhost"
 
 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"
 

DATA/production/calib/tpc-pulser.sh

@@ -19,7 +19,7 @@ CCDB_PATH="http://o2-ccdb.internal"
 
 HOST=localhost
 
-QC_CONFIG="/o2/components/qc/ANY/any/tpc-pulser-calib-qcmn"
+QC_CONFIG="components/qc/ANY/any/tpc-pulser-calib-qcmn"
 
 max_events=200
 publish_after=230
@@ -40,7 +40,7 @@ WORKFLOW=
 add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0
 add_W o2-tpc-calib-pad-raw "--input-spec \"$CALIB_INSPEC\" --calib-type pulser --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36 --check-calib-infos" "${CALIB_CONFIG}" 
 add_W o2-calibration-ccdb-populator-workflow "--ccdb-path \"http://o2-ccdb.internal\" " "" 0
-add_QC_from_consul "${QC_CONFIG}" "--local --host localhost"
+add_QC_from_apricot "${QC_CONFIG}" "--local --host localhost"
 
 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"
 

DATA/production/common/getStat.sh

@@ -12,7 +12,7 @@ nCTFsFilesFailed=`grep "FileFetcher: non-zero exit code [0-9]*" $1 | wc -l`
 nCTFsProcessed=`grep "Read CTF" $1 | tail -1 | sed 's/^.*Read CTF \([0-9]*\).*$/\1/'`
 nCTFsProcessed=$((nCTFsProcessed + 1))
 
-if [[ $nCTFsFilesInspected != $((nCTFsFilesFailed + nCTFsFilesOK)) ]]; then
+if [[ $nCTFsFilesInspected -gt 0 && $nCTFsFilesInspected != $((nCTFsFilesFailed + nCTFsFilesOK)) ]]; then
   echo "Something went wrong with parsing the log file: CTF files inspected ($nCTFsFilesInspected) is not the sum of those successfully processed ($nCTFsFilesOK) and those that failed ($nCTFsFilesFailed)"
   exit 8
 fi

DATA/production/configurations/CTFSkimming/ctf-skim-workflow.sh

@@ -3,6 +3,22 @@
 # Non-zero exit code already if one command in a pipe fails
 set -o pipefail
 
+if [[ -z "$NHBPERTF" ]]; then  # try to extract from JDL variables
+  NHBPERTF=32
+  if [[ ! -z "$ALIEN_JDL_NHBPERTF" ]] && [[ "$ALIEN_JDL_NHBPERTF" != 0 ]]; then
+    NHBPERTF=$ALIEN_JDL_NHBPERTF
+    echo "Set NHBF per TF to $NHBPERTF from ALIEN_JDL_NHBPERTF"
+  elif [[ ! -z "$ALIEN_JDL_LPMRUNNUMBER" ]]; then
+    [[ "$ALIEN_JDL_LPMRUNNUMBER" -lt 534125 ]] && NHBPERTF=128 || NHBPERTF=32
+    echo "Set NHBF per TF to $NHBPERTF from ALIEN_JDL_LPMRUNNUMBER=$ALIEN_JDL_LPMRUNNUMBER"
+  else
+    echo "Set NHBF per TF to $NHBPERTF as default"
+  fi
+else
+  echo "Set NHBF per TF to $NHBPERTF as requested explicitly"
+fi
+export NHBPERTF
+
 # ---------------------------------------------------------------------------------------------------------------------
 # Get this script's directory and load common settings (use zsh first (e.g. on Mac) and fallback on `readlink -f` if zsh is not there)
 : ${GEN_TOPO_MYDIR:="$O2DPG_ROOT/DATA/production"}
@@ -26,6 +42,7 @@ if [[ ! -z "$ALIEN_JDL_SHMSIZE" ]]; then export SHMSIZE=$ALIEN_JDL_SHMSIZE; fi
 if [[ ! -z "$ALIEN_JDL_MULTIPLICITYPROCESSTPCENTROPYDECODER" ]]; then export MULTIPLICITY_PROCESS_tpc_entropy_decoder=$ALIEN_JDL_MULTIPLICITYPROCESSTPCENTROPYDECODER; fi
 if [[ ! -z "$ALIEN_JDL_MULTIPLICITYPROCESSTPCENTROPYENCODER" ]]; then export MULTIPLICITY_PROCESS_tpc_entropy_encoder=$ALIEN_JDL_MULTIPLICITYPROCESSTPCENTROPYENCODER; fi
 
+
 # Set general arguments
 source $GEN_TOPO_MYDIR/getCommonArgs.sh || { echo "getCommonArgs.sh failed" 1>&2 && exit 1; }
 
@@ -86,7 +103,7 @@ if [[ -z ${NO_ITSMFT_MASKING:-} ]] ; then
   has_detector_ctf MFT && export ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow="$ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow --mft-digits"
 fi
 
-add_W o2-ctf-reader-workflow "--ctf-data-subspec 1 --ir-frames-files $IRFRAMES $SKIP_SKIMMED_OUT_TF --ctf-input $CTFLIST ${INPUT_FILE_COPY_CMD+--copy-cmd} ${INPUT_FILE_COPY_CMD} --onlyDet $WORKFLOW_DETECTORS $ALLOW_MISSING_DET --pipeline $(get_N tpc-entropy-decoder TPC REST 1 TPCENTDEC)"
+add_W o2-ctf-reader-workflow "--ctf-data-subspec 1 --ir-frames-files $IRFRAMES $SKIP_SKIMMED_OUT_TF --ctf-input $CTFLIST ${INPUT_FILE_COPY_CMD+--copy-cmd} ${INPUT_FILE_COPY_CMD} --onlyDet $WORKFLOW_DETECTORS $ALLOW_MISSING_DET --pipeline $(get_N tpc-entropy-decoder TPC REST 1 TPCENTDEC)" "HBFUtils.nHBFPerTF=$NHBPERTF"
 
 if [[ -z ${NO_ITSMFT_MASKING:-} ]] ; then
   has_detector_ctf ITS && add_W o2-its-reco-workflow "--digits-from-upstream --disable-mc --disable-tracking --disable-root-output --pipeline $(get_N its-tracker ITS REST 1 ITSTRK)" "ITSClustererParam.maxBCDiffToMaskBias=10;"

DATA/production/configurations/asyncReco/async_pass.sh

@@ -23,23 +23,28 @@ timeStartFullProcessing=`date +%s`
 export inputarg="${1}"
 
 if [[ "${1##*.}" == "root" ]]; then
-    #echo ${1##*.}
-    #echo "alien://${1}" > list.list
-    #export MODE="remote"
-    echo "${1}" > list.list
-    if [[ ! -z $ASYNC_BENCHMARK_ITERATIONS ]]; then
-      for i in `seq 1 $ASYNC_BENCHMARK_ITERATIONS`; do echo "${1}" >> list.list; done
-    fi
-    export MODE="LOCAL"
-    shift
+  #echo ${1##*.}
+  #echo "alien://${1}" > list.list
+  #export MODE="remote"
+  echo "${1}" > list.list
+  if [[ ! -z $ASYNC_BENCHMARK_ITERATIONS ]]; then
+    for i in `seq 1 $ASYNC_BENCHMARK_ITERATIONS`; do echo "${1}" >> list.list; done
+  fi
+  export MODE="LOCAL"
+  shift
 elif [[ "${1##*.}" == "xml" ]]; then
+  if [[ $ALIEN_JDL_DOWNLOADINPUTFILES == "1" ]]; then
+    echo "Downloading input files done by the job agent"
+    sed -rn 's/.*file\ name="(o2_ctf[^"]*)".*/\1/p' $1 > list.list
+  else
     sed -rn 's/.*turl="([^"]*)".*/\1/p' $1 > list.list
-    export MODE="remote"
-    shift
+  fi
+  export MODE="remote"
+  shift
 elif [[ $1 != "list.list" && "${1##*.}" == "list" ]]; then
-    cp $1 list.list
-    export MODE="remote"
-    shift
+  cp $1 list.list
+  export MODE="remote"
+  shift
 fi
 
 # Could need sometimes to iterate just a subset of the input files
@@ -717,15 +722,15 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then
     if (( $(echo "$PERCENT < $MIN_ALLOWED_AOD_PERCENT_SIZE" | bc -l) )); then
       AOD_LAST_BUT_ONE=`find . -name AO2D.root | sort | tail -2 | head -1`
       echo "Too small, merging $AOD_LAST with previous file $AOD_LAST_BUT_ONE"
-      ls $PWD/$AOD_LAST > list.list
-      ls $PWD/$AOD_LAST_BUT_ONE >> list.list
+      ls $PWD/$AOD_LAST > listAOD.list
+      ls $PWD/$AOD_LAST_BUT_ONE >> listAOD.list
       echo "List of files for merging:"
-      cat list.list
+      cat listAOD.list
       mkdir tmpAOD
       cd tmpAOD
-      ln -s ../list.list .
+      ln -s ../listAOD.list .
       timeStart=`date +%s`
-      time o2-aod-merger --input list.list
+      time o2-aod-merger --input listAOD.list
       exitcode=$?
       timeEnd=`date +%s`
       timeUsed=$(( $timeUsed+$timeEnd-$timeStart ))