diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1ea6093d3c2..12f9c77ef6d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -126,4 +126,6 @@ jobs: python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid GEN-Run3Summer22EEwmLHEGS-00403 python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TRK-Run3Summer23wmLHEGS-00002 python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TSG-Run3Summer23BPixGS-00007 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TOP-RunIISummer20UL16wmLHEGEN-00883 \ No newline at end of file + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TOP-RunIISummer20UL16wmLHEGEN-00883 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid BPH-Run3Winter24GS-00001 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid GEN-RunIII2024Summer24wmLHEGS-00001 diff --git a/bin/utils/BPH-Run3Winter24GS-00001 b/bin/utils/BPH-Run3Winter24GS-00001 new file mode 100644 index 00000000000..692f7e90b52 --- /dev/null +++ b/bin/utils/BPH-Run3Winter24GS-00001 @@ -0,0 +1,96 @@ +import FWCore.ParameterSet.Config as cms + +from Configuration.Generator.Pythia8CommonSettings_cfi import * +from GeneratorInterface.EvtGenInterface.EvtGenSetting_cff import * +from Configuration.Generator.MCTunesRun3ECM13p6TeV.PythiaCP5Settings_cfi import * + +_generator = cms.EDFilter("Pythia8GeneratorFilter", + pythiaPylistVerbosity = cms.untracked.int32(0), + pythiaHepMCVerbosity = cms.untracked.bool(False), + comEnergy = cms.double(13600.0), + maxEventsToPrint = cms.untracked.int32(0), + ExternalDecays = cms.PSet( + EvtGen130 = cms.untracked.PSet( + decay_table = cms.string('GeneratorInterface/EvtGenInterface/data/DECAY_2020_NOLONGLIFE.DEC'), + particle_property_file = cms.FileInPath('GeneratorInterface/EvtGenInterface/data/evt_2020.pdl'), + user_decay_embedded= cms.vstring ( + +'Define Hp 0.49', +'Define Hz 0.775', +'Define Hm 0.4', +'Define pHp 2.50', +'Define pHz 0.0', +'Define pHm -0.17', +'#', +'Alias MyB_s0 B_s0', +'Alias Myanti-B_s0 anti-B_s0', +'ChargeConj Myanti-B_s0 MyB_s0', +'Alias MyPhi phi', +'ChargeConj MyPhi MyPhi', +'#', +'Decay MyB_s0', +' 1.000 MyPhi MyPhi PVV_CPLH 0.02 1 Hp pHp Hz pHz Hm pHm;', +'#', +'Enddecay', +'Decay Myanti-B_s0', + ' 1.000 MyPhi MyPhi PVV_CPLH 0.02 1 Hp pHp Hz pHz Hm pHm;', +'Enddecay', +'#', +'Decay MyPhi', + ' 1.000 K+ K- VSS;', +'Enddecay', +'End' + ), + list_forced_decays = cms.vstring('MyB_s0','Myanti-B_s0'), + operates_on_particles = cms.vint32(), + convertPythiaCodes = cms.untracked.bool(False) + ), + parameterSets = cms.vstring('EvtGen130') + ), + PythiaParameters = cms.PSet( + pythia8CommonSettingsBlock, + pythia8CP5SettingsBlock, + processParameters = cms.vstring( + 'SoftQCD:nonDiffractive = on', + 'PTFilter:filter = on', # this turn on the filter + 'PTFilter:quarkToFilter = 5', # PDG id of q quark + 'PTFilter:scaleToFilter = 1.0'), + parameterSets = cms.vstring( + 'pythia8CommonSettings', + 'pythia8CP5Settings', + 'processParameters') + ) + ) + +from GeneratorInterface.Core.ExternalGeneratorFilter import ExternalGeneratorFilter +generator = ExternalGeneratorFilter(_generator) + +generator.PythiaParameters.processParameters.extend(EvtGenExtraParticles) + +bfilter = cms.EDFilter( + "PythiaFilter", + MaxEta = cms.untracked.double(9999.), + MinEta = cms.untracked.double(-9999.), + ParticleID = cms.untracked.int32(531) + ) + +decayfilter = cms.EDFilter( + "PythiaDauVFilter", + verbose = cms.untracked.int32(1), + NumberDaughters = cms.untracked.int32(2), + ParticleID = cms.untracked.int32(531), ## Bs0 + DaughterIDs = cms.untracked.vint32(333, 333), ## Phi and Phi + MinPt = cms.untracked.vdouble(-1., -1.), + MinEta = cms.untracked.vdouble(-9999., -9999.), + MaxEta = cms.untracked.vdouble( 9999., 9999.) + ) + +MuFilter = cms.EDFilter( + "PythiaFilter", + ParticleID = cms.untracked.int32(13), + MinPt = cms.untracked.double(4.0), + MinEta = cms.untracked.double(-2.4), + MaxEta = cms.untracked.double(2.4) +) + +ProductionFilterSequence = cms.Sequence(generator*bfilter*decayfilter*MuFilter) \ No newline at end of file diff --git a/bin/utils/BPH-Run3Winter24GS-00001_get_test b/bin/utils/BPH-Run3Winter24GS-00001_get_test new file mode 100644 index 00000000000..c9d46fe9b5c --- /dev/null +++ b/bin/utils/BPH-Run3Winter24GS-00001_get_test @@ -0,0 +1,155 @@ +#!/bin/bash + +# Binds for singularity containers +# Mount /afs, /eos, /cvmfs, /etc/grid-security for xrootd +export APPTAINER_BINDPATH='/afs,/cvmfs,/cvmfs/grid.cern.ch/etc/grid-security:/etc/grid-security,/eos,/etc/pki/ca-trust,/run/user,/var/run/user' + +cat <<'EndOfGenScriptFile' > BPH-Run3Winter24GS-00001_gen_script.sh +#!/bin/bash + +echo "Running CMS GEN request script using cms-sw containers. Architecture: el9:x86_64" +python3 -m venv cms_gen_venv_BPH-Run3Winter24GS-00001 && source ./cms_gen_venv_BPH-Run3Winter24GS-00001/bin/activate + +# Install the PdmV REST client +pip install git+https://github.com/cms-PdmV/mcm_scripts &> /dev/null + +echo "Packages installed" +pip freeze +echo "" + +# GEN Script begin +rm -f request_fragment_check.py +wget -q https://raw.githubusercontent.com/cms-sw/genproductions/master/bin/utils/request_fragment_check.py +chmod +x request_fragment_check.py + +./request_fragment_check.py --bypass_status --prepid BPH-Run3Winter24GS-00001 + +# End of CMS GEN script file: BPH-Run3Winter24GS-00001_gen_script.sh +EndOfGenScriptFile +chmod +x BPH-Run3Winter24GS-00001_gen_script.sh + +# Run in singularity container +singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el9:x86_64 $(echo $(pwd)/BPH-Run3Winter24GS-00001_gen_script.sh) + +GEN_ERR=$? +if [ $GEN_ERR -ne 0 ]; then + echo "GEN Checking Script returned exit code $GEN_ERR which means there are $GEN_ERR errors" + echo "Validation WILL NOT RUN" + echo "Please correct errors in the request and run validation again" + exit $GEN_ERR +fi +echo "Running VALIDATION. GEN Request Checking Script returned no errors" +# GEN Script end + +# Download fragment from McM +curl -s -k https://cms-pdmv-prod.web.cern.ch/mcm/public/restapi/requests/get_fragment/BPH-Run3Winter24GS-00001 --retry 3 --create-dirs -o Configuration/GenProduction/python/BPH-Run3Winter24GS-00001-fragment.py +[ -s Configuration/GenProduction/python/BPH-Run3Winter24GS-00001-fragment.py ] || exit $?; + +# Check if fragment contais gridpack path ant that it is in cvmfs +if grep -q "gridpacks" Configuration/GenProduction/python/BPH-Run3Winter24GS-00001-fragment.py; then + if ! grep -q "/cvmfs/cms.cern.ch/phys_generator/gridpacks" Configuration/GenProduction/python/BPH-Run3Winter24GS-00001-fragment.py; then + echo "Gridpack inside fragment is not in cvmfs." + exit -1 + fi +fi + +# Dump actual test code to a BPH-Run3Winter24GS-00001_test.sh file that can be run in Singularity +cat <<'EndOfTestFile' > BPH-Run3Winter24GS-00001_test.sh +#!/bin/bash + +export SCRAM_ARCH=el8_amd64_gcc12 + +source /cvmfs/cms.cern.ch/cmsset_default.sh +if [ -r CMSSW_13_3_0/src ] ; then + echo release CMSSW_13_3_0 already exists +else + scram p CMSSW CMSSW_13_3_0 +fi +cd CMSSW_13_3_0/src +eval `scram runtime -sh` + +mv ../../Configuration . +scram b +cd ../.. + +# Maximum validation duration: 28800s +# Margin for validation duration: 30% +# Validation duration with margin: 28800 * (1 - 0.30) = 20160s +# Time per event for each sequence: 0.5436s +# Threads for each sequence: 1 +# Time per event for single thread for each sequence: 1 * 0.5436s = 0.5436s +# Which adds up to 0.5436s per event +# Single core events that fit in validation duration: 20160s / 0.5436s = 37086 +# Produced events limit in McM is 10000 +# According to 0.0017 efficiency, validation should run 10000 / 0.0017 = 5882352 events to reach the limit of 10000 +# Take the minimum of 37086 and 5882352, but more than 0 -> 37086 +# It is estimated that this validation will produce: 37086 * 0.0017 = 63 events +EVENTS=37086 + + +# cmsDriver command +cmsDriver.py Configuration/GenProduction/python/BPH-Run3Winter24GS-00001-fragment.py --python_filename BPH-Run3Winter24GS-00001_1_cfg.py --eventcontent RAWSIM --customise Configuration/DataProcessing/Utils.addMonitoring --datatier GEN-SIM --fileout file:BPH-Run3Winter24GS-00001.root --conditions 133X_mcRun3_2024_realistic_v10 --beamspot Realistic25ns13p6TeVEarly2023Collision --customise_commands process.source.numberEventsInLuminosityBlock="cms.untracked.uint32(58823)" --step GEN,SIM --geometry DB:Extended --era Run3_2023 --no_exec --mc -n $EVENTS || exit $? ; + +# Run generated config +REPORT_NAME=BPH-Run3Winter24GS-00001_report.xml +# Run the cmsRun +cmsRun -e -j $REPORT_NAME BPH-Run3Winter24GS-00001_1_cfg.py || exit $? ; + +# Parse values from BPH-Run3Winter24GS-00001_report.xml report +processedEvents=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +producedEvents=$(grep -Po "(?<=)(\d*)(?=)" $REPORT_NAME | tail -n 1) +threads=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueRss=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueVsize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSizeAlt=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobCPU=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +eventThroughput=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +avgEventTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +if [ -z "$threads" ]; then + echo "Could not find NumberOfThreads in report, defaulting to 1" + threads=1 +fi +if [ -z "$eventThroughput" ]; then + eventThroughput=$(bc -l <<< "scale=4; 1 / ($avgEventTime / $threads)") +fi +if [ -z "$totalSize" ]; then + totalSize=$totalSizeAlt +fi +if [ -z "$processedEvents" ]; then + processedEvents=$EVENTS +fi +echo "Validation report of BPH-Run3Winter24GS-00001 sequence 1/1" +echo "Processed events: $processedEvents" +echo "Produced events: $producedEvents" +echo "Threads: $threads" +echo "Peak value RSS: $peakValueRss MB" +echo "Peak value Vsize: $peakValueVsize MB" +echo "Total size: $totalSize MB" +echo "Total job time: $totalJobTime s" +echo "Total CPU time: $totalJobCPU s" +echo "Event throughput: $eventThroughput" +echo "CPU efficiency: "$(bc -l <<< "scale=2; ($totalJobCPU * 100) / ($threads * $totalJobTime)")" %" +echo "Size per event: "$(bc -l <<< "scale=4; ($totalSize * 1024 / $producedEvents)")" kB" +echo "Time per event: "$(bc -l <<< "scale=4; (1 / $eventThroughput)")" s" +echo "Filter efficiency percent: "$(bc -l <<< "scale=8; ($producedEvents * 100) / $processedEvents")" %" +echo "Filter efficiency fraction: "$(bc -l <<< "scale=10; ($producedEvents) / $processedEvents") + +# End of BPH-Run3Winter24GS-00001_test.sh file +EndOfTestFile + +# Make file executable +chmod +x BPH-Run3Winter24GS-00001_test.sh + +if [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el8:amd64" ]; then + CONTAINER_NAME="el8:amd64" +elif [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el8:x86_64" ]; then + CONTAINER_NAME="el8:x86_64" +else + echo "Could not find amd64 or x86_64 for el8" + exit 1 +fi +# Run in singularity container +export SINGULARITY_CACHEDIR="/tmp/$(whoami)/singularity" +singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/$CONTAINER_NAME $(echo $(pwd)/BPH-Run3Winter24GS-00001_test.sh) diff --git a/bin/utils/GEN-RunIII2024Summer24wmLHEGS-00001 b/bin/utils/GEN-RunIII2024Summer24wmLHEGS-00001 new file mode 100644 index 00000000000..33a6d0fd5aa --- /dev/null +++ b/bin/utils/GEN-RunIII2024Summer24wmLHEGS-00001 @@ -0,0 +1,40 @@ +import FWCore.ParameterSet.Config as cms + +externalLHEProducer = cms.EDProducer('ExternalLHEProducer', + args = cms.vstring('/cvmfs/cms.cern.ch/phys_generator/gridpacks/PdmV/RunIII2024Summer24/Powheg/TT/hvq_slc7_amd64_gcc10_CMSSW_12_4_8_TTtoLNu2Q_powheg-pythia8.tgz'), + nEvents = cms.untracked.uint32(5000), + numberOfParameters = cms.uint32(1), + outputFile = cms.string('cmsgrid_final.lhe'), + scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh'), + generateConcurrently = cms.untracked.bool(True), +) + +from Configuration.Generator.Pythia8CommonSettings_cfi import * +from Configuration.Generator.MCTunesRun3ECM13p6TeV.PythiaCP5Settings_cfi import * +from Configuration.Generator.Pythia8PowhegEmissionVetoSettings_cfi import * +from Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import * + +generator = cms.EDFilter("Pythia8ConcurrentHadronizerFilter", + PythiaParameters = cms.PSet( + pythia8CommonSettingsBlock, + pythia8CP5SettingsBlock, + pythia8PowhegEmissionVetoSettingsBlock, + pythia8PSweightsSettingsBlock, + processParameters = cms.vstring( + 'POWHEG:nFinal = 2', + 'TimeShower:mMaxGamma = 4.0' + ), + parameterSets = cms.vstring( + 'pythia8CommonSettings', + 'pythia8CP5Settings', + 'pythia8PowhegEmissionVetoSettings', + 'processParameters', + 'pythia8PSweightsSettings', + ) + ), + comEnergy = cms.double(13600), + maxEventsToPrint = cms.untracked.int32(1), + pythiaHepMCVerbosity = cms.untracked.bool(False), + pythiaPylistVerbosity = cms.untracked.int32(1), +) + diff --git a/bin/utils/GEN-RunIII2024Summer24wmLHEGS-00001_get_test b/bin/utils/GEN-RunIII2024Summer24wmLHEGS-00001_get_test new file mode 100644 index 00000000000..e14f1edccab --- /dev/null +++ b/bin/utils/GEN-RunIII2024Summer24wmLHEGS-00001_get_test @@ -0,0 +1,158 @@ +#!/bin/bash + +# Binds for singularity containers +# Mount /afs, /eos, /cvmfs, /etc/grid-security for xrootd +export APPTAINER_BINDPATH='/afs,/cvmfs,/cvmfs/grid.cern.ch/etc/grid-security:/etc/grid-security,/eos,/etc/pki/ca-trust,/run/user,/var/run/user' + +cat <<'EndOfGenScriptFile' > GEN-RunIII2024Summer24wmLHEGS-00001_gen_script.sh +#!/bin/bash + +echo "Running CMS GEN request script using cms-sw containers. Architecture: el9:x86_64" +python3 -m venv cms_gen_venv_GEN-RunIII2024Summer24wmLHEGS-00001 && source ./cms_gen_venv_GEN-RunIII2024Summer24wmLHEGS-00001/bin/activate + +# Install the PdmV REST client +pip install git+https://github.com/cms-PdmV/mcm_scripts &> /dev/null + +echo "Packages installed" +pip freeze +echo "" + +# GEN Script begin +rm -f request_fragment_check.py +wget -q https://raw.githubusercontent.com/cms-sw/genproductions/master/bin/utils/request_fragment_check.py +chmod +x request_fragment_check.py + +./request_fragment_check.py --bypass_status --prepid GEN-RunIII2024Summer24wmLHEGS-00001 + +# End of CMS GEN script file: GEN-RunIII2024Summer24wmLHEGS-00001_gen_script.sh +EndOfGenScriptFile +chmod +x GEN-RunIII2024Summer24wmLHEGS-00001_gen_script.sh + +# Run in singularity container +singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el9:x86_64 $(echo $(pwd)/GEN-RunIII2024Summer24wmLHEGS-00001_gen_script.sh) + +GEN_ERR=$? +if [ $GEN_ERR -ne 0 ]; then + echo "GEN Checking Script returned exit code $GEN_ERR which means there are $GEN_ERR errors" + echo "Validation WILL NOT RUN" + echo "Please correct errors in the request and run validation again" + exit $GEN_ERR +fi +echo "Running VALIDATION. GEN Request Checking Script returned no errors" +# GEN Script end + +# Download fragment from McM +curl -s -k https://cms-pdmv-prod.web.cern.ch/mcm/public/restapi/requests/get_fragment/GEN-RunIII2024Summer24wmLHEGS-00001 --retry 3 --create-dirs -o Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py +[ -s Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py ] || exit $?; + +# Check if fragment contais gridpack path ant that it is in cvmfs +if grep -q "gridpacks" Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py; then + if ! grep -q "/cvmfs/cms.cern.ch/phys_generator/gridpacks" Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py; then + echo "Gridpack inside fragment is not in cvmfs." + exit -1 + fi +fi + +# Dump actual test code to a GEN-RunIII2024Summer24wmLHEGS-00001_test.sh file that can be run in Singularity +cat <<'EndOfTestFile' > GEN-RunIII2024Summer24wmLHEGS-00001_test.sh +#!/bin/bash + +export SCRAM_ARCH=el8_amd64_gcc12 + +source /cvmfs/cms.cern.ch/cmsset_default.sh +if [ -r CMSSW_14_0_18/src ] ; then + echo release CMSSW_14_0_18 already exists +else + scram p CMSSW CMSSW_14_0_18 +fi +cd CMSSW_14_0_18/src +eval `scram runtime -sh` + +mv ../../Configuration . +scram b +cd ../.. + +# Maximum validation duration: 28800s +# Margin for validation duration: 30% +# Validation duration with margin: 28800 * (1 - 0.30) = 20160s +# Time per event for each sequence: 22.1832s +# Threads for each sequence: 4 +# Time per event for single thread for each sequence: 4 * 22.1832s = 88.7329s +# Which adds up to 88.7329s per event +# Single core events that fit in validation duration: 20160s / 88.7329s = 227 +# Produced events limit in McM is 10000 +# According to 1.0000 efficiency, validation should run 10000 / 1.0000 = 10000 events to reach the limit of 10000 +# Take the minimum of 227 and 10000, but more than 0 -> 227 +# It is estimated that this validation will produce: 227 * 1.0000 = 227 events +EVENTS=227 + +# Random seed between 1 and 100 for externalLHEProducer +SEED=$(($(date +%s) % 100 + 1)) + + +# cmsDriver command +cmsDriver.py Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py --python_filename GEN-RunIII2024Summer24wmLHEGS-00001_1_cfg.py --eventcontent RAWSIM,LHE --customise Configuration/DataProcessing/Utils.addMonitoring --datatier GEN-SIM,LHE --fileout file:GEN-RunIII2024Summer24wmLHEGS-00001.root --conditions 140X_mcRun3_2024_realistic_v26 --beamspot DBrealistic --customise_commands process.RandomNumberGeneratorService.externalLHEProducer.initialSeed="int(${SEED})"\\nprocess.source.numberEventsInLuminosityBlock="cms.untracked.uint32(100)" --step LHE,GEN,SIM --geometry DB:Extended --era Run3_2024 --no_exec --mc -n $EVENTS || exit $? ; + +# Run generated config +REPORT_NAME=GEN-RunIII2024Summer24wmLHEGS-00001_report.xml +# Run the cmsRun +cmsRun -e -j $REPORT_NAME GEN-RunIII2024Summer24wmLHEGS-00001_1_cfg.py || exit $? ; + +# Parse values from GEN-RunIII2024Summer24wmLHEGS-00001_report.xml report +processedEvents=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +producedEvents=$(grep -Po "(?<=)(\d*)(?=)" $REPORT_NAME | tail -n 1) +threads=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueRss=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueVsize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSizeAlt=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobCPU=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +eventThroughput=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +avgEventTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +if [ -z "$threads" ]; then + echo "Could not find NumberOfThreads in report, defaulting to 1" + threads=1 +fi +if [ -z "$eventThroughput" ]; then + eventThroughput=$(bc -l <<< "scale=4; 1 / ($avgEventTime / $threads)") +fi +if [ -z "$totalSize" ]; then + totalSize=$totalSizeAlt +fi +if [ -z "$processedEvents" ]; then + processedEvents=$EVENTS +fi +echo "Validation report of GEN-RunIII2024Summer24wmLHEGS-00001 sequence 1/1" +echo "Processed events: $processedEvents" +echo "Produced events: $producedEvents" +echo "Threads: $threads" +echo "Peak value RSS: $peakValueRss MB" +echo "Peak value Vsize: $peakValueVsize MB" +echo "Total size: $totalSize MB" +echo "Total job time: $totalJobTime s" +echo "Total CPU time: $totalJobCPU s" +echo "Event throughput: $eventThroughput" +echo "CPU efficiency: "$(bc -l <<< "scale=2; ($totalJobCPU * 100) / ($threads * $totalJobTime)")" %" +echo "Size per event: "$(bc -l <<< "scale=4; ($totalSize * 1024 / $producedEvents)")" kB" +echo "Time per event: "$(bc -l <<< "scale=4; (1 / $eventThroughput)")" s" +echo "Filter efficiency percent: "$(bc -l <<< "scale=8; ($producedEvents * 100) / $processedEvents")" %" +echo "Filter efficiency fraction: "$(bc -l <<< "scale=10; ($producedEvents) / $processedEvents") + +# End of GEN-RunIII2024Summer24wmLHEGS-00001_test.sh file +EndOfTestFile + +# Make file executable +chmod +x GEN-RunIII2024Summer24wmLHEGS-00001_test.sh + +if [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el8:amd64" ]; then + CONTAINER_NAME="el8:amd64" +elif [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el8:x86_64" ]; then + CONTAINER_NAME="el8:x86_64" +else + echo "Could not find amd64 or x86_64 for el8" + exit 1 +fi +# Run in singularity container +export SINGULARITY_CACHEDIR="/tmp/$(whoami)/singularity" +singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/$CONTAINER_NAME $(echo $(pwd)/GEN-RunIII2024Summer24wmLHEGS-00001_test.sh) diff --git a/bin/utils/request_BPH-Run3Winter24GS-00001.json b/bin/utils/request_BPH-Run3Winter24GS-00001.json new file mode 100644 index 00000000000..657b49a700d --- /dev/null +++ b/bin/utils/request_BPH-Run3Winter24GS-00001.json @@ -0,0 +1 @@ +{"results": {"validation": {"content": "all", "valid": false, "results": {"1": [{"peak_value_rss": 1271.93, "total_events": 68, "estimated_events_per_lumi": 165.59004432939653, "cpu_efficiency": 0.9915296462381665, "filter_efficiency": 0.00183357601251146, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 0.31890195678240685, "expected_events": 37086, "size_per_event": 610.9876705882352}], "8": [{"peak_value_rss": 1756.94, "total_events": 77, "estimated_events_per_lumi": 1468.0672253680634, "cpu_efficiency": 0.9442697012877003, "filter_efficiency": 0.0020762551906379767, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 0.04073120662126495, "expected_events": 37086, "size_per_event": 583.2132155844156}], "2": [{"peak_value_rss": 1326.33, "total_events": 70, "estimated_events_per_lumi": 328.76249150622874, "cpu_efficiency": 0.978837657225031, "filter_efficiency": 0.001887504718761797, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 0.16534774283796252, "expected_events": 37086, "size_per_event": 547.0178742857142}], "4": [{"peak_value_rss": 1495.32, "total_events": 60, "estimated_events_per_lumi": 646.571881572561, "cpu_efficiency": 0.9724601838470821, "filter_efficiency": 0.0016178611875101116, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 0.07206376201663232, "expected_events": 37086, "size_per_event": 567.7687466666666}]}}, "total_events": 1000000, "config_id": [], "events_per_lumi": 0, "_rev": "9-4317e2845adca29eb1e8586a22e64b1e", "mcdb_id": -1, "transient_output_modules": [[]], "sequences": [{"eventcontent": ["RAWSIM"], "runsScenarioForMC": "", "donotDropOnInput": "", "extra": "", "slhc": "", "runsAndWeightsForMC": "", "outputCommand": "", "dropDescendant": false, "pileup": "", "hltProcess": "", "index": -1, "customise": "", "custom_conditions": "", "datatier": ["GEN-SIM"], "processName": "", "inputCommands": "", "particle_table": "", "conditions": "133X_mcRun3_2024_realistic_v10", "harvesting": "", "triggerResultsProcess": "", "inputEventContent": "", "beamspot": "Realistic25ns13p6TeVEarly2023Collision", "restoreRNDSeeds": "", "filtername": "", "repacked": "", "customise_commands": "", "step": ["GEN", "SIM"], "gflash": "", "procModifiers": "", "magField": "", "himix": false, "nThreads": 1, "scenario": "", "geometry": "DB:Extended", "nStreams": 0, "datamix": "", "inline_custom": false, "era": "Run3_2023"}], "block_black_list": [], "block_white_list": [], "priority": 20000, "process_string": "", "ppd_tags": [], "fragment_tag": "", "generator_parameters": [{"submission_details": {"submission_date": "2019-06-12-23-42", "author_email": "jordan.martins@cern.ch", "author_username": "jordanm", "author_name": "Jordan Martins"}, "match_efficiency_error": 0, "match_efficiency": 1, "filter_efficiency": 0.0017, "negative_weights_fraction": -1, "version": 0, "cross_section": 543600000, "filter_efficiency_error": 0.000412}, {"submission_details": {"author_email": "kirill.ivanov@cern.ch", "submission_date": "2024-10-06-22-12", "author_username": "kivanov", "author_name": "Kirill Ivanov"}, "match_efficiency_error": 0.0, "match_efficiency": 1.0, "filter_efficiency": 0.0017, "negative_weights_fraction": -1.0, "version": 1, "cross_section": 543600000.0, "filter_efficiency_error": 0.000412}], "flown_with": "", "interested_pwg": ["BPH"], "version": 0, "generators": ["pythia8", "evtGen"], "memory": 4000, "pilot": false, "type": "Prod", "cmssw_release": "CMSSW_13_3_0", "status": "validation", "keep_output": [true], "energy": 13.6, "tags": [], "fragment": "import FWCore.ParameterSet.Config as cms \n\nfrom Configuration.Generator.Pythia8CommonSettings_cfi import *\nfrom GeneratorInterface.EvtGenInterface.EvtGenSetting_cff import *\nfrom Configuration.Generator.MCTunesRun3ECM13p6TeV.PythiaCP5Settings_cfi import *\n\n_generator = cms.EDFilter(\"Pythia8GeneratorFilter\",\n pythiaPylistVerbosity = cms.untracked.int32(0),\n pythiaHepMCVerbosity = cms.untracked.bool(False),\n comEnergy = cms.double(13600.0),\n maxEventsToPrint = cms.untracked.int32(0),\n ExternalDecays = cms.PSet(\n EvtGen130 = cms.untracked.PSet(\n decay_table = cms.string('GeneratorInterface/EvtGenInterface/data/DECAY_2020_NOLONGLIFE.DEC'),\n particle_property_file = cms.FileInPath('GeneratorInterface/EvtGenInterface/data/evt_2020.pdl'),\n# user_decay_file = cms.FileInPath('GeneratorInterface/ExternalDecays/data/Bs_phiphi_4K.dec'),\n user_decay_embedded= cms.vstring (\n\n'Define Hp 0.49',\n'Define Hz 0.775',\n'Define Hm 0.4',\n'Define pHp 2.50',\n'Define pHz 0.0',\n'Define pHm -0.17',\n'#',\n'Alias MyB_s0 B_s0',\n'Alias Myanti-B_s0 anti-B_s0',\n'ChargeConj Myanti-B_s0 MyB_s0',\n'Alias MyPhi phi',\n'ChargeConj MyPhi MyPhi',\n'#',\n'Decay MyB_s0',\n' 1.000 MyPhi MyPhi PVV_CPLH 0.02 1 Hp pHp Hz pHz Hm pHm;',\n'#',\n'Enddecay',\n'Decay Myanti-B_s0',\n ' 1.000 MyPhi MyPhi PVV_CPLH 0.02 1 Hp pHp Hz pHz Hm pHm;',\n'Enddecay',\n'#',\n'Decay MyPhi',\n ' 1.000 K+ K- VSS;',\n'Enddecay',\n'End'\n ),\n list_forced_decays = cms.vstring('MyB_s0','Myanti-B_s0'),\n operates_on_particles = cms.vint32(),\n convertPythiaCodes = cms.untracked.bool(False)\n ),\n parameterSets = cms.vstring('EvtGen130')\n ),\n PythiaParameters = cms.PSet(\n pythia8CommonSettingsBlock,\n pythia8CP5SettingsBlock,\n processParameters = cms.vstring(\n 'SoftQCD:nonDiffractive = on',\n 'PTFilter:filter = on', # this turn on the filter\n 'PTFilter:quarkToFilter = 5', # PDG id of q quark\n 'PTFilter:scaleToFilter = 1.0'),\n parameterSets = cms.vstring(\n 'pythia8CommonSettings',\n 'pythia8CP5Settings',\n 'processParameters')\n )\n )\n\nfrom GeneratorInterface.Core.ExternalGeneratorFilter import ExternalGeneratorFilter\ngenerator = ExternalGeneratorFilter(_generator)\n\ngenerator.PythiaParameters.processParameters.extend(EvtGenExtraParticles)\n\nbfilter = cms.EDFilter(\n \"PythiaFilter\",\n MaxEta = cms.untracked.double(9999.),\n MinEta = cms.untracked.double(-9999.),\n ParticleID = cms.untracked.int32(531)\n )\n\ndecayfilter = cms.EDFilter(\n \"PythiaDauVFilter\",\n verbose = cms.untracked.int32(1),\n NumberDaughters = cms.untracked.int32(2),\n ParticleID = cms.untracked.int32(531), ## Bs0\n DaughterIDs = cms.untracked.vint32(333, 333), ## Phi and Phi\n MinPt = cms.untracked.vdouble(-1., -1.),\n MinEta = cms.untracked.vdouble(-9999., -9999.),\n MaxEta = cms.untracked.vdouble( 9999., 9999.)\n )\n\nMuFilter = cms.EDFilter(\n \"PythiaFilter\",\n ParticleID = cms.untracked.int32(13),\n # MotherID = cms.untracked.int32(531), #Bs0\n MinPt = cms.untracked.double(4.0),\n MinEta = cms.untracked.double(-2.4),\n MaxEta = cms.untracked.double(2.4)\n)\n\nProductionFilterSequence = cms.Sequence(generator*bfilter*decayfilter*MuFilter)", "time_event": [0.5435905241299833], "pwg": "BPH", "reqmgr_name": [], "approval": "validation", "name_of_fragment": "", "pileup_dataset_name": "", "analysis_id": [], "input_dataset": "", "member_of_chain": [], "prepid": "BPH-Run3Winter24GS-00001", "extension": 0, "size_event": [538], "notes": "", "completed_events": -1, "history": [{"action": "created", "updater": {"author_email": "kirill.ivanov@cern.ch", "submission_date": "2024-10-06-22-12", "author_username": "kivanov", "author_name": "Kirill Ivanov"}}, {"action": "clone", "step": "BPH-RunIIFall18GS-00198", "updater": {"author_email": "kirill.ivanov@cern.ch", "submission_date": "2024-10-06-22-12", "author_username": "kivanov", "author_name": "Kirill Ivanov"}}, {"action": "update", "step": "dataset_name, fragment", "updater": {"author_email": "kirill.ivanov@cern.ch", "submission_date": "2024-10-06-22-21", "author_username": "kivanov", "author_name": "Kirill Ivanov"}}, {"action": "update", "step": "fragment", "updater": {"author_email": "kirill.ivanov@cern.ch", "submission_date": "2024-10-06-23-07", "author_username": "kivanov", "author_name": "Kirill Ivanov"}}, {"action": "update", "step": "fragment", "updater": {"author_email": "kirill.ivanov@cern.ch", "submission_date": "2024-10-07-00-04", "author_username": "kivanov", "author_name": "Kirill Ivanov"}}, {"action": "update", "step": "fragment", "updater": {"author_email": "kirill.ivanov@cern.ch", "submission_date": "2024-10-07-13-43", "author_username": "kivanov", "author_name": "Kirill Ivanov"}}, {"action": "update", "step": "fragment", "updater": {"author_email": "kirill.ivanov@cern.ch", "submission_date": "2024-10-07-15-34", "author_username": "kivanov", "author_name": "Kirill Ivanov"}}, {"action": "approve", "step": "validation", "updater": {"author_email": "kirill.ivanov@cern.ch", "submission_date": "2024-10-07-15-35", "author_username": "kivanov", "author_name": "Kirill Ivanov"}}, {"action": "validation", "step": "succeeded", "updater": {"author_email": "", "submission_date": "2024-10-09-04-00", "author_username": "automatic", "author_name": ""}}], "output_dataset": [], "member_of_campaign": "Run3Winter24GS", "_id": "BPH-Run3Winter24GS-00001", "dataset_name": "BsToPhiPhi_4K_MuFilter_SoftQCDnonD_TuneCP5_13p6TeV-pythia8-evtgen", "cadi_line": ""}} diff --git a/bin/utils/request_GEN-RunIII2024Summer24wmLHEGS-00001.json b/bin/utils/request_GEN-RunIII2024Summer24wmLHEGS-00001.json new file mode 100644 index 00000000000..d621b4e6bd7 --- /dev/null +++ b/bin/utils/request_GEN-RunIII2024Summer24wmLHEGS-00001.json @@ -0,0 +1 @@ +{"results": {"validation": {"content": "all", "valid": false, "results": {"1": [{"peak_value_rss": 1353.18, "total_events": 227, "estimated_events_per_lumi": 2491.4303999999997, "cpu_efficiency": 0.9906964733938126, "filter_efficiency": 1, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 11.55962454339483, "expected_events": 227, "size_per_event": 894.145973568282}], "8": [{"peak_value_rss": 2046.88, "total_events": 227, "estimated_events_per_lumi": 16484.4288, "cpu_efficiency": 0.871975369494733, "filter_efficiency": 1, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 1.7471033027240834, "expected_events": 227, "size_per_event": 889.860511013216}], "2": [{"peak_value_rss": 1571.68, "total_events": 227, "estimated_events_per_lumi": 5081.8752, "cpu_efficiency": 0.9830421416875211, "filter_efficiency": 1, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 5.667199383408707, "expected_events": 227, "size_per_event": 846.7983788546255}], "4": [{"peak_value_rss": 1713.93, "total_events": 227, "estimated_events_per_lumi": 9597.456, "cpu_efficiency": 0.9382336828924716, "filter_efficiency": 1, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 3.0007952107308435, "expected_events": 227, "size_per_event": 843.0452158590308}]}}, "total_events": 500000000, "config_id": [], "events_per_lumi": 0, "_rev": "11-95a509edc0601cffbf2683cc564e82e2", "mcdb_id": 0, "transient_output_modules": [[]], "sequences": [{"eventcontent": ["RAWSIM", "LHE"], "runsScenarioForMC": "", "donotDropOnInput": "", "extra": "", "slhc": "", "runsAndWeightsForMC": "", "outputCommand": "", "dropDescendant": false, "pileup": "", "hltProcess": "", "index": -1, "customise": "", "custom_conditions": "", "datatier": ["GEN-SIM", "LHE"], "processName": "", "inputCommands": "", "particle_table": "", "conditions": "140X_mcRun3_2024_realistic_v26", "harvesting": "", "triggerResultsProcess": "", "inputEventContent": "", "beamspot": "DBrealistic", "restoreRNDSeeds": "", "filtername": "", "repacked": "", "customise_commands": "", "step": ["LHE", "GEN", "SIM"], "gflash": "", "procModifiers": "", "magField": "", "himix": false, "nThreads": "4", "scenario": "", "geometry": "DB:Extended", "nStreams": 0, "datamix": "", "inline_custom": false, "era": "Run3_2024"}], "block_black_list": [], "block_white_list": [], "priority": 20000, "process_string": "", "ppd_tags": [], "fragment_tag": "", "generator_parameters": [{"submission_details": {"submission_date": "2023-09-19-20-03", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}, "match_efficiency_error": 0, "match_efficiency": 1, "filter_efficiency": 1, "negative_weights_fraction": 0, "version": 0, "cross_section": 1, "filter_efficiency_error": 0}, {"submission_details": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-15-15-58", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}, "match_efficiency_error": 0, "match_efficiency": 1, "filter_efficiency": 1, "negative_weights_fraction": 0, "version": 1, "cross_section": 1, "filter_efficiency_error": 0}], "flown_with": "", "interested_pwg": ["GEN"], "version": 0, "generators": ["Powheg", "Pythia8"], "memory": 8000, "pilot": false, "type": "LHE", "cmssw_release": "CMSSW_14_0_18", "status": "approved", "keep_output": [false], "energy": 13.6, "tags": [], "fragment": "import FWCore.ParameterSet.Config as cms\n\nexternalLHEProducer = cms.EDProducer('ExternalLHEProducer',\n args = cms.vstring('/cvmfs/cms.cern.ch/phys_generator/gridpacks/PdmV/RunIII2024Summer24/Powheg/TT/hvq_slc7_amd64_gcc10_CMSSW_12_4_8_TTtoLNu2Q_powheg-pythia8.tgz'),\n nEvents = cms.untracked.uint32(5000),\n numberOfParameters = cms.uint32(1),\n outputFile = cms.string('cmsgrid_final.lhe'),\n scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh'),\n generateConcurrently = cms.untracked.bool(True),\n)\n\nfrom Configuration.Generator.Pythia8CommonSettings_cfi import *\nfrom Configuration.Generator.MCTunesRun3ECM13p6TeV.PythiaCP5Settings_cfi import *\nfrom Configuration.Generator.Pythia8PowhegEmissionVetoSettings_cfi import *\nfrom Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import *\n\ngenerator = cms.EDFilter(\"Pythia8ConcurrentHadronizerFilter\",\n PythiaParameters = cms.PSet(\n pythia8CommonSettingsBlock,\n pythia8CP5SettingsBlock,\n pythia8PowhegEmissionVetoSettingsBlock,\n pythia8PSweightsSettingsBlock,\n processParameters = cms.vstring(\n 'POWHEG:nFinal = 2',\n 'TimeShower:mMaxGamma = 4.0'\n ),\n parameterSets = cms.vstring(\n 'pythia8CommonSettings',\n 'pythia8CP5Settings',\n 'pythia8PowhegEmissionVetoSettings',\n 'processParameters',\n 'pythia8PSweightsSettings',\n )\n ),\n comEnergy = cms.double(13600),\n maxEventsToPrint = cms.untracked.int32(1),\n pythiaHepMCVerbosity = cms.untracked.bool(False),\n pythiaPylistVerbosity = cms.untracked.int32(1),\n)\n\n", "time_event": [22.18322274769774], "pwg": "GEN", "reqmgr_name": [], "approval": "approve", "name_of_fragment": "", "pileup_dataset_name": "", "analysis_id": [], "input_dataset": "", "member_of_chain": [], "prepid": "GEN-RunIII2024Summer24wmLHEGS-00001", "extension": 0, "size_event": [878], "notes": "", "completed_events": -1, "history": [{"action": "created", "updater": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-15-15-58", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "clone", "step": "GEN-Run3Summer23wmLHEGS-00009", "updater": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-15-15-58", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "update", "step": "fragment, total_events", "updater": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-15-16-11", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "update", "updater": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-15-16-15", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "approve", "step": "validation", "updater": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-15-16-15", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "validation", "step": "failed", "updater": {"author_email": "", "submission_date": "2024-11-15-16-41", "author_username": "automatic", "author_name": ""}}, {"action": "approve", "step": "validation", "updater": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-19-20-06", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "validation", "step": "succeeded", "updater": {"author_email": "", "submission_date": "2024-11-19-22-01", "author_username": "automatic", "author_name": ""}}, {"action": "approve", "step": "define", "updater": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-20-16-08", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "set status", "step": "defined", "updater": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-20-16-08", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "update", "step": "total_events", "updater": {"author_email": "si.hyun.jeon@cern.ch", "submission_date": "2024-11-20-16-09", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "approve", "step": "approve", "updater": {"author_email": "lorenzo.viliani@cern.ch", "submission_date": "2024-11-20-16-37", "author_username": "lviliani", "author_name": "Lorenzo Viliani"}}, {"action": "set status", "step": "approved", "updater": {"author_email": "lorenzo.viliani@cern.ch", "submission_date": "2024-11-20-16-37", "author_username": "lviliani", "author_name": "Lorenzo Viliani"}}], "output_dataset": [], "member_of_campaign": "RunIII2024Summer24wmLHEGS", "_id": "GEN-RunIII2024Summer24wmLHEGS-00001", "dataset_name": "TTtoLNu2Q_TuneCP5_13p6TeV_powheg-pythia8", "cadi_line": ""}} diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 1bd53c5ba95..64efa1b6929 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -845,7 +845,7 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): gp_size = len(gridpack_cvmfs_path_tmp) # additional data set name check for 2024 campaigns - if "Run3" in pi and "24" in pi: + if ("Run3" in pi or "RunIII" in pi) and "24" in pi: valid, message, feedback = validate_dataset_name(dn) if not valid: print("-----------------------------")