diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f0fdae9bef04..1ea6093d3c22 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -120,11 +120,10 @@ jobs: #shell: python# if: steps.filter.outputs.bin == 'true' || steps.filter.outputs.workflow == 'true' run: | - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid EXO-Run3Summer22wmLHEGS-01444 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid B2G-RunIISummer20UL18wmLHEGEN-03308 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SMP-RunIISummer20UL16GEN-00050 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00403 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TRK-Run3Summer23wmLHEGS-00002 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TSG-Run3Summer23BPixGS-00007 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TSG-Run3Summer23BPixGS-00045 \ No newline at end of file + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid EXO-Run3Summer22wmLHEGS-01444 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid B2G-RunIISummer20UL18wmLHEGEN-03308 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid GEN-Run3Summer22EEwmLHEGS-00403 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TRK-Run3Summer23wmLHEGS-00002 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TSG-Run3Summer23BPixGS-00007 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TOP-RunIISummer20UL16wmLHEGEN-00883 \ No newline at end of file diff --git a/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883 b/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883 new file mode 100644 index 000000000000..e71227af4903 --- /dev/null +++ b/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883 @@ -0,0 +1,33 @@ +import FWCore.ParameterSet.Config as cms + +externalLHEProducer = cms.EDProducer("ExternalLHEProducer", + args = cms.vstring('/cvmfs/cms.cern.ch/phys_generator/gridpacks/slc7_amd64_gcc700/13TeV/madgraph/V5_2.6.5/ttbar_LFV_SMEFT_splitFlavour/SMEFTsim_TT_vector_emutu_slc7_amd64_gcc700_CMSSW_10_6_19_tarball.tar.xz'), + nEvents = cms.untracked.uint32(5000), + numberOfParameters = cms.uint32(1), + outputFile = cms.string('cmsgrid_final.lhe'), + generateConcurrently = cms.untracked.bool(False), + scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh') +) + + +from Configuration.Generator.Pythia8CommonSettings_cfi import * +from Configuration.Generator.MCTunes2017.PythiaCP5Settings_cfi import * +from Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import * + +generator = cms.EDFilter("Pythia8ConcurrentHadronizerFilter", + maxEventsToPrint = cms.untracked.int32(1), + pythiaPylistVerbosity = cms.untracked.int32(1), + filterEfficiency = cms.untracked.double(1.0), + pythiaHepMCVerbosity = cms.untracked.bool(False), + comEnergy = cms.double(13000.), + PythiaParameters = cms.PSet( + pythia8CommonSettingsBlock, + pythia8CP5SettingsBlock, + pythia8PSweightsSettingsBlock, + parameterSets = cms.vstring('pythia8CommonSettings', + 'pythia8CP5Settings', + 'pythia8PSweightsSettings', + ) + ) +) +ProductionFilterSequence = cms.Sequence(generator) \ No newline at end of file diff --git a/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883_get_test b/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883_get_test new file mode 100644 index 000000000000..5bf31bf74286 --- /dev/null +++ b/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883_get_test @@ -0,0 +1,155 @@ +#!/bin/bash + +# Binds for singularity containers +# Mount /afs, /eos, /cvmfs, /etc/grid-security for xrootd +export APPTAINER_BINDPATH='/afs,/cvmfs,/cvmfs/grid.cern.ch/etc/grid-security:/etc/grid-security,/eos,/etc/pki/ca-trust,/run/user,/var/run/user' + +cat <<'EndOfGenScriptFile' > TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh +#!/bin/bash + +echo "Running CMS GEN request script using cms-sw containers. Architecture: el9:x86_64" +python3.9 -m venv cms_gen_venv_TOP-RunIISummer20UL16wmLHEGEN-00883 && source ./cms_gen_venv_TOP-RunIISummer20UL16wmLHEGEN-00883/bin/activate + +# Install the PdmV REST client +pip install git+https://github.com/cms-PdmV/mcm_scripts &> /dev/null + +echo "Packages installed" +pip freeze +echo "" + +# GEN Script begin +rm -f request_fragment_check.py +wget -q https://raw.githubusercontent.com/cms-sw/genproductions/master/bin/utils/request_fragment_check.py +chmod +x request_fragment_check.py + +./request_fragment_check.py --bypass_status --prepid TOP-RunIISummer20UL16wmLHEGEN-00883 + +# End of CMS GEN script file: TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh +EndOfGenScriptFile +chmod +x TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh + +# Run in singularity container +singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el9:x86_64 $(echo $(pwd)/TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh) + +GEN_ERR=$? +if [ $GEN_ERR -ne 0 ]; then + echo "GEN Checking Script returned exit code $GEN_ERR which means there are $GEN_ERR errors" + echo "Validation WILL NOT RUN" + echo "Please correct errors in the request and run validation again" + exit $GEN_ERR +fi +echo "Running VALIDATION. GEN Request Checking Script returned no errors" +# GEN Script end + +# Download fragment from McM +curl -s -k https://cms-pdmv-prod.web.cern.ch/mcm/public/restapi/requests/get_fragment/TOP-RunIISummer20UL16wmLHEGEN-00883 --retry 3 --create-dirs -o Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py +[ -s Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py ] || exit $?; + +# Check if fragment contais gridpack path ant that it is in cvmfs +if grep -q "gridpacks" Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py; then + if ! grep -q "/cvmfs/cms.cern.ch/phys_generator/gridpacks" Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py; then + echo "Gridpack inside fragment is not in cvmfs." + exit -1 + fi +fi + +# Dump actual test code to a TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh file that can be run in Singularity +cat <<'EndOfTestFile' > TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh +#!/bin/bash + +export SCRAM_ARCH=slc7_amd64_gcc700 + +source /cvmfs/cms.cern.ch/cmsset_default.sh +if [ -r CMSSW_10_6_43/src ] ; then + echo release CMSSW_10_6_43 already exists +else + scram p CMSSW CMSSW_10_6_43 +fi +cd CMSSW_10_6_43/src +eval `scram runtime -sh` + +mv ../../Configuration . +scram b +cd ../.. + +# Maximum validation duration: 57600s +# Margin for validation duration: 30% +# Validation duration with margin: 57600 * (1 - 0.30) = 40320s +# Time per event for each sequence: 2.0000s +# Threads for each sequence: 1 +# Time per event for single thread for each sequence: 1 * 2.0000s = 2.0000s +# Which adds up to 2.0000s per event +# Single core events that fit in validation duration: 40320s / 2.0000s = 20160 +# Produced events limit in McM is 10000 +# According to 1.0000 efficiency, validation should run 10000 / 1.0000 = 10000 events to reach the limit of 10000 +# Take the minimum of 20160 and 10000, but more than 0 -> 10000 +# It is estimated that this validation will produce: 10000 * 1.0000 = 10000 events +EVENTS=10000 + + +# cmsDriver command +cmsDriver.py Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py --python_filename TOP-RunIISummer20UL16wmLHEGEN-00883_1_cfg.py --eventcontent RAWSIM,LHE --customise Configuration/DataProcessing/Utils.addMonitoring --datatier GEN,LHE --fileout file:TOP-RunIISummer20UL16wmLHEGEN-00883.root --conditions 106X_mcRun2_asymptotic_v13 --beamspot Realistic25ns13TeV2016Collision --customise_commands process.source.numberEventsInLuminosityBlock="cms.untracked.uint32(100)" --step LHE,GEN --geometry DB:Extended --era Run2_2016 --no_exec --mc -n $EVENTS || exit $? ; + +# Run generated config +REPORT_NAME=TOP-RunIISummer20UL16wmLHEGEN-00883_report.xml +# Run the cmsRun +cmsRun -e -j $REPORT_NAME TOP-RunIISummer20UL16wmLHEGEN-00883_1_cfg.py || exit $? ; + +# Parse values from TOP-RunIISummer20UL16wmLHEGEN-00883_report.xml report +processedEvents=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +producedEvents=$(grep -Po "(?<=)(\d*)(?=)" $REPORT_NAME | tail -n 1) +threads=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueRss=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueVsize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSizeAlt=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobCPU=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +eventThroughput=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +avgEventTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +if [ -z "$threads" ]; then + echo "Could not find NumberOfThreads in report, defaulting to 1" + threads=1 +fi +if [ -z "$eventThroughput" ]; then + eventThroughput=$(bc -l <<< "scale=4; 1 / ($avgEventTime / $threads)") +fi +if [ -z "$totalSize" ]; then + totalSize=$totalSizeAlt +fi +if [ -z "$processedEvents" ]; then + processedEvents=$EVENTS +fi +echo "Validation report of TOP-RunIISummer20UL16wmLHEGEN-00883 sequence 1/1" +echo "Processed events: $processedEvents" +echo "Produced events: $producedEvents" +echo "Threads: $threads" +echo "Peak value RSS: $peakValueRss MB" +echo "Peak value Vsize: $peakValueVsize MB" +echo "Total size: $totalSize MB" +echo "Total job time: $totalJobTime s" +echo "Total CPU time: $totalJobCPU s" +echo "Event throughput: $eventThroughput" +echo "CPU efficiency: "$(bc -l <<< "scale=2; ($totalJobCPU * 100) / ($threads * $totalJobTime)")" %" +echo "Size per event: "$(bc -l <<< "scale=4; ($totalSize * 1024 / $producedEvents)")" kB" +echo "Time per event: "$(bc -l <<< "scale=4; (1 / $eventThroughput)")" s" +echo "Filter efficiency percent: "$(bc -l <<< "scale=8; ($producedEvents * 100) / $processedEvents")" %" +echo "Filter efficiency fraction: "$(bc -l <<< "scale=10; ($producedEvents) / $processedEvents") + +# End of TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh file +EndOfTestFile + +# Make file executable +chmod +x TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh + +if [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:amd64" ]; then + CONTAINER_NAME="el7:amd64" +elif [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:x86_64" ]; then + CONTAINER_NAME="el7:x86_64" +else + echo "Could not find amd64 or x86_64 for el7" + exit 1 +fi +# Run in singularity container +export SINGULARITY_CACHEDIR="/tmp/$(whoami)/singularity" +singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/$CONTAINER_NAME $(echo $(pwd)/TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh) diff --git a/bin/utils/request_TOP-RunIISummer20UL16wmLHEGEN-00883.json b/bin/utils/request_TOP-RunIISummer20UL16wmLHEGEN-00883.json new file mode 100644 index 000000000000..0251790b9727 --- /dev/null +++ b/bin/utils/request_TOP-RunIISummer20UL16wmLHEGEN-00883.json @@ -0,0 +1 @@ +{"results": {"validation": {"time_multiplier": 2, "results": {"1": [{"peak_value_rss": 665.695, "total_events": 10000, "estimated_events_per_lumi": 14218.5888, "cpu_efficiency": 0.9936923152855239, "filter_efficiency": 1.0, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 2.025517469075412, "expected_events": 10000, "size_per_event": 124.242944}], "8": [{"peak_value_rss": 666.508, "total_events": 10000, "estimated_events_per_lumi": 43582.17599999999, "cpu_efficiency": 0.9094780097297934, "filter_efficiency": 1.0, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 0.6608206070298097, "expected_events": 10000, "size_per_event": 123.57632}], "2": [{"peak_value_rss": 621.676, "total_events": 10000, "estimated_events_per_lumi": 26357.4432, "cpu_efficiency": 0.9893730851890804, "filter_efficiency": 1.0, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 1.0926704757159449, "expected_events": 10000, "size_per_event": 123.794432}], "4": [{"peak_value_rss": 696.875, "total_events": 10000, "estimated_events_per_lumi": 39914.208, "cpu_efficiency": 0.9495847701527331, "filter_efficiency": 1.0, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 0.7215475752393734, "expected_events": 10000, "size_per_event": 124.77951999999999}]}}, "total_events": 1000000, "config_id": [], "events_per_lumi": 0, "_rev": "9-5e70c27cdedf3c2bdfbdcc6f0c4318ac", "mcdb_id": 0, "transient_output_modules": [[]], "sequences": [{"eventcontent": ["RAWSIM", "LHE"], "runsScenarioForMC": "", "donotDropOnInput": "", "extra": "", "slhc": "", "runsAndWeightsForMC": "", "outputCommand": "", "dropDescendant": false, "pileup": "", "hltProcess": "", "index": -1, "customise": "", "custom_conditions": "", "datatier": ["GEN", "LHE"], "processName": "", "inputCommands": "", "particle_table": "", "conditions": "106X_mcRun2_asymptotic_v13", "harvesting": "", "triggerResultsProcess": "", "inputEventContent": "", "beamspot": "Realistic25ns13TeV2016Collision", "restoreRNDSeeds": "", "filtername": "", "repacked": "", "customise_commands": "", "step": ["LHE", "GEN"], "gflash": "", "procModifiers": "", "magField": "", "himix": false, "nThreads": "1", "scenario": "", "geometry": "DB:Extended", "nStreams": 0, "datamix": "", "inline_custom": false, "era": "Run2_2016"}], "block_black_list": [], "block_white_list": [], "priority": 20000, "process_string": "", "ppd_tags": [], "fragment_tag": "", "generator_parameters": [{"submission_details": {"submission_date": "2024-07-03-16-11", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}, "match_efficiency_error": 0, "match_efficiency": 1, "filter_efficiency": 1, "negative_weights_fraction": 0, "version": 0, "cross_section": 0.0436, "filter_efficiency_error": 1}, {"submission_details": {"submission_date": "2024-10-23-10-06", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}, "match_efficiency_error": 0.0, "match_efficiency": 1.0, "filter_efficiency": 1.0, "negative_weights_fraction": 0.0, "version": 1, "cross_section": 0.0436, "filter_efficiency_error": 1.0}], "flown_with": "", "interested_pwg": ["TOP"], "version": 0, "generators": ["Pythia8", "Madgraph V5 2.6.5"], "memory": 1900, "pilot": false, "type": "LHE", "cmssw_release": "CMSSW_10_6_43", "status": "defined", "keep_output": [false], "energy": 13.0, "tags": [], "fragment": "import FWCore.ParameterSet.Config as cms\n\nexternalLHEProducer = cms.EDProducer(\"ExternalLHEProducer\",\n args = cms.vstring('/cvmfs/cms.cern.ch/phys_generator/gridpacks/slc7_amd64_gcc700/13TeV/madgraph/V5_2.6.5/ttbar_LFV_SMEFT_splitFlavour/SMEFTsim_TT_vector_emutu_slc7_amd64_gcc700_CMSSW_10_6_19_tarball.tar.xz'),\n nEvents = cms.untracked.uint32(5000),\n numberOfParameters = cms.uint32(1),\n outputFile = cms.string('cmsgrid_final.lhe'),\n generateConcurrently = cms.untracked.bool(False),\n scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh')\n)\n\n# Link to datacards:\n# https://github.com/cms-sw/genproductions/tree/mg265UL/bin/MadGraph5_aMCatNLO/cards/production/13TeV/LFV_TopProduction_TopDecay/SMEFTsim/SMEFTsim_TopDecay\n\nfrom Configuration.Generator.Pythia8CommonSettings_cfi import *\nfrom Configuration.Generator.MCTunes2017.PythiaCP5Settings_cfi import *\nfrom Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import *\n\ngenerator = cms.EDFilter(\"Pythia8ConcurrentHadronizerFilter\",\n maxEventsToPrint = cms.untracked.int32(1),\n pythiaPylistVerbosity = cms.untracked.int32(1),\n filterEfficiency = cms.untracked.double(1.0),\n pythiaHepMCVerbosity = cms.untracked.bool(False),\n comEnergy = cms.double(13000.),\n PythiaParameters = cms.PSet(\n pythia8CommonSettingsBlock,\n pythia8CP5SettingsBlock,\n pythia8PSweightsSettingsBlock,\n parameterSets = cms.vstring('pythia8CommonSettings',\n 'pythia8CP5Settings',\n 'pythia8PSweightsSettings',\n )\n )\n)\nProductionFilterSequence = cms.Sequence(generator)", "time_event": [2], "pwg": "TOP", "reqmgr_name": [], "approval": "define", "name_of_fragment": "", "pileup_dataset_name": "", "analysis_id": [], "input_dataset": "", "member_of_chain": [], "prepid": "TOP-RunIISummer20UL16wmLHEGEN-00883", "extension": 0, "size_event": [130], "notes": "Overall cross-section summary\n--------------------------------------------------------------------------------------------------------------------------------------------------------------------------\nProcess xsec_before [pb] passed nposw nnegw tried nposw nnegw xsec_match [pb] accepted [%] event_eff [%]\n0 7.297e-03 +/- 2.998e-05 1673 1673 0 1673 1673 0 7.297e-03 +/- 2.998e-05 100.0 +/- 0.0 100.0 +/- 0.0\n1 7.287e-03 +/- 3.066e-05 1605 1605 0 1605 1605 0 7.287e-03 +/- 3.066e-05 100.0 +/- 0.0 100.0 +/- 0.0\n2 7.248e-03 +/- 3.051e-05 1737 1737 0 1737 1737 0 7.248e-03 +/- 3.051e-05 100.0 +/- 0.0 100.0 +/- 0.0\n3 7.274e-03 +/- 2.999e-05 1696 1696 0 1696 1696 0 7.274e-03 +/- 2.999e-05 100.0 +/- 0.0 100.0 +/- 0.0\n4 7.229e-03 +/- 3.153e-05 1643 1643 0 1643 1643 0 7.229e-03 +/- 3.153e-05 100.0 +/- 0.0 100.0 +/- 0.0\n5 7.268e-03 +/- 2.912e-05 1646 1646 0 1646 1646 0 7.268e-03 +/- 2.912e-05 100.0 +/- 0.0 100.0 +/- 0.0\n--------------------------------------------------------------------------------------------------------------------------------------------------------------------------\nTotal 4.360e-02 +/- 7.424e-05 10000 10000 0 10000 10000 0 4.360e-02 +/- 7.424e-05 100.0 +/- 0.0 100.0 +/- 0.0\n-", "completed_events": 0, "history": [{"action": "created", "updater": {"submission_date": "2024-10-23-10-06", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "clone", "step": "TOP-RunIISummer20UL16wmLHEGEN-00869", "updater": {"submission_date": "2024-10-23-10-06", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "approve", "step": "validation", "updater": {"submission_date": "2024-10-23-16-42", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "validation", "step": "succeeded", "updater": {"submission_date": "2024-10-24-04-00", "author_email": "", "author_username": "automatic", "author_name": ""}}, {"action": "approve", "step": "none", "updater": {"submission_date": "2024-10-24-08-12", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "set status", "step": "new", "updater": {"submission_date": "2024-10-24-08-12", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "update", "updater": {"author_email": "carlos.vico.villalba@cern.ch", "submission_date": "2024-10-24-08-39", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "approve", "step": "validation", "updater": {"author_email": "carlos.vico.villalba@cern.ch", "submission_date": "2024-10-24-08-46", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "validation", "step": "succeeded", "updater": {"author_email": "", "submission_date": "2024-10-24-20-21", "author_username": "automatic", "author_name": ""}}, {"action": "approve", "step": "define", "updater": {"author_email": "carlos.vico.villalba@cern.ch", "submission_date": "2024-10-27-16-34", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "set status", "step": "defined", "updater": {"author_email": "carlos.vico.villalba@cern.ch", "submission_date": "2024-10-27-16-34", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}], "output_dataset": [], "member_of_campaign": "RunIISummer20UL16wmLHEGEN", "_id": "TOP-RunIISummer20UL16wmLHEGEN-00883", "dataset_name": "TT_vector-emutu-LFV_TuneCP5_13TeV_madgraph-pythia8", "cadi_line": ""}} \ No newline at end of file diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 08af7cbf901a..ecb13d8365f2 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -11,6 +11,7 @@ import glob import json import ast +import subprocess from datetime import datetime from json import dumps @@ -31,6 +32,8 @@ parser.add_argument('--develop', help="Option to make modifications of the script", action='store_true') parser.add_argument('--local', help="Option to read fragment locally", action='store_true') parser.add_argument('--download_json', help="Download request json to read fragment locally in a next step", action='store_true') +parser.add_argument('--bypass_runcmsgrid_patch', help="apply the runcmsgrid patch if necessary", action='store_true') + args = parser.parse_args() @@ -313,34 +316,70 @@ def ul_consistency(dn,pi,jhu_gp): if not error_ul: print("UL consistency check is OK.") return warning_ul,error_ul +def gridpack_copy(gridpack_eos_path,pi): + error_gp_copy = [] + targz_flag = 0 + if "Run3" in pi: + copy_name = "_original_Run3_wo_runcmsgrid_sys_patch" + else: + copy_name = "_original" + if ".tar.gz" in gridpack_eos_path: + targz_flag = 1 + gridpack_eos_path_backup = gridpack_eos_path.replace('.tar.gz',copy_name+'.tar.gz') + if ".tgz" in gridpack_eos_path: gridpack_eos_path_backup = gridpack_eos_path.replace('.tgz',copy_name+'.tgz') + if ".tar.xz" in gridpack_eos_path: + gridpack_eos_path_backup = gridpack_eos_path.replace('.tar.xz',copy_name+'.tar.xz') + targz_flag = 2 + if not os.path.exists(gridpack_eos_path_backup): + print("Backup gridpack does not exist.") + print("Copying "+gridpack_eos_path+" to "+gridpack_eos_path_backup+" before patching runcms.grid") + os.system('cp -n -p '+gridpack_eos_path+' '+gridpack_eos_path_backup) + md5_1 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] + md5_2 = os.popen('md5sum'+' '+gridpack_eos_path_backup).read().split(' ')[0] + if md5_1 == md5_2: + print("Backup and original file checksums are equal.") + else: + error_gp_copy.append("backup gridpack has a problem.") + print("Backup gridpack: "+gridpack_eos_path_backup) + return error_gp_copy + +def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): + error_gridpack_repack = [] + gp_extension = ".tar.xz" + if ".tar.gz" in gridpack_eos_path: + gp_extension = ".tar.gz" + if ".tgz" in gridpack_eos_path: + gp_extension = ".tgz" + gp_name = "gridpack"+gp_extension + if os.path.isfile(gp_name): os.system('rm '+gp_name) + print("re-tarring to "+gp_name) + cur_dir = os.getcwd() + os.chdir(my_path+'/'+pi) + print(os.getcwd()) + os.environ['XZ_OPT'] = "--lzma2=preset=9,dict=512MiB" + os.system('XZ_OPT="$XZ_OPT" tar -cJpf '+gp_name+' --exclude='+gp_name+' --exclude='+pi+' ./*') + print('cp '+gp_name+' '+gridpack_eos_path) + os.system('cp '+gp_name+' '+gridpack_eos_path) + md5_1 = os.popen('md5sum '+gp_name).read().split(' ')[0] + md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] + if md5_1 == md5_2: + print("Updated gridpack copied succesfully.") + else: + error_gridpack_repack.append("There was a problem copying in the updated gridpack to eos.") + os.chdir(cur_dir) + return error_gridpack_repack + def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): xml = str(re.findall('xmllint.*',cont)) cur_dir = os.getcwd() warning_xml = [] error_xml = [] if "stream" not in xml or len(xml) < 3: - targz_flag = 0 if "stream" not in xml and len(xml) > 3: warning_xml.append(" --stream option is missing in XMLLINT, will update runcmsgrid.") if len(xml) < 3: warning_xml.append("[WARNING] XMLLINT does not exist in runcmsgrid, will update it.") - if ".tar.gz" in gridpack_eos_path: - targz_flag = 1 - gridpack_eos_path_backup = gridpack_eos_path.replace('.tar.gz','_original.tar.gz') - if ".tgz" in gridpack_eos_path: gridpack_eos_path_backup = gridpack_eos_path.replace('.tgz','_original.tgz') - if ".tar.xz" in gridpack_eos_path: - gridpack_eos_path_backup = gridpack_eos_path.replace('.tar.xz','_original.tar.xz') - targz_flag = 2 - if not os.path.exists(gridpack_eos_path_backup): - print("Backup gridpack does not exist.") - print("Copying "+gridpack_eos_path+" to "+gridpack_eos_path_backup+" before patching runcms.grid") - os.system('cp -n -p '+gridpack_eos_path+' '+gridpack_eos_path_backup) - md5_1 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] - md5_2 = os.popen('md5sum'+' '+gridpack_eos_path_backup).read().split(' ')[0] - if md5_1 == md5_2: - print("Backup and original file checksums are equal.") - else: - error_xml.append("backup gridpack has a problem.") + error_xml.append(gridpack_copy(gridpack_eos_path,pi)) print("Updating XMLLINT line in runcmsgrid.") os.chdir(my_path+'/'+pi) if "stream" not in xml and len(xml) > 3: cont = re.sub("xmllint","xmllint --stream",cont) @@ -351,19 +390,7 @@ def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): f.seek(0) f.write(cont) f.truncate() - if targz_flag == 0: gridpackname = "gridpack.tgz" - if targz_flag == 1: gridpackname = "gridpack.tar.gz" - if targz_flag == 2: gridpackname = "gridpack.tar.xz" - os.chdir(my_path+'/'+pi) - os.system('tar cfJ '+gridpackname+' ./* --exclude='+gridpackname+' --exclude='+pi) - os.system('cp '+gridpackname+' '+gridpack_eos_path) - md5_1 = os.popen('md5sum '+gridpackname).read().split(' ')[0] - md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] - if md5_1 == md5_2: - print("Updated gridpack copied succesfully.") - else: - error_xml.append("There was a problem copying in the updated gridpack to eos.") - os.chdir(cur_dir) + error_xml = gridpack_repack_and_copy(gridpack_eos_path,my_path,pi) return warning_xml,error_xml def evtgen_check(fragment): @@ -1150,7 +1177,7 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): warnings.append(pf[1]+" "+pf[2]+" You may try to request more events per phase-space region in the gridpack.") if os.path.isfile(gp_log_loc) and ('madgraph' in dn.lower() or 'amcatnlo' in dn.lower()): print("------------------------------------------------------------------------------------") - print("Summary for madgraph for experts fron gridpack log (cross section BEFORE matching (if there is matching/merging)):") + print("Summary for madgraph for experts from gridpack log (cross section BEFORE matching (if there is matching/merging)):") print(os.popen('grep Summary '+gp_log_loc+' -A 5 -B 1').read()) print("------------------------------------------------------------------------------------") if mg_gp: @@ -1302,10 +1329,10 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): errors.extend(check_replace(runcmsgrid_file)) match = re.search(r"""process=(["']?)([^"']*)\1""", content) print(match.group(0)) - warning1,error1 = xml_check_and_patch(f,content,gridpack_eos_path,my_path,pi) - warnings.extend(warning1) - errors.extend(error1) - + if args.bypass_runcmsgrid_patch is False: + warning1,error1 = xml_check_and_patch(f,content,gridpack_eos_path,my_path,pi) + warnings.extend(warning1) + errors.extend(error1) f.close() else: errors.append(my_path+'/'+pi+'/'+'runcmsgrid.sh does not exists') @@ -1514,6 +1541,23 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): if mg5_aMC_version >= 260: mg_lo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c madevent').read()) mg_nlo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c aMCatNLO').read()) + if mg_lo: print("LO gridpack") + if mg_nlo: print("NLO gridpack") + if ("Run3" in pi or "RunII" in pi) and args.bypass_runcmsgrid_patch is False: + if int(os.popen('grep -c "systematics $runlabel" '+str(runcmsgrid_file)).read()): + if int(os.popen('grep -c "Encounter Error in Running Systematics Module" '+str(runcmsgrid_file)).read()) < 1: + print("-----------------------------------------") + print("runcmsgrid script patch for Run3 missing!") + print("-----------------------------------------") + print("I will patch the runcmsgrid script.") + err_gpr = gridpack_copy(gridpack_eos_path,pi) + errors.extend(err_gpr) + if mg_nlo: + os.system("patch "+runcmsgrid_file+" < /eos/cms/store/group/phys_generator/cvmfs/gridpacks/mg_amg_patch/runcmsgrid_systematics_NLO.patch") + if mg_lo: + os.system("patch "+runcmsgrid_file+" < /eos/cms/store/group/phys_generator/cvmfs/gridpacks/mg_amg_patch/runcmsgrid_systematics_LO.patch") + err_gpr = gridpack_repack_and_copy(gridpack_eos_path,my_path,pi) + errors.extend(err_gpr) if mg5_aMC_version < 260: mg_lo = int(os.popen('grep -c syscalc '+str(runcmsgrid_file)).read()) if mg_nlo > 0: @@ -1622,7 +1666,7 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): if MGpatch2[0] == 0 and MGpatch2[1] == 1: print("[OK] MG5_aMC@NLO LO nthreads patch not made in CVMFS but done in EOS waiting for CVMFS-EOS synch") if MGpatch2[1] == 0 and args.local is False: errors.append("MG5_aMC@NLO LO nthreads patch not made in EOS") - if args.apply_many_threads_patch: + if args.apply_many_threads_patch and args.bypass_runcmsgrid_patch is False: print("Patching for nthreads problem... please be patient.") if slha_flag == 0: os.system('python2 ../../Utilities/scripts/update_gridpacks_mg242_thread.py --prepid '+pi)