Merge pull request #3600 from cms-sw/unmcm_wf

changes for wf to run with no access to mcm rest

bin/utils/SUS-RunIISummer20UL16wmLHEGEN-00768_get_test

@@ -0,0 +1,131 @@
+#!/bin/bash
+
+# GEN Script begin
+rm -f request_fragment_check.py
+wget -q https://raw.githubusercontent.com/cms-sw/genproductions/master/bin/utils/request_fragment_check.py
+chmod +x request_fragment_check.py
+./request_fragment_check.py --bypass_status --prepid SUS-RunIISummer20UL16wmLHEGEN-00768
+GEN_ERR=$?
+if [ $GEN_ERR -ne 0 ]; then
+  echo "GEN Checking Script returned exit code $GEN_ERR which means there are $GEN_ERR errors"
+  echo "Validation WILL NOT RUN"
+  echo "Please correct errors in the request and run validation again"
+  exit $GEN_ERR
+fi
+echo "Running VALIDATION. GEN Request Checking Script returned no errors"
+# GEN Script end
+
+# Download fragment from McM
+curl -s -k https://cms-pdmv-prod.web.cern.ch/mcm/public/restapi/requests/get_fragment/SUS-RunIISummer20UL16wmLHEGEN-00768 --retry 3 --create-dirs -o Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py
+[ -s Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py ] || exit $?;
+
+# Check if fragment contais gridpack path ant that it is in cvmfs
+if grep -q "gridpacks" Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py; then
+  if ! grep -q "/cvmfs/cms.cern.ch/phys_generator/gridpacks" Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py; then
+    echo "Gridpack inside fragment is not in cvmfs."
+    exit -1
+  fi
+fi
+
+# Dump actual test code to a SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh file that can be run in Singularity
+cat <<'EndOfTestFile' > SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh
+#!/bin/bash
+
+export SCRAM_ARCH=slc7_amd64_gcc700
+
+source /cvmfs/cms.cern.ch/cmsset_default.sh
+if [ -r CMSSW_10_6_32_patch1/src ] ; then
+  echo release CMSSW_10_6_32_patch1 already exists
+else
+  scram p CMSSW CMSSW_10_6_32_patch1
+fi
+cd CMSSW_10_6_32_patch1/src
+eval `scram runtime -sh`
+
+mv ../../Configuration .
+scram b
+cd ../..
+
+# Maximum validation duration: 28800s
+# Margin for validation duration: 30%
+# Validation duration with margin: 28800 * (1 - 0.30) = 20160s
+# Time per event for each sequence: 30.0000s
+# Threads for each sequence: 1
+# Time per event for single thread for each sequence: 1 * 30.0000s = 30.0000s
+# Which adds up to 30.0000s per event
+# Single core events that fit in validation duration: 20160s / 30.0000s = 672
+# Produced events limit in McM is 10000
+# According to 1.0000 efficiency, validation should run 10000 / 1.0000 = 10000 events to reach the limit of 10000
+# Take the minimum of 672 and 10000, but more than 0 -> 672
+# It is estimated that this validation will produce: 672 * 1.0000 = 672 events
+EVENTS=672
+
+
+# cmsDriver command
+cmsDriver.py Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py --python_filename SUS-RunIISummer20UL16wmLHEGEN-00768_1_cfg.py --eventcontent RAWSIM,LHE --customise Configuration/DataProcessing/Utils.addMonitoring --datatier GEN,LHE --fileout file:SUS-RunIISummer20UL16wmLHEGEN-00768.root --conditions 106X_mcRun2_asymptotic_v13 --beamspot Realistic25ns13TeV2016Collision --customise_commands process.source.numberEventsInLuminosityBlock="cms.untracked.uint32(100)" --step LHE,GEN --geometry DB:Extended --era Run2_2016 --no_exec --mc -n $EVENTS || exit $? ;
+
+# Run generated config
+REPORT_NAME=SUS-RunIISummer20UL16wmLHEGEN-00768_report.xml
+# Run the cmsRun
+cmsRun -e -j $REPORT_NAME SUS-RunIISummer20UL16wmLHEGEN-00768_1_cfg.py || exit $? ;
+
+# Parse values from SUS-RunIISummer20UL16wmLHEGEN-00768_report.xml report
+processedEvents=$(grep -Po "(?<=<Metric Name=\"NumberEvents\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+producedEvents=$(grep -Po "(?<=<TotalEvents>)(\d*)(?=</TotalEvents>)" $REPORT_NAME | tail -n 1)
+threads=$(grep -Po "(?<=<Metric Name=\"NumberOfThreads\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+peakValueRss=$(grep -Po "(?<=<Metric Name=\"PeakValueRss\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+peakValueVsize=$(grep -Po "(?<=<Metric Name=\"PeakValueVsize\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+totalSize=$(grep -Po "(?<=<Metric Name=\"Timing-tstoragefile-write-totalMegabytes\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+totalSizeAlt=$(grep -Po "(?<=<Metric Name=\"Timing-file-write-totalMegabytes\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+totalJobTime=$(grep -Po "(?<=<Metric Name=\"TotalJobTime\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+totalJobCPU=$(grep -Po "(?<=<Metric Name=\"TotalJobCPU\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+eventThroughput=$(grep -Po "(?<=<Metric Name=\"EventThroughput\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+avgEventTime=$(grep -Po "(?<=<Metric Name=\"AvgEventTime\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+if [ -z "$threads" ]; then
+  echo "Could not find NumberOfThreads in report, defaulting to 1"
+  threads=1
+fi
+if [ -z "$eventThroughput" ]; then
+  eventThroughput=$(bc -l <<< "scale=4; 1 / ($avgEventTime / $threads)")
+fi
+if [ -z "$totalSize" ]; then
+  totalSize=$totalSizeAlt
+fi
+if [ -z "$processedEvents" ]; then
+  processedEvents=$EVENTS
+fi
+echo "Validation report of SUS-RunIISummer20UL16wmLHEGEN-00768 sequence 1/1"
+echo "Processed events: $processedEvents"
+echo "Produced events: $producedEvents"
+echo "Threads: $threads"
+echo "Peak value RSS: $peakValueRss MB"
+echo "Peak value Vsize: $peakValueVsize MB"
+echo "Total size: $totalSize MB"
+echo "Total job time: $totalJobTime s"
+echo "Total CPU time: $totalJobCPU s"
+echo "Event throughput: $eventThroughput"
+echo "CPU efficiency: "$(bc -l <<< "scale=2; ($totalJobCPU * 100) / ($threads * $totalJobTime)")" %"
+echo "Size per event: "$(bc -l <<< "scale=4; ($totalSize * 1024 / $producedEvents)")" kB"
+echo "Time per event: "$(bc -l <<< "scale=4; (1 / $eventThroughput)")" s"
+echo "Filter efficiency percent: "$(bc -l <<< "scale=8; ($producedEvents * 100) / $processedEvents")" %"
+echo "Filter efficiency fraction: "$(bc -l <<< "scale=10; ($producedEvents) / $processedEvents")
+
+# End of SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh file
+EndOfTestFile
+
+# Make file executable
+chmod +x SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh
+
+if [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:amd64" ]; then
+  CONTAINER_NAME="el7:amd64"
+elif [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:x86_64" ]; then
+  CONTAINER_NAME="el7:x86_64"
+else
+  echo "Could not find amd64 or x86_64 for el7"
+  exit 1
+fi
+# Run in singularity container
+# Mount afs, eos, cvmfs
+# Mount /etc/grid-security for xrootd
+export SINGULARITY_CACHEDIR="/tmp/$(whoami)/singularity"
+singularity run -B /afs -B /eos -B /cvmfs -B /etc/grid-security -B /etc/pki/ca-trust --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/$CONTAINER_NAME $(echo $(pwd)/SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh)

bin/utils/request_fragment_check.py

@@ -11,8 +11,9 @@
 import json
 import ast
 from datetime import datetime
-sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/')
-from rest import McM
+#
+#sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/')
+#from rest import McM
 from json import dumps
 
 parser = argparse.ArgumentParser(
@@ -54,8 +55,11 @@
        sys.exit()
 
 # Use no-id as identification mode in order not to use a SSO cookie
-mcm = McM(id=None, dev=args.dev, debug=args.debug)
-mcm_link = mcm.server
+if args.local is False:
+    sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/')
+    from rest import McM
+    mcm = McM(id=None, dev=args.dev, debug=args.debug)
+    mcm_link = mcm.server
 
 def get_request(prepid):
     if args.local is False:
@@ -473,15 +477,16 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False):
     return requests
 
 
-if args.ticket is not None:
-    ticket = args.ticket
-    ticket = ticket[0]
-    print("------------------------------------")
-    print("--> Ticket = "+ticket)
-    print("------------------------------------")
-    prepid = []
-    for rr in root_requests_from_ticket(ticket):
-        if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr or 'FS' in rr: prepid.append(rr)
+if args.local is False:
+    if args.ticket is not None:
+        ticket = args.ticket
+        ticket = ticket[0]
+        print("------------------------------------")
+        print("--> Ticket = "+ticket)
+        print("------------------------------------")
+        prepid = []
+        for rr in root_requests_from_ticket(ticket):
+            if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr or 'FS' in rr: prepid.append(rr)
 
 
 
@@ -693,7 +698,8 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False):
         os.system('mkdir -p '+my_path+'/eos/'+pi)
         os.system('mv '+pi+'_tmp '+pi)
         os.system('cp '+pi+' '+my_path+'/'+pi+'/.')
-        os.system('wget -q '+mcm_link+'public/restapi/requests/get_test/'+pi+' -O '+pi+'_get_test')
+        if args.local is False:
+            os.system('wget -q '+mcm_link+'public/restapi/requests/get_test/'+pi+' -O '+pi+'_get_test')
         gettest = os.popen('grep cff '+pi+'_get_test'+' | grep curl').read()
         if os.path.getsize(pi+'_get_test') == 0:
             print("public/restapi/requests/get_test/ is not acessible for this request. Exiting! Please send an email to '[email protected]'")