diff --git a/crab/FrameworkJobReport.xml b/crab/FrameworkJobReport.xml new file mode 100644 index 00000000000..fd947d385b9 --- /dev/null +++ b/crab/FrameworkJobReport.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/crab/crab-test/pset.py b/crab/crab-test/pset.py new file mode 100644 index 00000000000..527e6884968 --- /dev/null +++ b/crab/crab-test/pset.py @@ -0,0 +1,130 @@ +# Auto generated configuration file +# using: +# Revision: 1.19 +# Source: /local/reps/CMSSW/CMSSW/Configuration/Applications/python/ConfigBuilder.py,v +# with command line options: MinBias_8TeV_cfi --conditions auto:startup -s GEN,SIM --datatier GEN-SIM -n 10 +# --relval 9000,300 --eventcontent RAWSIM --io MinBias.io --python MinBias.py --no_exec --fileout minbias.root + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("SIM") + +# Import of standard configurations +process.load("Configuration.StandardSequences.Services_cff") +process.load("SimGeneral.HepPDTESSource.pythiapdt_cfi") +process.load("FWCore.MessageService.MessageLogger_cfi") +process.load("Configuration.EventContent.EventContent_cff") +process.load("SimGeneral.MixingModule.mixNoPU_cfi") +process.load("Configuration.StandardSequences.GeometryRecoDB_cff") +process.load("Configuration.Geometry.GeometrySimDB_cff") +process.load("Configuration.StandardSequences.MagneticField_38T_cff") +process.load("Configuration.StandardSequences.Generator_cff") +process.load("IOMC.EventVertexGenerators.VtxSmearedRealistic8TeVCollision_cfi") +process.load("GeneratorInterface.Core.genFilterSummary_cff") +process.load("Configuration.StandardSequences.SimIdeal_cff") +process.load("Configuration.StandardSequences.EndOfProcess_cff") +process.load("Configuration.StandardSequences.FrontierConditions_GlobalTag_cff") + +process.maxEvents = cms.untracked.PSet(input=cms.untracked.int32(10)) + +# Input source +process.source = cms.Source("EmptySource") + +process.options = cms.untracked.PSet() +process.options.numberOfThreads = cms.untracked.uint32(1) + +# Production Info +process.configurationMetadata = cms.untracked.PSet( + version=cms.untracked.string("$Revision: 1.19 $"), + annotation=cms.untracked.string("MinBias_8TeV_cfi nevts:10"), + name=cms.untracked.string("Applications"), +) + +# Output definition +process.RAWSIMoutput = cms.OutputModule( + "PoolOutputModule", + splitLevel=cms.untracked.int32(0), + eventAutoFlushCompressedSize=cms.untracked.int32(5242880), + outputCommands=process.RAWSIMEventContent.outputCommands, + fileName=cms.untracked.string("minbias.root"), + dataset=cms.untracked.PSet( + filterName=cms.untracked.string(""), dataTier=cms.untracked.string("GEN-SIM") + ), + SelectEvents=cms.untracked.PSet(SelectEvents=cms.vstring("generation_step")), +) + +# Additional output definition + +# Other statements +process.genstepfilter.triggerConditions = cms.vstring("generation_step") +from Configuration.AlCa.GlobalTag import GlobalTag + +process.GlobalTag = GlobalTag(process.GlobalTag, "auto:startup", "") + +process.generator = cms.EDFilter( + "Pythia6GeneratorFilter", + pythiaPylistVerbosity=cms.untracked.int32(0), + filterEfficiency=cms.untracked.double(1.0), + pythiaHepMCVerbosity=cms.untracked.bool(False), + comEnergy=cms.double(8000.0), + maxEventsToPrint=cms.untracked.int32(0), + PythiaParameters=cms.PSet( + pythiaUESettings=cms.vstring( + "MSTU(21)=1 ! Check on possible errors during program execution", + "MSTJ(22)=2 ! Decay those unstable particles", + "PARJ(71)=10 . ! for which ctau 10 mm", + "MSTP(33)=0 ! no K factors in hard cross sections", + "MSTP(2)=1 ! which order running alphaS", + "MSTP(51)=10042 ! structure function chosen (external PDF CTEQ6L1)", + "MSTP(52)=2 ! work with LHAPDF", + "PARP(82)=1.921 ! pt cutoff for multiparton interactions", + "PARP(89)=1800. ! sqrts for which PARP82 is set", + "PARP(90)=0.227 ! Multiple interactions: rescaling power", + "MSTP(95)=6 ! CR (color reconnection parameters)", + "PARP(77)=1.016 ! CR", + "PARP(78)=0.538 ! CR", + "PARP(80)=0.1 ! Prob. colored parton from BBR", + "PARP(83)=0.356 ! Multiple interactions: matter distribution parameter", + "PARP(84)=0.651 ! Multiple interactions: matter distribution parameter", + "PARP(62)=1.025 ! ISR cutoff", + "MSTP(91)=1 ! Gaussian primordial kT", + "PARP(93)=10.0 ! primordial kT-max", + "MSTP(81)=21 ! multiple parton interactions 1 is Pythia default", + "MSTP(82)=4 ! Defines the multi-parton model", + ), + processParameters=cms.vstring( + "MSEL=0 ! User defined processes", + "MSUB(11)=1 ! Min bias process", + "MSUB(12)=1 ! Min bias process", + "MSUB(13)=1 ! Min bias process", + "MSUB(28)=1 ! Min bias process", + "MSUB(53)=1 ! Min bias process", + "MSUB(68)=1 ! Min bias process", + "MSUB(92)=1 ! Min bias process, single diffractive", + "MSUB(93)=1 ! Min bias process, single diffractive", + "MSUB(94)=1 ! Min bias process, double diffractive", + "MSUB(95)=1 ! Min bias process", + ), + parameterSets=cms.vstring("pythiaUESettings", "processParameters"), + ), +) + +# Path and EndPath definitions +process.generation_step = cms.Path(process.pgen) +process.simulation_step = cms.Path(process.psim) +process.genfiltersummary_step = cms.EndPath(process.genFilterSummary) +process.endjob_step = cms.EndPath(process.endOfProcess) +process.RAWSIMoutput_step = cms.EndPath(process.RAWSIMoutput) + +# Schedule definition +process.schedule = cms.Schedule( + process.generation_step, + process.genfiltersummary_step, + process.simulation_step, + process.endjob_step, + process.RAWSIMoutput_step, +) + +# Filter all path with the production filter sequence +for path in process.paths: + getattr(process, path)._seq = process.generator * getattr(process, path)._seq diff --git a/crab/crab-test/run.sh b/crab/crab-test/run.sh new file mode 100755 index 00000000000..134e923cd48 --- /dev/null +++ b/crab/crab-test/run.sh @@ -0,0 +1,69 @@ +#!/bin/bash -e +JENKINS_ID=$(echo ${CRAB_ReqName} | sed 's|.*_||') +if [ "${JENKINS_ID}" = "" ] ; then JENKINS_ID="$$" ; fi +req=$(date +%s) +rm -rf cmdrun +mkdir -p cmdrun +xstart=$(echo "START" | base64) +xend=$(echo "END" | base64) +pushd cmdrun + touch run.log + curl -s -L "https://muzaffar.web.cern.ch/cgi-bin/test-v2?START=1&req=${req}&uid=${JENKINS_ID}" >>run.log 2>&1 + req=$(date +%s) + LRUN=${req} + PRECMD="" + RUN_GAP=0 + while [ $RUN_GAP -lt 3600 ] ; do + req=$(date +%s) + curl -s -L -o cmd.txt "https://muzaffar.web.cern.ch/crab-test/cmd.txt?req=${req}&uid=${JENKINS_ID}" + cmd=$(grep "^cmd=${JENKINS_ID}=" cmd.txt || true) + if [ "${cmd}" = "" ] ; then + cmd=$(grep "^cmd=0=" cmd.txt || true) + fi + rm -f cmd.txt + cmd=$(echo "$cmd" | sed 's|^.*=|cmd=|') + if [ "$cmd" = "cmd=0" ] ; then + break + fi + if [ "${PRECMD}" = "$cmd" -o "${cmd}" = "" ] ; then + xt=$(date +%s) + xd=0 + while [ $xd -lt 9 ] ; do + for i in 0 1 2 3 4 5 6 7 8 9 ; do + for i in 0 1 2 3 4 5 6 7 8 9 ; do + for i in 0 1 2 3 4 5 6 7 8 9 ; do + true + done + done + done + let xd=$(date +%s)-${xt} || true + done + let RUN_GAP=$(date +%s)-${LRUN} + b64=$(echo "previous_${cmd}" | base64) + curl -s -L -X POST -d "$b64" "https://muzaffar.web.cern.ch/cgi-bin/test-v2?WAITING=1&${cmd}&req=${req}&uid=${JENKINS_ID}" >>run.log 2>&1 + else + curl -s -L -o run.sh "https://muzaffar.web.cern.ch/crab-test/run.sh?req=${req}&uid=${JENKINS_ID}" + curl -s -L -X POST -d "${xstart}" "https://muzaffar.web.cern.ch/cgi-bin/test-v2?RUN=START&req=${req}&${cmd}&uid=${JENKINS_ID}" >>run.log 2>&1 + chmod +x run.sh + ./run.sh > run.log 2>&1 || true + total_lines=$(cat run.log | wc -l) + sline=1 + xline=20 + while [ $sline -le $total_lines ] ; do + sed -n "${sline},+${xline}p" run.log | base64 > run.base64 + curl -s -L -X POST -d @run.base64 "https://muzaffar.web.cern.ch/cgi-bin/test-v2?result=${sline}of${total_lines}&${cmd}&req=${req}&uid=${JENKINS_ID}" >>run.log 2>&1 + sleep 1 + let sline=$sline+$xline+1 + done + curl -s -L -X POST -d "${xend}" "https://muzaffar.web.cern.ch/cgi-bin/test-v2?RUN=END&req=${req}&${cmd}&uid=${JENKINS_ID}" >>run.log 2>&1 + rm -f run.sh run.log run.base64 + PRECMD="${cmd}" + LRUN=$(date +%s) + RUN_GAP=0 + fi + done +popd +req=$(date +%s) +mv cmdrun/run.log . +rm -rf cmdrun +curl -s -L "https://muzaffar.web.cern.ch/cgi-bin/test-v2?END=1&req=${req}&uid=${JENKINS_ID}" >>run.log 2>&1 diff --git a/crab/crab-test/setup.sh b/crab/crab-test/setup.sh new file mode 100755 index 00000000000..418f76a53eb --- /dev/null +++ b/crab/crab-test/setup.sh @@ -0,0 +1,4 @@ +#!/bin/bash -ex +scram build enable-multi-targets +cp $(dirname $0)/../FrameworkJobReport.xml $WORKSPACE/ +cp $(dirname $0)/../minbias.root $WORKSPACE/ diff --git a/crab/crab-test/task.py b/crab/crab-test/task.py new file mode 100644 index 00000000000..b0ecc1ffd38 --- /dev/null +++ b/crab/crab-test/task.py @@ -0,0 +1,44 @@ +from CRABClient.UserUtilities import config +import os, re, time + +archs = os.environ["SCRAM_ARCH"].split("_") +osMajorVer = int(re.sub("[a-z]", "", archs[0])) + +config = config() +config.General.instance = os.getenv("CRABCONFIGINSTANCE", "prod") +config.General.requestName = os.getenv("CRAB_REQUEST", str(int(time.time()))) +config.General.transferOutputs = True +config.General.transferLogs = False + +config.Data.unitsPerJob = 10 +config.Data.totalUnits = 10 +config.Data.splitting = "EventBased" +config.Data.publication = False + +config.JobType.psetName = "pset.py" +config.JobType.pluginName = "PrivateMC" +config.JobType.maxJobRuntimeMin = int(os.getenv("CRAB_MAX_TIME", "600")) +config.JobType.maxMemoryMB = 2000 +config.JobType.numCores = 1 +config.JobType.allowUndistributedCMSSW = True +config.JobType.scriptExe = "run.sh" +config.JobType.disableAutomaticOutputCollection = True +config.JobType.outputFiles = ["run.log"] +config.JobType.inputFiles = ["FrameworkJobReport.xml", "minbias.root"] + +config.Site.storageSite = "T2_CH_CERN" + +if "CRAB_SCHEDD_NAME" in os.environ and os.environ["CRAB_SCHEDD_NAME"] != "": + config.Debug.scheddName = os.environ["CRAB_SCHEDD_NAME"] +if "CRAB_COLLECTOR" in os.environ and os.environ["CRAB_COLLECTOR"] != "": + config.Debug.collector = os.environ["CRAB_COLLECTOR"] + +config.Debug.extraJDL = ['+REQUIRED_OS="rhel%s"' % osMajorVer] +if "amd64" == archs[1]: + config.Debug.extraJDL.append( + '+DESIRED_Archs="%s"' % "X86_64" if ("amd64" == archs[1]) else archs[1] + ) +if "SINGULARITY_IMAGE" in os.environ and os.environ["SINGULARITY_IMAGE"] != "": + config.Debug.extraJDL.append('+SingularityImage="%s"' % os.environ["SINGULARITY_IMAGE"]) +if "CRAB_SITE" in os.environ and os.environ["CRAB_SITE"] != "": + config.Debug.extraJDL.append('+DESIRED_Sites="%s"' % os.environ["CRAB_SITE"]) diff --git a/crab/ib-run-crab.sh b/crab/ib-run-crab.sh index 2d04fc5708e..0ff5096672e 100755 --- a/crab/ib-run-crab.sh +++ b/crab/ib-run-crab.sh @@ -9,8 +9,16 @@ report() { fi } +thisdir=$(dirname $0) +[ "${CRABCLIENT_TYPE}" != "" ] || export CRABCLIENT_TYPE="prod" +[ "${BUILD_ID}" != "" ] || export BUILD_ID=$(date +%s) +[ "${WORKSPACE}" != "" ] || export WORKSPACE=$(pwd) +[ "${CRABCONFIGINSTANCE}" != "" ]|| export CRABCONFIGINSTANCE="prod" +[ "${JOB_DIR}" != "" ] || JOB_DIR="." + #Checkout a package git cms-addpkg FWCore/Version +[ -x ${thisdir}/${JOB_DIR}/setup.sh ] && ${thisdir}/${JOB_DIR}/setup.sh #Added test python module and script to make sure it is part of card sandbox mkdir -p ${CMSSW_BASE}/src/FWCore/Version/python ${CMSSW_BASE}/src/FWCore/Version/scripts echo 'CMSBOT_CRAB_TEST="OK"' > ${CMSSW_BASE}/src/FWCore/Version/python/cmsbot_crab_test.py @@ -23,11 +31,6 @@ if [ -d ${CMSSW_BASE}/biglib/${SCRAM_ARCH} ] ; then [ -e $l ] || rm -f $l done fi -[ "${CRABCLIENT_TYPE}" != "" ] || export CRABCLIENT_TYPE="prod" -[ "${BUILD_ID}" != "" ] || export BUILD_ID=$(date +%s) -[ "${WORKSPACE}" != "" ] || export WORKSPACE=$(pwd) && cd $WORKSPACE -[ "${CRABCONFIGINSTANCE}" != "" ]|| export CRABCONFIGINSTANCE="prod" -[ "${JOB_DIR}" != "" ] || JOB_DIR="." if [ "${SINGULARITY_IMAGE}" = "" ] ; then osver=$(echo ${SCRAM_ARCH} | tr '_' '\n' | head -1 | sed 's|^[a-z][a-z]*||') @@ -41,12 +44,17 @@ fi export CRAB_REQUEST="Jenkins_${CMSSW_VERSION}_${SCRAM_ARCH}_${BUILD_ID}" cmssw_queue=$(echo ${CMSSW_VERSION} | cut -d_ -f1-3)_X -thisdir=$(dirname $0) -cp ${thisdir}/${JOB_DIR}/run.sh . +cd $WORKSPACE +for f in run.sh task.py pset.py ; do + if [ -e ${thisdir}/${JOB_DIR}/$f ] ; then + cp ${thisdir}/${JOB_DIR}/$f . + else + cp ${thisdir}/$f . + fi +done if [ -e ${thisdir}/${cmssw_queue}/pset.py ] ; then + rm -f pset.py cp ${thisdir}/${cmssw_queue}/pset.py . -else - cp ${thisdir}/${JOB_DIR}/pset.py . fi if [ "${X509_USER_PROXY}" = "" ] ; then voms-proxy-init -voms cms @@ -54,7 +62,8 @@ if [ "${X509_USER_PROXY}" = "" ] ; then fi pyver=$(${CMSBOT_PYTHON_CMD} -c 'import sys;print("python%s%s" % (sys.version_info[0],sys.version_info[1]))') if [ -e ${thisdir}/${pyver} ] ; then export PYTHONPATH="${thisdir}/${pyver}:${PYTHONPATH}"; fi -crab submit --proxy ${X509_USER_PROXY} -c ${thisdir}/${JOB_DIR}/task.py +ls +crab submit --proxy ${X509_USER_PROXY} -c ./task.py rm -rf ${WORKSPACE}/crab mv crab_${CRAB_REQUEST} ${WORKSPACE}/crab echo "INPROGRESS" > $WORKSPACE/crab/statusfile diff --git a/crab/minbias.root b/crab/minbias.root new file mode 100644 index 00000000000..682f5dd73b3 Binary files /dev/null and b/crab/minbias.root differ diff --git a/crab/multiarch/pset.py b/crab/multiarch/pset.py deleted file mode 120000 index 02f61e7a58b..00000000000 --- a/crab/multiarch/pset.py +++ /dev/null @@ -1 +0,0 @@ -../pset.py \ No newline at end of file diff --git a/crab/multiarch/run.sh b/crab/multiarch/run.sh deleted file mode 100755 index 9de715b065f..00000000000 --- a/crab/multiarch/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -for f in minbias.root FrameworkJobReport.xml ; do - curl -L -o $f http://cern.ch/muzaffar/$f - [ -e $f ] || exit 1 -done -ls -env > run.log -echo "======================" >> run.log -pushd $CMSSW_BASE - for dir in lib biglib ; do - rm -rf $dir - rsync -a $CMSSW_RELEASE_BASE/${dir}/ ./${dir}/ || true - [ -d ${dir}/$SCRAM_ARCH/scram_x86-64-v2 ] || continue - pushd ${dir}/$SCRAM_ARCH/scram_x86-64-v2 - for pcm in $(ls *.pcm) ; do - rm -f $pcm - cp ../$pcm . - done - popd - done -pushd -export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr : '\n' | grep -E -v "$CMSSW_RELEASE_BASE/(big|)lib/" | grep -E -v "/${CMSSW_VERSION}/(big|)lib/${SCRAM_ARCH}$" | tr '\n' ':') -echo $LD_LIBRARY_PATH | tr : '\n' -ld.so --help | grep supported | grep x86-64-v -which cmsRun -strace -f cmsRun --help diff --git a/crab/multiarch/setup.sh b/crab/multiarch/setup.sh new file mode 100755 index 00000000000..fac6fa5b6a9 --- /dev/null +++ b/crab/multiarch/setup.sh @@ -0,0 +1,2 @@ +#!/bin/bash +scram build enable-multi-targets diff --git a/crab/multiarch/task.py b/crab/multiarch/task.py deleted file mode 120000 index 5e32f54411c..00000000000 --- a/crab/multiarch/task.py +++ /dev/null @@ -1 +0,0 @@ -../task.py \ No newline at end of file diff --git a/crab/run.sh b/crab/run.sh index 7f9a4cf63a1..131ba9822af 100755 --- a/crab/run.sh +++ b/crab/run.sh @@ -1,8 +1,5 @@ #!/bin/bash -ex -pwd -ls env > run.log ld.so --help | grep supported | grep x86-64-v which cmsRun -cmsRun --help -cmsRun -j FrameworkJobReport.xml PSet.py +cmsRun -j FrameworkJobReport.xml PSet.py >>run.log 2>&1 diff --git a/crab/scram-build/run.sh b/crab/scram-build/run.sh index 05ff887e747..837af2d5a8d 100755 --- a/crab/scram-build/run.sh +++ b/crab/scram-build/run.sh @@ -6,9 +6,6 @@ pushd $CMSSW_BASE scram build enable-multi-targets rm -rf src mkdir src - git clone https://github.com/cms-sw/cmssw-config - cd cmssw-config; git checkout V09-06-02 ; cd .. - rm -rf config/SCRAM; mv cmssw-config/SCRAM config/SCRAM git cms-addpkg '*' > $log scram b -v -k -j $(nproc) >$log 2>&1 || true eval `scram run -sh` @@ -17,5 +14,4 @@ pushd $CMSSW_BASE which cmsRun cmsRun --help >>$log || true popd -[ -f $log ] || mv $CMSSW_BASE/$log $log -cmsRun -j FrameworkJobReport.xml PSet.py || true +cmsRun -j FrameworkJobReport.xml PSet.py >run.log 2>&1 || true diff --git a/crab/scram-build/task.py b/crab/scram-build/task.py index b01b90de4ae..92fabbde40c 100644 --- a/crab/scram-build/task.py +++ b/crab/scram-build/task.py @@ -22,6 +22,7 @@ config.JobType.numCores = 8 config.JobType.allowUndistributedCMSSW = True config.JobType.scriptExe = "run.sh" +config.JobType.disableAutomaticOutputCollection = True config.JobType.outputFiles = ["run.log"] config.Site.storageSite = "T2_CH_CERN" diff --git a/crab/script/pset.py b/crab/script/pset.py new file mode 100644 index 00000000000..6c21333b697 --- /dev/null +++ b/crab/script/pset.py @@ -0,0 +1,115 @@ +# Auto generated configuration file +# using: +# Revision: 1.19 +# Source: /local/reps/CMSSW/CMSSW/Configuration/Applications/python/ConfigBuilder.py,v +# with command line options: MinBias_8TeV_cfi --conditions auto:startup -s GEN,SIM --datatier GEN-SIM -n 10 +# --relval 9000,300 --eventcontent RAWSIM --io MinBias.io --python MinBias.py --no_exec --fileout minbias.root + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("SIM") + +# Import of standard configurations +process.load("Configuration.StandardSequences.Services_cff") +process.load("SimGeneral.HepPDTESSource.pythiapdt_cfi") +process.load("FWCore.MessageService.MessageLogger_cfi") +process.load("Configuration.EventContent.EventContent_cff") +process.load("SimGeneral.MixingModule.mixNoPU_cfi") +process.load("Configuration.StandardSequences.GeometryRecoDB_cff") +process.load("Configuration.Geometry.GeometrySimDB_cff") +process.load("Configuration.StandardSequences.MagneticField_38T_cff") +process.load("Configuration.StandardSequences.Generator_cff") +process.load("IOMC.EventVertexGenerators.VtxSmearedRealistic8TeVCollision_cfi") +process.load("GeneratorInterface.Core.genFilterSummary_cff") +process.load("Configuration.StandardSequences.SimIdeal_cff") +process.load("Configuration.StandardSequences.EndOfProcess_cff") +process.load("Configuration.StandardSequences.FrontierConditions_GlobalTag_cff") + +process.maxEvents = cms.untracked.PSet(input=cms.untracked.int32(10)) + +# Input source +process.source = cms.Source("EmptySource") + +process.options = cms.untracked.PSet() +process.options.numberOfThreads = cms.untracked.uint32(1) + +# Production Info +process.configurationMetadata = cms.untracked.PSet( + version=cms.untracked.string("$Revision: 1.19 $"), + annotation=cms.untracked.string("MinBias_8TeV_cfi nevts:10"), + name=cms.untracked.string("Applications"), +) + +# Additional output definition + +# Other statements +process.genstepfilter.triggerConditions = cms.vstring("generation_step") +from Configuration.AlCa.GlobalTag import GlobalTag + +process.GlobalTag = GlobalTag(process.GlobalTag, "auto:startup", "") + +process.generator = cms.EDFilter( + "Pythia6GeneratorFilter", + pythiaPylistVerbosity=cms.untracked.int32(0), + filterEfficiency=cms.untracked.double(1.0), + pythiaHepMCVerbosity=cms.untracked.bool(False), + comEnergy=cms.double(8000.0), + maxEventsToPrint=cms.untracked.int32(0), + PythiaParameters=cms.PSet( + pythiaUESettings=cms.vstring( + "MSTU(21)=1 ! Check on possible errors during program execution", + "MSTJ(22)=2 ! Decay those unstable particles", + "PARJ(71)=10 . ! for which ctau 10 mm", + "MSTP(33)=0 ! no K factors in hard cross sections", + "MSTP(2)=1 ! which order running alphaS", + "MSTP(51)=10042 ! structure function chosen (external PDF CTEQ6L1)", + "MSTP(52)=2 ! work with LHAPDF", + "PARP(82)=1.921 ! pt cutoff for multiparton interactions", + "PARP(89)=1800. ! sqrts for which PARP82 is set", + "PARP(90)=0.227 ! Multiple interactions: rescaling power", + "MSTP(95)=6 ! CR (color reconnection parameters)", + "PARP(77)=1.016 ! CR", + "PARP(78)=0.538 ! CR", + "PARP(80)=0.1 ! Prob. colored parton from BBR", + "PARP(83)=0.356 ! Multiple interactions: matter distribution parameter", + "PARP(84)=0.651 ! Multiple interactions: matter distribution parameter", + "PARP(62)=1.025 ! ISR cutoff", + "MSTP(91)=1 ! Gaussian primordial kT", + "PARP(93)=10.0 ! primordial kT-max", + "MSTP(81)=21 ! multiple parton interactions 1 is Pythia default", + "MSTP(82)=4 ! Defines the multi-parton model", + ), + processParameters=cms.vstring( + "MSEL=0 ! User defined processes", + "MSUB(11)=1 ! Min bias process", + "MSUB(12)=1 ! Min bias process", + "MSUB(13)=1 ! Min bias process", + "MSUB(28)=1 ! Min bias process", + "MSUB(53)=1 ! Min bias process", + "MSUB(68)=1 ! Min bias process", + "MSUB(92)=1 ! Min bias process, single diffractive", + "MSUB(93)=1 ! Min bias process, single diffractive", + "MSUB(94)=1 ! Min bias process, double diffractive", + "MSUB(95)=1 ! Min bias process", + ), + parameterSets=cms.vstring("pythiaUESettings", "processParameters"), + ), +) + +# Path and EndPath definitions +process.generation_step = cms.Path(process.pgen) +process.simulation_step = cms.Path(process.psim) +process.genfiltersummary_step = cms.EndPath(process.genFilterSummary) +process.endjob_step = cms.EndPath(process.endOfProcess) + +# Schedule definition +process.schedule = cms.Schedule( + process.generation_step, + process.genfiltersummary_step, + process.simulation_step, + process.endjob_step, +) + +# Filter all path with the production filter sequence +for path in process.paths: + getattr(process, path)._seq = process.generator * getattr(process, path)._seq diff --git a/crab/script/run.sh b/crab/script/run.sh new file mode 100755 index 00000000000..11eb95c32a6 --- /dev/null +++ b/crab/script/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash -ex +pwd +ls +ld.so --help | grep supported | grep x86-64-v +which cmsRun +cmsRun --help >>run.log diff --git a/crab/script/setup.sh b/crab/script/setup.sh new file mode 100755 index 00000000000..03318c9117e --- /dev/null +++ b/crab/script/setup.sh @@ -0,0 +1,3 @@ +#!/bin/bash -ex +cp $(dirname $0)/../FrameworkJobReport.xml $WORKSPACE/ +cp $(dirname $0)/../minbias.root $WORKSPACE/ diff --git a/crab/script/task.py b/crab/script/task.py new file mode 100644 index 00000000000..50ca822fc28 --- /dev/null +++ b/crab/script/task.py @@ -0,0 +1,44 @@ +from CRABClient.UserUtilities import config +import os, re, time + +archs = os.environ["SCRAM_ARCH"].split("_") +osMajorVer = int(re.sub("[a-z]", "", archs[0])) + +config = config() +config.General.instance = os.getenv("CRABCONFIGINSTANCE", "prod") +config.General.requestName = os.getenv("CRAB_REQUEST", str(int(time.time()))) +config.General.transferOutputs = True +config.General.transferLogs = False + +config.Data.unitsPerJob = 10 +config.Data.totalUnits = 10 +config.Data.splitting = "EventBased" +config.Data.publication = False + +config.JobType.psetName = "pset.py" +config.JobType.pluginName = "PrivateMC" +config.JobType.maxJobRuntimeMin = 10 +config.JobType.maxMemoryMB = 2000 +config.JobType.numCores = 1 +config.JobType.allowUndistributedCMSSW = True +config.JobType.scriptExe = "run.sh" +config.JobType.disableAutomaticOutputCollection = True +config.JobType.outputFiles = ["run.log"] +config.JobType.inputFiles = ["FrameworkJobReport.xml"] + +config.Site.storageSite = "T2_CH_CERN" + +if "CRAB_SCHEDD_NAME" in os.environ and os.environ["CRAB_SCHEDD_NAME"] != "": + config.Debug.scheddName = os.environ["CRAB_SCHEDD_NAME"] +if "CRAB_COLLECTOR" in os.environ and os.environ["CRAB_COLLECTOR"] != "": + config.Debug.collector = os.environ["CRAB_COLLECTOR"] + +config.Debug.extraJDL = ['+REQUIRED_OS="rhel%s"' % osMajorVer] +if "amd64" == archs[1]: + config.Debug.extraJDL.append( + '+DESIRED_Archs="%s"' % "X86_64" if ("amd64" == archs[1]) else archs[1] + ) +if "SINGULARITY_IMAGE" in os.environ and os.environ["SINGULARITY_IMAGE"] != "": + config.Debug.extraJDL.append('+SingularityImage="%s"' % os.environ["SINGULARITY_IMAGE"]) +if "CRAB_SITE" in os.environ and os.environ["CRAB_SITE"] != "": + config.Debug.extraJDL.append('+DESIRED_Sites="%s"' % os.environ["CRAB_SITE"]) diff --git a/crab/short-matrix/pset.py b/crab/short-matrix/pset.py new file mode 100644 index 00000000000..d8a9b94ed9b --- /dev/null +++ b/crab/short-matrix/pset.py @@ -0,0 +1,130 @@ +# Auto generated configuration file +# using: +# Revision: 1.19 +# Source: /local/reps/CMSSW/CMSSW/Configuration/Applications/python/ConfigBuilder.py,v +# with command line options: MinBias_8TeV_cfi --conditions auto:startup -s GEN,SIM --datatier GEN-SIM -n 10 +# --relval 9000,300 --eventcontent RAWSIM --io MinBias.io --python MinBias.py --no_exec --fileout minbias.root + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("SIM") + +# Import of standard configurations +process.load("Configuration.StandardSequences.Services_cff") +process.load("SimGeneral.HepPDTESSource.pythiapdt_cfi") +process.load("FWCore.MessageService.MessageLogger_cfi") +process.load("Configuration.EventContent.EventContent_cff") +process.load("SimGeneral.MixingModule.mixNoPU_cfi") +process.load("Configuration.StandardSequences.GeometryRecoDB_cff") +process.load("Configuration.Geometry.GeometrySimDB_cff") +process.load("Configuration.StandardSequences.MagneticField_38T_cff") +process.load("Configuration.StandardSequences.Generator_cff") +process.load("IOMC.EventVertexGenerators.VtxSmearedRealistic8TeVCollision_cfi") +process.load("GeneratorInterface.Core.genFilterSummary_cff") +process.load("Configuration.StandardSequences.SimIdeal_cff") +process.load("Configuration.StandardSequences.EndOfProcess_cff") +process.load("Configuration.StandardSequences.FrontierConditions_GlobalTag_cff") + +process.maxEvents = cms.untracked.PSet(input=cms.untracked.int32(10)) + +# Input source +process.source = cms.Source("EmptySource") + +process.options = cms.untracked.PSet() +process.options.numberOfThreads = cms.untracked.uint32(8) + +# Production Info +process.configurationMetadata = cms.untracked.PSet( + version=cms.untracked.string("$Revision: 1.19 $"), + annotation=cms.untracked.string("MinBias_8TeV_cfi nevts:10"), + name=cms.untracked.string("Applications"), +) + +# Output definition +process.RAWSIMoutput = cms.OutputModule( + "PoolOutputModule", + splitLevel=cms.untracked.int32(0), + eventAutoFlushCompressedSize=cms.untracked.int32(5242880), + outputCommands=process.RAWSIMEventContent.outputCommands, + fileName=cms.untracked.string("minbias.root"), + dataset=cms.untracked.PSet( + filterName=cms.untracked.string(""), dataTier=cms.untracked.string("GEN-SIM") + ), + SelectEvents=cms.untracked.PSet(SelectEvents=cms.vstring("generation_step")), +) + +# Additional output definition + +# Other statements +process.genstepfilter.triggerConditions = cms.vstring("generation_step") +from Configuration.AlCa.GlobalTag import GlobalTag + +process.GlobalTag = GlobalTag(process.GlobalTag, "auto:startup", "") + +process.generator = cms.EDFilter( + "Pythia6GeneratorFilter", + pythiaPylistVerbosity=cms.untracked.int32(0), + filterEfficiency=cms.untracked.double(1.0), + pythiaHepMCVerbosity=cms.untracked.bool(False), + comEnergy=cms.double(8000.0), + maxEventsToPrint=cms.untracked.int32(0), + PythiaParameters=cms.PSet( + pythiaUESettings=cms.vstring( + "MSTU(21)=1 ! Check on possible errors during program execution", + "MSTJ(22)=2 ! Decay those unstable particles", + "PARJ(71)=10 . ! for which ctau 10 mm", + "MSTP(33)=0 ! no K factors in hard cross sections", + "MSTP(2)=1 ! which order running alphaS", + "MSTP(51)=10042 ! structure function chosen (external PDF CTEQ6L1)", + "MSTP(52)=2 ! work with LHAPDF", + "PARP(82)=1.921 ! pt cutoff for multiparton interactions", + "PARP(89)=1800. ! sqrts for which PARP82 is set", + "PARP(90)=0.227 ! Multiple interactions: rescaling power", + "MSTP(95)=6 ! CR (color reconnection parameters)", + "PARP(77)=1.016 ! CR", + "PARP(78)=0.538 ! CR", + "PARP(80)=0.1 ! Prob. colored parton from BBR", + "PARP(83)=0.356 ! Multiple interactions: matter distribution parameter", + "PARP(84)=0.651 ! Multiple interactions: matter distribution parameter", + "PARP(62)=1.025 ! ISR cutoff", + "MSTP(91)=1 ! Gaussian primordial kT", + "PARP(93)=10.0 ! primordial kT-max", + "MSTP(81)=21 ! multiple parton interactions 1 is Pythia default", + "MSTP(82)=4 ! Defines the multi-parton model", + ), + processParameters=cms.vstring( + "MSEL=0 ! User defined processes", + "MSUB(11)=1 ! Min bias process", + "MSUB(12)=1 ! Min bias process", + "MSUB(13)=1 ! Min bias process", + "MSUB(28)=1 ! Min bias process", + "MSUB(53)=1 ! Min bias process", + "MSUB(68)=1 ! Min bias process", + "MSUB(92)=1 ! Min bias process, single diffractive", + "MSUB(93)=1 ! Min bias process, single diffractive", + "MSUB(94)=1 ! Min bias process, double diffractive", + "MSUB(95)=1 ! Min bias process", + ), + parameterSets=cms.vstring("pythiaUESettings", "processParameters"), + ), +) + +# Path and EndPath definitions +process.generation_step = cms.Path(process.pgen) +process.simulation_step = cms.Path(process.psim) +process.genfiltersummary_step = cms.EndPath(process.genFilterSummary) +process.endjob_step = cms.EndPath(process.endOfProcess) +process.RAWSIMoutput_step = cms.EndPath(process.RAWSIMoutput) + +# Schedule definition +process.schedule = cms.Schedule( + process.generation_step, + process.genfiltersummary_step, + process.simulation_step, + process.endjob_step, + process.RAWSIMoutput_step, +) + +# Filter all path with the production filter sequence +for path in process.paths: + getattr(process, path)._seq = process.generator * getattr(process, path)._seq diff --git a/crab/short-matrix/run.sh b/crab/short-matrix/run.sh new file mode 100755 index 00000000000..741a61a7270 --- /dev/null +++ b/crab/short-matrix/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e +env > run.log +ld.so --help | grep supported | grep x86-64-v +mkdir matrix +pushd matrix + runTheMatrix.py -i all -s -j 3 -t 4 --ibeos >>../run.log 2>&1 || touch runall-report-step123-.log + for f in $(find . -name '*' -type f) ; do + case $f in + *.xml|*.txt|*.log|*.py|*.json|*/cmdLog ) ;; + * ) rm -rf $f ;; + esac + done +popd +mv matrix/runall-report-step123-.log matrix.log +tar -czvf matrix.tar.gz matrix diff --git a/crab/short-matrix/setup.sh b/crab/short-matrix/setup.sh new file mode 100755 index 00000000000..929bb2fc336 --- /dev/null +++ b/crab/short-matrix/setup.sh @@ -0,0 +1,5 @@ +#!/bin/bash -ex +scram build enable-multi-targets +cp $(dirname $0)/../FrameworkJobReport.xml $WORKSPACE/ +cp $(dirname $0)/../minbias.root $WORKSPACE/ + diff --git a/crab/short-matrix/task.py b/crab/short-matrix/task.py new file mode 100644 index 00000000000..a638014752c --- /dev/null +++ b/crab/short-matrix/task.py @@ -0,0 +1,44 @@ +from CRABClient.UserUtilities import config +import os, re, time + +archs = os.environ["SCRAM_ARCH"].split("_") +osMajorVer = int(re.sub("[a-z]", "", archs[0])) + +config = config() +config.General.instance = os.getenv("CRABCONFIGINSTANCE", "prod") +config.General.requestName = os.getenv("CRAB_REQUEST", str(int(time.time()))) +config.General.transferOutputs = True +config.General.transferLogs = False + +config.Data.unitsPerJob = 10 +config.Data.totalUnits = 10 +config.Data.splitting = "EventBased" +config.Data.publication = False + +config.JobType.psetName = "pset.py" +config.JobType.pluginName = "PrivateMC" +config.JobType.maxJobRuntimeMin = 300 +config.JobType.maxMemoryMB = 16000 +config.JobType.numCores = 8 +config.JobType.allowUndistributedCMSSW = True +config.JobType.scriptExe = "run.sh" +config.JobType.disableAutomaticOutputCollection = True +config.JobType.outputFiles = ["matrix.tar.gz", "matrix.log", "run.log"] +config.JobType.inputFiles = ["FrameworkJobReport.xml", "minbias.root"] + +config.Site.storageSite = "T2_CH_CERN" + +if "CRAB_SCHEDD_NAME" in os.environ and os.environ["CRAB_SCHEDD_NAME"] != "": + config.Debug.scheddName = os.environ["CRAB_SCHEDD_NAME"] +if "CRAB_COLLECTOR" in os.environ and os.environ["CRAB_COLLECTOR"] != "": + config.Debug.collector = os.environ["CRAB_COLLECTOR"] + +config.Debug.extraJDL = ['+REQUIRED_OS="rhel%s"' % osMajorVer] +if "amd64" == archs[1]: + config.Debug.extraJDL.append( + '+DESIRED_Archs="%s"' % "X86_64" if ("amd64" == archs[1]) else archs[1] + ) +if "SINGULARITY_IMAGE" in os.environ and os.environ["SINGULARITY_IMAGE"] != "": + config.Debug.extraJDL.append('+SingularityImage="%s"' % os.environ["SINGULARITY_IMAGE"]) +if "CRAB_SITE" in os.environ and os.environ["CRAB_SITE"] != "": + config.Debug.extraJDL.append('+DESIRED_Sites="%s"' % os.environ["CRAB_SITE"]) diff --git a/crab/task.py b/crab/task.py index de3499024fb..cdc7be64ee7 100644 --- a/crab/task.py +++ b/crab/task.py @@ -22,6 +22,7 @@ config.JobType.numCores = 1 config.JobType.allowUndistributedCMSSW = True config.JobType.scriptExe = "run.sh" +config.JobType.disableAutomaticOutputCollection = True config.JobType.outputFiles = ["run.log"] config.Site.storageSite = "T2_CH_CERN" diff --git a/es_relval_log.py b/es_relval_log.py index 48a6d39479e..2296642ac71 100755 --- a/es_relval_log.py +++ b/es_relval_log.py @@ -215,7 +215,6 @@ def es_parse_log(logFile): send_payload("ib-dataset-" + week, "relvals-dataset", idx, json.dumps(dataset)) ref = open(stamp, "w") ref.close() - print("Done processing :", logFile) if __name__ == "__main__":