From 6dc1c83607a9c641a1b23c6078ed1f0043c90da1 Mon Sep 17 00:00:00 2001 From: Efe Date: Mon, 28 Oct 2024 13:55:05 +0100 Subject: [PATCH 01/11] run3 gridpack runcmsgrid patch --- bin/utils/request_fragment_check.py | 99 ++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 31 deletions(-) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 08af7cbf901..c6004c5636f 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -11,6 +11,7 @@ import glob import json import ast +import subprocess from datetime import datetime from json import dumps @@ -313,34 +314,67 @@ def ul_consistency(dn,pi,jhu_gp): if not error_ul: print("UL consistency check is OK.") return warning_ul,error_ul +def gridpack_copy(gridpack_eos_path,pi): + error_gp_copy = 0 + targz_flag = 0 + if "Run3" in pi: + copy_name = "_original_Run3_wo_runcmsgrid_sys_patch" + else: + copy_name = "_original" + if ".tar.gz" in gridpack_eos_path: + targz_flag = 1 + gridpack_eos_path_backup = gridpack_eos_path.replace('.tar.gz',copy_name+'.tar.gz') + if ".tgz" in gridpack_eos_path: gridpack_eos_path_backup = gridpack_eos_path.replace('.tgz',copy_name+'.tgz') + if ".tar.xz" in gridpack_eos_path: + gridpack_eos_path_backup = gridpack_eos_path.replace('.tar.xz',copy_name+'.tar.xz') + targz_flag = 2 + if not os.path.exists(gridpack_eos_path_backup): + print("Backup gridpack does not exist.") + print("Copying "+gridpack_eos_path+" to "+gridpack_eos_path_backup+" before patching runcms.grid") + os.system('cp -n -p '+gridpack_eos_path+' '+gridpack_eos_path_backup) + md5_1 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] + md5_2 = os.popen('md5sum'+' '+gridpack_eos_path_backup).read().split(' ')[0] + if md5_1 == md5_2: + print("Backup and original file checksums are equal.") + else: + error_gp_copy.append("backup gridpack has a problem.") + print(gridpack_eos_path_backup) + return error_gp_copy + +def gridpack_repack_and_copy(gridpack_eos_path,pi): + error_gridpack_repack = [] + gp_extension = ".tar.xz" + if ".tar.gz" in gridpack_eos_path: + gp_extension = ".tar.gz" + if ".tgz" in gridpack_eos_path: + gp_extension = ".tgz" + gp_name = "gridpack"+gp_extension + print("re-tarring to "+gp_name) + cur_dir = os.getcwd() + os.chdir(my_path+'/'+pi) + os.environ['XZ_OPT'] = "--lzma2=preset=9,dict=512MiB" + os.system('XZ_OPT="$XZ_OPT" tar -cJpf '+gp_name+' --exclude='+gp_name+' --exclude='+pi+' ./*') + os.system('cp '+gp_name+' '+gridpack_eos_path) + md5_1 = os.popen('md5sum '+gp_name).read().split(' ')[0] + md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] + if md5_1 == md5_2: + print("Updated gridpack copied succesfully.") + else: + error.append("There was a problem copying in the updated gridpack to eos.") + os.chdir(cur_dir) + return error_gridpack_repack + def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): xml = str(re.findall('xmllint.*',cont)) cur_dir = os.getcwd() warning_xml = [] error_xml = [] if "stream" not in xml or len(xml) < 3: - targz_flag = 0 if "stream" not in xml and len(xml) > 3: warning_xml.append(" --stream option is missing in XMLLINT, will update runcmsgrid.") if len(xml) < 3: warning_xml.append("[WARNING] XMLLINT does not exist in runcmsgrid, will update it.") - if ".tar.gz" in gridpack_eos_path: - targz_flag = 1 - gridpack_eos_path_backup = gridpack_eos_path.replace('.tar.gz','_original.tar.gz') - if ".tgz" in gridpack_eos_path: gridpack_eos_path_backup = gridpack_eos_path.replace('.tgz','_original.tgz') - if ".tar.xz" in gridpack_eos_path: - gridpack_eos_path_backup = gridpack_eos_path.replace('.tar.xz','_original.tar.xz') - targz_flag = 2 - if not os.path.exists(gridpack_eos_path_backup): - print("Backup gridpack does not exist.") - print("Copying "+gridpack_eos_path+" to "+gridpack_eos_path_backup+" before patching runcms.grid") - os.system('cp -n -p '+gridpack_eos_path+' '+gridpack_eos_path_backup) - md5_1 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] - md5_2 = os.popen('md5sum'+' '+gridpack_eos_path_backup).read().split(' ')[0] - if md5_1 == md5_2: - print("Backup and original file checksums are equal.") - else: - error_xml.append("backup gridpack has a problem.") + error_xml.append(gridpack_copy(gridpack_eos_path)) print("Updating XMLLINT line in runcmsgrid.") os.chdir(my_path+'/'+pi) if "stream" not in xml and len(xml) > 3: cont = re.sub("xmllint","xmllint --stream",cont) @@ -351,19 +385,7 @@ def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): f.seek(0) f.write(cont) f.truncate() - if targz_flag == 0: gridpackname = "gridpack.tgz" - if targz_flag == 1: gridpackname = "gridpack.tar.gz" - if targz_flag == 2: gridpackname = "gridpack.tar.xz" - os.chdir(my_path+'/'+pi) - os.system('tar cfJ '+gridpackname+' ./* --exclude='+gridpackname+' --exclude='+pi) - os.system('cp '+gridpackname+' '+gridpack_eos_path) - md5_1 = os.popen('md5sum '+gridpackname).read().split(' ')[0] - md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] - if md5_1 == md5_2: - print("Updated gridpack copied succesfully.") - else: - error_xml.append("There was a problem copying in the updated gridpack to eos.") - os.chdir(cur_dir) + error_xml = gridpack_repack_and_copy(gridpack_eos_path,pi) return warning_xml,error_xml def evtgen_check(fragment): @@ -1511,6 +1533,21 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): errors.extend(check_replace(runcmsgrid_file)) fmg_f = re.sub(r'(?m)^ *#.*\n?', '',fmg_f) mg_me_pdf_list = re.findall('pdfsets=\S+',fmg_f) +######################################################################################################################################### + if "Run3" in pi: + if int(os.popen('grep -c "systematics $runlabel" '+str(runcmsgrid_file)).read()): + if int(os.popen('grep -c "Encounter Error in Running Systematics Module" '+str(runcmsgrid_file)).read()) < 1: + print("runcmsgrid script patch for Run3 missing!") + print("I will patch the runcmsgrid script.") + print("Backing up the gridpack to :") + gridpack_copy(gridpack_eos_path,pi) + if mg_nlo: + os.system("patch "+runcmsgrid_file+" < /eos/cms/store/group/phys_generator/cvmfs/gridpacks/mg_amg_patch/runcmsgrid_systematics_NLO.patch") + if mg_lo: + os.system("patch "+runcmsgrid_file+" < /eos/cms/store/group/phys_generator/cvmfs/gridpacks/mg_amg_patch/runcmsgrid_systematics_LO.patch") + err_gpr = gridpack_repack_and_copy(gridpack_eos_path,pi) + errors.append(err_gpr) +######################################################################################################################################### if mg5_aMC_version >= 260: mg_lo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c madevent').read()) mg_nlo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c aMCatNLO').read()) From b1afa8833b95709aa8f0f889e1f7c91bb759d1af Mon Sep 17 00:00:00 2001 From: Efe Date: Mon, 28 Oct 2024 13:56:56 +0100 Subject: [PATCH 02/11] run3 gridpack runcmsgrid patch --- bin/utils/request_fragment_check.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index c6004c5636f..d02901ae0b5 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -1533,7 +1533,6 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): errors.extend(check_replace(runcmsgrid_file)) fmg_f = re.sub(r'(?m)^ *#.*\n?', '',fmg_f) mg_me_pdf_list = re.findall('pdfsets=\S+',fmg_f) -######################################################################################################################################### if "Run3" in pi: if int(os.popen('grep -c "systematics $runlabel" '+str(runcmsgrid_file)).read()): if int(os.popen('grep -c "Encounter Error in Running Systematics Module" '+str(runcmsgrid_file)).read()) < 1: @@ -1547,7 +1546,6 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): os.system("patch "+runcmsgrid_file+" < /eos/cms/store/group/phys_generator/cvmfs/gridpacks/mg_amg_patch/runcmsgrid_systematics_LO.patch") err_gpr = gridpack_repack_and_copy(gridpack_eos_path,pi) errors.append(err_gpr) -######################################################################################################################################### if mg5_aMC_version >= 260: mg_lo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c madevent').read()) mg_nlo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c aMCatNLO').read()) From 5b8c2de6f0482c913bf9be5ac7a4a0c854c936a3 Mon Sep 17 00:00:00 2001 From: Efe Date: Mon, 28 Oct 2024 14:05:19 +0100 Subject: [PATCH 03/11] run3 gridpack runcmsgrid patch --- bin/utils/request_fragment_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index d02901ae0b5..d5391c26af3 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -360,7 +360,7 @@ def gridpack_repack_and_copy(gridpack_eos_path,pi): if md5_1 == md5_2: print("Updated gridpack copied succesfully.") else: - error.append("There was a problem copying in the updated gridpack to eos.") + error_gridpack_repack.append("There was a problem copying in the updated gridpack to eos.") os.chdir(cur_dir) return error_gridpack_repack From 23fa8210e72bf22645e3a1d3390e772b703c2784 Mon Sep 17 00:00:00 2001 From: Efe Date: Mon, 28 Oct 2024 14:21:30 +0100 Subject: [PATCH 04/11] run3 gridpack runcmsgrid patch --- bin/utils/request_fragment_check.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index d5391c26af3..fb65ec2e2ac 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -315,7 +315,7 @@ def ul_consistency(dn,pi,jhu_gp): return warning_ul,error_ul def gridpack_copy(gridpack_eos_path,pi): - error_gp_copy = 0 + error_gp_copy = [] targz_flag = 0 if "Run3" in pi: copy_name = "_original_Run3_wo_runcmsgrid_sys_patch" @@ -374,7 +374,7 @@ def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): warning_xml.append(" --stream option is missing in XMLLINT, will update runcmsgrid.") if len(xml) < 3: warning_xml.append("[WARNING] XMLLINT does not exist in runcmsgrid, will update it.") - error_xml.append(gridpack_copy(gridpack_eos_path)) + error_xml.append(gridpack_copy(gridpack_eos_path,pi)) print("Updating XMLLINT line in runcmsgrid.") os.chdir(my_path+'/'+pi) if "stream" not in xml and len(xml) > 3: cont = re.sub("xmllint","xmllint --stream",cont) @@ -1539,13 +1539,14 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): print("runcmsgrid script patch for Run3 missing!") print("I will patch the runcmsgrid script.") print("Backing up the gridpack to :") - gridpack_copy(gridpack_eos_path,pi) + err_gpr = gridpack_copy(gridpack_eos_path,pi) + errors.extend(err_gpr) if mg_nlo: os.system("patch "+runcmsgrid_file+" < /eos/cms/store/group/phys_generator/cvmfs/gridpacks/mg_amg_patch/runcmsgrid_systematics_NLO.patch") if mg_lo: os.system("patch "+runcmsgrid_file+" < /eos/cms/store/group/phys_generator/cvmfs/gridpacks/mg_amg_patch/runcmsgrid_systematics_LO.patch") - err_gpr = gridpack_repack_and_copy(gridpack_eos_path,pi) - errors.append(err_gpr) + err_gpr = gridpack_repack_and_copy(gridpack_eos_path,pi) + errors.extend(err_gpr) if mg5_aMC_version >= 260: mg_lo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c madevent').read()) mg_nlo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c aMCatNLO').read()) From 7863f3a31238e46699973526fc90a2100a7247d4 Mon Sep 17 00:00:00 2001 From: Efe Date: Mon, 28 Oct 2024 16:34:35 +0100 Subject: [PATCH 05/11] run3 runcmsgrid patch --- bin/utils/request_fragment_check.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index fb65ec2e2ac..73c80d77085 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -338,10 +338,10 @@ def gridpack_copy(gridpack_eos_path,pi): print("Backup and original file checksums are equal.") else: error_gp_copy.append("backup gridpack has a problem.") - print(gridpack_eos_path_backup) + print("Backup gridpack: "+gridpack_eos_path_backup) return error_gp_copy -def gridpack_repack_and_copy(gridpack_eos_path,pi): +def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): error_gridpack_repack = [] gp_extension = ".tar.xz" if ".tar.gz" in gridpack_eos_path: @@ -357,6 +357,7 @@ def gridpack_repack_and_copy(gridpack_eos_path,pi): os.system('cp '+gp_name+' '+gridpack_eos_path) md5_1 = os.popen('md5sum '+gp_name).read().split(' ')[0] md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] + print("Checksums = ",md5_1,md5_2) if md5_1 == md5_2: print("Updated gridpack copied succesfully.") else: @@ -385,7 +386,7 @@ def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): f.seek(0) f.write(cont) f.truncate() - error_xml = gridpack_repack_and_copy(gridpack_eos_path,pi) + error_xml = gridpack_repack_and_copy(gridpack_eos_path,my_path,pi) return warning_xml,error_xml def evtgen_check(fragment): @@ -1172,7 +1173,7 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): warnings.append(pf[1]+" "+pf[2]+" You may try to request more events per phase-space region in the gridpack.") if os.path.isfile(gp_log_loc) and ('madgraph' in dn.lower() or 'amcatnlo' in dn.lower()): print("------------------------------------------------------------------------------------") - print("Summary for madgraph for experts fron gridpack log (cross section BEFORE matching (if there is matching/merging)):") + print("Summary for madgraph for experts from gridpack log (cross section BEFORE matching (if there is matching/merging)):") print(os.popen('grep Summary '+gp_log_loc+' -A 5 -B 1').read()) print("------------------------------------------------------------------------------------") if mg_gp: @@ -1533,23 +1534,26 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): errors.extend(check_replace(runcmsgrid_file)) fmg_f = re.sub(r'(?m)^ *#.*\n?', '',fmg_f) mg_me_pdf_list = re.findall('pdfsets=\S+',fmg_f) + if mg5_aMC_version >= 260: + mg_lo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c madevent').read()) + mg_nlo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c aMCatNLO').read()) + if mg_lo: print("LO gridpack") + if mg_nlo: print("NLO gridpack") if "Run3" in pi: if int(os.popen('grep -c "systematics $runlabel" '+str(runcmsgrid_file)).read()): if int(os.popen('grep -c "Encounter Error in Running Systematics Module" '+str(runcmsgrid_file)).read()) < 1: + print("-----------------------------------------") print("runcmsgrid script patch for Run3 missing!") + print("-----------------------------------------") print("I will patch the runcmsgrid script.") - print("Backing up the gridpack to :") err_gpr = gridpack_copy(gridpack_eos_path,pi) errors.extend(err_gpr) if mg_nlo: os.system("patch "+runcmsgrid_file+" < /eos/cms/store/group/phys_generator/cvmfs/gridpacks/mg_amg_patch/runcmsgrid_systematics_NLO.patch") if mg_lo: os.system("patch "+runcmsgrid_file+" < /eos/cms/store/group/phys_generator/cvmfs/gridpacks/mg_amg_patch/runcmsgrid_systematics_LO.patch") - err_gpr = gridpack_repack_and_copy(gridpack_eos_path,pi) + err_gpr = gridpack_repack_and_copy(gridpack_eos_path,my_path,pi) errors.extend(err_gpr) - if mg5_aMC_version >= 260: - mg_lo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c madevent').read()) - mg_nlo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c aMCatNLO').read()) if mg5_aMC_version < 260: mg_lo = int(os.popen('grep -c syscalc '+str(runcmsgrid_file)).read()) if mg_nlo > 0: From 2da62644f91b4e4e9be4a0ed7221da29c8c483b5 Mon Sep 17 00:00:00 2001 From: Efe Date: Mon, 28 Oct 2024 16:41:56 +0100 Subject: [PATCH 06/11] run3 runcmsgrid patch --- bin/utils/request_fragment_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 73c80d77085..7bf0d0001e2 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -354,6 +354,7 @@ def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): os.chdir(my_path+'/'+pi) os.environ['XZ_OPT'] = "--lzma2=preset=9,dict=512MiB" os.system('XZ_OPT="$XZ_OPT" tar -cJpf '+gp_name+' --exclude='+gp_name+' --exclude='+pi+' ./*') + print('cp '+gp_name+' '+gridpack_eos_path) os.system('cp '+gp_name+' '+gridpack_eos_path) md5_1 = os.popen('md5sum '+gp_name).read().split(' ')[0] md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] From 5e9fdf340dced20eb49607fa92381a2299f8d8eb Mon Sep 17 00:00:00 2001 From: Efe Date: Mon, 28 Oct 2024 16:59:32 +0100 Subject: [PATCH 07/11] run3 runcmsgrid patch --- bin/utils/request_fragment_check.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 7bf0d0001e2..813b97ef1aa 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -354,10 +354,12 @@ def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): os.chdir(my_path+'/'+pi) os.environ['XZ_OPT'] = "--lzma2=preset=9,dict=512MiB" os.system('XZ_OPT="$XZ_OPT" tar -cJpf '+gp_name+' --exclude='+gp_name+' --exclude='+pi+' ./*') - print('cp '+gp_name+' '+gridpack_eos_path) - os.system('cp '+gp_name+' '+gridpack_eos_path) + print('cp -p '+gp_name+' '+gridpack_eos_path) + os.system('cp -p '+gp_name+' '+gridpack_eos_path) md5_1 = os.popen('md5sum '+gp_name).read().split(' ')[0] md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] + print(gp_name) + print(gridpack_eos_path) print("Checksums = ",md5_1,md5_2) if md5_1 == md5_2: print("Updated gridpack copied succesfully.") From 2dab35123acf205cb4078f796f4932187e44f2d4 Mon Sep 17 00:00:00 2001 From: Efe Date: Mon, 28 Oct 2024 17:38:46 +0100 Subject: [PATCH 08/11] debug run3 patch --- bin/utils/request_fragment_check.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 813b97ef1aa..3c887bf0b23 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -352,6 +352,7 @@ def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): print("re-tarring to "+gp_name) cur_dir = os.getcwd() os.chdir(my_path+'/'+pi) + print(os.getcwd()) os.environ['XZ_OPT'] = "--lzma2=preset=9,dict=512MiB" os.system('XZ_OPT="$XZ_OPT" tar -cJpf '+gp_name+' --exclude='+gp_name+' --exclude='+pi+' ./*') print('cp -p '+gp_name+' '+gridpack_eos_path) @@ -360,12 +361,14 @@ def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] print(gp_name) print(gridpack_eos_path) - print("Checksums = ",md5_1,md5_2) + print("Checksums = [local=",md5_1,"copied=",md5_2,"]") + print(type(md5_1),type(md5_2)) if md5_1 == md5_2: print("Updated gridpack copied succesfully.") else: error_gridpack_repack.append("There was a problem copying in the updated gridpack to eos.") os.chdir(cur_dir) + print(os.getcwd()) return error_gridpack_repack def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): @@ -1542,7 +1545,7 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): mg_nlo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c aMCatNLO').read()) if mg_lo: print("LO gridpack") if mg_nlo: print("NLO gridpack") - if "Run3" in pi: + if "Run3" or "RunII" in pi: if int(os.popen('grep -c "systematics $runlabel" '+str(runcmsgrid_file)).read()): if int(os.popen('grep -c "Encounter Error in Running Systematics Module" '+str(runcmsgrid_file)).read()) < 1: print("-----------------------------------------") From 173cfdceff1e6b757f4fda188876b405a6c32e35 Mon Sep 17 00:00:00 2001 From: Efe Date: Mon, 28 Oct 2024 21:41:28 +0100 Subject: [PATCH 09/11] test runcmsgrid patch --- bin/utils/request_fragment_check.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 3c887bf0b23..fa47a4dcb3e 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -349,20 +349,18 @@ def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): if ".tgz" in gridpack_eos_path: gp_extension = ".tgz" gp_name = "gridpack"+gp_extension + if os.path.isfile(gp_name): os.system('rm '+gp_name) print("re-tarring to "+gp_name) cur_dir = os.getcwd() os.chdir(my_path+'/'+pi) print(os.getcwd()) os.environ['XZ_OPT'] = "--lzma2=preset=9,dict=512MiB" os.system('XZ_OPT="$XZ_OPT" tar -cJpf '+gp_name+' --exclude='+gp_name+' --exclude='+pi+' ./*') - print('cp -p '+gp_name+' '+gridpack_eos_path) - os.system('cp -p '+gp_name+' '+gridpack_eos_path) + print('cp '+gp_name+' '+gridpack_eos_path) + os.system('cp '+gp_name+' '+gridpack_eos_path) md5_1 = os.popen('md5sum '+gp_name).read().split(' ')[0] md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] - print(gp_name) - print(gridpack_eos_path) print("Checksums = [local=",md5_1,"copied=",md5_2,"]") - print(type(md5_1),type(md5_2)) if md5_1 == md5_2: print("Updated gridpack copied succesfully.") else: From 881f55af75f9b10311862d167bbc0b614d11d8f5 Mon Sep 17 00:00:00 2001 From: Efe Date: Tue, 29 Oct 2024 14:21:13 +0100 Subject: [PATCH 10/11] exception for user runner --- bin/utils/request_fragment_check.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index fa47a4dcb3e..e7a5016cc96 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -131,6 +131,9 @@ def find_file(dir_path,patt): if file.endswith(patt): return root+'/'+str(file) +c_user = os.popen('echo $USER').read() +print("User=",c_user) + def check_replace(runcmsgridfile): error_check_replace = [] replace_mccont = os.popen('grep "_REPLACE" '+str(runcmsgridfile)).read() @@ -339,6 +342,7 @@ def gridpack_copy(gridpack_eos_path,pi): else: error_gp_copy.append("backup gridpack has a problem.") print("Backup gridpack: "+gridpack_eos_path_backup) + if "runner" in c_user: error_gp_copy=[] return error_gp_copy def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): @@ -360,13 +364,13 @@ def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): os.system('cp '+gp_name+' '+gridpack_eos_path) md5_1 = os.popen('md5sum '+gp_name).read().split(' ')[0] md5_2 = os.popen('md5sum'+' '+gridpack_eos_path).read().split(' ')[0] - print("Checksums = [local=",md5_1,"copied=",md5_2,"]") if md5_1 == md5_2: print("Updated gridpack copied succesfully.") else: error_gridpack_repack.append("There was a problem copying in the updated gridpack to eos.") os.chdir(cur_dir) print(os.getcwd()) + if "runner" in c_user: error_gridpack_repack = [] return error_gridpack_repack def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): @@ -391,6 +395,7 @@ def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): f.write(cont) f.truncate() error_xml = gridpack_repack_and_copy(gridpack_eos_path,my_path,pi) + if "runner" in c_user: error_xml = [] return warning_xml,error_xml def evtgen_check(fragment): @@ -1543,7 +1548,7 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): mg_nlo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c aMCatNLO').read()) if mg_lo: print("LO gridpack") if mg_nlo: print("NLO gridpack") - if "Run3" or "RunII" in pi: + if "Run3" in pi or "RunII" in pi: if int(os.popen('grep -c "systematics $runlabel" '+str(runcmsgrid_file)).read()): if int(os.popen('grep -c "Encounter Error in Running Systematics Module" '+str(runcmsgrid_file)).read()) < 1: print("-----------------------------------------") From 7d6b80616c830340d8f40d4861d56d39528abb27 Mon Sep 17 00:00:00 2001 From: Efe Date: Tue, 29 Oct 2024 15:31:02 +0100 Subject: [PATCH 11/11] bypass runcmsgrid patch for CI and update requests in CI --- .github/workflows/main.yml | 15 +- bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883 | 33 ++++ ...P-RunIISummer20UL16wmLHEGEN-00883_get_test | 155 ++++++++++++++++++ ...t_TOP-RunIISummer20UL16wmLHEGEN-00883.json | 1 + bin/utils/request_fragment_check.py | 21 +-- 5 files changed, 204 insertions(+), 21 deletions(-) create mode 100644 bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883 create mode 100644 bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883_get_test create mode 100644 bin/utils/request_TOP-RunIISummer20UL16wmLHEGEN-00883.json diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f0fdae9bef0..1ea6093d3c2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -120,11 +120,10 @@ jobs: #shell: python# if: steps.filter.outputs.bin == 'true' || steps.filter.outputs.workflow == 'true' run: | - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid EXO-Run3Summer22wmLHEGS-01444 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid B2G-RunIISummer20UL18wmLHEGEN-03308 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SMP-RunIISummer20UL16GEN-00050 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00403 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TRK-Run3Summer23wmLHEGS-00002 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TSG-Run3Summer23BPixGS-00007 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TSG-Run3Summer23BPixGS-00045 \ No newline at end of file + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid EXO-Run3Summer22wmLHEGS-01444 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid B2G-RunIISummer20UL18wmLHEGEN-03308 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid GEN-Run3Summer22EEwmLHEGS-00403 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TRK-Run3Summer23wmLHEGS-00002 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TSG-Run3Summer23BPixGS-00007 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TOP-RunIISummer20UL16wmLHEGEN-00883 \ No newline at end of file diff --git a/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883 b/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883 new file mode 100644 index 00000000000..e71227af490 --- /dev/null +++ b/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883 @@ -0,0 +1,33 @@ +import FWCore.ParameterSet.Config as cms + +externalLHEProducer = cms.EDProducer("ExternalLHEProducer", + args = cms.vstring('/cvmfs/cms.cern.ch/phys_generator/gridpacks/slc7_amd64_gcc700/13TeV/madgraph/V5_2.6.5/ttbar_LFV_SMEFT_splitFlavour/SMEFTsim_TT_vector_emutu_slc7_amd64_gcc700_CMSSW_10_6_19_tarball.tar.xz'), + nEvents = cms.untracked.uint32(5000), + numberOfParameters = cms.uint32(1), + outputFile = cms.string('cmsgrid_final.lhe'), + generateConcurrently = cms.untracked.bool(False), + scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh') +) + + +from Configuration.Generator.Pythia8CommonSettings_cfi import * +from Configuration.Generator.MCTunes2017.PythiaCP5Settings_cfi import * +from Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import * + +generator = cms.EDFilter("Pythia8ConcurrentHadronizerFilter", + maxEventsToPrint = cms.untracked.int32(1), + pythiaPylistVerbosity = cms.untracked.int32(1), + filterEfficiency = cms.untracked.double(1.0), + pythiaHepMCVerbosity = cms.untracked.bool(False), + comEnergy = cms.double(13000.), + PythiaParameters = cms.PSet( + pythia8CommonSettingsBlock, + pythia8CP5SettingsBlock, + pythia8PSweightsSettingsBlock, + parameterSets = cms.vstring('pythia8CommonSettings', + 'pythia8CP5Settings', + 'pythia8PSweightsSettings', + ) + ) +) +ProductionFilterSequence = cms.Sequence(generator) \ No newline at end of file diff --git a/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883_get_test b/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883_get_test new file mode 100644 index 00000000000..5bf31bf7428 --- /dev/null +++ b/bin/utils/TOP-RunIISummer20UL16wmLHEGEN-00883_get_test @@ -0,0 +1,155 @@ +#!/bin/bash + +# Binds for singularity containers +# Mount /afs, /eos, /cvmfs, /etc/grid-security for xrootd +export APPTAINER_BINDPATH='/afs,/cvmfs,/cvmfs/grid.cern.ch/etc/grid-security:/etc/grid-security,/eos,/etc/pki/ca-trust,/run/user,/var/run/user' + +cat <<'EndOfGenScriptFile' > TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh +#!/bin/bash + +echo "Running CMS GEN request script using cms-sw containers. Architecture: el9:x86_64" +python3.9 -m venv cms_gen_venv_TOP-RunIISummer20UL16wmLHEGEN-00883 && source ./cms_gen_venv_TOP-RunIISummer20UL16wmLHEGEN-00883/bin/activate + +# Install the PdmV REST client +pip install git+https://github.com/cms-PdmV/mcm_scripts &> /dev/null + +echo "Packages installed" +pip freeze +echo "" + +# GEN Script begin +rm -f request_fragment_check.py +wget -q https://raw.githubusercontent.com/cms-sw/genproductions/master/bin/utils/request_fragment_check.py +chmod +x request_fragment_check.py + +./request_fragment_check.py --bypass_status --prepid TOP-RunIISummer20UL16wmLHEGEN-00883 + +# End of CMS GEN script file: TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh +EndOfGenScriptFile +chmod +x TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh + +# Run in singularity container +singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el9:x86_64 $(echo $(pwd)/TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh) + +GEN_ERR=$? +if [ $GEN_ERR -ne 0 ]; then + echo "GEN Checking Script returned exit code $GEN_ERR which means there are $GEN_ERR errors" + echo "Validation WILL NOT RUN" + echo "Please correct errors in the request and run validation again" + exit $GEN_ERR +fi +echo "Running VALIDATION. GEN Request Checking Script returned no errors" +# GEN Script end + +# Download fragment from McM +curl -s -k https://cms-pdmv-prod.web.cern.ch/mcm/public/restapi/requests/get_fragment/TOP-RunIISummer20UL16wmLHEGEN-00883 --retry 3 --create-dirs -o Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py +[ -s Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py ] || exit $?; + +# Check if fragment contais gridpack path ant that it is in cvmfs +if grep -q "gridpacks" Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py; then + if ! grep -q "/cvmfs/cms.cern.ch/phys_generator/gridpacks" Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py; then + echo "Gridpack inside fragment is not in cvmfs." + exit -1 + fi +fi + +# Dump actual test code to a TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh file that can be run in Singularity +cat <<'EndOfTestFile' > TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh +#!/bin/bash + +export SCRAM_ARCH=slc7_amd64_gcc700 + +source /cvmfs/cms.cern.ch/cmsset_default.sh +if [ -r CMSSW_10_6_43/src ] ; then + echo release CMSSW_10_6_43 already exists +else + scram p CMSSW CMSSW_10_6_43 +fi +cd CMSSW_10_6_43/src +eval `scram runtime -sh` + +mv ../../Configuration . +scram b +cd ../.. + +# Maximum validation duration: 57600s +# Margin for validation duration: 30% +# Validation duration with margin: 57600 * (1 - 0.30) = 40320s +# Time per event for each sequence: 2.0000s +# Threads for each sequence: 1 +# Time per event for single thread for each sequence: 1 * 2.0000s = 2.0000s +# Which adds up to 2.0000s per event +# Single core events that fit in validation duration: 40320s / 2.0000s = 20160 +# Produced events limit in McM is 10000 +# According to 1.0000 efficiency, validation should run 10000 / 1.0000 = 10000 events to reach the limit of 10000 +# Take the minimum of 20160 and 10000, but more than 0 -> 10000 +# It is estimated that this validation will produce: 10000 * 1.0000 = 10000 events +EVENTS=10000 + + +# cmsDriver command +cmsDriver.py Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py --python_filename TOP-RunIISummer20UL16wmLHEGEN-00883_1_cfg.py --eventcontent RAWSIM,LHE --customise Configuration/DataProcessing/Utils.addMonitoring --datatier GEN,LHE --fileout file:TOP-RunIISummer20UL16wmLHEGEN-00883.root --conditions 106X_mcRun2_asymptotic_v13 --beamspot Realistic25ns13TeV2016Collision --customise_commands process.source.numberEventsInLuminosityBlock="cms.untracked.uint32(100)" --step LHE,GEN --geometry DB:Extended --era Run2_2016 --no_exec --mc -n $EVENTS || exit $? ; + +# Run generated config +REPORT_NAME=TOP-RunIISummer20UL16wmLHEGEN-00883_report.xml +# Run the cmsRun +cmsRun -e -j $REPORT_NAME TOP-RunIISummer20UL16wmLHEGEN-00883_1_cfg.py || exit $? ; + +# Parse values from TOP-RunIISummer20UL16wmLHEGEN-00883_report.xml report +processedEvents=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +producedEvents=$(grep -Po "(?<=)(\d*)(?=)" $REPORT_NAME | tail -n 1) +threads=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueRss=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueVsize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSizeAlt=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobCPU=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +eventThroughput=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +avgEventTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +if [ -z "$threads" ]; then + echo "Could not find NumberOfThreads in report, defaulting to 1" + threads=1 +fi +if [ -z "$eventThroughput" ]; then + eventThroughput=$(bc -l <<< "scale=4; 1 / ($avgEventTime / $threads)") +fi +if [ -z "$totalSize" ]; then + totalSize=$totalSizeAlt +fi +if [ -z "$processedEvents" ]; then + processedEvents=$EVENTS +fi +echo "Validation report of TOP-RunIISummer20UL16wmLHEGEN-00883 sequence 1/1" +echo "Processed events: $processedEvents" +echo "Produced events: $producedEvents" +echo "Threads: $threads" +echo "Peak value RSS: $peakValueRss MB" +echo "Peak value Vsize: $peakValueVsize MB" +echo "Total size: $totalSize MB" +echo "Total job time: $totalJobTime s" +echo "Total CPU time: $totalJobCPU s" +echo "Event throughput: $eventThroughput" +echo "CPU efficiency: "$(bc -l <<< "scale=2; ($totalJobCPU * 100) / ($threads * $totalJobTime)")" %" +echo "Size per event: "$(bc -l <<< "scale=4; ($totalSize * 1024 / $producedEvents)")" kB" +echo "Time per event: "$(bc -l <<< "scale=4; (1 / $eventThroughput)")" s" +echo "Filter efficiency percent: "$(bc -l <<< "scale=8; ($producedEvents * 100) / $processedEvents")" %" +echo "Filter efficiency fraction: "$(bc -l <<< "scale=10; ($producedEvents) / $processedEvents") + +# End of TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh file +EndOfTestFile + +# Make file executable +chmod +x TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh + +if [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:amd64" ]; then + CONTAINER_NAME="el7:amd64" +elif [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:x86_64" ]; then + CONTAINER_NAME="el7:x86_64" +else + echo "Could not find amd64 or x86_64 for el7" + exit 1 +fi +# Run in singularity container +export SINGULARITY_CACHEDIR="/tmp/$(whoami)/singularity" +singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/$CONTAINER_NAME $(echo $(pwd)/TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh) diff --git a/bin/utils/request_TOP-RunIISummer20UL16wmLHEGEN-00883.json b/bin/utils/request_TOP-RunIISummer20UL16wmLHEGEN-00883.json new file mode 100644 index 00000000000..0251790b972 --- /dev/null +++ b/bin/utils/request_TOP-RunIISummer20UL16wmLHEGEN-00883.json @@ -0,0 +1 @@ +{"results": {"validation": {"time_multiplier": 2, "results": {"1": [{"peak_value_rss": 665.695, "total_events": 10000, "estimated_events_per_lumi": 14218.5888, "cpu_efficiency": 0.9936923152855239, "filter_efficiency": 1.0, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 2.025517469075412, "expected_events": 10000, "size_per_event": 124.242944}], "8": [{"peak_value_rss": 666.508, "total_events": 10000, "estimated_events_per_lumi": 43582.17599999999, "cpu_efficiency": 0.9094780097297934, "filter_efficiency": 1.0, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 0.6608206070298097, "expected_events": 10000, "size_per_event": 123.57632}], "2": [{"peak_value_rss": 621.676, "total_events": 10000, "estimated_events_per_lumi": 26357.4432, "cpu_efficiency": 0.9893730851890804, "filter_efficiency": 1.0, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 1.0926704757159449, "expected_events": 10000, "size_per_event": 123.794432}], "4": [{"peak_value_rss": 696.875, "total_events": 10000, "estimated_events_per_lumi": 39914.208, "cpu_efficiency": 0.9495847701527331, "filter_efficiency": 1.0, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 0.7215475752393734, "expected_events": 10000, "size_per_event": 124.77951999999999}]}}, "total_events": 1000000, "config_id": [], "events_per_lumi": 0, "_rev": "9-5e70c27cdedf3c2bdfbdcc6f0c4318ac", "mcdb_id": 0, "transient_output_modules": [[]], "sequences": [{"eventcontent": ["RAWSIM", "LHE"], "runsScenarioForMC": "", "donotDropOnInput": "", "extra": "", "slhc": "", "runsAndWeightsForMC": "", "outputCommand": "", "dropDescendant": false, "pileup": "", "hltProcess": "", "index": -1, "customise": "", "custom_conditions": "", "datatier": ["GEN", "LHE"], "processName": "", "inputCommands": "", "particle_table": "", "conditions": "106X_mcRun2_asymptotic_v13", "harvesting": "", "triggerResultsProcess": "", "inputEventContent": "", "beamspot": "Realistic25ns13TeV2016Collision", "restoreRNDSeeds": "", "filtername": "", "repacked": "", "customise_commands": "", "step": ["LHE", "GEN"], "gflash": "", "procModifiers": "", "magField": "", "himix": false, "nThreads": "1", "scenario": "", "geometry": "DB:Extended", "nStreams": 0, "datamix": "", "inline_custom": false, "era": "Run2_2016"}], "block_black_list": [], "block_white_list": [], "priority": 20000, "process_string": "", "ppd_tags": [], "fragment_tag": "", "generator_parameters": [{"submission_details": {"submission_date": "2024-07-03-16-11", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}, "match_efficiency_error": 0, "match_efficiency": 1, "filter_efficiency": 1, "negative_weights_fraction": 0, "version": 0, "cross_section": 0.0436, "filter_efficiency_error": 1}, {"submission_details": {"submission_date": "2024-10-23-10-06", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}, "match_efficiency_error": 0.0, "match_efficiency": 1.0, "filter_efficiency": 1.0, "negative_weights_fraction": 0.0, "version": 1, "cross_section": 0.0436, "filter_efficiency_error": 1.0}], "flown_with": "", "interested_pwg": ["TOP"], "version": 0, "generators": ["Pythia8", "Madgraph V5 2.6.5"], "memory": 1900, "pilot": false, "type": "LHE", "cmssw_release": "CMSSW_10_6_43", "status": "defined", "keep_output": [false], "energy": 13.0, "tags": [], "fragment": "import FWCore.ParameterSet.Config as cms\n\nexternalLHEProducer = cms.EDProducer(\"ExternalLHEProducer\",\n args = cms.vstring('/cvmfs/cms.cern.ch/phys_generator/gridpacks/slc7_amd64_gcc700/13TeV/madgraph/V5_2.6.5/ttbar_LFV_SMEFT_splitFlavour/SMEFTsim_TT_vector_emutu_slc7_amd64_gcc700_CMSSW_10_6_19_tarball.tar.xz'),\n nEvents = cms.untracked.uint32(5000),\n numberOfParameters = cms.uint32(1),\n outputFile = cms.string('cmsgrid_final.lhe'),\n generateConcurrently = cms.untracked.bool(False),\n scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh')\n)\n\n# Link to datacards:\n# https://github.com/cms-sw/genproductions/tree/mg265UL/bin/MadGraph5_aMCatNLO/cards/production/13TeV/LFV_TopProduction_TopDecay/SMEFTsim/SMEFTsim_TopDecay\n\nfrom Configuration.Generator.Pythia8CommonSettings_cfi import *\nfrom Configuration.Generator.MCTunes2017.PythiaCP5Settings_cfi import *\nfrom Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import *\n\ngenerator = cms.EDFilter(\"Pythia8ConcurrentHadronizerFilter\",\n maxEventsToPrint = cms.untracked.int32(1),\n pythiaPylistVerbosity = cms.untracked.int32(1),\n filterEfficiency = cms.untracked.double(1.0),\n pythiaHepMCVerbosity = cms.untracked.bool(False),\n comEnergy = cms.double(13000.),\n PythiaParameters = cms.PSet(\n pythia8CommonSettingsBlock,\n pythia8CP5SettingsBlock,\n pythia8PSweightsSettingsBlock,\n parameterSets = cms.vstring('pythia8CommonSettings',\n 'pythia8CP5Settings',\n 'pythia8PSweightsSettings',\n )\n )\n)\nProductionFilterSequence = cms.Sequence(generator)", "time_event": [2], "pwg": "TOP", "reqmgr_name": [], "approval": "define", "name_of_fragment": "", "pileup_dataset_name": "", "analysis_id": [], "input_dataset": "", "member_of_chain": [], "prepid": "TOP-RunIISummer20UL16wmLHEGEN-00883", "extension": 0, "size_event": [130], "notes": "Overall cross-section summary\n--------------------------------------------------------------------------------------------------------------------------------------------------------------------------\nProcess xsec_before [pb] passed nposw nnegw tried nposw nnegw xsec_match [pb] accepted [%] event_eff [%]\n0 7.297e-03 +/- 2.998e-05 1673 1673 0 1673 1673 0 7.297e-03 +/- 2.998e-05 100.0 +/- 0.0 100.0 +/- 0.0\n1 7.287e-03 +/- 3.066e-05 1605 1605 0 1605 1605 0 7.287e-03 +/- 3.066e-05 100.0 +/- 0.0 100.0 +/- 0.0\n2 7.248e-03 +/- 3.051e-05 1737 1737 0 1737 1737 0 7.248e-03 +/- 3.051e-05 100.0 +/- 0.0 100.0 +/- 0.0\n3 7.274e-03 +/- 2.999e-05 1696 1696 0 1696 1696 0 7.274e-03 +/- 2.999e-05 100.0 +/- 0.0 100.0 +/- 0.0\n4 7.229e-03 +/- 3.153e-05 1643 1643 0 1643 1643 0 7.229e-03 +/- 3.153e-05 100.0 +/- 0.0 100.0 +/- 0.0\n5 7.268e-03 +/- 2.912e-05 1646 1646 0 1646 1646 0 7.268e-03 +/- 2.912e-05 100.0 +/- 0.0 100.0 +/- 0.0\n--------------------------------------------------------------------------------------------------------------------------------------------------------------------------\nTotal 4.360e-02 +/- 7.424e-05 10000 10000 0 10000 10000 0 4.360e-02 +/- 7.424e-05 100.0 +/- 0.0 100.0 +/- 0.0\n-", "completed_events": 0, "history": [{"action": "created", "updater": {"submission_date": "2024-10-23-10-06", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "clone", "step": "TOP-RunIISummer20UL16wmLHEGEN-00869", "updater": {"submission_date": "2024-10-23-10-06", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "approve", "step": "validation", "updater": {"submission_date": "2024-10-23-16-42", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "validation", "step": "succeeded", "updater": {"submission_date": "2024-10-24-04-00", "author_email": "", "author_username": "automatic", "author_name": ""}}, {"action": "approve", "step": "none", "updater": {"submission_date": "2024-10-24-08-12", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "set status", "step": "new", "updater": {"submission_date": "2024-10-24-08-12", "author_email": "carlos.vico.villalba@cern.ch", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "update", "updater": {"author_email": "carlos.vico.villalba@cern.ch", "submission_date": "2024-10-24-08-39", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "approve", "step": "validation", "updater": {"author_email": "carlos.vico.villalba@cern.ch", "submission_date": "2024-10-24-08-46", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "validation", "step": "succeeded", "updater": {"author_email": "", "submission_date": "2024-10-24-20-21", "author_username": "automatic", "author_name": ""}}, {"action": "approve", "step": "define", "updater": {"author_email": "carlos.vico.villalba@cern.ch", "submission_date": "2024-10-27-16-34", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}, {"action": "set status", "step": "defined", "updater": {"author_email": "carlos.vico.villalba@cern.ch", "submission_date": "2024-10-27-16-34", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}}], "output_dataset": [], "member_of_campaign": "RunIISummer20UL16wmLHEGEN", "_id": "TOP-RunIISummer20UL16wmLHEGEN-00883", "dataset_name": "TT_vector-emutu-LFV_TuneCP5_13TeV_madgraph-pythia8", "cadi_line": ""}} \ No newline at end of file diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index e7a5016cc96..ecb13d8365f 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -32,6 +32,8 @@ parser.add_argument('--develop', help="Option to make modifications of the script", action='store_true') parser.add_argument('--local', help="Option to read fragment locally", action='store_true') parser.add_argument('--download_json', help="Download request json to read fragment locally in a next step", action='store_true') +parser.add_argument('--bypass_runcmsgrid_patch', help="apply the runcmsgrid patch if necessary", action='store_true') + args = parser.parse_args() @@ -131,9 +133,6 @@ def find_file(dir_path,patt): if file.endswith(patt): return root+'/'+str(file) -c_user = os.popen('echo $USER').read() -print("User=",c_user) - def check_replace(runcmsgridfile): error_check_replace = [] replace_mccont = os.popen('grep "_REPLACE" '+str(runcmsgridfile)).read() @@ -342,7 +341,6 @@ def gridpack_copy(gridpack_eos_path,pi): else: error_gp_copy.append("backup gridpack has a problem.") print("Backup gridpack: "+gridpack_eos_path_backup) - if "runner" in c_user: error_gp_copy=[] return error_gp_copy def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): @@ -369,8 +367,6 @@ def gridpack_repack_and_copy(gridpack_eos_path,my_path,pi): else: error_gridpack_repack.append("There was a problem copying in the updated gridpack to eos.") os.chdir(cur_dir) - print(os.getcwd()) - if "runner" in c_user: error_gridpack_repack = [] return error_gridpack_repack def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): @@ -395,7 +391,6 @@ def xml_check_and_patch(f,cont,gridpack_eos_path,my_path,pi): f.write(cont) f.truncate() error_xml = gridpack_repack_and_copy(gridpack_eos_path,my_path,pi) - if "runner" in c_user: error_xml = [] return warning_xml,error_xml def evtgen_check(fragment): @@ -1334,10 +1329,10 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): errors.extend(check_replace(runcmsgrid_file)) match = re.search(r"""process=(["']?)([^"']*)\1""", content) print(match.group(0)) - warning1,error1 = xml_check_and_patch(f,content,gridpack_eos_path,my_path,pi) - warnings.extend(warning1) - errors.extend(error1) - + if args.bypass_runcmsgrid_patch is False: + warning1,error1 = xml_check_and_patch(f,content,gridpack_eos_path,my_path,pi) + warnings.extend(warning1) + errors.extend(error1) f.close() else: errors.append(my_path+'/'+pi+'/'+'runcmsgrid.sh does not exists') @@ -1548,7 +1543,7 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): mg_nlo = int(os.popen('grep "systematics" '+str(runcmsgrid_file)+' | grep -c aMCatNLO').read()) if mg_lo: print("LO gridpack") if mg_nlo: print("NLO gridpack") - if "Run3" in pi or "RunII" in pi: + if ("Run3" in pi or "RunII" in pi) and args.bypass_runcmsgrid_patch is False: if int(os.popen('grep -c "systematics $runlabel" '+str(runcmsgrid_file)).read()): if int(os.popen('grep -c "Encounter Error in Running Systematics Module" '+str(runcmsgrid_file)).read()) < 1: print("-----------------------------------------") @@ -1671,7 +1666,7 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): if MGpatch2[0] == 0 and MGpatch2[1] == 1: print("[OK] MG5_aMC@NLO LO nthreads patch not made in CVMFS but done in EOS waiting for CVMFS-EOS synch") if MGpatch2[1] == 0 and args.local is False: errors.append("MG5_aMC@NLO LO nthreads patch not made in EOS") - if args.apply_many_threads_patch: + if args.apply_many_threads_patch and args.bypass_runcmsgrid_patch is False: print("Patching for nthreads problem... please be patient.") if slha_flag == 0: os.system('python2 ../../Utilities/scripts/update_gridpacks_mg242_thread.py --prepid '+pi)