-
Notifications
You must be signed in to change notification settings - Fork 19
/
create_job.py
77 lines (60 loc) · 1.87 KB
/
create_job.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
#!/usr/bin/env python
import os
import sys
jdl = """\
universe = vanilla
executable = ./{PROCESS}.sh
output = out/$(ProcId).$(ClusterID).out
error = err/$(ProcId).$(ClusterID).err
log = log/$(ProcId).$(ClusterID).log
requirements = (OpSysAndVer =?= "SLCern6")
max_retries = 3
RequestCpus = 1
+MaxRuntime = 1800
queue arguments from arguments.txt\
"""
def mkdir(path):
if not os.path.exists(path):
os.mkdir(path)
def parse_arguments():
if not len(sys.argv) == 3:
raise Exception("./create_job.py PROCESS PATH_TO_JOBDIR")
return {"process": sys.argv[1], "jobdir": sys.argv[2]}
def main(args):
process = args["process"]
print("Process: %s" % process)
# Build argument list
print("Filelist:")
arguments = []
counter = 0
for filename in os.listdir("data/"):
if process in filename:
print(" %s." % filename)
for line in open("data/" + filename, "r").readlines():
arguments.append("%u %s %s" % (counter, process, line))
counter += 1
print("Number of jobs: %u" % len(arguments))
# Create jobdir and subdirectories
jobdir = os.path.join(args["jobdir"], process)
print("Jobdir: %s" % jobdir)
mkdir(jobdir)
mkdir(os.path.join(jobdir, "out"))
mkdir(os.path.join(jobdir, "log"))
mkdir(os.path.join(jobdir, "err"))
# Write jdl file
out = open(os.path.join(jobdir, "job.jdl"), "w")
out.write(jdl.format(PROCESS=process))
out.close()
# Write argument list
arglist = open(os.path.join(jobdir, "arguments.txt"), "w")
for a in arguments:
arglist.write(a)
arglist.close()
# Write job file
jobfile = open("job.sh", "r").read()
job = open(os.path.join(jobdir, "{PROCESS}.sh".format(PROCESS=process)), "w")
job.write(jobfile)
job.close()
if __name__ == "__main__":
args = parse_arguments()
main(args)