-
Notifications
You must be signed in to change notification settings - Fork 0
/
parseall.py
executable file
·153 lines (137 loc) · 6.54 KB
/
parseall.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
import sys
if sys.version_info < (3, 5):
raise "must use python 3.5 or greater"
import os
import os.path
from os import path
import subprocess
import numpy as np
def parse(command):
return subprocess.run([command, '-l'], stdout=subprocess.PIPE, shell=True).stdout.decode('utf-8')
def getdebugfile(casename, numthreads, numprocessors):
return "sims/"+casename+"_processors"+str(numprocessors)+"_threads"+str(numthreads)+'/'+casename+'.DBG'
def get_totalTime(casename, numthreads, maxproc):
res=np.empty(maxproc)
res[:] = np.nan
for numprocessors in np.arange(1,maxproc+1):
dbgf=getdebugfile(casename,numthreads, numprocessors)
if path.exists(dbgf):
command='tail -n30 '+dbgf+'|grep "Total time (seconds)"|cut -d ":" -f 2'
tmp=parse(command).strip()
if tmp:
res[numprocessors-1] = float(tmp)
return res
def get_numNewtonIterations(casename, numthreads, maxproc):
res=np.empty(maxproc)
res[:] = np.nan
for numprocessors in np.arange(1,maxproc+1):
dbgf=getdebugfile(casename,numthreads, numprocessors)
if path.exists(dbgf):
command='tail -n30 '+dbgf+'|grep "Newton Iterations"|cut -d ":" -f 2|cut -d " " -f 5'
tmp=parse(command).strip()
if tmp:
res[numprocessors-1] = int(tmp)
return res
def get_numLinearIterations(casename, numthreads, maxproc):
res=np.empty(maxproc)
res[:] = np.nan
for numprocessors in np.arange(1,maxproc+1):
dbgf=getdebugfile(casename,numthreads, numprocessors)
if path.exists(dbgf):
command='tail -n30 '+dbgf+'|grep "Linear Iterations"|cut -d ":" -f 2|cut -d " " -f 5'
tmp=parse(command).strip()
if tmp:
res[numprocessors-1] = int(tmp)
return res
def get_AssemplyTime(casename, numthreads, maxproc):
res=np.empty(maxproc)
res[:] = np.nan
for numprocessors in np.arange(1,maxproc+1):
dbgf=getdebugfile(casename,numthreads, numprocessors)
if path.exists(dbgf):
command='tail -n30 '+dbgf+'|grep "Assembly time (seconds)"|cut -d ":" -f 2|cut -d " " -f 6'
tmp=parse(command).strip()
if tmp:
res[numprocessors-1] = float(tmp)
return res
def get_linearSolveTime(casename, numthreads, maxproc):
res=np.empty(maxproc)
res[:] = np.nan
for numprocessors in np.arange(1,maxproc+1):
dbgf=getdebugfile(casename,numthreads, numprocessors)
if path.exists(dbgf):
command='tail -n30 '+dbgf+'|grep "Linear solve time (seconds)"|cut -d ":" -f 2|cut -d " " -f 2'
tmp=parse(command).strip()
if tmp:
res[numprocessors-1] = float(tmp)
return res
def get_numCells(casename, numthreads, maxproc):
owned=np.empty((maxproc,maxproc))
owned[:] = np.nan
overlap=np.empty((maxproc,maxproc))
overlap[:] = np.nan
total=np.empty((maxproc,maxproc))
total[:] = np.nan
for numprocessors in np.arange(2,maxproc+1):
dbgf=getdebugfile(casename,numthreads, numprocessors)
if path.exists(dbgf):
if numprocessors == 2:
command = 'grep -i load '+dbgf+'|cut -d " " -f 4'
tmp = parse(command)
if tmp:
total[0,0] = int(tmp)
command='line=`grep -i load '+dbgf+' -n`; linenumber=`echo $line|cut -d ":" -f 1`; linenumber=`expr $linenumber + 3`; echo $linenumber'
linenumber = str(parse(command)).strip()
if int(linenumber)>3:### if the line is not found, linenumber will be 3
#owned
rest=str(1)
command='data=`tail -n+'+linenumber+' '+dbgf+' | head -n'+str(numprocessors)+'`; count=0;for d in $data; do if [ `expr $count % 4` -eq '+rest+' ]; then echo $d", "; fi; count=`expr $count + 1`; done'
data = str(parse(command))
for i in np.arange(0,numprocessors):
owned[numprocessors-1,i] = float(data.split(', ')[i])
#overlap
rest=str(2)
command='data=`tail -n+'+linenumber+' '+dbgf+' | head -n'+str(numprocessors)+'`; count=0;for d in $data; do if [ `expr $count % 4` -eq '+rest+' ]; then echo $d", "; fi; count=`expr $count + 1`; done'
data = str(parse(command))
for i in np.arange(0,numprocessors):
overlap[numprocessors-1,i] = float(data.split(', ')[i])
#total
rest=str(3)
command='data=`tail -n+'+linenumber+' '+dbgf+' | head -n'+str(numprocessors)+'`; count=0;for d in $data; do if [ `expr $count % 4` -eq '+rest+' ]; then echo $d", "; fi; count=`expr $count + 1`; done'
data = str(parse(command))
for i in np.arange(0,numprocessors):
total[numprocessors-1,i] = float(data.split(', ')[i])
return owned, overlap, total
def writedata(foldername, casename, numthreads, maxproc):
name=foldername+"/"+casename
name+="_threads_"+str(numthreads)
res=get_totalTime(casename, numthreads, maxproc)
np.save(name+"_totaltime",res)
res=get_numNewtonIterations(casename, numthreads, maxproc)
np.save(name+"_newtoniterations",res)
res=get_numLinearIterations(casename, numthreads, maxproc)
np.save(name+"_lineariterations",res)
res=get_AssemplyTime(casename, numthreads, maxproc)
np.save(name+"_assemblytime",res)
res=get_linearSolveTime(casename, numthreads, maxproc)
np.save(name+"_linearsolvetime",res)
owned, overlap, total=get_numCells(casename, numthreads, maxproc)
np.save(name+"_numOwnedCells",owned)
np.save(name+"_numOverlapCells",overlap)
np.save(name+"_numTotalCells",total)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description='Parse and write data')
parser.add_argument('--maxproc', metavar='maxproc', required=True,
type=int, help='maximum number of processors')
parser.add_argument('--maxthreads', metavar='maxthreads', required=True,
type=int, help='maximum number of threads')
args = parser.parse_args()
foldername="data/"+str(parse("uname --nodename")).strip()
foldername+="_"+str(parse("whoami")).strip()
foldername+="_"+str(parse('date +"%T-%d-%m-%Y"')).strip()
if not os.path.exists(foldername):
os.makedirs(foldername)
for numthreads in np.arange(1,args.maxthreads+1):
writedata(foldername, "NORNE_ATW2013",numthreads, args.maxproc)
writedata(foldername, "SPE9",numthreads, args.maxproc)