9 print("Usage : %s datafile links latency bandwidth [size...]" % sys.argv[0])
10 print("where : datafile is a SkaMPI pingpong measurement log file");
11 print(" links is the number of links between nodes")
12 print(" latency is the nominal latency given in the platform file")
13 print(" bandwidth is the nominal bandwidth given in the platform file")
14 print(" size are segments limits")
17 ##-----------------------------------------
18 ## avg : return average of a list of values
19 ## param l list of values
20 ##-----------------------------------------
27 ##-------------------------------------------------
29 ## param X first data vector (..x_i..)
30 ## param Y second data vector (..x_i..)
31 ## = 1/n \Sum_{i=1}^n (x_i - avg(x)) * (y_i - avg(y))
32 ##--------------------------------------------------
34 assert len(X) == len(Y)
35 n = len(X) # n=len(X)=len(Y)
40 S_XY += (X[i] - avg_X) * (Y[i] - avg_Y)
43 ##---------------------------------------------------------------------
44 ## variance : variance
45 ## param X data vector ( ..x_i.. )
46 ## (S_X)^2 = (Sum ( x_i - avg(x) )^2 ) / n
47 ##---------------------------------------------------------------------
53 S_X2 += (X[i] - avg_X) ** 2
56 ##---------------------------------------------------------------------
57 ## calibrate : output correction factors, c_lat on latency, c_bw on bw
58 ## such that bandwidth * c_bw = bw_regr, latency * c_lat = lat_regr
60 ## where bw_obs and lat_obs are the values approximatong experimental
62 ## where bw_regr and lat_regr are the values approximatong experimental
63 >>>>>>> 496787ff18b9d4034acba3c5c68dc54ea416e887
66 ## param links number of links traversed during ping-pong
67 ## param latency as specified on command line, in s
68 ## param bandwidth as specified on command line, in Byte/s
69 ## param sizes vector of data sizes, in Bytes
70 ## param timings vector of time taken: timings[i] for sizes[i], in us
71 ##---------------------------------------------------------------------
72 def calibrate (links, latency, bandwidth, sizes, timings):
73 assert len(sizes) == len(timings)
77 # compute linear regression : find an affine form time = a*size+b
79 # compute linear regression : find an affine form a*size+b
80 >>>>>>> 496787ff18b9d4034acba3c5c68dc54ea416e887
81 S_XY = cov(sizes, timings)
82 S_X2 = variance(sizes)
84 b = avg(timings) - a * avg(sizes)
86 # corresponding bandwith, in byte/s (was in byte/us in skampi dat)
88 # corresponding latency, in s (was in us in skampi dat)
90 print("\nregression: {0} * x + {1}".format(a,b))
91 print("corr_bw = bw_regr/bandwidth= {0}/{1}={2} lat_regr/(lat_xml*links)={3}/({4}*{5}))".format(bw_regr,bandwidth,bw_regr/bandwidth,lat_regr,latency,links))
92 # return linear regression result and corresponding correction factors c_bw,c_lat
93 return a,b, bw_regr/bandwidth, lat_regr/(latency*links)
96 ##---------------------------------------------------------------------
97 ## outputs a C formatted conditional return value for factor
99 ## param lb lower bound
100 ## param ub upper bound
101 ## param lb_included boolean to tell if bound is included (<=) or exclude (<)
102 ## param ub_included boolean to tell if bound is included (<=) or exclude (<)
103 ##---------------------------------------------------------------------
104 def c_code_print (lb,ub, retval, lb_included, ub_included):
105 lb_cmp = ub_cmp = "<"
113 print("\t /* case {0:.1f} KiB {1} size {2} {3:.1f} KiB */".format(lb_kib,lb_cmp,ub_cmp,ub_kib))
114 print("\t if ({0:d} {1} size && size {2} {3:d}) ".format(lb,lb_cmp,ub_cmp,ub))
115 print("\t return({0});" . format(retval))
118 # corresponding bandwith, in s (was in us in skampi dat)
120 # corresponding latency, in s (was in us in skampi dat)
122 #print("\nregression: {0} * x + {1}".format(a,b))
123 #print("corr_bw = bw_regr/bandwidth= {0}/{1}={2} lat_regr/(lat_xml*links)={3}/({4}*{5}))".format(bw_regr,bandwidth,bw_regr/bandwidth,lat_regr,latency,links))
124 # return correction factors c_bw,c_lat
125 return bw_regr/bandwidth, lat_regr/(latency*links)
126 >>>>>>> 496787ff18b9d4034acba3c5c68dc54ea416e887
128 ##-----------------------------------------------------------------------------------------------
130 ##-----------------------------------------------------------------------------------------------
131 links = int(sys.argv[2])
132 latency = float(sys.argv[3])
133 bandwidth = float(sys.argv[4])
134 skampidat = open(sys.argv[1], "r")
136 ## read data from skampi logs.
140 for line in skampidat:
142 if line[0] != '#' and len(l) >= 3: # is it a comment ?
145 #count= 8388608 8388608 144916.1 7.6 32 144916.1 143262.0
146 #("%s %d %d %f %f %d %f %f\n" % (countlbl, count, countn, time, stddev, iter, mini, maxi)
147 readdata.append( (int(l[1]),float(l[3]) / 2) ); # divide by 2 because of ping-pong measured
149 ## These may not be sorted so sort it by message size before processing.
150 sorteddata = sorted( readdata, key=lambda pair: pair[0])
151 sizes,timings= zip(*sorteddata)
154 ## adds message sizes of interest: if values are specified starting from the 6th command line arg
155 ## and these values are found as message sizes in te log file, add it to the limits list.
156 ## Each of these value si considered a potential inflexion point between two segments.
158 ## If no value specified, a single segment is considered from 1st to last message size logged.
160 if len(sys.argv) > 5:
161 for i in range(5, len(sys.argv)):
162 limits += [idx for idx in range(len(sizes)) if sizes[idx] == int(sys.argv[i])]
163 limits.append(len(sizes) - 1)
168 correc = calibrate(links, latency, bandwidth, sizes[low:lim + 1], timings[low:lim + 1])
171 # save interval [lb,ub] correction, regression line direction and origin
172 # and corresponding correction factors for bw and lat resp,
173 (dircoef,origin,factor_bw,factor_lat) = correc
174 factors.append( (sizes[low],sizes[lim], dircoef, origin, factor_bw,factor_lat) )
175 print("Segment [%d:%d] --Bandwidth factor=%g --Latency factor=%g " % (sizes[low], sizes[lim], factor_bw,factor_lat))
178 # now computes joining lines between segments
181 print("\n/**\n *------------------ <copy/paste C code snippet in surf/network.c> ----------------------")
182 print(" *\n * produced by: {0}\n *".format(' '.join(sys.argv)))
183 print(" *---------------------------------------------------------------------------------------\n **/")
185 # print correction factor for bandwidth for each segment
186 print("static double smpi_bandwidth_factor(double size)\n{")
187 for (lb,ub,a,b,factor_bw,factor_lat) in factors:
188 c_code_print(lb,ub,factor_bw,True,True)
190 # save ends and starts of segments
192 joinseg.append( (lb,timings[sizes.index(lb)]) )
194 joinseg.append( (ub,timings[sizes.index(ub)]) )
196 # print correction factor for bandwidth between segments
198 print("\t /* ..:: inter-segment corrections ::.. */\n");
201 (x0,y0) = joinseg[inx]
203 (x1,y1) = joinseg[inx]
205 # line eq. is y = (y1-y0)/(x0-x1) * x + (y0 x1 - y1 x0)/(x1-x0)
206 a = (y1-y0) / (x1-x0)
208 factor_join_bw = bw_join / bandwidth
209 #print("Joining points (%f,%f) -> (%f,%f) : line dir : a=%g\n" % (x0,y0,x1,y1,a))
210 c_code_print(x0,x1,factor_join_bw,False,False)
214 # print correction factor for latency for each segment
215 print("static double smpi_latency_factor(double size)\n{")
216 for (lb,ub,a,b,factor_bw,factor_lat) in factors:
217 c_code_print(lb,ub,factor_lat,True,True)
219 print("\t /* ..:: inter-segment corrections ::.. */\n");
221 (x0,y0) = joinseg.pop()
222 (x1,y1) = joinseg.pop()
223 # line eq. is y = (y0-y1)/(x0-x1) * x + (y0 x1 - y1 x0)/(x1-x0)
224 #print("(%f,%f) -> (%f,%f)\n" % (x0,y0,x1,y1))
225 b = 1e-6 * (y0*x1-y1*x0) / (x1-x0)
226 factor_join_lat = b / (latency*links)
227 c_code_print(x0,x1,factor_join_lat,False,False)
231 # save interval [lb,ub] and correction factors for bw and lat resp.
232 factors.append( (sizes[low],sizes[lim], correc[0], correc[1]) )
233 print("Segment [%d:%d] --Bandwidth factor=%g --Latency factor=%g " % (sizes[low], sizes[lim], correc[0], correc[1]))
236 print("\n/**\n *------------------ <copy/paste C code snippet in surf/network.c> ----------------------")
237 print(" *\n * produced by: {0}\n *".format(' '.join(sys.argv)))
238 print(" *---------------------------------------------------------------------------------------\n **/")
239 print("static double smpi_bandwidth_factor(double size)\n{")
240 for (lb,ub,factor_bw,factor_lat) in factors:
241 print("\t /* case %d Bytes <= size <=%d Bytes */" % (lb,ub))
242 print("\t if (%d <= size && size <= %d) {" % (lb,ub))
243 print("\t return(%g);" % (factor_bw))
247 print("static double smpi_latency_factor(double size)\n{")
248 for (lb,ub,factor_bw,factor_lat) in factors:
249 print("\t /* case %d Bytes <= size <=%d Bytes */" % (lb,ub))
250 print("\t if (%d <= size && size <= %d) {" % (lb,ub))
251 print("\t return(%g);" % (factor_lat))
253 >>>>>>> 496787ff18b9d4034acba3c5c68dc54ea416e887
255 print("/**\n *------------------ </copy/paste C code snippet in surf/network.c> ---------------------\n **/")