3 # Copyright (c) 2010-2011, 2014. The SimGrid Team.
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the license (GNU LGPL) which comes with this package.
14 print("Usage : %s datafile links latency bandwidth [size...]" % sys.argv[0])
15 print("where : datafile is a SkaMPI pingpong measurement log file");
16 print(" links is the number of links between nodes")
17 print(" latency is the nominal latency given in the platform file")
18 print(" bandwidth is the nominal bandwidth given in the platform file")
19 print(" size are segments limits")
22 ##-----------------------------------------
23 ## avg : return average of a list of values
24 ## param l list of values
25 ##-----------------------------------------
32 ##-------------------------------------------------
34 ## param X first data vector (..x_i..)
35 ## param Y second data vector (..x_i..)
36 ## = 1/n \Sum_{i=1}^n (x_i - avg(x)) * (y_i - avg(y))
37 ##--------------------------------------------------
39 assert len(X) == len(Y)
40 n = len(X) # n=len(X)=len(Y)
45 S_XY += (X[i] - avg_X) * (Y[i] - avg_Y)
48 ##---------------------------------------------------------------------
49 ## variance : variance
50 ## param X data vector ( ..x_i.. )
51 ## (S_X)^2 = (Sum ( x_i - avg(x) )^2 ) / n
52 ##---------------------------------------------------------------------
58 S_X2 += (X[i] - avg_X) ** 2
61 ##---------------------------------------------------------------------
62 ## calibrate : output correction factors, c_lat on latency, c_bw on bw
63 ## such that bandwidth * c_bw = bw_regr, latency * c_lat = lat_regr
64 ## where bw_regr and lat_regr are the values approximating experimental
67 ## param links number of links traversed during ping-pong
68 ## param latency as specified on command line, in s
69 ## param bandwidth as specified on command line, in Byte/s
70 ## param sizes vector of data sizes, in Bytes
71 ## param timings vector of time taken: timings[i] for sizes[i], in us
72 ##---------------------------------------------------------------------
73 def calibrate (links, latency, bandwidth, sizes, timings):
74 assert len(sizes) == len(timings)
77 # compute linear regression : find an affine form time = a*size+b
78 S_XY = cov(sizes, timings)
79 S_X2 = variance(sizes)
81 b = avg(timings) - a * avg(sizes)
82 # corresponding bandwith, in byte/s (was in byte/us in skampi dat)
84 # corresponding latency, in s (was in us in skampi dat)
86 print("\nregression: {0} * x + {1}".format(a,b))
87 print("corr_bw = bw_regr/bandwidth= {0}/{1}={2} lat_regr/(lat_xml*links)={3}/({4}*{5}))".format(bw_regr,bandwidth,bw_regr/bandwidth,lat_regr,latency,links))
88 # return linear regression result and corresponding correction factors c_bw,c_lat
89 return a,b, bw_regr/bandwidth, lat_regr/(latency*links)
92 ##---------------------------------------------------------------------
93 ## outputs a C formatted conditional return value for factor
95 ## param lb lower bound
96 ## param ub upper bound
97 ## param lb_included boolean to tell if bound is included (<=) or exclude (<)
98 ## param ub_included boolean to tell if bound is included (<=) or exclude (<)
99 ##---------------------------------------------------------------------
100 def c_code_print (lb,ub, retval, lb_included, ub_included):
101 lb_cmp = ub_cmp = "<"
109 print("\t /* case {0:.1f} KiB {1} size {2} {3:.1f} KiB */".format(lb_kib,lb_cmp,ub_cmp,ub_kib))
110 print("\t if ({0:d} {1} size && size {2} {3:d}) ".format(lb,lb_cmp,ub_cmp,ub))
111 print("\t return({0});" . format(retval))
114 ##-----------------------------------------------------------------------------------------------
116 ##-----------------------------------------------------------------------------------------------
117 links = int(sys.argv[2])
118 latency = float(sys.argv[3])
119 bandwidth = float(sys.argv[4])
120 skampidat = open(sys.argv[1], "r")
122 ## read data from skampi logs.
126 for line in skampidat:
128 if line[0] != '#' and len(l) >= 3: # is it a comment ?
131 #count= 8388608 8388608 144916.1 7.6 32 144916.1 143262.0
132 #("%s %d %d %f %f %d %f %f\n" % (countlbl, count, countn, time, stddev, iter, mini, maxi)
133 readdata.append( (int(l[1]),float(l[3]) / 2) ); # divide by 2 because of ping-pong measured
135 ## These may not be sorted so sort it by message size before processing.
136 sorteddata = sorted( readdata, key=lambda pair: pair[0])
137 sizes,timings= zip(*sorteddata)
140 ## adds message sizes of interest: if values are specified starting from the 6th command line arg
141 ## and these values are found as message sizes in te log file, add it to the limits list.
142 ## Each of these value si considered a potential inflexion point between two segments.
144 ## If no value specified, a single segment is considered from 1st to last message size logged.
146 if len(sys.argv) > 5:
147 for i in range(5, len(sys.argv)):
148 limits += [idx for idx in range(len(sizes)) if sizes[idx] == int(sys.argv[i])]
149 limits.append(len(sizes) - 1)
154 correc = calibrate(links, latency, bandwidth, sizes[low:lim + 1], timings[low:lim + 1])
156 # save interval [lb,ub] correction, regression line direction and origin
157 # and corresponding correction factors for bw and lat resp.
158 (dircoef,origin,factor_bw,factor_lat) = correc
159 factors.append( (sizes[low],sizes[lim], dircoef, origin, factor_bw,factor_lat) )
160 print("Segment [%d:%d] --Bandwidth factor=%g --Latency factor=%g " % (sizes[low], sizes[lim], factor_bw,factor_lat))
163 # now computes joining lines between segments
166 print("\n/**\n *------------------ <copy/paste C code snippet in surf/network.c> ----------------------")
167 print(" *\n * produced by: {0}\n *".format(' '.join(sys.argv)))
168 print(" *---------------------------------------------------------------------------------------\n **/")
170 # print correction factor for bandwidth for each segment
171 print("static double smpi_bandwidth_factor(double size)\n{")
172 for (lb,ub,a,b,factor_bw,factor_lat) in factors:
173 c_code_print(lb,ub,factor_bw,True,True)
175 # save ends and starts of segments
177 joinseg.append( (lb,timings[sizes.index(lb)]) )
179 joinseg.append( (ub,timings[sizes.index(ub)]) )
181 # print correction factor for bandwidth between segments
183 print("\n\t /* ..:: inter-segment corrections ::.. */");
186 (x0,y0) = joinseg[inx]
188 (x1,y1) = joinseg[inx]
190 # line eq. is y = (y1-y0)/(x0-x1) * x + (y0 x1 - y1 x0)/(x1-x0)
191 a = (y1-y0) / (x1-x0)
193 factor_join_bw = bw_join / bandwidth
194 #print("Joining points (%f,%f) -> (%f,%f) : line dir : a=%g\n" % (x0,y0,x1,y1,a))
195 c_code_print(x0,x1,factor_join_bw,False,False)
199 # print correction factor for latency for each segment
200 print("static double smpi_latency_factor(double size)\n{")
201 for (lb,ub,a,b,factor_bw,factor_lat) in factors:
202 c_code_print(lb,ub,factor_lat,True,True)
204 print("\n\t /* ..:: inter-segment corrections ::.. */");
206 (x0,y0) = joinseg.pop()
207 (x1,y1) = joinseg.pop()
208 # line eq. is y = (y0-y1)/(x0-x1) * x + (y0 x1 - y1 x0)/(x1-x0)
209 #print("(%f,%f) -> (%f,%f)\n" % (x0,y0,x1,y1))
210 b = 1e-6 * (y0*x1-y1*x0) / (x1-x0)
211 factor_join_lat = b / (latency*links)
212 c_code_print(x0,x1,factor_join_lat,False,False)
216 print("\n/**\n *------------------ <copy/paste C code snippet in surf/network.c> ----------------------\n **/")