ouputs explicit C code to be included in the network model

[simgrid.git] / contrib / network_model / calibrate_piecewise.py

diff --git a/contrib/network_model/calibrate_piecewise.py b/contrib/network_model/calibrate_piecewise.py
index b8ce3b0..3433026 100755 (executable)
--- a/contrib/network_model/calibrate_piecewise.py
+++ b/contrib/network_model/calibrate_piecewise.py
@@ -7,10 +7,11 @@ from math import sqrt
 
 if len(sys.argv) < 5:
    print("Usage : %s datafile links latency bandwidth [size...]" % sys.argv[0])
 
 if len(sys.argv) < 5:
    print("Usage : %s datafile links latency bandwidth [size...]" % sys.argv[0])

-   print("where : links is the number of links between nodes")
+   print("where : datafile is a SkaMPI pingpong measurement log file"); 
+   print("        links is the number of links between nodes")

    print("        latency is the nominal latency given in the platform file")
    print("        bandwidth is the nominal bandwidth given in the platform file")
    print("        latency is the nominal latency given in the platform file")
    print("        bandwidth is the nominal bandwidth given in the platform file")

-   print("        size are segments limites")
+   print("        size are segments limits")

    sys.exit(-1)
 
 ##-----------------------------------------
    sys.exit(-1)
 
 ##-----------------------------------------


@@ -35,32 +36,52 @@ def cov (X, Y):
    avg_X = avg(X)
    avg_Y = avg(Y)
    S_XY = 0.0
    avg_X = avg(X)
    avg_Y = avg(Y)
    S_XY = 0.0

-   for i in xrange(n):
+   for i in range(n):

       S_XY += (X[i] - avg_X) * (Y[i] - avg_Y)
    return (S_XY / n)
 
       S_XY += (X[i] - avg_X) * (Y[i] - avg_Y)
    return (S_XY / n)
 

-##----------------------------------
+##---------------------------------------------------------------------

 ## variance : variance
 ## param X data vector ( ..x_i.. )
 ## (S_X)^2 = (Sum ( x_i - avg(x) )^2 ) / n
 ## variance : variance
 ## param X data vector ( ..x_i.. )
 ## (S_X)^2 = (Sum ( x_i - avg(x) )^2 ) / n

-##----------------------------------
+##---------------------------------------------------------------------

 def variance (X):
    n = len(X)
    avg_X = avg (X)
    S_X2 = 0.0
 def variance (X):
    n = len(X)
    avg_X = avg (X)
    S_X2 = 0.0

-   for i in xrange(n):
+   for i in range(n):

       S_X2 += (X[i] - avg_X) ** 2
    return (S_X2 / n)
 
       S_X2 += (X[i] - avg_X) ** 2
    return (S_X2 / n)
 

+##---------------------------------------------------------------------
+## calibrate : output correction factors, c_lat on latency, c_bw on bw
+## such that bandwidth * c_bw = bw_regr, latency * c_lat = lat_regr
+## where bw_regr and lat_regr are the values approximatong experimental
+## observations.
+##
+## param links number of links traversed during ping-pong
+## param latency as specified on command line, in s
+## param bandwidth as specified on command line, in Byte/s
+## param sizes vector of data sizes, in Bytes
+## param timings vector of time taken: timings[i] for sizes[i], in us
+##---------------------------------------------------------------------

 def calibrate (links, latency, bandwidth, sizes, timings):
    assert len(sizes) == len(timings)
    if len(sizes) < 2:
       return None
 def calibrate (links, latency, bandwidth, sizes, timings):
    assert len(sizes) == len(timings)
    if len(sizes) < 2:
       return None

+   # compute linear regression : find an affine form  a*size+b

    S_XY = cov(sizes, timings)
    S_X2 = variance(sizes)
    a = S_XY / S_X2
    b = avg(timings) - a * avg(sizes)
    S_XY = cov(sizes, timings)
    S_X2 = variance(sizes)
    a = S_XY / S_X2
    b = avg(timings) - a * avg(sizes)

-   return (b * 1e-6) / (latency * links), 1e6 / (a * bandwidth)
+   # corresponding bandwith, in s (was in us in skampi dat)
+   bw_regr = 1e6 / a     
+   # corresponding latency, in s (was in us in skampi dat)
+   lat_regr = b*1e-6
+   #print("\nregression: {0} * x + {1}".format(a,b))
+   #print("corr_bw = bw_regr/bandwidth= {0}/{1}={2}     lat_regr/(lat_xml*links)={3}/({4}*{5}))".format(bw_regr,bandwidth,bw_regr/bandwidth,lat_regr,latency,links))
+   # return correction factors c_bw,c_lat
+   return bw_regr/bandwidth, lat_regr/(latency*links)

 
 ##-----------------------------------------------------------------------------------------------
 ## main
 
 ##-----------------------------------------------------------------------------------------------
 ## main


@@ -70,17 +91,23 @@ latency = float(sys.argv[3])
 bandwidth = float(sys.argv[4])
 skampidat = open(sys.argv[1], "r")
 
 bandwidth = float(sys.argv[4])
 skampidat = open(sys.argv[1], "r")
 

+## read data from skampi logs.

 timings = []
 sizes = []
 timings = []
 sizes = []

+readdata =[]

 for line in skampidat:
 for line in skampidat:

-   l = line.split();
-   if line[0] != '#' and len(l) >= 3:   # is it a comment ?
+       l = line.split();
+       if line[0] != '#' and len(l) >= 3:   # is it a comment ?

       ## expected format
       ## ---------------
       #count= 8388608  8388608  144916.1       7.6       32  144916.1  143262.0
       #("%s %d %d %f %f %d %f %f\n" % (countlbl, count, countn, time, stddev, iter, mini, maxi)
       ## expected format
       ## ---------------
       #count= 8388608  8388608  144916.1       7.6       32  144916.1  143262.0
       #("%s %d %d %f %f %d %f %f\n" % (countlbl, count, countn, time, stddev, iter, mini, maxi)

-      timings.append(float(l[3]) / links)
-      sizes.append(int(l[1]))
+               readdata.append( (int(l[1]),float(l[3]) / 2) );   # divide by 2 because of ping-pong measured
+
+## These may not be sorted so sort it by message size before processing.
+sorteddata = sorted( readdata, key=lambda pair: pair[0])
+sizes,timings= zip(*sorteddata)
+

 
 ## adds message sizes of interest: if values are specified starting from the 6th command line arg 
 ## and these values are found as message sizes in te log file, add it to the limits list.
 
 ## adds message sizes of interest: if values are specified starting from the 6th command line arg 
 ## and these values are found as message sizes in te log file, add it to the limits list.


@@ -89,13 +116,36 @@ for line in skampidat:
 ## If no value specified, a single segment is considered from 1st to last message size logged.
 limits = []
 if len(sys.argv) > 5:
 ## If no value specified, a single segment is considered from 1st to last message size logged.
 limits = []
 if len(sys.argv) > 5:

-   for i in xrange(5, len(sys.argv)):
-      limits += [idx for idx in xrange(len(sizes)) if sizes[idx] == int(sys.argv[i])]
+   for i in range(5, len(sys.argv)):
+      limits += [idx for idx in range(len(sizes)) if sizes[idx] == int(sys.argv[i])]

 limits.append(len(sizes) - 1)
 
 limits.append(len(sizes) - 1)
 

+factors = []

 low = 0
 for lim in limits:
    correc = calibrate(links, latency, bandwidth, sizes[low:lim + 1], timings[low:lim + 1])
    if correc:
 low = 0
 for lim in limits:
    correc = calibrate(links, latency, bandwidth, sizes[low:lim + 1], timings[low:lim + 1])
    if correc:

-      print("Segment [%d:%d] -- Latency factor=%g -- Bandwidth factor=%g" % (sizes[low], sizes[lim], correc[0], correc[1]))
+       # save interval [lb,ub] and correction factors for bw and lat resp.
+       factors.append( (sizes[low],sizes[lim], correc[0], correc[1]) )
+       print("Segment [%d:%d] --Bandwidth factor=%g --Latency factor=%g " % (sizes[low], sizes[lim], correc[0], correc[1]))

    low = lim + 1
    low = lim + 1

+
+print("\n/**\n *------------------ <copy/paste C code snippet in surf/network.c> ----------------------")
+print(" *\n * produced by: {0}\n *".format(' '.join(sys.argv)))
+print(" *---------------------------------------------------------------------------------------\n **/")
+print("static double smpi_bandwidth_factor(double size)\n{")                                
+for (lb,ub,factor_bw,factor_lat) in factors:
+       print("\t /* case %d Bytes <= size <=%d Bytes */" % (lb,ub))
+       print("\t if (%d <= size && size <=  %d) {" % (lb,ub))
+       print("\t       return(%g);" % (factor_bw))
+       print("\t }")
+print("}\n")  
+
+print("static double smpi_latency_factor(double size)\n{")                                
+for (lb,ub,factor_bw,factor_lat) in factors:
+       print("\t /* case %d Bytes <= size <=%d Bytes */" % (lb,ub))
+       print("\t if (%d <= size && size <=  %d) {" % (lb,ub))
+       print("\t       return(%g);" % (factor_lat))
+       print("\t }")
+print("}\n")  
+print("/**\n *------------------ </copy/paste C code snippet in surf/network.c> ---------------------\n **/")