tools/simgrid_convert_TIT_traces.py

   1 #!/usr/bin/env python3
   2
   3 '''
   4 This script is intended to convert SMPI time independent traces (TIT) from the
   5 old format (simgrid version <= 3.19) to the new format.
   6
   7 On the previous format each MPI_wait calls were associated to the last ISend of
   8 IRecv call arbitrarily.
   9
  10 This new that includes tags field that links MPI_wait calls to the
  11 MPI_ISend or MPI_IRecv associated to this wait.
  12
  13 This script reproduce the old behavior of simgrid because informations are
  14 missing to add the tags properly.
  15
  16 It takes in input (as argument or in stdin) the trace list file that is only a
  17 simple text file that contains path of actual TIT files split by rank.
  18 '''
  19
  20 import os
  21 import pathlib
  22 import shutil
  23
  24
  25 def insert_elem(split_line, position, elem):
  26     split_line.insert(position, elem)
  27     return " ".join(split_line)
  28
  29
  30 def convert_trace(trace_path, base_path, output_path):
  31     old_file_path = os.path.join(base_path, trace_path)
  32     with open(old_file_path) as trace_file:
  33         new_file_path = os.path.join(output_path, trace_path)
  34         pathlib.Path(os.path.dirname(new_file_path)).mkdir(
  35                 parents=True, exist_ok=True)
  36         with open(new_file_path, "w") as new_trace:
  37             last_async_call_src = None
  38             last_async_call_dst = None
  39             for line_num, line in enumerate(trace_file.readlines()):
  40                 #print(line)
  41                 new_line = None
  42                 split_line = line.strip().split(" ")
  43                 mpi_call = split_line[1]
  44                 if mpi_call == "recv" or mpi_call == "send":
  45                     new_line = insert_elem(split_line, 3, "0")
  46                 if mpi_call == "Irecv" or mpi_call == "Isend":
  47                     last_async_call_src = split_line[0]
  48                     last_async_call_dst = split_line[2]
  49                     new_line = insert_elem(split_line, 3, "0")
  50                     # print("found async call in line: "+ str(line_num))
  51                 # print("mpi_call: " + mpi_call)
  52                 if mpi_call == "wait":
  53                     # print("found wait call in line: "+ str(line_num))
  54                     if (last_async_call_src is None
  55                             or last_async_call_dst is None):
  56                         raise Exception("Invalid traces: No Isend or Irecv "
  57                                 "found before the wait in line " +
  58                                 str(line_num) + " in file " + old_file_path )
  59                     new_line = insert_elem(split_line, 2, last_async_call_src)
  60                     new_line = insert_elem(split_line, 3, last_async_call_dst)
  61                     new_line = insert_elem(split_line, 4, "0")
  62
  63                 if new_line is not None:
  64                     # print("line: "+ str(line_num) + " in file " + old_file_path +
  65                     #        " processed\n:OLD: " + line + "\n:NEW: " + new_line)
  66                     new_trace.write(new_line + "\n")
  67                 else:
  68                     new_trace.write(line)
  69
  70
  71 if __name__ == "__main__":
  72     import argparse
  73     import sys
  74
  75
  76     parser = argparse.ArgumentParser(description=__doc__)
  77
  78     parser.add_argument('trace_list_file', type=argparse.FileType('r'),
  79             default=sys.stdin, help="the trace list file")
  80
  81     parser.add_argument('--output_path', '-o', default="converted_traces",
  82             help="the path where converted traces will be put")
  83
  84     args = parser.parse_args()
  85
  86     trace_list_file_path = args.trace_list_file.name
  87
  88     # creates results dir
  89     pathlib.Path(args.output_path).mkdir(
  90             parents=True, exist_ok=True)
  91
  92     # copy trace list file
  93     shutil.copy(trace_list_file_path, args.output_path)
  94
  95     with open(trace_list_file_path) as tracelist_file:
  96         trace_list = tracelist_file.readlines()
  97
  98     # get based path relative to trace list file
  99     base_path = os.path.dirname(trace_list_file_path)
 100
 101     trace_list = [x.strip() for x in trace_list]
 102
 103     # process trace files
 104     for trace_path in trace_list:
 105         convert_trace(trace_path, base_path, args.output_path)
 106
 107     print("Done!")
 108     print("Result in:\n" + args.output_path)