tools/simgrid_convert_TI_traces.py

   1 #!/usr/bin/env python3
   2
   3 '''
   4 This script is intended to convert SMPI time independent traces (TIT) from the
   5 old format (simgrid version <= 3.19) to the new format.
   6
   7 On the previous format each MPI_wait calls were associated to the last ISend of
   8 IRecv call arbitrarily.
   9
  10 This new that includes tags field that links MPI_wait calls to the
  11 MPI_ISend or MPI_IRecv associated to this wait.
  12
  13 This script reproduce the old behavior of simgrid because informations are
  14 missing to add the tags properly. It also lower case all the mpi calls.
  15
  16 It takes in input (as argument or in stdin) the trace list file that is only a
  17 simple text file that contains path of actual TIT files split by rank.
  18
  19 It creates a directory to put the traces on ("converted_traces" by default)
  20 '''
  21
  22 import os
  23 import pathlib
  24 import shutil
  25
  26
  27 def insert_elem(split_line, position, elem):
  28     split_line.insert(position, elem)
  29     return " ".join(split_line)
  30
  31
  32 def convert_trace(trace_path, base_path, output_path, trace_version="1.0"):
  33     old_file_path = os.path.join(base_path, trace_path)
  34     with open(old_file_path) as trace_file:
  35
  36         new_file_path = os.path.join(output_path, trace_path)
  37         pathlib.Path(os.path.dirname(new_file_path)).mkdir(
  38             parents=True, exist_ok=True)
  39
  40         with open(new_file_path, "w") as new_trace:
  41             # Write header
  42             new_trace.write("# version: " + trace_version + "\n")
  43
  44             last_async_call_src = None
  45             last_async_call_dst = None
  46             for line_num, line in enumerate(trace_file.readlines()):
  47                 # print(line)
  48                 new_line = None
  49                 split_line = line.lower().strip().split(" ")
  50                 mpi_call = split_line[1]
  51
  52                 if mpi_call == "recv" or mpi_call == "send":
  53                     new_line = insert_elem(split_line, 3, "0")
  54
  55                 elif mpi_call == "irecv" or mpi_call == "isend":
  56                     last_async_call_src = split_line[0]
  57                     last_async_call_dst = split_line[2]
  58                     new_line = insert_elem(split_line, 3, "0")
  59                     # print("found async call in line: "+ str(line_num))
  60
  61                 elif mpi_call == "wait":
  62                     # print("found wait call in line: "+ str(line_num))
  63                     if (last_async_call_src is None
  64                             or last_async_call_dst is None):
  65                         raise Exception("Invalid traces: No Isend or Irecv "
  66                                         "found before the wait in line " +
  67                                         str(line_num) + " in file " + old_file_path)
  68                     new_line = insert_elem(split_line, 2, last_async_call_src)
  69                     new_line = insert_elem(split_line, 3, last_async_call_dst)
  70                     new_line = insert_elem(split_line, 4, "0")
  71
  72                 if new_line is not None:
  73                     # print("line: "+ str(line_num) + " in file " + old_file_path +
  74                     #        " processed\n:OLD: " + line + "\n:NEW: " + new_line)
  75                     new_trace.write(new_line + "\n")
  76                 else:
  77                     new_trace.write(line.lower())
  78
  79
  80 if __name__ == "__main__":
  81     import argparse
  82     import sys
  83
  84     parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
  85
  86     parser.add_argument('trace_list_file', type=argparse.FileType('r'),
  87                         default=sys.stdin, help="The trace list file (e.g. smpi_simgrid.txt)")
  88
  89     parser.add_argument('--output_path', '-o', default="converted_traces",
  90                         help="The path where converted traces will be put")
  91
  92     args = parser.parse_args()
  93
  94     trace_list_file_path = args.trace_list_file.name
  95
  96     # creates results dir
  97     pathlib.Path(args.output_path).mkdir(
  98         parents=True, exist_ok=True)
  99
 100     # copy trace list file
 101     try:
 102         shutil.copy(trace_list_file_path, args.output_path)
 103     except shutil.SameFileError:
 104         print("ERROR: Inplace replacement of the trace is not supported: "
 105               "Please, select another output path")
 106         sys.exit(-1)
 107
 108     with open(trace_list_file_path) as tracelist_file:
 109         trace_list = tracelist_file.readlines()
 110
 111     # get based path relative to trace list file
 112     base_path = os.path.dirname(trace_list_file_path)
 113
 114     trace_list = [x.strip() for x in trace_list]
 115
 116     # process trace files
 117     for trace_path in trace_list:
 118         if os.path.isabs(trace_path):
 119             sys.exit("ERROR: Absolute path in the trace list file is not supported")
 120         convert_trace(trace_path, base_path, args.output_path)
 121
 122     print("Traces converted!")
 123     print("Result directory:\n" + args.output_path)