tools/simgrid_convert_TI_traces.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (c) 2018-2023. The SimGrid Team. All rights reserved.
   4
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the license (GNU LGPL) which comes with this package.
   7
   8 '''
   9 This script is intended to convert SMPI time independent traces (TIT) from the
  10 old format (SimGrid version <= 3.19) to the new format.
  11
  12 On the previous format each MPI_wait calls were associated to the last ISend of
  13 IRecv call arbitrarily.
  14
  15 This new that includes tags field that links MPI_wait calls to the
  16 MPI_ISend or MPI_IRecv associated to this wait.
  17
  18 This script reproduce the old behavior of SimGrid because information are
  19 missing to add the tags properly. It also lower case all the mpi calls.
  20
  21 It takes in input (as argument or in stdin) the trace list file that is only a
  22 simple text file that contains path of actual TIT files split by rank.
  23
  24 It creates a directory to put the traces on ("converted_traces" by default)
  25 '''
  26
  27 import os
  28 import pathlib
  29 import shutil
  30
  31
  32 def insert_elem(split_line, position, elem):
  33     split_line.insert(position, elem)
  34     return " ".join(split_line)
  35
  36
  37 def convert_trace(trace_path, base_path, output_path, trace_version="1.0"):
  38     old_file_path = os.path.join(base_path, trace_path)
  39     with open(old_file_path) as trace_file:
  40
  41         new_file_path = os.path.join(output_path, trace_path)
  42         pathlib.Path(os.path.dirname(new_file_path)).mkdir(
  43             parents=True, exist_ok=True)
  44
  45         with open(new_file_path, "w") as new_trace:
  46             # Write header
  47             new_trace.write("# version: " + trace_version + "\n")
  48
  49             last_async_call_src = None
  50             last_async_call_dst = None
  51             for line_num, line in enumerate(trace_file.readlines()):
  52                 # print(line)
  53                 new_line = None
  54                 split_line = line.lower().strip().split(" ")
  55                 mpi_call = split_line[1]
  56
  57                 if mpi_call == "recv" or mpi_call == "send":
  58                     new_line = insert_elem(split_line, 3, "0")
  59
  60                 elif mpi_call == "irecv" or mpi_call == "isend":
  61                     last_async_call_src = split_line[0]
  62                     last_async_call_dst = split_line[2]
  63                     new_line = insert_elem(split_line, 3, "0")
  64                     # print("found async call in line: "+ str(line_num))
  65
  66                 elif mpi_call == "wait":
  67                     # print("found wait call in line: "+ str(line_num))
  68                     if (last_async_call_src is None
  69                             or last_async_call_dst is None):
  70                         raise Exception("Invalid traces: No Isend or Irecv "
  71                                         "found before the wait in line " +
  72                                         str(line_num) + " in file " + old_file_path)
  73                     insert_elem(split_line, 2, last_async_call_src)
  74                     insert_elem(split_line, 3, last_async_call_dst)
  75                     new_line = insert_elem(split_line, 4, "0")
  76
  77                 if new_line is not None:
  78                     # print("line: "+ str(line_num) + " in file " + old_file_path +
  79                     #        " processed\n:OLD: " + line + "\n:NEW: " + new_line)
  80                     new_trace.write(new_line + "\n")
  81                 else:
  82                     new_trace.write(line.lower())
  83
  84
  85 if __name__ == "__main__":
  86     import argparse
  87     import sys
  88
  89     parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
  90
  91     parser.add_argument('trace_list_file', type=argparse.FileType('r'),
  92                         default=sys.stdin, help="The trace list file (e.g. smpi_simgrid.txt)")
  93
  94     parser.add_argument('--output_path', '-o', default="converted_traces",
  95                         help="The path where converted traces will be put")
  96
  97     args = parser.parse_args()
  98
  99     trace_list_file_path = args.trace_list_file.name
 100
 101     # creates results dir
 102     pathlib.Path(args.output_path).mkdir(
 103         parents=True, exist_ok=True)
 104
 105     # copy trace list file
 106     try:
 107         shutil.copy(trace_list_file_path, args.output_path)
 108     except shutil.SameFileError:
 109         print("ERROR: Inplace replacement of the trace is not supported: "
 110               "Please, select another output path")
 111         sys.exit(-1)
 112
 113     with open(trace_list_file_path) as tracelist_file:
 114         trace_list = tracelist_file.readlines()
 115
 116     # get based path relative to trace list file
 117     base_path = os.path.dirname(trace_list_file_path)
 118
 119     trace_list = [x.strip() for x in trace_list]
 120
 121     # process trace files
 122     for trace_path in trace_list:
 123         if os.path.isabs(trace_path):
 124             sys.exit("ERROR: Absolute path in the trace list file is not supported")
 125         convert_trace(trace_path, base_path, args.output_path)
 126
 127     print("Traces converted!")
 128     print("Result directory:\n" + args.output_path)