src/smpi/smpirun.in

   1 #!/usr/bin/env sh
   2
   3 # Copyright (c) 2007-2023. The SimGrid Team. All rights reserved.
   4
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the license (GNU LGPL) which comes with this package.
   7
   8 @CMAKE_SMPI_COMMAND@
   9
  10 SIMGRID_VERSION="@SIMGRID_VERSION_STRING@"
  11 SIMGRID_GITHASH="@GIT_VERSION@"
  12
  13 DEFAULT_LOOPBACK_BANDWIDTH="498000000Bps"
  14 DEFAULT_LOOPBACK_LATENCY="0.000004s"
  15 DEFAULT_NETWORK_BANDWIDTH="$((26 * 1024 * 1024))Bps"
  16 DEFAULT_NETWORK_LATENCY="0.000005s"
  17 DEFAULT_SPEED="100flops"
  18
  19 LOOPBACK_BANDWIDTH="${DEFAULT_LOOPBACK_BANDWIDTH}"
  20 LOOPBACK_LATENCY="${DEFAULT_LOOPBACK_LATENCY}"
  21 NETWORK_BANDWIDTH="${DEFAULT_NETWORK_BANDWIDTH}"
  22 NETWORK_LATENCY="${DEFAULT_NETWORK_LATENCY}"
  23 SPEED="${DEFAULT_SPEED}"
  24
  25 PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-@HAVE_PRIVATIZATION@}"
  26
  27 NUMPROCS=0
  28 DEPLOYOPTS=""
  29
  30 SIMOPTS="--cfg=precision/timing:1e-9 --cfg=network/model:SMPI"
  31
  32 SMPITMPDIR="$(dirname $(mktemp -u))"
  33
  34 #usage to print the way this script should be called
  35 usage () {
  36     cat <<EOF
  37 Usage: $0 [OPTIONS] -platform <xmldesc|so> -hostfile <hostfile> program [program-options]
  38        $0 [OPTIONS] -platform <xmldesc|so> -hostfile <hostfile> -replay <tracefile> [program] [program-options]
  39 Options:
  40   -analyze                   # show information about allocations and timings at the end of execution
  41   -keep-temps                # don't remove the generated files after execution
  42   -wrapper <command>         # use command to run the program (e.g. "valgrind" or "gdb --args")
  43   -gdb                       # run within GDB (-wrapper "gdb --args" -keep-temps)
  44   -lldb                      # run within LLDB (-wrapper "lldb --" -keep-temps)
  45   -vgdb                      # run within Valgrind+GDB (-wrapper "valgrind --vgdb=yes --vgdb-error=0" -keep-temps)
  46   -map                       # display the machine on which each process rank is mapped
  47   -np <numprocs>             # use that amount of processes from the hostfile.
  48                              # By default, all processes of the hostfile are used.
  49   -no-privatize              # Disable the globals privatization, that is activated by default
  50   -tmpdir                    # Directory used to store temporary files. Defaults to system's.
  51   -trace-ti                  # activate time independent tracing (for replay, default in smpi_simgrid.txt)
  52   -trace                     # activate tracing (Paje, default in smpi_simgrid.trace)
  53   -trace-comment <comment>   # put a comment on the top of the trace file
  54   -trace-comment-file <file> # put file contents on the top of the trace file as comment
  55   -trace-grouped             # group MPI processes by location
  56   -trace-resource            # trace resource utilization
  57   -trace-file <tracefile>    # name of the tracefile (simgrid_smpi.trace)
  58   -replay <tracefile>        # replays a trace instead of actually executing an application
  59
  60   -version                   # Displays the SimGrid version (human readable)
  61   -git-version               # Displays the git hash of SimGrid
  62
  63 or (deprecated usage):
  64   $0 [-keep-temps] [-np <numprocs>] [-bandwidth <bytes/sec>] [-latency <secs>] program [program-options]
  65
  66 EOF
  67 }
  68
  69 #check if we have at least one parameter
  70 if [ $# -eq 0 ]
  71 then
  72     usage
  73     exit
  74 fi
  75
  76 WRAPPER=""
  77 HOSTFILE=""
  78 QUIET=""
  79
  80 unset pid
  81
  82 trapped_signals="HUP INT QUIT ILL ABRT SEGV FPE ALRM TERM USR1 USR2 BUS"
  83
  84 die () {
  85     printf '[%s] ** error: %s. Aborting.\n' "$(basename "$0")" "$*" >&2
  86     exit 1
  87 }
  88
  89 smpirun_cleanup()
  90 {
  91   if [ -z "${KEEP}" ] ; then
  92       if [ -z "${PLATFORM}" ] && [ -n "$PLATFORMTMP" ]; then
  93         rm -f "${PLATFORMTMP}"
  94         PLATFORMTMP=""
  95       fi
  96       if [ "${UNROLLEDHOSTFILETMP}" = 1 ] && [ -n "$UNROLLEDHOSTFILE" ] ; then
  97           rm -f "${UNROLLEDHOSTFILE}"
  98           UNROLLEDHOSTFILE=""
  99       fi
 100   fi
 101 }
 102
 103 smpirun_trap() {
 104   local sig
 105   sig="$1"
 106
 107   # Cleanup and kill the child process:
 108   smpirun_cleanup
 109   if [ -n "$pid" ]; then
 110     kill -TERM "$pid"
 111   fi
 112   unset pid
 113
 114   # Raise the same signal again (remove the traps first):
 115   trap - $trapped_signals
 116   kill -"$sig" $$
 117
 118   # This should never happen:
 119   kill -ABRT $$
 120   kill -TERM $$
 121 }
 122
 123 for s in $trapped_signals; do
 124   trap "smpirun_trap $s" "$s"
 125 done
 126
 127 while true; do
 128     case "$1" in
 129         "-np" | "-n")
 130             NUMPROCS="$2"
 131             shift 2
 132             ;;
 133         "-bandwidth")
 134             NETWORK_BANDWIDTH="$2"
 135             shift 2
 136             ;;
 137         "-latency")
 138             NETWORK_LATENCY="$2"
 139             shift 2
 140             ;;
 141         "-platform")
 142             PLATFORM="$2"
 143             if [ ! -f "${PLATFORM}" ]; then
 144                 die "the file '${PLATFORM}' does not exist"
 145             fi
 146             shift 2
 147             ;;
 148         "-hostfile" | "-machinefile")
 149             HOSTFILE="$2"
 150             if [ ! -f "${HOSTFILE}" ]; then
 151                 die "the file '${HOSTFILE}' does not exist"
 152             fi
 153             shift 2
 154             ;;
 155         "-replay")
 156             APP_TRACES="$2"
 157             if [ ! -f "${APP_TRACES}" ]; then
 158                 die "the file '${APP_TRACES}' does not exist"
 159             fi
 160             DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/replay:${APP_TRACES}"
 161             shift 2
 162             ;;
 163         "-no-privatize")
 164             PRIVATIZE="--cfg=smpi/privatization:no"
 165             shift 1
 166             ;;
 167         "-map")
 168             DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/map:1"
 169             shift 1
 170             ;;
 171         "-tmpdir")
 172             SMPITMPDIR="$2"
 173             shift 1
 174             ;;
 175         "-trace")
 176             TRACE_ACTIVE="true"
 177             shift 1
 178             ;;
 179         "-trace-ti")
 180             TRACE_ACTIVE="true"
 181             TRACE_TI_ACTIVE="true"
 182             shift 1
 183             ;;
 184         "-trace-comment")
 185             TRACE_COMMENT="$2"
 186             shift 2
 187             ;;
 188         "-trace-comment-file")
 189             TRACE_COMMENT_FILE="$2"
 190             shift 2
 191             ;;
 192         "-trace-file")
 193             TRACE_FILENAME="$2"
 194             shift 2
 195             ;;
 196         "-trace-grouped")
 197             TRACE_GROUPED="true"
 198             shift 1
 199             ;;
 200         "-trace-resource")
 201             TRACE_RESOURCE="true"
 202             shift 1
 203             ;;
 204         "-keep-temps")
 205             KEEP="true"
 206             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 207             shift 1
 208             ;;
 209         "-quiet")
 210             QUIET="true"
 211             shift 1
 212             ;;
 213         "-wrapper")
 214             WRAPPER="$2"
 215             shift 2
 216             ;;
 217         "-gdb")
 218             WRAPPER="gdb --args"
 219             KEEP="true"
 220             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 221             shift 1
 222             ;;
 223         "-vgdb")
 224             WRAPPER="valgrind --vgdb=yes --vgdb-error=0"
 225             KEEP="true"
 226             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 227             shift 1
 228             ;;
 229         "-lldb")
 230             WRAPPER="lldb --"
 231             KEEP="true"
 232             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 233             shift 1
 234             ;;
 235         "-analyze")
 236             SIMOPTS="$SIMOPTS --cfg=smpi/display-timing:yes --cfg=smpi/display-allocs:yes --cfg=smpi/list-leaks:50 --cfg=smpi/pedantic:true --cfg=smpi/barrier-collectives:true"
 237             shift 1
 238             ;;
 239         "-help" | "--help" | "-h")
 240             usage
 241             exit 0
 242             ;;
 243         "--help-coll")
 244             ${WRAPPER} "@SMPIMAIN@" --help-coll
 245             exit 0
 246             ;;
 247         "-version" | "--version" | "-v")
 248             printf '%b\n' "$SIMGRID_VERSION"
 249             exit 0
 250             ;;
 251         "-git-version" | "--git-version")
 252             printf '%b\n' "$SIMGRID_GITHASH"
 253             exit 0
 254             ;;
 255         "--cfg="*|"--log="*)
 256             for OPT in ${1#*=}
 257             do
 258                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 259             done
 260             shift 1
 261             ;;
 262         "-foreground")
 263             # Nothing to do, compatibility.
 264             shift 1
 265             ;;
 266         *)
 267             break
 268             ;;
 269     esac
 270 done
 271
 272 #setup tmp dir
 273 SIMOPTS="$SIMOPTS --cfg=smpi/tmpdir:$SMPITMPDIR"
 274 export LD_LIBRARY_PATH="$SMPITMPDIR:$LD_LIBRARY_PATH"
 275
 276 if [ -n "${APP_TRACES}" ] ; then
 277     if [ $# -eq 0 ] ; then
 278         EXEC="@SMPIREPLAYMAIN@"
 279     else
 280         EXEC="$1"
 281         shift
 282     fi
 283 else
 284     # check if we still have at least one parameter beyond options
 285     if [ $# -eq 0 ]
 286     then
 287         echo "Error: no program to execute!"
 288         usage
 289         exit
 290     fi
 291
 292     EXEC="$1"
 293     shift
 294 fi
 295
 296 # steal --cfg and --logs options
 297 while [ $# -gt 0 ]; do
 298     case "$1" in
 299         "--cfg="*|"--log="*)
 300             for OPT in ${1#*=}
 301             do
 302                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 303             done
 304             shift 1
 305             ;;
 306         *)
 307             PROC_ARGS="${PROC_ARGS:+$PROC_ARGS }$1"
 308             shift
 309             ;;
 310     esac
 311 done
 312
 313 if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then
 314     echo "No hostfile nor platform specified."
 315     usage
 316     exit 1
 317 fi
 318
 319 UNROLLEDHOSTFILETMP=0
 320
 321 # parse if our lines are terminated by :num_process
 322 if [ -n "${HOSTFILE}" ] && grep -q ':' "${HOSTFILE}" ; then
 323     UNROLLEDHOSTFILETMP=1
 324     UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 325     @PYTHON_EXECUTABLE@ -c '
 326 import sys
 327 import re
 328
 329 for line in sys.stdin:
 330     m = re.match("(.*):(.*)", line)
 331     if m:
 332         for i in range(0, int(m.group(2))):
 333             print(m.group(1))
 334     else:
 335         print(line.strip())
 336 ' < "${HOSTFILE}"  > "${UNROLLEDHOSTFILE}"
 337     HOSTFILE=$UNROLLEDHOSTFILE
 338 fi
 339
 340 DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/np:${NUMPROCS}"
 341 DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/hostfile:${HOSTFILE}"
 342
 343 ##-------------------------------- DEFAULT or SPECIFIED PLATFORM --------------------------------------
 344 if [ -z "${PLATFORM}" ]; then
 345     PLATFORMTMP="$(mktemp smpitmp-platfXXXXXX)"
 346
 347     cat > "${PLATFORMTMP}" <<PLATFORMHEAD
 348 <?xml version='1.0'?>
 349 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 350 <platform version="4.1">
 351 <zone id="AS0" routing="Full">
 352 PLATFORMHEAD
 353
 354     i=${NUMPROCS}
 355     while [ "$i" -gt 0 ]; do
 356         {
 357         echo "  <host id=\"host$i\" speed=\"${SPEED}\"/>"
 358         echo "  <link id=\"loop$i\" bandwidth=\"${LOOPBACK_BANDWIDTH}\" latency=\"${LOOPBACK_LATENCY}\"/>"
 359         echo "  <link id=\"link$i\" bandwidth=\"${NETWORK_BANDWIDTH}\" latency=\"${NETWORK_LATENCY}\"/>"
 360         } >> "${PLATFORMTMP}"
 361         i=$((i - 1))
 362     done
 363
 364     i=${NUMPROCS}
 365     while [ "$i" -gt 0 ]; do
 366         j=${NUMPROCS}
 367         while [ "$j" -gt 0 ]; do
 368             if [ "$i" -eq "$j" ]; then
 369                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"loop$i\"/></route>" >> "${PLATFORMTMP}"
 370             else
 371                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"link$i\"/><link_ctn id=\"link$j\"/></route>" >> "${PLATFORMTMP}"
 372             fi
 373             j=$((j - 1))
 374         done
 375         i=$((i - 1))
 376     done
 377
 378     cat >> "${PLATFORMTMP}" <<PLATFORMFOOT
 379 </zone>
 380 </platform>
 381 PLATFORMFOOT
 382
 383 else
 384     PLATFORMTMP=${PLATFORM}
 385 fi
 386 ##-------------------------------- end DEFAULT or SPECIFIED PLATFORM --------------------------------------
 387 ##---------------------- SMPI TRACING OPTIONS ---------------------------------
 388 if [ -n "${TRACE_ACTIVE}" ]; then
 389     #define trace filename
 390     if [ -n "${TRACE_TI_ACTIVE}" ]; then
 391         if [ -z "${TRACE_FILENAME}" ]; then
 392             TRACE_FILENAME="smpi_simgrid.txt"
 393         fi
 394         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes --cfg=tracing/smpi/format:TI --cfg=tracing/smpi/computing:yes"
 395     else
 396         if [ -z "${TRACE_FILENAME}" ]; then
 397             TRACE_FILENAME="smpi_simgrid.trace"
 398         fi
 399         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes"
 400     fi
 401
 402     if [ -n "${TRACE_COMMENT}" ]; then
 403         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment:${TRACE_COMMENT}"
 404     fi
 405
 406     if [ -n "${TRACE_COMMENT_FILE}" ]; then
 407         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment-file:${TRACE_COMMENT_FILE}"
 408     fi
 409
 410     if [ -n "${TRACE_GROUPED}" ]; then
 411         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/smpi/group:yes"
 412     fi
 413
 414     if [ -n "${TRACE_RESOURCE}" ]; then
 415         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/categorized:yes --cfg=tracing/uncategorized:yes"
 416     fi
 417 fi
 418 ##---------------------- end SMPI TRACING OPTIONS ---------------------------------
 419
 420 # Do not remove, this variable may be used by user code (e.g. StarPU)
 421 export SMPI_GLOBAL_SIZE=${NUMPROCS}
 422 if [ -n "${KEEP}" ] && [ -z "${QUIET}" ] ; then
 423     echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}"
 424     if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 425         echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 426     fi
 427 fi
 428
 429 # Execute the process
 430 #
 431 # The shell still need to be alive for the duration in order to do some cleanup after the process.
 432 #
 433 # We are going through great lengths in order to both keep stdin and be able to handle signals:
 434 #
 435 # * The job is launched in the background in order to be able to handle signals.
 436 #
 437 # * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command
 438 #   is launched in the background: this can be overridden in bash but not in standard bourne shell.
 439 exec 3<&0
 440 ${WRAPPER} "@SMPIMAIN@" "${EXEC}" ${PRIVATIZE} ${DEPLOYOPTS} ${TRACEOPTIONS} ${SIMOPTS} "${PLATFORMTMP}" ${PROC_ARGS} <&3 3>&- &
 441 pid=$!
 442 exec 3>&-
 443 wait $pid
 444 status=$?
 445 # With dash on Windows WSL/Ubuntu, "wait" sometimes returns early with an exit
 446 # status of 128. Try again.
 447 while test $status -eq 128 && kill -0 $pid 2>/dev/null; do
 448     wait $pid
 449     status=$?
 450 done
 451 pid=""
 452
 453 # Keep temporary files on failures to help debugging
 454 #
 455 if [ ${status} -ne 0 ] ; then
 456     if [ -z "${KEEP}" ] && [ -z "${QUIET}" ]; then
 457         echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}"
 458         if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 459             echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 460         fi
 461         KEEP=true
 462     fi
 463     echo "Execution failed with code ${status}."
 464 fi
 465
 466 smpirun_cleanup
 467
 468 exit $status