src/smpi/smpirun.in

   1 #!/usr/bin/env sh
   2
   3 # Copyright (c) 2007-2020. The SimGrid Team. All rights reserved.
   4
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the license (GNU LGPL) which comes with this package.
   7
   8 @CMAKE_SMPI_COMMAND@
   9
  10 SIMGRID_VERSION="@SIMGRID_VERSION_STRING@"
  11 SIMGRID_GITHASH="@SIMGRID_GITHASH@"
  12
  13 DEFAULT_LOOPBACK_BANDWIDTH="498000000Bps"
  14 DEFAULT_LOOPBACK_LATENCY="0.000004s"
  15 DEFAULT_NETWORK_BANDWIDTH="$((26 * 1024 * 1024))Bps"
  16 DEFAULT_NETWORK_LATENCY="0.000005s"
  17 DEFAULT_SPEED="100flops"
  18
  19 LOOPBACK_BANDWIDTH="${DEFAULT_LOOPBACK_BANDWIDTH}"
  20 LOOPBACK_LATENCY="${DEFAULT_LOOPBACK_LATENCY}"
  21 NETWORK_BANDWIDTH="${DEFAULT_NETWORK_BANDWIDTH}"
  22 NETWORK_LATENCY="${DEFAULT_NETWORK_LATENCY}"
  23 SPEED="${DEFAULT_SPEED}"
  24
  25 PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-@HAVE_PRIVATIZATION@}"
  26
  27 SIMOPTS="--cfg=surf/precision:1e-9 --cfg=network/model:SMPI"
  28
  29 #usage to print the way this script should be called
  30 usage () {
  31     cat <<EOF
  32 Usage: $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> program [program-options]
  33        $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> -replay <tracefile> [program] [program-options]
  34 Options:
  35   -keep-temps                # don't remove the generated files after execution
  36   -wrapper <command>         # use command to run the program (e.g. "valgrind" or "gdb --args")
  37   -gdb                       # run within GDB (-wrapper "gdb --args" -keep-temps)
  38   -lldb                      # run within LLDB (-wrapper "lldb --" -keep-temps)
  39   -vgdb                      # run within Valgrind+GDB (-wrapper "valgrind --vgdb=yes --vgdb-error=0" -keep-temps)
  40   -map                       # display the machine on which each process rank is mapped
  41   -np <numprocs>             # use that amount of processes from the hostfile.
  42                              # By default, all processes of the hostfile are used.
  43   -no-privatize              # Disable the globals privatization, that is activated by default
  44   -trace-ti                  # activate time independent tracing (for replay, default in smpi_simgrid.txt)
  45   -trace                     # activate tracing (Paje, default in smpi_simgrid.trace)
  46   -trace-comment <comment>   # put a comment on the top of the trace file
  47   -trace-comment-file <file> # put file contents on the top of the trace file as comment
  48   -trace-grouped             # group MPI processes by location
  49   -trace-resource            # trace resource utilization
  50   -trace-file <tracefile>    # name of the tracefile (simgrid_smpi.trace)
  51   -replay <tracefile>        # replays a trace instead of actually executing an application
  52
  53   -version                   # Displays the SimGrid version (human readable)
  54   -git-version               # Displays the git hash of SimGrid
  55
  56 or (deprecated usage):
  57   $0 [-keep-temps] [-np <numprocs>] [-bandwidth <bytes/sec>] [-latency <secs>] program [program-options]
  58
  59 EOF
  60 }
  61
  62 #check if we have at least one parameter
  63 if [ $# -eq 0 ]
  64 then
  65     usage
  66     exit
  67 fi
  68
  69 WRAPPER=""
  70 HOSTFILE=""
  71 HOSTFILETMP=0
  72 MAPOPT=0
  73 REPLAY=0
  74 QUIET=""
  75
  76 unset pid
  77
  78 trapped_signals="HUP INT QUIT ILL ABRT SEGV FPE ALRM TERM USR1 USR2 BUS"
  79
  80 die () {
  81     printf '[%s] ** error: %s. Aborting.\n' "$(basename "$0")" "$*" >&2
  82     exit 1
  83 }
  84
  85 smpirun_cleanup()
  86 {
  87   if [ -z "${KEEP}" ] ; then
  88       if [ -z "${PLATFORM}" ] && [ -n "$PLATFORMTMP" ]; then
  89         rm -f "${PLATFORMTMP}"
  90         PLATFORMTMP=""
  91       fi
  92       if [ ${HOSTFILETMP} = 1 ] && [ -n "$HOSTFILE" ] ; then
  93           rm -f "${HOSTFILE}"
  94           HOSTFILE=""
  95       fi
  96       if [ "${UNROLLEDHOSTFILETMP}" = 1 ] && [ -n "$UNROLLEDHOSTFILE" ] ; then
  97           rm -f "${UNROLLEDHOSTFILE}"
  98           UNROLLEDHOSTFILE=""
  99       fi
 100       if [ -n "${APPLICATIONTMP}" ]; then
 101         rm -f "${APPLICATIONTMP}"
 102         APPLICATIONTMP=""
 103       fi
 104   fi
 105 }
 106
 107 smpirun_trap() {
 108   local sig
 109   sig="$1"
 110
 111   # Cleanup and kill the child process:
 112   smpirun_cleanup
 113   if [ -n "$pid" ]; then
 114     kill -TERM "$pid"
 115   fi
 116   unset pid
 117
 118   # Raise the same signal again (remove the traps first):
 119   trap - $trapped_signals
 120   kill -"$sig" $$
 121
 122   # This should never happen:
 123   kill -ABRT $$
 124   kill -TERM $$
 125 }
 126
 127 for s in $trapped_signals; do
 128   trap "smpirun_trap $s" "$s"
 129 done
 130
 131 while true; do
 132     case "$1" in
 133         "-np" | "-n")
 134             NUMPROCS="$2"
 135             shift 2
 136             ;;
 137         "-bandwidth")
 138             NETWORK_BANDWIDTH="$2"
 139             shift 2
 140             ;;
 141         "-latency")
 142             NETWORK_LATENCY="$2"
 143             shift 2
 144             ;;
 145         "-platform")
 146             PLATFORM="$2"
 147             if [ ! -f "${PLATFORM}" ]; then
 148                 die "the file '${PLATFORM}' does not exist"
 149             fi
 150             shift 2
 151             ;;
 152         "-hostfile" | "-machinefile")
 153             HOSTFILE="$2"
 154             if [ ! -f "${HOSTFILE}" ]; then
 155                 die "the file '${HOSTFILE}' does not exist"
 156             fi
 157             shift 2
 158             ;;
 159         "-replay")
 160             APP_TRACES="$2"
 161             shift 2
 162             ;;
 163         "-no-privatize")
 164             PRIVATIZE="--cfg=smpi/privatization:no"
 165             shift 1
 166             ;;
 167         "-map")
 168             MAPOPT=1
 169             shift 1
 170             ;;
 171         "-trace")
 172             TRACE_ACTIVE="true"
 173             shift 1
 174             ;;
 175         "-trace-ti")
 176             TRACE_ACTIVE="true"
 177             TRACE_TI_ACTIVE="true"
 178             shift 1
 179             ;;
 180         "-trace-comment")
 181             TRACE_COMMENT="$2"
 182             shift 2
 183             ;;
 184         "-trace-comment-file")
 185             TRACE_COMMENT_FILE="$2"
 186             shift 2
 187             ;;
 188         "-trace-file")
 189             TRACE_FILENAME="$2"
 190             shift 2
 191             ;;
 192         "-trace-grouped")
 193             TRACE_GROUPED="true"
 194             shift 1
 195             ;;
 196         "-trace-resource")
 197             TRACE_RESOURCE="true"
 198             shift 1
 199             ;;
 200         "-keep-temps")
 201             KEEP="true"
 202             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 203             shift 1
 204             ;;
 205         "-quiet")
 206             QUIET="true"
 207             shift 1
 208             ;;
 209         "-wrapper")
 210             WRAPPER="$2"
 211             shift 2
 212             ;;
 213         "-gdb")
 214             WRAPPER="gdb --args"
 215             KEEP="true"
 216             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 217             shift 1
 218             ;;
 219         "-vgdb")
 220             WRAPPER="valgrind --vgdb=yes --vgdb-error=0"
 221             KEEP="true"
 222             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 223             shift 1
 224             ;;
 225         "-lldb")
 226             WRAPPER="lldb --"
 227             KEEP="true"
 228             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 229             shift 1
 230             ;;
 231         "-help" | "--help" | "-h")
 232             usage
 233             exit 0
 234             ;;
 235         "-version" | "--version" | "-v")
 236             printf '%b\n' "$SIMGRID_VERSION"
 237             exit 0
 238             ;;
 239         "-git-version" | "--git-version")
 240             printf '%b\n' "$SIMGRID_GITHASH"
 241             exit 0
 242             ;;
 243         "--cfg="*|"--log="*)
 244             for OPT in ${1#*=}
 245             do
 246                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 247             done
 248             shift 1
 249             ;;
 250         "-foreground")
 251             # Nothing to do, compatibility.
 252             shift 1
 253             ;;
 254         *)
 255             break
 256             ;;
 257     esac
 258 done
 259
 260 if [ -n "${APP_TRACES}" ] ; then
 261     if [ $# -eq 0 ] ; then
 262         EXEC="@SMPIREPLAYMAIN@"
 263     else
 264         EXEC="$1"
 265         shift
 266     fi
 267 else
 268     # check if we still have at least one parameter beyond options
 269     if [ $# -eq 0 ]
 270     then
 271         echo "Error: no program to execute!"
 272         usage
 273         exit
 274     fi
 275
 276     EXEC="$1"
 277     shift
 278 fi
 279
 280 # steal --cfg and --logs options
 281 while [ $# -gt 0 ]; do
 282     case "$1" in
 283         "--cfg="*|"--log="*)
 284             for OPT in ${1#*=}
 285             do
 286                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 287             done
 288             shift 1
 289             ;;
 290         *)
 291             PROC_ARGS="${PROC_ARGS:+$PROC_ARGS }$1"
 292             shift
 293             ;;
 294     esac
 295 done
 296
 297 if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then
 298     echo "No hostfile nor platform specified."
 299     usage
 300     exit 1
 301 fi
 302
 303 if [ -z "${HOSTFILE}" ] ; then
 304     HOSTFILETMP=1
 305     HOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 306     @PYTHON_EXECUTABLE@ -c '
 307 import xml.etree.ElementTree as ET
 308 import sys
 309 import re
 310
 311 tree = ET.parse(sys.stdin)
 312
 313 for elem in tree.findall(".//host"):
 314     print(elem.attrib["id"])
 315
 316 for elem in tree.findall(".//cluster"):
 317     prefix = elem.attrib["prefix"]
 318     radical = elem.attrib["radical"]
 319     suffix = elem.attrib["suffix"]
 320     for r in radical.split(","):
 321         m = re.match("^([^-]*?)-([^-]*)$", r)
 322         if m:
 323             for i in range(int(m.group(1)), int(m.group(2))):
 324                 print(prefix + str(i) + suffix)
 325         else:
 326             print(prefix + r + suffix)
 327             ' < "${PLATFORM}" > "${HOSTFILE}"
 328 fi
 329 UNROLLEDHOSTFILETMP=0
 330
 331 # parse if our lines are terminated by :num_process
 332 if grep -q ':' "$HOSTFILE" ; then
 333     UNROLLEDHOSTFILETMP=1
 334     UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 335     @PYTHON_EXECUTABLE@ -c '
 336 import sys
 337 import re
 338
 339 for line in sys.stdin:
 340     m = re.match("(.*):(.*)", line)
 341     if m:
 342         for i in range(0, int(m.group(2))):
 343             print(m.group(1))
 344     else:
 345         print(line.strip())
 346 ' < "${HOSTFILE}"  > "${UNROLLEDHOSTFILE}"
 347     if [ ${HOSTFILETMP} = 1 ] ; then
 348         rm "${HOSTFILE}"
 349         HOSTFILETMP=0
 350     fi
 351     HOSTFILE=$UNROLLEDHOSTFILE
 352 fi
 353
 354 # Don't use wc -l to compute it to avoid issues with trailing \n at EOF
 355 hostfile_procs=$(grep -c "[a-zA-Z0-9]" "$HOSTFILE")
 356 if [ "${hostfile_procs}" = 0 ] ; then
 357     die "the hostfile '${HOSTFILE}' is empty"
 358 fi
 359
 360 if [ -z "${NUMPROCS}" ] ; then
 361     # Use the amount of processes in the hostfile as default value for the -np parameter
 362     NUMPROCS=$hostfile_procs
 363 fi
 364
 365 if [ "${NUMPROCS}" -gt "${hostfile_procs}" ] ; then
 366     echo "You requested to use ${NUMPROCS} ranks, but there is only ${hostfile_procs} processes in your hostfile..." >&2
 367 fi
 368
 369 ##-------------------------------- DEFAULT or SPECIFIED PLATFORM --------------------------------------
 370 if [ -z "${PLATFORM}" ]; then
 371     PLATFORMTMP="$(mktemp smpitmp-platfXXXXXX)"
 372
 373     cat > "${PLATFORMTMP}" <<PLATFORMHEAD
 374 <?xml version='1.0'?>
 375 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 376 <platform version="4.1">
 377 <zone id="AS0" routing="Full">
 378 PLATFORMHEAD
 379
 380     i=${NUMPROCS}
 381     while [ "$i" -gt 0 ]; do
 382         {
 383         echo "  <host id=\"host$i\" speed=\"${SPEED}\"/>"
 384         echo "  <link id=\"loop$i\" bandwidth=\"${LOOPBACK_BANDWIDTH}\" latency=\"${LOOPBACK_LATENCY}\"/>"
 385         echo "  <link id=\"link$i\" bandwidth=\"${NETWORK_BANDWIDTH}\" latency=\"${NETWORK_LATENCY}\"/>"
 386         } >> "${PLATFORMTMP}"
 387         i=$((i - 1))
 388     done
 389
 390     i=${NUMPROCS}
 391     while [ "$i" -gt 0 ]; do
 392         j=${NUMPROCS}
 393         while [ "$j" -gt 0 ]; do
 394             if [ "$i" -eq "$j" ]; then
 395                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"loop$i\"/></route>" >> "${PLATFORMTMP}"
 396             else
 397                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"link$i\"/><link_ctn id=\"link$j\"/></route>" >> "${PLATFORMTMP}"
 398             fi
 399             j=$((j - 1))
 400         done
 401         i=$((i - 1))
 402     done
 403
 404     cat >> "${PLATFORMTMP}" <<PLATFORMFOOT
 405 </zone>
 406 </platform>
 407 PLATFORMFOOT
 408
 409 else
 410     PLATFORMTMP=${PLATFORM}
 411 fi
 412 ##-------------------------------- end DEFAULT or SPECIFIED PLATFORM --------------------------------------
 413 ##-------------------------------- DEFAULT APPLICATION --------------------------------------
 414 APPLICATIONTMP="$(mktemp smpitmp-appXXXXXX)"
 415 #APPLICATIONTMP="app.xml"
 416
 417 cat > "${APPLICATIONTMP}" <<APPLICATIONHEAD
 418 <?xml version='1.0'?>
 419 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 420 <platform version="4.1">
 421 APPLICATIONHEAD
 422
 423 ##---- cache hostnames of hostfile---------------
 424 if [ -n "${HOSTFILE}" ] && [ -f "${HOSTFILE}" ]; then
 425     hostnames=$(< "${HOSTFILE}" tr '\n\r' '  ')
 426 fi
 427
 428 if [ -n "${APP_TRACES}" ]; then
 429     if [ -f "${APP_TRACES}" ]; then
 430         hosttraces=$(< "${APP_TRACES}" tr '\n\r' '  ' )
 431         NUMTRACES=$(< "${APP_TRACES}" wc -l)
 432         REPLAY=1
 433     else
 434         printf "File not found: %s\n" "${APP_TRACES:-\${APP_TRACES\}}" >&2
 435         exit 1
 436     fi
 437 fi
 438
 439 ##----------------------------------------------------------
 440 ##  generate application.xml with hostnames from hostfile:
 441 ##  the name of host_i (1<=i<=p, where -np p) is the line i in hostfile (where -hostfile hostfile), or "host$i" if
 442 ##  hostfile has less than i lines.
 443 ##----------------------------------------------------------
 444
 445 HAVE_SEQ="$(which seq 2>/dev/null)"
 446
 447 if [ -n "${HAVE_SEQ}" ]; then
 448     SEQ=$(${HAVE_SEQ} 0 $(( NUMPROCS - 1)))
 449 else
 450     cnt=0
 451     while [ $cnt -lt "${NUMPROCS}" ] ; do
 452         SEQ="$SEQ $cnt"
 453         cnt=$((cnt + 1));
 454     done
 455 fi
 456
 457 set -- $hostnames
 458
 459 ##---- generate <actor> tags------------------------------
 460 #prepare arguments at once
 461 for ARG in $PROC_ARGS; do
 462   XML_ARGS="${XML_ARGS}""<argument value=\"${ARG}\"/>
 463 "
 464 done
 465
 466 for i in ${SEQ}
 467 do
 468     j=$(( i % hostfile_procs + 1 ))
 469     host=$(eval "echo \${$j}")
 470
 471     ##---- optional display of ranks to actor mapping
 472     if [ ${MAPOPT} = 1 ]; then
 473       echo "[rank $i] -> $host"
 474     fi
 475     {
 476     echo "  <actor host=\"${host}\" function=\"$i\"> <!-- function name used only for logging -->
 477     <prop id=\"instance_id\" value=\"smpirun\"/>
 478     <prop id=\"rank\" value=\"$i\"/>"
 479     if [ ${REPLAY} = 1 ]; then
 480         echo "    <prop id=\"smpi_replay\" value=\"true\"/>"
 481         if  [ "${NUMTRACES}" -gt 1 ]; then
 482             echo "    <argument value=\"$(echo "$hosttraces"|cut -d' ' -f$j)\"/>"
 483         else
 484             echo "    <argument value=\"$(echo "$hosttraces"|cut -d' ' -f1)\"/>"
 485         fi
 486     else
 487     echo "${XML_ARGS}"
 488     fi
 489     echo "  </actor>"
 490     }    >> "${APPLICATIONTMP}"
 491 done
 492
 493 cat >> "${APPLICATIONTMP}" <<APPLICATIONFOOT
 494 </platform>
 495 APPLICATIONFOOT
 496 ##-------------------------------- end DEFAULT APPLICATION --------------------------------------
 497 ##---------------------- SMPI TRACING OPTIONS ---------------------------------
 498 if [ -n "${TRACE_ACTIVE}" ]; then
 499     #define trace filename
 500     if [ -n "${TRACE_TI_ACTIVE}" ]; then
 501         if [ -z "${TRACE_FILENAME}" ]; then
 502             TRACE_FILENAME="smpi_simgrid.txt"
 503         fi
 504         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes --cfg=tracing/smpi/format:TI --cfg=tracing/smpi/computing:yes"
 505     else
 506         if [ -z "${TRACE_FILENAME}" ]; then
 507             TRACE_FILENAME="smpi_simgrid.trace"
 508         fi
 509         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes"
 510     fi
 511
 512     if [ -n "${TRACE_COMMENT}" ]; then
 513         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment:${TRACE_COMMENT}"
 514     fi
 515
 516     if [ -n "${TRACE_COMMENT_FILE}" ]; then
 517         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment-file:${TRACE_COMMENT_FILE}"
 518     fi
 519
 520     if [ -n "${TRACE_GROUPED}" ]; then
 521         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/smpi/group:yes"
 522     fi
 523
 524     if [ -n "${TRACE_RESOURCE}" ]; then
 525         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/categorized:yes --cfg=tracing/uncategorized:yes"
 526     fi
 527 fi
 528 ##---------------------- end SMPI TRACING OPTIONS ---------------------------------
 529
 530 # Do not remove, this variable may be used by user code (e.g. StarPU)
 531 export SMPI_GLOBAL_SIZE=${NUMPROCS}
 532 if [ -n "${KEEP}" ] && [ -z "${QUIET}" ] ; then
 533     echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" "${APPLICATIONTMP}"
 534     if [ ${HOSTFILETMP} = 1 ] ; then
 535         echo "Generated hostfile ${HOSTFILE} kept."
 536     fi
 537     if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 538         echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 539     fi
 540 fi
 541
 542 # Execute the process
 543 #
 544 # The shell still need to be alive for the duration in order to do some cleanup after the process.
 545 #
 546 # We are going through great lengths in order to both keep stdin and be able to handle signals:
 547 #
 548 # * The job is launched in the background in order to be able to handle signals.
 549 #
 550 # * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command
 551 #   is launched in the background: this can be overridden in bash but not in standard bourne shell.
 552 exec 3<&0
 553 ${WRAPPER} "@SMPIMAIN@" "${EXEC}" ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} "${PLATFORMTMP}" "${APPLICATIONTMP}" <&3 3>&- &
 554 pid=$!
 555 exec 3>&-
 556 wait $pid
 557 status=$?
 558 # With dash on Windows WSL/Ubuntu, "wait" sometimes returns early with an exit
 559 # status of 128. Try again.
 560 while test $status -eq 128 && kill -0 $pid 2>/dev/null; do
 561     wait $pid
 562     status=$?
 563 done
 564 pid=""
 565
 566 # Keep temporary files on failures to help debugging
 567 #
 568 if [ ${status} -ne 0 ] ; then
 569     if [ -z "${KEEP}" ] && [ -z "${QUIET}" ]; then
 570         echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" "${APPLICATIONTMP}"
 571         if [ ${HOSTFILETMP} = 1 ] ; then
 572             echo "Generated hostfile ${HOSTFILE} kept."
 573         fi
 574         if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 575             echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 576         fi
 577         KEEP=true
 578     fi
 579     echo "Execution failed with code ${status}."
 580 fi
 581
 582 smpirun_cleanup
 583
 584 exit $status