src/smpi/smpirun.in

   1 #!/usr/bin/env sh
   2
   3 # Copyright (c) 2007-2021. The SimGrid Team. All rights reserved.
   4
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the license (GNU LGPL) which comes with this package.
   7
   8 @CMAKE_SMPI_COMMAND@
   9
  10 SIMGRID_VERSION="@SIMGRID_VERSION_STRING@"
  11 SIMGRID_GITHASH="@SIMGRID_GITHASH@"
  12
  13 DEFAULT_LOOPBACK_BANDWIDTH="498000000Bps"
  14 DEFAULT_LOOPBACK_LATENCY="0.000004s"
  15 DEFAULT_NETWORK_BANDWIDTH="$((26 * 1024 * 1024))Bps"
  16 DEFAULT_NETWORK_LATENCY="0.000005s"
  17 DEFAULT_SPEED="100flops"
  18
  19 LOOPBACK_BANDWIDTH="${DEFAULT_LOOPBACK_BANDWIDTH}"
  20 LOOPBACK_LATENCY="${DEFAULT_LOOPBACK_LATENCY}"
  21 NETWORK_BANDWIDTH="${DEFAULT_NETWORK_BANDWIDTH}"
  22 NETWORK_LATENCY="${DEFAULT_NETWORK_LATENCY}"
  23 SPEED="${DEFAULT_SPEED}"
  24
  25 PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-@HAVE_PRIVATIZATION@}"
  26
  27 SIMOPTS="--cfg=surf/precision:1e-9 --cfg=network/model:SMPI"
  28
  29 SMPITMPDIR="$(dirname $(mktemp -u))"
  30
  31 #usage to print the way this script should be called
  32 usage () {
  33     cat <<EOF
  34 Usage: $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> program [program-options]
  35        $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> -replay <tracefile> [program] [program-options]
  36 Options:
  37   -keep-temps                # don't remove the generated files after execution
  38   -wrapper <command>         # use command to run the program (e.g. "valgrind" or "gdb --args")
  39   -gdb                       # run within GDB (-wrapper "gdb --args" -keep-temps)
  40   -lldb                      # run within LLDB (-wrapper "lldb --" -keep-temps)
  41   -vgdb                      # run within Valgrind+GDB (-wrapper "valgrind --vgdb=yes --vgdb-error=0" -keep-temps)
  42   -map                       # display the machine on which each process rank is mapped
  43   -np <numprocs>             # use that amount of processes from the hostfile.
  44                              # By default, all processes of the hostfile are used.
  45   -no-privatize              # Disable the globals privatization, that is activated by default
  46   -tmpdir                    # Directory used to store temporary files. Defaults to system's.
  47   -trace-ti                  # activate time independent tracing (for replay, default in smpi_simgrid.txt)
  48   -trace                     # activate tracing (Paje, default in smpi_simgrid.trace)
  49   -trace-comment <comment>   # put a comment on the top of the trace file
  50   -trace-comment-file <file> # put file contents on the top of the trace file as comment
  51   -trace-grouped             # group MPI processes by location
  52   -trace-resource            # trace resource utilization
  53   -trace-file <tracefile>    # name of the tracefile (simgrid_smpi.trace)
  54   -replay <tracefile>        # replays a trace instead of actually executing an application
  55
  56   -version                   # Displays the SimGrid version (human readable)
  57   -git-version               # Displays the git hash of SimGrid
  58
  59 or (deprecated usage):
  60   $0 [-keep-temps] [-np <numprocs>] [-bandwidth <bytes/sec>] [-latency <secs>] program [program-options]
  61
  62 EOF
  63 }
  64
  65 #check if we have at least one parameter
  66 if [ $# -eq 0 ]
  67 then
  68     usage
  69     exit
  70 fi
  71
  72 WRAPPER=""
  73 HOSTFILE=""
  74 HOSTFILETMP=0
  75 MAPOPT=0
  76 REPLAY=0
  77 QUIET=""
  78
  79 unset pid
  80
  81 trapped_signals="HUP INT QUIT ILL ABRT SEGV FPE ALRM TERM USR1 USR2 BUS"
  82
  83 die () {
  84     printf '[%s] ** error: %s. Aborting.\n' "$(basename "$0")" "$*" >&2
  85     exit 1
  86 }
  87
  88 smpirun_cleanup()
  89 {
  90   if [ -z "${KEEP}" ] ; then
  91       if [ -z "${PLATFORM}" ] && [ -n "$PLATFORMTMP" ]; then
  92         rm -f "${PLATFORMTMP}"
  93         PLATFORMTMP=""
  94       fi
  95       if [ ${HOSTFILETMP} = 1 ] && [ -n "$HOSTFILE" ] ; then
  96           rm -f "${HOSTFILE}"
  97           HOSTFILE=""
  98       fi
  99       if [ "${UNROLLEDHOSTFILETMP}" = 1 ] && [ -n "$UNROLLEDHOSTFILE" ] ; then
 100           rm -f "${UNROLLEDHOSTFILE}"
 101           UNROLLEDHOSTFILE=""
 102       fi
 103       if [ -n "${APPLICATIONTMP}" ]; then
 104         rm -f "${APPLICATIONTMP}"
 105         APPLICATIONTMP=""
 106       fi
 107   fi
 108 }
 109
 110 smpirun_trap() {
 111   local sig
 112   sig="$1"
 113
 114   # Cleanup and kill the child process:
 115   smpirun_cleanup
 116   if [ -n "$pid" ]; then
 117     kill -TERM "$pid"
 118   fi
 119   unset pid
 120
 121   # Raise the same signal again (remove the traps first):
 122   trap - $trapped_signals
 123   kill -"$sig" $$
 124
 125   # This should never happen:
 126   kill -ABRT $$
 127   kill -TERM $$
 128 }
 129
 130 for s in $trapped_signals; do
 131   trap "smpirun_trap $s" "$s"
 132 done
 133
 134 while true; do
 135     case "$1" in
 136         "-np" | "-n")
 137             NUMPROCS="$2"
 138             shift 2
 139             ;;
 140         "-bandwidth")
 141             NETWORK_BANDWIDTH="$2"
 142             shift 2
 143             ;;
 144         "-latency")
 145             NETWORK_LATENCY="$2"
 146             shift 2
 147             ;;
 148         "-platform")
 149             PLATFORM="$2"
 150             if [ ! -f "${PLATFORM}" ]; then
 151                 die "the file '${PLATFORM}' does not exist"
 152             fi
 153             shift 2
 154             ;;
 155         "-hostfile" | "-machinefile")
 156             HOSTFILE="$2"
 157             if [ ! -f "${HOSTFILE}" ]; then
 158                 die "the file '${HOSTFILE}' does not exist"
 159             fi
 160             shift 2
 161             ;;
 162         "-replay")
 163             APP_TRACES="$2"
 164             shift 2
 165             ;;
 166         "-no-privatize")
 167             PRIVATIZE="--cfg=smpi/privatization:no"
 168             shift 1
 169             ;;
 170         "-map")
 171             MAPOPT=1
 172             shift 1
 173             ;;
 174         "-tmpdir")
 175             SMPITMPDIR="$2"
 176             shift 1
 177             ;;
 178         "-trace")
 179             TRACE_ACTIVE="true"
 180             shift 1
 181             ;;
 182         "-trace-ti")
 183             TRACE_ACTIVE="true"
 184             TRACE_TI_ACTIVE="true"
 185             shift 1
 186             ;;
 187         "-trace-comment")
 188             TRACE_COMMENT="$2"
 189             shift 2
 190             ;;
 191         "-trace-comment-file")
 192             TRACE_COMMENT_FILE="$2"
 193             shift 2
 194             ;;
 195         "-trace-file")
 196             TRACE_FILENAME="$2"
 197             shift 2
 198             ;;
 199         "-trace-grouped")
 200             TRACE_GROUPED="true"
 201             shift 1
 202             ;;
 203         "-trace-resource")
 204             TRACE_RESOURCE="true"
 205             shift 1
 206             ;;
 207         "-keep-temps")
 208             KEEP="true"
 209             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 210             shift 1
 211             ;;
 212         "-quiet")
 213             QUIET="true"
 214             shift 1
 215             ;;
 216         "-wrapper")
 217             WRAPPER="$2"
 218             shift 2
 219             ;;
 220         "-gdb")
 221             WRAPPER="gdb --args"
 222             KEEP="true"
 223             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 224             shift 1
 225             ;;
 226         "-vgdb")
 227             WRAPPER="valgrind --vgdb=yes --vgdb-error=0"
 228             KEEP="true"
 229             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 230             shift 1
 231             ;;
 232         "-lldb")
 233             WRAPPER="lldb --"
 234             KEEP="true"
 235             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 236             shift 1
 237             ;;
 238         "-help" | "--help" | "-h")
 239             usage
 240             exit 0
 241             ;;
 242         "-version" | "--version" | "-v")
 243             printf '%b\n' "$SIMGRID_VERSION"
 244             exit 0
 245             ;;
 246         "-git-version" | "--git-version")
 247             printf '%b\n' "$SIMGRID_GITHASH"
 248             exit 0
 249             ;;
 250         "--cfg="*|"--log="*)
 251             for OPT in ${1#*=}
 252             do
 253                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 254             done
 255             shift 1
 256             ;;
 257         "-foreground")
 258             # Nothing to do, compatibility.
 259             shift 1
 260             ;;
 261         *)
 262             break
 263             ;;
 264     esac
 265 done
 266
 267 #setup tmp dir
 268 SIMOPTS="$SIMOPTS --cfg=smpi/tmpdir:$SMPITMPDIR"
 269 export LD_LIBRARY_PATH="$SMPITMPDIR:$LD_LIBRARY_PATH"
 270
 271 if [ -n "${APP_TRACES}" ] ; then
 272     if [ $# -eq 0 ] ; then
 273         EXEC="@SMPIREPLAYMAIN@"
 274     else
 275         EXEC="$1"
 276         shift
 277     fi
 278 else
 279     # check if we still have at least one parameter beyond options
 280     if [ $# -eq 0 ]
 281     then
 282         echo "Error: no program to execute!"
 283         usage
 284         exit
 285     fi
 286
 287     EXEC="$1"
 288     shift
 289 fi
 290
 291 # steal --cfg and --logs options
 292 while [ $# -gt 0 ]; do
 293     case "$1" in
 294         "--cfg="*|"--log="*)
 295             for OPT in ${1#*=}
 296             do
 297                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 298             done
 299             shift 1
 300             ;;
 301         *)
 302             PROC_ARGS="${PROC_ARGS:+$PROC_ARGS }$1"
 303             shift
 304             ;;
 305     esac
 306 done
 307
 308 if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then
 309     echo "No hostfile nor platform specified."
 310     usage
 311     exit 1
 312 fi
 313
 314 if [ -z "${HOSTFILE}" ] ; then
 315     HOSTFILETMP=1
 316     HOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 317     @PYTHON_EXECUTABLE@ -c '
 318 import xml.etree.ElementTree as ET
 319 import sys
 320 import re
 321
 322 tree = ET.parse(sys.stdin)
 323
 324 for elem in tree.findall(".//host"):
 325     print(elem.attrib["id"])
 326
 327 for elem in tree.findall(".//cluster"):
 328     prefix = elem.attrib["prefix"]
 329     radical = elem.attrib["radical"]
 330     suffix = elem.attrib["suffix"]
 331     for r in radical.split(","):
 332         m = re.match("^([^-]*?)-([^-]*)$", r)
 333         if m:
 334             for i in range(int(m.group(1)), int(m.group(2))):
 335                 print(prefix + str(i) + suffix)
 336         else:
 337             print(prefix + r + suffix)
 338             ' < "${PLATFORM}" > "${HOSTFILE}"
 339 fi
 340 UNROLLEDHOSTFILETMP=0
 341
 342 # parse if our lines are terminated by :num_process
 343 if grep -q ':' "$HOSTFILE" ; then
 344     UNROLLEDHOSTFILETMP=1
 345     UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 346     @PYTHON_EXECUTABLE@ -c '
 347 import sys
 348 import re
 349
 350 for line in sys.stdin:
 351     m = re.match("(.*):(.*)", line)
 352     if m:
 353         for i in range(0, int(m.group(2))):
 354             print(m.group(1))
 355     else:
 356         print(line.strip())
 357 ' < "${HOSTFILE}"  > "${UNROLLEDHOSTFILE}"
 358     if [ ${HOSTFILETMP} = 1 ] ; then
 359         rm "${HOSTFILE}"
 360         HOSTFILETMP=0
 361     fi
 362     HOSTFILE=$UNROLLEDHOSTFILE
 363 fi
 364
 365 # Don't use wc -l to compute it to avoid issues with trailing \n at EOF
 366 hostfile_procs=$(grep -c "[a-zA-Z0-9]" "$HOSTFILE")
 367 if [ "${hostfile_procs}" = 0 ] ; then
 368     die "the hostfile '${HOSTFILE}' is empty"
 369 fi
 370
 371 if [ -z "${NUMPROCS}" ] ; then
 372     # Use the amount of processes in the hostfile as default value for the -np parameter
 373     NUMPROCS=$hostfile_procs
 374 fi
 375
 376 if [ "${NUMPROCS}" -gt "${hostfile_procs}" ] ; then
 377     echo "You requested to use ${NUMPROCS} ranks, but there is only ${hostfile_procs} processes in your hostfile..." >&2
 378 fi
 379
 380 ##-------------------------------- DEFAULT or SPECIFIED PLATFORM --------------------------------------
 381 if [ -z "${PLATFORM}" ]; then
 382     PLATFORMTMP="$(mktemp smpitmp-platfXXXXXX)"
 383
 384     cat > "${PLATFORMTMP}" <<PLATFORMHEAD
 385 <?xml version='1.0'?>
 386 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 387 <platform version="4.1">
 388 <zone id="AS0" routing="Full">
 389 PLATFORMHEAD
 390
 391     i=${NUMPROCS}
 392     while [ "$i" -gt 0 ]; do
 393         {
 394         echo "  <host id=\"host$i\" speed=\"${SPEED}\"/>"
 395         echo "  <link id=\"loop$i\" bandwidth=\"${LOOPBACK_BANDWIDTH}\" latency=\"${LOOPBACK_LATENCY}\"/>"
 396         echo "  <link id=\"link$i\" bandwidth=\"${NETWORK_BANDWIDTH}\" latency=\"${NETWORK_LATENCY}\"/>"
 397         } >> "${PLATFORMTMP}"
 398         i=$((i - 1))
 399     done
 400
 401     i=${NUMPROCS}
 402     while [ "$i" -gt 0 ]; do
 403         j=${NUMPROCS}
 404         while [ "$j" -gt 0 ]; do
 405             if [ "$i" -eq "$j" ]; then
 406                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"loop$i\"/></route>" >> "${PLATFORMTMP}"
 407             else
 408                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"link$i\"/><link_ctn id=\"link$j\"/></route>" >> "${PLATFORMTMP}"
 409             fi
 410             j=$((j - 1))
 411         done
 412         i=$((i - 1))
 413     done
 414
 415     cat >> "${PLATFORMTMP}" <<PLATFORMFOOT
 416 </zone>
 417 </platform>
 418 PLATFORMFOOT
 419
 420 else
 421     PLATFORMTMP=${PLATFORM}
 422 fi
 423 ##-------------------------------- end DEFAULT or SPECIFIED PLATFORM --------------------------------------
 424 ##-------------------------------- DEFAULT APPLICATION --------------------------------------
 425 APPLICATIONTMP="$(mktemp smpitmp-appXXXXXX)"
 426 #APPLICATIONTMP="app.xml"
 427
 428 cat > "${APPLICATIONTMP}" <<APPLICATIONHEAD
 429 <?xml version='1.0'?>
 430 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 431 <platform version="4.1">
 432 APPLICATIONHEAD
 433
 434 ##---- cache hostnames of hostfile---------------
 435 if [ -n "${HOSTFILE}" ] && [ -f "${HOSTFILE}" ]; then
 436     hostnames=$(< "${HOSTFILE}" tr '\n\r' '  ')
 437 fi
 438
 439 if [ -n "${APP_TRACES}" ]; then
 440     if [ -f "${APP_TRACES}" ]; then
 441         hosttraces=$(< "${APP_TRACES}" tr '\n\r' '  ' )
 442         NUMTRACES=$(< "${APP_TRACES}" wc -l)
 443         REPLAY=1
 444     else
 445         printf "File not found: %s\n" "${APP_TRACES:-\${APP_TRACES\}}" >&2
 446         exit 1
 447     fi
 448 fi
 449
 450 ##----------------------------------------------------------
 451 ##  generate application.xml with hostnames from hostfile:
 452 ##  the name of host_i (1<=i<=p, where -np p) is the line i in hostfile (where -hostfile hostfile), or "host$i" if
 453 ##  hostfile has less than i lines.
 454 ##----------------------------------------------------------
 455
 456 HAVE_SEQ="$(which seq 2>/dev/null)"
 457
 458 if [ -n "${HAVE_SEQ}" ]; then
 459     SEQ=$(${HAVE_SEQ} 0 $(( NUMPROCS - 1)))
 460 else
 461     cnt=0
 462     while [ $cnt -lt "${NUMPROCS}" ] ; do
 463         SEQ="$SEQ $cnt"
 464         cnt=$((cnt + 1));
 465     done
 466 fi
 467
 468 set -- $hostnames
 469
 470 ##---- generate <actor> tags------------------------------
 471 #prepare arguments at once
 472 for ARG in $PROC_ARGS; do
 473   XML_ARGS="${XML_ARGS}""<argument value=\"${ARG}\"/>
 474 "
 475 done
 476
 477 for i in ${SEQ}
 478 do
 479     j=$(( i % hostfile_procs + 1 ))
 480     host=$(eval "echo \${$j}")
 481
 482     ##---- optional display of ranks to actor mapping
 483     if [ ${MAPOPT} = 1 ]; then
 484       echo "[rank $i] -> $host"
 485     fi
 486     {
 487     echo "  <actor host=\"${host}\" function=\"$i\"> <!-- function name used only for logging -->
 488     <prop id=\"instance_id\" value=\"smpirun\"/>
 489     <prop id=\"rank\" value=\"$i\"/>"
 490     if [ ${REPLAY} = 1 ]; then
 491         echo "    <prop id=\"smpi_replay\" value=\"true\"/>"
 492         if  [ "${NUMTRACES}" -gt 1 ]; then
 493             echo "    <argument value=\"$(echo "$hosttraces"|cut -d' ' -f$j)\"/>"
 494         else
 495             echo "    <argument value=\"$(echo "$hosttraces"|cut -d' ' -f1)\"/>"
 496         fi
 497     else
 498     echo "${XML_ARGS}"
 499     fi
 500     echo "  </actor>"
 501     }    >> "${APPLICATIONTMP}"
 502 done
 503
 504 cat >> "${APPLICATIONTMP}" <<APPLICATIONFOOT
 505 </platform>
 506 APPLICATIONFOOT
 507 ##-------------------------------- end DEFAULT APPLICATION --------------------------------------
 508 ##---------------------- SMPI TRACING OPTIONS ---------------------------------
 509 if [ -n "${TRACE_ACTIVE}" ]; then
 510     #define trace filename
 511     if [ -n "${TRACE_TI_ACTIVE}" ]; then
 512         if [ -z "${TRACE_FILENAME}" ]; then
 513             TRACE_FILENAME="smpi_simgrid.txt"
 514         fi
 515         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes --cfg=tracing/smpi/format:TI --cfg=tracing/smpi/computing:yes"
 516     else
 517         if [ -z "${TRACE_FILENAME}" ]; then
 518             TRACE_FILENAME="smpi_simgrid.trace"
 519         fi
 520         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes"
 521     fi
 522
 523     if [ -n "${TRACE_COMMENT}" ]; then
 524         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment:${TRACE_COMMENT}"
 525     fi
 526
 527     if [ -n "${TRACE_COMMENT_FILE}" ]; then
 528         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment-file:${TRACE_COMMENT_FILE}"
 529     fi
 530
 531     if [ -n "${TRACE_GROUPED}" ]; then
 532         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/smpi/group:yes"
 533     fi
 534
 535     if [ -n "${TRACE_RESOURCE}" ]; then
 536         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/categorized:yes --cfg=tracing/uncategorized:yes"
 537     fi
 538 fi
 539 ##---------------------- end SMPI TRACING OPTIONS ---------------------------------
 540
 541 # Do not remove, this variable may be used by user code (e.g. StarPU)
 542 export SMPI_GLOBAL_SIZE=${NUMPROCS}
 543 if [ -n "${KEEP}" ] && [ -z "${QUIET}" ] ; then
 544     echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" "${APPLICATIONTMP}"
 545     if [ ${HOSTFILETMP} = 1 ] ; then
 546         echo "Generated hostfile ${HOSTFILE} kept."
 547     fi
 548     if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 549         echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 550     fi
 551 fi
 552
 553 # Execute the process
 554 #
 555 # The shell still need to be alive for the duration in order to do some cleanup after the process.
 556 #
 557 # We are going through great lengths in order to both keep stdin and be able to handle signals:
 558 #
 559 # * The job is launched in the background in order to be able to handle signals.
 560 #
 561 # * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command
 562 #   is launched in the background: this can be overridden in bash but not in standard bourne shell.
 563 exec 3<&0
 564 ${WRAPPER} "@SMPIMAIN@" "${EXEC}" ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} "${PLATFORMTMP}" "${APPLICATIONTMP}" <&3 3>&- &
 565 pid=$!
 566 exec 3>&-
 567 wait $pid
 568 status=$?
 569 # With dash on Windows WSL/Ubuntu, "wait" sometimes returns early with an exit
 570 # status of 128. Try again.
 571 while test $status -eq 128 && kill -0 $pid 2>/dev/null; do
 572     wait $pid
 573     status=$?
 574 done
 575 pid=""
 576
 577 # Keep temporary files on failures to help debugging
 578 #
 579 if [ ${status} -ne 0 ] ; then
 580     if [ -z "${KEEP}" ] && [ -z "${QUIET}" ]; then
 581         echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" "${APPLICATIONTMP}"
 582         if [ ${HOSTFILETMP} = 1 ] ; then
 583             echo "Generated hostfile ${HOSTFILE} kept."
 584         fi
 585         if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 586             echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 587         fi
 588         KEEP=true
 589     fi
 590     echo "Execution failed with code ${status}."
 591 fi
 592
 593 smpirun_cleanup
 594
 595 exit $status