src/smpi/smpirun.in

   1 #!/usr/bin/env sh
   2
   3 # Copyright (c) 2007-2021. The SimGrid Team. All rights reserved.
   4
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the license (GNU LGPL) which comes with this package.
   7
   8 @CMAKE_SMPI_COMMAND@
   9
  10 SIMGRID_VERSION="@SIMGRID_VERSION_STRING@"
  11 SIMGRID_GITHASH="@SIMGRID_GITHASH@"
  12
  13 DEFAULT_LOOPBACK_BANDWIDTH="498000000Bps"
  14 DEFAULT_LOOPBACK_LATENCY="0.000004s"
  15 DEFAULT_NETWORK_BANDWIDTH="$((26 * 1024 * 1024))Bps"
  16 DEFAULT_NETWORK_LATENCY="0.000005s"
  17 DEFAULT_SPEED="100flops"
  18
  19 LOOPBACK_BANDWIDTH="${DEFAULT_LOOPBACK_BANDWIDTH}"
  20 LOOPBACK_LATENCY="${DEFAULT_LOOPBACK_LATENCY}"
  21 NETWORK_BANDWIDTH="${DEFAULT_NETWORK_BANDWIDTH}"
  22 NETWORK_LATENCY="${DEFAULT_NETWORK_LATENCY}"
  23 SPEED="${DEFAULT_SPEED}"
  24
  25 PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-@HAVE_PRIVATIZATION@}"
  26
  27 SIMOPTS="--cfg=surf/precision:1e-9 --cfg=network/model:SMPI"
  28
  29 SMPITMPDIR="$(dirname $(mktemp -u))"
  30
  31 #usage to print the way this script should be called
  32 usage () {
  33     cat <<EOF
  34 Usage: $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> program [program-options]
  35        $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> -replay <tracefile> [program] [program-options]
  36 Options:
  37   -analyze                   # show information about allocations and timings at the end of execution
  38   -keep-temps                # don't remove the generated files after execution
  39   -wrapper <command>         # use command to run the program (e.g. "valgrind" or "gdb --args")
  40   -gdb                       # run within GDB (-wrapper "gdb --args" -keep-temps)
  41   -lldb                      # run within LLDB (-wrapper "lldb --" -keep-temps)
  42   -vgdb                      # run within Valgrind+GDB (-wrapper "valgrind --vgdb=yes --vgdb-error=0" -keep-temps)
  43   -map                       # display the machine on which each process rank is mapped
  44   -np <numprocs>             # use that amount of processes from the hostfile.
  45                              # By default, all processes of the hostfile are used.
  46   -no-privatize              # Disable the globals privatization, that is activated by default
  47   -tmpdir                    # Directory used to store temporary files. Defaults to system's.
  48   -trace-ti                  # activate time independent tracing (for replay, default in smpi_simgrid.txt)
  49   -trace                     # activate tracing (Paje, default in smpi_simgrid.trace)
  50   -trace-comment <comment>   # put a comment on the top of the trace file
  51   -trace-comment-file <file> # put file contents on the top of the trace file as comment
  52   -trace-grouped             # group MPI processes by location
  53   -trace-resource            # trace resource utilization
  54   -trace-file <tracefile>    # name of the tracefile (simgrid_smpi.trace)
  55   -replay <tracefile>        # replays a trace instead of actually executing an application
  56
  57   -version                   # Displays the SimGrid version (human readable)
  58   -git-version               # Displays the git hash of SimGrid
  59
  60 or (deprecated usage):
  61   $0 [-keep-temps] [-np <numprocs>] [-bandwidth <bytes/sec>] [-latency <secs>] program [program-options]
  62
  63 EOF
  64 }
  65
  66 #check if we have at least one parameter
  67 if [ $# -eq 0 ]
  68 then
  69     usage
  70     exit
  71 fi
  72
  73 WRAPPER=""
  74 HOSTFILE=""
  75 HOSTFILETMP=0
  76 MAPOPT=0
  77 REPLAY=0
  78 QUIET=""
  79
  80 unset pid
  81
  82 trapped_signals="HUP INT QUIT ILL ABRT SEGV FPE ALRM TERM USR1 USR2 BUS"
  83
  84 die () {
  85     printf '[%s] ** error: %s. Aborting.\n' "$(basename "$0")" "$*" >&2
  86     exit 1
  87 }
  88
  89 smpirun_cleanup()
  90 {
  91   if [ -z "${KEEP}" ] ; then
  92       if [ -z "${PLATFORM}" ] && [ -n "$PLATFORMTMP" ]; then
  93         rm -f "${PLATFORMTMP}"
  94         PLATFORMTMP=""
  95       fi
  96       if [ ${HOSTFILETMP} = 1 ] && [ -n "$HOSTFILE" ] ; then
  97           rm -f "${HOSTFILE}"
  98           HOSTFILE=""
  99       fi
 100       if [ "${UNROLLEDHOSTFILETMP}" = 1 ] && [ -n "$UNROLLEDHOSTFILE" ] ; then
 101           rm -f "${UNROLLEDHOSTFILE}"
 102           UNROLLEDHOSTFILE=""
 103       fi
 104       if [ -n "${APPLICATIONTMP}" ]; then
 105         rm -f "${APPLICATIONTMP}"
 106         APPLICATIONTMP=""
 107       fi
 108   fi
 109 }
 110
 111 smpirun_trap() {
 112   local sig
 113   sig="$1"
 114
 115   # Cleanup and kill the child process:
 116   smpirun_cleanup
 117   if [ -n "$pid" ]; then
 118     kill -TERM "$pid"
 119   fi
 120   unset pid
 121
 122   # Raise the same signal again (remove the traps first):
 123   trap - $trapped_signals
 124   kill -"$sig" $$
 125
 126   # This should never happen:
 127   kill -ABRT $$
 128   kill -TERM $$
 129 }
 130
 131 for s in $trapped_signals; do
 132   trap "smpirun_trap $s" "$s"
 133 done
 134
 135 while true; do
 136     case "$1" in
 137         "-np" | "-n")
 138             NUMPROCS="$2"
 139             shift 2
 140             ;;
 141         "-bandwidth")
 142             NETWORK_BANDWIDTH="$2"
 143             shift 2
 144             ;;
 145         "-latency")
 146             NETWORK_LATENCY="$2"
 147             shift 2
 148             ;;
 149         "-platform")
 150             PLATFORM="$2"
 151             if [ ! -f "${PLATFORM}" ]; then
 152                 die "the file '${PLATFORM}' does not exist"
 153             fi
 154             shift 2
 155             ;;
 156         "-hostfile" | "-machinefile")
 157             HOSTFILE="$2"
 158             if [ ! -f "${HOSTFILE}" ]; then
 159                 die "the file '${HOSTFILE}' does not exist"
 160             fi
 161             shift 2
 162             ;;
 163         "-replay")
 164             APP_TRACES="$2"
 165             shift 2
 166             ;;
 167         "-no-privatize")
 168             PRIVATIZE="--cfg=smpi/privatization:no"
 169             shift 1
 170             ;;
 171         "-map")
 172             MAPOPT=1
 173             shift 1
 174             ;;
 175         "-tmpdir")
 176             SMPITMPDIR="$2"
 177             shift 1
 178             ;;
 179         "-trace")
 180             TRACE_ACTIVE="true"
 181             shift 1
 182             ;;
 183         "-trace-ti")
 184             TRACE_ACTIVE="true"
 185             TRACE_TI_ACTIVE="true"
 186             shift 1
 187             ;;
 188         "-trace-comment")
 189             TRACE_COMMENT="$2"
 190             shift 2
 191             ;;
 192         "-trace-comment-file")
 193             TRACE_COMMENT_FILE="$2"
 194             shift 2
 195             ;;
 196         "-trace-file")
 197             TRACE_FILENAME="$2"
 198             shift 2
 199             ;;
 200         "-trace-grouped")
 201             TRACE_GROUPED="true"
 202             shift 1
 203             ;;
 204         "-trace-resource")
 205             TRACE_RESOURCE="true"
 206             shift 1
 207             ;;
 208         "-keep-temps")
 209             KEEP="true"
 210             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 211             shift 1
 212             ;;
 213         "-quiet")
 214             QUIET="true"
 215             shift 1
 216             ;;
 217         "-wrapper")
 218             WRAPPER="$2"
 219             shift 2
 220             ;;
 221         "-gdb")
 222             WRAPPER="gdb --args"
 223             KEEP="true"
 224             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 225             shift 1
 226             ;;
 227         "-vgdb")
 228             WRAPPER="valgrind --vgdb=yes --vgdb-error=0"
 229             KEEP="true"
 230             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 231             shift 1
 232             ;;
 233         "-lldb")
 234             WRAPPER="lldb --"
 235             KEEP="true"
 236             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 237             shift 1
 238             ;;
 239         "-analyze")
 240             SIMOPTS="$SIMOPTS --cfg=smpi/display-timing:yes --cfg=smpi/display-allocs:yes"
 241             shift 1
 242             ;;
 243         "-help" | "--help" | "-h")
 244             usage
 245             exit 0
 246             ;;
 247         "-version" | "--version" | "-v")
 248             printf '%b\n' "$SIMGRID_VERSION"
 249             exit 0
 250             ;;
 251         "-git-version" | "--git-version")
 252             printf '%b\n' "$SIMGRID_GITHASH"
 253             exit 0
 254             ;;
 255         "--cfg="*|"--log="*)
 256             for OPT in ${1#*=}
 257             do
 258                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 259             done
 260             shift 1
 261             ;;
 262         "-foreground")
 263             # Nothing to do, compatibility.
 264             shift 1
 265             ;;
 266         *)
 267             break
 268             ;;
 269     esac
 270 done
 271
 272 #setup tmp dir
 273 SIMOPTS="$SIMOPTS --cfg=smpi/tmpdir:$SMPITMPDIR"
 274 export LD_LIBRARY_PATH="$SMPITMPDIR:$LD_LIBRARY_PATH"
 275
 276 if [ -n "${APP_TRACES}" ] ; then
 277     if [ $# -eq 0 ] ; then
 278         EXEC="@SMPIREPLAYMAIN@"
 279     else
 280         EXEC="$1"
 281         shift
 282     fi
 283 else
 284     # check if we still have at least one parameter beyond options
 285     if [ $# -eq 0 ]
 286     then
 287         echo "Error: no program to execute!"
 288         usage
 289         exit
 290     fi
 291
 292     EXEC="$1"
 293     shift
 294 fi
 295
 296 # steal --cfg and --logs options
 297 while [ $# -gt 0 ]; do
 298     case "$1" in
 299         "--cfg="*|"--log="*)
 300             for OPT in ${1#*=}
 301             do
 302                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 303             done
 304             shift 1
 305             ;;
 306         *)
 307             PROC_ARGS="${PROC_ARGS:+$PROC_ARGS }$1"
 308             shift
 309             ;;
 310     esac
 311 done
 312
 313 if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then
 314     echo "No hostfile nor platform specified."
 315     usage
 316     exit 1
 317 fi
 318
 319 if [ -z "${HOSTFILE}" ] ; then
 320     HOSTFILETMP=1
 321     HOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 322     @PYTHON_EXECUTABLE@ -c '
 323 import xml.etree.ElementTree as ET
 324 import sys
 325 import re
 326
 327 tree = ET.parse(sys.stdin)
 328
 329 for elem in tree.findall(".//host"):
 330     print(elem.attrib["id"])
 331
 332 for elem in tree.findall(".//cluster"):
 333     prefix = elem.attrib["prefix"]
 334     radical = elem.attrib["radical"]
 335     suffix = elem.attrib["suffix"]
 336     for r in radical.split(","):
 337         m = re.match("^([^-]*?)-([^-]*)$", r)
 338         if m:
 339             for i in range(int(m.group(1)), int(m.group(2))):
 340                 print(prefix + str(i) + suffix)
 341         else:
 342             print(prefix + r + suffix)
 343             ' < "${PLATFORM}" > "${HOSTFILE}"
 344 fi
 345 UNROLLEDHOSTFILETMP=0
 346
 347 # parse if our lines are terminated by :num_process
 348 if grep -q ':' "$HOSTFILE" ; then
 349     UNROLLEDHOSTFILETMP=1
 350     UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 351     @PYTHON_EXECUTABLE@ -c '
 352 import sys
 353 import re
 354
 355 for line in sys.stdin:
 356     m = re.match("(.*):(.*)", line)
 357     if m:
 358         for i in range(0, int(m.group(2))):
 359             print(m.group(1))
 360     else:
 361         print(line.strip())
 362 ' < "${HOSTFILE}"  > "${UNROLLEDHOSTFILE}"
 363     if [ ${HOSTFILETMP} = 1 ] ; then
 364         rm "${HOSTFILE}"
 365         HOSTFILETMP=0
 366     fi
 367     HOSTFILE=$UNROLLEDHOSTFILE
 368 fi
 369
 370 # Don't use wc -l to compute it to avoid issues with trailing \n at EOF
 371 hostfile_procs=$(grep -c "[a-zA-Z0-9]" "$HOSTFILE")
 372 if [ "${hostfile_procs}" = 0 ] ; then
 373     die "the hostfile '${HOSTFILE}' is empty"
 374 fi
 375
 376 if [ -z "${NUMPROCS}" ] ; then
 377     # Use the amount of processes in the hostfile as default value for the -np parameter
 378     NUMPROCS=$hostfile_procs
 379 fi
 380
 381 if [ "${NUMPROCS}" -gt "${hostfile_procs}" ] ; then
 382     echo "You requested to use ${NUMPROCS} ranks, but there is only ${hostfile_procs} processes in your hostfile..." >&2
 383 fi
 384
 385 ##-------------------------------- DEFAULT or SPECIFIED PLATFORM --------------------------------------
 386 if [ -z "${PLATFORM}" ]; then
 387     PLATFORMTMP="$(mktemp smpitmp-platfXXXXXX)"
 388
 389     cat > "${PLATFORMTMP}" <<PLATFORMHEAD
 390 <?xml version='1.0'?>
 391 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 392 <platform version="4.1">
 393 <zone id="AS0" routing="Full">
 394 PLATFORMHEAD
 395
 396     i=${NUMPROCS}
 397     while [ "$i" -gt 0 ]; do
 398         {
 399         echo "  <host id=\"host$i\" speed=\"${SPEED}\"/>"
 400         echo "  <link id=\"loop$i\" bandwidth=\"${LOOPBACK_BANDWIDTH}\" latency=\"${LOOPBACK_LATENCY}\"/>"
 401         echo "  <link id=\"link$i\" bandwidth=\"${NETWORK_BANDWIDTH}\" latency=\"${NETWORK_LATENCY}\"/>"
 402         } >> "${PLATFORMTMP}"
 403         i=$((i - 1))
 404     done
 405
 406     i=${NUMPROCS}
 407     while [ "$i" -gt 0 ]; do
 408         j=${NUMPROCS}
 409         while [ "$j" -gt 0 ]; do
 410             if [ "$i" -eq "$j" ]; then
 411                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"loop$i\"/></route>" >> "${PLATFORMTMP}"
 412             else
 413                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"link$i\"/><link_ctn id=\"link$j\"/></route>" >> "${PLATFORMTMP}"
 414             fi
 415             j=$((j - 1))
 416         done
 417         i=$((i - 1))
 418     done
 419
 420     cat >> "${PLATFORMTMP}" <<PLATFORMFOOT
 421 </zone>
 422 </platform>
 423 PLATFORMFOOT
 424
 425 else
 426     PLATFORMTMP=${PLATFORM}
 427 fi
 428 ##-------------------------------- end DEFAULT or SPECIFIED PLATFORM --------------------------------------
 429 ##-------------------------------- DEFAULT APPLICATION --------------------------------------
 430 APPLICATIONTMP="$(mktemp smpitmp-appXXXXXX)"
 431 #APPLICATIONTMP="app.xml"
 432
 433 cat > "${APPLICATIONTMP}" <<APPLICATIONHEAD
 434 <?xml version='1.0'?>
 435 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 436 <platform version="4.1">
 437 APPLICATIONHEAD
 438
 439 ##---- cache hostnames of hostfile---------------
 440 if [ -n "${HOSTFILE}" ] && [ -f "${HOSTFILE}" ]; then
 441     hostnames=$(< "${HOSTFILE}" tr '\n\r' '  ')
 442 fi
 443
 444 if [ -n "${APP_TRACES}" ]; then
 445     if [ -f "${APP_TRACES}" ]; then
 446         hosttraces=$(< "${APP_TRACES}" tr '\n\r' '  ' )
 447         NUMTRACES=$(< "${APP_TRACES}" wc -l)
 448         REPLAY=1
 449     else
 450         printf "File not found: %s\n" "${APP_TRACES:-\${APP_TRACES\}}" >&2
 451         exit 1
 452     fi
 453 fi
 454
 455 ##----------------------------------------------------------
 456 ##  generate application.xml with hostnames from hostfile:
 457 ##  the name of host_i (1<=i<=p, where -np p) is the line i in hostfile (where -hostfile hostfile), or "host$i" if
 458 ##  hostfile has less than i lines.
 459 ##----------------------------------------------------------
 460
 461 HAVE_SEQ="$(which seq 2>/dev/null)"
 462
 463 if [ -n "${HAVE_SEQ}" ]; then
 464     SEQ=$(${HAVE_SEQ} 0 $(( NUMPROCS - 1)))
 465 else
 466     cnt=0
 467     while [ $cnt -lt "${NUMPROCS}" ] ; do
 468         SEQ="$SEQ $cnt"
 469         cnt=$((cnt + 1));
 470     done
 471 fi
 472
 473 set -- $hostnames
 474
 475 ##---- generate <actor> tags------------------------------
 476 #prepare arguments at once
 477 for ARG in $PROC_ARGS; do
 478   XML_ARGS="${XML_ARGS}""<argument value=\"${ARG}\"/>
 479 "
 480 done
 481
 482 for i in ${SEQ}
 483 do
 484     j=$(( i % hostfile_procs + 1 ))
 485     host=$(eval "echo \${$j}")
 486
 487     ##---- optional display of ranks to actor mapping
 488     if [ ${MAPOPT} = 1 ]; then
 489       echo "[rank $i] -> $host"
 490     fi
 491     {
 492     echo "  <actor host=\"${host}\" function=\"$i\"> <!-- function name used only for logging -->
 493     <prop id=\"instance_id\" value=\"smpirun\"/>
 494     <prop id=\"rank\" value=\"$i\"/>"
 495     if [ ${REPLAY} = 1 ]; then
 496         echo "    <prop id=\"smpi_replay\" value=\"true\"/>"
 497         if  [ "${NUMTRACES}" -gt 1 ]; then
 498             # Give each process its trace file (as an argument)
 499             echo "    <argument value=\"$(echo "$hosttraces"|cut -d' ' -f$j)\"/>"
 500         elif [ ${i} = 0 ]; then
 501             # Give the shared trace file to rank 0 (as a property)
 502             echo "    <prop id=\"tracefile\" value=\"$(echo "$hosttraces"|cut -d' ' -f1)\"/>"
 503         fi
 504     else
 505     echo "${XML_ARGS}"
 506     fi
 507     echo "  </actor>"
 508     }    >> "${APPLICATIONTMP}"
 509 done
 510
 511 cat >> "${APPLICATIONTMP}" <<APPLICATIONFOOT
 512 </platform>
 513 APPLICATIONFOOT
 514 ##-------------------------------- end DEFAULT APPLICATION --------------------------------------
 515 ##---------------------- SMPI TRACING OPTIONS ---------------------------------
 516 if [ -n "${TRACE_ACTIVE}" ]; then
 517     #define trace filename
 518     if [ -n "${TRACE_TI_ACTIVE}" ]; then
 519         if [ -z "${TRACE_FILENAME}" ]; then
 520             TRACE_FILENAME="smpi_simgrid.txt"
 521         fi
 522         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes --cfg=tracing/smpi/format:TI --cfg=tracing/smpi/computing:yes"
 523     else
 524         if [ -z "${TRACE_FILENAME}" ]; then
 525             TRACE_FILENAME="smpi_simgrid.trace"
 526         fi
 527         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes"
 528     fi
 529
 530     if [ -n "${TRACE_COMMENT}" ]; then
 531         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment:${TRACE_COMMENT}"
 532     fi
 533
 534     if [ -n "${TRACE_COMMENT_FILE}" ]; then
 535         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment-file:${TRACE_COMMENT_FILE}"
 536     fi
 537
 538     if [ -n "${TRACE_GROUPED}" ]; then
 539         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/smpi/group:yes"
 540     fi
 541
 542     if [ -n "${TRACE_RESOURCE}" ]; then
 543         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/categorized:yes --cfg=tracing/uncategorized:yes"
 544     fi
 545 fi
 546 ##---------------------- end SMPI TRACING OPTIONS ---------------------------------
 547
 548 # Do not remove, this variable may be used by user code (e.g. StarPU)
 549 export SMPI_GLOBAL_SIZE=${NUMPROCS}
 550 if [ -n "${KEEP}" ] && [ -z "${QUIET}" ] ; then
 551     echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" "${APPLICATIONTMP}"
 552     if [ ${HOSTFILETMP} = 1 ] ; then
 553         echo "Generated hostfile ${HOSTFILE} kept."
 554     fi
 555     if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 556         echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 557     fi
 558 fi
 559
 560 # Execute the process
 561 #
 562 # The shell still need to be alive for the duration in order to do some cleanup after the process.
 563 #
 564 # We are going through great lengths in order to both keep stdin and be able to handle signals:
 565 #
 566 # * The job is launched in the background in order to be able to handle signals.
 567 #
 568 # * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command
 569 #   is launched in the background: this can be overridden in bash but not in standard bourne shell.
 570 exec 3<&0
 571 ${WRAPPER} "@SMPIMAIN@" "${EXEC}" ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} "${PLATFORMTMP}" "${APPLICATIONTMP}" <&3 3>&- &
 572 pid=$!
 573 exec 3>&-
 574 wait $pid
 575 status=$?
 576 # With dash on Windows WSL/Ubuntu, "wait" sometimes returns early with an exit
 577 # status of 128. Try again.
 578 while test $status -eq 128 && kill -0 $pid 2>/dev/null; do
 579     wait $pid
 580     status=$?
 581 done
 582 pid=""
 583
 584 # Keep temporary files on failures to help debugging
 585 #
 586 if [ ${status} -ne 0 ] ; then
 587     if [ -z "${KEEP}" ] && [ -z "${QUIET}" ]; then
 588         echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" "${APPLICATIONTMP}"
 589         if [ ${HOSTFILETMP} = 1 ] ; then
 590             echo "Generated hostfile ${HOSTFILE} kept."
 591         fi
 592         if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 593             echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 594         fi
 595         KEEP=true
 596     fi
 597     echo "Execution failed with code ${status}."
 598 fi
 599
 600 smpirun_cleanup
 601
 602 exit $status