src/smpi/smpirun.in

   1 #!/usr/bin/env sh
   2
   3 # Copyright (c) 2007-2019. The SimGrid Team. All rights reserved.
   4
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the license (GNU LGPL) which comes with this package.
   7
   8 @CMAKE_SMPI_COMMAND@
   9
  10 SIMGRID_VERSION="@SIMGRID_VERSION_STRING@"
  11 SIMGRID_GITHASH="@SIMGRID_GITHASH@"
  12
  13 DEFAULT_LOOPBACK_BANDWIDTH="498000000Bps"
  14 DEFAULT_LOOPBACK_LATENCY="0.000004s"
  15 DEFAULT_NETWORK_BANDWIDTH="$((26 * 1024 * 1024))Bps"
  16 DEFAULT_NETWORK_LATENCY="0.000005s"
  17 DEFAULT_NUMPROCS="4"
  18 DEFAULT_SPEED="100flops"
  19
  20 LOOPBACK_BANDWIDTH="${DEFAULT_LOOPBACK_BANDWIDTH}"
  21 LOOPBACK_LATENCY="${DEFAULT_LOOPBACK_LATENCY}"
  22 NETWORK_BANDWIDTH="${DEFAULT_NETWORK_BANDWIDTH}"
  23 NETWORK_LATENCY="${DEFAULT_NETWORK_LATENCY}"
  24 SPEED="${DEFAULT_SPEED}"
  25
  26 PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-@HAVE_PRIVATIZATION@}"
  27
  28 SIMOPTS="--cfg=surf/precision:1e-9 --cfg=network/model:SMPI"
  29
  30 #usage to print the way this script should be called
  31 usage () {
  32     cat <<EOF
  33 Usage: $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> program [program-options]
  34        $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> -replay <tracefile> [program] [program-options]
  35 Options:
  36   -keep-temps                # don't remove the generated files after execution
  37   -wrapper <command>         # use command to run the program (e.g. "valgrind" or "gdb --args")
  38   -gdb                       # run within GDB (-wrapper "gdb --args" -keep-temps)
  39   -lldb                      # run within LLDB (-wrapper "lldb --" -keep-temps)
  40   -vgdb                      # run within Valgrind+GDB (-wrapper "valgrind --vgdb=yes --vgdb-error=0" -keep-temps)
  41   -map                       # display the machine on which each process rank is mapped
  42   -np <numprocs>             # use that amount of processes from the hostfile.
  43                              # By default, all processes of the hostfile are used.
  44   -no-privatize              # Disable the globals privatization, that is activated by default
  45   -trace-ti                  # activate time independent tracing (for replay, default in smpi_simgrid.txt)
  46   -trace                     # activate tracing (Paje, default in smpi_simgrid.trace)
  47   -trace-comment <comment>   # put a comment on the top of the trace file
  48   -trace-comment-file <file> # put file contents on the top of the trace file as comment
  49   -trace-grouped             # group MPI processes by location
  50   -trace-resource            # trace resource utilization
  51   -trace-file <tracefile>    # name of the tracefile (simgrid_smpi.trace)
  52   -replay <tracefile>        # replays a trace instead of actually executing an application
  53
  54   -version                   # Displays the SimGrid version (human readable)
  55   -git-version               # Displays the git hash of SimGrid
  56
  57 or (deprecated usage):
  58   $0 [-keep-temps] [-np <numprocs>] [-bandwidth <bytes/sec>] [-latency <secs>] program [program-options]
  59
  60 EOF
  61 }
  62
  63 #check if we have at least one parameter
  64 if [ $# -eq 0 ]
  65 then
  66     usage
  67     exit
  68 fi
  69
  70 WRAPPER=""
  71 HOSTFILE=""
  72 HOSTFILETMP=0
  73 MAPOPT=0
  74 REPLAY=0
  75 QUIET=""
  76
  77 unset pid
  78
  79 trapped_signals="HUP INT QUIT ILL ABRT SEGV FPE ALRM TERM USR1 USR2 BUS"
  80
  81 die () {
  82     printf '[%s] ** error: %s. Aborting.\n' "$(basename $0)" "$*" >&2
  83     exit 1
  84 }
  85
  86 smpirun_cleanup()
  87 {
  88   if [ -z "${KEEP}" ] ; then
  89       if [ -z "${PLATFORM}" -a -n "$PLATFORMTMP" ]; then
  90         rm -f ${PLATFORMTMP}
  91         PLATFORMTMP=""
  92       fi
  93       if [ ${HOSTFILETMP} = 1 -a -n "$HOSTFILE" ] ; then
  94           rm -f ${HOSTFILE}
  95           HOSTFILE=""
  96       fi
  97       if [ ${UNROLLEDHOSTFILETMP} = 1 -a -n "$UNROLLEDHOSTFILE" ] ; then
  98           rm -f ${UNROLLEDHOSTFILE}
  99           UNROLLEDHOSTFILE=""
 100       fi
 101       if [ -n ${APPLICATIONTMP} ]; then
 102         rm -f ${APPLICATIONTMP}
 103         APPLICATIONTMP=""
 104       fi
 105   fi
 106 }
 107
 108 smpirun_trap() {
 109   local sig
 110   sig="$1"
 111
 112   # Cleanup and kill the child process:
 113   smpirun_cleanup
 114   if ! [ -z "$pid" ]; then
 115     kill -TERM $pid
 116   fi
 117   unset pid
 118
 119   # Raise the same signal again (remove the traps first):
 120   trap - $trapped_signals
 121   kill -$sig $$
 122
 123   # This should never happen:
 124   kill -ABRT $$
 125   kill -TERM $$
 126 }
 127
 128 for s in $trapped_signals; do
 129   trap "smpirun_trap $s" $s
 130 done
 131
 132 while true; do
 133     case "$1" in
 134         "-np" | "-n")
 135             NUMPROCS="$2"
 136             shift 2
 137             ;;
 138         "-bandwidth")
 139             NETWORK_BANDWIDTH="$2"
 140             shift 2
 141             ;;
 142         "-latency")
 143             NETWORK_LATENCY="$2"
 144             shift 2
 145             ;;
 146         "-platform")
 147             PLATFORM="$2"
 148             if [ ! -f "${PLATFORM}" ]; then
 149                 die "the file '${PLATFORM}' does not exist"
 150             fi
 151             shift 2
 152             ;;
 153         "-hostfile" | "-machinefile")
 154             HOSTFILE="$2"
 155             if [ ! -f "${HOSTFILE}" ]; then
 156                 die "the file '${HOSTFILE}' does not exist"
 157             fi
 158             shift 2
 159             ;;
 160         "-replay")
 161             APP_TRACES="$2"
 162             shift 2
 163             ;;
 164         "-no-privatize")
 165             PRIVATIZE="--cfg=smpi/privatization:no"
 166             shift 1
 167             ;;
 168         "-map")
 169             MAPOPT=1
 170             shift 1
 171             ;;
 172         "-trace")
 173             TRACE_ACTIVE="true"
 174             shift 1
 175             ;;
 176         "-trace-ti")
 177             TRACE_ACTIVE="true"
 178             TRACE_TI_ACTIVE="true"
 179             shift 1
 180             ;;
 181         "-trace-comment")
 182             TRACE_COMMENT="$2"
 183             shift 2
 184             ;;
 185         "-trace-comment-file")
 186             TRACE_COMMENT_FILE="$2"
 187             shift 2
 188             ;;
 189         "-trace-file")
 190             TRACE_FILENAME="$2"
 191             shift 2
 192             ;;
 193         "-trace-grouped")
 194             TRACE_GROUPED="true"
 195             shift 1
 196             ;;
 197         "-trace-resource")
 198             TRACE_RESOURCE="true"
 199             shift 1
 200             ;;
 201         "-keep-temps")
 202             KEEP="true"
 203             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 204             shift 1
 205             ;;
 206         "-quiet")
 207             QUIET="true"
 208             shift 1
 209             ;;
 210         "-wrapper")
 211             WRAPPER="$2"
 212             shift 2
 213             ;;
 214         "-gdb")
 215             WRAPPER="gdb --args"
 216             KEEP="true"
 217             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 218             shift 1
 219             ;;
 220         "-vgdb")
 221             WRAPPER="valgrind --vgdb=yes --vgdb-error=0"
 222             KEEP="true"
 223             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 224             shift 1
 225             ;;
 226         "-lldb")
 227             WRAPPER="lldb --"
 228             KEEP="true"
 229             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 230             shift 1
 231             ;;
 232         "-help" | "--help" | "-h")
 233             usage
 234             exit 0
 235             ;;
 236         "-version" | "--version" | "-v")
 237             printf '%b\n' "$SIMGRID_VERSION"
 238             exit 0
 239             ;;
 240         "-git-version" | "--git-version")
 241             printf '%b\n' "$SIMGRID_GITHASH"
 242             exit 0
 243             ;;
 244         "--cfg="*|"--log="*)
 245             for OPT in ${1#*=}
 246             do
 247                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 248             done
 249             shift 1
 250             ;;
 251         "-foreground")
 252             # Nothing to do, compatibility.
 253             shift 1
 254             ;;
 255         *)
 256             break
 257             ;;
 258     esac
 259 done
 260
 261 if [ -n "${APP_TRACES}" ] ; then
 262     if [ $# -eq 0 ] ; then
 263         EXEC="@SMPIREPLAYMAIN@"
 264     else
 265         EXEC="$1"
 266         shift
 267     fi
 268 else
 269     # check if we still have at least one parameter beyond options
 270     if [ $# -eq 0 ]
 271     then
 272         echo "Error: no program to execute!"
 273         usage
 274         exit
 275     fi
 276
 277     EXEC="$1"
 278     shift
 279 fi
 280
 281 # steal --cfg and --logs options
 282 while [ $# -gt 0 ]; do
 283     case "$1" in
 284         "--cfg="*|"--log="*)
 285             for OPT in ${1#*=}
 286             do
 287                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 288             done
 289             shift 1
 290             ;;
 291         *)
 292             PROC_ARGS="${PROC_ARGS:+$PROC_ARGS }$1"
 293             shift
 294             ;;
 295     esac
 296 done
 297
 298 if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then
 299     echo "No hostfile nor platform specified."
 300     usage
 301     exit 1
 302 fi
 303
 304 if [ -z "${HOSTFILE}" ] ; then
 305     HOSTFILETMP=1
 306     HOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 307     @PYTHON_EXECUTABLE@ -c '
 308 import xml.etree.ElementTree as ET
 309 import sys
 310 import re
 311
 312 tree = ET.parse(sys.stdin)
 313
 314 for elem in tree.findall(".//host"):
 315     print(elem.attrib["id"])
 316
 317 for elem in tree.findall(".//cluster"):
 318     prefix = elem.attrib["prefix"]
 319     radical = elem.attrib["radical"]
 320     suffix = elem.attrib["suffix"]
 321     for r in radical.split(","):
 322         m = re.match("^([^-]*?)-([^-]*)$", r)
 323         if m:
 324             for i in range(int(m.group(1)), int(m.group(2))):
 325                 print(prefix + str(i) + suffix)
 326         else:
 327             print(prefix + r + suffix)
 328             ' < ${PLATFORM} > ${HOSTFILE}
 329 fi
 330 UNROLLEDHOSTFILETMP=0
 331
 332 # parse if our lines are terminated by :num_process
 333 if grep -q ':' $HOSTFILE ; then
 334     UNROLLEDHOSTFILETMP=1
 335     UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 336     @PYTHON_EXECUTABLE@ -c '
 337 import sys
 338 import re
 339
 340 for line in sys.stdin:
 341     m = re.match("(.*):(.*)", line)
 342     if m:
 343         for i in range(0, int(m.group(2))):
 344             print(m.group(1))
 345     else:
 346         print(line.strip())
 347 ' < ${HOSTFILE}  > ${UNROLLEDHOSTFILE}
 348     if [ ${HOSTFILETMP} = 1 ] ; then
 349         rm ${HOSTFILE}
 350         HOSTFILETMP=0
 351     fi
 352     HOSTFILE=$UNROLLEDHOSTFILE
 353 fi
 354
 355 # Don't use wc -l to compute it to avoid issues with trailing \n at EOF
 356 hostfile_procs=$(grep -c "[a-zA-Z0-9]" $HOSTFILE)
 357 if [ ${hostfile_procs} = 0 ] ; then
 358     die "the hostfile '${HOSTFILE}' is empty"
 359 fi
 360
 361 if [ -z "${NUMPROCS}" ] ; then
 362     # Use the amount of processes in the hostfile as default value for the -np parameter
 363     NUMPROCS=$hostfile_procs
 364 fi
 365
 366 if [ ${NUMPROCS} -gt ${hostfile_procs} ] ; then
 367     echo "You requested to use ${NUMPROCS} ranks, but there is only ${hostfile_procs} processes in your hostfile..." >&2
 368 fi
 369
 370 ##-------------------------------- DEFAULT or SPECIFIED PLATFORM --------------------------------------
 371 if [ -z "${PLATFORM}" ]; then
 372     PLATFORMTMP="$(mktemp smpitmp-platfXXXXXX)"
 373
 374     cat > ${PLATFORMTMP} <<PLATFORMHEAD
 375 <?xml version='1.0'?>
 376 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 377 <platform version="4.1">
 378 <zone id="AS0" routing="Full">
 379 PLATFORMHEAD
 380
 381     i=${NUMPROCS}
 382     while [ $i -gt 0 ]; do
 383         echo "  <host id=\"host$i\" speed=\"${SPEED}\"/>" >> ${PLATFORMTMP}
 384         echo "  <link id=\"loop$i\" bandwidth=\"${LOOPBACK_BANDWIDTH}\" latency=\"${LOOPBACK_LATENCY}\"/>" >> ${PLATFORMTMP}
 385         echo "  <link id=\"link$i\" bandwidth=\"${NETWORK_BANDWIDTH}\" latency=\"${NETWORK_LATENCY}\"/>" >> ${PLATFORMTMP}
 386         i=$((i - 1))
 387     done
 388
 389     i=${NUMPROCS}
 390     while [ $i -gt 0 ]; do
 391         j=${NUMPROCS}
 392         while [ $j -gt 0 ]; do
 393             if [ $i -eq $j ]; then
 394                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"loop$i\"/></route>" >> ${PLATFORMTMP}
 395             else
 396                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"link$i\"/><link_ctn id=\"link$j\"/></route>" >> ${PLATFORMTMP}
 397             fi
 398             j=$((j - 1))
 399         done
 400         i=$((i - 1))
 401     done
 402
 403     cat >> ${PLATFORMTMP} <<PLATFORMFOOT
 404 </zone>
 405 </platform>
 406 PLATFORMFOOT
 407
 408 else
 409     PLATFORMTMP=${PLATFORM}
 410 fi
 411 ##-------------------------------- end DEFAULT or SPECIFIED PLATFORM --------------------------------------
 412 ##-------------------------------- DEFAULT APPLICATION --------------------------------------
 413 APPLICATIONTMP="$(mktemp smpitmp-appXXXXXX)"
 414 #APPLICATIONTMP="app.xml"
 415
 416 cat > ${APPLICATIONTMP} <<APPLICATIONHEAD
 417 <?xml version='1.0'?>
 418 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 419 <platform version="4.1">
 420 APPLICATIONHEAD
 421
 422 ##---- cache hostnames of hostfile---------------
 423 if [ -n "${HOSTFILE}" ] && [ -f ${HOSTFILE} ]; then
 424     hostnames=$(cat ${HOSTFILE} | tr '\n\r' '  ')
 425 fi
 426
 427 if [ -n "${APP_TRACES}" ]; then
 428     if [ -f "${APP_TRACES}" ]; then
 429         hosttraces=$(cat ${APP_TRACES} | tr '\n\r' '  ' )
 430         NUMTRACES=$(cat ${APP_TRACES} | wc -l)
 431         REPLAY=1
 432     else
 433         printf "File not found: %s\n" "${APP_TRACES:-\${APP_TRACES\}}" >&2
 434         exit 1
 435     fi
 436 fi
 437
 438 ##----------------------------------------------------------
 439 ##  generate application.xml with hostnames from hostfile:
 440 ##  the name of host_i (1<=i<=p, where -np p) is the line i in hostfile (where -hostfile hostfile), or "host$i" if
 441 ##  hostfile has less than i lines.
 442 ##----------------------------------------------------------
 443
 444 HAVE_SEQ="$(which seq 2>/dev/null)"
 445
 446 if [ -n "${HAVE_SEQ}" ]; then
 447     SEQ=$(${HAVE_SEQ} 0 $(( NUMPROCS - 1)))
 448 else
 449     cnt=0
 450     while [ $cnt -lt ${NUMPROCS} ] ; do
 451         SEQ="$SEQ $cnt"
 452         cnt=$((cnt + 1));
 453     done
 454 fi
 455
 456 set -- $hostnames
 457
 458 ##---- generate <actor> tags------------------------------
 459 #prepare arguments at once
 460 for ARG in $PROC_ARGS; do
 461   XML_ARGS="${XML_ARGS}""<argument value=\"${ARG}\"/>
 462 "
 463 done
 464
 465 for i in ${SEQ}
 466 do
 467     j=$(( i % hostfile_procs + 1 ))
 468     host=$(eval "echo \${$j}")
 469
 470     ##---- optional display of ranks to actor mapping
 471     if [ ${MAPOPT} = 1 ]; then
 472       echo "[rank $i] -> $host"
 473     fi
 474
 475     echo "  <actor host=\"${host}\" function=\"$i\"> <!-- function name used only for logging -->
 476     <prop id=\"instance_id\" value=\"smpirun\"/>
 477     <prop id=\"rank\" value=\"$i\"/>" >> ${APPLICATIONTMP}
 478     if [ ${REPLAY} = 1 ]; then
 479         echo "    <prop id=\"smpi_replay\" value=\"true\"/>" >> ${APPLICATIONTMP}
 480         if  [ ${NUMTRACES} -gt 1 ]; then
 481             echo "    <argument value=\"$(echo $hosttraces|cut -d' ' -f$j)\"/>" >> ${APPLICATIONTMP}
 482         else
 483             echo "    <argument value=\"$(echo $hosttraces|cut -d' ' -f1)\"/>" >> ${APPLICATIONTMP}
 484         fi
 485     else
 486     echo ${XML_ARGS} >> ${APPLICATIONTMP}
 487     fi
 488     echo "  </actor>" >> ${APPLICATIONTMP}
 489 done
 490
 491 cat >> ${APPLICATIONTMP} <<APPLICATIONFOOT
 492 </platform>
 493 APPLICATIONFOOT
 494 ##-------------------------------- end DEFAULT APPLICATION --------------------------------------
 495 ##---------------------- SMPI TRACING OPTIONS ---------------------------------
 496 if [ -n "${TRACE_ACTIVE}" ]; then
 497     #define trace filename
 498     if [ -n "${TRACE_TI_ACTIVE}" ]; then
 499         if [ -z "${TRACE_FILENAME}" ]; then
 500             TRACE_FILENAME="smpi_simgrid.txt"
 501         fi
 502         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes --cfg=tracing/smpi/format:TI --cfg=tracing/smpi/computing:yes"
 503     else
 504         if [ -z "${TRACE_FILENAME}" ]; then
 505             TRACE_FILENAME="smpi_simgrid.trace"
 506         fi
 507         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes"
 508     fi
 509
 510     if [ -n "${TRACE_COMMENT}" ]; then
 511         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment:${TRACE_COMMENT}"
 512     fi
 513
 514     if [ -n "${TRACE_COMMENT_FILE}" ]; then
 515         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment-file:${TRACE_COMMENT_FILE}"
 516     fi
 517
 518     if [ -n "${TRACE_GROUPED}" ]; then
 519         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/smpi/group:yes"
 520     fi
 521
 522     if [ -n "${TRACE_RESOURCE}" ]; then
 523         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/categorized:yes --cfg=tracing/uncategorized:yes"
 524     fi
 525 fi
 526 ##---------------------- end SMPI TRACING OPTIONS ---------------------------------
 527
 528 # Do not remove, this variable may be used by user code (e.g. StarPU)
 529 export SMPI_GLOBAL_SIZE=${NUMPROCS}
 530 if [ -n "${KEEP}" ] ; then
 531     echo ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP}
 532     if [ ${HOSTFILETMP} = 1 ] ; then
 533         echo "Generated hostfile ${HOSTFILE} kept."
 534     fi
 535     if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 536         echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 537     fi
 538 fi
 539
 540 # Execute the process
 541 #
 542 # The shell still need to be alive for the duration in order to do some cleanup after the process.
 543 #
 544 # We are going through great lengths in order to both keep stdin and be able to handle signals:
 545 #
 546 # * The job is launched in the background in order to be able to handle signals.
 547 #
 548 # * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command
 549 #   is launched in the background: this can be overridden in bash but not in standard bourne shell.
 550 exec 3<&0
 551 ${WRAPPER} "@SMPIMAIN@" ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP} <&3 3>&- &
 552 pid=$!
 553 exec 3>&-
 554 wait $pid
 555 status=$?
 556 # With dash on Windows WSL/Ubuntu, "wait" sometimes returns early with an exit
 557 # status of 128. Try again.
 558 while test $status -eq 128 && kill -0 $pid 2>/dev/null; do
 559     wait $pid
 560     status=$?
 561 done
 562 pid=""
 563
 564 # Keep temporary files on failures to help debugging
 565 #
 566 if [ ${status} -ne 0 ] ; then
 567     if [ -z "${KEEP}" -a -z "${QUIET}" ]; then
 568         echo ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP}
 569         if [ ${HOSTFILETMP} = 1 ] ; then
 570             echo "Generated hostfile ${HOSTFILE} kept."
 571         fi
 572         if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 573             echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 574         fi
 575         KEEP=true
 576     fi
 577     echo "Execution failed with code ${status}."
 578 fi
 579
 580 smpirun_cleanup
 581
 582 exit $status