src/smpi/smpirun.in

   1 #!/usr/bin/env sh
   2
   3 # Copyright (c) 2007-2019. The SimGrid Team. All rights reserved.
   4
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the license (GNU LGPL) which comes with this package.
   7
   8 @CMAKE_SMPI_COMMAND@
   9
  10 SIMGRID_VERSION="@SIMGRID_VERSION_STRING@"
  11 SIMGRID_GITHASH="@SIMGRID_GITHASH@"
  12
  13 DEFAULT_LOOPBACK_BANDWIDTH="498000000Bps"
  14 DEFAULT_LOOPBACK_LATENCY="0.000004s"
  15 DEFAULT_NETWORK_BANDWIDTH="$((26 * 1024 * 1024))Bps"
  16 DEFAULT_NETWORK_LATENCY="0.000005s"
  17 DEFAULT_NUMPROCS="4"
  18 DEFAULT_SPEED="100flops"
  19
  20 LOOPBACK_BANDWIDTH="${DEFAULT_LOOPBACK_BANDWIDTH}"
  21 LOOPBACK_LATENCY="${DEFAULT_LOOPBACK_LATENCY}"
  22 NETWORK_BANDWIDTH="${DEFAULT_NETWORK_BANDWIDTH}"
  23 NETWORK_LATENCY="${DEFAULT_NETWORK_LATENCY}"
  24 SPEED="${DEFAULT_SPEED}"
  25
  26 PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-@HAVE_PRIVATIZATION@}"
  27
  28 SIMOPTS="--cfg=surf/precision:1e-9 --cfg=network/model:SMPI"
  29
  30 #usage to print the way this script should be called
  31 usage () {
  32     cat <<EOF
  33 Usage: $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> program [program-options]
  34        $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> -replay <tracefile> [program] [program-options]
  35 Options:
  36   -keep-temps                # don't remove the generated files after execution
  37   -wrapper <command>         # use command to run the program (e.g. "valgrind" or "gdb --args")
  38   -gdb                       # run within GDB (-wrapper "gdb --args" -keep-temps)
  39   -lldb                      # run within LLDB (-wrapper "lldb --" -keep-temps)
  40   -map                       # display the machine on which each process rank is mapped
  41   -np <numprocs>             # use that amount of processes from the hostfile.
  42                              # By default, all processes of the hostfile are used.
  43   -no-privatize              # Disable the globals privatization, that is activated by default
  44   -trace-ti                  # activate time independant tracing (for replay, default in smpi_simgrid.txt)
  45   -trace                     # activate tracing (Paje, default in smpi_simgrid.trace)
  46   -trace-comment <comment>   # put a comment on the top of the trace file
  47   -trace-comment-file <file> # put file contents on the top of the trace file as comment
  48   -trace-grouped             # group MPI processes by location
  49   -trace-resource            # trace resource utilization
  50   -trace-file <tracefile>    # name of the tracefile (simgrid_smpi.trace)
  51   -replay <tracefile>        # replays a trace instead of actually executing an application
  52
  53   -version                   # Displays the SimGrid version (human readable)
  54   -git-version               # Displays the git hash of SimGrid
  55
  56 or (deprecated usage):
  57   $0 [-keep-temps] [-np <numprocs>] [-bandwidth <bytes/sec>] [-latency <secs>] program [program-options]
  58
  59 EOF
  60 }
  61
  62 #check if we have at least one parameter
  63 if [ $# -eq 0 ]
  64 then
  65     usage
  66     exit
  67 fi
  68
  69 WRAPPER=""
  70 HOSTFILE=""
  71 HOSTFILETMP=0
  72 MAPOPT=0
  73 REPLAY=0
  74
  75 unset pid
  76
  77 trapped_signals="HUP INT QUIT ILL ABRT SEGV FPE ALRM TERM USR1 USR2 BUS"
  78
  79 die () {
  80     printf '[%s] ** error: %s. Aborting.\n' "$(basename $0)" "$*" >&2
  81     exit 1
  82 }
  83
  84 smpirun_cleanup()
  85 {
  86   if [ -z "${KEEP}" ] ; then
  87       if [ -z "${PLATFORM}" -a -n "$PLATFORMTMP" ]; then
  88         rm -f ${PLATFORMTMP}
  89         PLATFORMTMP=""
  90       fi
  91       if [ ${HOSTFILETMP} = 1 -a -n "$HOSTFILE" ] ; then
  92           rm -f ${HOSTFILE}
  93           HOSTFILE=""
  94       fi
  95       if [ ${UNROLLEDHOSTFILETMP} = 1 -a -n "$UNROLLEDHOSTFILE" ] ; then
  96           rm -f ${UNROLLEDHOSTFILE}
  97           UNROLLEDHOSTFILE=""
  98       fi
  99       if [ -n ${APPLICATIONTMP} ]; then
 100         rm -f ${APPLICATIONTMP}
 101         APPLICATIONTMP=""
 102       fi
 103   fi
 104 }
 105
 106 smpirun_trap() {
 107   local sig
 108   sig="$1"
 109
 110   # Cleanup and kill the child process:
 111   smpirun_cleanup
 112   if ! [ -z "$pid" ]; then
 113     kill -TERM $pid
 114   fi
 115   unset pid
 116
 117   # Raise the same signal again (remove the traps first):
 118   trap - $trapped_signals
 119   kill -$sig $$
 120
 121   # This should never happen:
 122   kill -ABRT $$
 123   kill -TERM $$
 124 }
 125
 126 for s in $trapped_signals; do
 127   trap "smpirun_trap $s" $s
 128 done
 129
 130 while true; do
 131     case "$1" in
 132         "-np" | "-n")
 133             NUMPROCS="$2"
 134             shift 2
 135             ;;
 136         "-bandwidth")
 137             NETWORK_BANDWIDTH="$2"
 138             shift 2
 139             ;;
 140         "-latency")
 141             NETWORK_LATENCY="$2"
 142             shift 2
 143             ;;
 144         "-platform")
 145             PLATFORM="$2"
 146             if [ ! -f "${PLATFORM}" ]; then
 147                 die "the file '${PLATFORM}' does not exist"
 148             fi
 149             shift 2
 150             ;;
 151         "-hostfile" | "-machinefile")
 152             HOSTFILE="$2"
 153             if [ ! -f "${HOSTFILE}" ]; then
 154                 die "the file '${HOSTFILE}' does not exist"
 155             fi
 156             shift 2
 157             ;;
 158         "-replay")
 159             APP_TRACES="$2"
 160             shift 2
 161             ;;
 162         "-no-privatize")
 163             PRIVATIZE="--cfg=smpi/privatization:no"
 164             shift 1
 165             ;;
 166         "-map")
 167             MAPOPT=1
 168             shift 1
 169             ;;
 170         "-trace")
 171             TRACE_ACTIVE="true"
 172             shift 1
 173             ;;
 174         "-trace-ti")
 175             TRACE_ACTIVE="true"
 176             TRACE_TI_ACTIVE="true"
 177             shift 1
 178             ;;
 179         "-trace-comment")
 180             TRACE_COMMENT="$2"
 181             shift 2
 182             ;;
 183         "-trace-comment-file")
 184             TRACE_COMMENT_FILE="$2"
 185             shift 2
 186             ;;
 187         "-trace-file")
 188             TRACE_FILENAME="$2"
 189             shift 2
 190             ;;
 191         "-trace-grouped")
 192             TRACE_GROUPED="true"
 193             shift 1
 194             ;;
 195         "-trace-resource")
 196             TRACE_RESOURCE="true"
 197             shift 1
 198             ;;
 199         "-keep-temps")
 200             KEEP="true"
 201             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 202             shift 1
 203             ;;
 204         "-wrapper")
 205             WRAPPER="$2"
 206             shift 2
 207             ;;
 208         "-gdb")
 209             WRAPPER="gdb --args"
 210             KEEP="true"
 211             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 212             shift 1
 213             ;;
 214         "-lldb")
 215             WRAPPER="lldb --"
 216             KEEP="true"
 217             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 218             shift 1
 219             ;;
 220         "-help" | "--help" | "-h")
 221             usage
 222             exit 0
 223             ;;
 224         "-version" | "--version" | "-v")
 225             printf '%b\n' "$SIMGRID_VERSION"
 226             exit 0
 227             ;;
 228         "-git-version" | "--git-version")
 229             printf '%b\n' "$SIMGRID_GITHASH"
 230             exit 0
 231             ;;
 232         "--cfg="*|"--log="*)
 233             for OPT in ${1#*=}
 234             do
 235                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 236             done
 237             shift 1
 238             ;;
 239         "-foreground")
 240             # Nothing to do, compatibility.
 241             shift 1
 242             ;;
 243         *)
 244             break
 245             ;;
 246     esac
 247 done
 248
 249 if [ -n "${APP_TRACES}" ] ; then
 250     if [ $# -eq 0 ] ; then
 251         EXEC="@SMPIREPLAYMAIN@"
 252     else
 253         EXEC="$1"
 254         shift
 255     fi
 256 else
 257     # check if we still have at least one parameter beyond options
 258     if [ $# -eq 0 ]
 259     then
 260         echo "Error: no program to execute!"
 261         usage
 262         exit
 263     fi
 264
 265     EXEC="$1"
 266     shift
 267 fi
 268
 269 # steel --cfg and --logs options
 270 while [ $# -gt 0 ]; do
 271     case "$1" in
 272         "--cfg="*|"--log="*)
 273             for OPT in ${1#*=}
 274             do
 275                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 276             done
 277             shift 1
 278             ;;
 279         *)
 280             PROC_ARGS="${PROC_ARGS:+$PROC_ARGS }$1"
 281             shift
 282             ;;
 283     esac
 284 done
 285
 286 if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then
 287     echo "No hostfile nor platform specified."
 288     usage
 289     exit 1
 290 fi
 291
 292 if [ -z "${HOSTFILE}" ] ; then
 293     HOSTFILETMP=1
 294     HOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 295     perl -ne 'print "$1\n" if /.*<host.*?id="(.*?)".*?\/>.*/' ${PLATFORM} > ${HOSTFILE}
 296     # put all <cluster tag on its own line.
 297     cat ${PLATFORM} | tr '\n' ' ' | sed 's/<cluster/\n<cluster/' | \
 298       perl -ne 'if (m/.*<cluster.*?prefix="(.*?)".*?radical="(.*?)".*?suffix="(.*?)".*/s) {
 299                 my ($pre,$rad,$post)=($1,$2,$3);
 300                 for my $elm (split(",",$rad)) {
 301                   if ($elm=~/^([^-]*?)-([^-]*)$/) {
 302                      for (my $i=$1; $i<=$2;$i++) {
 303                         print "$pre$i$post\n";
 304                      }
 305                   } else {
 306                      print "$pre$elm$post\n";
 307                   }
 308                 }
 309               } elsif (/<cluster/) {
 310             die ("Unparsable cluster tag. smpirun uses a primitive regular expression to parse cluster tags. Either provide a hostfile yourself or give the attributes prefix, radical and suffix IN THAT ORDER.");
 311               }' ${PLATFORM} >> ${HOSTFILE}
 312 fi
 313 UNROLLEDHOSTFILETMP=0
 314
 315 #parse if our lines are terminated by :num_process
 316 multiple_processes=$(grep -c ":" $HOSTFILE)
 317 if [ "${multiple_processes}" -gt 0 ] ; then
 318     UNROLLEDHOSTFILETMP=1
 319     UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 320     perl -ne ' do{ for ( 1 .. $2 ) { print "$1\n" } } if /(.*?):(\d+).*/'  ${HOSTFILE}  > ${UNROLLEDHOSTFILE}
 321     if [ ${HOSTFILETMP} = 1 ] ; then
 322         rm ${HOSTFILE}
 323         HOSTFILETMP=0
 324     fi
 325     HOSTFILE=$UNROLLEDHOSTFILE
 326 fi
 327
 328 # Don't use wc -l to compute it to avoid issues with trailing \n at EOF
 329 hostfile_procs=$(grep -c "[a-zA-Z0-9]" $HOSTFILE)
 330 if [ ${hostfile_procs} = 0 ] ; then
 331     die "the hostfile '${HOSTFILE}' is empty"
 332 fi
 333
 334 if [ -z "${NUMPROCS}" ] ; then
 335     # Use the amount of processes in the hostfile as default value for the -np parameter
 336     NUMPROCS=$hostfile_procs
 337 fi
 338
 339 if [ ${NUMPROCS} -gt ${hostfile_procs} ] ; then
 340     echo "You requested to use ${NUMPROCS} ranks, but there is only ${hostfile_procs} processes in your hostfile..." >&2
 341 fi
 342
 343 ##-------------------------------- DEFAULT or SPECIFIED PLATFORM --------------------------------------
 344 if [ -z "${PLATFORM}" ]; then
 345     PLATFORMTMP="$(mktemp smpitmp-platfXXXXXX)"
 346
 347     cat > ${PLATFORMTMP} <<PLATFORMHEAD
 348 <?xml version='1.0'?>
 349 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 350 <platform version="4.1">
 351 <zone id="AS0" routing="Full">
 352 PLATFORMHEAD
 353
 354     i=${NUMPROCS}
 355     while [ $i -gt 0 ]; do
 356         echo "  <host id=\"host$i\" speed=\"${SPEED}\"/>" >> ${PLATFORMTMP}
 357         echo "  <link id=\"loop$i\" bandwidth=\"${LOOPBACK_BANDWIDTH}\" latency=\"${LOOPBACK_LATENCY}\"/>" >> ${PLATFORMTMP}
 358         echo "  <link id=\"link$i\" bandwidth=\"${NETWORK_BANDWIDTH}\" latency=\"${NETWORK_LATENCY}\"/>" >> ${PLATFORMTMP}
 359         i=$((i - 1))
 360     done
 361
 362     i=${NUMPROCS}
 363     while [ $i -gt 0 ]; do
 364         j=${NUMPROCS}
 365         while [ $j -gt 0 ]; do
 366             if [ $i -eq $j ]; then
 367                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"loop$i\"/></route>" >> ${PLATFORMTMP}
 368             else
 369                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"link$i\"/><link_ctn id=\"link$j\"/></route>" >> ${PLATFORMTMP}
 370             fi
 371             j=$((j - 1))
 372         done
 373         i=$((i - 1))
 374     done
 375
 376     cat >> ${PLATFORMTMP} <<PLATFORMFOOT
 377 </zone>
 378 </platform>
 379 PLATFORMFOOT
 380
 381 else
 382     PLATFORMTMP=${PLATFORM}
 383 fi
 384 ##-------------------------------- end DEFAULT or SPECIFIED PLATFORM --------------------------------------
 385 ##-------------------------------- DEFAULT APPLICATION --------------------------------------
 386 APPLICATIONTMP="$(mktemp smpitmp-appXXXXXX)"
 387 #APPLICATIONTMP="app.xml"
 388
 389 cat > ${APPLICATIONTMP} <<APPLICATIONHEAD
 390 <?xml version='1.0'?>
 391 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 392 <platform version="4.1">
 393 APPLICATIONHEAD
 394
 395 ##---- cache hostnames of hostfile---------------
 396 if [ -n "${HOSTFILE}" ] && [ -f ${HOSTFILE} ]; then
 397     hostnames=$(cat ${HOSTFILE} | tr '\n\r' '  ')
 398 fi
 399
 400 if [ -n "${APP_TRACES}" ]; then
 401     if [ -f "${APP_TRACES}" ]; then
 402         hosttraces=$(cat ${APP_TRACES} | tr '\n\r' '  ' )
 403         NUMTRACES=$(cat ${APP_TRACES} | wc -l)
 404         REPLAY=1
 405     else
 406         printf "File not found: %s\n" "${APP_TRACES:-\${APP_TRACES\}}" >&2
 407         exit 1
 408     fi
 409 fi
 410
 411 ##----------------------------------------------------------
 412 ##  generate application.xml with hostnames from hostfile:
 413 ##  the name of host_i (1<=i<=p, where -np p) is the line i in hostfile (where -hostfile hostfile), or "host$i" if
 414 ##  hostfile has less than i lines.
 415 ##----------------------------------------------------------
 416
 417 HAVE_SEQ="$(which seq 2>/dev/null)"
 418
 419 if [ -n "${HAVE_SEQ}" ]; then
 420     SEQ=$(${HAVE_SEQ} 0 $(( NUMPROCS - 1)))
 421 else
 422     cnt=0
 423     while [ $cnt -lt ${NUMPROCS} ] ; do
 424         SEQ="$SEQ $cnt"
 425         cnt=$((cnt + 1));
 426     done
 427 fi
 428
 429 set -- $hostnames
 430
 431 ##---- generate <actor> tags------------------------------
 432 #prepare arguments at once
 433 for ARG in $PROC_ARGS; do
 434   XML_ARGS="${XML_ARGS}""<argument value=\"${ARG}\"/>
 435 "
 436 done
 437
 438 for i in ${SEQ}
 439 do
 440     j=$(( i % hostfile_procs + 1 ))
 441     host=$(eval "echo \${$j}")
 442
 443     ##---- optional display of ranks to actor mapping
 444     if [ ${MAPOPT} = 1 ]; then
 445       echo "[rank $i] -> $host"
 446     fi
 447
 448     echo "  <actor host=\"${host}\" function=\"$i\"> <!-- function name used only for logging -->
 449     <prop id=\"instance_id\" value=\"smpirun\"/>
 450     <prop id=\"rank\" value=\"$i\"/>" >> ${APPLICATIONTMP}
 451     if [ ${REPLAY} = 1 ]; then
 452         echo "    <prop id=\"smpi_replay\" value=\"true\"/>" >> ${APPLICATIONTMP}
 453         if  [ ${NUMTRACES} -gt 1 ]; then
 454             echo "    <argument value=\"$(echo $hosttraces|cut -d' ' -f$j)\"/>" >> ${APPLICATIONTMP}
 455         else
 456             echo "    <argument value=\"$(echo $hosttraces|cut -d' ' -f1)\"/>" >> ${APPLICATIONTMP}
 457         fi
 458     else
 459     echo ${XML_ARGS} >> ${APPLICATIONTMP}
 460     fi
 461     echo "  </actor>" >> ${APPLICATIONTMP}
 462 done
 463
 464 cat >> ${APPLICATIONTMP} <<APPLICATIONFOOT
 465 </platform>
 466 APPLICATIONFOOT
 467 ##-------------------------------- end DEFAULT APPLICATION --------------------------------------
 468 ##---------------------- SMPI TRACING OPTIONS ---------------------------------
 469 if [ -n "${TRACE_ACTIVE}" ]; then
 470     #define trace filename
 471     if [ -n "${TRACE_TI_ACTIVE}" ]; then
 472         if [ -z "${TRACE_FILENAME}" ]; then
 473             TRACE_FILENAME="smpi_simgrid.txt"
 474         fi
 475         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes --cfg=tracing/smpi/format:TI --cfg=tracing/smpi/computing:yes"
 476     else
 477         if [ -z "${TRACE_FILENAME}" ]; then
 478             TRACE_FILENAME="smpi_simgrid.trace"
 479         fi
 480         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes"
 481     fi
 482
 483     if [ -n "${TRACE_COMMENT}" ]; then
 484         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment:${TRACE_COMMENT}"
 485     fi
 486
 487     if [ -n "${TRACE_COMMENT_FILE}" ]; then
 488         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment-file:${TRACE_COMMENT_FILE}"
 489     fi
 490
 491     if [ -n "${TRACE_GROUPED}" ]; then
 492         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/smpi/group:yes"
 493     fi
 494
 495     if [ -n "${TRACE_RESOURCE}" ]; then
 496         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/categorized:yes --cfg=tracing/uncategorized:yes"
 497     fi
 498 fi
 499 ##---------------------- end SMPI TRACING OPTIONS ---------------------------------
 500
 501 export SMPI_GLOBAL_SIZE=${NUMPROCS}
 502 if [ -n "${KEEP}" ] ; then
 503     echo ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP}
 504     if [ ${HOSTFILETMP} = 1 ] ; then
 505         echo "Generated hostfile ${HOSTFILE} kept."
 506     fi
 507     if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 508         echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 509     fi
 510 fi
 511
 512 # Execute the process
 513 #
 514 # The shell still need to be alive for the duration in order to do some cleanup after the process.
 515 #
 516 # We are going through great lengths in order to both keep stdin and be able to handle signals:
 517 #
 518 # * The job is launched in the background in order to be able to handle signals.
 519 #
 520 # * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command
 521 #   is launched in the background: this can be overriden in bash but not in standard bourne shell.
 522 exec 3<&0
 523 ${WRAPPER} "@SMPIMAIN@" ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP} <&3 3>&- &
 524 pid=$!
 525 exec 3>&-
 526 wait $pid
 527 status=$?
 528 # With dash on Windows WSL/Ubuntu, "wait" sometimes returns early with an exit
 529 # status of 128. Try again.
 530 while test $status -eq 128 && kill -0 $pid 2>/dev/null; do
 531     wait $pid
 532     status=$?
 533 done
 534 pid=""
 535
 536 # Keep temporary files on failures to help debugging
 537 #
 538 if [ ${status} -ne 0 ] ; then
 539     if [ -z "${KEEP}" ]; then
 540         echo ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP}
 541         if [ ${HOSTFILETMP} = 1 ] ; then
 542             echo "Generated hostfile ${HOSTFILE} kept."
 543         fi
 544         if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 545             echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 546         fi
 547     fi
 548     echo "Execution failed with code ${status}."
 549     KEEP=true
 550 fi
 551
 552 smpirun_cleanup
 553
 554 exit $status