src/smpi/smpirun.in

   1 #!/usr/bin/env sh
   2
   3 # Copyright (c) 2007-2019. The SimGrid Team. All rights reserved.
   4
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the license (GNU LGPL) which comes with this package.
   7
   8 @CMAKE_SMPI_COMMAND@
   9
  10 SIMGRID_VERSION="@SIMGRID_VERSION_STRING@"
  11 SIMGRID_GITHASH="@SIMGRID_GITHASH@"
  12
  13 DEFAULT_LOOPBACK_BANDWIDTH="498000000Bps"
  14 DEFAULT_LOOPBACK_LATENCY="0.000004s"
  15 DEFAULT_NETWORK_BANDWIDTH="$((26 * 1024 * 1024))Bps"
  16 DEFAULT_NETWORK_LATENCY="0.000005s"
  17 DEFAULT_NUMPROCS="4"
  18 DEFAULT_SPEED="100flops"
  19
  20 LOOPBACK_BANDWIDTH="${DEFAULT_LOOPBACK_BANDWIDTH}"
  21 LOOPBACK_LATENCY="${DEFAULT_LOOPBACK_LATENCY}"
  22 NETWORK_BANDWIDTH="${DEFAULT_NETWORK_BANDWIDTH}"
  23 NETWORK_LATENCY="${DEFAULT_NETWORK_LATENCY}"
  24 SPEED="${DEFAULT_SPEED}"
  25
  26 PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-@HAVE_PRIVATIZATION@}"
  27
  28 SIMOPTS="--cfg=surf/precision:1e-9 --cfg=network/model:SMPI"
  29
  30 #usage to print the way this script should be called
  31 usage () {
  32     cat <<EOF
  33 Usage: $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> program [program-options]
  34        $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> -replay <tracefile> [program] [program-options]
  35 Options:
  36   -keep-temps                # don't remove the generated files after execution
  37   -wrapper <command>         # use command to run the program (e.g. "valgrind" or "gdb --args")
  38   -gdb                       # run within GDB (-wrapper "gdb --args" -keep-temps)
  39   -lldb                      # run within LLDB (-wrapper "lldb --" -keep-temps)
  40   -vgdb                      # run within Valgrind+GDB (-wrapper "valgrind --vgdb=yes --vgdb-error=0" -keep-temps)
  41   -map                       # display the machine on which each process rank is mapped
  42   -np <numprocs>             # use that amount of processes from the hostfile.
  43                              # By default, all processes of the hostfile are used.
  44   -no-privatize              # Disable the globals privatization, that is activated by default
  45   -trace-ti                  # activate time independant tracing (for replay, default in smpi_simgrid.txt)
  46   -trace                     # activate tracing (Paje, default in smpi_simgrid.trace)
  47   -trace-comment <comment>   # put a comment on the top of the trace file
  48   -trace-comment-file <file> # put file contents on the top of the trace file as comment
  49   -trace-grouped             # group MPI processes by location
  50   -trace-resource            # trace resource utilization
  51   -trace-file <tracefile>    # name of the tracefile (simgrid_smpi.trace)
  52   -replay <tracefile>        # replays a trace instead of actually executing an application
  53
  54   -version                   # Displays the SimGrid version (human readable)
  55   -git-version               # Displays the git hash of SimGrid
  56
  57 or (deprecated usage):
  58   $0 [-keep-temps] [-np <numprocs>] [-bandwidth <bytes/sec>] [-latency <secs>] program [program-options]
  59
  60 EOF
  61 }
  62
  63 #check if we have at least one parameter
  64 if [ $# -eq 0 ]
  65 then
  66     usage
  67     exit
  68 fi
  69
  70 WRAPPER=""
  71 HOSTFILE=""
  72 HOSTFILETMP=0
  73 MAPOPT=0
  74 REPLAY=0
  75
  76 unset pid
  77
  78 trapped_signals="HUP INT QUIT ILL ABRT SEGV FPE ALRM TERM USR1 USR2 BUS"
  79
  80 die () {
  81     printf '[%s] ** error: %s. Aborting.\n' "$(basename $0)" "$*" >&2
  82     exit 1
  83 }
  84
  85 smpirun_cleanup()
  86 {
  87   if [ -z "${KEEP}" ] ; then
  88       if [ -z "${PLATFORM}" -a -n "$PLATFORMTMP" ]; then
  89         rm -f ${PLATFORMTMP}
  90         PLATFORMTMP=""
  91       fi
  92       if [ ${HOSTFILETMP} = 1 -a -n "$HOSTFILE" ] ; then
  93           rm -f ${HOSTFILE}
  94           HOSTFILE=""
  95       fi
  96       if [ ${UNROLLEDHOSTFILETMP} = 1 -a -n "$UNROLLEDHOSTFILE" ] ; then
  97           rm -f ${UNROLLEDHOSTFILE}
  98           UNROLLEDHOSTFILE=""
  99       fi
 100       if [ -n ${APPLICATIONTMP} ]; then
 101         rm -f ${APPLICATIONTMP}
 102         APPLICATIONTMP=""
 103       fi
 104   fi
 105 }
 106
 107 smpirun_trap() {
 108   local sig
 109   sig="$1"
 110
 111   # Cleanup and kill the child process:
 112   smpirun_cleanup
 113   if ! [ -z "$pid" ]; then
 114     kill -TERM $pid
 115   fi
 116   unset pid
 117
 118   # Raise the same signal again (remove the traps first):
 119   trap - $trapped_signals
 120   kill -$sig $$
 121
 122   # This should never happen:
 123   kill -ABRT $$
 124   kill -TERM $$
 125 }
 126
 127 for s in $trapped_signals; do
 128   trap "smpirun_trap $s" $s
 129 done
 130
 131 while true; do
 132     case "$1" in
 133         "-np" | "-n")
 134             NUMPROCS="$2"
 135             shift 2
 136             ;;
 137         "-bandwidth")
 138             NETWORK_BANDWIDTH="$2"
 139             shift 2
 140             ;;
 141         "-latency")
 142             NETWORK_LATENCY="$2"
 143             shift 2
 144             ;;
 145         "-platform")
 146             PLATFORM="$2"
 147             if [ ! -f "${PLATFORM}" ]; then
 148                 die "the file '${PLATFORM}' does not exist"
 149             fi
 150             shift 2
 151             ;;
 152         "-hostfile" | "-machinefile")
 153             HOSTFILE="$2"
 154             if [ ! -f "${HOSTFILE}" ]; then
 155                 die "the file '${HOSTFILE}' does not exist"
 156             fi
 157             shift 2
 158             ;;
 159         "-replay")
 160             APP_TRACES="$2"
 161             shift 2
 162             ;;
 163         "-no-privatize")
 164             PRIVATIZE="--cfg=smpi/privatization:no"
 165             shift 1
 166             ;;
 167         "-map")
 168             MAPOPT=1
 169             shift 1
 170             ;;
 171         "-trace")
 172             TRACE_ACTIVE="true"
 173             shift 1
 174             ;;
 175         "-trace-ti")
 176             TRACE_ACTIVE="true"
 177             TRACE_TI_ACTIVE="true"
 178             shift 1
 179             ;;
 180         "-trace-comment")
 181             TRACE_COMMENT="$2"
 182             shift 2
 183             ;;
 184         "-trace-comment-file")
 185             TRACE_COMMENT_FILE="$2"
 186             shift 2
 187             ;;
 188         "-trace-file")
 189             TRACE_FILENAME="$2"
 190             shift 2
 191             ;;
 192         "-trace-grouped")
 193             TRACE_GROUPED="true"
 194             shift 1
 195             ;;
 196         "-trace-resource")
 197             TRACE_RESOURCE="true"
 198             shift 1
 199             ;;
 200         "-keep-temps")
 201             KEEP="true"
 202             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 203             shift 1
 204             ;;
 205         "-wrapper")
 206             WRAPPER="$2"
 207             shift 2
 208             ;;
 209         "-gdb")
 210             WRAPPER="gdb --args"
 211             KEEP="true"
 212             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 213             shift 1
 214             ;;
 215         "-vgdb")
 216             WRAPPER="valgrind --vgdb=yes --vgdb-error=0"
 217             KEEP="true"
 218             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 219             shift 1
 220             ;;
 221         "-lldb")
 222             WRAPPER="lldb --"
 223             KEEP="true"
 224             SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
 225             shift 1
 226             ;;
 227         "-help" | "--help" | "-h")
 228             usage
 229             exit 0
 230             ;;
 231         "-version" | "--version" | "-v")
 232             printf '%b\n' "$SIMGRID_VERSION"
 233             exit 0
 234             ;;
 235         "-git-version" | "--git-version")
 236             printf '%b\n' "$SIMGRID_GITHASH"
 237             exit 0
 238             ;;
 239         "--cfg="*|"--log="*)
 240             for OPT in ${1#*=}
 241             do
 242                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 243             done
 244             shift 1
 245             ;;
 246         "-foreground")
 247             # Nothing to do, compatibility.
 248             shift 1
 249             ;;
 250         *)
 251             break
 252             ;;
 253     esac
 254 done
 255
 256 if [ -n "${APP_TRACES}" ] ; then
 257     if [ $# -eq 0 ] ; then
 258         EXEC="@SMPIREPLAYMAIN@"
 259     else
 260         EXEC="$1"
 261         shift
 262     fi
 263 else
 264     # check if we still have at least one parameter beyond options
 265     if [ $# -eq 0 ]
 266     then
 267         echo "Error: no program to execute!"
 268         usage
 269         exit
 270     fi
 271
 272     EXEC="$1"
 273     shift
 274 fi
 275
 276 # steel --cfg and --logs options
 277 while [ $# -gt 0 ]; do
 278     case "$1" in
 279         "--cfg="*|"--log="*)
 280             for OPT in ${1#*=}
 281             do
 282                 SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
 283             done
 284             shift 1
 285             ;;
 286         *)
 287             PROC_ARGS="${PROC_ARGS:+$PROC_ARGS }$1"
 288             shift
 289             ;;
 290     esac
 291 done
 292
 293 if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then
 294     echo "No hostfile nor platform specified."
 295     usage
 296     exit 1
 297 fi
 298
 299 if [ -z "${HOSTFILE}" ] ; then
 300     HOSTFILETMP=1
 301     HOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 302     perl -ne 'print "$1\n" if /.*<host.*?id="(.*?)".*?\/>.*/' ${PLATFORM} > ${HOSTFILE}
 303     # put all <cluster tag on its own line.
 304     cat ${PLATFORM} | tr '\n' ' ' | sed 's/<cluster/\n<cluster/' | \
 305       perl -ne 'if (m/.*<cluster.*?prefix="(.*?)".*?radical="(.*?)".*?suffix="(.*?)".*/s) {
 306                 my ($pre,$rad,$post)=($1,$2,$3);
 307                 for my $elm (split(",",$rad)) {
 308                   if ($elm=~/^([^-]*?)-([^-]*)$/) {
 309                      for (my $i=$1; $i<=$2;$i++) {
 310                         print "$pre$i$post\n";
 311                      }
 312                   } else {
 313                      print "$pre$elm$post\n";
 314                   }
 315                 }
 316               } elsif (/<cluster/) {
 317             die ("Unparsable cluster tag. smpirun uses a primitive regular expression to parse cluster tags. Either provide a hostfile yourself or give the attributes prefix, radical and suffix IN THAT ORDER.");
 318               }' ${PLATFORM} >> ${HOSTFILE}
 319 fi
 320 UNROLLEDHOSTFILETMP=0
 321
 322 #parse if our lines are terminated by :num_process
 323 multiple_processes=$(grep -c ":" $HOSTFILE)
 324 if [ "${multiple_processes}" -gt 0 ] ; then
 325     UNROLLEDHOSTFILETMP=1
 326     UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
 327     perl -ne ' do{ for ( 1 .. $2 ) { print "$1\n" } } if /(.*?):(\d+).*/'  ${HOSTFILE}  > ${UNROLLEDHOSTFILE}
 328     if [ ${HOSTFILETMP} = 1 ] ; then
 329         rm ${HOSTFILE}
 330         HOSTFILETMP=0
 331     fi
 332     HOSTFILE=$UNROLLEDHOSTFILE
 333 fi
 334
 335 # Don't use wc -l to compute it to avoid issues with trailing \n at EOF
 336 hostfile_procs=$(grep -c "[a-zA-Z0-9]" $HOSTFILE)
 337 if [ ${hostfile_procs} = 0 ] ; then
 338     die "the hostfile '${HOSTFILE}' is empty"
 339 fi
 340
 341 if [ -z "${NUMPROCS}" ] ; then
 342     # Use the amount of processes in the hostfile as default value for the -np parameter
 343     NUMPROCS=$hostfile_procs
 344 fi
 345
 346 if [ ${NUMPROCS} -gt ${hostfile_procs} ] ; then
 347     echo "You requested to use ${NUMPROCS} ranks, but there is only ${hostfile_procs} processes in your hostfile..." >&2
 348 fi
 349
 350 ##-------------------------------- DEFAULT or SPECIFIED PLATFORM --------------------------------------
 351 if [ -z "${PLATFORM}" ]; then
 352     PLATFORMTMP="$(mktemp smpitmp-platfXXXXXX)"
 353
 354     cat > ${PLATFORMTMP} <<PLATFORMHEAD
 355 <?xml version='1.0'?>
 356 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 357 <platform version="4.1">
 358 <zone id="AS0" routing="Full">
 359 PLATFORMHEAD
 360
 361     i=${NUMPROCS}
 362     while [ $i -gt 0 ]; do
 363         echo "  <host id=\"host$i\" speed=\"${SPEED}\"/>" >> ${PLATFORMTMP}
 364         echo "  <link id=\"loop$i\" bandwidth=\"${LOOPBACK_BANDWIDTH}\" latency=\"${LOOPBACK_LATENCY}\"/>" >> ${PLATFORMTMP}
 365         echo "  <link id=\"link$i\" bandwidth=\"${NETWORK_BANDWIDTH}\" latency=\"${NETWORK_LATENCY}\"/>" >> ${PLATFORMTMP}
 366         i=$((i - 1))
 367     done
 368
 369     i=${NUMPROCS}
 370     while [ $i -gt 0 ]; do
 371         j=${NUMPROCS}
 372         while [ $j -gt 0 ]; do
 373             if [ $i -eq $j ]; then
 374                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"loop$i\"/></route>" >> ${PLATFORMTMP}
 375             else
 376                 echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"link$i\"/><link_ctn id=\"link$j\"/></route>" >> ${PLATFORMTMP}
 377             fi
 378             j=$((j - 1))
 379         done
 380         i=$((i - 1))
 381     done
 382
 383     cat >> ${PLATFORMTMP} <<PLATFORMFOOT
 384 </zone>
 385 </platform>
 386 PLATFORMFOOT
 387
 388 else
 389     PLATFORMTMP=${PLATFORM}
 390 fi
 391 ##-------------------------------- end DEFAULT or SPECIFIED PLATFORM --------------------------------------
 392 ##-------------------------------- DEFAULT APPLICATION --------------------------------------
 393 APPLICATIONTMP="$(mktemp smpitmp-appXXXXXX)"
 394 #APPLICATIONTMP="app.xml"
 395
 396 cat > ${APPLICATIONTMP} <<APPLICATIONHEAD
 397 <?xml version='1.0'?>
 398 <!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
 399 <platform version="4.1">
 400 APPLICATIONHEAD
 401
 402 ##---- cache hostnames of hostfile---------------
 403 if [ -n "${HOSTFILE}" ] && [ -f ${HOSTFILE} ]; then
 404     hostnames=$(cat ${HOSTFILE} | tr '\n\r' '  ')
 405 fi
 406
 407 if [ -n "${APP_TRACES}" ]; then
 408     if [ -f "${APP_TRACES}" ]; then
 409         hosttraces=$(cat ${APP_TRACES} | tr '\n\r' '  ' )
 410         NUMTRACES=$(cat ${APP_TRACES} | wc -l)
 411         REPLAY=1
 412     else
 413         printf "File not found: %s\n" "${APP_TRACES:-\${APP_TRACES\}}" >&2
 414         exit 1
 415     fi
 416 fi
 417
 418 ##----------------------------------------------------------
 419 ##  generate application.xml with hostnames from hostfile:
 420 ##  the name of host_i (1<=i<=p, where -np p) is the line i in hostfile (where -hostfile hostfile), or "host$i" if
 421 ##  hostfile has less than i lines.
 422 ##----------------------------------------------------------
 423
 424 HAVE_SEQ="$(which seq 2>/dev/null)"
 425
 426 if [ -n "${HAVE_SEQ}" ]; then
 427     SEQ=$(${HAVE_SEQ} 0 $(( NUMPROCS - 1)))
 428 else
 429     cnt=0
 430     while [ $cnt -lt ${NUMPROCS} ] ; do
 431         SEQ="$SEQ $cnt"
 432         cnt=$((cnt + 1));
 433     done
 434 fi
 435
 436 set -- $hostnames
 437
 438 ##---- generate <actor> tags------------------------------
 439 #prepare arguments at once
 440 for ARG in $PROC_ARGS; do
 441   XML_ARGS="${XML_ARGS}""<argument value=\"${ARG}\"/>
 442 "
 443 done
 444
 445 for i in ${SEQ}
 446 do
 447     j=$(( i % hostfile_procs + 1 ))
 448     host=$(eval "echo \${$j}")
 449
 450     ##---- optional display of ranks to actor mapping
 451     if [ ${MAPOPT} = 1 ]; then
 452       echo "[rank $i] -> $host"
 453     fi
 454
 455     echo "  <actor host=\"${host}\" function=\"$i\"> <!-- function name used only for logging -->
 456     <prop id=\"instance_id\" value=\"smpirun\"/>
 457     <prop id=\"rank\" value=\"$i\"/>" >> ${APPLICATIONTMP}
 458     if [ ${REPLAY} = 1 ]; then
 459         echo "    <prop id=\"smpi_replay\" value=\"true\"/>" >> ${APPLICATIONTMP}
 460         if  [ ${NUMTRACES} -gt 1 ]; then
 461             echo "    <argument value=\"$(echo $hosttraces|cut -d' ' -f$j)\"/>" >> ${APPLICATIONTMP}
 462         else
 463             echo "    <argument value=\"$(echo $hosttraces|cut -d' ' -f1)\"/>" >> ${APPLICATIONTMP}
 464         fi
 465     else
 466     echo ${XML_ARGS} >> ${APPLICATIONTMP}
 467     fi
 468     echo "  </actor>" >> ${APPLICATIONTMP}
 469 done
 470
 471 cat >> ${APPLICATIONTMP} <<APPLICATIONFOOT
 472 </platform>
 473 APPLICATIONFOOT
 474 ##-------------------------------- end DEFAULT APPLICATION --------------------------------------
 475 ##---------------------- SMPI TRACING OPTIONS ---------------------------------
 476 if [ -n "${TRACE_ACTIVE}" ]; then
 477     #define trace filename
 478     if [ -n "${TRACE_TI_ACTIVE}" ]; then
 479         if [ -z "${TRACE_FILENAME}" ]; then
 480             TRACE_FILENAME="smpi_simgrid.txt"
 481         fi
 482         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes --cfg=tracing/smpi/format:TI --cfg=tracing/smpi/computing:yes"
 483     else
 484         if [ -z "${TRACE_FILENAME}" ]; then
 485             TRACE_FILENAME="smpi_simgrid.trace"
 486         fi
 487         TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes"
 488     fi
 489
 490     if [ -n "${TRACE_COMMENT}" ]; then
 491         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment:${TRACE_COMMENT}"
 492     fi
 493
 494     if [ -n "${TRACE_COMMENT_FILE}" ]; then
 495         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment-file:${TRACE_COMMENT_FILE}"
 496     fi
 497
 498     if [ -n "${TRACE_GROUPED}" ]; then
 499         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/smpi/group:yes"
 500     fi
 501
 502     if [ -n "${TRACE_RESOURCE}" ]; then
 503         TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/categorized:yes --cfg=tracing/uncategorized:yes"
 504     fi
 505 fi
 506 ##---------------------- end SMPI TRACING OPTIONS ---------------------------------
 507
 508 export SMPI_GLOBAL_SIZE=${NUMPROCS}
 509 if [ -n "${KEEP}" ] ; then
 510     echo ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP}
 511     if [ ${HOSTFILETMP} = 1 ] ; then
 512         echo "Generated hostfile ${HOSTFILE} kept."
 513     fi
 514     if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 515         echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 516     fi
 517 fi
 518
 519 # Execute the process
 520 #
 521 # The shell still need to be alive for the duration in order to do some cleanup after the process.
 522 #
 523 # We are going through great lengths in order to both keep stdin and be able to handle signals:
 524 #
 525 # * The job is launched in the background in order to be able to handle signals.
 526 #
 527 # * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command
 528 #   is launched in the background: this can be overriden in bash but not in standard bourne shell.
 529 exec 3<&0
 530 ${WRAPPER} "@SMPIMAIN@" ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP} <&3 3>&- &
 531 pid=$!
 532 exec 3>&-
 533 wait $pid
 534 status=$?
 535 # With dash on Windows WSL/Ubuntu, "wait" sometimes returns early with an exit
 536 # status of 128. Try again.
 537 while test $status -eq 128 && kill -0 $pid 2>/dev/null; do
 538     wait $pid
 539     status=$?
 540 done
 541 pid=""
 542
 543 # Keep temporary files on failures to help debugging
 544 #
 545 if [ ${status} -ne 0 ] ; then
 546     if [ -z "${KEEP}" ]; then
 547         echo ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP}
 548         if [ ${HOSTFILETMP} = 1 ] ; then
 549             echo "Generated hostfile ${HOSTFILE} kept."
 550         fi
 551         if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
 552             echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
 553         fi
 554     fi
 555     echo "Execution failed with code ${status}."
 556     KEEP=true
 557 fi
 558
 559 smpirun_cleanup
 560
 561 exit $status