#!/usr/bin/env sh

# Copyright (c) 2007-2023. The SimGrid Team. All rights reserved.

# This program is free software; you can redistribute it and/or modify it
# under the terms of the license (GNU LGPL) which comes with this package.

@CMAKE_SMPI_COMMAND@

SIMGRID_VERSION="@SIMGRID_VERSION_STRING@"
SIMGRID_GITHASH="@GIT_VERSION@"

DEFAULT_LOOPBACK_BANDWIDTH="498000000Bps"
DEFAULT_LOOPBACK_LATENCY="0.000004s"
DEFAULT_NETWORK_BANDWIDTH="$((26 * 1024 * 1024))Bps"
DEFAULT_NETWORK_LATENCY="0.000005s"
DEFAULT_SPEED="100flops"

LOOPBACK_BANDWIDTH="${DEFAULT_LOOPBACK_BANDWIDTH}"
LOOPBACK_LATENCY="${DEFAULT_LOOPBACK_LATENCY}"
NETWORK_BANDWIDTH="${DEFAULT_NETWORK_BANDWIDTH}"
NETWORK_LATENCY="${DEFAULT_NETWORK_LATENCY}"
SPEED="${DEFAULT_SPEED}"

PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-@HAVE_PRIVATIZATION@}"

NUMPROCS=0
DEPLOYOPTS=""

SIMOPTS="--cfg=precision/timing:1e-9 --cfg=network/model:SMPI"

SMPITMPDIR="$(dirname $(mktemp -u))"

#usage to print the way this script should be called
usage () {
    cat <<EOF
Usage: $0 [OPTIONS] -platform <xmldesc|so> -hostfile <hostfile> program [program-options]
       $0 [OPTIONS] -platform <xmldesc|so> -hostfile <hostfile> -replay <tracefile> [program] [program-options]
Options:
  -analyze                   # show information about allocations and timings at the end of execution
  -keep-temps                # don't remove the generated files after execution
  -wrapper <command>         # use command to run the program (e.g. "valgrind" or "gdb --args")
  -gdb                       # run within GDB (-wrapper "gdb --args" -keep-temps)
  -lldb                      # run within LLDB (-wrapper "lldb --" -keep-temps)
  -vgdb                      # run within Valgrind+GDB (-wrapper "valgrind --vgdb=yes --vgdb-error=0" -keep-temps)
  -map                       # display the machine on which each process rank is mapped
  -np <numprocs>             # use that amount of processes from the hostfile.
                             # By default, all processes of the hostfile are used.
  -no-privatize              # Disable the globals privatization, that is activated by default
  -tmpdir                    # Directory used to store temporary files. Defaults to system's.
  -trace-ti                  # activate time independent tracing (for replay, default in smpi_simgrid.txt)
  -trace                     # activate tracing (Paje, default in smpi_simgrid.trace)
  -trace-comment <comment>   # put a comment on the top of the trace file
  -trace-comment-file <file> # put file contents on the top of the trace file as comment
  -trace-grouped             # group MPI processes by location
  -trace-resource            # trace resource utilization
  -trace-file <tracefile>    # name of the tracefile (simgrid_smpi.trace)
  -replay <tracefile>        # replays a trace instead of actually executing an application

  -version                   # Displays the SimGrid version (human readable)
  -git-version               # Displays the git hash of SimGrid

or (deprecated usage):
  $0 [-keep-temps] [-np <numprocs>] [-bandwidth <bytes/sec>] [-latency <secs>] program [program-options]

EOF
}

#check if we have at least one parameter
if [ $# -eq 0 ]
then
    usage
    exit
fi

WRAPPER=""
HOSTFILE=""
QUIET=""

unset pid

trapped_signals="HUP INT QUIT ILL ABRT SEGV FPE ALRM TERM USR1 USR2 BUS"

die () {
    printf '[%s] ** error: %s. Aborting.\n' "$(basename "$0")" "$*" >&2
    exit 1
}

smpirun_cleanup()
{
  if [ -z "${KEEP}" ] ; then
      if [ -z "${PLATFORM}" ] && [ -n "$PLATFORMTMP" ]; then
        rm -f "${PLATFORMTMP}"
        PLATFORMTMP=""
      fi
      if [ "${UNROLLEDHOSTFILETMP}" = 1 ] && [ -n "$UNROLLEDHOSTFILE" ] ; then
          rm -f "${UNROLLEDHOSTFILE}"
          UNROLLEDHOSTFILE=""
      fi
  fi
}

smpirun_trap() {
  local sig
  sig="$1"

  # Cleanup and kill the child process:
  smpirun_cleanup
  if [ -n "$pid" ]; then
    kill -TERM "$pid"
  fi
  unset pid

  # Raise the same signal again (remove the traps first):
  trap - $trapped_signals
  kill -"$sig" $$

  # This should never happen:
  kill -ABRT $$
  kill -TERM $$
}

for s in $trapped_signals; do
  trap "smpirun_trap $s" "$s"
done

while true; do
    case "$1" in
        "-np" | "-n")
            NUMPROCS="$2"
            shift 2
            ;;
        "-bandwidth")
            NETWORK_BANDWIDTH="$2"
            shift 2
            ;;
        "-latency")
            NETWORK_LATENCY="$2"
            shift 2
            ;;
        "-platform")
            PLATFORM="$2"
            if [ ! -f "${PLATFORM}" ]; then
                die "the file '${PLATFORM}' does not exist"
            fi
            shift 2
            ;;
        "-hostfile" | "-machinefile")
            HOSTFILE="$2"
            if [ ! -f "${HOSTFILE}" ]; then
                die "the file '${HOSTFILE}' does not exist"
            fi
            shift 2
            ;;
        "-replay")
            APP_TRACES="$2"
            if [ ! -f "${APP_TRACES}" ]; then
                die "the file '${APP_TRACES}' does not exist"
            fi
            DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/replay:${APP_TRACES}"
            shift 2
            ;;
        "-no-privatize")
            PRIVATIZE="--cfg=smpi/privatization:no"
            shift 1
            ;;
        "-map")
            DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/map:1"
            shift 1
            ;;
        "-tmpdir")
            SMPITMPDIR="$2"
            shift 1
            ;;
        "-trace")
            TRACE_ACTIVE="true"
            shift 1
            ;;
        "-trace-ti")
            TRACE_ACTIVE="true"
            TRACE_TI_ACTIVE="true"
            shift 1
            ;;
        "-trace-comment")
            TRACE_COMMENT="$2"
            shift 2
            ;;
        "-trace-comment-file")
            TRACE_COMMENT_FILE="$2"
            shift 2
            ;;
        "-trace-file")
            TRACE_FILENAME="$2"
            shift 2
            ;;
        "-trace-grouped")
            TRACE_GROUPED="true"
            shift 1
            ;;
        "-trace-resource")
            TRACE_RESOURCE="true"
            shift 1
            ;;
        "-keep-temps")
            KEEP="true"
            SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
            shift 1
            ;;
        "-quiet")
            QUIET="true"
            shift 1
            ;;
        "-wrapper")
            WRAPPER="$2"
            shift 2
            ;;
        "-gdb")
            WRAPPER="gdb --args"
            KEEP="true"
            SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
            shift 1
            ;;
        "-vgdb")
            WRAPPER="valgrind --vgdb=yes --vgdb-error=0"
            KEEP="true"
            SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
            shift 1
            ;;
        "-lldb")
            WRAPPER="lldb --"
            KEEP="true"
            SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
            shift 1
            ;;
        "-analyze")
            SIMOPTS="$SIMOPTS --cfg=smpi/display-timing:yes --cfg=smpi/display-allocs:yes --cfg=smpi/list-leaks:50 --cfg=smpi/pedantic:true --cfg=smpi/barrier-collectives:true"
            shift 1
            ;;
        "-help" | "--help" | "-h")
            usage
            exit 0
            ;;
        "--help-coll")
            ${WRAPPER} "@SMPIMAIN@" --help-coll
            exit 0
            ;;
        "-version" | "--version" | "-v")
            printf '%b\n' "$SIMGRID_VERSION"
            exit 0
            ;;
        "-git-version" | "--git-version")
            printf '%b\n' "$SIMGRID_GITHASH"
            exit 0
            ;;
        "--cfg="*|"--log="*)
            for OPT in ${1#*=}
            do
                SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
            done
            shift 1
            ;;
        "-foreground")
            # Nothing to do, compatibility.
            shift 1
            ;;
        *)
            break
            ;;
    esac
done

#setup tmp dir
SIMOPTS="$SIMOPTS --cfg=smpi/tmpdir:$SMPITMPDIR"
export LD_LIBRARY_PATH="$SMPITMPDIR:$LD_LIBRARY_PATH"

if [ -n "${APP_TRACES}" ] ; then
    if [ $# -eq 0 ] ; then
        EXEC="@SMPIREPLAYMAIN@"
    else
        EXEC="$1"
        shift
    fi
else
    # check if we still have at least one parameter beyond options
    if [ $# -eq 0 ]
    then
        echo "Error: no program to execute!"
        usage
        exit
    fi

    EXEC="$1"
    shift
fi

# steal --cfg and --logs options
while [ $# -gt 0 ]; do
    case "$1" in
        "--cfg="*|"--log="*)
            for OPT in ${1#*=}
            do
                SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
            done
            shift 1
            ;;
        *)
            PROC_ARGS="${PROC_ARGS:+$PROC_ARGS }$1"
            shift
            ;;
    esac
done

if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then
    echo "No hostfile nor platform specified."
    usage
    exit 1
fi

UNROLLEDHOSTFILETMP=0

# parse if our lines are terminated by :num_process
if [ -n "${HOSTFILE}" ] && grep -q ':' "${HOSTFILE}" ; then
    UNROLLEDHOSTFILETMP=1
    UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
    @PYTHON_EXECUTABLE@ -c '
import sys
import re

for line in sys.stdin:
    m = re.match("(.*):(.*)", line)
    if m:
        for i in range(0, int(m.group(2))):
            print(m.group(1))
    else:
        print(line.strip())
' < "${HOSTFILE}"  > "${UNROLLEDHOSTFILE}"
    HOSTFILE=$UNROLLEDHOSTFILE
fi

DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/np:${NUMPROCS}"
DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/hostfile:${HOSTFILE}"

##-------------------------------- DEFAULT or SPECIFIED PLATFORM --------------------------------------
if [ -z "${PLATFORM}" ]; then
    PLATFORMTMP="$(mktemp smpitmp-platfXXXXXX)"

    cat > "${PLATFORMTMP}" <<PLATFORMHEAD
<?xml version='1.0'?>
<!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
<platform version="4.1">
<zone id="AS0" routing="Full">
PLATFORMHEAD

    i=${NUMPROCS}
    while [ "$i" -gt 0 ]; do
        {
        echo "  <host id=\"host$i\" speed=\"${SPEED}\"/>"
        echo "  <link id=\"loop$i\" bandwidth=\"${LOOPBACK_BANDWIDTH}\" latency=\"${LOOPBACK_LATENCY}\"/>"
        echo "  <link id=\"link$i\" bandwidth=\"${NETWORK_BANDWIDTH}\" latency=\"${NETWORK_LATENCY}\"/>"
        } >> "${PLATFORMTMP}"
        i=$((i - 1))
    done

    i=${NUMPROCS}
    while [ "$i" -gt 0 ]; do
        j=${NUMPROCS}
        while [ "$j" -gt 0 ]; do
            if [ "$i" -eq "$j" ]; then
                echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"loop$i\"/></route>" >> "${PLATFORMTMP}"
            else
                echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"link$i\"/><link_ctn id=\"link$j\"/></route>" >> "${PLATFORMTMP}"
            fi
            j=$((j - 1))
        done
        i=$((i - 1))
    done

    cat >> "${PLATFORMTMP}" <<PLATFORMFOOT
</zone>
</platform>
PLATFORMFOOT

else
    PLATFORMTMP=${PLATFORM}
fi
##-------------------------------- end DEFAULT or SPECIFIED PLATFORM --------------------------------------
##---------------------- SMPI TRACING OPTIONS ---------------------------------
if [ -n "${TRACE_ACTIVE}" ]; then
    #define trace filename
    if [ -n "${TRACE_TI_ACTIVE}" ]; then
        if [ -z "${TRACE_FILENAME}" ]; then
            TRACE_FILENAME="smpi_simgrid.txt"
        fi
        TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes --cfg=tracing/smpi/format:TI --cfg=tracing/smpi/computing:yes"
    else
        if [ -z "${TRACE_FILENAME}" ]; then
            TRACE_FILENAME="smpi_simgrid.trace"
        fi
        TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes"
    fi

    if [ -n "${TRACE_COMMENT}" ]; then
        TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment:${TRACE_COMMENT}"
    fi

    if [ -n "${TRACE_COMMENT_FILE}" ]; then
        TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment-file:${TRACE_COMMENT_FILE}"
    fi

    if [ -n "${TRACE_GROUPED}" ]; then
        TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/smpi/group:yes"
    fi

    if [ -n "${TRACE_RESOURCE}" ]; then
        TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/categorized:yes --cfg=tracing/uncategorized:yes"
    fi
fi
##---------------------- end SMPI TRACING OPTIONS ---------------------------------

# Do not remove, this variable may be used by user code (e.g. StarPU)
export SMPI_GLOBAL_SIZE=${NUMPROCS}
if [ -n "${KEEP}" ] && [ -z "${QUIET}" ] ; then
    echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}"
    if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
        echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
    fi
fi

# Execute the process
#
# The shell still need to be alive for the duration in order to do some cleanup after the process.
#
# We are going through great lengths in order to both keep stdin and be able to handle signals:
#
# * The job is launched in the background in order to be able to handle signals.
#
# * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command
#   is launched in the background: this can be overridden in bash but not in standard bourne shell.
exec 3<&0
${WRAPPER} "@SMPIMAIN@" "${EXEC}" ${PRIVATIZE} ${DEPLOYOPTS} ${TRACEOPTIONS} ${SIMOPTS} "${PLATFORMTMP}" ${PROC_ARGS} <&3 3>&- &
pid=$!
exec 3>&-
wait $pid
status=$?
# With dash on Windows WSL/Ubuntu, "wait" sometimes returns early with an exit
# status of 128. Try again.
while test $status -eq 128 && kill -0 $pid 2>/dev/null; do
    wait $pid
    status=$?
done
pid=""

# Keep temporary files on failures to help debugging
#
if [ ${status} -ne 0 ] ; then
    if [ -z "${KEEP}" ] && [ -z "${QUIET}" ]; then
        echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}"
        if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
            echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
        fi
        KEEP=true
    fi
    echo "Execution failed with code ${status}."
fi

smpirun_cleanup

exit $status