From: Bruno Donassolo Date: Wed, 2 Jun 2021 18:11:09 +0000 (+0200) Subject: Moving SMPI app deployment to C++ code X-Git-Tag: v3.28~150 X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/c02acff354ef915c0c88943e73959ca5c081b524 Moving SMPI app deployment to C++ code Enable the deployment of SMPI experiments with C++ platform description. Move application deployment from smpirun to smpi_main function. The smpirun script used to parse the platform XML to create an application deployment. This isn't possible anymore since we may don't have a platform XML anymore. Move the necessary input to smpi_main through specific cfg variables: - smpi/hostfile: host file - smpi/replay: replay file - smpi/np: number of processes - smpi/map: mapping process/rank This cfg isn't used by users, they are cached inside the smpirun script. --- diff --git a/src/smpi/include/private.hpp b/src/smpi/include/private.hpp index 8c5eb133fc..7791846013 100644 --- a/src/smpi/include/private.hpp +++ b/src/smpi/include/private.hpp @@ -84,6 +84,8 @@ XBT_PRIVATE void smpi_deployment_unregister_process(const std::string& instance_ XBT_PRIVATE MPI_Comm* smpi_deployment_comm_world(const std::string& instance_id); XBT_PRIVATE void smpi_deployment_cleanup_instances(); +XBT_PRIVATE int smpi_deployment_smpirun(simgrid::s4u::Engine* e, const std::string& hostfile, int np, + const std::string& replayfile, int map, int argc, char* argv[]); XBT_PRIVATE void smpi_comm_copy_buffer_callback(simgrid::kernel::activity::CommImpl* comm, void* buff, size_t buff_size); diff --git a/src/smpi/internals/smpi_deployment.cpp b/src/smpi/internals/smpi_deployment.cpp index 09addebdb7..d69dbed712 100644 --- a/src/smpi/internals/smpi_deployment.cpp +++ b/src/smpi/internals/smpi_deployment.cpp @@ -103,3 +103,98 @@ int smpi_get_universe_size() { return simgrid::smpi::app::universe_size; } + +/** @brief Auxiliary method to get list of hosts to deploy app */ +static std::vector smpi_get_hosts(simgrid::s4u::Engine* e, const std::string& hostfile) +{ + if (hostfile == "") { + return e->get_all_hosts(); + } + std::vector hosts; + std::ifstream in(hostfile.c_str()); + xbt_assert(in, "smpirun: Cannot open the host file: %s", hostfile.c_str()); + std::string str; + while (std::getline(in, str)) { + if (str.size() > 0) + hosts.emplace_back(e->host_by_name(str)); + } + xbt_assert(hosts.size(), "smpirun: the hostfile '%s' is empty", hostfile.c_str()); + return hosts; +} + +/** @brief Read replay configuration from file */ +static std::vector smpi_read_replay(const std::string& replayfile) +{ + std::vector replay; + if (replayfile == "") + return replay; + + std::ifstream in(replayfile.c_str()); + xbt_assert(in, "smpirun: Cannot open the replay file: %s", replayfile.c_str()); + std::string str; + while (std::getline(in, str)) { + if (str.size() > 0) + replay.emplace_back(str); + } + + return replay; +} + +/** @brief Build argument vector to pass to process */ +static std::vector smpi_deployment_get_args(int rank_id, const std::vector& replay, int argc, + char* argv[]) +{ + std::vector args{std::to_string(rank_id)}; + // pass arguments to process only if not a replay execution + if (replay.size() == 0) { + for (int i = 0; i < argc; i++) { + args.push_back(argv[i]); + } + } + /* one trace per process */ + if (replay.size() > 1) { + args.push_back(replay[rank_id]); + } + return args; +} + +/** + * @brief Deploy an SMPI application from a smpirun call + * + * This used to be done at smpirun script, parsing either the hostfile or the platform XML. + * If hostfile isn't provided, get the list of hosts from engine. + */ +int smpi_deployment_smpirun(simgrid::s4u::Engine* e, const std::string& hostfile, int np, const std::string& replayfile, + int map, int argc, char* argv[]) +{ + auto hosts = smpi_get_hosts(e, hostfile); + auto replay = smpi_read_replay(replayfile); + int hosts_size = static_cast(hosts.size()); + if (np == 0) + np = hosts_size; + + xbt_assert(np > 0, "Invalid number of process (np must be > 0). Check your np parameter, platform or hostfile"); + + if (np > hosts_size) { + printf("You requested to use %d ranks, but there is only %d processes in your hostfile...\n", np, hosts_size); + } + + for (int i = 0; i < np; i++) { + simgrid::s4u::Host* host = hosts[i % hosts_size]; + std::string rank_id = std::to_string(i); + auto args = smpi_deployment_get_args(i, replay, argc, argv); + auto actor = simgrid::s4u::Actor::create(rank_id, host, rank_id, args); + /* keeping the same behavior as done in smpirun script, print mapping rank/process */ + if (map != 0) { + printf("[rank %d] -> %s\n", i, host->get_cname()); + } + actor->set_property("instance_id", "smpirun"); + actor->set_property("rank", rank_id); + if (replay.size() > 0) + actor->set_property("smpi_replay", "true"); + /* shared trace file, set it to rank 0 */ + if (i == 0 && replay.size() == 1) + actor->set_property("tracefile", replay[0]); + } + return np; +} \ No newline at end of file diff --git a/src/smpi/internals/smpi_global.cpp b/src/smpi/internals/smpi_global.cpp index e2569677f9..87b7b0781f 100644 --- a/src/smpi/internals/smpi_global.cpp +++ b/src/smpi/internals/smpi_global.cpp @@ -80,6 +80,19 @@ static const std::string smpi_default_instance_name("smpirun"); static simgrid::config::Flag smpi_init_sleep( "smpi/init", "Time to inject inside a call to MPI_Init", 0.0); +static simgrid::config::Flag + smpi_hostfile("smpi/hostfile", + "Classical MPI hostfile containing list of machines to dispatch " + "the processes, one per line", + ""); + +static simgrid::config::Flag smpi_replay("smpi/replay", + "Replay a trace instead of executing the application", ""); + +static simgrid::config::Flag smpi_np("smpi/np", "Number of processes to be created", 0); + +static simgrid::config::Flag smpi_map("smpi/map", "Display the mapping between nodes and processes", 0); + void (*smpi_comm_copy_data_callback)(simgrid::kernel::activity::CommImpl*, void*, size_t) = &smpi_comm_copy_buffer_callback; @@ -548,13 +561,8 @@ int smpi_main(const char* executable, int argc, char* argv[]) SMPI_init(); - /* This is a ... heavy way to count the MPI ranks */ - int rank_counts = 0; - simgrid::s4u::Actor::on_creation.connect([&rank_counts](const simgrid::s4u::Actor& actor) { - if (not actor.is_daemon()) - rank_counts++; - }); - engine->load_deployment(argv[2]); + int rank_counts = smpi_deployment_smpirun(engine, smpi_hostfile.get(), smpi_np.get(), smpi_replay.get(), + smpi_map.get(), argc - 2, argv + 2); SMPI_app_instance_register(smpi_default_instance_name.c_str(), nullptr, rank_counts); MPI_COMM_WORLD = *smpi_deployment_comm_world(smpi_default_instance_name); diff --git a/src/smpi/smpirun.in b/src/smpi/smpirun.in index ac3e393e6d..33fb52425c 100755 --- a/src/smpi/smpirun.in +++ b/src/smpi/smpirun.in @@ -24,6 +24,9 @@ SPEED="${DEFAULT_SPEED}" PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-@HAVE_PRIVATIZATION@}" +NUMPROCS=0 +DEPLOYOPTS="" + SIMOPTS="--cfg=surf/precision:1e-9 --cfg=network/model:SMPI" SMPITMPDIR="$(dirname $(mktemp -u))" @@ -72,9 +75,6 @@ fi WRAPPER="" HOSTFILE="" -HOSTFILETMP=0 -MAPOPT=0 -REPLAY=0 QUIET="" unset pid @@ -93,18 +93,10 @@ smpirun_cleanup() rm -f "${PLATFORMTMP}" PLATFORMTMP="" fi - if [ ${HOSTFILETMP} = 1 ] && [ -n "$HOSTFILE" ] ; then - rm -f "${HOSTFILE}" - HOSTFILE="" - fi if [ "${UNROLLEDHOSTFILETMP}" = 1 ] && [ -n "$UNROLLEDHOSTFILE" ] ; then rm -f "${UNROLLEDHOSTFILE}" UNROLLEDHOSTFILE="" fi - if [ -n "${APPLICATIONTMP}" ]; then - rm -f "${APPLICATIONTMP}" - APPLICATIONTMP="" - fi fi } @@ -162,6 +154,10 @@ while true; do ;; "-replay") APP_TRACES="$2" + if [ ! -f "${APP_TRACES}" ]; then + die "the file '${APP_TRACES}' does not exist" + fi + DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/replay:${APP_TRACES}" shift 2 ;; "-no-privatize") @@ -169,7 +165,7 @@ while true; do shift 1 ;; "-map") - MAPOPT=1 + DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/map:1" shift 1 ;; "-tmpdir") @@ -316,36 +312,10 @@ if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then exit 1 fi -if [ -z "${HOSTFILE}" ] ; then - HOSTFILETMP=1 - HOSTFILE="$(mktemp smpitmp-hostfXXXXXX)" - @PYTHON_EXECUTABLE@ -c ' -import xml.etree.ElementTree as ET -import sys -import re - -tree = ET.parse(sys.stdin) - -for elem in tree.findall(".//host"): - print(elem.attrib["id"]) - -for elem in tree.findall(".//cluster"): - prefix = elem.attrib["prefix"] - radical = elem.attrib["radical"] - suffix = elem.attrib["suffix"] - for r in radical.split(","): - m = re.match("^([^-]*?)-([^-]*)$", r) - if m: - for i in range(int(m.group(1)), int(m.group(2))): - print(prefix + str(i) + suffix) - else: - print(prefix + r + suffix) - ' < "${PLATFORM}" > "${HOSTFILE}" -fi UNROLLEDHOSTFILETMP=0 # parse if our lines are terminated by :num_process -if grep -q ':' "$HOSTFILE" ; then +if [ -n "${HOSTFILE}" ] && grep -q ':' "${HOSTFILE}" ; then UNROLLEDHOSTFILETMP=1 UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)" @PYTHON_EXECUTABLE@ -c ' @@ -360,27 +330,15 @@ for line in sys.stdin: else: print(line.strip()) ' < "${HOSTFILE}" > "${UNROLLEDHOSTFILE}" - if [ ${HOSTFILETMP} = 1 ] ; then - rm "${HOSTFILE}" - HOSTFILETMP=0 - fi HOSTFILE=$UNROLLEDHOSTFILE fi -# Don't use wc -l to compute it to avoid issues with trailing \n at EOF -hostfile_procs=$(grep -c "[a-zA-Z0-9]" "$HOSTFILE") -if [ "${hostfile_procs}" = 0 ] ; then - die "the hostfile '${HOSTFILE}' is empty" -fi - -if [ -z "${NUMPROCS}" ] ; then - # Use the amount of processes in the hostfile as default value for the -np parameter - NUMPROCS=$hostfile_procs -fi +DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/np:${NUMPROCS}" +DEPLOYOPTS="${DEPLOYOPTS} --cfg=smpi/hostfile:${HOSTFILE}" -if [ "${NUMPROCS}" -gt "${hostfile_procs}" ] ; then - echo "You requested to use ${NUMPROCS} ranks, but there is only ${hostfile_procs} processes in your hostfile..." >&2 -fi +#if [ "${NUMPROCS}" -gt "${hostfile_procs}" ] ; then +# echo "You requested to use ${NUMPROCS} ranks, but there is only ${hostfile_procs} processes in your hostfile..." >&2 +#fi ##-------------------------------- DEFAULT or SPECIFIED PLATFORM -------------------------------------- if [ -z "${PLATFORM}" ]; then @@ -426,92 +384,6 @@ else PLATFORMTMP=${PLATFORM} fi ##-------------------------------- end DEFAULT or SPECIFIED PLATFORM -------------------------------------- -##-------------------------------- DEFAULT APPLICATION -------------------------------------- -APPLICATIONTMP="$(mktemp smpitmp-appXXXXXX)" -#APPLICATIONTMP="app.xml" - -cat > "${APPLICATIONTMP}" < - - -APPLICATIONHEAD - -##---- cache hostnames of hostfile--------------- -if [ -n "${HOSTFILE}" ] && [ -f "${HOSTFILE}" ]; then - hostnames=$(< "${HOSTFILE}" tr '\n\r' ' ') -fi - -if [ -n "${APP_TRACES}" ]; then - if [ -f "${APP_TRACES}" ]; then - hosttraces=$(< "${APP_TRACES}" tr '\n\r' ' ' ) - NUMTRACES=$(< "${APP_TRACES}" wc -l) - REPLAY=1 - else - printf "File not found: %s\n" "${APP_TRACES:-\${APP_TRACES\}}" >&2 - exit 1 - fi -fi - -##---------------------------------------------------------- -## generate application.xml with hostnames from hostfile: -## the name of host_i (1<=i<=p, where -np p) is the line i in hostfile (where -hostfile hostfile), or "host$i" if -## hostfile has less than i lines. -##---------------------------------------------------------- - -HAVE_SEQ="$(which seq 2>/dev/null)" - -if [ -n "${HAVE_SEQ}" ]; then - SEQ=$(${HAVE_SEQ} 0 $(( NUMPROCS - 1))) -else - cnt=0 - while [ $cnt -lt "${NUMPROCS}" ] ; do - SEQ="$SEQ $cnt" - cnt=$((cnt + 1)); - done -fi - -set -- $hostnames - -##---- generate tags------------------------------ -#prepare arguments at once -for ARG in $PROC_ARGS; do - XML_ARGS="${XML_ARGS}"" -" -done - -for i in ${SEQ} -do - j=$(( i % hostfile_procs + 1 )) - host=$(eval "echo \${$j}") - - ##---- optional display of ranks to actor mapping - if [ ${MAPOPT} = 1 ]; then - echo "[rank $i] -> $host" - fi - { - echo " - - " - if [ ${REPLAY} = 1 ]; then - echo " " - if [ "${NUMTRACES}" -gt 1 ]; then - # Give each process its trace file (as an argument) - echo " " - elif [ ${i} = 0 ]; then - # Give the shared trace file to rank 0 (as a property) - echo " " - fi - else - echo "${XML_ARGS}" - fi - echo " " - } >> "${APPLICATIONTMP}" -done - -cat >> "${APPLICATIONTMP}" < -APPLICATIONFOOT -##-------------------------------- end DEFAULT APPLICATION -------------------------------------- ##---------------------- SMPI TRACING OPTIONS --------------------------------- if [ -n "${TRACE_ACTIVE}" ]; then #define trace filename @@ -548,10 +420,7 @@ fi # Do not remove, this variable may be used by user code (e.g. StarPU) export SMPI_GLOBAL_SIZE=${NUMPROCS} if [ -n "${KEEP}" ] && [ -z "${QUIET}" ] ; then - echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" "${APPLICATIONTMP}" - if [ ${HOSTFILETMP} = 1 ] ; then - echo "Generated hostfile ${HOSTFILE} kept." - fi + echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept." fi @@ -568,7 +437,7 @@ fi # * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command # is launched in the background: this can be overridden in bash but not in standard bourne shell. exec 3<&0 -${WRAPPER} "@SMPIMAIN@" "${EXEC}" ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} "${PLATFORMTMP}" "${APPLICATIONTMP}" <&3 3>&- & +${WRAPPER} "@SMPIMAIN@" "${EXEC}" ${PRIVATIZE} ${DEPLOYOPTS} ${TRACEOPTIONS} ${SIMOPTS} "${PLATFORMTMP}" ${PROC_ARGS} <&3 3>&- & pid=$! exec 3>&- wait $pid @@ -585,10 +454,7 @@ pid="" # if [ ${status} -ne 0 ] ; then if [ -z "${KEEP}" ] && [ -z "${QUIET}" ]; then - echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" "${APPLICATIONTMP}" - if [ ${HOSTFILETMP} = 1 ] ; then - echo "Generated hostfile ${HOSTFILE} kept." - fi + echo "${EXEC}" ${PRIVATIZE} "${TRACEOPTIONS}" "${SIMOPTS}" "${PLATFORMTMP}" if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept." fi