X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/093591688e3facd6c1421b8072a66c9d343a0ad1..a50c2dca84b5c5863c0ac4e73a07dff3dc070d4b:/src/smpi/internals/smpi_global.cpp diff --git a/src/smpi/internals/smpi_global.cpp b/src/smpi/internals/smpi_global.cpp index 749fd2f046..db19072c98 100644 --- a/src/smpi/internals/smpi_global.cpp +++ b/src/smpi/internals/smpi_global.cpp @@ -4,14 +4,15 @@ * under the terms of the license (GNU LGPL) which comes with this package. */ #include "mc/mc.h" -#include "simgrid/s4u/Engine.hpp" +#include "simgrid/Exception.hpp" #include "simgrid/plugins/file_system.h" +#include "simgrid/s4u/Engine.hpp" #include "smpi_coll.hpp" +#include "smpi_config.hpp" #include "smpi_f2c.hpp" #include "smpi_host.hpp" -#include "smpi_config.hpp" +#include "src/kernel/EngineImpl.hpp" #include "src/kernel/activity/CommImpl.hpp" -#include "src/simix/smx_private.hpp" #include "src/smpi/include/smpi_actor.hpp" #include "xbt/config.hpp" #include "xbt/file.hpp" @@ -64,32 +65,31 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_kernel, smpi, "Logging specific to SMPI (ke #endif #if HAVE_PAPI -std::string papi_default_config_name = "default"; std::map> units2papi_setup; #endif std::unordered_map location2speedup; static int smpi_exit_status = 0; -extern double smpi_total_benched_time; xbt_os_timer_t global_timer; static std::vector privatize_libs_paths; -/** - * Setting MPI_COMM_WORLD to MPI_COMM_UNINITIALIZED (it's a variable) - * is important because the implementation of MPI_Comm checks - * "this == MPI_COMM_UNINITIALIZED"? If yes, it uses smpi_process()->comm_world() - * instead of "this". - * This is basically how we only have one global variable but all processes have - * different communicators (the one their SMPI instance uses). - * - * See smpi_comm.cpp and the functions therein for details. - */ -MPI_Comm MPI_COMM_WORLD = MPI_COMM_UNINITIALIZED; + // No instance gets manually created; check also the smpirun.in script as // this default name is used there as well (when the tag is generated). static const std::string smpi_default_instance_name("smpirun"); -static simgrid::config::Flag smpi_init_sleep( - "smpi/init", "Time to inject inside a call to MPI_Init", 0.0); + +static simgrid::config::Flag + smpi_hostfile("smpi/hostfile", + "Classical MPI hostfile containing list of machines to dispatch " + "the processes, one per line", + ""); + +static simgrid::config::Flag smpi_replay("smpi/replay", + "Replay a trace instead of executing the application", ""); + +static simgrid::config::Flag smpi_np("smpi/np", "Number of processes to be created", 0); + +static simgrid::config::Flag smpi_map("smpi/map", "Display the mapping between nodes and processes", 0); void (*smpi_comm_copy_data_callback)(simgrid::kernel::activity::CommImpl*, void*, size_t) = &smpi_comm_copy_buffer_callback; @@ -195,21 +195,15 @@ void smpi_comm_copy_buffer_callback(simgrid::kernel::activity::CommImpl* comm, v auto private_blocks = merge_private_blocks(src_private_blocks, dst_private_blocks); check_blocks(private_blocks, buff_size); void* tmpbuff=buff; - if ((smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) && - (static_cast(buff) >= smpi_data_exe_start) && - (static_cast(buff) < smpi_data_exe_start + smpi_data_exe_size)) { + if (smpi_switch_data_segment(comm->src_actor_->get_iface(), buff)) { XBT_DEBUG("Privatization : We are copying from a zone inside global memory... Saving data to temp buffer !"); - smpi_switch_data_segment(comm->src_actor_->get_iface()); tmpbuff = xbt_malloc(buff_size); memcpy_private(tmpbuff, buff, private_blocks); } - if ((smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) && - ((char*)comm->dst_buff_ >= smpi_data_exe_start) && - ((char*)comm->dst_buff_ < smpi_data_exe_start + smpi_data_exe_size)) { + if (smpi_switch_data_segment(comm->dst_actor_->get_iface(), comm->dst_buff_)) XBT_DEBUG("Privatization : We are copying to a zone inside global memory - Switch data segment"); - smpi_switch_data_segment(comm->dst_actor_->get_iface()); - } + XBT_DEBUG("Copying %zu bytes from %p to %p", buff_size, tmpbuff, comm->dst_buff_); memcpy_private(comm->dst_buff_, tmpbuff, private_blocks); @@ -234,57 +228,61 @@ static void smpi_init_papi() // the configuration as given by the user (counter data as a pair of (counter_name, counter_counter)) // and the (computed) event_set. - if (not smpi_cfg_papi_events_file().empty()) { - if (PAPI_library_init(PAPI_VER_CURRENT) != PAPI_VER_CURRENT) - XBT_ERROR("Could not initialize PAPI library; is it correctly installed and linked?" - " Expected version is %u", PAPI_VER_CURRENT); - - using Tokenizer = boost::tokenizer>; - boost::char_separator separator_units(";"); - std::string str = smpi_cfg_papi_events_file(); - Tokenizer tokens(str, separator_units); - - // Iterate over all the computational units. This could be processes, hosts, threads, ranks... You name it. - // I'm not exactly sure what we will support eventually, so I'll leave it at the general term "units". - for (auto const& unit_it : tokens) { - boost::char_separator separator_events(":"); - Tokenizer event_tokens(unit_it, separator_events); - - int event_set = PAPI_NULL; - if (PAPI_create_eventset(&event_set) != PAPI_OK) { - // TODO: Should this let the whole simulation die? - XBT_CRITICAL("Could not create PAPI event set during init."); - } + if (smpi_cfg_papi_events_file().empty()) + return; - // NOTE: We cannot use a map here, as we must obey the order of the counters - // This is important for PAPI: We need to map the values of counters back to the event_names (so, when PAPI_read() - // has finished)! - papi_counter_t counters2values; - - // Iterate over all counters that were specified for this specific unit. - // Note that we need to remove the name of the unit (that could also be the "default" value), which always comes - // first. Hence, we start at ++(events.begin())! - for (Tokenizer::iterator events_it = ++(event_tokens.begin()); events_it != event_tokens.end(); ++events_it) { - int event_code = PAPI_NULL; - auto* event_name = const_cast((*events_it).c_str()); - if (PAPI_event_name_to_code(event_name, &event_code) != PAPI_OK) { - XBT_CRITICAL("Could not find PAPI event '%s'. Skipping.", event_name); - continue; - } - if (PAPI_add_event(event_set, event_code) != PAPI_OK) { - XBT_ERROR("Could not add PAPI event '%s'. Skipping.", event_name); - continue; - } - XBT_DEBUG("Successfully added PAPI event '%s' to the event set.", event_name); + if (PAPI_library_init(PAPI_VER_CURRENT) != PAPI_VER_CURRENT) { + XBT_ERROR("Could not initialize PAPI library; is it correctly installed and linked? Expected version is %u", + PAPI_VER_CURRENT); + return; + } - counters2values.emplace_back(*events_it, 0LL); - } + using Tokenizer = boost::tokenizer>; + boost::char_separator separator_units(";"); + std::string str = smpi_cfg_papi_events_file(); + Tokenizer tokens(str, separator_units); + + // Iterate over all the computational units. This could be processes, hosts, threads, ranks... You name it. + // I'm not exactly sure what we will support eventually, so I'll leave it at the general term "units". + for (auto const& unit_it : tokens) { + boost::char_separator separator_events(":"); + Tokenizer event_tokens(unit_it, separator_events); + + int event_set = PAPI_NULL; + if (PAPI_create_eventset(&event_set) != PAPI_OK) { + // TODO: Should this let the whole simulation die? + XBT_CRITICAL("Could not create PAPI event set during init."); + break; + } - std::string unit_name = *(event_tokens.begin()); - papi_process_data config = {.counter_data = std::move(counters2values), .event_set = event_set}; + // NOTE: We cannot use a map here, as we must obey the order of the counters + // This is important for PAPI: We need to map the values of counters back to the event_names (so, when PAPI_read() + // has finished)! + papi_counter_t counters2values; + + // Iterate over all counters that were specified for this specific unit. + // Note that we need to remove the name of the unit (that could also be the "default" value), which always comes + // first. Hence, we start at ++(events.begin())! + for (Tokenizer::iterator events_it = ++(event_tokens.begin()); events_it != event_tokens.end(); ++events_it) { + int event_code = PAPI_NULL; + auto* event_name = const_cast((*events_it).c_str()); + if (PAPI_event_name_to_code(event_name, &event_code) != PAPI_OK) { + XBT_CRITICAL("Could not find PAPI event '%s'. Skipping.", event_name); + continue; + } + if (PAPI_add_event(event_set, event_code) != PAPI_OK) { + XBT_ERROR("Could not add PAPI event '%s'. Skipping.", event_name); + continue; + } + XBT_DEBUG("Successfully added PAPI event '%s' to the event set.", event_name); - units2papi_setup.insert(std::make_pair(unit_name, std::move(config))); + counters2values.emplace_back(*events_it, 0LL); } + + std::string unit_name = *(event_tokens.begin()); + papi_process_data config = {.counter_data = std::move(counters2values), .event_set = event_set}; + + units2papi_setup.insert(std::make_pair(unit_name, std::move(config))); } #endif } @@ -294,52 +292,41 @@ using smpi_c_entry_point_type = int (*)(int argc, char** argv); using smpi_fortran_entry_point_type = void (*)(); template -static int smpi_run_entry_point(const F& entry_point, const std::string& executable_path, std::vector args) +static int smpi_run_entry_point(const F& entry_point, const std::string& executable_path, + const std::vector& args) { // copy C strings, we need them writable - auto* args4argv = new std::vector(args.size()); - std::transform(begin(args), end(args), begin(*args4argv), [](const std::string& s) { return xbt_strdup(s.c_str()); }); + std::vector args4argv(args.size()); + std::transform(begin(args) + 1, end(args), begin(args4argv) + 1, + [](const std::string& s) { return xbt_strdup(s.c_str()); }); // set argv[0] to executable_path - xbt_free((*args4argv)[0]); - (*args4argv)[0] = xbt_strdup(executable_path.c_str()); + args4argv[0] = xbt_strdup(executable_path.c_str()); + // add the final NULL + args4argv.push_back(nullptr); -#if !SMPI_IFORT // take a copy of args4argv to keep reference of the allocated strings - const std::vector args2str(*args4argv); -#endif - int argc = args4argv->size(); - args4argv->push_back(nullptr); - char** argv = args4argv->data(); - -#if SMPI_IFORT - for_rtl_init_ (&argc, argv); -#elif SMPI_FLANG - __io_set_argc(argc); - __io_set_argv(argv); -#elif SMPI_GFORTRAN - _gfortran_set_args(argc, argv); -#endif - int res = entry_point(argc, argv); + const std::vector args2str(args4argv); + + try { + int argc = static_cast(args4argv.size() - 1); + char** argv = args4argv.data(); + int res = entry_point(argc, argv); + if (res != 0) { + XBT_WARN("SMPI process did not return 0. Return value : %d", res); + if (smpi_exit_status == 0) + smpi_exit_status = res; + } + } catch (simgrid::ForcefulKillException const& e) { + XBT_DEBUG("Caught a ForcefulKillException: %s", e.what()); + } -#if SMPI_IFORT - for_rtl_finish_ (); -#else for (char* s : args2str) xbt_free(s); - delete args4argv; -#endif - if (res != 0){ - XBT_WARN("SMPI process did not return 0. Return value : %d", res); - if (smpi_exit_status == 0) - smpi_exit_status = res; - } return 0; } - -// TODO, remove the number of functions involved here static smpi_entry_point_type smpi_resolve_function(void* handle) { auto* entry_point_fortran = reinterpret_cast(dlsym(handle, "user_main_")); @@ -362,8 +349,8 @@ static void smpi_copy_file(const std::string& src, const std::string& target, of { int fdin = open(src.c_str(), O_RDONLY); xbt_assert(fdin >= 0, "Cannot read from %s. Please make sure that the file exists and is executable.", src.c_str()); - XBT_ATTRIB_UNUSED int unlink_status = unlink(target.c_str()); - xbt_assert(unlink_status == 0 || errno == ENOENT, "Failed to unlink file %s: %s", target.c_str(), strerror(errno)); + xbt_assert(unlink(target.c_str()) == 0 || errno == ENOENT, "Failed to unlink file %s: %s", target.c_str(), + strerror(errno)); int fdout = open(target.c_str(), O_CREAT | O_RDWR | O_EXCL, S_IRWXU); xbt_assert(fdout >= 0, "Cannot write into %s: %s", target.c_str(), strerror(errno)); @@ -374,27 +361,27 @@ static void smpi_copy_file(const std::string& src, const std::string& target, of close(fdin); close(fdout); return; - } else if (sent_size != -1 || errno != ENOSYS) { - xbt_die("Error while copying %s: only %zd bytes copied instead of %" PRIdMAX " (errno: %d -- %s)", target.c_str(), - sent_size, static_cast(fdin_size), errno, strerror(errno)); } + xbt_assert(sent_size == -1 && errno == ENOSYS, + "Error while copying %s: only %zd bytes copied instead of %" PRIdMAX " (errno: %d -- %s)", target.c_str(), + sent_size, static_cast(fdin_size), errno, strerror(errno)); #endif // If this point is reached, sendfile() actually is not available. Copy file by hand. std::vector buf(1024 * 1024 * 4); while (ssize_t got = read(fdin, buf.data(), buf.size())) { if (got == -1) { xbt_assert(errno == EINTR, "Cannot read from %s", src.c_str()); - } else { - const unsigned char* p = buf.data(); - ssize_t todo = got; - while (ssize_t done = write(fdout, p, todo)) { - if (done == -1) { - xbt_assert(errno == EINTR, "Cannot write into %s", target.c_str()); - } else { - p += done; - todo -= done; - } + continue; + } + const unsigned char* p = buf.data(); + ssize_t todo = got; + while (ssize_t done = write(fdout, p, todo)) { + if (done == -1) { + xbt_assert(errno == EINTR, "Cannot write into %s", target.c_str()); + continue; } + p += done; + todo -= done; } } close(fdin); @@ -430,14 +417,13 @@ static void smpi_init_privatization_dlopen(const std::string& executable) for (auto const& libname : privatize_libs) { // load the library once to add it to the local libs, to get the absolute path void* libhandle = dlopen(libname.c_str(), RTLD_LAZY); - xbt_assert(libhandle != nullptr, - "Cannot dlopen %s - check your settings in smpi/privatize-libs", libname.c_str()); + xbt_assert(libhandle != nullptr, "Cannot dlopen %s - check your settings in smpi/privatize-libs", + libname.c_str()); // get library name from path std::string fullpath = libname; #if not defined(__APPLE__) && not defined(__HAIKU__) - XBT_ATTRIB_UNUSED int dl_iterate_res = dl_iterate_phdr(visit_libs, &fullpath); - xbt_assert(dl_iterate_res != 0, "Can't find a linked %s - check your settings in smpi/privatize-libs", - fullpath.c_str()); + xbt_assert(dl_iterate_phdr(visit_libs, &fullpath) != 0, + "Can't find a linked %s - check your settings in smpi/privatize-libs", fullpath.c_str()); XBT_DEBUG("Extra lib to privatize '%s' found", fullpath.c_str()); #else xbt_die("smpi/privatize-libs is not (yet) compatible with OSX nor with Haiku"); @@ -448,7 +434,7 @@ static void smpi_init_privatization_dlopen(const std::string& executable) } simgrid::s4u::Engine::get_instance()->register_default([executable, fdin_size](std::vector args) { - return std::function([executable, fdin_size, args] { + return simgrid::kernel::actor::ActorCode([executable, fdin_size, args = std::move(args)] { static std::size_t rank = 0; // Copy the dynamic library: simgrid::xbt::Path path(executable); @@ -473,9 +459,7 @@ static void smpi_init_privatization_dlopen(const std::string& executable) // Copy the dynamic library, the new name must be the same length as the old one // just replace the name with 7 digits for the rank and the rest of the name. - unsigned int pad = 7; - if (libname.length() < pad) - pad = libname.length(); + auto pad = std::min(7, libname.length()); std::string target_libname = std::string(pad - std::to_string(rank).length(), '0') + std::to_string(rank) + libname.substr(pad); std::string target_lib = simgrid::config::get_value("smpi/tmpdir") + "/" + target_libname; target_libs.push_back(target_lib); @@ -503,7 +487,7 @@ static void smpi_init_privatization_dlopen(const std::string& executable) dlerror(), saved_errno, strerror(saved_errno)); smpi_entry_point_type entry_point = smpi_resolve_function(handle); - xbt_assert(entry_point, "Could not resolve entry point"); + xbt_assert(entry_point, "Could not resolve entry point. Does your program contain a main() function?"); smpi_run_entry_point(entry_point, executable, args); }); }); @@ -526,8 +510,16 @@ static void smpi_init_privatization_no_dlopen(const std::string& executable) // Execute the same entry point for each simulated process: simgrid::s4u::Engine::get_instance()->register_default([entry_point, executable](std::vector args) { - return std::function( - [entry_point, executable, args] { smpi_run_entry_point(entry_point, executable, args); }); + return simgrid::kernel::actor::ActorCode([entry_point, executable, args = std::move(args)] { + if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { + simgrid::smpi::ActorExt* ext = smpi_process(); + /* Now using the segment index of this process */ + ext->set_privatized_region(smpi_init_global_memory_segment_process()); + /* Done at the process's creation */ + smpi_switch_data_segment(simgrid::s4u::Actor::self()); + } + smpi_run_entry_point(entry_point, executable, args); + }); }); } @@ -538,18 +530,14 @@ int smpi_main(const char* executable, int argc, char* argv[]) * configuration tools */ return 0; } - - SMPI_switch_data_segment = &smpi_switch_data_segment; - smpi_init_options(); - simgrid::instr::init(); - SIMIX_global_init(&argc, argv); - auto engine = simgrid::s4u::Engine::get_instance(); + smpi_init_options_internal(true); + simgrid::s4u::Engine engine(&argc, argv); sg_storage_file_system_init(); // parse the platform file: get the host list - engine->load_platform(argv[1]); - SIMIX_comm_set_copy_data_callback(smpi_comm_copy_buffer_callback); + engine.load_platform(argv[1]); + engine.set_default_comm_data_copy_callback(smpi_comm_copy_buffer_callback); if (smpi_cfg_privatization() == SmpiPrivStrategies::DLOPEN) smpi_init_privatization_dlopen(executable); @@ -558,16 +546,27 @@ int smpi_main(const char* executable, int argc, char* argv[]) simgrid::smpi::colls::set_collectives(); simgrid::smpi::colls::smpi_coll_cleanup_callback = nullptr; - + + std::vector args4argv(argv + 1, argv + argc + 1); // last element is NULL + args4argv[0] = xbt_strdup(executable); + int real_argc = argc - 1; + char** real_argv = args4argv.data(); + + // Setup argc/argv for the Fortran run-time environment +#if SMPI_IFORT + for_rtl_init_(&real_argc, real_argv); +#elif SMPI_FLANG + __io_set_argc(real_argc); + __io_set_argv(real_argv); +#elif SMPI_GFORTRAN + _gfortran_set_args(real_argc, real_argv); +#endif + SMPI_init(); - /* This is a ... heavy way to count the MPI ranks */ - int rank_counts = 0; - simgrid::s4u::Actor::on_creation.connect([&rank_counts](const simgrid::s4u::Actor& actor) { - if (not actor.is_daemon()) - rank_counts++; - }); - engine->load_deployment(argv[2]); + const std::vector args(real_argv + 1, real_argv + real_argc); + int rank_counts = + smpi_deployment_smpirun(&engine, smpi_hostfile.get(), smpi_np.get(), smpi_replay.get(), smpi_map.get(), args); SMPI_app_instance_register(smpi_default_instance_name.c_str(), nullptr, rank_counts); MPI_COMM_WORLD = *smpi_deployment_comm_world(smpi_default_instance_name); @@ -579,29 +578,24 @@ int smpi_main(const char* executable, int argc, char* argv[]) if (MC_is_active()) { MC_run(); } else { - SIMIX_run(); + engine.get_impl()->run(); xbt_os_walltimer_stop(global_timer); - if (simgrid::config::get_value("smpi/display-timing")) { - double global_time = xbt_os_timer_elapsed(global_timer); - XBT_INFO("Simulated time: %g seconds. \n\n" - "The simulation took %g seconds (after parsing and platform setup)\n" - "%g seconds were actual computation of the application", - SIMIX_get_clock(), global_time , smpi_total_benched_time); - - if (smpi_total_benched_time/global_time>=0.75) - XBT_INFO("More than 75%% of the time was spent inside the application code.\n" - "You may want to use sampling functions or trace replay to reduce this."); - } + simgrid::smpi::utils::print_time_analysis(xbt_os_timer_elapsed(global_timer)); } SMPI_finalize(); +#if SMPI_IFORT + for_rtl_finish_(); +#endif + xbt_free(args4argv[0]); + return smpi_exit_status; } // Called either directly from the user code, or from the code called by smpirun void SMPI_init(){ - smpi_init_options(); + smpi_init_options_internal(false); simgrid::s4u::Actor::on_creation.connect([](simgrid::s4u::Actor& actor) { if (not actor.is_daemon()) actor.extension_set(new simgrid::smpi::ActorExt(&actor)); @@ -636,16 +630,29 @@ void SMPI_finalize() if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) smpi_destroy_global_memory_segments(); - if (simgrid::smpi::F2C::lookup() != nullptr) - simgrid::smpi::F2C::delete_lookup(); + + simgrid::smpi::utils::print_memory_analysis(); } void smpi_mpi_init() { smpi_init_fortran_types(); - if(smpi_init_sleep > 0) - simgrid::s4u::this_actor::sleep_for(smpi_init_sleep); + if(_smpi_init_sleep > 0) + simgrid::s4u::this_actor::sleep_for(_smpi_init_sleep); + if (not MC_is_active()) { + smpi_deployment_startup_barrier(smpi_process()->get_instance_id()); + } } void SMPI_thread_create() { TRACE_smpi_init(simgrid::s4u::this_actor::get_pid(), __func__); + smpi_process()->mark_as_initialized(); +} + +void smpi_exit(int res){ + if(res != 0){ + XBT_WARN("SMPI process did not return 0. Return value : %d", res); + smpi_exit_status = res; + } + simgrid::s4u::this_actor::exit(); + THROW_IMPOSSIBLE; }