X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/711f572ba93269d5151544df3a6df09018c803a0..0def08e689276880f2f012a9ed335ccef2c0103e:/src/smpi/mpi/smpi_comm.cpp diff --git a/src/smpi/mpi/smpi_comm.cpp b/src/smpi/mpi/smpi_comm.cpp index 0d7b5c76a8..017d010a79 100644 --- a/src/smpi/mpi/smpi_comm.cpp +++ b/src/smpi/mpi/smpi_comm.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2020. The SimGrid Team. All rights reserved. */ +/* Copyright (c) 2010-2021. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ @@ -12,12 +12,22 @@ #include "src/smpi/include/smpi_actor.hpp" #include "src/surf/HostImpl.hpp" -#include +#include XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_comm, smpi, "Logging specific to SMPI (comm)"); -simgrid::smpi::Comm mpi_MPI_COMM_UNINITIALIZED; -MPI_Comm MPI_COMM_UNINITIALIZED=&mpi_MPI_COMM_UNINITIALIZED; +simgrid::smpi::Comm smpi_MPI_COMM_UNINITIALIZED; +MPI_Comm MPI_COMM_UNINITIALIZED=&smpi_MPI_COMM_UNINITIALIZED; +/** + * Setting MPI_COMM_WORLD to MPI_COMM_UNINITIALIZED (it's a variable) + * is important because the implementation of MPI_Comm checks + * "this == MPI_COMM_UNINITIALIZED"? If yes, it uses smpi_process()->comm_world() + * instead of "this". + * This is basically how we only have one global variable but all processes have + * different communicators (the one their SMPI instance uses). + * + */ +MPI_Comm MPI_COMM_WORLD = MPI_COMM_UNINITIALIZED; /* Support for cartesian topology was added, but there are 2 other types of topology, graph et dist graph. In order to * support them, we have to add a field SMPI_Topo_type, and replace the MPI_Topology field by an union. */ @@ -34,6 +44,8 @@ Comm::Comm(MPI_Group group, MPI_Topology topo, bool smp, int in_id) errhandler_->ref(); //First creation of comm is done before SIMIX_run, so only do comms for others if(in_id==MPI_UNDEFINED && smp==0 && this->rank()!=MPI_UNDEFINED ){ + this->add_f(); + group->c2f(); int id; if(this->rank()==0){ static int global_id_ = 0; @@ -53,54 +65,64 @@ void Comm::destroy(Comm* comm) Comm::destroy(smpi_process()->comm_world()); return; } + if (comm != MPI_COMM_WORLD && not comm->deleted()) { + comm->cleanup_attr(); + comm->mark_as_deleted(); + } Comm::unref(comm); } int Comm::dup(MPI_Comm* newcomm){ - if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { - // we need to switch as the called function may silently touch global variables - smpi_switch_data_segment(s4u::Actor::self()); - } + // we need to switch as the called function may silently touch global variables + smpi_switch_data_segment(s4u::Actor::self()); + auto* cp = new Group(this->group()); (*newcomm) = new Comm(cp, this->topo()); - int ret = MPI_SUCCESS; - - if (not attributes()->empty()) { - int flag=0; - void* value_out=nullptr; - for (auto const& it : *attributes()) { - smpi_key_elem elem = keyvals_.at(it.first); - if (elem != nullptr){ - if( elem->copy_fn.comm_copy_fn != MPI_NULL_COPY_FN && - elem->copy_fn.comm_copy_fn != MPI_COMM_DUP_FN) - ret = elem->copy_fn.comm_copy_fn(this, it.first, elem->extra_state, it.second, &value_out, &flag); - else if ( elem->copy_fn.comm_copy_fn_fort != MPI_NULL_COPY_FN && - *(int*)*elem->copy_fn.comm_copy_fn_fort != 1){ - value_out=(int*)xbt_malloc(sizeof(int)); - elem->copy_fn.comm_copy_fn_fort(this, it.first, elem->extra_state, it.second, value_out, &flag,&ret); - } - if (ret != MPI_SUCCESS) { - Comm::destroy(*newcomm); - *newcomm = MPI_COMM_NULL; - return ret; - } - if (elem->copy_fn.comm_copy_fn == MPI_COMM_DUP_FN || - ((elem->copy_fn.comm_copy_fn_fort != MPI_NULL_COPY_FN) && *(int*)*elem->copy_fn.comm_copy_fn_fort == 1)){ - elem->refcount++; - (*newcomm)->attributes()->insert({it.first, it.second}); - }else if (flag){ - elem->refcount++; - (*newcomm)->attributes()->insert({it.first, value_out}); - } + + for (auto const& it : attributes()) { + auto elem_it = keyvals_.find(it.first); + xbt_assert(elem_it != keyvals_.end(), "Keyval not found for Comm: %d", it.first); + + smpi_key_elem& elem = elem_it->second; + int ret = MPI_SUCCESS; + int flag = 0; + void* value_out = nullptr; + if (elem.copy_fn.comm_copy_fn == MPI_COMM_DUP_FN) { + value_out = it.second; + flag = 1; + } else if (elem.copy_fn.comm_copy_fn != MPI_NULL_COPY_FN) { + ret = elem.copy_fn.comm_copy_fn(this, it.first, elem.extra_state, it.second, &value_out, &flag); + } + if (elem.copy_fn.comm_copy_fn_fort != MPI_NULL_COPY_FN) { + value_out = xbt_new(int, 1); + if (*(int*)*elem.copy_fn.comm_copy_fn_fort == 1) { // MPI_COMM_DUP_FN + memcpy(value_out, it.second, sizeof(int)); + flag = 1; + } else { // not null, nor dup + elem.copy_fn.comm_copy_fn_fort(this, it.first, elem.extra_state, it.second, value_out, &flag, &ret); } + if (ret != MPI_SUCCESS) + xbt_free(value_out); + } + if (ret != MPI_SUCCESS) { + Comm::destroy(*newcomm); + *newcomm = MPI_COMM_NULL; + return ret; + } + if (flag) { + elem.refcount++; + (*newcomm)->attributes().emplace(it.first, value_out); } } //duplicate info if present if(info_!=MPI_INFO_NULL) (*newcomm)->info_ = new simgrid::smpi::Info(info_); //duplicate errhandler - (*newcomm)->set_errhandler(errhandler_); - return ret; + if (errhandlers_ != nullptr)//MPI_COMM_WORLD, only grab our own + (*newcomm)->set_errhandler(errhandlers_[this->rank()]); + else + (*newcomm)->set_errhandler(errhandler_); + return MPI_SUCCESS; } int Comm::dup_with_info(MPI_Info info, MPI_Comm* newcomm){ @@ -136,7 +158,7 @@ int Comm::rank() const { if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->rank(); - return group_->rank(s4u::Actor::self()); + return group_->rank(s4u::this_actor::get_pid()); } int Comm::id() const @@ -153,14 +175,23 @@ void Comm::get_name(char* name, int* len) const if(this == MPI_COMM_WORLD && name_.empty()) { strncpy(name, "MPI_COMM_WORLD", 15); *len = 14; - } else if(this == MPI_COMM_SELF && name_.empty()) { - strncpy(name, "MPI_COMM_SELF", 14); - *len = 13; } else { *len = snprintf(name, MPI_MAX_NAME_STRING+1, "%s", name_.c_str()); } } +std::string Comm::name() const +{ + int size; + char name[MPI_MAX_NAME_STRING+1]; + this->get_name(name, &size); + if (name[0]=='\0') + return std::string("MPI_Comm"); + else + return std::string(name); +} + + void Comm::set_name (const char* name) { if (this == MPI_COMM_UNINITIALIZED){ @@ -233,27 +264,21 @@ MPI_Comm Comm::split(int color, int key) if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->split(color, key); int system_tag = -123; - int* recvbuf; MPI_Group group_root = nullptr; MPI_Group group_out = nullptr; - MPI_Group group = this->group(); + const Group* group = this->group(); int myrank = this->rank(); int size = this->size(); /* Gather all colors and keys on rank 0 */ - int* sendbuf = xbt_new(int, 2); - sendbuf[0] = color; - sendbuf[1] = key; - if (myrank == 0) { - recvbuf = xbt_new(int, 2 * size); - } else { - recvbuf = nullptr; - } - gather__default(sendbuf, 2, MPI_INT, recvbuf, 2, MPI_INT, 0, this); - xbt_free(sendbuf); + const std::array sendbuf = {{color, key}}; + std::vector recvbuf; + if (myrank == 0) + recvbuf.resize(2 * size); + gather__default(sendbuf.data(), 2, MPI_INT, recvbuf.data(), 2, MPI_INT, 0, this); /* Do the actual job */ if (myrank == 0) { - MPI_Group* group_snd = xbt_new(MPI_Group, size); + std::vector group_snd(size); std::vector> rankmap; rankmap.reserve(size); for (int i = 0; i < size; i++) { @@ -274,10 +299,10 @@ MPI_Comm Comm::split(int color, int key) group_root = group_out; /* Save root's group */ } for (unsigned j = 0; j < rankmap.size(); j++) { - s4u::Actor* actor = group->actor(rankmap[j].second); + aid_t actor = group->actor(rankmap[j].second); group_out->set_mapping(actor, j); } - MPI_Request* requests = xbt_new(MPI_Request, rankmap.size()); + std::vector requests(rankmap.size()); int reqs = 0; for (auto const& rank : rankmap) { if (rank.second != 0) { @@ -289,12 +314,9 @@ MPI_Comm Comm::split(int color, int key) if(i != 0 && group_out != MPI_COMM_WORLD->group() && group_out != MPI_GROUP_EMPTY) Group::unref(group_out); - Request::waitall(reqs, requests, MPI_STATUS_IGNORE); - xbt_free(requests); + Request::waitall(reqs, requests.data(), MPI_STATUS_IGNORE); } } - xbt_free(recvbuf); - xbt_free(group_snd); group_out = group_root; /* exit with root's group */ } else { if(color != MPI_UNDEFINED) { @@ -328,41 +350,48 @@ void Comm::unref(Comm* comm){ return; } comm->refcount_--; - Group::unref(comm->group_); if(comm->refcount_==0){ + if(simgrid::smpi::F2C::lookup() != nullptr) + F2C::free_f(comm->f2c_id()); comm->cleanup_smp(); comm->cleanup_attr(); if (comm->info_ != MPI_INFO_NULL) simgrid::smpi::Info::unref(comm->info_); - if (comm->errhandler_ != MPI_ERRHANDLER_NULL) + if(comm->errhandlers_!=nullptr){ + for (int i=0; isize(); i++) + if (comm->errhandlers_[i]!=MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(comm->errhandlers_[i]); + delete[] comm->errhandlers_; + } else if (comm->errhandler_ != MPI_ERRHANDLER_NULL) simgrid::smpi::Errhandler::unref(comm->errhandler_); - delete comm; } + Group::unref(comm->group_); + if(comm->refcount_==0) + delete comm; } MPI_Comm Comm::find_intra_comm(int * leader){ //get the indices of all processes sharing the same simix host - auto& actor_list = sg_host_self()->pimpl_->actor_list_; int intra_comm_size = 0; - int min_index = INT_MAX; // the minimum index will be the leader - for (auto& actor : actor_list) { - int index = actor.get_pid(); - if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { // Is this process in the current group? + aid_t min_index = std::numeric_limits::max(); // the minimum index will be the leader + sg_host_self()->get_impl()->foreach_actor([this, &intra_comm_size, &min_index](auto& actor) { + aid_t index = actor.get_pid(); + if (this->group()->rank(index) != MPI_UNDEFINED) { // Is this process in the current group? intra_comm_size++; if (index < min_index) min_index = index; } - } + }); XBT_DEBUG("number of processes deployed on my node : %d", intra_comm_size); auto* group_intra = new Group(intra_comm_size); int i = 0; - for (auto& actor : actor_list) { - if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { - group_intra->set_mapping(actor.get_ciface(), i); + sg_host_self()->get_impl()->foreach_actor([this, group_intra, &i](auto& actor) { + if (this->group()->rank(actor.get_pid()) != MPI_UNDEFINED) { + group_intra->set_mapping(actor.get_pid(), i); i++; } - } + }); *leader=min_index; return new Comm(group_intra, nullptr, true); } @@ -383,10 +412,9 @@ void Comm::init_smp(){ smpi_process()->set_replaying(false); } - if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { - // we need to switch as the called function may silently touch global variables - smpi_switch_data_segment(s4u::Actor::self()); - } + // we need to switch as the called function may silently touch global variables + smpi_switch_data_segment(s4u::Actor::self()); + // identify neighbors in comm MPI_Comm comm_intra = find_intra_comm(&leader); @@ -397,11 +425,6 @@ void Comm::init_smp(){ allgather__ring(&leader, 1, MPI_INT , leaders_map, 1, MPI_INT, this); - if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { - // we need to switch as the called function may silently touch global variables - smpi_switch_data_segment(s4u::Actor::self()); - } - if(leaders_map_==nullptr){ leaders_map_= leaders_map; }else{ @@ -429,7 +452,7 @@ void Comm::init_smp(){ if(MPI_COMM_WORLD!=MPI_COMM_UNINITIALIZED && this!=MPI_COMM_WORLD){ //create leader_communicator for (i=0; i< leader_group_size;i++) - leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]).get(), i); + leaders_group->set_mapping(leader_list[i], i); leader_comm = new Comm(leaders_group, nullptr, true); this->set_leaders_comm(leader_comm); this->set_intra_comm(comm_intra); @@ -437,7 +460,7 @@ void Comm::init_smp(){ // create intracommunicator }else{ for (i=0; i< leader_group_size;i++) - leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]).get(), i); + leaders_group->set_mapping(leader_list[i], i); if(this->get_leaders_comm()==MPI_COMM_NULL){ leader_comm = new Comm(leaders_group, nullptr, true); @@ -453,7 +476,7 @@ void Comm::init_smp(){ int my_local_size=comm_intra->size(); if(comm_intra->rank()==0) { int is_uniform = 1; - int* non_uniform_map = xbt_new0(int,leader_group_size); + auto* non_uniform_map = xbt_new0(int, leader_group_size); allgather__ring(&my_local_size, 1, MPI_INT, non_uniform_map, 1, MPI_INT, leader_comm); for(i=0; i < leader_group_size; i++) { @@ -469,15 +492,14 @@ void Comm::init_smp(){ } is_uniform_=is_uniform; } - bcast__scatter_LR_allgather(&(is_uniform_),1, MPI_INT, 0, comm_intra ); + bcast__scatter_LR_allgather(&is_uniform_, 1, MPI_INT, 0, comm_intra); + + // we need to switch as the called function may silently touch global variables + smpi_switch_data_segment(s4u::Actor::self()); - if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { - // we need to switch as the called function may silently touch global variables - smpi_switch_data_segment(s4u::Actor::self()); - } // Are the ranks blocked ? = allocated contiguously on the SMP nodes int is_blocked=1; - int prev=this->group()->rank(comm_intra->group()->actor(0)); + int prev = this->group()->rank(comm_intra->group()->actor(0)); for (i = 1; i < my_local_size; i++) { int that = this->group()->rank(comm_intra->group()->actor(i)); if (that != prev + 1) { @@ -488,7 +510,7 @@ void Comm::init_smp(){ } int global_blocked; - allreduce__default(&is_blocked, &(global_blocked), 1, MPI_INT, MPI_LAND, this); + allreduce__default(&is_blocked, &global_blocked, 1, MPI_INT, MPI_LAND, this); if(MPI_COMM_WORLD==MPI_COMM_UNINITIALIZED || this==MPI_COMM_WORLD){ if(this->rank()==0){ @@ -508,8 +530,8 @@ MPI_Comm Comm::f2c(int id) { return MPI_COMM_SELF; } else if(id==0){ return MPI_COMM_WORLD; - } else if(F2C::f2c_lookup() != nullptr && id >= 0) { - const auto& lookup = F2C::f2c_lookup(); + } else if (F2C::lookup() != nullptr && id >= 0) { + const auto& lookup = F2C::lookup(); auto comm = lookup->find(id); return comm == lookup->end() ? MPI_COMM_NULL : static_cast(comm->second); } else { @@ -518,7 +540,7 @@ MPI_Comm Comm::f2c(int id) { } void Comm::free_f(int id) { - F2C::f2c_lookup()->erase(id); + F2C::lookup()->erase(id); } void Comm::add_rma_win(MPI_Win win){ @@ -531,19 +553,17 @@ void Comm::remove_rma_win(MPI_Win win){ void Comm::finish_rma_calls() const { + const int myrank = rank(); for (auto const& it : rma_wins_) { - if(it->rank()==this->rank()){//is it ours (for MPI_COMM_WORLD)? + if (it->rank() == myrank) { // is it ours (for MPI_COMM_WORLD)? int finished = it->finish_comms(); - XBT_DEBUG("Barrier for rank %d - Finished %d RMA calls",this->rank(), finished); + XBT_DEBUG("Barrier for rank %d - Finished %d RMA calls", myrank, finished); } } } MPI_Info Comm::info() { - if (info_ == MPI_INFO_NULL) - info_ = new Info(); - info_->ref(); return info_; } @@ -558,18 +578,36 @@ void Comm::set_info(MPI_Info info) MPI_Errhandler Comm::errhandler() { - if (errhandler_ != MPI_ERRHANDLER_NULL) - errhandler_->ref(); - return errhandler_; + if (this != MPI_COMM_WORLD){ + if (errhandler_ != MPI_ERRHANDLER_NULL) + errhandler_->ref(); + return errhandler_; + } else { + if(errhandlers_==nullptr) + return MPI_ERRORS_ARE_FATAL; + else { + if(errhandlers_[this->rank()] != MPI_ERRHANDLER_NULL) + errhandlers_[this->rank()]->ref(); + return errhandlers_[this->rank()]; + } + } } void Comm::set_errhandler(MPI_Errhandler errhandler) { - if (errhandler_ != MPI_ERRHANDLER_NULL) - simgrid::smpi::Errhandler::unref(errhandler_); - errhandler_ = errhandler; - if (errhandler_ != MPI_ERRHANDLER_NULL) - errhandler_->ref(); + if(this != MPI_COMM_WORLD){ + if (errhandler_ != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(errhandler_); + errhandler_ = errhandler; + }else{ + if(errhandlers_==nullptr) + errhandlers_= new MPI_Errhandler[this->size()]{MPI_ERRHANDLER_NULL}; + if(errhandlers_[this->rank()] != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(errhandlers_[this->rank()]); + errhandlers_[this->rank()]=errhandler; + } + if (errhandler != MPI_ERRHANDLER_NULL) + errhandler->ref(); } MPI_Comm Comm::split_type(int type, int /*key*/, const Info*)