X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/98755faee042e94d1ff52f6e9508b18015bb1ae5..ea02d47c2ee462e84798356e4dda87b1467e22b8:/src/smpi/mpi/smpi_comm.cpp diff --git a/src/smpi/mpi/smpi_comm.cpp b/src/smpi/mpi/smpi_comm.cpp index a670b30bcd..117d76f8b6 100644 --- a/src/smpi/mpi/smpi_comm.cpp +++ b/src/smpi/mpi/smpi_comm.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2019. The SimGrid Team. All rights reserved. */ +/* Copyright (c) 2010-2021. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ @@ -16,8 +16,18 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_comm, smpi, "Logging specific to SMPI (comm)"); -simgrid::smpi::Comm mpi_MPI_COMM_UNINITIALIZED; -MPI_Comm MPI_COMM_UNINITIALIZED=&mpi_MPI_COMM_UNINITIALIZED; +simgrid::smpi::Comm smpi_MPI_COMM_UNINITIALIZED; +MPI_Comm MPI_COMM_UNINITIALIZED=&smpi_MPI_COMM_UNINITIALIZED; +/** + * Setting MPI_COMM_WORLD to MPI_COMM_UNINITIALIZED (it's a variable) + * is important because the implementation of MPI_Comm checks + * "this == MPI_COMM_UNINITIALIZED"? If yes, it uses smpi_process()->comm_world() + * instead of "this". + * This is basically how we only have one global variable but all processes have + * different communicators (the one their SMPI instance uses). + * + */ +MPI_Comm MPI_COMM_WORLD = MPI_COMM_UNINITIALIZED; /* Support for cartesian topology was added, but there are 2 other types of topology, graph et dist graph. In order to * support them, we have to add a field SMPI_Topo_type, and replace the MPI_Topology field by an union. */ @@ -28,30 +38,24 @@ namespace smpi{ std::unordered_map Comm::keyvals_; int Comm::keyval_id_=0; -Comm::Comm(MPI_Group group, MPI_Topology topo, int smp, int in_id) : group_(group), topo_(topo),is_smp_comm_(smp), id_(in_id) +Comm::Comm(MPI_Group group, MPI_Topology topo, bool smp, int in_id) + : group_(group), topo_(topo), is_smp_comm_(smp), id_(in_id) { - refcount_ = 1; - topoType_ = MPI_INVALID_TOPO; - intra_comm_ = MPI_COMM_NULL; - leaders_comm_ = MPI_COMM_NULL; - is_uniform_ = 1; - non_uniform_map_ = nullptr; - leaders_map_ = nullptr; - is_blocked_ = 0; - info_ = MPI_INFO_NULL; - errhandler_ = MPI_ERRORS_RETURN; - static int global_id_=0; + errhandler_->ref(); //First creation of comm is done before SIMIX_run, so only do comms for others if(in_id==MPI_UNDEFINED && smp==0 && this->rank()!=MPI_UNDEFINED ){ + this->add_f(); + group->c2f(); int id; if(this->rank()==0){ + static int global_id_ = 0; id=global_id_; global_id_++; } - Colls::bcast(&id, 1, MPI_INT, 0, this); + colls::bcast(&id, 1, MPI_INT, 0, this); XBT_DEBUG("Communicator %p has id %d", this, id); id_=id;//only set here, as we don't want to change it in the middle of the bcast - Colls::barrier(this); + colls::barrier(this); } } @@ -61,16 +65,17 @@ void Comm::destroy(Comm* comm) Comm::destroy(smpi_process()->comm_world()); return; } - delete comm->topo_; // there's no use count on topos + if(comm != MPI_COMM_WORLD) + comm->mark_as_deleted(); Comm::unref(comm); } int Comm::dup(MPI_Comm* newcomm){ - if (smpi_privatize_global_variables == SmpiPrivStrategies::MMAP) { + if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { // we need to switch as the called function may silently touch global variables smpi_switch_data_segment(s4u::Actor::self()); } - MPI_Group cp = new Group(this->group()); + auto* cp = new Group(this->group()); (*newcomm) = new Comm(cp, this->topo()); int ret = MPI_SUCCESS; @@ -108,7 +113,10 @@ int Comm::dup(MPI_Comm* newcomm){ if(info_!=MPI_INFO_NULL) (*newcomm)->info_ = new simgrid::smpi::Info(info_); //duplicate errhandler - (*newcomm)->set_errhandler(errhandler_); + if (errhandlers_ != nullptr)//MPI_COMM_WORLD, only grab our own + (*newcomm)->set_errhandler(errhandlers_[this->rank()]); + else + (*newcomm)->set_errhandler(errhandler_); return ret; } @@ -121,6 +129,7 @@ int Comm::dup_with_info(MPI_Info info, MPI_Comm* newcomm){ (*newcomm)->info_=MPI_INFO_NULL; } if(info != MPI_INFO_NULL){ + info->ref(); (*newcomm)->info_=info; } return ret; @@ -133,26 +142,26 @@ MPI_Group Comm::group() return group_; } -int Comm::size() +int Comm::size() const { if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->size(); return group_->size(); } -int Comm::rank() +int Comm::rank() const { if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->rank(); return group_->rank(s4u::Actor::self()); } -int Comm::id() +int Comm::id() const { return id_; } -void Comm::get_name (char* name, int* len) +void Comm::get_name(char* name, int* len) const { if (this == MPI_COMM_UNINITIALIZED){ smpi_process()->comm_world()->get_name(name, len); @@ -187,43 +196,50 @@ void Comm::set_leaders_comm(MPI_Comm leaders){ leaders_comm_=leaders; } -int* Comm::get_non_uniform_map(){ +int* Comm::get_non_uniform_map() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->get_non_uniform_map(); return non_uniform_map_; } -int* Comm::get_leaders_map(){ +int* Comm::get_leaders_map() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->get_leaders_map(); return leaders_map_; } -MPI_Comm Comm::get_leaders_comm(){ +MPI_Comm Comm::get_leaders_comm() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->get_leaders_comm(); return leaders_comm_; } -MPI_Comm Comm::get_intra_comm(){ +MPI_Comm Comm::get_intra_comm() const +{ if (this == MPI_COMM_UNINITIALIZED || this==MPI_COMM_WORLD) return smpi_process()->comm_intra(); else return intra_comm_; } -int Comm::is_uniform(){ +bool Comm::is_uniform() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->is_uniform(); - return is_uniform_; + return is_uniform_ != 0; } -int Comm::is_blocked(){ +bool Comm::is_blocked() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->is_blocked(); - return is_blocked_; + return is_blocked_ != 0; } -int Comm::is_smp_comm(){ +bool Comm::is_smp_comm() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->is_smp_comm(); return is_smp_comm_; @@ -234,7 +250,6 @@ MPI_Comm Comm::split(int color, int key) if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->split(color, key); int system_tag = -123; - int* recvbuf; MPI_Group group_root = nullptr; MPI_Group group_out = nullptr; @@ -242,19 +257,14 @@ MPI_Comm Comm::split(int color, int key) int myrank = this->rank(); int size = this->size(); /* Gather all colors and keys on rank 0 */ - int* sendbuf = xbt_new(int, 2); - sendbuf[0] = color; - sendbuf[1] = key; - if (myrank == 0) { - recvbuf = xbt_new(int, 2 * size); - } else { - recvbuf = nullptr; - } - Coll_gather_default::gather(sendbuf, 2, MPI_INT, recvbuf, 2, MPI_INT, 0, this); - xbt_free(sendbuf); + const std::array sendbuf = {{color, key}}; + std::vector recvbuf; + if (myrank == 0) + recvbuf.resize(2 * size); + gather__default(sendbuf.data(), 2, MPI_INT, recvbuf.data(), 2, MPI_INT, 0, this); /* Do the actual job */ if (myrank == 0) { - MPI_Group* group_snd = xbt_new(MPI_Group, size); + std::vector group_snd(size); std::vector> rankmap; rankmap.reserve(size); for (int i = 0; i < size; i++) { @@ -263,22 +273,22 @@ MPI_Comm Comm::split(int color, int key) for (int j = i + 1; j < size; j++) { if(recvbuf[2 * i] == recvbuf[2 * j]) { recvbuf[2 * j] = MPI_UNDEFINED; - rankmap.push_back({recvbuf[2 * j + 1], j}); + rankmap.emplace_back(recvbuf[2 * j + 1], j); } } /* Add self in the group */ recvbuf[2 * i] = MPI_UNDEFINED; - rankmap.push_back({recvbuf[2 * i + 1], i}); + rankmap.emplace_back(recvbuf[2 * i + 1], i); std::sort(begin(rankmap), end(rankmap)); group_out = new Group(rankmap.size()); if (i == 0) { group_root = group_out; /* Save root's group */ } for (unsigned j = 0; j < rankmap.size(); j++) { - s4u::ActorPtr actor = group->actor(rankmap[j].second); + s4u::Actor* actor = group->actor(rankmap[j].second); group_out->set_mapping(actor, j); } - MPI_Request* requests = xbt_new(MPI_Request, rankmap.size()); + std::vector requests(rankmap.size()); int reqs = 0; for (auto const& rank : rankmap) { if (rank.second != 0) { @@ -290,19 +300,16 @@ MPI_Comm Comm::split(int color, int key) if(i != 0 && group_out != MPI_COMM_WORLD->group() && group_out != MPI_GROUP_EMPTY) Group::unref(group_out); - Request::waitall(reqs, requests, MPI_STATUS_IGNORE); - xbt_free(requests); + Request::waitall(reqs, requests.data(), MPI_STATUS_IGNORE); } } - xbt_free(recvbuf); - xbt_free(group_snd); group_out = group_root; /* exit with root's group */ } else { if(color != MPI_UNDEFINED) { Request::recv(&group_out, 1, MPI_PTR, 0, system_tag, this, MPI_STATUS_IGNORE); } /* otherwise, exit with group_out == nullptr */ } - return group_out!=nullptr ? new Comm(group_out, nullptr) : MPI_COMM_NULL; + return group_out!=nullptr ? new Comm(group_out, topo_) : MPI_COMM_NULL; } void Comm::ref(){ @@ -329,39 +336,50 @@ void Comm::unref(Comm* comm){ return; } comm->refcount_--; - Group::unref(comm->group_); if(comm->refcount_==0){ + if(simgrid::smpi::F2C::lookup() != nullptr) + F2C::free_f(comm->f2c_id()); comm->cleanup_smp(); comm->cleanup_attr(); - delete comm; + if (comm->info_ != MPI_INFO_NULL) + simgrid::smpi::Info::unref(comm->info_); + if(comm->errhandlers_!=nullptr){ + for (int i=0; isize(); i++) + if (comm->errhandlers_[i]!=MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(comm->errhandlers_[i]); + delete[] comm->errhandlers_; + } else if (comm->errhandler_ != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(comm->errhandler_); } + Group::unref(comm->group_); + if(comm->refcount_==0) + delete comm; } MPI_Comm Comm::find_intra_comm(int * leader){ //get the indices of all processes sharing the same simix host - auto& process_list = sg_host_self()->pimpl_->process_list_; int intra_comm_size = 0; int min_index = INT_MAX; // the minimum index will be the leader - for (auto& actor : process_list) { + sg_host_self()->get_impl()->foreach_actor([this, &intra_comm_size, &min_index](auto& actor) { int index = actor.get_pid(); - if (this->group()->rank(actor.iface()) != MPI_UNDEFINED) { // Is this process in the current group? + if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { // Is this process in the current group? intra_comm_size++; if (index < min_index) min_index = index; } - } + }); XBT_DEBUG("number of processes deployed on my node : %d", intra_comm_size); - MPI_Group group_intra = new Group(intra_comm_size); + auto* group_intra = new Group(intra_comm_size); int i = 0; - for (auto& actor : process_list) { - if (this->group()->rank(actor.iface()) != MPI_UNDEFINED) { - group_intra->set_mapping(actor.iface(), i); + sg_host_self()->get_impl()->foreach_actor([this, group_intra, &i](auto& actor) { + if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { + group_intra->set_mapping(actor.get_ciface(), i); i++; } - } + }); *leader=min_index; - return new Comm(group_intra, nullptr, 1); + return new Comm(group_intra, nullptr, true); } void Comm::init_smp(){ @@ -380,22 +398,21 @@ void Comm::init_smp(){ smpi_process()->set_replaying(false); } - if (smpi_privatize_global_variables == SmpiPrivStrategies::MMAP) { + if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { // we need to switch as the called function may silently touch global variables smpi_switch_data_segment(s4u::Actor::self()); } - //identify neighbours in comm + // identify neighbors in comm MPI_Comm comm_intra = find_intra_comm(&leader); - - int* leaders_map = new int[comm_size]; - int* leader_list = new int[comm_size]; + auto* leaders_map = new int[comm_size]; + auto* leader_list = new int[comm_size]; std::fill_n(leaders_map, comm_size, 0); std::fill_n(leader_list, comm_size, -1); - Coll_allgather_ring::allgather(&leader, 1, MPI_INT , leaders_map, 1, MPI_INT, this); + allgather__ring(&leader, 1, MPI_INT , leaders_map, 1, MPI_INT, this); - if (smpi_privatize_global_variables == SmpiPrivStrategies::MMAP) { + if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { // we need to switch as the called function may silently touch global variables smpi_switch_data_segment(s4u::Actor::self()); } @@ -418,26 +435,27 @@ void Comm::init_smp(){ leader_group_size++; } } + xbt_assert(leader_group_size > 0); std::sort(leader_list, leader_list + leader_group_size); - MPI_Group leaders_group = new Group(leader_group_size); + auto* leaders_group = new Group(leader_group_size); MPI_Comm leader_comm = MPI_COMM_NULL; if(MPI_COMM_WORLD!=MPI_COMM_UNINITIALIZED && this!=MPI_COMM_WORLD){ //create leader_communicator for (i=0; i< leader_group_size;i++) - leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]), i); - leader_comm = new Comm(leaders_group, nullptr,1); + leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]).get(), i); + leader_comm = new Comm(leaders_group, nullptr, true); this->set_leaders_comm(leader_comm); this->set_intra_comm(comm_intra); // create intracommunicator }else{ for (i=0; i< leader_group_size;i++) - leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]), i); + leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]).get(), i); if(this->get_leaders_comm()==MPI_COMM_NULL){ - leader_comm = new Comm(leaders_group, nullptr,1); + leader_comm = new Comm(leaders_group, nullptr, true); this->set_leaders_comm(leader_comm); }else{ leader_comm=this->get_leaders_comm(); @@ -451,7 +469,7 @@ void Comm::init_smp(){ if(comm_intra->rank()==0) { int is_uniform = 1; int* non_uniform_map = xbt_new0(int,leader_group_size); - Coll_allgather_ring::allgather(&my_local_size, 1, MPI_INT, + allgather__ring(&my_local_size, 1, MPI_INT, non_uniform_map, 1, MPI_INT, leader_comm); for(i=0; i < leader_group_size; i++) { if(non_uniform_map[0] != non_uniform_map[i]) { @@ -459,16 +477,16 @@ void Comm::init_smp(){ break; } } - if(is_uniform==0 && this->is_uniform()!=0){ + if (is_uniform == 0 && this->is_uniform()) { non_uniform_map_ = non_uniform_map; }else{ xbt_free(non_uniform_map); } is_uniform_=is_uniform; } - Coll_bcast_scatter_LR_allgather::bcast(&(is_uniform_),1, MPI_INT, 0, comm_intra ); + bcast__scatter_LR_allgather(&is_uniform_, 1, MPI_INT, 0, comm_intra); - if (smpi_privatize_global_variables == SmpiPrivStrategies::MMAP) { + if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { // we need to switch as the called function may silently touch global variables smpi_switch_data_segment(s4u::Actor::self()); } @@ -485,7 +503,7 @@ void Comm::init_smp(){ } int global_blocked; - Coll_allreduce_default::allreduce(&is_blocked, &(global_blocked), 1, MPI_INT, MPI_LAND, this); + allreduce__default(&is_blocked, &global_blocked, 1, MPI_INT, MPI_LAND, this); if(MPI_COMM_WORLD==MPI_COMM_UNINITIALIZED || this==MPI_COMM_WORLD){ if(this->rank()==0){ @@ -505,10 +523,9 @@ MPI_Comm Comm::f2c(int id) { return MPI_COMM_SELF; } else if(id==0){ return MPI_COMM_WORLD; - } else if(F2C::f2c_lookup() != nullptr && id >= 0) { - char key[KEY_SIZE]; - const auto& lookup = F2C::f2c_lookup(); - auto comm = lookup->find(get_key(key, id)); + } else if (F2C::lookup() != nullptr && id >= 0) { + const auto& lookup = F2C::lookup(); + auto comm = lookup->find(id); return comm == lookup->end() ? MPI_COMM_NULL : static_cast(comm->second); } else { return MPI_COMM_NULL; @@ -516,8 +533,7 @@ MPI_Comm Comm::f2c(int id) { } void Comm::free_f(int id) { - char key[KEY_SIZE]; - F2C::f2c_lookup()->erase(get_key(key, id)); + F2C::lookup()->erase(id); } void Comm::add_rma_win(MPI_Win win){ @@ -528,7 +544,8 @@ void Comm::remove_rma_win(MPI_Win win){ rma_wins_.remove(win); } -void Comm::finish_rma_calls(){ +void Comm::finish_rma_calls() const +{ for (auto const& it : rma_wins_) { if(it->rank()==this->rank()){//is it ours (for MPI_COMM_WORLD)? int finished = it->finish_comms(); @@ -537,30 +554,58 @@ void Comm::finish_rma_calls(){ } } -MPI_Info Comm::info(){ - if(info_== MPI_INFO_NULL) +MPI_Info Comm::info() +{ + if (info_ == MPI_INFO_NULL) info_ = new Info(); info_->ref(); return info_; } -void Comm::set_info(MPI_Info info){ - if(info_!= MPI_INFO_NULL) +void Comm::set_info(MPI_Info info) +{ + if (info_ != MPI_INFO_NULL) + simgrid::smpi::Info::unref(info); + info_ = info; + if (info_ != MPI_INFO_NULL) info->ref(); - info_=info; } -MPI_Errhandler Comm::errhandler(){ - return errhandler_; +MPI_Errhandler Comm::errhandler() +{ + if (this != MPI_COMM_WORLD){ + if (errhandler_ != MPI_ERRHANDLER_NULL) + errhandler_->ref(); + return errhandler_; + } else { + if(errhandlers_==nullptr) + return MPI_ERRORS_ARE_FATAL; + else { + if(errhandlers_[this->rank()] != MPI_ERRHANDLER_NULL) + errhandlers_[this->rank()]->ref(); + return errhandlers_[this->rank()]; + } + } } -void Comm::set_errhandler(MPI_Errhandler errhandler){ - errhandler_=errhandler; - if(errhandler_!= MPI_ERRHANDLER_NULL) +void Comm::set_errhandler(MPI_Errhandler errhandler) +{ + if(this != MPI_COMM_WORLD){ + if (errhandler_ != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(errhandler_); + errhandler_ = errhandler; + }else{ + if(errhandlers_==nullptr) + errhandlers_= new MPI_Errhandler[this->size()]{MPI_ERRHANDLER_NULL}; + if(errhandlers_[this->rank()] != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(errhandlers_[this->rank()]); + errhandlers_[this->rank()]=errhandler; + } + if (errhandler != MPI_ERRHANDLER_NULL) errhandler->ref(); } -MPI_Comm Comm::split_type(int type, int /*key*/, MPI_Info) +MPI_Comm Comm::split_type(int type, int /*key*/, const Info*) { //MPI_UNDEFINED can be given to some nodes... but we need them to still perform the smp part which is collective if(type != MPI_COMM_TYPE_SHARED && type != MPI_UNDEFINED){ @@ -571,6 +616,7 @@ MPI_Comm Comm::split_type(int type, int /*key*/, MPI_Info) if(type != MPI_UNDEFINED) return res; else{ + xbt_assert(res->refcount_ == 1); // ensure the next call to Comm::destroy really frees the comm Comm::destroy(res); return MPI_COMM_NULL; }