X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/55a08c7439db5b0335a5b26e40b3a46e1e03d3e8..ea02d47c2ee462e84798356e4dda87b1467e22b8:/src/smpi/mpi/smpi_comm.cpp diff --git a/src/smpi/mpi/smpi_comm.cpp b/src/smpi/mpi/smpi_comm.cpp index d3e4978d1e..117d76f8b6 100644 --- a/src/smpi/mpi/smpi_comm.cpp +++ b/src/smpi/mpi/smpi_comm.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2020. The SimGrid Team. All rights reserved. */ +/* Copyright (c) 2010-2021. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ @@ -16,8 +16,18 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_comm, smpi, "Logging specific to SMPI (comm)"); -simgrid::smpi::Comm mpi_MPI_COMM_UNINITIALIZED; -MPI_Comm MPI_COMM_UNINITIALIZED=&mpi_MPI_COMM_UNINITIALIZED; +simgrid::smpi::Comm smpi_MPI_COMM_UNINITIALIZED; +MPI_Comm MPI_COMM_UNINITIALIZED=&smpi_MPI_COMM_UNINITIALIZED; +/** + * Setting MPI_COMM_WORLD to MPI_COMM_UNINITIALIZED (it's a variable) + * is important because the implementation of MPI_Comm checks + * "this == MPI_COMM_UNINITIALIZED"? If yes, it uses smpi_process()->comm_world() + * instead of "this". + * This is basically how we only have one global variable but all processes have + * different communicators (the one their SMPI instance uses). + * + */ +MPI_Comm MPI_COMM_WORLD = MPI_COMM_UNINITIALIZED; /* Support for cartesian topology was added, but there are 2 other types of topology, graph et dist graph. In order to * support them, we have to add a field SMPI_Topo_type, and replace the MPI_Topology field by an union. */ @@ -34,6 +44,8 @@ Comm::Comm(MPI_Group group, MPI_Topology topo, bool smp, int in_id) errhandler_->ref(); //First creation of comm is done before SIMIX_run, so only do comms for others if(in_id==MPI_UNDEFINED && smp==0 && this->rank()!=MPI_UNDEFINED ){ + this->add_f(); + group->c2f(); int id; if(this->rank()==0){ static int global_id_ = 0; @@ -53,6 +65,8 @@ void Comm::destroy(Comm* comm) Comm::destroy(smpi_process()->comm_world()); return; } + if(comm != MPI_COMM_WORLD) + comm->mark_as_deleted(); Comm::unref(comm); } @@ -61,7 +75,7 @@ int Comm::dup(MPI_Comm* newcomm){ // we need to switch as the called function may silently touch global variables smpi_switch_data_segment(s4u::Actor::self()); } - MPI_Group cp = new Group(this->group()); + auto* cp = new Group(this->group()); (*newcomm) = new Comm(cp, this->topo()); int ret = MPI_SUCCESS; @@ -99,7 +113,10 @@ int Comm::dup(MPI_Comm* newcomm){ if(info_!=MPI_INFO_NULL) (*newcomm)->info_ = new simgrid::smpi::Info(info_); //duplicate errhandler - (*newcomm)->set_errhandler(errhandler_); + if (errhandlers_ != nullptr)//MPI_COMM_WORLD, only grab our own + (*newcomm)->set_errhandler(errhandlers_[this->rank()]); + else + (*newcomm)->set_errhandler(errhandler_); return ret; } @@ -233,7 +250,6 @@ MPI_Comm Comm::split(int color, int key) if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->split(color, key); int system_tag = -123; - int* recvbuf; MPI_Group group_root = nullptr; MPI_Group group_out = nullptr; @@ -241,19 +257,14 @@ MPI_Comm Comm::split(int color, int key) int myrank = this->rank(); int size = this->size(); /* Gather all colors and keys on rank 0 */ - int* sendbuf = xbt_new(int, 2); - sendbuf[0] = color; - sendbuf[1] = key; - if (myrank == 0) { - recvbuf = xbt_new(int, 2 * size); - } else { - recvbuf = nullptr; - } - gather__default(sendbuf, 2, MPI_INT, recvbuf, 2, MPI_INT, 0, this); - xbt_free(sendbuf); + const std::array sendbuf = {{color, key}}; + std::vector recvbuf; + if (myrank == 0) + recvbuf.resize(2 * size); + gather__default(sendbuf.data(), 2, MPI_INT, recvbuf.data(), 2, MPI_INT, 0, this); /* Do the actual job */ if (myrank == 0) { - MPI_Group* group_snd = xbt_new(MPI_Group, size); + std::vector group_snd(size); std::vector> rankmap; rankmap.reserve(size); for (int i = 0; i < size; i++) { @@ -262,12 +273,12 @@ MPI_Comm Comm::split(int color, int key) for (int j = i + 1; j < size; j++) { if(recvbuf[2 * i] == recvbuf[2 * j]) { recvbuf[2 * j] = MPI_UNDEFINED; - rankmap.push_back({recvbuf[2 * j + 1], j}); + rankmap.emplace_back(recvbuf[2 * j + 1], j); } } /* Add self in the group */ recvbuf[2 * i] = MPI_UNDEFINED; - rankmap.push_back({recvbuf[2 * i + 1], i}); + rankmap.emplace_back(recvbuf[2 * i + 1], i); std::sort(begin(rankmap), end(rankmap)); group_out = new Group(rankmap.size()); if (i == 0) { @@ -277,7 +288,7 @@ MPI_Comm Comm::split(int color, int key) s4u::Actor* actor = group->actor(rankmap[j].second); group_out->set_mapping(actor, j); } - MPI_Request* requests = xbt_new(MPI_Request, rankmap.size()); + std::vector requests(rankmap.size()); int reqs = 0; for (auto const& rank : rankmap) { if (rank.second != 0) { @@ -289,12 +300,9 @@ MPI_Comm Comm::split(int color, int key) if(i != 0 && group_out != MPI_COMM_WORLD->group() && group_out != MPI_GROUP_EMPTY) Group::unref(group_out); - Request::waitall(reqs, requests, MPI_STATUS_IGNORE); - xbt_free(requests); + Request::waitall(reqs, requests.data(), MPI_STATUS_IGNORE); } } - xbt_free(recvbuf); - xbt_free(group_snd); group_out = group_root; /* exit with root's group */ } else { if(color != MPI_UNDEFINED) { @@ -328,41 +336,48 @@ void Comm::unref(Comm* comm){ return; } comm->refcount_--; - Group::unref(comm->group_); if(comm->refcount_==0){ + if(simgrid::smpi::F2C::lookup() != nullptr) + F2C::free_f(comm->f2c_id()); comm->cleanup_smp(); comm->cleanup_attr(); if (comm->info_ != MPI_INFO_NULL) simgrid::smpi::Info::unref(comm->info_); - if (comm->errhandler_ != MPI_ERRHANDLER_NULL) + if(comm->errhandlers_!=nullptr){ + for (int i=0; isize(); i++) + if (comm->errhandlers_[i]!=MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(comm->errhandlers_[i]); + delete[] comm->errhandlers_; + } else if (comm->errhandler_ != MPI_ERRHANDLER_NULL) simgrid::smpi::Errhandler::unref(comm->errhandler_); - delete comm; } + Group::unref(comm->group_); + if(comm->refcount_==0) + delete comm; } MPI_Comm Comm::find_intra_comm(int * leader){ //get the indices of all processes sharing the same simix host - auto& actor_list = sg_host_self()->pimpl_->actor_list_; int intra_comm_size = 0; int min_index = INT_MAX; // the minimum index will be the leader - for (auto& actor : actor_list) { + sg_host_self()->get_impl()->foreach_actor([this, &intra_comm_size, &min_index](auto& actor) { int index = actor.get_pid(); - if (this->group()->rank(actor.ciface()) != MPI_UNDEFINED) { // Is this process in the current group? + if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { // Is this process in the current group? intra_comm_size++; if (index < min_index) min_index = index; } - } + }); XBT_DEBUG("number of processes deployed on my node : %d", intra_comm_size); - MPI_Group group_intra = new Group(intra_comm_size); + auto* group_intra = new Group(intra_comm_size); int i = 0; - for (auto& actor : actor_list) { - if (this->group()->rank(actor.ciface()) != MPI_UNDEFINED) { - group_intra->set_mapping(actor.ciface(), i); + sg_host_self()->get_impl()->foreach_actor([this, group_intra, &i](auto& actor) { + if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { + group_intra->set_mapping(actor.get_ciface(), i); i++; } - } + }); *leader=min_index; return new Comm(group_intra, nullptr, true); } @@ -390,9 +405,8 @@ void Comm::init_smp(){ // identify neighbors in comm MPI_Comm comm_intra = find_intra_comm(&leader); - - int* leaders_map = new int[comm_size]; - int* leader_list = new int[comm_size]; + auto* leaders_map = new int[comm_size]; + auto* leader_list = new int[comm_size]; std::fill_n(leaders_map, comm_size, 0); std::fill_n(leader_list, comm_size, -1); @@ -424,7 +438,7 @@ void Comm::init_smp(){ xbt_assert(leader_group_size > 0); std::sort(leader_list, leader_list + leader_group_size); - MPI_Group leaders_group = new Group(leader_group_size); + auto* leaders_group = new Group(leader_group_size); MPI_Comm leader_comm = MPI_COMM_NULL; if(MPI_COMM_WORLD!=MPI_COMM_UNINITIALIZED && this!=MPI_COMM_WORLD){ @@ -470,7 +484,7 @@ void Comm::init_smp(){ } is_uniform_=is_uniform; } - bcast__scatter_LR_allgather(&(is_uniform_),1, MPI_INT, 0, comm_intra ); + bcast__scatter_LR_allgather(&is_uniform_, 1, MPI_INT, 0, comm_intra); if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { // we need to switch as the called function may silently touch global variables @@ -489,7 +503,7 @@ void Comm::init_smp(){ } int global_blocked; - allreduce__default(&is_blocked, &(global_blocked), 1, MPI_INT, MPI_LAND, this); + allreduce__default(&is_blocked, &global_blocked, 1, MPI_INT, MPI_LAND, this); if(MPI_COMM_WORLD==MPI_COMM_UNINITIALIZED || this==MPI_COMM_WORLD){ if(this->rank()==0){ @@ -509,10 +523,9 @@ MPI_Comm Comm::f2c(int id) { return MPI_COMM_SELF; } else if(id==0){ return MPI_COMM_WORLD; - } else if(F2C::f2c_lookup() != nullptr && id >= 0) { - char key[KEY_SIZE]; - const auto& lookup = F2C::f2c_lookup(); - auto comm = lookup->find(get_key(key, id)); + } else if (F2C::lookup() != nullptr && id >= 0) { + const auto& lookup = F2C::lookup(); + auto comm = lookup->find(id); return comm == lookup->end() ? MPI_COMM_NULL : static_cast(comm->second); } else { return MPI_COMM_NULL; @@ -520,8 +533,7 @@ MPI_Comm Comm::f2c(int id) { } void Comm::free_f(int id) { - char key[KEY_SIZE]; - F2C::f2c_lookup()->erase(get_key(key, id)); + F2C::lookup()->erase(id); } void Comm::add_rma_win(MPI_Win win){ @@ -561,18 +573,36 @@ void Comm::set_info(MPI_Info info) MPI_Errhandler Comm::errhandler() { - if (errhandler_ != MPI_ERRHANDLER_NULL) - errhandler_->ref(); - return errhandler_; + if (this != MPI_COMM_WORLD){ + if (errhandler_ != MPI_ERRHANDLER_NULL) + errhandler_->ref(); + return errhandler_; + } else { + if(errhandlers_==nullptr) + return MPI_ERRORS_ARE_FATAL; + else { + if(errhandlers_[this->rank()] != MPI_ERRHANDLER_NULL) + errhandlers_[this->rank()]->ref(); + return errhandlers_[this->rank()]; + } + } } void Comm::set_errhandler(MPI_Errhandler errhandler) { - if (errhandler_ != MPI_ERRHANDLER_NULL) - simgrid::smpi::Errhandler::unref(errhandler_); - errhandler_ = errhandler; - if (errhandler_ != MPI_ERRHANDLER_NULL) - errhandler_->ref(); + if(this != MPI_COMM_WORLD){ + if (errhandler_ != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(errhandler_); + errhandler_ = errhandler; + }else{ + if(errhandlers_==nullptr) + errhandlers_= new MPI_Errhandler[this->size()]{MPI_ERRHANDLER_NULL}; + if(errhandlers_[this->rank()] != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(errhandlers_[this->rank()]); + errhandlers_[this->rank()]=errhandler; + } + if (errhandler != MPI_ERRHANDLER_NULL) + errhandler->ref(); } MPI_Comm Comm::split_type(int type, int /*key*/, const Info*)