X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/2750bde086c69304587076350db922bfd9da04f6..cfbf3bda3a828313326d322fa92fb6719904d1a4:/src/smpi/mpi/smpi_comm.cpp diff --git a/src/smpi/mpi/smpi_comm.cpp b/src/smpi/mpi/smpi_comm.cpp index e735a364f9..1722cec5de 100644 --- a/src/smpi/mpi/smpi_comm.cpp +++ b/src/smpi/mpi/smpi_comm.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2020. The SimGrid Team. All rights reserved. */ +/* Copyright (c) 2010-2021. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ @@ -16,8 +16,18 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_comm, smpi, "Logging specific to SMPI (comm)"); -simgrid::smpi::Comm mpi_MPI_COMM_UNINITIALIZED; -MPI_Comm MPI_COMM_UNINITIALIZED=&mpi_MPI_COMM_UNINITIALIZED; +simgrid::smpi::Comm smpi_MPI_COMM_UNINITIALIZED; +MPI_Comm MPI_COMM_UNINITIALIZED=&smpi_MPI_COMM_UNINITIALIZED; +/** + * Setting MPI_COMM_WORLD to MPI_COMM_UNINITIALIZED (it's a variable) + * is important because the implementation of MPI_Comm checks + * "this == MPI_COMM_UNINITIALIZED"? If yes, it uses smpi_process()->comm_world() + * instead of "this". + * This is basically how we only have one global variable but all processes have + * different communicators (the one their SMPI instance uses). + * + */ +MPI_Comm MPI_COMM_WORLD = MPI_COMM_UNINITIALIZED; /* Support for cartesian topology was added, but there are 2 other types of topology, graph et dist graph. In order to * support them, we have to add a field SMPI_Topo_type, and replace the MPI_Topology field by an union. */ @@ -34,6 +44,8 @@ Comm::Comm(MPI_Group group, MPI_Topology topo, bool smp, int in_id) errhandler_->ref(); //First creation of comm is done before SIMIX_run, so only do comms for others if(in_id==MPI_UNDEFINED && smp==0 && this->rank()!=MPI_UNDEFINED ){ + this->add_f(); + group->c2f(); int id; if(this->rank()==0){ static int global_id_ = 0; @@ -233,7 +245,6 @@ MPI_Comm Comm::split(int color, int key) if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->split(color, key); int system_tag = -123; - int* recvbuf; MPI_Group group_root = nullptr; MPI_Group group_out = nullptr; @@ -241,19 +252,14 @@ MPI_Comm Comm::split(int color, int key) int myrank = this->rank(); int size = this->size(); /* Gather all colors and keys on rank 0 */ - int* sendbuf = xbt_new(int, 2); - sendbuf[0] = color; - sendbuf[1] = key; - if (myrank == 0) { - recvbuf = xbt_new(int, 2 * size); - } else { - recvbuf = nullptr; - } - gather__default(sendbuf, 2, MPI_INT, recvbuf, 2, MPI_INT, 0, this); - xbt_free(sendbuf); + const std::array sendbuf = {{color, key}}; + std::vector recvbuf; + if (myrank == 0) + recvbuf.resize(2 * size); + gather__default(sendbuf.data(), 2, MPI_INT, recvbuf.data(), 2, MPI_INT, 0, this); /* Do the actual job */ if (myrank == 0) { - MPI_Group* group_snd = xbt_new(MPI_Group, size); + std::vector group_snd(size); std::vector> rankmap; rankmap.reserve(size); for (int i = 0; i < size; i++) { @@ -262,12 +268,12 @@ MPI_Comm Comm::split(int color, int key) for (int j = i + 1; j < size; j++) { if(recvbuf[2 * i] == recvbuf[2 * j]) { recvbuf[2 * j] = MPI_UNDEFINED; - rankmap.push_back({recvbuf[2 * j + 1], j}); + rankmap.emplace_back(recvbuf[2 * j + 1], j); } } /* Add self in the group */ recvbuf[2 * i] = MPI_UNDEFINED; - rankmap.push_back({recvbuf[2 * i + 1], i}); + rankmap.emplace_back(recvbuf[2 * i + 1], i); std::sort(begin(rankmap), end(rankmap)); group_out = new Group(rankmap.size()); if (i == 0) { @@ -277,7 +283,7 @@ MPI_Comm Comm::split(int color, int key) s4u::Actor* actor = group->actor(rankmap[j].second); group_out->set_mapping(actor, j); } - MPI_Request* requests = xbt_new(MPI_Request, rankmap.size()); + std::vector requests(rankmap.size()); int reqs = 0; for (auto const& rank : rankmap) { if (rank.second != 0) { @@ -289,12 +295,9 @@ MPI_Comm Comm::split(int color, int key) if(i != 0 && group_out != MPI_COMM_WORLD->group() && group_out != MPI_GROUP_EMPTY) Group::unref(group_out); - Request::waitall(reqs, requests, MPI_STATUS_IGNORE); - xbt_free(requests); + Request::waitall(reqs, requests.data(), MPI_STATUS_IGNORE); } } - xbt_free(recvbuf); - xbt_free(group_snd); group_out = group_root; /* exit with root's group */ } else { if(color != MPI_UNDEFINED) { @@ -328,41 +331,48 @@ void Comm::unref(Comm* comm){ return; } comm->refcount_--; - Group::unref(comm->group_); if(comm->refcount_==0){ + if(simgrid::smpi::F2C::lookup() != nullptr) + F2C::free_f(comm->c2f()); comm->cleanup_smp(); comm->cleanup_attr(); if (comm->info_ != MPI_INFO_NULL) simgrid::smpi::Info::unref(comm->info_); - if (comm->errhandler_ != MPI_ERRHANDLER_NULL) + if(comm->errhandlers_!=nullptr){ + for (int i=0; isize(); i++) + if (comm->errhandlers_[i]!=MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(comm->errhandlers_[i]); + delete[] comm->errhandlers_; + } else if (comm->errhandler_ != MPI_ERRHANDLER_NULL) simgrid::smpi::Errhandler::unref(comm->errhandler_); - delete comm; } + Group::unref(comm->group_); + if(comm->refcount_==0) + delete comm; } MPI_Comm Comm::find_intra_comm(int * leader){ //get the indices of all processes sharing the same simix host - auto& actor_list = sg_host_self()->pimpl_->actor_list_; int intra_comm_size = 0; int min_index = INT_MAX; // the minimum index will be the leader - for (auto& actor : actor_list) { + sg_host_self()->pimpl_->foreach_actor([this, &intra_comm_size, &min_index](auto& actor) { int index = actor.get_pid(); - if (this->group()->rank(actor.ciface()) != MPI_UNDEFINED) { // Is this process in the current group? + if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { // Is this process in the current group? intra_comm_size++; if (index < min_index) min_index = index; } - } + }); XBT_DEBUG("number of processes deployed on my node : %d", intra_comm_size); auto* group_intra = new Group(intra_comm_size); int i = 0; - for (auto& actor : actor_list) { - if (this->group()->rank(actor.ciface()) != MPI_UNDEFINED) { - group_intra->set_mapping(actor.ciface(), i); + sg_host_self()->pimpl_->foreach_actor([this, group_intra, &i](auto& actor) { + if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { + group_intra->set_mapping(actor.get_ciface(), i); i++; } - } + }); *leader=min_index; return new Comm(group_intra, nullptr, true); } @@ -508,10 +518,9 @@ MPI_Comm Comm::f2c(int id) { return MPI_COMM_SELF; } else if(id==0){ return MPI_COMM_WORLD; - } else if(F2C::f2c_lookup() != nullptr && id >= 0) { - char key[KEY_SIZE]; - const auto& lookup = F2C::f2c_lookup(); - auto comm = lookup->find(get_key(key, id)); + } else if (F2C::lookup() != nullptr && id >= 0) { + const auto& lookup = F2C::lookup(); + auto comm = lookup->find(id); return comm == lookup->end() ? MPI_COMM_NULL : static_cast(comm->second); } else { return MPI_COMM_NULL; @@ -519,8 +528,7 @@ MPI_Comm Comm::f2c(int id) { } void Comm::free_f(int id) { - char key[KEY_SIZE]; - F2C::f2c_lookup()->erase(get_key(key, id)); + F2C::lookup()->erase(id); } void Comm::add_rma_win(MPI_Win win){ @@ -560,18 +568,36 @@ void Comm::set_info(MPI_Info info) MPI_Errhandler Comm::errhandler() { - if (errhandler_ != MPI_ERRHANDLER_NULL) - errhandler_->ref(); - return errhandler_; + if (this != MPI_COMM_WORLD){ + if (errhandler_ != MPI_ERRHANDLER_NULL) + errhandler_->ref(); + return errhandler_; + } else { + if(errhandlers_==nullptr) + return MPI_ERRORS_ARE_FATAL; + else { + if(errhandlers_[this->rank()] != MPI_ERRHANDLER_NULL) + errhandlers_[this->rank()]->ref(); + return errhandlers_[this->rank()]; + } + } } void Comm::set_errhandler(MPI_Errhandler errhandler) { - if (errhandler_ != MPI_ERRHANDLER_NULL) - simgrid::smpi::Errhandler::unref(errhandler_); - errhandler_ = errhandler; - if (errhandler_ != MPI_ERRHANDLER_NULL) - errhandler_->ref(); + if(this != MPI_COMM_WORLD){ + if (errhandler_ != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(errhandler_); + errhandler_ = errhandler; + }else{ + if(errhandlers_==nullptr) + errhandlers_= new MPI_Errhandler[this->size()]{MPI_ERRHANDLER_NULL}; + if(errhandlers_[this->rank()] != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(errhandlers_[this->rank()]); + errhandlers_[this->rank()]=errhandler; + } + if (errhandler != MPI_ERRHANDLER_NULL) + errhandler->ref(); } MPI_Comm Comm::split_type(int type, int /*key*/, const Info*)