X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/cf507642628052d29599e6b7703e049ef28c0505..cc88d06e4fd1c32dddf17295ff750c60792588e5:/src/smpi/mpi/smpi_comm.cpp diff --git a/src/smpi/mpi/smpi_comm.cpp b/src/smpi/mpi/smpi_comm.cpp index 7722473d21..84b9e95c40 100644 --- a/src/smpi/mpi/smpi_comm.cpp +++ b/src/smpi/mpi/smpi_comm.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2019. The SimGrid Team. All rights reserved. */ +/* Copyright (c) 2010-2020. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ @@ -28,16 +28,23 @@ namespace smpi{ std::unordered_map Comm::keyvals_; int Comm::keyval_id_=0; -Comm::Comm(MPI_Group group, MPI_Topology topo, int smp) : group_(group), topo_(topo),is_smp_comm_(smp) +Comm::Comm(MPI_Group group, MPI_Topology topo, bool smp, int in_id) + : group_(group), topo_(topo), is_smp_comm_(smp), id_(in_id) { - refcount_ = 1; - topoType_ = MPI_INVALID_TOPO; - intra_comm_ = MPI_COMM_NULL; - leaders_comm_ = MPI_COMM_NULL; - is_uniform_ = 1; - non_uniform_map_ = nullptr; - leaders_map_ = nullptr; - is_blocked_ = 0; + errhandler_->ref(); + //First creation of comm is done before SIMIX_run, so only do comms for others + if(in_id==MPI_UNDEFINED && smp==0 && this->rank()!=MPI_UNDEFINED ){ + int id; + if(this->rank()==0){ + static int global_id_ = 0; + id=global_id_; + global_id_++; + } + colls::bcast(&id, 1, MPI_INT, 0, this); + XBT_DEBUG("Communicator %p has id %d", this, id); + id_=id;//only set here, as we don't want to change it in the middle of the bcast + colls::barrier(this); + } } void Comm::destroy(Comm* comm) @@ -46,16 +53,15 @@ void Comm::destroy(Comm* comm) Comm::destroy(smpi_process()->comm_world()); return; } - delete comm->topo_; // there's no use count on topos Comm::unref(comm); } int Comm::dup(MPI_Comm* newcomm){ - if (smpi_privatize_global_variables == SmpiPrivStrategies::MMAP) { + if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { // we need to switch as the called function may silently touch global variables smpi_switch_data_segment(s4u::Actor::self()); } - MPI_Group cp = new Group(this->group()); + auto* cp = new Group(this->group()); (*newcomm) = new Comm(cp, this->topo()); int ret = MPI_SUCCESS; @@ -89,6 +95,26 @@ int Comm::dup(MPI_Comm* newcomm){ } } } + //duplicate info if present + if(info_!=MPI_INFO_NULL) + (*newcomm)->info_ = new simgrid::smpi::Info(info_); + //duplicate errhandler + (*newcomm)->set_errhandler(errhandler_); + return ret; +} + +int Comm::dup_with_info(MPI_Info info, MPI_Comm* newcomm){ + int ret = dup(newcomm); + if(ret != MPI_SUCCESS) + return ret; + if((*newcomm)->info_!=MPI_INFO_NULL){ + simgrid::smpi::Info::unref((*newcomm)->info_); + (*newcomm)->info_=MPI_INFO_NULL; + } + if(info != MPI_INFO_NULL){ + info->ref(); + (*newcomm)->info_=info; + } return ret; } @@ -99,21 +125,26 @@ MPI_Group Comm::group() return group_; } -int Comm::size() +int Comm::size() const { if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->size(); return group_->size(); } -int Comm::rank() +int Comm::rank() const { if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->rank(); return group_->rank(s4u::Actor::self()); } -void Comm::get_name (char* name, int* len) +int Comm::id() const +{ + return id_; +} + +void Comm::get_name(char* name, int* len) const { if (this == MPI_COMM_UNINITIALIZED){ smpi_process()->comm_world()->get_name(name, len); @@ -130,7 +161,7 @@ void Comm::get_name (char* name, int* len) } } -void Comm::set_name (char* name) +void Comm::set_name (const char* name) { if (this == MPI_COMM_UNINITIALIZED){ smpi_process()->comm_world()->set_name(name); @@ -148,43 +179,50 @@ void Comm::set_leaders_comm(MPI_Comm leaders){ leaders_comm_=leaders; } -int* Comm::get_non_uniform_map(){ +int* Comm::get_non_uniform_map() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->get_non_uniform_map(); return non_uniform_map_; } -int* Comm::get_leaders_map(){ +int* Comm::get_leaders_map() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->get_leaders_map(); return leaders_map_; } -MPI_Comm Comm::get_leaders_comm(){ +MPI_Comm Comm::get_leaders_comm() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->get_leaders_comm(); return leaders_comm_; } -MPI_Comm Comm::get_intra_comm(){ +MPI_Comm Comm::get_intra_comm() const +{ if (this == MPI_COMM_UNINITIALIZED || this==MPI_COMM_WORLD) return smpi_process()->comm_intra(); else return intra_comm_; } -int Comm::is_uniform(){ +bool Comm::is_uniform() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->is_uniform(); - return is_uniform_; + return is_uniform_ != 0; } -int Comm::is_blocked(){ +bool Comm::is_blocked() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->is_blocked(); - return is_blocked_; + return is_blocked_ != 0; } -int Comm::is_smp_comm(){ +bool Comm::is_smp_comm() const +{ if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->is_smp_comm(); return is_smp_comm_; @@ -194,8 +232,7 @@ MPI_Comm Comm::split(int color, int key) { if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->split(color, key); - int system_tag = 123; - int* recvbuf; + int system_tag = -123; MPI_Group group_root = nullptr; MPI_Group group_out = nullptr; @@ -203,19 +240,19 @@ MPI_Comm Comm::split(int color, int key) int myrank = this->rank(); int size = this->size(); /* Gather all colors and keys on rank 0 */ - int* sendbuf = xbt_new(int, 2); - sendbuf[0] = color; - sendbuf[1] = key; + const std::array sendbuf = {{color, key}}; + std::vector recvbuf; + int* recvbuf_ptr; if (myrank == 0) { - recvbuf = xbt_new(int, 2 * size); + recvbuf.resize(2 * size); + recvbuf_ptr = recvbuf.data(); } else { - recvbuf = nullptr; + recvbuf_ptr = nullptr; } - Coll_gather_default::gather(sendbuf, 2, MPI_INT, recvbuf, 2, MPI_INT, 0, this); - xbt_free(sendbuf); + gather__default(sendbuf.data(), 2, MPI_INT, recvbuf_ptr, 2, MPI_INT, 0, this); /* Do the actual job */ if (myrank == 0) { - MPI_Group* group_snd = xbt_new(MPI_Group, size); + std::vector group_snd(size); std::vector> rankmap; rankmap.reserve(size); for (int i = 0; i < size; i++) { @@ -224,22 +261,22 @@ MPI_Comm Comm::split(int color, int key) for (int j = i + 1; j < size; j++) { if(recvbuf[2 * i] == recvbuf[2 * j]) { recvbuf[2 * j] = MPI_UNDEFINED; - rankmap.push_back({recvbuf[2 * j + 1], j}); + rankmap.emplace_back(recvbuf[2 * j + 1], j); } } /* Add self in the group */ recvbuf[2 * i] = MPI_UNDEFINED; - rankmap.push_back({recvbuf[2 * i + 1], i}); + rankmap.emplace_back(recvbuf[2 * i + 1], i); std::sort(begin(rankmap), end(rankmap)); group_out = new Group(rankmap.size()); if (i == 0) { group_root = group_out; /* Save root's group */ } for (unsigned j = 0; j < rankmap.size(); j++) { - s4u::ActorPtr actor = group->actor(rankmap[j].second); + s4u::Actor* actor = group->actor(rankmap[j].second); group_out->set_mapping(actor, j); } - MPI_Request* requests = xbt_new(MPI_Request, rankmap.size()); + std::vector requests(rankmap.size()); int reqs = 0; for (auto const& rank : rankmap) { if (rank.second != 0) { @@ -251,19 +288,16 @@ MPI_Comm Comm::split(int color, int key) if(i != 0 && group_out != MPI_COMM_WORLD->group() && group_out != MPI_GROUP_EMPTY) Group::unref(group_out); - Request::waitall(reqs, requests, MPI_STATUS_IGNORE); - xbt_free(requests); + Request::waitall(reqs, requests.data(), MPI_STATUS_IGNORE); } } - xbt_free(recvbuf); - xbt_free(group_snd); group_out = group_root; /* exit with root's group */ } else { if(color != MPI_UNDEFINED) { Request::recv(&group_out, 1, MPI_PTR, 0, system_tag, this, MPI_STATUS_IGNORE); } /* otherwise, exit with group_out == nullptr */ } - return group_out!=nullptr ? new Comm(group_out, nullptr) : MPI_COMM_NULL; + return group_out!=nullptr ? new Comm(group_out, topo_) : MPI_COMM_NULL; } void Comm::ref(){ @@ -295,13 +329,43 @@ void Comm::unref(Comm* comm){ if(comm->refcount_==0){ comm->cleanup_smp(); comm->cleanup_attr(); + if (comm->info_ != MPI_INFO_NULL) + simgrid::smpi::Info::unref(comm->info_); + if (comm->errhandler_ != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(comm->errhandler_); delete comm; } } +MPI_Comm Comm::find_intra_comm(int * leader){ + //get the indices of all processes sharing the same simix host + auto& actor_list = sg_host_self()->pimpl_->actor_list_; + int intra_comm_size = 0; + int min_index = INT_MAX; // the minimum index will be the leader + for (auto& actor : actor_list) { + int index = actor.get_pid(); + if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { // Is this process in the current group? + intra_comm_size++; + if (index < min_index) + min_index = index; + } + } + XBT_DEBUG("number of processes deployed on my node : %d", intra_comm_size); + auto* group_intra = new Group(intra_comm_size); + int i = 0; + for (auto& actor : actor_list) { + if (this->group()->rank(actor.get_ciface()) != MPI_UNDEFINED) { + group_intra->set_mapping(actor.get_ciface(), i); + i++; + } + } + *leader=min_index; + return new Comm(group_intra, nullptr, true); +} + void Comm::init_smp(){ int leader = -1; - + int i = 0; if (this == MPI_COMM_UNINITIALIZED) smpi_process()->comm_world()->init_smp(); @@ -315,44 +379,21 @@ void Comm::init_smp(){ smpi_process()->set_replaying(false); } - if (smpi_privatize_global_variables == SmpiPrivStrategies::MMAP) { + if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { // we need to switch as the called function may silently touch global variables smpi_switch_data_segment(s4u::Actor::self()); } - //identify neighbours in comm - //get the indices of all processes sharing the same simix host - auto& process_list = sg_host_self()->pimpl_->process_list_; - int intra_comm_size = 0; - int min_index = INT_MAX; // the minimum index will be the leader - for (auto& actor : process_list) { - int index = actor.get_pid(); - if (this->group()->rank(actor.iface()) != MPI_UNDEFINED) { // Is this process in the current group? - intra_comm_size++; - if (index < min_index) - min_index = index; - } - } - XBT_DEBUG("number of processes deployed on my node : %d", intra_comm_size); - MPI_Group group_intra = new Group(intra_comm_size); - int i = 0; - for (auto& actor : process_list) { - if (this->group()->rank(actor.iface()) != MPI_UNDEFINED) { - group_intra->set_mapping(actor.iface(), i); - i++; - } - } - - MPI_Comm comm_intra = new Comm(group_intra, nullptr, 1); - leader=min_index; + // identify neighbors in comm + MPI_Comm comm_intra = find_intra_comm(&leader); - int* leaders_map = new int[comm_size]; - int* leader_list = new int[comm_size]; + auto* leaders_map = new int[comm_size]; + auto* leader_list = new int[comm_size]; std::fill_n(leaders_map, comm_size, 0); std::fill_n(leader_list, comm_size, -1); - Coll_allgather_ring::allgather(&leader, 1, MPI_INT , leaders_map, 1, MPI_INT, this); + allgather__ring(&leader, 1, MPI_INT , leaders_map, 1, MPI_INT, this); - if (smpi_privatize_global_variables == SmpiPrivStrategies::MMAP) { + if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { // we need to switch as the called function may silently touch global variables smpi_switch_data_segment(s4u::Actor::self()); } @@ -375,26 +416,27 @@ void Comm::init_smp(){ leader_group_size++; } } + xbt_assert(leader_group_size > 0); std::sort(leader_list, leader_list + leader_group_size); - MPI_Group leaders_group = new Group(leader_group_size); + auto* leaders_group = new Group(leader_group_size); MPI_Comm leader_comm = MPI_COMM_NULL; if(MPI_COMM_WORLD!=MPI_COMM_UNINITIALIZED && this!=MPI_COMM_WORLD){ //create leader_communicator for (i=0; i< leader_group_size;i++) - leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]), i); - leader_comm = new Comm(leaders_group, nullptr,1); + leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]).get(), i); + leader_comm = new Comm(leaders_group, nullptr, true); this->set_leaders_comm(leader_comm); this->set_intra_comm(comm_intra); // create intracommunicator }else{ for (i=0; i< leader_group_size;i++) - leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]), i); + leaders_group->set_mapping(s4u::Actor::by_pid(leader_list[i]).get(), i); if(this->get_leaders_comm()==MPI_COMM_NULL){ - leader_comm = new Comm(leaders_group, nullptr,1); + leader_comm = new Comm(leaders_group, nullptr, true); this->set_leaders_comm(leader_comm); }else{ leader_comm=this->get_leaders_comm(); @@ -408,7 +450,7 @@ void Comm::init_smp(){ if(comm_intra->rank()==0) { int is_uniform = 1; int* non_uniform_map = xbt_new0(int,leader_group_size); - Coll_allgather_ring::allgather(&my_local_size, 1, MPI_INT, + allgather__ring(&my_local_size, 1, MPI_INT, non_uniform_map, 1, MPI_INT, leader_comm); for(i=0; i < leader_group_size; i++) { if(non_uniform_map[0] != non_uniform_map[i]) { @@ -416,16 +458,16 @@ void Comm::init_smp(){ break; } } - if(is_uniform==0 && this->is_uniform()!=0){ + if (is_uniform == 0 && this->is_uniform()) { non_uniform_map_ = non_uniform_map; }else{ xbt_free(non_uniform_map); } is_uniform_=is_uniform; } - Coll_bcast_scatter_LR_allgather::bcast(&(is_uniform_),1, MPI_INT, 0, comm_intra ); + bcast__scatter_LR_allgather(&(is_uniform_),1, MPI_INT, 0, comm_intra ); - if (smpi_privatize_global_variables == SmpiPrivStrategies::MMAP) { + if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { // we need to switch as the called function may silently touch global variables smpi_switch_data_segment(s4u::Actor::self()); } @@ -442,7 +484,7 @@ void Comm::init_smp(){ } int global_blocked; - Coll_allreduce_default::allreduce(&is_blocked, &(global_blocked), 1, MPI_INT, MPI_LAND, this); + allreduce__default(&is_blocked, &(global_blocked), 1, MPI_INT, MPI_LAND, this); if(MPI_COMM_WORLD==MPI_COMM_UNINITIALIZED || this==MPI_COMM_WORLD){ if(this->rank()==0){ @@ -462,10 +504,9 @@ MPI_Comm Comm::f2c(int id) { return MPI_COMM_SELF; } else if(id==0){ return MPI_COMM_WORLD; - } else if(F2C::f2c_lookup() != nullptr && id >= 0) { - char key[KEY_SIZE]; - const auto& lookup = F2C::f2c_lookup(); - auto comm = lookup->find(get_key_id(key, id)); + } else if (F2C::lookup() != nullptr && id >= 0) { + const auto& lookup = F2C::lookup(); + auto comm = lookup->find(id); return comm == lookup->end() ? MPI_COMM_NULL : static_cast(comm->second); } else { return MPI_COMM_NULL; @@ -473,18 +514,7 @@ MPI_Comm Comm::f2c(int id) { } void Comm::free_f(int id) { - char key[KEY_SIZE]; - F2C::f2c_lookup()->erase(id == 0 ? get_key(key, id) : get_key_id(key, id)); -} - -int Comm::add_f() { - if(F2C::f2c_lookup()==nullptr){ - F2C::set_f2c_lookup(new std::unordered_map); - } - char key[KEY_SIZE]; - (*(F2C::f2c_lookup()))[this == MPI_COMM_WORLD ? get_key(key, F2C::f2c_id()) : get_key_id(key, F2C::f2c_id())] = this; - f2c_id_increment(); - return F2C::f2c_id()-1; + F2C::lookup()->erase(id); } void Comm::add_rma_win(MPI_Win win){ @@ -495,7 +525,8 @@ void Comm::remove_rma_win(MPI_Win win){ rma_wins_.remove(win); } -void Comm::finish_rma_calls(){ +void Comm::finish_rma_calls() const +{ for (auto const& it : rma_wins_) { if(it->rank()==this->rank()){//is it ours (for MPI_COMM_WORLD)? int finished = it->finish_comms(); @@ -504,28 +535,54 @@ void Comm::finish_rma_calls(){ } } -MPI_Info Comm::info(){ - if(info_== MPI_INFO_NULL) +MPI_Info Comm::info() +{ + if (info_ == MPI_INFO_NULL) info_ = new Info(); info_->ref(); return info_; } -void Comm::set_info(MPI_Info info){ - if(info_!= MPI_INFO_NULL) +void Comm::set_info(MPI_Info info) +{ + if (info_ != MPI_INFO_NULL) + simgrid::smpi::Info::unref(info); + info_ = info; + if (info_ != MPI_INFO_NULL) info->ref(); - info_=info; } -MPI_Comm Comm::split_type(int type, int key, MPI_Info info) +MPI_Errhandler Comm::errhandler() { - if(type != MPI_COMM_TYPE_SHARED){ + if (errhandler_ != MPI_ERRHANDLER_NULL) + errhandler_->ref(); + return errhandler_; +} + +void Comm::set_errhandler(MPI_Errhandler errhandler) +{ + if (errhandler_ != MPI_ERRHANDLER_NULL) + simgrid::smpi::Errhandler::unref(errhandler_); + errhandler_ = errhandler; + if (errhandler_ != MPI_ERRHANDLER_NULL) + errhandler_->ref(); +} + +MPI_Comm Comm::split_type(int type, int /*key*/, const Info*) +{ + //MPI_UNDEFINED can be given to some nodes... but we need them to still perform the smp part which is collective + if(type != MPI_COMM_TYPE_SHARED && type != MPI_UNDEFINED){ + return MPI_COMM_NULL; + } + int leader=0; + MPI_Comm res= this->find_intra_comm(&leader); + if(type != MPI_UNDEFINED) + return res; + else{ + xbt_assert(res->refcount_ == 1); // ensure the next call to Comm::destroy really frees the comm + Comm::destroy(res); return MPI_COMM_NULL; } - this->init_smp(); - this->ref(); - this->get_intra_comm()->ref(); - return this->get_intra_comm(); } } // namespace smpi