-/* Copyright (c) 2006-2022. The SimGrid Team. All rights reserved. */
+/* Copyright (c) 2006-2023. The SimGrid Team. All rights reserved. */
/* This program is free software; you can redistribute it and/or modify it
* under the terms of the license (GNU LGPL) which comes with this package. */
-//#include "src/msg/msg_private.hpp"
-//#include "xbt/log.h"
-
+#include <cmath>
#include <simgrid/Exception.hpp>
#include <simgrid/comm.h>
#include <simgrid/s4u/Comm.hpp>
#include <simgrid/s4u/Engine.hpp>
#include <simgrid/s4u/Mailbox.hpp>
+#include "mc/mc.h"
#include "src/kernel/activity/CommImpl.hpp"
#include "src/kernel/actor/ActorImpl.hpp"
#include "src/kernel/actor/SimcallObserver.hpp"
+#include "src/mc/mc_replay.hpp"
XBT_LOG_NEW_DEFAULT_SUBCATEGORY(s4u_comm, s4u_activity, "S4U asynchronous communications");
-namespace simgrid {
-namespace s4u {
+namespace simgrid::s4u {
xbt::signal<void(Comm const&)> Comm::on_send;
xbt::signal<void(Comm const&)> Comm::on_recv;
-xbt::signal<void(Comm const&)> Comm::on_completion;
+
+CommPtr Comm::set_copy_data_callback(const std::function<void(kernel::activity::CommImpl*, void*, size_t)>& callback)
+{
+ copy_data_function_ = callback;
+ return this;
+}
+
+void Comm::copy_buffer_callback(kernel::activity::CommImpl* comm, void* buff, size_t buff_size)
+{
+ XBT_DEBUG("Copy the data over");
+ memcpy(comm->dst_buff_, buff, buff_size);
+ if (comm->is_detached()) { // if this is a detached send, the source buffer was duplicated by SMPI sender to make the
+ // original buffer available to the application ASAP
+ xbt_free(buff);
+ comm->src_buff_ = nullptr;
+ }
+}
+
+void Comm::copy_pointer_callback(kernel::activity::CommImpl* comm, void* buff, size_t buff_size)
+{
+ xbt_assert((buff_size == sizeof(void*)), "Cannot copy %zu bytes: must be sizeof(void*)", buff_size);
+ *(void**)(comm->dst_buff_) = buff;
+}
Comm::~Comm()
{
}
}
-ssize_t Comm::wait_any_for(const std::vector<CommPtr>& comms, double timeout)
-{
- std::vector<ActivityPtr> activities;
- for (const auto& comm : comms)
- activities.push_back(boost::dynamic_pointer_cast<Activity>(comm));
- ssize_t changed_pos;
- try {
- changed_pos = Activity::wait_any_for(activities, timeout);
- } catch (const NetworkFailureException& e) {
- for (auto c : comms) {
- if (c->pimpl_->get_state() == kernel::activity::State::FAILED) {
- c->complete(State::FAILED);
- }
+void Comm::send(kernel::actor::ActorImpl* sender, const Mailbox* mbox, double task_size, double rate, void* src_buff,
+ size_t src_buff_size,
+ const std::function<bool(void*, void*, simgrid::kernel::activity::CommImpl*)>& match_fun,
+ const std::function<void(simgrid::kernel::activity::CommImpl*, void*, size_t)>& copy_data_fun,
+ void* data, double timeout)
+{
+ /* checking for infinite values */
+ xbt_assert(std::isfinite(task_size), "task_size is not finite!");
+ xbt_assert(std::isfinite(rate), "rate is not finite!");
+ xbt_assert(std::isfinite(timeout), "timeout is not finite!");
+
+ xbt_assert(mbox, "No rendez-vous point defined for send");
+
+ if (MC_is_active() || MC_record_replay_is_active()) {
+ /* the model-checker wants two separate simcalls, and wants comm to be nullptr during the simcall */
+ simgrid::kernel::activity::ActivityImplPtr comm = nullptr;
+
+ simgrid::kernel::actor::CommIsendSimcall send_observer{
+ sender, mbox->get_impl(), task_size, rate, static_cast<unsigned char*>(src_buff), src_buff_size, match_fun,
+ nullptr, copy_data_fun, data, false};
+ comm = simgrid::kernel::actor::simcall_answered(
+ [&send_observer] { return simgrid::kernel::activity::CommImpl::isend(&send_observer); }, &send_observer);
+
+ if (simgrid::kernel::actor::ActivityWaitSimcall wait_observer{sender, comm.get(), timeout};
+ simgrid::kernel::actor::simcall_blocking(
+ [&wait_observer] {
+ wait_observer.get_activity()->wait_for(wait_observer.get_issuer(), wait_observer.get_timeout());
+ },
+ &wait_observer)) {
+ throw simgrid::TimeoutException(XBT_THROW_POINT, "Timeouted");
}
- e.rethrow_nested(XBT_THROW_POINT, boost::core::demangle(typeid(e).name()) + " raised in kernel mode.");
+ comm = nullptr;
+ } else {
+ simgrid::kernel::actor::CommIsendSimcall observer(sender, mbox->get_impl(), task_size, rate,
+ static_cast<unsigned char*>(src_buff), src_buff_size, match_fun,
+ nullptr, copy_data_fun, data, false);
+ simgrid::kernel::actor::simcall_blocking([&observer, timeout] {
+ simgrid::kernel::activity::ActivityImplPtr comm = simgrid::kernel::activity::CommImpl::isend(&observer);
+ comm->wait_for(observer.get_issuer(), timeout);
+ });
}
- return changed_pos;
}
-void Comm::wait_all(const std::vector<CommPtr>& comms)
+void Comm::recv(kernel::actor::ActorImpl* receiver, const Mailbox* mbox, void* dst_buff, size_t* dst_buff_size,
+ const std::function<bool(void*, void*, simgrid::kernel::activity::CommImpl*)>& match_fun,
+ const std::function<void(simgrid::kernel::activity::CommImpl*, void*, size_t)>& copy_data_fun,
+ void* data, double timeout, double rate)
+{
+ xbt_assert(std::isfinite(timeout), "timeout is not finite!");
+ xbt_assert(mbox, "No rendez-vous point defined for recv");
+
+ if (MC_is_active() || MC_record_replay_is_active()) {
+ /* the model-checker wants two separate simcalls, and wants comm to be nullptr during the simcall */
+ simgrid::kernel::activity::ActivityImplPtr comm = nullptr;
+
+ simgrid::kernel::actor::CommIrecvSimcall observer{receiver,
+ mbox->get_impl(),
+ static_cast<unsigned char*>(dst_buff),
+ dst_buff_size,
+ match_fun,
+ copy_data_fun,
+ data,
+ rate};
+ comm = simgrid::kernel::actor::simcall_answered(
+ [&observer] { return simgrid::kernel::activity::CommImpl::irecv(&observer); }, &observer);
+
+ if (simgrid::kernel::actor::ActivityWaitSimcall wait_observer{receiver, comm.get(), timeout};
+ simgrid::kernel::actor::simcall_blocking(
+ [&wait_observer] {
+ wait_observer.get_activity()->wait_for(wait_observer.get_issuer(), wait_observer.get_timeout());
+ },
+ &wait_observer)) {
+ throw simgrid::TimeoutException(XBT_THROW_POINT, "Timeouted");
+ }
+ comm = nullptr;
+ } else {
+ simgrid::kernel::actor::CommIrecvSimcall observer(receiver, mbox->get_impl(), static_cast<unsigned char*>(dst_buff),
+ dst_buff_size, match_fun, copy_data_fun, data, rate);
+ simgrid::kernel::actor::simcall_blocking([&observer, timeout] {
+ simgrid::kernel::activity::ActivityImplPtr comm = simgrid::kernel::activity::CommImpl::irecv(&observer);
+ comm->wait_for(observer.get_issuer(), timeout);
+ });
+ }
+}
+
+CommPtr Comm::sendto_init()
{
- // TODO: this should be a simcall or something
- for (auto& comm : comms)
- comm->wait();
+ CommPtr res(new Comm());
+ res->pimpl_ = kernel::activity::CommImplPtr(new kernel::activity::CommImpl());
+ boost::static_pointer_cast<kernel::activity::CommImpl>(res->pimpl_)->detach();
+ res->sender_ = kernel::actor::ActorImpl::self();
+ return res;
}
-size_t Comm::wait_all_for(const std::vector<CommPtr>& comms, double timeout)
+CommPtr Comm::sendto_init(Host* from, Host* to)
{
- if (timeout < 0.0) {
- wait_all(comms);
- return comms.size();
- }
+ auto res = Comm::sendto_init()->set_source(from)->set_destination(to);
+ res->set_state(State::STARTING);
+ return res;
+}
- double deadline = Engine::get_clock() + timeout;
- std::vector<CommPtr> waited_comm(1, nullptr);
- for (size_t i = 0; i < comms.size(); i++) {
- double wait_timeout = std::max(0.0, deadline - Engine::get_clock());
- waited_comm[0] = comms[i];
- // Using wait_any_for() here (and not wait_for) because we don't want comms to be invalidated on timeout
- if (wait_any_for(waited_comm, wait_timeout) == -1) {
- XBT_DEBUG("Timeout (%g): i = %zu", wait_timeout, i);
- return i;
- }
- }
- return comms.size();
+CommPtr Comm::sendto_async(Host* from, Host* to, uint64_t simulated_size_in_bytes)
+{
+ return Comm::sendto_init()->set_payload_size(simulated_size_in_bytes)->set_source(from)->set_destination(to);
+}
+
+void Comm::sendto(Host* from, Host* to, uint64_t simulated_size_in_bytes)
+{
+ sendto_async(from, to, simulated_size_in_bytes)->wait();
}
CommPtr Comm::set_source(Host* from)
{
xbt_assert(state_ == State::INITED || state_ == State::STARTING,
"Cannot change the source of a Comm once it's started (state: %s)", to_c_str(state_));
- from_ = from;
- // Setting 'from_' may allow to start the activity, let's try
- vetoable_start();
+ boost::static_pointer_cast<kernel::activity::CommImpl>(pimpl_)->set_source(from);
+ // Setting 'source' may allow to start the activity, let's try
+ if (state_ == State::STARTING && remains_ <= 0)
+ XBT_DEBUG("This communication has a payload size of 0 byte. It cannot start yet");
+ else
+ vetoable_start();
return this;
}
+Host* Comm::get_source() const
+{
+ return pimpl_ ? boost::static_pointer_cast<kernel::activity::CommImpl>(pimpl_)->get_source() : nullptr;
+}
CommPtr Comm::set_destination(Host* to)
{
xbt_assert(state_ == State::INITED || state_ == State::STARTING,
"Cannot change the destination of a Comm once it's started (state: %s)", to_c_str(state_));
- to_ = to;
- // Setting 'to_' may allow to start the activity, let's try
- vetoable_start();
+ boost::static_pointer_cast<kernel::activity::CommImpl>(pimpl_)->set_destination(to);
+ // Setting 'destination' may allow to start the activity, let's try
+ if (state_ == State::STARTING && remains_ <= 0)
+ XBT_DEBUG("This communication has a payload size of 0 byte. It cannot start yet");
+ else
+ vetoable_start();
return this;
}
+Host* Comm::get_destination() const
+{
+ return pimpl_ ? boost::static_pointer_cast<kernel::activity::CommImpl>(pimpl_)->get_destination() : nullptr;
+}
+
CommPtr Comm::set_rate(double rate)
{
xbt_assert(state_ == State::INITED, "You cannot use %s() once your communication started (not implemented)",
return this;
}
+CommPtr Comm::set_mailbox(Mailbox* mailbox)
+{
+ xbt_assert(state_ == State::INITED, "You cannot use %s() once your communication started (not implemented)",
+ __FUNCTION__);
+ mailbox_ = mailbox;
+ return this;
+}
+
CommPtr Comm::set_src_data(void* buff)
{
xbt_assert(state_ == State::INITED, "You cannot use %s() once your communication started (not implemented)",
dst_buff_ = buff;
return this;
}
-void* Comm::get_dst_data()
-{
- return dst_buff_;
-}
-size_t Comm::get_dst_data_size() const
-{
- return dst_buff_size_;
-}
CommPtr Comm::set_dst_data(void** buff, size_t size)
{
xbt_assert(state_ == State::INITED, "You cannot use %s() once your communication started (not implemented)",
dst_buff_size_ = size;
return this;
}
+
CommPtr Comm::set_payload_size(uint64_t bytes)
{
- Activity::set_remaining(bytes);
+ set_remaining(bytes);
+ if (pimpl_) {
+ boost::static_pointer_cast<kernel::activity::CommImpl>(pimpl_)->set_size(bytes);
+ }
return this;
}
-CommPtr Comm::sendto_init()
-{
- CommPtr res(new Comm());
- res->sender_ = kernel::actor::ActorImpl::self();
- return res;
-}
-
-CommPtr Comm::sendto_init(Host* from, Host* to)
-{
- auto res = Comm::sendto_init();
- res->from_ = from;
- res->to_ = to;
-
- return res;
-}
-
-CommPtr Comm::sendto_async(Host* from, Host* to, uint64_t simulated_size_in_bytes)
+Actor* Comm::get_sender() const
{
- auto res = Comm::sendto_init(from, to)->set_payload_size(simulated_size_in_bytes);
- res->vetoable_start();
- return res;
+ kernel::actor::ActorImplPtr sender = nullptr;
+ if (pimpl_)
+ sender = boost::static_pointer_cast<kernel::activity::CommImpl>(pimpl_)->src_actor_;
+ return sender ? sender->get_ciface() : nullptr;
}
-void Comm::sendto(Host* from, Host* to, uint64_t simulated_size_in_bytes)
+bool Comm::is_assigned() const
{
- sendto_async(from, to, simulated_size_in_bytes)->wait();
+ return (pimpl_ && boost::static_pointer_cast<kernel::activity::CommImpl>(pimpl_)->is_assigned()) ||
+ mailbox_ != nullptr;
}
Comm* Comm::start()
{
xbt_assert(get_state() == State::INITED || get_state() == State::STARTING,
"You cannot use %s() once your communication started (not implemented)", __FUNCTION__);
- if (from_ != nullptr || to_ != nullptr) {
- xbt_assert(from_ != nullptr && to_ != nullptr, "When either from_ or to_ is specified, both must be.");
+ if (get_source() != nullptr || get_destination() != nullptr) {
+ xbt_assert(is_assigned(), "When either from_ or to_ is specified, both must be.");
xbt_assert(src_buff_ == nullptr && dst_buff_ == nullptr,
"Direct host-to-host communications cannot carry any data.");
- pimpl_ = kernel::actor::simcall([this] {
- kernel::activity::CommImplPtr res(new kernel::activity::CommImpl(this->from_, this->to_, this->get_remaining()));
- res->start();
- return res;
+ XBT_DEBUG("host-to-host Comm. Pimpl already created and set, just start it.");
+ kernel::actor::simcall_answered([this] {
+ pimpl_->set_state(kernel::activity::State::READY);
+ boost::static_pointer_cast<kernel::activity::CommImpl>(pimpl_)->start();
});
-
} else if (src_buff_ != nullptr) { // Sender side
on_send(*this);
kernel::actor::CommIsendSimcall observer{sender_,
copy_data_function_,
get_data<void>(),
detached_};
- pimpl_ = kernel::actor::simcall_blocking(
- [&observer] {
- return kernel::activity::CommImpl::isend(
- observer.get_issuer(), observer.get_mailbox(), observer.get_payload_size(), observer.get_rate(),
- observer.get_src_buff(), observer.get_src_buff_size(), observer.match_fun_, observer.clean_fun_,
- observer.copy_data_fun_, observer.get_payload(), observer.is_detached());
- },
- &observer);
+ pimpl_ = kernel::actor::simcall_answered([&observer] { return kernel::activity::CommImpl::isend(&observer); },
+ &observer);
} else if (dst_buff_ != nullptr) { // Receiver side
xbt_assert(not detached_, "Receive cannot be detached");
on_recv(*this);
copy_data_function_,
get_data<void>(),
rate_};
- pimpl_ = kernel::actor::simcall_blocking(
- [&observer] {
- return kernel::activity::CommImpl::irecv(
- observer.get_issuer(), observer.get_mailbox(), observer.get_dst_buff(), observer.get_dst_buff_size(),
- observer.match_fun_, observer.copy_data_fun_, observer.get_payload(), observer.get_rate());
- },
- &observer);
+ pimpl_ = kernel::actor::simcall_answered([&observer] { return kernel::activity::CommImpl::irecv(&observer); },
+ &observer);
} else {
xbt_die("Cannot start a communication before specifying whether we are the sender or the receiver");
}
return this;
}
+Comm* Comm::detach()
+{
+ xbt_assert(state_ == State::INITED || state_ == State::STARTING,
+ "You cannot use %s() once your communication is %s (not implemented)", __FUNCTION__, get_state_str());
+ xbt_assert(dst_buff_ == nullptr && dst_buff_size_ == 0, "You can only detach sends, not recvs");
+ detached_ = true;
+ vetoable_start();
+ return this;
+}
+
+ssize_t Comm::test_any(const std::vector<CommPtr>& comms)
+{
+ std::vector<ActivityPtr> activities;
+ for (const auto& comm : comms)
+ activities.push_back(boost::dynamic_pointer_cast<Activity>(comm));
+ return Activity::test_any(activities);
+}
+
/** @brief Block the calling actor until the communication is finished, or until timeout
*
* On timeout, an exception is thrown and the communication is invalidated.
throw NetworkFailureException(XBT_THROW_POINT, "Cannot wait for a failed communication");
case State::INITED:
case State::STARTING: // It's not started yet. Do it in one simcall if it's a regular communication
- if (from_ != nullptr || to_ != nullptr) {
+ if (get_source() != nullptr || get_destination() != nullptr) {
return vetoable_start()->wait_for(timeout); // In the case of host2host comm, do it in two simcalls
} else if (src_buff_ != nullptr) {
on_send(*this);
- simcall_comm_send(sender_, mailbox_->get_impl(), remains_, rate_, src_buff_, src_buff_size_, match_fun_,
- copy_data_function_, get_data<void>(), timeout);
+ send(sender_, mailbox_, remains_, rate_, src_buff_, src_buff_size_, match_fun_, copy_data_function_,
+ get_data<void>(), timeout);
} else { // Receiver
on_recv(*this);
- simcall_comm_recv(receiver_, mailbox_->get_impl(), dst_buff_, &dst_buff_size_, match_fun_, copy_data_function_,
- get_data<void>(), timeout, rate_);
+ recv(receiver_, mailbox_, dst_buff_, &dst_buff_size_, match_fun_, copy_data_function_, get_data<void>(),
+ timeout, rate_);
}
break;
case State::STARTED:
return this;
}
-ssize_t Comm::test_any(const std::vector<CommPtr>& comms)
+ssize_t Comm::wait_any_for(const std::vector<CommPtr>& comms, double timeout)
{
std::vector<ActivityPtr> activities;
for (const auto& comm : comms)
activities.push_back(boost::dynamic_pointer_cast<Activity>(comm));
- return Activity::test_any(activities);
-}
-
-Comm* Comm::detach()
-{
- xbt_assert(state_ == State::INITED, "You cannot use %s() once your communication is %s (not implemented)",
- __FUNCTION__, get_state_str());
- xbt_assert(dst_buff_ == nullptr && dst_buff_size_ == 0, "You can only detach sends, not recvs");
- detached_ = true;
- vetoable_start();
- return this;
-}
-
-Mailbox* Comm::get_mailbox() const
-{
- return mailbox_;
+ ssize_t changed_pos;
+ try {
+ changed_pos = Activity::wait_any_for(activities, timeout);
+ } catch (const NetworkFailureException& e) {
+ changed_pos = -1;
+ for (auto c : comms) {
+ if (c->pimpl_->get_state() == kernel::activity::State::FAILED) {
+ c->complete(State::FAILED);
+ }
+ }
+ e.rethrow_nested(XBT_THROW_POINT, boost::core::demangle(typeid(e).name()) + " raised in kernel mode.");
+ }
+ return changed_pos;
}
-Actor* Comm::get_sender() const
+void Comm::wait_all(const std::vector<CommPtr>& comms)
{
- kernel::actor::ActorImplPtr sender = nullptr;
- if (pimpl_)
- sender = boost::static_pointer_cast<kernel::activity::CommImpl>(pimpl_)->src_actor_;
- return sender ? sender->get_ciface() : nullptr;
+ // TODO: this should be a simcall or something
+ for (auto& comm : comms)
+ comm->wait();
}
-CommPtr Comm::set_copy_data_callback(void (*callback)(kernel::activity::CommImpl*, void*, size_t))
-{
- copy_data_function_ = callback;
- return this;
-}
-void Comm::copy_buffer_callback(kernel::activity::CommImpl* comm, void* buff, size_t buff_size)
+size_t Comm::wait_all_for(const std::vector<CommPtr>& comms, double timeout)
{
- XBT_DEBUG("Copy the data over");
- memcpy(comm->dst_buff_, buff, buff_size);
- if (comm->detached()) { // if this is a detached send, the source buffer was duplicated by SMPI sender to make the
- // original buffer available to the application ASAP
- xbt_free(buff);
- comm->src_buff_ = nullptr;
+ if (timeout < 0.0) {
+ wait_all(comms);
+ return comms.size();
}
-}
-void Comm::copy_pointer_callback(kernel::activity::CommImpl* comm, void* buff, size_t buff_size)
-{
- xbt_assert((buff_size == sizeof(void*)), "Cannot copy %zu bytes: must be sizeof(void*)", buff_size);
- *(void**)(comm->dst_buff_) = buff;
+ double deadline = Engine::get_clock() + timeout;
+ std::vector<CommPtr> waited_comm(1, nullptr);
+ for (size_t i = 0; i < comms.size(); i++) {
+ double wait_timeout = std::max(0.0, deadline - Engine::get_clock());
+ waited_comm[0] = comms[i];
+ // Using wait_any_for() here (and not wait_for) because we don't want comms to be invalidated on timeout
+ if (wait_any_for(waited_comm, wait_timeout) == -1) {
+ XBT_DEBUG("Timeout (%g): i = %zu", wait_timeout, i);
+ return i;
+ }
+ }
+ return comms.size();
}
-
-} // namespace s4u
-} // namespace simgrid
+} // namespace simgrid::s4u
/* **************************** Public C interface *************************** */
void sg_comm_detach(sg_comm_t comm, void (*clean_function)(void*))
{
