1 /* Copyright (c) 2013-2017. The SimGrid Team. All rights reserved. */
3 /* This program is free software; you can redistribute it and/or modify it
4 * under the terms of the license (GNU LGPL) which comes with this package. */
6 #include <simgrid/plugins/live_migration.h>
7 #include <simgrid/s4u.hpp>
8 #include <simgrid/s4u/VirtualMachine.hpp>
9 #include <simgrid/vm.h>
10 #include <src/instr/instr_private.hpp>
11 #include <src/plugins/vm/VirtualMachineImpl.hpp>
12 #include <src/plugins/vm/VmHostExt.hpp>
13 #include <src/plugins/vm/VmLiveMigration.hpp>
16 XBT_LOG_NEW_DEFAULT_CATEGORY(vm_live_migration, "S4U virtual machines live migration");
20 simgrid::xbt::Extension<s4u::Host, VmMigrationExt> VmMigrationExt::EXTENSION_ID;
22 void VmMigrationExt::ensureVmMigrationExtInstalled()
24 if (not EXTENSION_ID.valid())
25 EXTENSION_ID = simgrid::s4u::Host::extension_create<VmMigrationExt>();
28 void MigrationRx::operator()()
30 XBT_DEBUG("mig: rx_start");
31 bool received_finalize = false;
33 std::string finalize_task_name =
34 std::string("__mig_stage3:") + vm_->getCname() + "(" + src_pm_->getCname() + "-" + dst_pm_->getCname() + ")";
36 while (not received_finalize) {
37 std::string* payload = static_cast<std::string*>(mbox->get());
39 if (finalize_task_name == *payload)
40 received_finalize = true;
45 // Here Stage 1, 2 and 3 have been performed.
46 // Hence complete the migration
48 /* Update the vm location */
49 /* precopy migration makes the VM temporally paused */
50 xbt_assert(vm_->getState() == SURF_VM_STATE_SUSPENDED);
52 /* Update the vm location and resume it */
56 // Now the VM is running on the new host (the migration is completed) (even if the SRC crash)
57 vm_->getImpl()->isMigrating = false;
58 XBT_DEBUG("VM(%s) moved from PM(%s) to PM(%s)", vm_->getCname(), src_pm_->getCname(), dst_pm_->getCname());
60 if (TRACE_msg_vm_is_enabled()) {
61 static long long int counter = 0;
62 std::string key = std::to_string(counter);
66 container_t msg = simgrid::instr::Container::byName(vm_->getName());
67 simgrid::instr::Container::getRoot()->getLink("MSG_VM_LINK")->startEvent(msg, "M", key);
69 // destroy existing container of this vm
70 container_t existing_container = simgrid::instr::Container::byName(vm_->getName());
71 existing_container->removeFromParent();
72 delete existing_container;
74 // create new container on the new_host location
75 new simgrid::instr::Container(vm_->getCname(), "MSG_VM", simgrid::instr::Container::byName(dst_pm_->getName()));
78 msg = simgrid::instr::Container::byName(vm_->getName());
79 simgrid::instr::Container::getRoot()->getLink("MSG_VM_LINK")->endEvent(msg, "M", key);
81 // Inform the SRC that the migration has been correctly performed
82 std::string* payload = new std::string("__mig_stage4:");
83 *payload = *payload + vm_->getCname() + "(" + src_pm_->getCname() + "-" + dst_pm_->getCname() + ")";
85 mbox_ctl->put(payload, 0);
87 XBT_DEBUG("mig: rx_done");
90 static sg_size_t get_updated_size(double computed, double dp_rate, sg_size_t dp_cap)
92 sg_size_t updated_size = static_cast<sg_size_t>(computed * dp_rate);
93 XBT_DEBUG("updated_size %llu dp_rate %f", updated_size, dp_rate);
94 if (updated_size > dp_cap) {
95 updated_size = dp_cap;
101 sg_size_t MigrationTx::sendMigrationData(sg_size_t size, int stage, int stage2_round, double mig_speed, double timeout)
103 sg_size_t sent = size;
104 std::string* msg = new std::string("__mig_stage");
105 *msg = *msg + std::to_string(stage) + ":" + vm_->getCname() + "(" + src_pm_->getCname() + "-" + dst_pm_->getCname() +
108 double clock_sta = s4u::Engine::getClock();
110 s4u::Activity* comm = nullptr;
113 comm = mbox->put_init(msg, size)->setRate(mig_speed)->wait(timeout);
115 comm = mbox->put_async(msg, size)->wait();
116 } catch (xbt_ex& e) {
118 sg_size_t remaining = static_cast<sg_size_t>(comm->getRemains());
119 XBT_VERB("timeout (%lf s) in sending_migration_data, remaining %llu bytes of %llu", timeout, remaining, size);
125 double clock_end = s4u::Engine::getClock();
126 double duration = clock_end - clock_sta;
127 double actual_speed = size / duration;
130 XBT_DEBUG("mig-stage%d.%d: sent %llu duration %f actual_speed %f (target %f)", stage, stage2_round, size, duration,
131 actual_speed, mig_speed);
133 XBT_DEBUG("mig-stage%d: sent %llu duration %f actual_speed %f (target %f)", stage, size, duration, actual_speed,
139 void MigrationTx::operator()()
141 XBT_DEBUG("mig: tx_start");
143 double host_speed = vm_->getPm()->getSpeed();
144 const sg_size_t ramsize = vm_->getRamsize();
145 const double dp_rate =
146 host_speed ? (sg_vm_get_migration_speed(vm_) * sg_vm_get_dirty_page_intensity(vm_)) / host_speed : 1;
147 const sg_size_t dp_cap = sg_vm_get_working_set_memory(vm_);
148 const double mig_speed = sg_vm_get_migration_speed(vm_);
149 double max_downtime = sg_vm_get_max_downtime(vm_);
151 double mig_timeout = 10000000.0;
152 bool skip_stage2 = false;
154 size_t remaining_size = ramsize;
155 size_t threshold = 0.0;
157 /* check parameters */
159 XBT_WARN("migrate a VM, but ramsize is zero");
161 if (max_downtime <= 0) {
162 XBT_WARN("use the default max_downtime value 30ms");
166 /* Stage1: send all memory pages to the destination. */
167 XBT_DEBUG("mig-stage1: remaining_size %zu", remaining_size);
168 sg_vm_start_dirty_page_tracking(vm_);
170 double computed_during_stage1 = 0;
171 double clock_prev_send = s4u::Engine::getClock();
174 /* At stage 1, we do not need timeout. We have to send all the memory pages even though the duration of this
175 * transfer exceeds the timeout value. */
176 XBT_VERB("Stage 1: Gonna send %llu bytes", ramsize);
177 sg_size_t sent = sendMigrationData(ramsize, 1, 0, mig_speed, -1);
178 remaining_size -= sent;
179 computed_during_stage1 = sg_vm_lookup_computed_flops(vm_);
181 if (sent < ramsize) {
182 XBT_VERB("mig-stage1: timeout, force moving to stage 3");
184 } else if (sent > ramsize)
187 } catch (xbt_ex& e) {
188 // hostfailure (if you want to know whether this is the SRC or the DST check directly in send_migration_data code)
189 // Stop the dirty page tracking an return (there is no memory space to release)
190 sg_vm_stop_dirty_page_tracking(vm_);
194 double clock_post_send = s4u::Engine::getClock();
195 mig_timeout -= (clock_post_send - clock_prev_send);
196 if (mig_timeout < 0) {
197 XBT_VERB("The duration of stage 1 exceeds the timeout value, skip stage 2");
201 /* estimate bandwidth */
202 double bandwidth = ramsize / (clock_post_send - clock_prev_send);
203 threshold = bandwidth * max_downtime;
204 XBT_DEBUG("actual bandwidth %f (MB/s), threshold %zu", bandwidth / 1024 / 1024, threshold);
206 /* Stage2: send update pages iteratively until the size of remaining states becomes smaller than threshold value. */
207 if (not skip_stage2) {
209 int stage2_round = 0;
211 sg_size_t updated_size = 0;
212 if (stage2_round == 0) {
213 /* just after stage1, nothing has been updated. But, we have to send the data updated during stage1 */
214 updated_size = get_updated_size(computed_during_stage1, dp_rate, dp_cap);
216 double computed = sg_vm_lookup_computed_flops(vm_);
217 updated_size = get_updated_size(computed, dp_rate, dp_cap);
220 XBT_DEBUG("mig-stage 2:%d updated_size %llu computed_during_stage1 %f dp_rate %f dp_cap %llu", stage2_round,
221 updated_size, computed_during_stage1, dp_rate, dp_cap);
223 /* Check whether the remaining size is below the threshold value. If so, move to stage 3. */
224 remaining_size += updated_size;
225 XBT_DEBUG("mig-stage2.%d: remaining_size %zu (%s threshold %zu)", stage2_round, remaining_size,
226 (remaining_size < threshold) ? "<" : ">", threshold);
227 if (remaining_size < threshold)
231 double clock_prev_send = s4u::Engine::getClock();
233 XBT_DEBUG("Stage 2, gonna send %llu", updated_size);
234 sent = sendMigrationData(updated_size, 2, stage2_round, mig_speed, mig_timeout);
235 } catch (xbt_ex& e) {
236 // hostfailure (if you want to know whether this is the SRC or the DST check directly in send_migration_data
238 // Stop the dirty page tracking an return (there is no memory space to release)
239 sg_vm_stop_dirty_page_tracking(vm_);
242 double clock_post_send = s4u::Engine::getClock();
244 if (sent == updated_size) {
245 /* timeout did not happen */
246 double bandwidth = updated_size / (clock_post_send - clock_prev_send);
247 threshold = bandwidth * max_downtime;
248 XBT_DEBUG("actual bandwidth %f, threshold %zu", bandwidth / 1024 / 1024, threshold);
249 remaining_size -= sent;
251 mig_timeout -= (clock_post_send - clock_prev_send);
252 xbt_assert(mig_timeout > 0);
254 } else if (sent < updated_size) {
255 /* When timeout happens, we move to stage 3. The size of memory pages
256 * updated before timeout must be added to the remaining size. */
257 XBT_VERB("mig-stage2.%d: timeout, force moving to stage 3. sent %llu / %llu, eta %lf", stage2_round, sent,
258 updated_size, (clock_post_send - clock_prev_send));
259 remaining_size -= sent;
261 double computed = sg_vm_lookup_computed_flops(vm_);
262 updated_size = get_updated_size(computed, dp_rate, dp_cap);
263 remaining_size += updated_size;
270 /* Stage3: stop the VM and copy the rest of states. */
271 XBT_DEBUG("mig-stage3: remaining_size %zu", remaining_size);
273 sg_vm_stop_dirty_page_tracking(vm_);
276 XBT_DEBUG("Stage 3: Gonna send %zu bytes", remaining_size);
277 sendMigrationData(remaining_size, 3, 0, mig_speed, -1);
278 } catch (xbt_ex& e) {
279 // hostfailure (if you want to know whether this is the SRC or the DST check directly in send_migration_data code)
280 // Stop the dirty page tracking an return (there is no memory space to release)
285 // At that point the Migration is considered valid for the SRC node but remind that the DST side should relocate
286 // effectively the VM on the DST node.
287 XBT_DEBUG("mig: tx_done");
294 static void onVirtualMachineShutdown(simgrid::s4u::VirtualMachine* vm)
296 if (vm->getImpl()->isMigrating) {
297 vm->extension<simgrid::vm::VmMigrationExt>()->rx_->kill();
298 vm->extension<simgrid::vm::VmMigrationExt>()->tx_->kill();
299 vm->extension<simgrid::vm::VmMigrationExt>()->issuer_->kill();
300 vm->getImpl()->isMigrating = false;
304 void sg_vm_live_migration_plugin_init()
306 sg_vm_dirty_page_tracking_init();
307 simgrid::vm::VmMigrationExt::ensureVmMigrationExtInstalled();
308 simgrid::s4u::VirtualMachine::onVmShutdown.connect(&onVirtualMachineShutdown);
311 /* Deprecated. Please use MSG_vm_create_migratable() instead */
312 msg_vm_t MSG_vm_create(msg_host_t ind_pm, const char* name, int coreAmount, int ramsize, int mig_netspeed,
315 return sg_vm_create_migratable(ind_pm, name, coreAmount, ramsize, mig_netspeed, dp_intensity);
318 simgrid::s4u::VirtualMachine* sg_vm_create_migratable(simgrid::s4u::Host* pm, const char* name, int coreAmount,
319 int ramsize, int mig_netspeed, int dp_intensity)
321 simgrid::vm::VmHostExt::ensureVmExtInstalled();
323 /* For the moment, intensity_rate is the percentage against the migration bandwidth */
325 sg_vm_t vm = new simgrid::s4u::VirtualMachine(name, pm, coreAmount, static_cast<sg_size_t>(ramsize) * 1024 * 1024);
326 sg_vm_set_dirty_page_intensity(vm, dp_intensity / 100.0);
327 sg_vm_set_working_set_memory(vm, vm->getRamsize() * 0.9); // assume working set memory is 90% of ramsize
328 sg_vm_set_migration_speed(vm, mig_netspeed * 1024 * 1024.0);
330 XBT_DEBUG("migspeed : %f intensity mem : %d", mig_netspeed * 1024 * 1024.0, dp_intensity);
335 int sg_vm_is_migrating(simgrid::s4u::VirtualMachine* vm)
337 return vm->getImpl()->isMigrating;
340 void sg_vm_migrate(simgrid::s4u::VirtualMachine* vm, simgrid::s4u::Host* dst_pm)
342 simgrid::s4u::Host* src_pm = vm->getPm();
345 THROWF(vm_error, 0, "Cannot migrate VM '%s' from host '%s', which is offline.", vm->getCname(), src_pm->getCname());
347 THROWF(vm_error, 0, "Cannot migrate VM '%s' to host '%s', which is offline.", vm->getCname(), dst_pm->getCname());
348 if (vm->getState() != SURF_VM_STATE_RUNNING)
349 THROWF(vm_error, 0, "Cannot migrate VM '%s' that is not running yet.", vm->getCname());
350 if (vm->getImpl()->isMigrating)
351 THROWF(vm_error, 0, "Cannot migrate VM '%s' that is already migrating.", vm->getCname());
353 vm->getImpl()->isMigrating = true;
355 std::string rx_name =
356 std::string("__pr_mig_rx:") + vm->getCname() + "(" + src_pm->getCname() + "-" + dst_pm->getCname() + ")";
357 std::string tx_name =
358 std::string("__pr_mig_tx:") + vm->getCname() + "(" + src_pm->getCname() + "-" + dst_pm->getCname() + ")";
360 simgrid::s4u::ActorPtr rx =
361 simgrid::s4u::Actor::createActor(rx_name.c_str(), dst_pm, simgrid::vm::MigrationRx(vm, dst_pm));
362 simgrid::s4u::ActorPtr tx =
363 simgrid::s4u::Actor::createActor(tx_name.c_str(), src_pm, simgrid::vm::MigrationTx(vm, dst_pm));
365 vm->extension_set<simgrid::vm::VmMigrationExt>(new simgrid::vm::VmMigrationExt(simgrid::s4u::Actor::self(), rx, tx));
367 /* wait until the migration have finished or on error has occurred */
368 XBT_DEBUG("wait for reception of the final ACK (i.e. migration has been correctly performed");
369 simgrid::s4u::MailboxPtr mbox_ctl = simgrid::s4u::Mailbox::byName(
370 std::string("__mbox_mig_ctl:") + vm->getCname() + "(" + src_pm->getCname() + "-" + dst_pm->getCname() + ")");
371 delete static_cast<std::string*>(mbox_ctl->get());
375 vm->getImpl()->isMigrating = false;