From: Frederic Suter Date: Mon, 16 Sep 2019 09:28:01 +0000 (+0200) Subject: Merge branch 'master' into disk X-Git-Tag: v3.24~87 X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/5ab070a2054636f5dcf5f0b56d691b089c5d16e4?hp=219b128344e14ff691c5f9615741d006ac79aeec Merge branch 'master' into disk --- diff --git a/MANIFEST.in b/MANIFEST.in index 2caa104a12..5a4621f5c1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -377,6 +377,8 @@ include examples/s4u/exec-waitany/s4u-exec-waitany.cpp include examples/s4u/exec-waitany/s4u-exec-waitany.tesh include examples/s4u/io-async/s4u-io-async.cpp include examples/s4u/io-async/s4u-io-async.tesh +include examples/s4u/io-disk-raw/s4u-io-disk-raw.cpp +include examples/s4u/io-disk-raw/s4u-io-disk-raw.tesh include examples/s4u/io-file-remote/s4u-io-file-remote.cpp include examples/s4u/io-file-remote/s4u-io-file-remote.tesh include examples/s4u/io-file-remote/s4u-io-file-remote_d.xml @@ -1832,6 +1834,7 @@ include examples/platforms/energy_platform.xml include examples/platforms/faulty_host.xml include examples/platforms/g5k.xml include examples/platforms/griffon.xml +include examples/platforms/hosts_with_disks.xml include examples/platforms/meta_cluster.xml include examples/platforms/multicore_machine.xml include examples/platforms/onelink.xml @@ -1943,6 +1946,7 @@ include include/simgrid/s4u/Actor.hpp include include/simgrid/s4u/Barrier.hpp include include/simgrid/s4u/Comm.hpp include include/simgrid/s4u/ConditionVariable.hpp +include include/simgrid/s4u/Disk.hpp include include/simgrid/s4u/Engine.hpp include include/simgrid/s4u/Exec.hpp include include/simgrid/s4u/Host.hpp @@ -2143,6 +2147,8 @@ include src/kernel/lmm/maxmin.cpp include src/kernel/lmm/maxmin.hpp include src/kernel/lmm/maxmin_test.cpp include src/kernel/resource/Action.cpp +include src/kernel/resource/DiskImpl.cpp +include src/kernel/resource/DiskImpl.hpp include src/kernel/resource/Model.cpp include src/kernel/resource/Resource.cpp include src/kernel/resource/profile/DatedValue.cpp @@ -2271,6 +2277,7 @@ include src/s4u/s4u_Actor.cpp include src/s4u/s4u_Barrier.cpp include src/s4u/s4u_Comm.cpp include src/s4u/s4u_ConditionVariable.cpp +include src/s4u/s4u_Disk.cpp include src/s4u/s4u_Engine.cpp include src/s4u/s4u_Exec.cpp include src/s4u/s4u_Host.cpp @@ -2503,6 +2510,8 @@ include src/surf/cpu_interface.cpp include src/surf/cpu_interface.hpp include src/surf/cpu_ti.cpp include src/surf/cpu_ti.hpp +include src/surf/disk_s19.cpp +include src/surf/disk_s19.hpp include src/surf/host_clm03.cpp include src/surf/host_clm03.hpp include src/surf/network_cm02.cpp diff --git a/examples/platforms/hosts_with_disks.xml b/examples/platforms/hosts_with_disks.xml new file mode 100644 index 0000000000..cb22063545 --- /dev/null +++ b/examples/platforms/hosts_with_disks.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/s4u/CMakeLists.txt b/examples/s4u/CMakeLists.txt index e6d1930592..5672278463 100644 --- a/examples/s4u/CMakeLists.txt +++ b/examples/s4u/CMakeLists.txt @@ -10,7 +10,7 @@ foreach (example actor-create actor-daemon actor-exiting actor-join actor-kill energy-exec energy-boot energy-link energy-vm engine-filtering exec-async exec-basic exec-dvfs exec-ptask exec-remote exec-waitany - io-async io-file-system io-file-remote io-storage-raw + io-async io-file-system io-file-remote io-storage-raw io-disk-raw platform-failures platform-profile platform-properties plugin-hostload replay-comm replay-storage diff --git a/examples/s4u/io-async/s4u-io-async.cpp b/examples/s4u/io-async/s4u-io-async.cpp index 13304cc3df..597284fcc9 100644 --- a/examples/s4u/io-async/s4u-io-async.cpp +++ b/examples/s4u/io-async/s4u-io-async.cpp @@ -9,10 +9,10 @@ XBT_LOG_NEW_DEFAULT_CATEGORY(s4u_test, "Messages specific for this s4u example") static void test(sg_size_t size) { - simgrid::s4u::Storage* storage = simgrid::s4u::Storage::by_name("Disk1"); - XBT_INFO("Hello! read %llu bytes from Storage %s", size, storage->get_cname()); + simgrid::s4u::Disk* disk = simgrid::s4u::Host::current()->get_disks().front(); + XBT_INFO("Hello! read %llu bytes from %s", size, disk->get_cname()); - simgrid::s4u::IoPtr activity = storage->io_init(size, simgrid::s4u::Io::OpType::READ); + simgrid::s4u::IoPtr activity = disk->io_init(size, simgrid::s4u::Io::OpType::READ); activity->start(); activity->wait(); @@ -21,10 +21,10 @@ static void test(sg_size_t size) static void test_cancel(sg_size_t size) { - simgrid::s4u::Storage* storage = simgrid::s4u::Storage::by_name("Disk2"); - XBT_INFO("Hello! write %llu bytes from Storage %s", size, storage->get_cname()); + simgrid::s4u::Disk* disk = simgrid::s4u::Host::current()->get_disks().front(); + XBT_INFO("Hello! write %llu bytes from %s", size, disk->get_cname()); - simgrid::s4u::IoPtr activity = storage->write_async(size); + simgrid::s4u::IoPtr activity = disk->write_async(size); simgrid::s4u::this_actor::sleep_for(0.5); XBT_INFO("I changed my mind, cancel!"); activity->cancel(); diff --git a/examples/s4u/io-async/s4u-io-async.tesh b/examples/s4u/io-async/s4u-io-async.tesh index d2a6acbc23..0ff50eb845 100644 --- a/examples/s4u/io-async/s4u-io-async.tesh +++ b/examples/s4u/io-async/s4u-io-async.tesh @@ -1,8 +1,8 @@ #!/usr/bin/env tesh -$ ${bindir:=.}/s4u-io-async ${platfdir}/storage/storage.xml "--log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n" -> [ 0.000000] (1:test@bob) Hello! read 20000000 bytes from Storage Disk1 -> [ 0.000000] (2:test_cancel@alice) Hello! write 50000000 bytes from Storage Disk2 +$ ${bindir:=.}/s4u-io-async ${platfdir}/hosts_with_disks.xml "--log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n" +> [ 0.000000] (1:test@bob) Hello! read 20000000 bytes from Disk1 +> [ 0.000000] (2:test_cancel@alice) Hello! write 50000000 bytes from Disk1 > [ 0.200000] (1:test@bob) Goodbye now! > [ 0.500000] (2:test_cancel@alice) I changed my mind, cancel! > [ 0.500000] (2:test_cancel@alice) Goodbye now! diff --git a/examples/s4u/io-disk-raw/s4u-io-disk-raw.cpp b/examples/s4u/io-disk-raw/s4u-io-disk-raw.cpp new file mode 100644 index 0000000000..2152a10b53 --- /dev/null +++ b/examples/s4u/io-disk-raw/s4u-io-disk-raw.cpp @@ -0,0 +1,57 @@ +/* Copyright (c) 2017-2019. The SimGrid Team. All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#include "simgrid/s4u.hpp" +#include +#include + +XBT_LOG_NEW_DEFAULT_CATEGORY(disk, "Messages specific for this simulation"); + +static void host() +{ + /* - Display information on the disks mounted by the current host */ + XBT_INFO("*** Storage info on %s ***", simgrid::s4u::Host::current()->get_cname()); + + /* - Retrieve all disks from current host */ + std::vector const& disk_list = simgrid::s4u::Host::current()->get_disks(); + + /* - For each disk mounted on host, display disk name and mount point */ + for (auto disk : disk_list) + XBT_INFO("Disk name: %s", disk->get_cname()); + + /* - Write 400,000 bytes on Disk1 */ + simgrid::s4u::Disk* disk = disk_list.front(); + sg_size_t write = disk->write(400000); + XBT_INFO("Wrote %llu bytes on '%s'", write, disk->get_cname()); + + /* - Now read 200,000 bytes */ + sg_size_t read = disk->read(200000); + XBT_INFO("Read %llu bytes on '%s'", read, disk->get_cname()); + + /* - Attach some user data to disk1 */ + XBT_INFO("*** Get/set data for storage element: Disk1 ***"); + + std::string* data = static_cast(disk->get_data()); + + XBT_INFO("Get storage data: '%s'", data ? data->c_str() : "No user data"); + + disk->set_data(new std::string("Some user data")); + data = static_cast(disk->get_data()); + XBT_INFO("Set and get data: '%s'", data->c_str()); + delete data; +} + +int main(int argc, char** argv) +{ + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + simgrid::s4u::Actor::create("", simgrid::s4u::Host::by_name("bob"), host); + + e.run(); + XBT_INFO("Simulated time: %g", simgrid::s4u::Engine::get_clock()); + + return 0; +} diff --git a/examples/s4u/io-disk-raw/s4u-io-disk-raw.tesh b/examples/s4u/io-disk-raw/s4u-io-disk-raw.tesh new file mode 100644 index 0000000000..f1906a81c2 --- /dev/null +++ b/examples/s4u/io-disk-raw/s4u-io-disk-raw.tesh @@ -0,0 +1,12 @@ +#!/usr/bin/env tesh + +$ ${bindir}/s4u-io-disk-raw ${platfdir}/hosts_with_disks.xml "--log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n" +> [ 0.000000] (1:@bob) *** Storage info on bob *** +> [ 0.000000] (1:@bob) Disk name: Disk1 +> [ 0.000000] (1:@bob) Disk name: Disk2 +> [ 0.010000] (1:@bob) Wrote 400000 bytes on 'Disk1' +> [ 0.012000] (1:@bob) Read 200000 bytes on 'Disk1' +> [ 0.012000] (1:@bob) *** Get/set data for storage element: Disk1 *** +> [ 0.012000] (1:@bob) Get storage data: 'No user data' +> [ 0.012000] (1:@bob) Set and get data: 'Some user data' +> [ 0.012000] (0:maestro@) Simulated time: 0.012 diff --git a/examples/s4u/io-file-remote/s4u-io-file-remote.cpp b/examples/s4u/io-file-remote/s4u-io-file-remote.cpp index b4e63504d7..35793244f7 100644 --- a/examples/s4u/io-file-remote/s4u-io-file-remote.cpp +++ b/examples/s4u/io-file-remote/s4u-io-file-remote.cpp @@ -17,33 +17,21 @@ static int host(int argc, char* argv[]) const char* filename = file.get_path(); XBT_INFO("Opened file '%s'", filename); file.dump(); - - XBT_INFO("Try to read %llu from '%s'", file.size(), filename); - sg_size_t read = file.read(file.size()); - XBT_INFO("Have read %llu from '%s'. Offset is now at: %llu", read, filename, file.tell()); - XBT_INFO("Seek back to the beginning of the stream..."); - file.seek(0, SEEK_SET); - XBT_INFO("Offset is now at: %llu", file.tell()); - - if (argc > 5) { - simgrid::s4u::File remoteFile(argv[2], nullptr); - filename = remoteFile.get_path(); - XBT_INFO("Opened file '%s'", filename); - XBT_INFO("Try to write %llu MiB to '%s'", remoteFile.size() / 1024, filename); - sg_size_t write = remoteFile.write(remoteFile.size() * 1024); - XBT_INFO("Have written %llu bytes to '%s'.", write, filename); - - if (std::stoi(argv[5]) != 0) { - XBT_INFO("Move '%s' (of size %llu) from '%s' to '%s'", filename, remoteFile.size(), - simgrid::s4u::Host::current()->get_cname(), argv[3]); - remoteFile.remote_move(simgrid::s4u::Host::by_name(argv[3]), argv[4]); + XBT_INFO("Try to write %llu MiB to '%s'", file.size() / 1024, filename); + sg_size_t write = file.write(file.size() * 1024); + XBT_INFO("Have written %llu MiB to '%s'.", write / (1024 * 1024), filename); + + if (argc > 4) { + if (std::stoi(argv[4]) != 0) { + XBT_INFO("Move '%s' (of size %llu) from '%s' to '%s'", filename, file.size(), + simgrid::s4u::Host::current()->get_cname(), argv[2]); + file.remote_move(simgrid::s4u::Host::by_name(argv[2]), argv[3]); } else { - XBT_INFO("Copy '%s' (of size %llu) from '%s' to '%s'", filename, remoteFile.size(), - simgrid::s4u::Host::current()->get_cname(), argv[3]); - remoteFile.remote_copy(simgrid::s4u::Host::by_name(argv[3]), argv[4]); + XBT_INFO("Copy '%s' (of size %llu) from '%s' to '%s'", filename, file.size(), + simgrid::s4u::Host::current()->get_cname(), argv[2]); + file.remote_copy(simgrid::s4u::Host::by_name(argv[2]), argv[3]); } } - return 0; } @@ -54,18 +42,20 @@ int main(int argc, char** argv) e.load_platform(argv[1]); e.register_function("host", host); e.load_deployment(argv[2]); - std::vector allStorages = e.get_all_storages(); + std::vector all_hosts = e.get_all_hosts(); - for (auto const& s : allStorages) { - XBT_INFO("Init: %llu/%llu MiB used/free on '%s'", sg_storage_get_size_used(s) / INMEGA, - sg_storage_get_size_free(s) / INMEGA, s->get_cname()); + for (auto const& h : all_hosts) { + for (auto const& d : h->get_disks()) + XBT_INFO("Init: %s: %llu/%llu MiB used/free on '%s@%s'", h->get_cname(), sg_disk_get_size_used(d) / INMEGA, + sg_disk_get_size_free(d) / INMEGA, d->get_cname(), d->get_host()->get_cname()); } e.run(); - for (auto const& s : allStorages) { - XBT_INFO("End: %llu/%llu MiB used/free on '%s'", sg_storage_get_size_used(s) / INMEGA, - sg_storage_get_size_free(s) / INMEGA, s->get_cname()); + for (auto const& h : all_hosts) { + for (auto const& d : h->get_disks()) + XBT_INFO("End: %llu/%llu MiB used/free on '%s@%s'", sg_disk_get_size_used(d) / INMEGA, + sg_disk_get_size_free(d) / INMEGA, d->get_cname(), h->get_cname()); } XBT_INFO("Simulation time %g", simgrid::s4u::Engine::get_clock()); diff --git a/examples/s4u/io-file-remote/s4u-io-file-remote.tesh b/examples/s4u/io-file-remote/s4u-io-file-remote.tesh index 1f5c26c3c5..1b13cbae4d 100644 --- a/examples/s4u/io-file-remote/s4u-io-file-remote.tesh +++ b/examples/s4u/io-file-remote/s4u-io-file-remote.tesh @@ -1,64 +1,43 @@ #!/usr/bin/env tesh -$ ${bindir:=.}/s4u-io-file-remote ${platfdir}/storage/remote_io.xml s4u-io-file-remote_d.xml "--log=root.fmt:[%10.6r]%e(%i@%5h)%e%m%n" -> [ 0.000000] (0@ ) Init: 12/476824 MiB used/free on 'Disk1' -> [ 0.000000] (0@ ) Init: 2280/474556 MiB used/free on 'Disk2' -> [ 0.000000] (1@alice) Opened file 'c:\Windows\setupact.log' +$ ${bindir:=.}/s4u-io-file-remote ${platfdir}/hosts_with_disks.xml s4u-io-file-remote_d.xml "--log=root.fmt:[%10.6r]%e(%i@%5h)%e%m%n" +> [ 0.000000] (0@ ) Init: alice: 12/511987 MiB used/free on 'Disk1@alice' +> [ 0.000000] (0@ ) Init: bob: 35/511964 MiB used/free on 'Disk1@bob' +> [ 0.000000] (0@ ) Init: bob: 0/512000 MiB used/free on 'Disk2@bob' +> [ 0.000000] (0@ ) Init: carl: 35/511964 MiB used/free on 'Disk1@bob' +> [ 0.000000] (1@alice) Opened file '/lib/libsimgrid.so.3.6.2' > [ 0.000000] (1@alice) File Descriptor information: -> Full path: 'c:\Windows\setupact.log' -> Size: 101663 -> Mount point: 'c:' -> Storage Id: 'Disk2' -> Storage Type: 'SATA-II_HDD' +> Full path: '/lib/libsimgrid.so.3.6.2' +> Size: 12710497 +> Mount point: '/' +> Disk Id: 'Disk1' +> Host Id: 'alice' > File Descriptor Id: 0 -> [ 0.000000] (1@alice) Try to read 101663 from 'c:\Windows\setupact.log' -> [ 0.000000] (2@ bob) Opened file '/scratch/lib/libsimgrid.so.3.6.2' +> [ 0.000000] (1@alice) Try to write 12412 MiB to '/lib/libsimgrid.so.3.6.2' +> [ 0.000000] (2@ bob) Opened file '/scratch/doc/simgrid/examples/platforms/g5k.xml' > [ 0.000000] (2@ bob) File Descriptor information: -> Full path: '/scratch/lib/libsimgrid.so.3.6.2' -> Size: 12710497 +> Full path: '/scratch/doc/simgrid/examples/platforms/g5k.xml' +> Size: 17028 > Mount point: '/scratch' -> Storage Id: 'Disk1' -> Storage Type: 'SATA-II_HDD' +> Disk Id: 'Disk1' +> Host Id: 'bob' > File Descriptor Id: 0 -> [ 0.000000] (2@ bob) Try to read 12710497 from '/scratch/lib/libsimgrid.so.3.6.2' +> [ 0.000000] (2@ bob) Try to write 16 MiB to '/scratch/doc/simgrid/examples/platforms/g5k.xml' > [ 0.000000] (3@ carl) Opened file '/scratch/lib/libsimgrid.so.3.6.2' > [ 0.000000] (3@ carl) File Descriptor information: > Full path: '/scratch/lib/libsimgrid.so.3.6.2' > Size: 12710497 > Mount point: '/scratch' -> Storage Id: 'Disk1' -> Storage Type: 'SATA-II_HDD' -> File Descriptor Id: 0 -> [ 0.000000] (3@ carl) Try to read 12710497 from '/scratch/lib/libsimgrid.so.3.6.2' -> [ 0.000000] (4@ dave) Opened file 'c:\Windows\bootstat.dat' -> [ 0.000000] (4@ dave) File Descriptor information: -> Full path: 'c:\Windows\bootstat.dat' -> Size: 67584 -> Mount point: 'c:' -> Storage Id: 'Disk2' -> Storage Type: 'SATA-II_HDD' +> Disk Id: 'Disk1' +> Host Id: 'bob' > File Descriptor Id: 0 -> [ 0.000000] (4@ dave) Try to read 67584 from 'c:\Windows\bootstat.dat' -> [ 0.001469] (4@ dave) Have read 67584 from 'c:\Windows\bootstat.dat'. Offset is now at: 67584 -> [ 0.001469] (4@ dave) Seek back to the beginning of the stream... -> [ 0.001469] (4@ dave) Offset is now at: 0 -> [ 0.001469] (4@ dave) Opened file 'c:\Windows\Professional.xml' -> [ 0.001469] (4@ dave) Try to write 31 MiB to 'c:\Windows\Professional.xml' -> [ 0.003741] (1@alice) Have read 101663 from 'c:\Windows\setupact.log'. Offset is now at: 101663 -> [ 0.003741] (1@alice) Seek back to the beginning of the stream... -> [ 0.003741] (1@alice) Offset is now at: 0 -> [ 0.276315] (3@ carl) Have read 12710497 from '/scratch/lib/libsimgrid.so.3.6.2'. Offset is now at: 12710497 -> [ 0.276315] (3@ carl) Seek back to the beginning of the stream... -> [ 0.276315] (3@ carl) Offset is now at: 0 -> [ 0.387036] (2@ bob) Have read 12710497 from '/scratch/lib/libsimgrid.so.3.6.2'. Offset is now at: 12710497 -> [ 0.387036] (2@ bob) Seek back to the beginning of the stream... -> [ 0.387036] (2@ bob) Offset is now at: 0 -> [ 0.387036] (2@ bob) Opened file '/scratch/doc/simgrid/examples/platforms/g5k.xml' -> [ 0.387036] (2@ bob) Try to write 16 MiB to '/scratch/doc/simgrid/examples/platforms/g5k.xml' -> [ 0.528211] (4@ dave) Have written 32646144 bytes to 'c:\Windows\Professional.xml'. -> [ 0.528211] (4@ dave) Move 'c:\Windows\Professional.xml' (of size 32646144) from 'dave' to 'carl' -> [ 0.819921] (2@ bob) Have written 17436672 bytes to '/scratch/doc/simgrid/examples/platforms/g5k.xml'. -> [ 0.819921] (2@ bob) Copy '/scratch/doc/simgrid/examples/platforms/g5k.xml' (of size 17436672) from 'bob' to 'alice' -> [ 1.843969] (0@ ) End: 60/476776 MiB used/free on 'Disk1' -> [ 1.843969] (0@ ) End: 2297/474539 MiB used/free on 'Disk2' -> [ 1.843969] (0@ ) Simulation time 1.84397 +> [ 0.000000] (3@ carl) Try to write 12412 MiB to '/scratch/lib/libsimgrid.so.3.6.2' +> [ 0.435917] (2@ bob) Have written 16 MiB to '/scratch/doc/simgrid/examples/platforms/g5k.xml'. +> [ 0.435917] (2@ bob) Copy '/scratch/doc/simgrid/examples/platforms/g5k.xml' (of size 17436672) from 'bob' to 'alice' +> [162.912320] (1@alice) Have written 12412 MiB to '/lib/libsimgrid.so.3.6.2'. +> [162.912320] (1@alice) Move '/lib/libsimgrid.so.3.6.2' (of size 13015548928) from 'alice' to 'bob' +> [438.102645] (3@ carl) Have written 12412 MiB to '/scratch/lib/libsimgrid.so.3.6.2'. +> [778.806631] (0@ ) End: 17/511982 MiB used/free on 'Disk1@alice' +> [778.806631] (0@ ) End: 24852/487147 MiB used/free on 'Disk1@bob' +> [778.806631] (0@ ) End: 0/512000 MiB used/free on 'Disk2@bob' +> [778.806631] (0@ ) Simulation time 778.807 diff --git a/examples/s4u/io-file-remote/s4u-io-file-remote_d.xml b/examples/s4u/io-file-remote/s4u-io-file-remote_d.xml index 3b3f1ab705..956f940a69 100644 --- a/examples/s4u/io-file-remote/s4u-io-file-remote_d.xml +++ b/examples/s4u/io-file-remote/s4u-io-file-remote_d.xml @@ -2,23 +2,18 @@ - + + + + - - + - - - - - - - diff --git a/examples/s4u/io-file-system/s4u-io-file-system.cpp b/examples/s4u/io-file-system/s4u-io-file-system.cpp index c706b97b9e..4c91e69c3c 100644 --- a/examples/s4u/io-file-system/s4u-io-file-system.cpp +++ b/examples/s4u/io-file-system/s4u-io-file-system.cpp @@ -4,7 +4,7 @@ * under the terms of the license (GNU LGPL) which comes with this package. */ #include -#include +#include #include "simgrid/plugins/file_system.h" #include "simgrid/s4u.hpp" @@ -13,36 +13,32 @@ XBT_LOG_NEW_DEFAULT_CATEGORY(s4u_test, "a sample log category"); class MyHost { public: - void show_info(std::unordered_map const& mounts) + void show_info(std::vector const& disks) { XBT_INFO("Storage info on %s:", simgrid::s4u::Host::current()->get_cname()); - for (auto const& kv : mounts) { - std::string mountpoint = kv.first; - simgrid::s4u::Storage* storage = kv.second; - + for (auto const& d : disks) { // Retrieve disk's information - XBT_INFO(" %s (%s) Used: %llu; Free: %llu; Total: %llu.", storage->get_cname(), mountpoint.c_str(), - sg_storage_get_size_used(storage), sg_storage_get_size_free(storage), sg_storage_get_size(storage)); + XBT_INFO(" %s (%s) Used: %llu; Free: %llu; Total: %llu.", d->get_cname(), sg_disk_get_mount_point(d), + sg_disk_get_size_used(d), sg_disk_get_size_free(d), sg_disk_get_size(d)); } } void operator()() { - std::unordered_map const& mounts = - simgrid::s4u::Host::current()->get_mounted_storages(); + std::vector const& disks = simgrid::s4u::Host::current()->get_disks(); - show_info(mounts); + show_info(disks); // Open an non-existing file to create it - std::string filename = "/home/tmp/data.txt"; + std::string filename = "/scratch/tmp/data.txt"; simgrid::s4u::File* file = new simgrid::s4u::File(filename, nullptr); sg_size_t write = file->write(200000); // Write 200,000 bytes - XBT_INFO("Create a %llu bytes file named '%s' on /sd1", write, filename.c_str()); + XBT_INFO("Create a %llu bytes file named '%s' on /scratch", write, filename.c_str()); // check that sizes have changed - show_info(mounts); + show_info(disks); // Now retrieve the size of created file and read it completely const sg_size_t file_size = file->size(); @@ -54,10 +50,8 @@ public: write = file->write(100000); // Write 100,000 bytes XBT_INFO("Write %llu bytes on %s", write, filename.c_str()); - simgrid::s4u::Storage* storage = simgrid::s4u::Storage::by_name("Disk4"); - // Now rename file from ./tmp/data.txt to ./tmp/simgrid.readme - std::string newpath = "/home/tmp/simgrid.readme"; + std::string newpath = "/scratch/tmp/simgrid.readme"; XBT_INFO("Move '%s' to '%s'", file->get_path(), newpath.c_str()); file->move(newpath); @@ -70,23 +64,15 @@ public: // Close the file delete file; - // Now attach some user data to disk1 - XBT_INFO("Get/set data for storage element: %s", storage->get_cname()); - XBT_INFO(" Uninitialized storage data: '%s'", static_cast(storage->get_data())); - - storage->set_data(new std::string("Some user data")); - std::string* storage_data = static_cast(storage->get_data()); - XBT_INFO(" Set and get data: '%s'", storage_data->c_str()); - - delete storage_data; + show_info(disks); // Reopen the file and then unlink it - file = new simgrid::s4u::File("/home/tmp/simgrid.readme", nullptr); + file = new simgrid::s4u::File("/scratch/tmp/simgrid.readme", nullptr); XBT_INFO("Unlink file: '%s'", file->get_path()); file->unlink(); delete file; // Unlinking the file on "disk" does not free the object - show_info(mounts); + show_info(disks); } }; @@ -95,7 +81,7 @@ int main(int argc, char** argv) simgrid::s4u::Engine e(&argc, argv); sg_storage_file_system_init(); e.load_platform(argv[1]); - simgrid::s4u::Actor::create("host", simgrid::s4u::Host::by_name("denise"), MyHost()); + simgrid::s4u::Actor::create("host", simgrid::s4u::Host::by_name("bob"), MyHost()); e.run(); return 0; diff --git a/examples/s4u/io-file-system/s4u-io-file-system.tesh b/examples/s4u/io-file-system/s4u-io-file-system.tesh index 54e4fa380d..b2ad152bd1 100644 --- a/examples/s4u/io-file-system/s4u-io-file-system.tesh +++ b/examples/s4u/io-file-system/s4u-io-file-system.tesh @@ -1,21 +1,21 @@ #!/usr/bin/env tesh -$ ${bindir:=.}/s4u-io-file-system ${platfdir}/storage/storage.xml -> [denise:host:(1) 0.000000] [s4u_test/INFO] Storage info on denise: -> [denise:host:(1) 0.000000] [s4u_test/INFO] Disk2 (c:) Used: 2391537133; Free: 534479374867; Total: 536870912000. -> [denise:host:(1) 0.000000] [s4u_test/INFO] Disk4 (/home) Used: 13221994; Free: 536857690006; Total: 536870912000. -> [denise:host:(1) 0.003333] [s4u_test/INFO] Create a 200000 bytes file named '/home/tmp/data.txt' on /sd1 -> [denise:host:(1) 0.003333] [s4u_test/INFO] Storage info on denise: -> [denise:host:(1) 0.003333] [s4u_test/INFO] Disk2 (c:) Used: 2391537133; Free: 534479374867; Total: 536870912000. -> [denise:host:(1) 0.003333] [s4u_test/INFO] Disk4 (/home) Used: 13421994; Free: 536857490006; Total: 536870912000. -> [denise:host:(1) 0.004333] [s4u_test/INFO] Read 200000 bytes on /home/tmp/data.txt -> [denise:host:(1) 0.006000] [s4u_test/INFO] Write 100000 bytes on /home/tmp/data.txt -> [denise:host:(1) 0.006000] [s4u_test/INFO] Move '/home/tmp/data.txt' to '/home/tmp/simgrid.readme' -> [denise:host:(1) 0.006000] [s4u_test/INFO] User data attached to the file: 777 -> [denise:host:(1) 0.006000] [s4u_test/INFO] Get/set data for storage element: Disk4 -> [denise:host:(1) 0.006000] [s4u_test/INFO] Uninitialized storage data: '(null)' -> [denise:host:(1) 0.006000] [s4u_test/INFO] Set and get data: 'Some user data' -> [denise:host:(1) 0.006000] [s4u_test/INFO] Unlink file: '/home/tmp/simgrid.readme' -> [denise:host:(1) 0.006000] [s4u_test/INFO] Storage info on denise: -> [denise:host:(1) 0.006000] [s4u_test/INFO] Disk2 (c:) Used: 2391537133; Free: 534479374867; Total: 536870912000. -> [denise:host:(1) 0.006000] [s4u_test/INFO] Disk4 (/home) Used: 13221994; Free: 536857690006; Total: 536870912000. +$ ${bindir:=.}/s4u-io-file-system ${platfdir}/hosts_with_disks.xml +> [bob:host:(1) 0.000000] [s4u_test/INFO] Storage info on bob: +> [bob:host:(1) 0.000000] [s4u_test/INFO] Disk1 (/scratch) Used: 36933331; Free: 536833978669; Total: 536870912000. +> [bob:host:(1) 0.000000] [s4u_test/INFO] Disk2 (/) Used: 0; Free: 536870912000; Total: 536870912000. +> [bob:host:(1) 0.005000] [s4u_test/INFO] Create a 200000 bytes file named '/scratch/tmp/data.txt' on /scratch +> [bob:host:(1) 0.005000] [s4u_test/INFO] Storage info on bob: +> [bob:host:(1) 0.005000] [s4u_test/INFO] Disk1 (/scratch) Used: 37133331; Free: 536833778669; Total: 536870912000. +> [bob:host:(1) 0.005000] [s4u_test/INFO] Disk2 (/) Used: 0; Free: 536870912000; Total: 536870912000. +> [bob:host:(1) 0.007000] [s4u_test/INFO] Read 200000 bytes on /scratch/tmp/data.txt +> [bob:host:(1) 0.009500] [s4u_test/INFO] Write 100000 bytes on /scratch/tmp/data.txt +> [bob:host:(1) 0.009500] [s4u_test/INFO] Move '/scratch/tmp/data.txt' to '/scratch/tmp/simgrid.readme' +> [bob:host:(1) 0.009500] [s4u_test/INFO] User data attached to the file: 777 +> [bob:host:(1) 0.009500] [s4u_test/INFO] Storage info on bob: +> [bob:host:(1) 0.009500] [s4u_test/INFO] Disk1 (/scratch) Used: 37233331; Free: 536833678669; Total: 536870912000. +> [bob:host:(1) 0.009500] [s4u_test/INFO] Disk2 (/) Used: 0; Free: 536870912000; Total: 536870912000. +> [bob:host:(1) 0.009500] [s4u_test/INFO] Unlink file: '/scratch/tmp/simgrid.readme' +> [bob:host:(1) 0.009500] [s4u_test/INFO] Storage info on bob: +> [bob:host:(1) 0.009500] [s4u_test/INFO] Disk1 (/scratch) Used: 36933331; Free: 536833978669; Total: 536870912000. +> [bob:host:(1) 0.009500] [s4u_test/INFO] Disk2 (/) Used: 0; Free: 536870912000; Total: 536870912000. diff --git a/include/simgrid/forward.h b/include/simgrid/forward.h index 85eff8e2ae..8ce7d7b8e6 100644 --- a/include/simgrid/forward.h +++ b/include/simgrid/forward.h @@ -81,6 +81,7 @@ typedef boost::intrusive_ptr SemaphorePtr; XBT_PUBLIC void intrusive_ptr_release(Semaphore* m); XBT_PUBLIC void intrusive_ptr_add_ref(Semaphore* m); +class Disk; class Storage; } // namespace s4u @@ -140,6 +141,8 @@ class CpuModel; class NetworkModel; class LinkImpl; class NetworkAction; +class DiskImpl; +class DiskModel; class StorageImpl; class StorageType; class StorageModel; @@ -183,6 +186,7 @@ typedef simgrid::s4u::File s4u_File; typedef simgrid::s4u::ConditionVariable s4u_ConditionVariable; typedef simgrid::s4u::Mutex s4u_Mutex; typedef simgrid::s4u::Semaphore s4u_Semaphore; +typedef simgrid::s4u::Disk s4u_Disk; typedef simgrid::s4u::Storage s4u_Storage; typedef simgrid::s4u::NetZone s4u_NetZone; typedef simgrid::s4u::VirtualMachine s4u_VM; @@ -205,6 +209,7 @@ typedef struct s4u_File s4u_File; typedef struct s4u_ConditionVariable s4u_ConditionVariable; typedef struct s4u_Mutex s4u_Mutex; typedef struct s4u_Semaphore s4u_Semaphore; +typedef struct s4u_Disk s4u_Disk; typedef struct s4u_Storage s4u_Storage; typedef struct s4u_NetZone s4u_NetZone; typedef struct s4u_VM s4u_VM; @@ -226,6 +231,7 @@ typedef s4u_Semaphore* sg_sem_t; typedef s4u_NetZone* sg_netzone_t; typedef s4u_Host* sg_host_t; typedef s4u_Link* sg_link_t; +typedef s4u_Disk* sg_disk_t; typedef s4u_Storage* sg_storage_t; typedef s4u_File* sg_file_t; typedef s4u_VM* sg_vm_t; diff --git a/include/simgrid/plugins/file_system.h b/include/simgrid/plugins/file_system.h index 099cf337fb..8f8b4d55d7 100644 --- a/include/simgrid/plugins/file_system.h +++ b/include/simgrid/plugins/file_system.h @@ -41,6 +41,11 @@ XBT_PUBLIC void sg_file_unlink(sg_file_t fd); XBT_PUBLIC int sg_file_rcopy(sg_file_t file, sg_host_t host, const char* fullpath); XBT_PUBLIC int sg_file_rmove(sg_file_t file, sg_host_t host, const char* fullpath); +XBT_PUBLIC sg_size_t sg_disk_get_size_free(sg_disk_t d); +XBT_PUBLIC sg_size_t sg_disk_get_size_used(sg_disk_t d); +XBT_PUBLIC sg_size_t sg_disk_get_size(sg_disk_t d); +XBT_PUBLIC const char* sg_disk_get_mount_point(sg_disk_t d); + XBT_PUBLIC sg_size_t sg_storage_get_size_free(sg_storage_t st); XBT_PUBLIC sg_size_t sg_storage_get_size_used(sg_storage_t st); XBT_PUBLIC sg_size_t sg_storage_get_size(sg_storage_t st); @@ -126,7 +131,8 @@ public: void dump(); int desc_id = 0; - Storage* local_storage_; + Disk* local_disk_ = nullptr; + Storage* local_storage_ = nullptr; std::string mount_point_; private: @@ -137,6 +143,30 @@ private: void* userdata_ = nullptr; }; +class XBT_PUBLIC FileSystemDiskExt { +public: + static simgrid::xbt::Extension EXTENSION_ID; + explicit FileSystemDiskExt(Disk* ptr); + FileSystemDiskExt(const FileSystemDiskExt&) = delete; + FileSystemDiskExt& operator=(const FileSystemDiskExt&) = delete; + std::map* parse_content(const std::string& filename); + std::map* get_content() const { return content_.get(); } + const char* get_mount_point() { return mount_point_.c_str(); } + const char* get_mount_point(s4u::Host* remote_host) { return remote_mount_points_[remote_host].c_str(); } + void add_remote_mount(Host* host, std::string mount_point) { remote_mount_points_.insert({host, mount_point}); } + sg_size_t get_size() const { return size_; } + sg_size_t get_used_size() const { return used_size_; } + void decr_used_size(sg_size_t size) { used_size_ -= size; } + void incr_used_size(sg_size_t size) { used_size_ += size; } + +private: + std::unique_ptr> content_; + std::map remote_mount_points_; + std::string mount_point_; + sg_size_t used_size_ = 0; + sg_size_t size_ = static_cast(500 * 1024) * 1024 * 1024; +}; + class XBT_PUBLIC FileSystemStorageExt { public: static simgrid::xbt::Extension EXTENSION_ID; diff --git a/include/simgrid/s4u.hpp b/include/simgrid/s4u.hpp index dcbc463eb3..7dc400515a 100644 --- a/include/simgrid/s4u.hpp +++ b/include/simgrid/s4u.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/include/simgrid/s4u/Disk.hpp b/include/simgrid/s4u/Disk.hpp new file mode 100644 index 0000000000..771509594c --- /dev/null +++ b/include/simgrid/s4u/Disk.hpp @@ -0,0 +1,78 @@ +/* Copyright (c) 2019. The SimGrid Team. All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#ifndef INCLUDE_SIMGRID_S4U_DISK_HPP_ +#define INCLUDE_SIMGRID_S4U_DISK_HPP_ + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace simgrid { +namespace s4u { + +/** Disk represent the disk resources associated to a host + * + * By default, SimGrid does not keep track of the actual data being written but + * only computes the time taken by the corresponding data movement. + */ + +class XBT_PUBLIC Disk : public xbt::Extendable { + friend Engine; + friend Io; + friend kernel::resource::DiskImpl; + +public: + explicit Disk(const std::string& name, kernel::resource::DiskImpl* pimpl) : pimpl_(pimpl), name_(name) {} + +protected: + virtual ~Disk() = default; + +public: + /** @brief Callback signal fired when a new Storage is created */ + static xbt::signal on_creation; + /** @brief Callback signal fired when a Storage is destroyed */ + static xbt::signal on_destruction; + /** @brief Callback signal fired when a Storage's state changes */ + static xbt::signal on_state_change; + + /** @brief Retrieves the name of that storage as a C++ string */ + std::string const& get_name() const { return name_; } + /** @brief Retrieves the name of that storage as a C string */ + const char* get_cname() const { return name_.c_str(); } + + const std::unordered_map* get_properties() const; + const char* get_property(const std::string& key) const; + void set_property(const std::string&, const std::string& value); + Host* get_host(); + + void set_data(void* data) { userdata_ = data; } + void* get_data() { return userdata_; } + + IoPtr io_init(sg_size_t size, s4u::Io::OpType type); + + IoPtr read_async(sg_size_t size); + sg_size_t read(sg_size_t size); + + IoPtr write_async(sg_size_t size); + sg_size_t write(sg_size_t size); + kernel::resource::DiskImpl* get_impl() const { return pimpl_; } + +private: + kernel::resource::DiskImpl* const pimpl_; + std::string name_; + void* userdata_ = nullptr; +}; + +} // namespace s4u +} // namespace simgrid + +#endif /* INCLUDE_SIMGRID_S4U_DISK_HPP_ */ diff --git a/include/simgrid/s4u/Engine.hpp b/include/simgrid/s4u/Engine.hpp index 242a9799ce..ea53d1c8e4 100644 --- a/include/simgrid/s4u/Engine.hpp +++ b/include/simgrid/s4u/Engine.hpp @@ -87,6 +87,7 @@ protected: #ifndef DOXYGEN friend Host; friend Link; + friend Disk; friend Storage; friend kernel::routing::NetPoint; friend kernel::routing::NetZoneImpl; diff --git a/include/simgrid/s4u/Host.hpp b/include/simgrid/s4u/Host.hpp index c136667a63..45038bc156 100644 --- a/include/simgrid/s4u/Host.hpp +++ b/include/simgrid/s4u/Host.hpp @@ -115,6 +115,10 @@ public: void set_pstate(int pstate_index); int get_pstate() const; + std::vector get_disks() const; + void add_disk(Disk* disk); + void remove_disk(std::string disk_name); + std::vector get_attached_storages() const; /** Get an associative list [mount point]->[Storage] of all local mount points. diff --git a/include/simgrid/s4u/Io.hpp b/include/simgrid/s4u/Io.hpp index d486b342f1..73593abf41 100644 --- a/include/simgrid/s4u/Io.hpp +++ b/include/simgrid/s4u/Io.hpp @@ -26,17 +26,20 @@ public: private: Storage* storage_ = nullptr; + Disk* disk_ = nullptr; sg_size_t size_ = 0; OpType type_ = OpType::READ; std::string name_ = ""; std::atomic_int_fast32_t refcount_{0}; explicit Io(sg_storage_t storage, sg_size_t size, OpType type); + explicit Io(sg_disk_t disk, sg_size_t size, OpType type); public: #ifndef DOXYGEN friend XBT_PUBLIC void intrusive_ptr_release(simgrid::s4u::Io* i); friend XBT_PUBLIC void intrusive_ptr_add_ref(simgrid::s4u::Io* i); + friend Disk; // Factory of IOs friend Storage; // Factory of IOs #endif diff --git a/src/kernel/EngineImpl.cpp b/src/kernel/EngineImpl.cpp index 3a96a578de..4d98ab4ed8 100644 --- a/src/kernel/EngineImpl.cpp +++ b/src/kernel/EngineImpl.cpp @@ -7,6 +7,7 @@ #include "simgrid/kernel/routing/NetPoint.hpp" #include "simgrid/kernel/routing/NetZoneImpl.hpp" #include "simgrid/s4u/Host.hpp" +#include "src/kernel/resource/DiskImpl.hpp" #include "src/surf/StorageImpl.hpp" #include "src/surf/network_interface.hpp" diff --git a/src/kernel/activity/IoImpl.cpp b/src/kernel/activity/IoImpl.cpp index 6022c2d9bb..63f5381293 100644 --- a/src/kernel/activity/IoImpl.cpp +++ b/src/kernel/activity/IoImpl.cpp @@ -7,6 +7,7 @@ #include "simgrid/Exception.hpp" #include "simgrid/kernel/resource/Action.hpp" #include "simgrid/s4u/Host.hpp" +#include "src/kernel/resource/DiskImpl.hpp" #include "src/mc/mc_replay.hpp" #include "src/simix/smx_private.hpp" #include "src/surf/StorageImpl.hpp" @@ -44,6 +45,12 @@ IoImpl& IoImpl::set_size(sg_size_t size) return *this; } +IoImpl& IoImpl::set_disk(resource::DiskImpl* disk) +{ + disk_ = disk; + return *this; +} + IoImpl& IoImpl::set_storage(resource::StorageImpl* storage) { storage_ = storage; @@ -53,7 +60,10 @@ IoImpl& IoImpl::set_storage(resource::StorageImpl* storage) IoImpl* IoImpl::start() { state_ = SIMIX_RUNNING; - surf_action_ = storage_->io_start(size_, type_); + if (storage_) + surf_action_ = storage_->io_start(size_, type_); + else + surf_action_ = disk_->io_start(size_, type_); surf_action_->set_activity(this); XBT_DEBUG("Create IO synchro %p %s", this, get_cname()); @@ -66,7 +76,7 @@ void IoImpl::post() { performed_ioops_ = surf_action_->get_cost(); if (surf_action_->get_state() == resource::Action::State::FAILED) { - if (storage_ && not storage_->is_on()) + if ((storage_ && not storage_->is_on()) || (disk_ && not disk_->is_on())) state_ = SIMIX_FAILED; else state_ = SIMIX_CANCELED; diff --git a/src/kernel/activity/IoImpl.hpp b/src/kernel/activity/IoImpl.hpp index 2ab4ff700d..8d8f04c8c6 100644 --- a/src/kernel/activity/IoImpl.hpp +++ b/src/kernel/activity/IoImpl.hpp @@ -16,6 +16,7 @@ namespace activity { class XBT_PUBLIC IoImpl : public ActivityImpl_T { resource::StorageImpl* storage_ = nullptr; + resource::DiskImpl* disk_ = nullptr; sg_size_t size_ = 0; s4u::Io::OpType type_ = s4u::Io::OpType::READ; sg_size_t performed_ioops_ = 0; @@ -24,6 +25,7 @@ public: IoImpl& set_size(sg_size_t size); IoImpl& set_type(s4u::Io::OpType type); IoImpl& set_storage(resource::StorageImpl* storage); + IoImpl& set_disk(resource::DiskImpl* disk); sg_size_t get_performed_ioops() { return performed_ioops_; } diff --git a/src/kernel/resource/DiskImpl.cpp b/src/kernel/resource/DiskImpl.cpp new file mode 100644 index 0000000000..f2877f2684 --- /dev/null +++ b/src/kernel/resource/DiskImpl.cpp @@ -0,0 +1,105 @@ +/* Copyright (c) 2019. The SimGrid Team. All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#include "DiskImpl.hpp" + +#include "simgrid/s4u/Engine.hpp" +#include "src/kernel/EngineImpl.hpp" +#include "src/kernel/lmm/maxmin.hpp" + +XBT_LOG_NEW_DEFAULT_SUBCATEGORY(disk_kernel, surf, "Logging specific to the disk kernel resource"); + +simgrid::kernel::resource::DiskModel* surf_disk_model = nullptr; + +namespace simgrid { +namespace kernel { +namespace resource { + +/********* + * Model * + *********/ + +DiskModel::DiskModel() : Model(Model::UpdateAlgo::FULL) +{ + set_maxmin_system(new simgrid::kernel::lmm::System(true /* selective update */)); +} + +DiskModel::~DiskModel() +{ + surf_disk_model = nullptr; +} + +/************ + * Resource * + ************/ + +DiskImpl::DiskImpl(kernel::resource::Model* model, const std::string& name, kernel::lmm::System* maxminSystem, + double read_bw, double write_bw) + : Resource(model, name, maxminSystem->constraint_new(this, std::max(read_bw, write_bw))), piface_(name, this) +{ + DiskImpl::turn_on(); + XBT_DEBUG("Create resource with read_bw '%f' write_bw '%f'", read_bw, write_bw); + constraint_read_ = maxminSystem->constraint_new(this, read_bw); + constraint_write_ = maxminSystem->constraint_new(this, write_bw); +} + +DiskImpl::~DiskImpl() +{ + xbt_assert(currently_destroying_, "Don't delete Disks directly. Call destroy() instead."); +} + +/** @brief Fire the required callbacks and destroy the object + * + * Don't delete directly a Disk, call d->destroy() instead. + */ +void DiskImpl::destroy() +{ + if (not currently_destroying_) { + currently_destroying_ = true; + s4u::Disk::on_destruction(this->piface_); + delete this; + } +} + +bool DiskImpl::is_used() +{ + THROW_UNIMPLEMENTED; +} + +void DiskImpl::apply_event(kernel::profile::Event* /*event*/, double /*value*/) +{ + THROW_UNIMPLEMENTED; +} + +void DiskImpl::turn_on() +{ + if (not is_on()) { + Resource::turn_on(); + s4u::Disk::on_state_change(this->piface_); + } +} +void DiskImpl::turn_off() +{ + if (is_on()) { + Resource::turn_off(); + s4u::Disk::on_state_change(this->piface_); + } +} + +xbt::signal + DiskAction::on_state_change; + +/********** + * Action * + **********/ +void DiskAction::set_state(Action::State state) +{ + Action::State old = get_state(); + Action::set_state(state); + on_state_change(*this, old, state); +} +} // namespace resource +} // namespace kernel +} // namespace simgrid diff --git a/src/kernel/resource/DiskImpl.hpp b/src/kernel/resource/DiskImpl.hpp new file mode 100644 index 0000000000..80ae7a66e0 --- /dev/null +++ b/src/kernel/resource/DiskImpl.hpp @@ -0,0 +1,118 @@ +/* Copyright (c) 2019. The SimGrid Team. All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#include "simgrid/kernel/resource/Action.hpp" +#include "simgrid/kernel/resource/Model.hpp" +#include "simgrid/kernel/resource/Resource.hpp" +#include "simgrid/s4u/Disk.hpp" +#include "simgrid/s4u/Io.hpp" +#include "src/surf/PropertyHolder.hpp" +#include "src/surf/surf_interface.hpp" + +#include + +#ifndef DISK_INTERFACE_HPP_ +#define DISK_INTERFACE_HPP_ + +/********* + * Model * + *********/ + +XBT_PUBLIC_DATA simgrid::kernel::resource::DiskModel* surf_disk_model; + +namespace simgrid { +namespace kernel { +namespace resource { +/*********** + * Classes * + ***********/ + +class DiskAction; + +/********* + * Model * + *********/ +class DiskModel : public kernel::resource::Model { +public: + DiskModel(); + DiskModel(const DiskModel&) = delete; + DiskModel& operator=(const DiskModel&) = delete; + ~DiskModel(); + + virtual DiskImpl* createDisk(const std::string& id, double read_bw, double write_bw) = 0; +}; + +/************ + * Resource * + ************/ +class DiskImpl : public Resource, public surf::PropertyHolder { + bool currently_destroying_ = false; + s4u::Host* host_ = nullptr; + +public: + DiskImpl(Model* model, const std::string& name, kernel::lmm::System* maxmin_system, double read_bw, double bwrite_bw); + DiskImpl(const DiskImpl&) = delete; + DiskImpl& operator=(const DiskImpl&) = delete; + + ~DiskImpl() override; + + /** @brief Public interface */ + s4u::Disk piface_; + s4u::Disk* get_iface() { return &piface_; } + /** @brief Check if the Storage is used (if an action currently uses its resources) */ + bool is_used() override; + + void apply_event(profile::Event* event, double value) override; + + void turn_on() override; + void turn_off() override; + + s4u::Host* get_host() { return host_; } + void set_host(s4u::Host* host) { host_ = host; } + + void destroy(); // Must be called instead of the destructor + virtual DiskAction* io_start(sg_size_t size, s4u::Io::OpType type) = 0; + virtual DiskAction* read(sg_size_t size) = 0; + virtual DiskAction* write(sg_size_t size) = 0; + + lmm::Constraint* constraint_write_; /* Constraint for maximum write bandwidth*/ + lmm::Constraint* constraint_read_; /* Constraint for maximum write bandwidth*/ +}; + +/********** + * Action * + **********/ + +class DiskAction : public Action { +public: + static xbt::signal on_state_change; + + DiskAction(Model* model, double cost, bool failed, DiskImpl* disk, s4u::Io::OpType type) + : Action(model, cost, failed), type_(type), disk_(disk){}; + + /** + * @brief diskAction constructor + * + * @param model The StorageModel associated to this DiskAction + * @param cost The cost of this DiskAction in bytes + * @param failed [description] + * @param var The lmm variable associated to this DiskAction if it is part of a LMM component + * @param storage The Storage associated to this DiskAction + * @param type [description] + */ + DiskAction(kernel::resource::Model* model, double cost, bool failed, kernel::lmm::Variable* var, DiskImpl* disk, + s4u::Io::OpType type) + : Action(model, cost, failed, var), type_(type), disk_(disk){}; + + void set_state(simgrid::kernel::resource::Action::State state) override; + + s4u::Io::OpType type_; + DiskImpl* disk_; +}; + +} // namespace resource +} // namespace kernel +} // namespace simgrid +#endif /* DISK_INTERFACE_HPP_ */ diff --git a/src/plugins/file_system/s4u_FileSystem.cpp b/src/plugins/file_system/s4u_FileSystem.cpp index b32fd5137e..f23b028b52 100644 --- a/src/plugins/file_system/s4u_FileSystem.cpp +++ b/src/plugins/file_system/s4u_FileSystem.cpp @@ -5,7 +5,9 @@ #include "simgrid/plugins/file_system.h" #include "simgrid/s4u/Actor.hpp" +#include "simgrid/s4u/Engine.hpp" #include "src/surf/HostImpl.hpp" +#include "src/surf/xml/platf_private.hpp" #include "xbt/config.hpp" #include @@ -20,6 +22,7 @@ int sg_storage_max_file_descriptors = 1024; namespace simgrid { namespace s4u { +simgrid::xbt::Extension FileSystemDiskExt::EXTENSION_ID; simgrid::xbt::Extension FileSystemStorageExt::EXTENSION_ID; simgrid::xbt::Extension FileDescriptorHostExt::EXTENSION_ID; @@ -28,27 +31,56 @@ File::File(const std::string& fullpath, void* userdata) : File(fullpath, Host::c File::File(const std::string& fullpath, sg_host_t host, void* userdata) : fullpath_(fullpath), userdata_(userdata) { // this cannot fail because we get a xbt_die if the mountpoint does not exist - Storage* st = nullptr; - size_t longest_prefix_length = 0; - XBT_DEBUG("Search for storage name for '%s' on '%s'", fullpath_.c_str(), host->get_cname()); - - for (auto const& mnt : host->get_mounted_storages()) { - XBT_DEBUG("See '%s'", mnt.first.c_str()); - mount_point_ = fullpath_.substr(0, mnt.first.length()); - - if (mount_point_ == mnt.first && mnt.first.length() > longest_prefix_length) { - /* The current mount name is found in the full path and is bigger than the previous*/ - longest_prefix_length = mnt.first.length(); - st = mnt.second; + if (not host->get_mounted_storages().empty()) { + Storage* st = nullptr; + size_t longest_prefix_length = 0; + XBT_DEBUG("Search for storage name for '%s' on '%s'", fullpath_.c_str(), host->get_cname()); + + for (auto const& mnt : host->get_mounted_storages()) { + XBT_DEBUG("See '%s'", mnt.first.c_str()); + mount_point_ = fullpath_.substr(0, mnt.first.length()); + + if (mount_point_ == mnt.first && mnt.first.length() > longest_prefix_length) { + /* The current mount name is found in the full path and is bigger than the previous*/ + longest_prefix_length = mnt.first.length(); + st = mnt.second; + } } - } - if (longest_prefix_length > 0) { /* Mount point found, split fullpath_ into mount_name and path+filename*/ - mount_point_ = fullpath_.substr(0, longest_prefix_length); - path_ = fullpath_.substr(longest_prefix_length, fullpath_.length()); - } else - xbt_die("Can't find mount point for '%s' on '%s'", fullpath_.c_str(), host->get_cname()); + if (longest_prefix_length > 0) { /* Mount point found, split fullpath_ into mount_name and path+filename*/ + mount_point_ = fullpath_.substr(0, longest_prefix_length); + path_ = fullpath_.substr(longest_prefix_length, fullpath_.length()); + } else + xbt_die("Can't find mount point for '%s' on '%s'", fullpath_.c_str(), host->get_cname()); - local_storage_ = st; + local_storage_ = st; + } + if (not host->get_disks().empty()) { + Disk* d = nullptr; + size_t longest_prefix_length = 0; + for (auto const& disk : host->get_disks()) { + std::string current_mount; + if (disk->get_host() != host) + current_mount = disk->extension()->get_mount_point(disk->get_host()); + else + current_mount = disk->extension()->get_mount_point(); + mount_point_ = fullpath_.substr(0, current_mount.length()); + if (mount_point_ == current_mount && current_mount.length() > longest_prefix_length) { + /* The current mount name is found in the full path and is bigger than the previous*/ + longest_prefix_length = current_mount.length(); + d = disk; + } + if (longest_prefix_length > 0) { /* Mount point found, split fullpath_ into mount_name and path+filename*/ + mount_point_ = fullpath_.substr(0, longest_prefix_length); + if (mount_point_ == std::string("/")) + path_ = fullpath_; + else + path_ = fullpath_.substr(longest_prefix_length, fullpath_.length()); + XBT_DEBUG("%s + %s", mount_point_.c_str(), path_.c_str()); + } else + xbt_die("Can't find mount point for '%s' on '%s'", fullpath_.c_str(), host->get_cname()); + } + local_disk_ = d; + } // assign a file descriptor id to the newly opened File FileDescriptorHostExt* ext = host->extension(); @@ -61,7 +93,13 @@ File::File(const std::string& fullpath, sg_host_t host, void* userdata) : fullpa ext->file_descriptor_table->pop_back(); XBT_DEBUG("\tOpen file '%s'", path_.c_str()); - std::map* content = local_storage_->extension()->get_content(); + std::map* content; + if (local_storage_) + content = local_storage_->extension()->get_content(); + + if (local_disk_) + content = local_disk_->extension()->get_content(); + // if file does not exist create an empty file auto sz = content->find(path_); if (sz != content->end()) { @@ -80,27 +118,50 @@ File::~File() void File::dump() { - XBT_INFO("File Descriptor information:\n" - "\t\tFull path: '%s'\n" - "\t\tSize: %llu\n" - "\t\tMount point: '%s'\n" - "\t\tStorage Id: '%s'\n" - "\t\tStorage Type: '%s'\n" - "\t\tFile Descriptor Id: %d", - get_path(), size_, mount_point_.c_str(), local_storage_->get_cname(), local_storage_->get_type(), desc_id); + if (local_storage_) + XBT_INFO("File Descriptor information:\n" + "\t\tFull path: '%s'\n" + "\t\tSize: %llu\n" + "\t\tMount point: '%s'\n" + "\t\tStorage Id: '%s'\n" + "\t\tStorage Type: '%s'\n" + "\t\tFile Descriptor Id: %d", + get_path(), size_, mount_point_.c_str(), local_storage_->get_cname(), local_storage_->get_type(), desc_id); + if (local_disk_) + XBT_INFO("File Descriptor information:\n" + "\t\tFull path: '%s'\n" + "\t\tSize: %llu\n" + "\t\tMount point: '%s'\n" + "\t\tDisk Id: '%s'\n" + "\t\tHost Id: '%s'\n" + "\t\tFile Descriptor Id: %d", + get_path(), size_, mount_point_.c_str(), local_disk_->get_cname(), local_disk_->get_host()->get_cname(), + desc_id); } sg_size_t File::read(sg_size_t size) { if (size_ == 0) /* Nothing to read, return */ return 0; + sg_size_t read_size = 0; + Host* host = nullptr; + if (local_storage_) { + /* Find the host where the file is physically located and read it */ + host = local_storage_->get_host(); + XBT_DEBUG("READ %s on disk '%s'", get_path(), local_storage_->get_cname()); + // if the current position is close to the end of the file, we may not be able to read the requested size + read_size = local_storage_->read(std::min(size, size_ - current_position_)); + current_position_ += read_size; + } - /* Find the host where the file is physically located and read it */ - Host* host = local_storage_->get_host(); - XBT_DEBUG("READ %s on disk '%s'", get_path(), local_storage_->get_cname()); - // if the current position is close to the end of the file, we may not be able to read the requested size - sg_size_t read_size = local_storage_->read(std::min(size, size_ - current_position_)); - current_position_ += read_size; + if (local_disk_) { + /* Find the host where the file is physically located and read it */ + host = local_disk_->get_host(); + XBT_DEBUG("READ %s on disk '%s'", get_path(), local_disk_->get_cname()); + // if the current position is close to the end of the file, we may not be able to read the requested size + read_size = local_disk_->read(std::min(size, size_ - current_position_)); + current_position_ += read_size; + } if (host->get_name() != Host::current()->get_name() && read_size > 0) { /* the file is hosted on a remote host, initiate a communication between src and dest hosts for data transfer */ @@ -120,9 +181,14 @@ sg_size_t File::write(sg_size_t size, int write_inside) { if (size == 0) /* Nothing to write, return */ return 0; + sg_size_t write_size = 0; + Host* host = nullptr; /* Find the host where the file is physically located (remote or local)*/ - Host* host = local_storage_->get_host(); + if (local_storage_) + host = local_storage_->get_host(); + if (local_disk_) + host = local_disk_->get_host(); if (host->get_name() != Host::current()->get_name()) { /* the file is hosted on a remote host, initiate a communication between src and dest hosts for data transfer */ @@ -130,29 +196,55 @@ sg_size_t File::write(sg_size_t size, int write_inside) Host::current()->send_to(host, size); } - XBT_DEBUG("WRITE %s on disk '%s'. size '%llu/%llu' '%llu:%llu'", get_path(), local_storage_->get_cname(), size, size_, sg_storage_get_size_used(local_storage_), sg_storage_get_size(local_storage_)); - // If the storage is full before even starting to write - if (sg_storage_get_size_used(local_storage_) >= sg_storage_get_size(local_storage_)) - return 0; - sg_size_t write_size=0; - if(write_inside==0){ - /* Substract the part of the file that might disappear from the used sized on the storage element */ - local_storage_->extension()->decr_used_size(size_ - current_position_); - write_size = local_storage_->write(size); - local_storage_->extension()->incr_used_size(write_size); - current_position_ += write_size; - size_ = current_position_; - }else { - write_size = local_storage_->write(size); - current_position_ += write_size; - if(current_position_>size_) + if (local_storage_) { + XBT_DEBUG("WRITE %s on disk '%s'. size '%llu/%llu' '%llu:%llu'", get_path(), local_storage_->get_cname(), size, + size_, sg_storage_get_size_used(local_storage_), sg_storage_get_size(local_storage_)); + // If the storage is full before even starting to write + if (sg_storage_get_size_used(local_storage_) >= sg_storage_get_size(local_storage_)) + return 0; + if (write_inside == 0) { + /* Substract the part of the file that might disappear from the used sized on the storage element */ + local_storage_->extension()->decr_used_size(size_ - current_position_); + write_size = local_storage_->write(size); + local_storage_->extension()->incr_used_size(write_size); + current_position_ += write_size; size_ = current_position_; + } else { + write_size = local_storage_->write(size); + current_position_ += write_size; + if (current_position_ > size_) + size_ = current_position_; + } + std::map* content = local_storage_->extension()->get_content(); + + content->erase(path_); + content->insert({path_, size_}); } - std::map* content = local_storage_->extension()->get_content(); - content->erase(path_); - content->insert({path_, size_}); + if (local_disk_) { + XBT_DEBUG("WRITE %s on disk '%s'. size '%llu/%llu' '%llu:%llu'", get_path(), local_disk_->get_cname(), size, size_, + sg_disk_get_size_used(local_disk_), sg_disk_get_size(local_disk_)); + // If the storage is full before even starting to write + if (sg_disk_get_size_used(local_disk_) >= sg_disk_get_size(local_disk_)) + return 0; + if (write_inside == 0) { + /* Substract the part of the file that might disappear from the used sized on the storage element */ + local_disk_->extension()->decr_used_size(size_ - current_position_); + write_size = local_disk_->write(size); + local_disk_->extension()->incr_used_size(write_size); + current_position_ += write_size; + size_ = current_position_; + } else { + write_size = local_disk_->write(size); + current_position_ += write_size; + if (current_position_ > size_) + size_ = current_position_; + } + std::map* content = local_disk_->extension()->get_content(); + content->erase(path_); + content->insert({path_, size_}); + } return write_size; } @@ -192,7 +284,11 @@ void File::move(const std::string& fullpath) { /* Check if the new full path is on the same mount point */ if (fullpath.compare(0, mount_point_.length(), mount_point_) == 0) { - std::map* content = local_storage_->extension()->get_content(); + std::map* content; + if (local_storage_) + content = local_storage_->extension()->get_content(); + if (local_disk_) + content = local_disk_->extension()->get_content(); auto sz = content->find(path_); if (sz != content->end()) { // src file exists sg_size_t new_size = sz->second; @@ -211,14 +307,28 @@ void File::move(const std::string& fullpath) int File::unlink() { /* Check if the file is on local storage */ - std::map* content = local_storage_->extension()->get_content(); + std::map* content; + const char* name = nullptr; + if (local_storage_) { + content = local_storage_->extension()->get_content(); + name = local_storage_->get_cname(); + } + if (local_disk_) { + content = local_disk_->extension()->get_content(); + name = local_disk_->get_cname(); + } if (content->find(path_) == content->end()) { - XBT_WARN("File %s is not on disk %s. Impossible to unlink", path_.c_str(), local_storage_->get_cname()); + XBT_WARN("File %s is not on disk %s. Impossible to unlink", path_.c_str(), name); return -1; } else { - XBT_DEBUG("UNLINK %s on disk '%s'", path_.c_str(), local_storage_->get_cname()); - local_storage_->extension()->decr_used_size(size_); + XBT_DEBUG("UNLINK %s on disk '%s'", path_.c_str(), name); + + if (local_storage_) + local_storage_->extension()->decr_used_size(size_); + + if (local_disk_) + local_disk_->extension()->decr_used_size(size_); // Remove the file from storage content->erase(fullpath_); @@ -230,45 +340,84 @@ int File::unlink() int File::remote_copy(sg_host_t host, const char* fullpath) { /* Find the host where the file is physically located and read it */ - Storage* storage_src = local_storage_; - Host* src_host = storage_src->get_host(); + Host* src_host; + if (local_storage_) { + src_host = local_storage_->get_host(); + XBT_DEBUG("READ %s on disk '%s'", get_path(), local_storage_->get_cname()); + } + + if (local_disk_) { + src_host = local_disk_->get_host(); + XBT_DEBUG("READ %s on disk '%s'", get_path(), local_disk_->get_cname()); + } + seek(0, SEEK_SET); - XBT_DEBUG("READ %s on disk '%s'", get_path(), local_storage_->get_cname()); // if the current position is close to the end of the file, we may not be able to read the requested size - sg_size_t read_size = local_storage_->read(size_); + sg_size_t read_size = 0; + if (local_storage_) + read_size = local_storage_->read(size_); + if (local_disk_) + read_size = local_disk_->read(size_); + current_position_ += read_size; - /* Find the host that owns the storage where the file has to be copied */ - Storage* storage_dest = nullptr; - Host* dst_host; - size_t longest_prefix_length = 0; - - for (auto const& elm : host->get_mounted_storages()) { - std::string mount_point = std::string(fullpath).substr(0, elm.first.size()); - if (mount_point == elm.first && elm.first.length() > longest_prefix_length) { - /* The current mount name is found in the full path and is bigger than the previous*/ - longest_prefix_length = elm.first.length(); - storage_dest = elm.second; + Host* dst_host = host; + if (local_storage_) { + /* Find the host that owns the storage where the file has to be copied */ + Storage* storage_dest = nullptr; + size_t longest_prefix_length = 0; + + for (auto const& elm : host->get_mounted_storages()) { + std::string mount_point = std::string(fullpath).substr(0, elm.first.size()); + if (mount_point == elm.first && elm.first.length() > longest_prefix_length) { + /* The current mount name is found in the full path and is bigger than the previous*/ + longest_prefix_length = elm.first.length(); + storage_dest = elm.second; + } + } + + if (storage_dest != nullptr) { + /* Mount point found, retrieve the host the storage is attached to */ + dst_host = storage_dest->get_host(); + } else { + XBT_WARN("Can't find mount point for '%s' on destination host '%s'", fullpath, host->get_cname()); + return -1; } } - if (storage_dest != nullptr) { - /* Mount point found, retrieve the host the storage is attached to */ - dst_host = storage_dest->get_host(); - } else { - XBT_WARN("Can't find mount point for '%s' on destination host '%s'", fullpath, host->get_cname()); - return -1; + if (local_disk_) { + size_t longest_prefix_length = 0; + Disk* dst_disk = nullptr; + + for (auto const& disk : host->get_disks()) { + std::string current_mount = disk->extension()->get_mount_point(); + std::string mount_point = std::string(fullpath).substr(0, current_mount.length()); + if (mount_point == current_mount && current_mount.length() > longest_prefix_length) { + /* The current mount name is found in the full path and is bigger than the previous*/ + longest_prefix_length = current_mount.length(); + dst_disk = disk; + } + } + + if (dst_disk == nullptr) { + XBT_WARN("Can't find mount point for '%s' on destination host '%s'", fullpath, host->get_cname()); + return -1; + } } XBT_DEBUG("Initiate data transfer of %llu bytes between %s and %s.", read_size, src_host->get_cname(), - storage_dest->get_host()->get_cname()); + dst_host->get_cname()); src_host->send_to(dst_host, read_size); /* Create file on remote host, write it and close it */ File* fd = new File(fullpath, dst_host, nullptr); - sg_size_t write_size = fd->local_storage_->write(read_size); - fd->local_storage_->extension()->incr_used_size(write_size); - (*(fd->local_storage_->extension()->get_content()))[path_] = size_; + if (local_storage_) { + sg_size_t write_size = fd->local_storage_->write(read_size); + fd->local_storage_->extension()->incr_used_size(write_size); + (*(fd->local_storage_->extension()->get_content()))[path_] = size_; + } + if (local_disk_) + fd->write(read_size); delete fd; return 0; } @@ -280,12 +429,56 @@ int File::remote_move(sg_host_t host, const char* fullpath) return res; } +FileSystemDiskExt::FileSystemDiskExt(simgrid::s4u::Disk* ptr) +{ + const char* size_str = ptr->get_property("size"); + if (size_str) + size_ = surf_parse_get_size(size_str, "disk size", ptr->get_name()); + + const char* current_mount_str = ptr->get_property("mount"); + if (current_mount_str) + mount_point_ = std::string(current_mount_str); + else + mount_point_ = std::string("/"); + + const char* content_str = ptr->get_property("content"); + if (content_str) + content_.reset(parse_content(content_str)); +} + FileSystemStorageExt::FileSystemStorageExt(simgrid::s4u::Storage* ptr) { content_.reset(parse_content(ptr->get_impl()->content_name_)); size_ = ptr->get_impl()->size_; } +std::map* FileSystemDiskExt::parse_content(const std::string& filename) +{ + if (filename.empty()) + return nullptr; + + std::map* parse_content = new std::map(); + + std::ifstream* fs = surf_ifsopen(filename); + + std::string line; + std::vector tokens; + do { + std::getline(*fs, line); + boost::trim(line); + if (line.length() > 0) { + boost::split(tokens, line, boost::is_any_of(" \t"), boost::token_compress_on); + xbt_assert(tokens.size() == 2, "Parse error in %s: %s", filename.c_str(), line.c_str()); + sg_size_t size = std::stoull(tokens.at(1)); + + used_size_ += size; + parse_content->insert({tokens.front(), size}); + } + } while (not fs->eof()); + delete fs; + return parse_content; +} + std::map* FileSystemStorageExt::parse_content(const std::string& filename) { if (filename.empty()) @@ -315,9 +508,14 @@ std::map* FileSystemStorageExt::parse_content(const std: } } -using simgrid::s4u::FileSystemStorageExt; using simgrid::s4u::FileDescriptorHostExt; +using simgrid::s4u::FileSystemDiskExt; +using simgrid::s4u::FileSystemStorageExt; +static void on_disk_creation(simgrid::s4u::Disk& d) +{ + d.extension_set(new FileSystemDiskExt(&d)); +} static void on_storage_creation(simgrid::s4u::Storage& st) { st.extension_set(new FileSystemStorageExt(&st)); @@ -328,6 +526,51 @@ static void on_host_creation(simgrid::s4u::Host& host) host.extension_set(new FileDescriptorHostExt()); } +static void on_platform_created() +{ + for (auto const& host : simgrid::s4u::Engine::get_instance()->get_all_hosts()) { + const char* remote_disk_str = host->get_property("remote_disk"); + if (remote_disk_str) { + std::vector tokens; + boost::split(tokens, remote_disk_str, boost::is_any_of(":")); + std::string mount_point = tokens[0]; + simgrid::s4u::Host* remote_host = simgrid::s4u::Host::by_name_or_null(tokens[2]); + xbt_assert(remote_host, "You're trying to access a host that does not exist. Please check your platform file"); + + simgrid::s4u::Disk* disk = nullptr; + for (auto const& d : remote_host->get_disks()) + if (d->get_name() == tokens[1]) { + disk = d; + break; + } + + xbt_assert(disk, "You're trying to mount a disk that does not exist. Please check your platform file"); + disk->extension()->add_remote_mount(remote_host, mount_point); + host->add_disk(disk); + + XBT_DEBUG("Host '%s' wants to mount a remote disk: %s of %s mounted on %s", host->get_cname(), disk->get_cname(), + remote_host->get_cname(), mount_point.c_str()); + XBT_DEBUG("Host '%s' now has %lu disks", host->get_cname(), host->get_disks().size()); + } + } +} + +static void on_simulation_end() +{ + XBT_DEBUG("Simulation is over, time to unregister remote disks if any"); + for (auto const& host : simgrid::s4u::Engine::get_instance()->get_all_hosts()) { + const char* remote_disk_str = host->get_property("remote_disk"); + if (remote_disk_str) { + std::vector tokens; + boost::split(tokens, remote_disk_str, boost::is_any_of(":")); + XBT_DEBUG("Host '%s' wants to unmount a remote disk: %s of %s mounted on %s", host->get_cname(), + tokens[1].c_str(), tokens[2].c_str(), tokens[0].c_str()); + host->remove_disk(tokens[1]); + XBT_DEBUG("Host '%s' now has %lu disks", host->get_cname(), host->get_disks().size()); + } + } +} + /* **************************** Public interface *************************** */ void sg_storage_file_system_init() { @@ -340,10 +583,17 @@ void sg_storage_file_system_init() simgrid::s4u::Storage::on_creation.connect(&on_storage_creation); } + if (not FileSystemDiskExt::EXTENSION_ID.valid()) { + FileSystemDiskExt::EXTENSION_ID = simgrid::s4u::Disk::extension_create(); + simgrid::s4u::Disk::on_creation.connect(&on_disk_creation); + } + if (not FileDescriptorHostExt::EXTENSION_ID.valid()) { FileDescriptorHostExt::EXTENSION_ID = simgrid::s4u::Host::extension_create(); simgrid::s4u::Host::on_creation.connect(&on_host_creation); } + simgrid::s4u::on_platform_created.connect(&on_platform_created); + simgrid::s4u::on_simulation_end.connect(&on_simulation_end); } sg_file_t sg_file_open(const char* fullpath, void* data) @@ -447,6 +697,26 @@ int sg_file_rmove(sg_file_t file, sg_host_t host, const char* fullpath) return file->remote_move(host, fullpath); } +sg_size_t sg_disk_get_size_free(sg_disk_t d) +{ + return d->extension()->get_size() - d->extension()->get_used_size(); +} + +sg_size_t sg_disk_get_size_used(sg_disk_t d) +{ + return d->extension()->get_used_size(); +} + +sg_size_t sg_disk_get_size(sg_disk_t d) +{ + return d->extension()->get_size(); +} + +const char* sg_disk_get_mount_point(sg_disk_t d) +{ + return d->extension()->get_mount_point(); +} + sg_size_t sg_storage_get_size_free(sg_storage_t st) { return st->extension()->get_size() - st->extension()->get_used_size(); diff --git a/src/s4u/s4u_Disk.cpp b/src/s4u/s4u_Disk.cpp new file mode 100644 index 0000000000..562e7423b2 --- /dev/null +++ b/src/s4u/s4u_Disk.cpp @@ -0,0 +1,70 @@ +/* Copyright (c) 2019. The SimGrid Team. All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#include "simgrid/s4u/Disk.hpp" +#include "simgrid/s4u/Engine.hpp" +#include "simgrid/s4u/Host.hpp" +#include "simgrid/s4u/Io.hpp" +#include "src/kernel/resource/DiskImpl.hpp" + +namespace simgrid { +namespace xbt { +template class Extendable; +} // namespace xbt + +namespace s4u { + +xbt::signal Disk::on_creation; +xbt::signal Disk::on_destruction; +xbt::signal Disk::on_state_change; + +Host* Disk::get_host() +{ + return pimpl_->get_host(); +} + +const std::unordered_map* Disk::get_properties() const +{ + return pimpl_->get_properties(); +} + +const char* Disk::get_property(const std::string& key) const +{ + return this->pimpl_->get_property(key); +} + +void Disk::set_property(const std::string& key, const std::string& value) +{ + kernel::actor::simcall([this, &key, &value] { this->pimpl_->set_property(key, value); }); +} + +IoPtr Disk::io_init(sg_size_t size, Io::OpType type) +{ + return IoPtr(new Io(this, size, type)); +} + +IoPtr Disk::read_async(sg_size_t size) +{ + return IoPtr(io_init(size, Io::OpType::READ))->start(); +} + +sg_size_t Disk::read(sg_size_t size) +{ + return IoPtr(io_init(size, Io::OpType::READ))->start()->wait()->get_performed_ioops(); +} + +IoPtr Disk::write_async(sg_size_t size) +{ + + return IoPtr(io_init(size, Io::OpType::WRITE)->start()); +} + +sg_size_t Disk::write(sg_size_t size) +{ + return IoPtr(io_init(size, Io::OpType::WRITE))->start()->wait()->get_performed_ioops(); +} + +} // namespace s4u +} // namespace simgrid diff --git a/src/s4u/s4u_Engine.cpp b/src/s4u/s4u_Engine.cpp index 5339a77f3e..1d12c11bdb 100644 --- a/src/s4u/s4u_Engine.cpp +++ b/src/s4u/s4u_Engine.cpp @@ -8,6 +8,7 @@ #include "mc/mc.h" #include "simgrid/kernel/routing/NetPoint.hpp" #include "simgrid/kernel/routing/NetZoneImpl.hpp" +#include "simgrid/s4u/Disk.hpp" #include "simgrid/s4u/Engine.hpp" #include "simgrid/s4u/Host.hpp" #include "simgrid/s4u/Mailbox.hpp" diff --git a/src/s4u/s4u_Host.cpp b/src/s4u/s4u_Host.cpp index 5e90311284..f86b99071f 100644 --- a/src/s4u/s4u_Host.cpp +++ b/src/s4u/s4u_Host.cpp @@ -282,6 +282,19 @@ int Host::get_pstate() const return this->pimpl_cpu->get_pstate(); } +std::vector Host::get_disks() const +{ + return kernel::actor::simcall([this] { return this->pimpl_->get_disks(); }); +} + +void Host::add_disk(Disk* disk) +{ + kernel::actor::simcall([this, disk] { this->pimpl_->add_disk(disk); }); +} +void Host::remove_disk(std::string disk_name) +{ + kernel::actor::simcall([this, disk_name] { this->pimpl_->remove_disk(disk_name); }); +} /** * @ingroup simix_storage_management * @brief Returns the list of storages attached to a host. diff --git a/src/s4u/s4u_Io.cpp b/src/s4u/s4u_Io.cpp index 924dfec086..4cf33bc8d4 100644 --- a/src/s4u/s4u_Io.cpp +++ b/src/s4u/s4u_Io.cpp @@ -3,6 +3,7 @@ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ +#include "simgrid/s4u/Disk.hpp" #include "simgrid/s4u/Io.hpp" #include "simgrid/s4u/Storage.hpp" #include "src/kernel/activity/IoImpl.hpp" @@ -13,6 +14,12 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(s4u_io, s4u_activity, "S4U asynchronous IOs"); namespace simgrid { namespace s4u { +Io::Io(sg_disk_t disk, sg_size_t size, OpType type) : disk_(disk), size_(size), type_(type) +{ + Activity::set_remaining(size_); + pimpl_ = kernel::activity::IoImplPtr(new kernel::activity::IoImpl()); +} + Io::Io(sg_storage_t storage, sg_size_t size, OpType type) : storage_(storage), size_(size), type_(type) { Activity::set_remaining(size_); @@ -22,12 +29,21 @@ Io::Io(sg_storage_t storage, sg_size_t size, OpType type) : storage_(storage), s Io* Io::start() { kernel::actor::simcall([this] { - (*boost::static_pointer_cast(pimpl_)) - .set_name(name_) - .set_storage(storage_->get_impl()) - .set_size(size_) - .set_type(type_) - .start(); + if (storage_) { + (*boost::static_pointer_cast(pimpl_)) + .set_name(name_) + .set_storage(storage_->get_impl()) + .set_size(size_) + .set_type(type_) + .start(); + } else { + (*boost::static_pointer_cast(pimpl_)) + .set_name(name_) + .set_disk(disk_->get_impl()) + .set_size(size_) + .set_type(type_) + .start(); + } }); state_ = State::STARTED; return this; diff --git a/src/simgrid/sg_config.cpp b/src/simgrid/sg_config.cpp index 1bf0f9f91c..2acb020ae2 100644 --- a/src/simgrid/sg_config.cpp +++ b/src/simgrid/sg_config.cpp @@ -171,8 +171,20 @@ static void _sg_cfg_cb__optimization_mode(const std::string& value) find_model_description(surf_optimization_mode_description, value); } +static void _sg_cfg_cb__disk_model(const std::string& value) +{ + xbt_assert(_sg_cfg_init_status < 2, "Cannot change the model after the initialization"); + + if (value == "help") { + model_help("disk", surf_disk_model_description); + exit(0); + } + + find_model_description(surf_disk_model_description, value); +} + /* callback of the cpu/model variable */ -static void _sg_cfg_cb__storage_mode(const std::string& value) +static void _sg_cfg_cb__storage_model(const std::string& value) { xbt_assert(_sg_cfg_init_status < 2, "Cannot change the model after the initialization"); @@ -243,7 +255,10 @@ void sg_config_init(int *argc, char **argv) declare_model_flag("cpu/model", "Cas01", &_sg_cfg_cb__cpu_model, surf_cpu_model_description, "model", "The model to use for the CPU"); - declare_model_flag("storage/model", "default", &_sg_cfg_cb__storage_mode, surf_storage_model_description, "model", + declare_model_flag("disk/model", "default", &_sg_cfg_cb__disk_model, surf_disk_model_description, "model", + "The model to use for the disk"); + + declare_model_flag("storage/model", "default", &_sg_cfg_cb__storage_model, surf_storage_model_description, "model", "The model to use for the storage"); declare_model_flag("network/model", "LV08", &_sg_cfg_cb__network_model, surf_network_model_description, "model", diff --git a/src/surf/HostImpl.cpp b/src/surf/HostImpl.cpp index e598461f5f..c037bf9022 100644 --- a/src/surf/HostImpl.cpp +++ b/src/surf/HostImpl.cpp @@ -92,6 +92,9 @@ HostImpl::~HostImpl() for (auto const& arg : actors_at_boot_) delete arg; actors_at_boot_.clear(); + + for (auto const& d : disks_) + d->destroy(); } /** Re-starts all the actors that are marked as restartable. @@ -147,6 +150,32 @@ size_t HostImpl::get_actor_count() { return process_list_.size(); } + +std::vector HostImpl::get_disks() +{ + std::vector disks; + for (auto const& d : disks_) + disks.push_back(&d->piface_); + return disks; +} + +void HostImpl::add_disk(s4u::Disk* disk) +{ + disks_.push_back(disk->get_impl()); +} + +void HostImpl::remove_disk(std::string disk_name) +{ + auto position = disks_.begin(); + for (auto const& d : disks_) { + if (d->get_name() == disk_name) { + disks_.erase(position); + break; + } + position++; + } +} + std::vector HostImpl::get_attached_storages() { std::vector storages; diff --git a/src/surf/HostImpl.hpp b/src/surf/HostImpl.hpp index 628569bf2d..791b13500f 100644 --- a/src/surf/HostImpl.hpp +++ b/src/surf/HostImpl.hpp @@ -7,6 +7,7 @@ #define SURF_HOST_INTERFACE_HPP_ #include "src/kernel/actor/ActorImpl.hpp" +#include "src/kernel/resource/DiskImpl.hpp" #include "src/surf/PropertyHolder.hpp" #include "src/surf/StorageImpl.hpp" #include "src/surf/cpu_interface.hpp" @@ -47,10 +48,16 @@ public: explicit HostImpl(s4u::Host* host); virtual ~HostImpl(); + std::vector get_disks(); + void add_disk(s4u::Disk* disk); + void remove_disk(std::string disk_name); + /** @brief Get the vector of storages (by names) attached to the Host */ virtual std::vector get_attached_storages(); std::map storage_; + std::vector disks_; + s4u::Host* piface_ = nullptr; void turn_on(); diff --git a/src/surf/disk_s19.cpp b/src/surf/disk_s19.cpp new file mode 100644 index 0000000000..4198e10035 --- /dev/null +++ b/src/surf/disk_s19.cpp @@ -0,0 +1,145 @@ +/* Copyright (c) 2013-2019. The SimGrid Team. All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#include "disk_s19.hpp" +#include "simgrid/kernel/routing/NetPoint.hpp" +#include "simgrid/s4u/Engine.hpp" +#include "simgrid/s4u/Host.hpp" +#include "src/kernel/lmm/maxmin.hpp" +#include "src/surf/xml/platf.hpp" +#include "surf/surf.hpp" + +XBT_LOG_EXTERNAL_DEFAULT_CATEGORY(disk_kernel); + +/********* + * Model * + *********/ + +void surf_disk_model_init_default() +{ + surf_disk_model = new simgrid::kernel::resource::DiskS19Model(); +} + +namespace simgrid { +namespace kernel { +namespace resource { + +DiskS19Model::DiskS19Model() +{ + all_existing_models.push_back(this); +} + +DiskImpl* DiskS19Model::createDisk(const std::string& id, double read_bw, double write_bw) +{ + XBT_DEBUG("SURF disk create resource\n\t\tid '%s'\n\t\tread_bw '%f'\n", id.c_str(), read_bw); + + return new DiskS19(this, id, get_maxmin_system(), read_bw, write_bw); +} + +double DiskS19Model::next_occuring_event(double now) +{ + return DiskModel::next_occuring_event_full(now); +} + +void DiskS19Model::update_actions_state(double /*now*/, double delta) +{ + for (auto it = std::begin(*get_started_action_set()); it != std::end(*get_started_action_set());) { + auto& action = *it; + ++it; // increment iterator here since the following calls to action.finish() may invalidate it + action.update_remains(lrint(action.get_variable()->get_value() * delta)); + action.update_max_duration(delta); + + if (((action.get_remains_no_update() <= 0) && (action.get_variable()->get_penalty() > 0)) || + ((action.get_max_duration() != NO_MAX_DURATION) && (action.get_max_duration() <= 0))) { + action.finish(Action::State::FINISHED); + } + } +} + +/************ + * Resource * + ************/ + +DiskS19::DiskS19(DiskModel* model, const std::string& name, lmm::System* maxminSystem, double read_bw, double write_bw) + : DiskImpl(model, name, maxminSystem, read_bw, write_bw) +{ + XBT_DEBUG("Create resource with read_bw '%f' write_bw '%f'", read_bw, write_bw); +} + +DiskAction* DiskS19::io_start(sg_size_t size, s4u::Io::OpType type) +{ + return new DiskS19Action(get_model(), size, not is_on(), this, type); +} + +DiskAction* DiskS19::read(sg_size_t size) +{ + return new DiskS19Action(get_model(), size, not is_on(), this, s4u::Io::OpType::READ); +} + +DiskAction* DiskS19::write(sg_size_t size) +{ + return new DiskS19Action(get_model(), size, not is_on(), this, s4u::Io::OpType::WRITE); +} + +/********** + * Action * + **********/ + +DiskS19Action::DiskS19Action(Model* model, double cost, bool failed, DiskImpl* disk, s4u::Io::OpType type) + : DiskAction(model, cost, failed, model->get_maxmin_system()->variable_new(this, 1.0, -1.0, 3), disk, type) +{ + XBT_IN("(%s,%g", disk->get_cname(), cost); + + // Must be less than the max bandwidth for all actions + model->get_maxmin_system()->expand(disk->get_constraint(), get_variable(), 1.0); + switch (type) { + case s4u::Io::OpType::READ: + model->get_maxmin_system()->expand(disk->constraint_read_, get_variable(), 1.0); + break; + case s4u::Io::OpType::WRITE: + model->get_maxmin_system()->expand(disk->constraint_write_, get_variable(), 1.0); + break; + default: + THROW_UNIMPLEMENTED; + } + XBT_OUT(); +} + +void DiskS19Action::cancel() +{ + set_state(Action::State::FAILED); +} + +void DiskS19Action::suspend() +{ + XBT_IN("(%p)", this); + if (is_running()) { + get_model()->get_maxmin_system()->update_variable_penalty(get_variable(), 0.0); + set_suspend_state(Action::SuspendStates::SUSPENDED); + } + XBT_OUT(); +} + +void DiskS19Action::resume() +{ + THROW_UNIMPLEMENTED; +} + +void DiskS19Action::set_max_duration(double /*duration*/) +{ + THROW_UNIMPLEMENTED; +} + +void DiskS19Action::set_sharing_penalty(double) +{ + THROW_UNIMPLEMENTED; +} +void DiskS19Action::update_remains_lazy(double /*now*/) +{ + THROW_IMPOSSIBLE; +} +} // namespace resource +} // namespace kernel +} // namespace simgrid diff --git a/src/surf/disk_s19.hpp b/src/surf/disk_s19.hpp new file mode 100644 index 0000000000..5b537ef58a --- /dev/null +++ b/src/surf/disk_s19.hpp @@ -0,0 +1,69 @@ +/* Copyright (c) 2013-2019. The SimGrid Team. All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#include + +#include "src/kernel/resource/DiskImpl.hpp" + +#ifndef DISK_S19_HPP_ +#define DISK_S19_HPP_ + +namespace simgrid { +namespace kernel { +namespace resource { + +/*********** + * Classes * + ***********/ + +class XBT_PRIVATE DiskS19Model; +class XBT_PRIVATE DiskS19; +class XBT_PRIVATE DiskS19Action; + +/********* + * Model * + *********/ + +class DiskS19Model : public DiskModel { +public: + DiskS19Model(); + DiskImpl* createDisk(const std::string& id, double read_bw, double write_bw) override; + double next_occuring_event(double now) override; + void update_actions_state(double now, double delta) override; +}; + +/************ + * Resource * + ************/ + +class DiskS19 : public DiskImpl { +public: + DiskS19(DiskModel* model, const std::string& name, kernel::lmm::System* maxminSystem, double read_bw, + double write_bw); + virtual ~DiskS19() = default; + DiskAction* io_start(sg_size_t size, s4u::Io::OpType type) override; + DiskAction* read(sg_size_t size) override; + DiskAction* write(sg_size_t size) override; +}; + +/********** + * Action * + **********/ + +class DiskS19Action : public DiskAction { +public: + DiskS19Action(Model* model, double cost, bool failed, DiskImpl* disk, s4u::Io::OpType type); + void suspend() override; + void cancel() override; + void resume() override; + void set_max_duration(double duration) override; + void set_sharing_penalty(double sharing_penalty) override; + void update_remains_lazy(double now) override; +}; + +} // namespace resource +} // namespace kernel +} // namespace simgrid +#endif /* DISK_S19_HPP_ */ diff --git a/src/surf/host_clm03.cpp b/src/surf/host_clm03.cpp index 7aa840b985..955e6b8252 100644 --- a/src/surf/host_clm03.cpp +++ b/src/surf/host_clm03.cpp @@ -36,17 +36,19 @@ double HostCLM03Model::next_occuring_event(double now) double min_by_net = surf_network_model->next_occuring_event_is_idempotent() ? surf_network_model->next_occuring_event(now) : -1; double min_by_sto = surf_storage_model->next_occuring_event(now); + double min_by_dsk = surf_disk_model->next_occuring_event(now); - XBT_DEBUG("model %p, %s min_by_cpu %f, %s min_by_net %f, %s min_by_sto %f", - this, typeid(surf_cpu_model_pm).name(), min_by_cpu, - typeid(surf_network_model).name(), min_by_net, - typeid(surf_storage_model).name(), min_by_sto); + XBT_DEBUG("model %p, %s min_by_cpu %f, %s min_by_net %f, %s min_by_sto %f, %s min_by_dsk %f", this, + typeid(surf_cpu_model_pm).name(), min_by_cpu, typeid(surf_network_model).name(), min_by_net, + typeid(surf_storage_model).name(), min_by_sto, typeid(surf_disk_model).name(), min_by_dsk); double res = min_by_cpu; if (res < 0 || (min_by_net >= 0.0 && min_by_net < res)) res = min_by_net; if (res < 0 || (min_by_sto >= 0.0 && min_by_sto < res)) res = min_by_sto; + if (res < 0 || (min_by_dsk >= 0.0 && min_by_dsk < res)) + res = min_by_dsk; return res; } diff --git a/src/surf/sg_platf.cpp b/src/surf/sg_platf.cpp index b91dbb05a6..67a353805d 100644 --- a/src/surf/sg_platf.cpp +++ b/src/surf/sg_platf.cpp @@ -19,6 +19,7 @@ #include "src/include/simgrid/sg_config.hpp" #include "src/include/surf/surf.hpp" #include "src/kernel/EngineImpl.hpp" +#include "src/kernel/resource/DiskImpl.hpp" #include "src/kernel/resource/profile/Profile.hpp" #include "src/simix/smx_private.hpp" #include "src/surf/HostImpl.hpp" @@ -80,6 +81,10 @@ void sg_platf_new_host(simgrid::kernel::routing::HostCreationArgs* args) host->pimpl_->storage_ = mount_list; mount_list.clear(); + host->pimpl_->disks_ = std::move(args->disks); + for (auto d : host->pimpl_->disks_) + d->set_host(host); + /* Change from the defaults */ if (args->state_trace) host->pimpl_cpu->set_state_profile(args->state_trace); @@ -331,6 +336,17 @@ void sg_platf_new_cabinet(simgrid::kernel::routing::CabinetCreationArgs* cabinet delete cabinet->radicals; } +simgrid::kernel::resource::DiskImpl* sg_platf_new_disk(simgrid::kernel::routing::DiskCreationArgs* disk) +{ + simgrid::kernel::resource::DiskImpl* d = surf_disk_model->createDisk(disk->id, disk->read_bw, disk->write_bw); + if (disk->properties) { + d->set_properties(*disk->properties); + delete disk->properties; + } + simgrid::s4u::Disk::on_creation(*d->get_iface()); + return d; +} + void sg_platf_new_storage(simgrid::kernel::routing::StorageCreationArgs* storage) { xbt_assert(std::find(known_storages.begin(), known_storages.end(), storage->id) == known_storages.end(), @@ -502,6 +518,7 @@ static void surf_config_models_setup() std::string host_model_name = simgrid::config::get_value("host/model"); std::string network_model_name = simgrid::config::get_value("network/model"); std::string cpu_model_name = simgrid::config::get_value("cpu/model"); + std::string disk_model_name = simgrid::config::get_value("disk/model"); std::string storage_model_name = simgrid::config::get_value("storage/model"); /* The compound host model is needed when using non-default net/cpu models */ @@ -530,6 +547,10 @@ static void surf_config_models_setup() XBT_DEBUG("Call vm_model_init"); surf_vm_model_init_HL13(); + XBT_DEBUG("Call disk_model_init"); + int disk_id = find_model_description(surf_disk_model_description, disk_model_name); + surf_disk_model_description[disk_id].model_init_preparse(); + XBT_DEBUG("Call storage_model_init"); int storage_id = find_model_description(surf_storage_model_description, storage_model_name); surf_storage_model_description[storage_id].model_init_preparse(); diff --git a/src/surf/surf_c_bindings.cpp b/src/surf/surf_c_bindings.cpp index 3b80a2d730..f559bb2231 100644 --- a/src/surf/surf_c_bindings.cpp +++ b/src/surf/surf_c_bindings.cpp @@ -6,6 +6,7 @@ #include "simgrid/s4u/Engine.hpp" #include "src/include/surf/surf.hpp" #include "src/instr/instr_private.hpp" +#include "src/kernel/resource/DiskImpl.hpp" #include "src/kernel/resource/profile/FutureEvtSet.hpp" #include "src/plugins/vm/VirtualMachineImpl.hpp" @@ -71,7 +72,7 @@ double surf_solve(double max_date) for (auto const& model : all_existing_models) { if (model != surf_host_model && model != surf_vm_model && model != surf_network_model && - model != surf_storage_model) { + model != surf_storage_model && model != surf_disk_model) { double next_event_model = model->next_occuring_event(NOW); if ((time_delta < 0.0 || next_event_model < time_delta) && next_event_model >= 0.0) time_delta = next_event_model; diff --git a/src/surf/surf_interface.cpp b/src/surf/surf_interface.cpp index 61231583c4..0b2ea3caca 100644 --- a/src/surf/surf_interface.cpp +++ b/src/surf/surf_interface.cpp @@ -112,6 +112,10 @@ const std::vector surf_optimization_mode_description = {"Full", "Full update of remaining and variables. Slow but may be useful when debugging.", nullptr}, }; +const std::vector surf_disk_model_description = { + {"default", "Simplistic disk model.", &surf_disk_model_init_default}, +}; + const std::vector surf_storage_model_description = { {"default", "Simplistic storage model.", &surf_storage_model_init_default}, }; diff --git a/src/surf/surf_interface.hpp b/src/surf/surf_interface.hpp index b9f2233479..da361ca582 100644 --- a/src/surf/surf_interface.hpp +++ b/src/surf/surf_interface.hpp @@ -173,6 +173,8 @@ XBT_PUBLIC void surf_host_model_init_ptask_L07(); */ XBT_PUBLIC void surf_storage_model_init_default(); +XBT_PUBLIC void surf_disk_model_init_default(); + /* -------------------- * Model Descriptions * -------------------- */ @@ -203,6 +205,8 @@ XBT_PUBLIC_DATA const std::vector surf_optimization_mo XBT_PUBLIC_DATA const std::vector surf_cpu_model_description; /** @brief The list of all network models (pick one with --cfg=network/model) */ XBT_PUBLIC_DATA const std::vector surf_network_model_description; +/** @brief The list of all disk models (pick one with --cfg=disk/model) */ +XBT_PUBLIC_DATA const std::vector surf_disk_model_description; /** @brief The list of all storage models (pick one with --cfg=storage/model) */ XBT_PUBLIC_DATA const std::vector surf_storage_model_description; /** @brief The list of all host models (pick one with --cfg=host/model:) */ diff --git a/src/surf/xml/platf_private.hpp b/src/surf/xml/platf_private.hpp index 09aaaad7fb..679a31259d 100644 --- a/src/surf/xml/platf_private.hpp +++ b/src/surf/xml/platf_private.hpp @@ -41,6 +41,7 @@ struct HostCreationArgs { profile::Profile* state_trace = nullptr; std::string coord = ""; std::unordered_map* properties = nullptr; + std::vector disks; }; class HostLinkCreationArgs { @@ -138,6 +139,14 @@ public: sg_size_t size; }; +class DiskCreationArgs { +public: + std::string id; + std::unordered_map* properties; + double read_bw; + double write_bw; +}; + class MountCreationArgs { public: std::string storageId; @@ -203,6 +212,9 @@ XBT_PUBLIC void sg_platf_new_bypassRoute(simgrid::kernel::routing::RouteCreation XBT_PUBLIC void sg_platf_new_trace(simgrid::kernel::routing::ProfileCreationArgs* trace); +XBT_PUBLIC simgrid::kernel::resource::DiskImpl* +sg_platf_new_disk(simgrid::kernel::routing::DiskCreationArgs* disk); // Add a disk to the current host + XBT_PUBLIC void sg_platf_new_storage(simgrid::kernel::routing::StorageCreationArgs* storage); // Add a storage to the current Zone XBT_PUBLIC void sg_platf_new_storage_type(simgrid::kernel::routing::StorageTypeCreationArgs* storage_type); XBT_PUBLIC void sg_platf_new_mount(simgrid::kernel::routing::MountCreationArgs* mount); diff --git a/src/surf/xml/surfxml_sax_cb.cpp b/src/surf/xml/surfxml_sax_cb.cpp index ce18fa9a58..86ab5cb2d6 100644 --- a/src/surf/xml/surfxml_sax_cb.cpp +++ b/src/surf/xml/surfxml_sax_cb.cpp @@ -28,8 +28,8 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(surf_parse, surf, "Logging specific to the SURF static std::string surf_parsed_filename; // Currently parsed file (for the error messages) std::vector - parsed_link_list; /* temporary store of current list link of a route */ - + parsed_link_list; /* temporary store of current link list of a route */ +std::vector parsed_disk_list; /* temporary store of current disk list of a host */ /* * Helping functions */ @@ -458,14 +458,27 @@ void ETag_surfxml_host() { : nullptr; host.pstate = surf_parse_get_int(A_surfxml_host_pstate); host.coord = A_surfxml_host_coordinates; + host.disks.swap(parsed_disk_list); sg_platf_new_host(&host); } void STag_surfxml_disk() { - THROW_UNIMPLEMENTED; + ZONE_TAG = 0; + xbt_assert(current_property_set == nullptr, + "Someone forgot to reset the property set to nullptr in its closing tag (or XML malformed)"); } + void ETag_surfxml_disk() { + simgrid::kernel::routing::DiskCreationArgs disk; + disk.properties = current_property_set; + current_property_set = nullptr; + + disk.id = A_surfxml_disk_id; + disk.read_bw = surf_parse_get_bandwidth(A_surfxml_disk_read___bw, "read_bw of disk ", disk.id); + disk.write_bw = surf_parse_get_bandwidth(A_surfxml_disk_write___bw, "write_bw of disk ", disk.id); + + parsed_disk_list.push_back(sg_platf_new_disk(&disk)); } void STag_surfxml_host___link(){ diff --git a/teshsuite/smpi/io-all-at/io-all-at.tesh b/teshsuite/smpi/io-all-at/io-all-at.tesh index 67a7278380..57f3e23b00 100644 --- a/teshsuite/smpi/io-all-at/io-all-at.tesh +++ b/teshsuite/smpi/io-all-at/io-all-at.tesh @@ -1,6 +1,6 @@ # Test for MPI_File_read and MPI_File_write ! output sort -$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/storage/remote_io.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-all-at +$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/hosts_with_disks.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-all-at > You requested to use 4 ranks, but there is only 2 processes in your hostfile... > [rank 0] -> bob > [rank 1] -> carl diff --git a/teshsuite/smpi/io-all/io-all.tesh b/teshsuite/smpi/io-all/io-all.tesh index 1e90e16834..d60d27ec1b 100644 --- a/teshsuite/smpi/io-all/io-all.tesh +++ b/teshsuite/smpi/io-all/io-all.tesh @@ -1,6 +1,6 @@ # Test for MPI_File_read and MPI_File_write ! output sort -$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/storage/remote_io.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-all +$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/hosts_with_disks.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-all > You requested to use 4 ranks, but there is only 2 processes in your hostfile... > [rank 0] -> bob > [rank 1] -> carl diff --git a/teshsuite/smpi/io-ordered/io-ordered.tesh b/teshsuite/smpi/io-ordered/io-ordered.tesh index db525341e0..dedb55072d 100644 --- a/teshsuite/smpi/io-ordered/io-ordered.tesh +++ b/teshsuite/smpi/io-ordered/io-ordered.tesh @@ -1,6 +1,6 @@ # Test for MPI_File_read and MPI_File_write ! output sort -$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/storage/remote_io.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-ordered +$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/hosts_with_disks.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-ordered > You requested to use 4 ranks, but there is only 2 processes in your hostfile... > [rank 0] -> bob > [rank 1] -> carl diff --git a/teshsuite/smpi/io-simple-at/io-simple-at.tesh b/teshsuite/smpi/io-simple-at/io-simple-at.tesh index 5f6e87dc8e..f8ff02771a 100644 --- a/teshsuite/smpi/io-simple-at/io-simple-at.tesh +++ b/teshsuite/smpi/io-simple-at/io-simple-at.tesh @@ -1,6 +1,6 @@ # Test for MPI_File_read and MPI_File_write ! output sort -$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/storage/remote_io.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-simple-at +$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/hosts_with_disks.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-simple-at > You requested to use 4 ranks, but there is only 2 processes in your hostfile... > [rank 0] -> bob > [rank 1] -> carl diff --git a/teshsuite/smpi/io-simple/io-simple.tesh b/teshsuite/smpi/io-simple/io-simple.tesh index 2bb859882b..bd56168279 100644 --- a/teshsuite/smpi/io-simple/io-simple.tesh +++ b/teshsuite/smpi/io-simple/io-simple.tesh @@ -1,6 +1,6 @@ # Test for MPI_File_read and MPI_File_write ! output sort -$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/storage/remote_io.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-simple +$ ${bindir:=.}/../../../smpi_script/bin/smpirun -map -hostfile ../hostfile_io -platform ../../../examples/platforms/hosts_with_disks.xml -np 4 --log=xbt_cfg.thres:critical --log=smpi_kernel.thres:warning --log=smpi_mpi.thres:error --log=smpi_io.thres:verbose "--log=root.fmt:(%P@%h)%e%m%n" --cfg=smpi/simulate-computation:0 ${bindir:=.}/io-simple > You requested to use 4 ranks, but there is only 2 processes in your hostfile... > [rank 0] -> bob > [rank 1] -> carl diff --git a/tools/cmake/DefinePackages.cmake b/tools/cmake/DefinePackages.cmake index 2be9a3884e..6b18fd179c 100644 --- a/tools/cmake/DefinePackages.cmake +++ b/tools/cmake/DefinePackages.cmake @@ -48,6 +48,7 @@ set(EXTRA_DIST src/surf/xml/simgrid_dtd.c src/surf/xml/surfxml_sax_cb.cpp + src/surf/disk_s19.hpp src/surf/StorageImpl.hpp src/surf/storage_n11.hpp src/surf/surf_interface.hpp @@ -313,6 +314,8 @@ set(SURF_SRC src/kernel/resource/Action.cpp src/kernel/resource/Model.cpp src/kernel/resource/Resource.cpp + src/kernel/resource/DiskImpl.cpp + src/kernel/resource/DiskImpl.hpp src/kernel/resource/profile/DatedValue.cpp src/kernel/resource/profile/DatedValue.hpp @@ -341,6 +344,7 @@ set(SURF_SRC src/surf/cpu_cas01.cpp src/surf/cpu_interface.cpp src/surf/cpu_ti.cpp + src/surf/disk_s19.cpp src/surf/network_cm02.cpp src/surf/network_constant.cpp src/surf/network_interface.cpp @@ -438,6 +442,7 @@ set(S4U_SRC src/s4u/s4u_Barrier.cpp src/s4u/s4u_ConditionVariable.cpp src/s4u/s4u_Comm.cpp + src/s4u/s4u_Disk.cpp src/s4u/s4u_Engine.cpp src/s4u/s4u_Exec.cpp src/s4u/s4u_Host.cpp @@ -707,6 +712,7 @@ set(headers_to_install include/simgrid/s4u/Barrier.hpp include/simgrid/s4u/Comm.hpp include/simgrid/s4u/ConditionVariable.hpp + include/simgrid/s4u/Disk.hpp include/simgrid/s4u/Engine.hpp include/simgrid/s4u/Exec.hpp include/simgrid/s4u/Host.hpp @@ -1131,6 +1137,7 @@ set(PLATFORMS_EXAMPLES examples/platforms/faulty_host.xml examples/platforms/g5k.xml examples/platforms/griffon.xml + examples/platforms/hosts_with_disks.xml examples/platforms/meta_cluster.xml examples/platforms/multicore_machine.xml examples/platforms/onelink.xml