X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/49b53eb334bc6f19b530603201d00bcc312bfd29..HEAD:/src/plugins/file_system/s4u_FileSystem.cpp diff --git a/src/plugins/file_system/s4u_FileSystem.cpp b/src/plugins/file_system/s4u_FileSystem.cpp index 1807a0d392..31b7fff93b 100644 --- a/src/plugins/file_system/s4u_FileSystem.cpp +++ b/src/plugins/file_system/s4u_FileSystem.cpp @@ -1,27 +1,26 @@ -/* Copyright (c) 2015-2021. The SimGrid Team. All rights reserved. */ +/* Copyright (c) 2015-2023. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ -#include "simgrid/plugins/file_system.h" -#include "simgrid/s4u/Actor.hpp" -#include "simgrid/s4u/Comm.hpp" -#include "simgrid/s4u/Engine.hpp" -#include "src/surf/HostImpl.hpp" -#include "src/surf/xml/platf_private.hpp" -#include "xbt/config.hpp" -#include "xbt/parse_units.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include -#include #include -#include #include #include -#include #include XBT_LOG_NEW_DEFAULT_SUBCATEGORY(s4u_file, s4u, "S4U files"); -int sg_storage_max_file_descriptors = 1024; /** @defgroup plugin_filesystem Plugin FileSystem * @@ -36,10 +35,11 @@ template class xbt::Extendable; namespace s4u { simgrid::xbt::Extension FileSystemDiskExt::EXTENSION_ID; simgrid::xbt::Extension FileDescriptorHostExt::EXTENSION_ID; +int FileDescriptorHostExt::max_file_descriptors; -Disk* File::find_local_disk_on(const Host* host) +const Disk* File::find_local_disk_on(const Host* host) { - Disk* d = nullptr; + const Disk* d = nullptr; size_t longest_prefix_length = 0; for (auto const& disk : host->get_disks()) { std::string current_mount; @@ -57,7 +57,7 @@ Disk* File::find_local_disk_on(const Host* host) host->get_cname()); /* Mount point found, split fullpath_ into mount_name and path+filename*/ mount_point_ = fullpath_.substr(0, longest_prefix_length); - if (mount_point_ == std::string("/")) + if (mount_point_ == "/") path_ = fullpath_; else path_ = fullpath_.substr(longest_prefix_length, fullpath_.length()); @@ -70,22 +70,21 @@ File::File(const std::string& fullpath, void* userdata) : File(fullpath, Host::c File::File(const std::string& fullpath, const_sg_host_t host, void* userdata) : fullpath_(fullpath) { - kernel::actor::simcall([this, &host, userdata] { + kernel::actor::simcall_answered([this, &host, userdata] { this->set_data(userdata); // this cannot fail because we get a xbt_die if the mountpoint does not exist local_disk_ = find_local_disk_on(host); // assign a file descriptor id to the newly opened File - auto* ext = host->extension(); + auto* ext = host->extension(); if (ext->file_descriptor_table == nullptr) { - ext->file_descriptor_table = std::make_unique>(sg_storage_max_file_descriptors); + ext->file_descriptor_table = std::make_unique>(FileDescriptorHostExt::max_file_descriptors); std::iota(ext->file_descriptor_table->rbegin(), ext->file_descriptor_table->rend(), 0); // Fill with ..., 1, 0. } xbt_assert(not ext->file_descriptor_table->empty(), "Too much files are opened! Some have to be closed."); desc_id = ext->file_descriptor_table->back(); ext->file_descriptor_table->pop_back(); - XBT_DEBUG("\tOpen file '%s'", path_.c_str()); std::map>* content = nullptr; content = local_disk_->extension()->get_content(); @@ -94,6 +93,7 @@ File::File(const std::string& fullpath, const_sg_host_t host, void* userdata) : auto sz = content->find(path_); if (sz != content->end()) { size_ = sz->second; + XBT_DEBUG("\tOpen file '%s', size %llu", path_.c_str(), size_); } else { size_ = 0; content->insert({path_, size_}); @@ -103,11 +103,23 @@ File::File(const std::string& fullpath, const_sg_host_t host, void* userdata) : }); } -File::~File() +File::~File() = default; + +File* File::open(const std::string& fullpath, void* userdata) +{ + return new File(fullpath, userdata); +} + +File* File::open(const std::string& fullpath, const_sg_host_t host, void* userdata) { - std::vector* desc_table = - Host::current()->extension()->file_descriptor_table.get(); - kernel::actor::simcall([this, desc_table] { desc_table->push_back(this->desc_id); }); + return new File(fullpath, host, userdata); +} + +void File::close() +{ + std::vector* desc_table = Host::current()->extension()->file_descriptor_table.get(); + kernel::actor::simcall_answered([this, desc_table] { desc_table->push_back(this->desc_id); }); + delete this; } void File::dump() const @@ -152,6 +164,7 @@ sg_size_t File::read(sg_size_t size) * @ingroup plugin_filesystem * * @param size of the file to write + * @param write_inside * @return the number of bytes successfully write or -1 if an error occurred */ sg_size_t File::write(sg_size_t size, bool write_inside) @@ -161,9 +174,7 @@ sg_size_t File::write(sg_size_t size, bool write_inside) sg_size_t write_size = 0; /* Find the host where the file is physically located (remote or local)*/ - Host* host = local_disk_->get_host(); - - if (host && host->get_name() != Host::current()->get_name()) { + if (Host* host = local_disk_->get_host(); host && host->get_name() != Host::current()->get_name()) { /* the file is hosted on a remote host, initiate a communication between src and dest hosts for data transfer */ XBT_DEBUG("File is on %s remote host, initiate data transfer of %llu bytes.", host->get_cname(), size); Comm::sendto(Host::current(), host, size); @@ -173,25 +184,11 @@ sg_size_t File::write(sg_size_t size, bool write_inside) // If the disk is full before even starting to write if (sg_disk_get_size_used(local_disk_) >= sg_disk_get_size(local_disk_)) return 0; - if (not write_inside) { + if (not write_inside) /* Subtract the part of the file that might disappear from the used sized on the storage element */ local_disk_->extension()->decr_used_size(size_ - current_position_); - write_size = local_disk_->write(size); - local_disk_->extension()->incr_used_size(write_size); - current_position_ += write_size; - size_ = current_position_; - } else { - write_size = local_disk_->write(size); - current_position_ += write_size; - if (current_position_ > size_) - size_ = current_position_; - } - kernel::actor::simcall([this] { - std::map>* content = local_disk_->extension()->get_content(); - - content->erase(path_); - content->insert({path_, size_}); - }); + write_size = local_disk_->write(size); + update_position(current_position_ + write_size); return write_size; } @@ -210,19 +207,36 @@ void File::seek(sg_offset_t offset, int origin) { switch (origin) { case SEEK_SET: - current_position_ = offset; - break; + update_position(offset); + break; case SEEK_CUR: - current_position_ += offset; + update_position(current_position_ + offset); break; case SEEK_END: - current_position_ = size_ + offset; + update_position(size_ + offset); break; default: break; } } +void File::update_position(sg_offset_t position) +{ + xbt_assert(position >= 0, "Error in seek, cannot seek before file %s", get_path()); + current_position_ = position; + if(current_position_>size_){ + XBT_DEBUG("Updating size of file %s from %llu to %lld", path_.c_str(), size_, position); + local_disk_->extension()->incr_used_size(current_position_-size_); + size_ = current_position_; + + kernel::actor::simcall_answered([this] { + std::map>* content = local_disk_->extension()->get_content(); + content->erase(path_); + content->insert({path_, size_}); + }); + } +} + sg_size_t File::tell() const { return current_position_; @@ -231,7 +245,7 @@ sg_size_t File::tell() const void File::move(const std::string& fullpath) const { /* Check if the new full path is on the same mount point */ - if (fullpath.compare(0, mount_point_.length(), mount_point_) == 0) { + if (fullpath.rfind(mount_point_, 0) == 0) { std::map>* content = nullptr; content = local_disk_->extension()->get_content(); if (content) { @@ -261,10 +275,9 @@ int File::unlink() const XBT_WARN("File %s is not on disk %s. Impossible to unlink", path_.c_str(), name); return -1; } else { - XBT_DEBUG("UNLINK %s on disk '%s'", path_.c_str(), name); + XBT_DEBUG("UNLINK %s of size %llu on disk '%s'", path_.c_str(), size_, name); local_disk_->extension()->decr_used_size(size_); - // Remove the file from storage content->erase(path_); @@ -292,7 +305,7 @@ int File::remote_copy(sg_host_t host, const std::string& fullpath) for (auto const& disk : host->get_disks()) { std::string current_mount = disk->extension()->get_mount_point(); - std::string mount_point = std::string(fullpath).substr(0, current_mount.length()); + std::string mount_point = fullpath.substr(0, current_mount.length()); if (mount_point == current_mount && current_mount.length() > longest_prefix_length) { /* The current mount name is found in the full path and is bigger than the previous*/ longest_prefix_length = current_mount.length(); @@ -312,8 +325,9 @@ int File::remote_copy(sg_host_t host, const std::string& fullpath) } /* Create file on remote host, write it and close it */ - File fd(fullpath, dst_host, nullptr); - fd.write(read_size); + auto* fd = File::open(fullpath, dst_host, nullptr); + fd->write(read_size); + fd->close(); return 0; } @@ -326,19 +340,17 @@ int File::remote_move(sg_host_t host, const std::string& fullpath) FileSystemDiskExt::FileSystemDiskExt(const Disk* ptr) { - const char* size_str = ptr->get_property("size"); - std::string dummyfile; - if (size_str) - size_ = surf_parse_get_size(dummyfile, -1, size_str, "disk size", ptr->get_name()); + if (const char* size_str = ptr->get_property("size")) { + std::string dummyfile; + size_ = xbt_parse_get_size(dummyfile, -1, size_str, "disk size " + ptr->get_name()); + } - const char* current_mount_str = ptr->get_property("mount"); - if (current_mount_str) - mount_point_ = std::string(current_mount_str); + if (const char* current_mount_str = ptr->get_property("mount")) + mount_point_ = current_mount_str; else - mount_point_ = std::string("/"); + mount_point_ = "/"; - const char* content_str = ptr->get_property("content"); - if (content_str) + if (const char* content_str = ptr->get_property("content")) content_.reset(parse_content(content_str)); } @@ -349,9 +361,9 @@ std::map>* FileSystemDiskExt::parse_content( auto* parse_content = new std::map>(); - auto fs = std::unique_ptr(surf_ifsopen(filename)); + auto fs = std::unique_ptr(simgrid::xbt::path_ifsopen(filename)); xbt_assert(not fs->fail(), "Cannot open file '%s' (path=%s)", filename.c_str(), - (boost::join(surf_path, ":")).c_str()); + simgrid::xbt::path_to_string().c_str()); std::string line; std::vector tokens; @@ -370,14 +382,19 @@ std::map>* FileSystemDiskExt::parse_content( return parse_content; } +void FileSystemDiskExt::add_remote_mount(Host* host, const std::string& mount_point) +{ + remote_mount_points_.try_emplace(host, mount_point); +} + void FileSystemDiskExt::decr_used_size(sg_size_t size) { - simgrid::kernel::actor::simcall([this, size] { used_size_ -= size; }); + simgrid::kernel::actor::simcall_answered([this, size] { used_size_ -= size; }); } void FileSystemDiskExt::incr_used_size(sg_size_t size) { - simgrid::kernel::actor::simcall([this, size] { used_size_ += size; }); + simgrid::kernel::actor::simcall_answered([this, size] { used_size_ += size; }); } } } @@ -395,49 +412,28 @@ static void on_host_creation(simgrid::s4u::Host& host) host.extension_set(new FileDescriptorHostExt()); } -static void on_platform_created() -{ - for (auto const& host : simgrid::s4u::Engine::get_instance()->get_all_hosts()) { - const char* remote_disk_str = host->get_property("remote_disk"); - if (remote_disk_str) { - std::vector tokens; - boost::split(tokens, remote_disk_str, boost::is_any_of(":")); - std::string mount_point = tokens[0]; - simgrid::s4u::Host* remote_host = simgrid::s4u::Host::by_name_or_null(tokens[2]); - xbt_assert(remote_host, "You're trying to access a host that does not exist. Please check your platform file"); - - const simgrid::s4u::Disk* disk = nullptr; - for (auto const& d : remote_host->get_disks()) - if (d->get_name() == tokens[1]) { - disk = d; - break; - } + static void on_platform_created() + { + for (auto const& host : simgrid::s4u::Engine::get_instance()->get_all_hosts()) { + const char* remote_disk_str = host->get_property("remote_disk"); + if (not remote_disk_str) + continue; + std::vector tokens; + boost::split(tokens, remote_disk_str, boost::is_any_of(":")); + std::string mount_point = tokens[0]; + simgrid::s4u::Host* remote_host = simgrid::s4u::Host::by_name_or_null(tokens[2]); + xbt_assert(remote_host, "You're trying to access a host that does not exist. Please check your platform file"); - xbt_assert(disk, "You're trying to mount a disk that does not exist. Please check your platform file"); - disk->extension()->add_remote_mount(remote_host, mount_point); - host->add_disk(disk); - - XBT_DEBUG("Host '%s' wants to mount a remote disk: %s of %s mounted on %s", host->get_cname(), disk->get_cname(), - remote_host->get_cname(), mount_point.c_str()); - XBT_DEBUG("Host '%s' now has %zu disks", host->get_cname(), host->get_disks().size()); - } - } -} + const simgrid::s4u::Disk* disk = nullptr; + for (auto const& d : remote_host->get_disks()) + if (d->get_name() == tokens[1]) { + disk = d; + break; + } -static void on_simulation_end() -{ - XBT_DEBUG("Simulation is over, time to unregister remote disks if any"); - for (auto const& host : simgrid::s4u::Engine::get_instance()->get_all_hosts()) { - const char* remote_disk_str = host->get_property("remote_disk"); - if (remote_disk_str) { - std::vector tokens; - boost::split(tokens, remote_disk_str, boost::is_any_of(":")); - XBT_DEBUG("Host '%s' wants to unmount a remote disk: %s of %s mounted on %s", host->get_cname(), - tokens[1].c_str(), tokens[2].c_str(), tokens[0].c_str()); - host->remove_disk(tokens[1]); - XBT_DEBUG("Host '%s' now has %zu disks", host->get_cname(), host->get_disks().size()); - } - } + xbt_assert(disk, "You're trying to mount a disk that does not exist. Please check your platform file"); + disk->extension()->add_remote_mount(remote_host, mount_point); + } } /* **************************** Public interface *************************** */ @@ -450,26 +446,25 @@ static void on_simulation_end() */ void sg_storage_file_system_init() { - sg_storage_max_file_descriptors = 1024; - simgrid::config::bind_flag(sg_storage_max_file_descriptors, "storage/max_file_descriptors", + FileDescriptorHostExt::max_file_descriptors = 1024; + simgrid::config::bind_flag(FileDescriptorHostExt::max_file_descriptors, "storage/max_file_descriptors", "Maximum number of concurrently opened files per host. Default is 1024"); if (not FileSystemDiskExt::EXTENSION_ID.valid()) { FileSystemDiskExt::EXTENSION_ID = simgrid::s4u::Disk::extension_create(); - simgrid::s4u::Disk::on_creation.connect(&on_disk_creation); + simgrid::s4u::Disk::on_creation_cb(&on_disk_creation); } if (not FileDescriptorHostExt::EXTENSION_ID.valid()) { FileDescriptorHostExt::EXTENSION_ID = simgrid::s4u::Host::extension_create(); - simgrid::s4u::Host::on_creation.connect(&on_host_creation); + simgrid::s4u::Host::on_creation_cb(&on_host_creation); } - simgrid::s4u::Engine::on_platform_created.connect(&on_platform_created); - simgrid::s4u::Engine::on_simulation_end.connect(&on_simulation_end); + simgrid::s4u::Engine::on_platform_created_cb(&on_platform_created); } sg_file_t sg_file_open(const char* fullpath, void* data) { - return new simgrid::s4u::File(fullpath, data); + return simgrid::s4u::File::open(fullpath, data); } sg_size_t sg_file_read(sg_file_t fd, sg_size_t size) @@ -482,9 +477,9 @@ sg_size_t sg_file_write(sg_file_t fd, sg_size_t size) return fd->write(size); } -void sg_file_close(const_sg_file_t fd) +void sg_file_close(sg_file_t fd) { - delete fd; + fd->close(); } /** Retrieves the path to the file @@ -514,7 +509,7 @@ void sg_file_dump(const_sg_file_t fd) */ void* sg_file_get_data(const_sg_file_t fd) { - return fd->get_data(); + return fd->get_data(); } /** Changes the user data associated with the file @@ -553,7 +548,7 @@ void sg_file_move(const_sg_file_t fd, const char* fullpath) void sg_file_unlink(sg_file_t fd) { fd->unlink(); - delete fd; + fd->close(); } /**