X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/4c7ee8190b86e9eb2adb13479325f586ead44ed0..173b3916e2bf66ae93d66d3a07ad9974610ac7ce:/src/plugins/file_system/s4u_FileSystem.cpp diff --git a/src/plugins/file_system/s4u_FileSystem.cpp b/src/plugins/file_system/s4u_FileSystem.cpp index d48c7d3afc..b1b6bafc55 100644 --- a/src/plugins/file_system/s4u_FileSystem.cpp +++ b/src/plugins/file_system/s4u_FileSystem.cpp @@ -1,14 +1,11 @@ -/* Copyright (c) 2015-2017. The SimGrid Team. All rights reserved. */ +/* Copyright (c) 2015-2018. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ -#include "xbt/log.h" -#include "simgrid/s4u/Host.hpp" -#include "simgrid/s4u/Storage.hpp" -#include "simgrid/simix.hpp" -#include "src/plugins/file_system/FileSystem.hpp" +#include "simgrid/plugins/file_system.h" +#include "simgrid/s4u/Actor.hpp" #include "src/surf/HostImpl.hpp" #include @@ -16,12 +13,15 @@ #include #include #include +#include XBT_LOG_NEW_DEFAULT_CATEGORY(s4u_file, "S4U files"); +int sg_storage_max_file_descriptors = 1024; namespace simgrid { namespace s4u { -simgrid::xbt::Extension FileSystemStorageExt::EXTENSION_ID; +simgrid::xbt::Extension FileSystemStorageExt::EXTENSION_ID; +simgrid::xbt::Extension FileDescriptorHostExt::EXTENSION_ID; File::File(std::string fullpath, void* userdata) : File(fullpath, Host::current(), userdata){}; @@ -30,7 +30,7 @@ File::File(std::string fullpath, sg_host_t host, void* userdata) : fullpath_(ful // this cannot fail because we get a xbt_die if the mountpoint does not exist Storage* st = nullptr; size_t longest_prefix_length = 0; - XBT_DEBUG("Search for storage name for '%s' on '%s'", fullpath.c_str(), host->getCname()); + XBT_DEBUG("Search for storage name for '%s' on '%s'", fullpath.c_str(), host->get_cname()); for (auto const& mnt : host->getMountedStorages()) { XBT_DEBUG("See '%s'", mnt.first.c_str()); @@ -46,10 +46,20 @@ File::File(std::string fullpath, sg_host_t host, void* userdata) : fullpath_(ful mount_point_ = fullpath.substr(0, longest_prefix_length); path_ = fullpath.substr(longest_prefix_length, fullpath.length()); } else - xbt_die("Can't find mount point for '%s' on '%s'", fullpath.c_str(), host->getCname()); + xbt_die("Can't find mount point for '%s' on '%s'", fullpath.c_str(), host->get_cname()); localStorage = st; + // assign a file descriptor id to the newly opened File + FileDescriptorHostExt* ext = host->extension(); + if (ext->file_descriptor_table == nullptr) { + ext->file_descriptor_table = new std::vector(sg_storage_max_file_descriptors); + std::iota(ext->file_descriptor_table->rbegin(), ext->file_descriptor_table->rend(), 0); // Fill with ..., 1, 0. + } + xbt_assert(not ext->file_descriptor_table->empty(), "Too much files are opened! Some have to be closed."); + desc_id = ext->file_descriptor_table->back(); + ext->file_descriptor_table->pop_back(); + XBT_DEBUG("\tOpen file '%s'", path_.c_str()); std::map* content = localStorage->extension()->getContent(); // if file does not exist create an empty file @@ -63,18 +73,73 @@ File::File(std::string fullpath, sg_host_t host, void* userdata) : fullpath_(ful } } +File::~File() +{ + Host::current()->extension()->file_descriptor_table->push_back(desc_id); +} + +void File::dump() +{ + XBT_INFO("File Descriptor information:\n" + "\t\tFull path: '%s'\n" + "\t\tSize: %llu\n" + "\t\tMount point: '%s'\n" + "\t\tStorage Id: '%s'\n" + "\t\tStorage Type: '%s'\n" + "\t\tFile Descriptor Id: %d", + getPath(), size_, mount_point_.c_str(), localStorage->get_cname(), localStorage->getType(), desc_id); +} + sg_size_t File::read(sg_size_t size) { - XBT_DEBUG("READ %s on disk '%s'", getPath(), localStorage->getCname()); + if (size_ == 0) /* Nothing to read, return */ + return 0; + + /* Find the host where the file is physically located and read it */ + Host* host = localStorage->getHost(); + XBT_DEBUG("READ %s on disk '%s'", getPath(), localStorage->get_cname()); // if the current position is close to the end of the file, we may not be able to read the requested size sg_size_t read_size = localStorage->read(std::min(size, size_ - current_position_)); current_position_ += read_size; + + if (strcmp(host->get_cname(), Host::current()->get_cname())) { + /* the file is hosted on a remote host, initiate a communication between src and dest hosts for data transfer */ + XBT_DEBUG("File is on %s remote host, initiate data transfer of %llu bytes.", host->get_cname(), read_size); + Host* m_host_list[] = {Host::current(), host}; + double* flops_amount = new double[2]{0, 0}; + double* bytes_amount = new double[4]{0, 0, static_cast(read_size), 0}; + + this_actor::parallel_execute(2, m_host_list, flops_amount, bytes_amount); + } + return read_size; } +/** \brief Write into a file (local or remote) + * + * \param size of the file to write + * \param fd is a the file descriptor + * \return the number of bytes successfully write or -1 if an error occurred + */ sg_size_t File::write(sg_size_t size) { - XBT_DEBUG("WRITE %s on disk '%s'. size '%llu/%llu'", getPath(), localStorage->getCname(), size, size_); + if (size == 0) /* Nothing to write, return */ + return 0; + + /* Find the host where the file is physically located (remote or local)*/ + Host* host = localStorage->getHost(); + + if (strcmp(host->get_cname(), Host::current()->get_cname())) { + /* the file is hosted on a remote host, initiate a communication between src and dest hosts for data transfer */ + XBT_DEBUG("File is on %s remote host, initiate data transfer of %llu bytes.", host->get_cname(), size); + Host* m_host_list[] = {Host::current(), host}; + double* flops_amount = new double[2]{0, 0}; + double* bytes_amount = new double[4]{0, static_cast(size), 0, 0}; + + this_actor::parallel_execute(2, m_host_list, flops_amount, bytes_amount); + } + + XBT_DEBUG("WRITE %s on disk '%s'. size '%llu/%llu'", getPath(), localStorage->get_cname(), size, size_); // If the storage is full before even starting to write if (sg_storage_get_size_used(localStorage) >= sg_storage_get_size(localStorage)) return 0; @@ -152,10 +217,10 @@ int File::unlink() std::map* content = localStorage->extension()->getContent(); if (content->find(path_) == content->end()) { - XBT_WARN("File %s is not on disk %s. Impossible to unlink", path_.c_str(), localStorage->getCname()); + XBT_WARN("File %s is not on disk %s. Impossible to unlink", path_.c_str(), localStorage->get_cname()); return -1; } else { - XBT_DEBUG("UNLINK %s on disk '%s'", path_.c_str(), localStorage->getCname()); + XBT_DEBUG("UNLINK %s on disk '%s'", path_.c_str(), localStorage->get_cname()); localStorage->extension()->decrUsedSize(size_); // Remove the file from storage @@ -165,6 +230,63 @@ int File::unlink() } } +int File::remoteCopy(sg_host_t host, const char* fullpath) +{ + /* Find the host where the file is physically located and read it */ + Storage* storage_src = localStorage; + Host* src_host = storage_src->getHost(); + seek(0, SEEK_SET); + XBT_DEBUG("READ %s on disk '%s'", getPath(), localStorage->get_cname()); + // if the current position is close to the end of the file, we may not be able to read the requested size + sg_size_t read_size = localStorage->read(size_); + current_position_ += read_size; + + /* Find the host that owns the storage where the file has to be copied */ + Storage* storage_dest = nullptr; + Host* dst_host; + size_t longest_prefix_length = 0; + + for (auto const& elm : host->getMountedStorages()) { + std::string mount_point = std::string(fullpath).substr(0, elm.first.size()); + if (mount_point == elm.first && elm.first.length() > longest_prefix_length) { + /* The current mount name is found in the full path and is bigger than the previous*/ + longest_prefix_length = elm.first.length(); + storage_dest = elm.second; + } + } + + if (storage_dest != nullptr) { + /* Mount point found, retrieve the host the storage is attached to */ + dst_host = storage_dest->getHost(); + } else { + XBT_WARN("Can't find mount point for '%s' on destination host '%s'", fullpath, host->get_cname()); + return -1; + } + + XBT_DEBUG("Initiate data transfer of %llu bytes between %s and %s.", read_size, src_host->get_cname(), + storage_dest->getHost()->get_cname()); + Host* m_host_list[] = {src_host, dst_host}; + double* flops_amount = new double[2]{0, 0}; + double* bytes_amount = new double[4]{0, static_cast(read_size), 0, 0}; + + this_actor::parallel_execute(2, m_host_list, flops_amount, bytes_amount); + + /* Create file on remote host, write it and close it */ + File* fd = new File(fullpath, dst_host, nullptr); + sg_size_t write_size = fd->localStorage->write(read_size); + fd->localStorage->extension()->incrUsedSize(write_size); + (*(fd->localStorage->extension()->getContent()))[path_] = size_; + delete fd; + return 0; +} + +int File::remoteMove(sg_host_t host, const char* fullpath) +{ + int res = remoteCopy(host, fullpath); + unlink(); + return res; +} + FileSystemStorageExt::FileSystemStorageExt(simgrid::s4u::Storage* ptr) { content_ = parseContent(ptr->getImpl()->content_name); @@ -206,6 +328,7 @@ std::map* FileSystemStorageExt::parseContent(std::string } using simgrid::s4u::FileSystemStorageExt; +using simgrid::s4u::FileDescriptorHostExt; static void onStorageCreation(simgrid::s4u::Storage& st) { @@ -217,19 +340,125 @@ static void onStorageDestruction(simgrid::s4u::Storage& st) delete st.extension(); } -/* **************************** Public interface *************************** */ -SG_BEGIN_DECL() +static void onHostCreation(simgrid::s4u::Host& host) +{ + host.extension_set(new FileDescriptorHostExt()); +} +/* **************************** Public interface *************************** */ void sg_storage_file_system_init() { + if (not FileSystemStorageExt::EXTENSION_ID.valid()) { + FileSystemStorageExt::EXTENSION_ID = simgrid::s4u::Storage::extension_create(); + simgrid::s4u::Storage::onCreation.connect(&onStorageCreation); + simgrid::s4u::Storage::onDestruction.connect(&onStorageDestruction); + } - if (FileSystemStorageExt::EXTENSION_ID.valid()) - return; + if (not FileDescriptorHostExt::EXTENSION_ID.valid()) { + FileDescriptorHostExt::EXTENSION_ID = simgrid::s4u::Host::extension_create(); + simgrid::s4u::Host::onCreation.connect(&onHostCreation); + } +} - FileSystemStorageExt::EXTENSION_ID = simgrid::s4u::Storage::extension_create(); +sg_file_t sg_file_open(const char* fullpath, void* data) +{ + return new simgrid::s4u::File(fullpath, data); +} - simgrid::s4u::Storage::onCreation.connect(&onStorageCreation); - simgrid::s4u::Storage::onDestruction.connect(&onStorageDestruction); +sg_size_t sg_file_read(sg_file_t fd, sg_size_t size) +{ + return fd->read(size); +} + +sg_size_t sg_file_write(sg_file_t fd, sg_size_t size) +{ + return fd->write(size); +} + +void sg_file_close(sg_file_t fd) +{ + delete fd; +} + +const char* sg_file_get_name(sg_file_t fd) +{ + xbt_assert((fd != nullptr), "Invalid file descriptor"); + return fd->getPath(); +} + +sg_size_t sg_file_get_size(sg_file_t fd) +{ + return fd->size(); +} + +void sg_file_dump(sg_file_t fd) +{ + fd->dump(); +} + +void* sg_file_get_data(sg_file_t fd) +{ + return fd->getUserdata(); +} + +void sg_file_set_data(sg_file_t fd, void* data) +{ + fd->setUserdata(data); +} + +/** + * \brief Set the file position indicator in the msg_file_t by adding offset bytes + * to the position specified by origin (either SEEK_SET, SEEK_CUR, or SEEK_END). + * + * \param fd : file object that identifies the stream + * \param offset : number of bytes to offset from origin + * \param origin : Position used as reference for the offset. It is specified by one of the following constants defined + * in \ exclusively to be used as arguments for this function (SEEK_SET = beginning of file, + * SEEK_CUR = current position of the file pointer, SEEK_END = end of file) + */ +void sg_file_seek(sg_file_t fd, sg_offset_t offset, int origin) +{ + fd->seek(offset, origin); +} + +sg_size_t sg_file_tell(sg_file_t fd) +{ + return fd->tell(); +} + +void sg_file_move(sg_file_t fd, const char* fullpath) +{ + fd->move(fullpath); +} + +void sg_file_unlink(sg_file_t fd) +{ + fd->unlink(); + delete fd; +} + +/** + * \brief Copy a file to another location on a remote host. + * \param file : the file to move + * \param host : the remote host where the file has to be copied + * \param fullpath : the complete path destination on the remote host + * \return If successful, the function returns 0. Otherwise, it returns -1. + */ +int sg_file_rcopy(sg_file_t file, sg_host_t host, const char* fullpath) +{ + return file->remoteCopy(host, fullpath); +} + +/** + * \brief Move a file to another location on a remote host. + * \param file : the file to move + * \param host : the remote host where the file has to be moved + * \param fullpath : the complete path destination on the remote host + * \return If successful, the function returns 0. Otherwise, it returns -1. + */ +int sg_file_rmove(sg_file_t file, sg_host_t host, const char* fullpath) +{ + return file->remoteMove(host, fullpath); } sg_size_t sg_storage_get_size_free(sg_storage_t st) @@ -247,4 +476,26 @@ sg_size_t sg_storage_get_size(sg_storage_t st) return st->extension()->getSize(); } -SG_END_DECL() +xbt_dict_t sg_storage_get_content(sg_storage_t storage) +{ + std::map* content = storage->extension()->getContent(); + // Note: ::operator delete is ok here (no destructor called) since the dict elements are of POD type sg_size_t. + xbt_dict_t content_as_dict = xbt_dict_new_homogeneous(::operator delete); + + for (auto const& entry : *content) { + sg_size_t* psize = new sg_size_t; + *psize = entry.second; + xbt_dict_set(content_as_dict, entry.first.c_str(), psize, nullptr); + } + return content_as_dict; +} + +xbt_dict_t sg_host_get_storage_content(sg_host_t host) +{ + xbt_assert((host != nullptr), "Invalid parameters"); + xbt_dict_t contents = xbt_dict_new_homogeneous(nullptr); + for (auto const& elm : host->getMountedStorages()) + xbt_dict_set(contents, elm.first.c_str(), sg_storage_get_content(elm.second), nullptr); + + return contents; +}