From 3a79499977cb9778f06ee95abdc7e3a2202ed5e0 Mon Sep 17 00:00:00 2001 From: Frederic Suter Date: Thu, 30 Nov 2017 20:11:34 +0100 Subject: [PATCH] remote copy and move in the plugin --- include/simgrid/msg.h | 2 - include/simgrid/plugins/file_system.h | 4 ++ src/msg/msg_io.cpp | 68 ------------------- src/plugins/file_system/FileSystem.hpp | 2 + src/plugins/file_system/s4u_FileSystem.cpp | 79 ++++++++++++++++++++++ 5 files changed, 85 insertions(+), 70 deletions(-) diff --git a/include/simgrid/msg.h b/include/simgrid/msg.h index 16a8b7b6a2..e84fc8d911 100644 --- a/include/simgrid/msg.h +++ b/include/simgrid/msg.h @@ -219,8 +219,6 @@ XBT_ATTRIB_DEPRECATED_v319("Use MSG_zone_get_hosts() instead: v3.19 will remove /************************** File handling ***********************************/ XBT_PUBLIC(sg_size_t) MSG_file_read(msg_file_t fd, sg_size_t size); XBT_PUBLIC(sg_size_t) MSG_file_write(msg_file_t fd, sg_size_t size); -XBT_PUBLIC(msg_error_t) MSG_file_rcopy(msg_file_t fd, msg_host_t host, const char* fullpath); -XBT_PUBLIC(msg_error_t) MSG_file_rmove(msg_file_t fd, msg_host_t host, const char* fullpath); /************************** Storage handling ***********************************/ XBT_PUBLIC(const char *) MSG_storage_get_name(msg_storage_t storage); XBT_PUBLIC(msg_storage_t) MSG_storage_get_by_name(const char *name); diff --git a/include/simgrid/plugins/file_system.h b/include/simgrid/plugins/file_system.h index fd1ab19832..2430be6cd4 100644 --- a/include/simgrid/plugins/file_system.h +++ b/include/simgrid/plugins/file_system.h @@ -25,6 +25,8 @@ XBT_PUBLIC(void) sg_file_seek(sg_file_t fd, sg_offset_t offset, int origin); XBT_PUBLIC(sg_size_t) sg_file_tell(sg_file_t fd); XBT_PUBLIC(void) sg_file_move(sg_file_t fd, const char* fullpath); XBT_PUBLIC(void) sg_file_unlink(sg_file_t fd); +XBT_PUBLIC(int) sg_file_rcopy(sg_file_t file, sg_host_t host, const char* fullpath); +XBT_PUBLIC(int) sg_file_rmove(sg_file_t file, sg_host_t host, const char* fullpath); XBT_PUBLIC(void*) sg_storage_get_data(sg_storage_t storage); XBT_PUBLIC(void) sg_storage_set_data(sg_storage_t storage, void* data); @@ -46,6 +48,8 @@ XBT_PUBLIC(xbt_dict_t) sg_host_get_storage_content(sg_host_t host); #define MSG_file_tell(fd) sg_file_tell(fd) #define MSG_file_move(fd, fullpath) sg_file_get_size(fd, fullpath) #define MSG_file_unlink(fd) sg_file_unlink(fd) +#define MSG_file_rcopy(file, host, fullpath) sg_file_rcopy(file, host, fullpath) +#define MSG_file_rmove(file, host, fullpath) sg_file_rmove(file, host, fullpath) #define MSG_storage_file_system_init() sg_storage_file_system_init() #define MSG_storage_get_free_size(st) sg_storage_get_size_free(st) diff --git a/src/msg/msg_io.cpp b/src/msg/msg_io.cpp index 2806561aad..8a8e8fba30 100644 --- a/src/msg/msg_io.cpp +++ b/src/msg/msg_io.cpp @@ -3,7 +3,6 @@ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ -#include "simgrid/s4u/Actor.hpp" #include "simgrid/s4u/Host.hpp" #include "simgrid/s4u/Storage.hpp" #include "src/msg/msg_private.hpp" @@ -106,73 +105,6 @@ sg_size_t MSG_file_write(msg_file_t fd, sg_size_t size) return write_size; } -/** - * \ingroup msg_file - * \brief Copy a file to another location on a remote host. - * \param file : the file to move - * \param host : the remote host where the file has to be copied - * \param fullpath : the complete path destination on the remote host - * \return If successful, the function returns MSG_OK. Otherwise, it returns MSG_TASK_CANCELED. - */ -msg_error_t MSG_file_rcopy (msg_file_t file, msg_host_t host, const char* fullpath) -{ - /* Find the host where the file is physically located and read it */ - msg_storage_t storage_src = file->localStorage; - msg_host_t src_host = storage_src->getHost(); - file->seek(0, SEEK_SET); - sg_size_t read_size = file->read(file->size()); - - /* Find the host that owns the storage where the file has to be copied */ - msg_storage_t storage_dest = nullptr; - msg_host_t dst_host; - size_t longest_prefix_length = 0; - - for (auto const& elm : host->getMountedStorages()) { - std::string mount_point = std::string(fullpath).substr(0, elm.first.size()); - if (mount_point == elm.first && elm.first.length() > longest_prefix_length) { - /* The current mount name is found in the full path and is bigger than the previous*/ - longest_prefix_length = elm.first.length(); - storage_dest = elm.second; - } - } - - if (storage_dest != nullptr) { - /* Mount point found, retrieve the host the storage is attached to */ - dst_host = storage_dest->getHost(); - }else{ - XBT_WARN("Can't find mount point for '%s' on destination host '%s'", fullpath, host->getCname()); - return MSG_TASK_CANCELED; - } - - XBT_DEBUG("Initiate data transfer of %llu bytes between %s and %s.", read_size, src_host->getCname(), - storage_dest->getHost()->getCname()); - msg_host_t m_host_list[] = {src_host, dst_host}; - double* flops_amount = new double[2]{0, 0}; - double* bytes_amount = new double[4]{0, static_cast(read_size), 0, 0}; - - simgrid::s4u::this_actor::parallel_execute(2, m_host_list, flops_amount, bytes_amount); - - /* Create file on remote host, write it and close it */ - msg_file_t fd = new simgrid::s4u::File(fullpath, dst_host, nullptr); - fd->write(read_size); - delete fd; - return MSG_OK; -} - -/** - * \ingroup msg_file - * \brief Move a file to another location on a remote host. - * \param file : the file to move - * \param host : the remote host where the file has to be moved - * \param fullpath : the complete path destination on the remote host - * \return If successful, the function returns MSG_OK. Otherwise, it returns MSG_TASK_CANCELED. - */ -msg_error_t MSG_file_rmove (msg_file_t file, msg_host_t host, const char* fullpath) -{ - msg_error_t res = MSG_file_rcopy(file, host, fullpath); - file->unlink(); - return res; -} /********************************* Storage **************************************/ /** @addtogroup msg_storage_management diff --git a/src/plugins/file_system/FileSystem.hpp b/src/plugins/file_system/FileSystem.hpp index 7711c272e8..7a75abeff1 100644 --- a/src/plugins/file_system/FileSystem.hpp +++ b/src/plugins/file_system/FileSystem.hpp @@ -52,6 +52,8 @@ public: /** Rename a file. WARNING: It is forbidden to move the file to another mount point */ void move(std::string fullpath); + int remoteCopy(sg_host_t host, const char* fullpath); + int remoteMove(sg_host_t host, const char* fullpath); int unlink(); /** Remove a file from the contents of a disk */ void dump(); diff --git a/src/plugins/file_system/s4u_FileSystem.cpp b/src/plugins/file_system/s4u_FileSystem.cpp index 7608580805..8f71c8bf4e 100644 --- a/src/plugins/file_system/s4u_FileSystem.cpp +++ b/src/plugins/file_system/s4u_FileSystem.cpp @@ -5,6 +5,7 @@ #include "xbt/log.h" +#include "simgrid/s4u/Actor.hpp" #include "simgrid/s4u/Host.hpp" #include "simgrid/s4u/Storage.hpp" #include "simgrid/simix.hpp" @@ -95,6 +96,7 @@ void File::dump() sg_size_t File::read(sg_size_t size) { + XBT_DEBUG("READ %s on disk '%s'", getPath(), localStorage->getCname()); // if the current position is close to the end of the file, we may not be able to read the requested size sg_size_t read_size = localStorage->read(std::min(size, size_ - current_position_)); @@ -195,6 +197,58 @@ int File::unlink() } } +int File::remoteCopy(sg_host_t host, const char* fullpath) +{ + /* Find the host where the file is physically located and read it */ + Storage* storage_src = localStorage; + Host* src_host = storage_src->getHost(); + seek(0, SEEK_SET); + sg_size_t read_size = read(size_); + + /* Find the host that owns the storage where the file has to be copied */ + Storage* storage_dest = nullptr; + Host* dst_host; + size_t longest_prefix_length = 0; + + for (auto const& elm : host->getMountedStorages()) { + std::string mount_point = std::string(fullpath).substr(0, elm.first.size()); + if (mount_point == elm.first && elm.first.length() > longest_prefix_length) { + /* The current mount name is found in the full path and is bigger than the previous*/ + longest_prefix_length = elm.first.length(); + storage_dest = elm.second; + } + } + + if (storage_dest != nullptr) { + /* Mount point found, retrieve the host the storage is attached to */ + dst_host = storage_dest->getHost(); + } else { + XBT_WARN("Can't find mount point for '%s' on destination host '%s'", fullpath, host->getCname()); + return -1; + } + + XBT_DEBUG("Initiate data transfer of %llu bytes between %s and %s.", read_size, src_host->getCname(), + storage_dest->getHost()->getCname()); + sg_host_t m_host_list[] = {src_host, dst_host}; + double* flops_amount = new double[2]{0, 0}; + double* bytes_amount = new double[4]{0, static_cast(read_size), 0, 0}; + + this_actor::parallel_execute(2, m_host_list, flops_amount, bytes_amount); + + /* Create file on remote host, write it and close it */ + File* fd = new File(fullpath, dst_host, nullptr); + fd->write(read_size); + delete fd; + return 0; +} + +int File::remoteMove(sg_host_t host, const char* fullpath) +{ + int res = remoteCopy(host, fullpath); + unlink(); + return res; +} + FileSystemStorageExt::FileSystemStorageExt(simgrid::s4u::Storage* ptr) { content_ = parseContent(ptr->getImpl()->content_name); @@ -337,6 +391,31 @@ void sg_file_unlink(sg_file_t fd) delete fd; } +/** + * \brief Copy a file to another location on a remote host. + * \param file : the file to move + * \param host : the remote host where the file has to be copied + * \param fullpath : the complete path destination on the remote host + * \return If successful, the function returns 0. Otherwise, it returns -1. + */ +int sg_file_rcopy(sg_file_t file, sg_host_t host, const char* fullpath) +{ + return file->remoteCopy(host, fullpath); +} + +/** + * \brief Move a file to another location on a remote host. + * \param file : the file to move + * \param host : the remote host where the file has to be moved + * \param fullpath : the complete path destination on the remote host + * \return If successful, the function returns 0. Otherwise, it returns -1. + */ +int sg_file_rmove(sg_file_t file, sg_host_t host, const char* fullpath) +{ + return file->remoteMove(host, fullpath); + ; +} + sg_size_t sg_storage_get_size_free(sg_storage_t st) { return st->extension()->getSize() - st->extension()->getUsedSize(); -- 2.20.1