From 04d3d36e9e87d194bee4f88859f96d66975cce31 Mon Sep 17 00:00:00 2001 From: Frederic Suter Date: Thu, 30 Nov 2017 16:08:57 +0100 Subject: [PATCH] MSG_file_open/close now are in the plugin as long as the Host extension to handle maximum number of open files only the remote operation remain outside of the plugin. they imply some parallel tasks (pushed send) that require a bit of thinking --- include/simgrid/msg.h | 3 -- include/simgrid/plugins/file_system.h | 4 ++ src/msg/msg_global.cpp | 5 --- src/msg/msg_host.cpp | 4 -- src/msg/msg_io.cpp | 52 +--------------------- src/msg/msg_private.hpp | 10 ----- src/plugins/file_system/FileSystem.hpp | 14 ++++-- src/plugins/file_system/s4u_FileSystem.cpp | 48 +++++++++++++++++--- 8 files changed, 59 insertions(+), 81 deletions(-) diff --git a/include/simgrid/msg.h b/include/simgrid/msg.h index 65a5e93497..16a8b7b6a2 100644 --- a/include/simgrid/msg.h +++ b/include/simgrid/msg.h @@ -219,9 +219,6 @@ XBT_ATTRIB_DEPRECATED_v319("Use MSG_zone_get_hosts() instead: v3.19 will remove /************************** File handling ***********************************/ XBT_PUBLIC(sg_size_t) MSG_file_read(msg_file_t fd, sg_size_t size); XBT_PUBLIC(sg_size_t) MSG_file_write(msg_file_t fd, sg_size_t size); -XBT_PUBLIC(msg_file_t) MSG_file_open(const char* fullpath, void* data); -XBT_PUBLIC(int) MSG_file_close(msg_file_t fd); -XBT_PUBLIC(msg_error_t) MSG_file_move(msg_file_t fd, const char* fullpath); XBT_PUBLIC(msg_error_t) MSG_file_rcopy(msg_file_t fd, msg_host_t host, const char* fullpath); XBT_PUBLIC(msg_error_t) MSG_file_rmove(msg_file_t fd, msg_host_t host, const char* fullpath); /************************** Storage handling ***********************************/ diff --git a/include/simgrid/plugins/file_system.h b/include/simgrid/plugins/file_system.h index d1339db704..fd1ab19832 100644 --- a/include/simgrid/plugins/file_system.h +++ b/include/simgrid/plugins/file_system.h @@ -13,6 +13,8 @@ SG_BEGIN_DECL() XBT_PUBLIC(void) sg_storage_file_system_init(); +XBT_PUBLIC(sg_file_t) sg_file_open(const char* fullpath, void* data); +XBT_PUBLIC(void) sg_file_close(sg_file_t fd); XBT_PUBLIC(const char*) sg_file_get_name(sg_file_t fd); XBT_PUBLIC(sg_size_t) sg_file_get_size(sg_file_t fd); @@ -33,6 +35,8 @@ XBT_PUBLIC(xbt_dict_t) sg_storage_get_content(sg_storage_t storage); XBT_PUBLIC(xbt_dict_t) sg_host_get_storage_content(sg_host_t host); +#define MSG_file_open(fullpath, data) sg_file_open(fullpath, data) +#define MSG_file_close(fd) sg_file_close(fd) #define MSG_file_get_name(fd) sg_file_get_name(fd) #define MSG_file_get_size(fd) sg_file_get_size(fd) #define MSG_file_dump(fd) sg_file_dump(fd) diff --git a/src/msg/msg_global.cpp b/src/msg/msg_global.cpp index 5651178070..aad562018c 100644 --- a/src/msg/msg_global.cpp +++ b/src/msg/msg_global.cpp @@ -49,11 +49,6 @@ void MSG_init_nocheck(int *argc, char **argv) { SIMIX_function_register_process_create(MSG_process_create_from_SIMIX); SIMIX_function_register_process_cleanup(MSG_process_cleanup_from_SIMIX); - - simgrid::MsgHostExt::EXTENSION_ID = simgrid::s4u::Host::extension_create(); - simgrid::s4u::Host::onCreation.connect([](simgrid::s4u::Host& host) { - host.extension_set(new simgrid::MsgHostExt()); - }); } if(MC_is_active()){ diff --git a/src/msg/msg_host.cpp b/src/msg/msg_host.cpp index 7da4076bf2..5db2de1d5c 100644 --- a/src/msg/msg_host.cpp +++ b/src/msg/msg_host.cpp @@ -11,12 +11,8 @@ XBT_LOG_EXTERNAL_DEFAULT_CATEGORY(msg); -simgrid::xbt::Extension simgrid::MsgHostExt::EXTENSION_ID; - extern "C" { -int sg_storage_max_file_descriptors = 1024; - /** @addtogroup m_host_management * (#msg_host_t) and the functions for managing it. * diff --git a/src/msg/msg_io.cpp b/src/msg/msg_io.cpp index c454f18337..31c6ca082f 100644 --- a/src/msg/msg_io.cpp +++ b/src/msg/msg_io.cpp @@ -19,24 +19,6 @@ extern "C" { * \see #msg_file_t */ -static int MSG_host_get_file_descriptor_id(msg_host_t host) -{ - simgrid::MsgHostExt* priv = host->extension(); - if (priv->file_descriptor_table == nullptr) { - priv->file_descriptor_table = new std::vector(sg_storage_max_file_descriptors); - std::iota(priv->file_descriptor_table->rbegin(), priv->file_descriptor_table->rend(), 0); // Fill with ..., 1, 0. - } - xbt_assert(not priv->file_descriptor_table->empty(), "Too much files are opened! Some have to be closed."); - int desc = priv->file_descriptor_table->back(); - priv->file_descriptor_table->pop_back(); - return desc; -} - -static void MSG_host_release_file_descriptor_id(msg_host_t host, int id) -{ - host->extension()->file_descriptor_table->push_back(id); -} - /** \ingroup msg_file * \brief Read a file (local or remote) * @@ -123,36 +105,6 @@ sg_size_t MSG_file_write(msg_file_t fd, sg_size_t size) return write_size; } -/** \ingroup msg_file - * \brief Opens the file whose name is the string pointed to by path - * - * \param fullpath is the file location on the storage - * \param data user data to attach to the file - * - * \return An #msg_file_t associated to the file - */ -msg_file_t MSG_file_open(const char* fullpath, void* data) -{ - msg_file_t fd = new simgrid::s4u::File(fullpath, MSG_host_self()); - fd->desc_id = MSG_host_get_file_descriptor_id(MSG_host_self()); - fd->setUserdata(data); - return fd; -} - -/** \ingroup msg_file - * \brief Close the file - * - * \param fd is the file to close - * \return 0 on success or 1 on error - */ -int MSG_file_close(msg_file_t fd) -{ - MSG_host_release_file_descriptor_id(MSG_host_self(), fd->desc_id); - delete fd; - - return MSG_OK; -} - /** * \ingroup msg_file * \brief Copy a file to another location on a remote host. @@ -166,7 +118,7 @@ msg_error_t MSG_file_rcopy (msg_file_t file, msg_host_t host, const char* fullpa /* Find the host where the file is physically located and read it */ msg_storage_t storage_src = file->localStorage; msg_host_t src_host = storage_src->getHost(); - MSG_file_seek(file, 0, SEEK_SET); + file->seek(0, SEEK_SET); sg_size_t read_size = file->read(file->size()); /* Find the host that owns the storage where the file has to be copied */ @@ -229,7 +181,7 @@ msg_error_t MSG_file_rcopy (msg_file_t file, msg_host_t host, const char* fullpa msg_error_t MSG_file_rmove (msg_file_t file, msg_host_t host, const char* fullpath) { msg_error_t res = MSG_file_rcopy(file, host, fullpath); - MSG_file_unlink(file); + file->unlink(); return res; } diff --git a/src/msg/msg_private.hpp b/src/msg/msg_private.hpp index 62b4285340..6f2539fea7 100644 --- a/src/msg/msg_private.hpp +++ b/src/msg/msg_private.hpp @@ -15,16 +15,6 @@ #include /**************** datatypes **********************************/ -/**************************** Host Extension *********************************/ -namespace simgrid { -class MsgHostExt { -public: - static simgrid::xbt::Extension EXTENSION_ID; - - ~MsgHostExt() { delete file_descriptor_table; } - std::vector* file_descriptor_table = nullptr; // Created lazily on need -}; -} /********************************* Task **************************************/ struct s_simdata_task_t { diff --git a/src/plugins/file_system/FileSystem.hpp b/src/plugins/file_system/FileSystem.hpp index 205c879a76..7711c272e8 100644 --- a/src/plugins/file_system/FileSystem.hpp +++ b/src/plugins/file_system/FileSystem.hpp @@ -29,7 +29,7 @@ XBT_PUBLIC_CLASS File public: File(std::string fullpath, void* userdata); File(std::string fullpath, sg_host_t host, void* userdata); - ~File() = default; + ~File(); /** Retrieves the path to the file */ const char* getPath() { return fullpath_.c_str(); } @@ -70,8 +70,8 @@ private: class FileSystemStorageExt { public: - static simgrid::xbt::Extension EXTENSION_ID; - explicit FileSystemStorageExt(simgrid::s4u::Storage* ptr); + static simgrid::xbt::Extension EXTENSION_ID; + explicit FileSystemStorageExt(Storage* ptr); ~FileSystemStorageExt(); std::map* parseContent(std::string filename); std::map* getContent() { return content_; } @@ -84,6 +84,14 @@ private: sg_size_t usedSize_ = 0; sg_size_t size_ = 0; }; + +class FileDescriptorHostExt { +public: + static simgrid::xbt::Extension EXTENSION_ID; + FileDescriptorHostExt() = default; + ~FileDescriptorHostExt() { delete file_descriptor_table; } + std::vector* file_descriptor_table = nullptr; // Created lazily on need +}; } } // namespace simgrid::s4u diff --git a/src/plugins/file_system/s4u_FileSystem.cpp b/src/plugins/file_system/s4u_FileSystem.cpp index b5460bf847..d62f41723b 100644 --- a/src/plugins/file_system/s4u_FileSystem.cpp +++ b/src/plugins/file_system/s4u_FileSystem.cpp @@ -18,10 +18,12 @@ #include XBT_LOG_NEW_DEFAULT_CATEGORY(s4u_file, "S4U files"); +int sg_storage_max_file_descriptors = 1024; namespace simgrid { namespace s4u { -simgrid::xbt::Extension FileSystemStorageExt::EXTENSION_ID; +simgrid::xbt::Extension FileSystemStorageExt::EXTENSION_ID; +simgrid::xbt::Extension FileDescriptorHostExt::EXTENSION_ID; File::File(std::string fullpath, void* userdata) : File(fullpath, Host::current(), userdata){}; @@ -50,6 +52,16 @@ File::File(std::string fullpath, sg_host_t host, void* userdata) : fullpath_(ful localStorage = st; + // assign a file descriptor id to the newly opened File + FileDescriptorHostExt* ext = host->extension(); + if (ext->file_descriptor_table == nullptr) { + ext->file_descriptor_table = new std::vector(sg_storage_max_file_descriptors); + std::iota(ext->file_descriptor_table->rbegin(), ext->file_descriptor_table->rend(), 0); // Fill with ..., 1, 0. + } + xbt_assert(not ext->file_descriptor_table->empty(), "Too much files are opened! Some have to be closed."); + desc_id = ext->file_descriptor_table->back(); + ext->file_descriptor_table->pop_back(); + XBT_DEBUG("\tOpen file '%s'", path_.c_str()); std::map* content = localStorage->extension()->getContent(); // if file does not exist create an empty file @@ -63,6 +75,11 @@ File::File(std::string fullpath, sg_host_t host, void* userdata) : fullpath_(ful } } +File::~File() +{ + Host::current()->extension()->file_descriptor_table->push_back(desc_id); +} + void File::dump() { XBT_INFO("File Descriptor information:\n" @@ -218,6 +235,7 @@ std::map* FileSystemStorageExt::parseContent(std::string } using simgrid::s4u::FileSystemStorageExt; +using simgrid::s4u::FileDescriptorHostExt; static void onStorageCreation(simgrid::s4u::Storage& st) { @@ -229,18 +247,36 @@ static void onStorageDestruction(simgrid::s4u::Storage& st) delete st.extension(); } +static void onHostCreation(simgrid::s4u::Host& host) +{ + host.extension_set(new FileDescriptorHostExt()); +} + /* **************************** Public interface *************************** */ SG_BEGIN_DECL() void sg_storage_file_system_init() { - if (FileSystemStorageExt::EXTENSION_ID.valid()) - return; + if (not FileSystemStorageExt::EXTENSION_ID.valid()) { + FileSystemStorageExt::EXTENSION_ID = simgrid::s4u::Storage::extension_create(); + simgrid::s4u::Storage::onCreation.connect(&onStorageCreation); + simgrid::s4u::Storage::onDestruction.connect(&onStorageDestruction); + } + + if (not FileDescriptorHostExt::EXTENSION_ID.valid()) { + FileDescriptorHostExt::EXTENSION_ID = simgrid::s4u::Host::extension_create(); + simgrid::s4u::Host::onCreation.connect(&onHostCreation); + } +} - FileSystemStorageExt::EXTENSION_ID = simgrid::s4u::Storage::extension_create(); +sg_file_t sg_file_open(const char* fullpath, void* data) +{ + return new simgrid::s4u::File(fullpath, data); +} - simgrid::s4u::Storage::onCreation.connect(&onStorageCreation); - simgrid::s4u::Storage::onDestruction.connect(&onStorageDestruction); +void sg_file_close(sg_file_t fd) +{ + delete fd; } const char* sg_file_get_name(sg_file_t fd) -- 2.20.1