From 6063afbddf453571e145b5236294c8631d9eecb2 Mon Sep 17 00:00:00 2001 From: Frederic Suter Date: Thu, 23 Mar 2017 12:22:14 +0100 Subject: [PATCH] start to mess up with storage + Storage size cannot change, do not ask surf all the time + replace internal dict by a map + use ifstream and c++ string tools --- examples/msg/io-storage/io-storage.tesh | 2 +- include/simgrid/msg.h | 1 + include/simgrid/s4u/storage.hpp | 1 + src/msg/msg_io.cpp | 11 ++-- src/s4u/s4u_storage.cpp | 3 +- src/surf/HostImpl.cpp | 17 +++--- src/surf/storage_interface.cpp | 58 +++++++++---------- src/surf/storage_interface.hpp | 17 ++---- src/surf/storage_n11.cpp | 22 +++---- src/surf/surf_c_bindings.cpp | 2 +- src/surf/surf_interface.cpp | 21 +++++++ src/surf/surf_private.h | 1 + .../storage_client_server.tesh | 4 +- 13 files changed, 92 insertions(+), 68 deletions(-) diff --git a/examples/msg/io-storage/io-storage.tesh b/examples/msg/io-storage/io-storage.tesh index cf79aa250e..90d6b07d1d 100644 --- a/examples/msg/io-storage/io-storage.tesh +++ b/examples/msg/io-storage/io-storage.tesh @@ -47,8 +47,8 @@ $ ${bindir:=.}/io-storage$EXEEXT ${srcdir:=.}/storage/storage.xml "--log=root.fm > [ 0.006000] (1:@denise) \Windows\winhlp32.exe size: 10752 bytes > [ 0.006000] (1:@denise) \Windows\setuperr.log size: 0 bytes > [ 0.006000] (1:@denise) \Windows\system.ini size: 219 bytes -> [ 0.006000] (1:@denise) \Windows\hapint.exe size: 382056 bytes > [ 0.006000] (1:@denise) \Windows\Professional.xml size: 31881 bytes +> [ 0.006000] (1:@denise) \Windows\hapint.exe size: 382056 bytes > [ 0.006000] (1:@denise) \Windows\regedit.exe size: 159232 bytes > [ 0.006000] (1:@denise) \Windows\setupact.log size: 101663 bytes > [ 0.006000] (1:@denise) \Windows\WindowsUpdate.log size: 1518934 bytes diff --git a/include/simgrid/msg.h b/include/simgrid/msg.h index 64596fddf3..c65ac44bb7 100644 --- a/include/simgrid/msg.h +++ b/include/simgrid/msg.h @@ -100,6 +100,7 @@ typedef xbt_dictelm_t msg_storage_t; struct msg_storage_priv { const char *hostname; + size_t size; void *data; }; typedef struct msg_storage_priv s_msg_storage_priv_t; diff --git a/include/simgrid/s4u/storage.hpp b/include/simgrid/s4u/storage.hpp index b99651b339..4297f7c701 100644 --- a/include/simgrid/s4u/storage.hpp +++ b/include/simgrid/s4u/storage.hpp @@ -54,6 +54,7 @@ private: static boost::unordered_map *storages_; std::string name_; + size_t size_; smx_storage_t pimpl_ = nullptr; void *userdata_ = nullptr; }; diff --git a/src/msg/msg_io.cpp b/src/msg/msg_io.cpp index 0aa6823f34..00f5105b2d 100644 --- a/src/msg/msg_io.cpp +++ b/src/msg/msg_io.cpp @@ -443,10 +443,12 @@ void __MSG_file_destroy(msg_file_priv_t file) { msg_storage_t __MSG_storage_create(smx_storage_t storage) { - const char *name = SIMIX_storage_get_name(storage); - const char *host = SIMIX_storage_get_host(storage); + const char* name = SIMIX_storage_get_name(storage); msg_storage_priv_t storage_private = xbt_new0(s_msg_storage_priv_t, 1); - storage_private->hostname = host; + + storage_private->hostname = SIMIX_storage_get_host(storage); + storage_private->size = SIMIX_storage_get_size(storage); + xbt_lib_set(storage_lib,name,MSG_STORAGE_LEVEL,storage_private); return xbt_lib_get_elm_or_null(storage_lib, name); } @@ -594,7 +596,8 @@ xbt_dict_t MSG_storage_get_content(msg_storage_t storage) */ sg_size_t MSG_storage_get_size(msg_storage_t storage) { - return SIMIX_storage_get_size(storage); + msg_storage_priv_t priv = MSG_storage_priv(storage); + return priv->size; } /** \ingroup msg_storage_management diff --git a/src/s4u/s4u_storage.cpp b/src/s4u/s4u_storage.cpp index c6e33340c2..fae9e32aba 100644 --- a/src/s4u/s4u_storage.cpp +++ b/src/s4u/s4u_storage.cpp @@ -16,6 +16,7 @@ boost::unordered_map *Storage::storages_ = new boost::u Storage::Storage(std::string name, smx_storage_t inferior) : name_(name), pimpl_(inferior) { + size_ = SIMIX_storage_get_size(pimpl_); storages_->insert({name, this}); } @@ -49,7 +50,7 @@ sg_size_t Storage::sizeUsed() { return simcall_storage_get_used_size(pimpl_); } sg_size_t Storage::size() { - return SIMIX_storage_get_size(pimpl_); + return size_; } } /* namespace s4u */ diff --git a/src/surf/HostImpl.cpp b/src/surf/HostImpl.cpp index a692a0317e..81faf9574b 100644 --- a/src/surf/HostImpl.cpp +++ b/src/surf/HostImpl.cpp @@ -220,7 +220,7 @@ int HostImpl::unlink(surf_file_t fd) simgrid::surf::Storage* st = findStorageOnMountList(fd->mount); /* Check if the file is on this storage */ - if (!xbt_dict_get_or_null(st->content_, fd->name)) { + if (st->content_->find(fd->name) == st->content_->end()) { XBT_WARN("File %s is not on disk %s. Impossible to unlink", fd->name, st->cname()); return -1; } else { @@ -228,7 +228,9 @@ int HostImpl::unlink(surf_file_t fd) st->usedSize_ -= fd->size; // Remove the file from storage - xbt_dict_remove(st->content_, fd->name); + sg_size_t* psize = st->content_->at(fd->name); + delete psize; + st->content_->erase(fd->name); xbt_free(fd->name); xbt_free(fd->mount); @@ -265,7 +267,6 @@ sg_size_t HostImpl::fileTell(surf_file_t fd) int HostImpl::fileSeek(surf_file_t fd, sg_offset_t offset, int origin) { - switch (origin) { case SEEK_SET: fd->current_position = offset; @@ -285,13 +286,15 @@ int HostImpl::fileMove(surf_file_t fd, const char* fullpath) { /* Check if the new full path is on the same mount point */ if (!strncmp((const char*)fd->mount, fullpath, strlen(fd->mount))) { - sg_size_t* psize = (sg_size_t*)xbt_dict_get_or_null(findStorageOnMountList(fd->mount)->content_, fd->name); - if (psize) { // src file exists + std::map* content = findStorageOnMountList(fd->mount)->content_; + if (content->find(fd->name) != content->end()) { // src file exists + sg_size_t* psize = content->at(std::string(fd->name)); sg_size_t* new_psize = xbt_new(sg_size_t, 1); *new_psize = *psize; - xbt_dict_remove(findStorageOnMountList(fd->mount)->content_, fd->name); + delete psize; + content->erase(fd->name); std::string path = std::string(fullpath).substr(strlen(fd->mount), strlen(fullpath)); - xbt_dict_set(findStorageOnMountList(fd->mount)->content_, path.c_str(), new_psize, nullptr); + content->insert({path.c_str(), new_psize}); XBT_DEBUG("Move file from %s to %s, size '%llu'", fd->name, fullpath, *psize); return 0; } else { diff --git a/src/surf/storage_interface.cpp b/src/surf/storage_interface.cpp index 7fe5fd05a1..799f67c41d 100644 --- a/src/surf/storage_interface.cpp +++ b/src/surf/storage_interface.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015. The SimGrid Team. +/* Copyright (c) 2013-2017. The SimGrid Team. * All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it @@ -7,7 +7,10 @@ #include "storage_interface.hpp" #include "surf_private.h" #include "xbt/file.h" /* xbt_getline */ +#include #include +#include +#include XBT_LOG_NEW_DEFAULT_SUBCATEGORY(surf_storage, surf, "Logging specific to the SURF storage module"); @@ -64,7 +67,7 @@ Storage::Storage(Model* model, const char* name, lmm_system_t maxminSystem, doub , writeActions_(std::vector()) { content_ = parseContent(content_name); - attach_ = xbt_strdup(attach); + attach_ = xbt_strdup(attach); turnOn(); XBT_DEBUG("Create resource with Bconnection '%f' Bread '%f' Bwrite '%f' and Size '%llu'", bconnection, bread, bwrite, size); constraintRead_ = lmm_constraint_new(maxminSystem, this, bread); @@ -73,41 +76,43 @@ Storage::Storage(Model* model, const char* name, lmm_system_t maxminSystem, doub Storage::~Storage(){ storageDestructedCallbacks(this); - xbt_dict_free(&content_); + if (content_ != nullptr) { + for (auto entry : *content_) + delete entry.second; + delete content_; + } free(typeId_); free(contentType_); free(attach_); } -xbt_dict_t Storage::parseContent(const char *filename) +std::map* Storage::parseContent(const char* filename) { usedSize_ = 0; if ((!filename) || (strcmp(filename, "") == 0)) return nullptr; - xbt_dict_t parse_content = xbt_dict_new_homogeneous(xbt_free_f); - - FILE *file = surf_fopen(filename, "r"); - xbt_assert(file, "Cannot open file '%s' (path=%s)", filename, (boost::join(surf_path, ":")).c_str()); + std::map* parse_content = new std::map(); - char *line = nullptr; - size_t len = 0; - ssize_t read; - char path[1024]; - sg_size_t size; + std::ifstream* fs = surf_ifsopen(filename); - while ((read = xbt_getline(&line, &len, file)) != -1) { - if (read){ - xbt_assert(sscanf(line,"%s %llu", path, &size) == 2, "Parse error in %s: %s",filename,line); + std::string line; + std::vector tokens; + do { + std::getline(*fs, line); + boost::trim(line); + if (line.length() > 0) { + boost::split(tokens, line, boost::is_any_of(" \t"), boost::token_compress_on); + xbt_assert(tokens.size() == 2, "Parse error in %s: %s", filename, line.c_str()); + sg_size_t size = std::stoull(tokens.at(1)); usedSize_ += size; - sg_size_t *psize = xbt_new(sg_size_t, 1); + sg_size_t* psize = new sg_size_t; *psize = size; - xbt_dict_set(parse_content,path,psize,nullptr); + parse_content->insert({tokens.front(), psize}); } - } - free(line); - fclose(file); + } while (!fs->eof()); + delete fs; return parse_content; } @@ -140,20 +145,13 @@ xbt_dict_t Storage::getContent() /* For the moment this action has no cost, but in the future we could take in account access latency of the disk */ xbt_dict_t content_dict = xbt_dict_new_homogeneous(nullptr); - xbt_dict_cursor_t cursor = nullptr; - char *file; - sg_size_t *psize; - xbt_dict_foreach(content_, cursor, file, psize){ - xbt_dict_set(content_dict,file,psize,nullptr); + for (auto entry : *content_) { + xbt_dict_set(content_dict, entry.first.c_str(), entry.second, nullptr); } return content_dict; } -sg_size_t Storage::getSize(){ - return size_; -} - sg_size_t Storage::getFreeSize(){ return size_ - usedSize_; } diff --git a/src/surf/storage_interface.hpp b/src/surf/storage_interface.hpp index e2b39efafc..b1d56c76e3 100644 --- a/src/surf/storage_interface.hpp +++ b/src/surf/storage_interface.hpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2004-2015. The SimGrid Team. +/* Copyright (c) 2004-2017. The SimGrid Team. * All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it @@ -7,8 +7,9 @@ #include #include -#include "surf_interface.hpp" #include "src/surf/PropertyHolder.hpp" +#include "surf_interface.hpp" +#include #ifndef STORAGE_INTERFACE_HPP_ #define STORAGE_INTERFACE_HPP_ @@ -101,7 +102,7 @@ public: void turnOn() override; void turnOff() override; - xbt_dict_t content_; + std::map* content_; char* contentType_; sg_size_t size_; sg_size_t usedSize_; @@ -151,13 +152,6 @@ public: */ virtual xbt_dict_t getContent(); - /** - * @brief Get the size in bytes of the current Storage - * - * @return The size in bytes of the current Storage - */ - virtual sg_size_t getSize(); - /** * @brief Get the available size in bytes of the current Storage * @@ -172,8 +166,7 @@ public: */ virtual sg_size_t getUsedSize(); - - xbt_dict_t parseContent(const char *filename); + std::map* parseContent(const char* filename); std::vector writeActions_; diff --git a/src/surf/storage_n11.cpp b/src/surf/storage_n11.cpp index b89e1956ee..5e3b24b9f5 100644 --- a/src/surf/storage_n11.cpp +++ b/src/surf/storage_n11.cpp @@ -146,10 +146,12 @@ void StorageN11Model::updateActionsState(double /*now*/, double delta) // which becomes the new file size action->file_->size = action->file_->current_position; - sg_size_t *psize = xbt_new(sg_size_t,1); - *psize = action->file_->size; - xbt_dict_t content_dict = action->storage_->content_; - xbt_dict_set(content_dict, action->file_->name, psize, nullptr); + sg_size_t* psize = new sg_size_t; + *psize = action->file_->size; + std::map* content_dict = action->storage_->content_; + auto entry = content_dict->find(action->file_->name); + delete entry->second; + entry->second = psize; } action->updateRemains(lmm_variable_getvalue(action->getVariable()) * delta); @@ -186,15 +188,15 @@ StorageAction *StorageN11::open(const char* mount, const char* path) XBT_DEBUG("\tOpen file '%s'",path); sg_size_t size; - sg_size_t* psize = (sg_size_t*)xbt_dict_get_or_null(content_, path); + sg_size_t* psize = nullptr; // if file does not exist create an empty file - if(psize) - size = *psize; + if (content_->find(path) != content_->end()) + size = *(content_->at(path)); else { - psize = xbt_new(sg_size_t,1); - size = 0; + psize = new sg_size_t; + size = 0; *psize = size; - xbt_dict_set(content_, path, psize, nullptr); + content_->insert({path, psize}); XBT_DEBUG("File '%s' was not found, file created.",path); } surf_file_t file = xbt_new0(s_surf_file_t,1); diff --git a/src/surf/surf_c_bindings.cpp b/src/surf/surf_c_bindings.cpp index 1b00946611..502eaccb05 100644 --- a/src/surf/surf_c_bindings.cpp +++ b/src/surf/surf_c_bindings.cpp @@ -213,7 +213,7 @@ xbt_dict_t surf_storage_get_content(surf_resource_t resource){ } sg_size_t surf_storage_get_size(surf_resource_t resource){ - return static_cast(surf_storage_resource_priv(resource))->getSize(); + return static_cast(surf_storage_resource_priv(resource))->size_; } sg_size_t surf_storage_get_free_size(surf_resource_t resource){ diff --git a/src/surf/surf_interface.cpp b/src/surf/surf_interface.cpp index f606e08e11..be75f0a219 100644 --- a/src/surf/surf_interface.cpp +++ b/src/surf/surf_interface.cpp @@ -16,6 +16,7 @@ #include "src/simix/smx_host_private.h" #include "src/surf/HostImpl.hpp" #include "surf_private.h" +#include #include XBT_LOG_NEW_CATEGORY(surf, "All SURF categories"); @@ -127,6 +128,26 @@ double surf_get_clock() # define FILE_DELIM "/" /* FIXME: move to better location */ #endif +std::ifstream* surf_ifsopen(const char* name) +{ + std::ifstream* fs = new std::ifstream(); + xbt_assert(name); + if (__surf_is_absolute_file_path(name)) { /* don't mess with absolute file names */ + fs->open(name, std::ifstream::in); + } + /* search relative files in the path */ + for (auto path_elm : surf_path) { + std::string buff = path_elm + FILE_DELIM + name; + fs->open(buff.c_str(), std::ifstream::in); + + if (!fs->fail()) { + XBT_DEBUG("Found file at %s", buff.c_str()); + return fs; + } + } + + return fs; +} FILE *surf_fopen(const char *name, const char *mode) { char *buff; diff --git a/src/surf/surf_private.h b/src/surf/surf_private.h index abf66e4f55..5a53d7ada7 100644 --- a/src/surf/surf_private.h +++ b/src/surf/surf_private.h @@ -33,6 +33,7 @@ typedef enum { /* Generic functions common to all models */ XBT_PRIVATE FILE *surf_fopen(const char *name, const char *mode); +XBT_PRIVATE std::ifstream* surf_ifsopen(const char* name); /* The __surf_is_absolute_file_path() returns 1 if * file_path is a absolute file path, in the other diff --git a/teshsuite/msg/storage_client_server/storage_client_server.tesh b/teshsuite/msg/storage_client_server/storage_client_server.tesh index 6399cd7c45..b1f0d740ec 100644 --- a/teshsuite/msg/storage_client_server/storage_client_server.tesh +++ b/teshsuite/msg/storage_client_server/storage_client_server.tesh @@ -15,8 +15,8 @@ $ ./storage_client_server$EXEEXT ${srcdir:=.}/../../../examples/platforms/storag > [ 0.000000] (1:server@alice) \Windows\winhlp32.exe size: 10752 bytes > [ 0.000000] (1:server@alice) \Windows\setuperr.log size: 0 bytes > [ 0.000000] (1:server@alice) \Windows\system.ini size: 219 bytes -> [ 0.000000] (1:server@alice) \Windows\hapint.exe size: 382056 bytes > [ 0.000000] (1:server@alice) \Windows\Professional.xml size: 31881 bytes +> [ 0.000000] (1:server@alice) \Windows\hapint.exe size: 382056 bytes > [ 0.000000] (1:server@alice) \Windows\regedit.exe size: 159232 bytes > [ 0.000000] (1:server@alice) \Windows\setupact.log size: 101663 bytes > [ 0.000000] (1:server@alice) \Windows\WindowsUpdate.log size: 1518934 bytes @@ -72,8 +72,8 @@ $ ./storage_client_server$EXEEXT ${srcdir:=.}/../../../examples/platforms/storag > [ 1.207952] (1:server@alice) \Windows\winhlp32.exe size: 10752 bytes > [ 1.207952] (1:server@alice) \Windows\setuperr.log size: 0 bytes > [ 1.207952] (1:server@alice) \Windows\system.ini size: 219 bytes -> [ 1.207952] (1:server@alice) \Windows\hapint.exe size: 382056 bytes > [ 1.207952] (1:server@alice) \Windows\Professional.xml size: 31881 bytes +> [ 1.207952] (1:server@alice) \Windows\hapint.exe size: 382056 bytes > [ 1.207952] (1:server@alice) \Windows\regedit.exe size: 159232 bytes > [ 1.207952] (1:server@alice) \Windows\setupact.log size: 101663 bytes > [ 1.207952] (1:server@alice) \Windows\WindowsUpdate.log size: 1518934 bytes -- 2.20.1