X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/198b09ec16ca1b8fc05053bcae9e75c0ad689711..22ad42bc0e39c33058fa8f47d3b41966a45ffa80:/src/smpi/smpi_bench.cpp diff --git a/src/smpi/smpi_bench.cpp b/src/smpi/smpi_bench.cpp index e9b4773355..a4f05211ca 100644 --- a/src/smpi/smpi_bench.cpp +++ b/src/smpi/smpi_bench.cpp @@ -1,5 +1,4 @@ -/* Copyright (c) 2007, 2009-2015. The SimGrid Team. - * All rights reserved. */ +/* Copyright (c) 2007, 2009-2017. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ @@ -12,6 +11,7 @@ #include "src/internal_config.h" #include "private.h" #include "private.hpp" +#include #include "xbt/dict.h" #include "xbt/sysdep.h" #include "xbt/ex.h" @@ -20,11 +20,11 @@ #include "simgrid/modelchecker.h" #include "src/mc/mc_replay.h" +#include #ifndef WIN32 #include #endif #include -#include #include #include #include // sqrt @@ -32,6 +32,10 @@ #include #include +#if HAVE_PAPI +#include +#endif + #ifndef MAP_ANONYMOUS #define MAP_ANONYMOUS MAP_ANON #endif @@ -75,7 +79,7 @@ xbt_dict_t samples = nullptr; /* Allocated on first use */ xbt_dict_t calls = nullptr; /* Allocated on first use */ double smpi_cpu_threshold; -double smpi_running_power; +double smpi_host_speed; int smpi_loaded_page = -1; char* smpi_start_data_exe = nullptr; @@ -93,10 +97,10 @@ namespace { class smpi_source_location { public: smpi_source_location(const char* filename, int line) - : filename(filename), filename_length(strlen(filename)), line(line) {} + : filename(xbt_strdup(filename)), filename_length(strlen(filename)), line(line) {} /** Pointer to a static string containing the file name */ - const char* filename = nullptr; + char* filename = nullptr; int filename_length = 0; int line = 0; @@ -160,7 +164,6 @@ static size_t shm_size(int fd) { #ifndef WIN32 static void* shm_map(int fd, size_t size, shared_data_key_type* data) { - void* mem; char loc[PTR_STRLEN]; shared_metadata_t meta; @@ -168,9 +171,14 @@ static void* shm_map(int fd, size_t size, shared_data_key_type* data) { xbt_die("Could not truncate fd %d to %zu: %s", fd, size, strerror(errno)); } - mem = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + void* mem = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if(mem == MAP_FAILED) { - xbt_die("Could not map fd %d: %s", fd, strerror(errno)); + xbt_die( + "Failed to map fd %d with size %zu: %s\n" + "If you are running a lot of ranks, you may be exceeding the amount of mappings allowed per process.\n" + "On Linux systems, change this value with sudo sysctl -w vm.max_map_count=newvalue (default value: 65536)\n" + "Please see http://simgrid.gforge.inria.fr/simgrid/latest/doc/html/options.html#options_virt for more info.", + fd, size, strerror(errno)); } snprintf(loc, PTR_STRLEN, "%p", mem); meta.size = size; @@ -181,7 +189,7 @@ static void* shm_map(int fd, size_t size, shared_data_key_type* data) { } #endif -void smpi_bench_destroy(void) +void smpi_bench_destroy() { allocs.clear(); allocs_metadata.clear(); @@ -202,9 +210,8 @@ void smpi_execute_(double *duration) } void smpi_execute_flops(double flops) { - smx_synchro_t action; XBT_DEBUG("Handle real computation time: %f flops", flops); - action = simcall_execution_start("computation", flops, 1, 0, 0); + smx_activity_t action = simcall_execution_start("computation", flops, 1, 0); simcall_set_category (action, TRACE_internal_smpi_get_category()); simcall_execution_wait(action); smpi_switch_data_segment(smpi_process_index()); @@ -214,12 +221,13 @@ void smpi_execute(double duration) { if (duration >= smpi_cpu_threshold) { XBT_DEBUG("Sleep for %g to handle real computation time", duration); - double flops = duration * smpi_running_power; + double flops = duration * smpi_host_speed; int rank = smpi_process_index(); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); extra->type=TRACING_COMPUTING; extra->comp_size=flops; TRACE_smpi_computing_in(rank, extra); + smpi_execute_flops(flops); TRACE_smpi_computing_out(rank); @@ -230,7 +238,7 @@ void smpi_execute(double duration) } } -void smpi_bench_begin(void) +void smpi_bench_begin() { if (smpi_privatize_global_variables) { smpi_switch_data_segment(smpi_process_index()); @@ -239,18 +247,54 @@ void smpi_bench_begin(void) if (MC_is_active() || MC_record_replay_is_active()) return; +#if HAVE_PAPI + if (xbt_cfg_get_string("smpi/papi-events")[0] != '\0') { + int event_set = smpi_process_papi_event_set(); + // PAPI_start sets everything to 0! See man(3) PAPI_start + if (PAPI_LOW_LEVEL_INITED == PAPI_is_initialized()) { + if (PAPI_start(event_set) != PAPI_OK) { + // TODO This needs some proper handling. + XBT_CRITICAL("Could not start PAPI counters.\n"); + xbt_die("Error."); + } + } + } +#endif xbt_os_threadtimer_start(smpi_process_timer()); } -void smpi_bench_end(void) +void smpi_bench_end() { - if (MC_is_active() || MC_record_replay_is_active()) return; double speedup = 1; xbt_os_timer_t timer = smpi_process_timer(); xbt_os_threadtimer_stop(timer); + +#if HAVE_PAPI + /** + * An MPI function has been called and now is the right time to update + * our PAPI counters for this process. + */ + if (xbt_cfg_get_string("smpi/papi-events")[0] != '\0') { + papi_counter_t& counter_data = smpi_process_papi_counters(); + int event_set = smpi_process_papi_event_set(); + std::vector event_values = std::vector(counter_data.size()); + + if (PAPI_stop(event_set, &event_values[0]) != PAPI_OK) { // Error + XBT_CRITICAL("Could not stop PAPI counters.\n"); + xbt_die("Error."); + } else { + for (unsigned int i = 0; i < counter_data.size(); i++) { + counter_data[i].second += event_values[i]; + // XBT_DEBUG("[%i] PAPI: Counter %s: Value is now %lli (got increment by %lli\n", smpi_process_index(), + // counter_data[i].first.c_str(), counter_data[i].second, event_values[i]); + } + } + } +#endif + if (smpi_process_get_sampling()) { XBT_CRITICAL("Cannot do recursive benchmarks."); XBT_CRITICAL("Are you trying to make a call to MPI within a SMPI_SAMPLE_ block?"); @@ -274,6 +318,20 @@ void smpi_bench_end(void) smpi_execute(xbt_os_timer_elapsed(timer)/speedup); } +#if HAVE_PAPI + if (xbt_cfg_get_string("smpi/papi-events")[0] != '\0' && TRACE_smpi_is_enabled()) { + char container_name[INSTR_DEFAULT_STR_SIZE]; + smpi_container(smpi_process_index(), container_name, INSTR_DEFAULT_STR_SIZE); + container_t container = PJ_container_get(container_name); + papi_counter_t& counter_data = smpi_process_papi_counters(); + + for (auto& pair : counter_data) { + new_pajeSetVariable(surf_get_clock(), container, + PJ_type_get(/* countername */ pair.first.c_str(), container->type), pair.second); + } + } +#endif + smpi_total_benched_time += xbt_os_timer_elapsed(timer); } @@ -316,9 +374,8 @@ int smpi_nanosleep(const struct timespec *tp, struct timespec * t) int smpi_gettimeofday(struct timeval *tv, void* tz) { - double now; smpi_bench_end(); - now = SIMIX_get_clock(); + double now = SIMIX_get_clock(); if (tv) { tv->tv_sec = static_cast(now); #ifdef WIN32 @@ -335,9 +392,8 @@ int smpi_gettimeofday(struct timeval *tv, void* tz) int smpi_clock_gettime(clockid_t clk_id, struct timespec *tp) { //there is only one time in SMPI, so clk_id is ignored. - double now; smpi_bench_end(); - now = SIMIX_get_clock(); + double now = SIMIX_get_clock(); if (tp) { tp->tv_sec = static_cast(now); tp->tv_nsec = static_cast((now - tp->tv_sec) * 1e9); @@ -348,7 +404,7 @@ int smpi_clock_gettime(clockid_t clk_id, struct timespec *tp) #endif extern double sg_surf_precision; -unsigned long long smpi_rastro_resolution (void) +unsigned long long smpi_rastro_resolution () { smpi_bench_end(); double resolution = (1/sg_surf_precision); @@ -356,7 +412,7 @@ unsigned long long smpi_rastro_resolution (void) return static_cast(resolution); } -unsigned long long smpi_rastro_timestamp (void) +unsigned long long smpi_rastro_timestamp () { smpi_bench_end(); double now = SIMIX_get_clock(); @@ -403,7 +459,6 @@ static int sample_enough_benchs(local_data_t *data) { void smpi_sample_1(int global, const char *file, int line, int iters, double threshold) { char *loc = sample_location(global, file, line); - local_data_t *data; smpi_bench_end(); /* Take time from previous, unrelated computation into account */ smpi_process_set_sampling(1); @@ -411,7 +466,7 @@ void smpi_sample_1(int global, const char *file, int line, int iters, double thr if (samples==nullptr) samples = xbt_dict_new_homogeneous(free); - data = static_cast(xbt_dict_get_or_null(samples, loc)); + local_data_t *data = static_cast(xbt_dict_get_or_null(samples, loc)); if (data==nullptr) { xbt_assert(threshold>0 || iters>0, "You should provide either a positive amount of iterations to bench, or a positive maximal stderr (or both)"); @@ -445,11 +500,10 @@ void smpi_sample_1(int global, const char *file, int line, int iters, double thr int smpi_sample_2(int global, const char *file, int line) { char *loc = sample_location(global, file, line); - local_data_t *data; int res; xbt_assert(samples, "Y U NO use SMPI_SAMPLE_* macros? Stop messing directly with smpi_sample_* functions!"); - data = static_cast(xbt_dict_get(samples, loc)); + local_data_t *data = static_cast(xbt_dict_get(samples, loc)); XBT_DEBUG("sample2 %s",loc); xbt_free(loc); @@ -474,27 +528,24 @@ int smpi_sample_2(int global, const char *file, int line) void smpi_sample_3(int global, const char *file, int line) { char *loc = sample_location(global, file, line); - local_data_t *data; xbt_assert(samples, "Y U NO use SMPI_SAMPLE_* macros? Stop messing directly with smpi_sample_* functions!"); - data = static_cast(xbt_dict_get(samples, loc)); + local_data_t *data = static_cast(xbt_dict_get(samples, loc)); XBT_DEBUG("sample3 %s",loc); xbt_free(loc); - if (data->benching==0) { + if (data->benching==0) THROW_IMPOSSIBLE; - } // ok, benchmarking this loop is over xbt_os_threadtimer_stop(smpi_process_timer()); // update the stats - double sample, n; data->count++; - sample = xbt_os_timer_elapsed(smpi_process_timer()); + double sample = xbt_os_timer_elapsed(smpi_process_timer()); data->sum += sample; data->sum_pow2 += sample * sample; - n = static_cast(data->count); + double n = static_cast(data->count); data->mean = data->sum / n; data->relstderr = sqrt((data->sum_pow2 / n - data->mean * data->mean) / n) / data->mean; if (sample_enough_benchs(data)==0) { @@ -513,7 +564,7 @@ void smpi_sample_3(int global, const char *file, int line) void *smpi_shared_malloc(size_t size, const char *file, int line) { void* mem; - if (xbt_cfg_get_boolean("smpi/use-shared-malloc")){ + if (size > 0 && xbt_cfg_get_boolean("smpi/use-shared-malloc")){ int fd; smpi_source_location loc(file, line); auto res = allocs.insert(std::make_pair(loc, shared_data_t())); @@ -558,7 +609,7 @@ void smpi_shared_free(void *ptr) snprintf(loc, PTR_STRLEN, "%p", ptr); auto meta = allocs_metadata.find(ptr); if (meta == allocs_metadata.end()) { - XBT_WARN("Cannot free: %p was not shared-allocated by SMPI", ptr); + XBT_WARN("Cannot free: %p was not shared-allocated by SMPI - maybe its size was 0?", ptr); return; } shared_data_t* data = &meta->second.data->second; @@ -566,11 +617,12 @@ void smpi_shared_free(void *ptr) XBT_WARN("Unmapping of fd %d failed: %s", data->fd, strerror(errno)); } data->count--; - XBT_DEBUG("Shared free - no removal - of %p, count = %d", ptr, data->count); if (data->count <= 0) { close(data->fd); allocs.erase(allocs.find(meta->second.data->first)); XBT_DEBUG("Shared free - with removal - of %p", ptr); + }else{ + XBT_DEBUG("Shared free - no removal - of %p, count = %d", ptr, data->count); } }else{ XBT_DEBUG("Classic free of %p", ptr); @@ -606,12 +658,10 @@ int smpi_shared_known_call(const char* func, const char* input) void* smpi_shared_get_call(const char* func, const char* input) { char* loc = bprintf("%s:%s", func, input); - void* data; - if(calls==nullptr) { + if (calls==nullptr) calls = xbt_dict_new_homogeneous(nullptr); - } - data = xbt_dict_get(calls, loc); + void* data = xbt_dict_get(calls, loc); xbt_free(loc); return data; } @@ -619,9 +669,8 @@ void* smpi_shared_get_call(const char* func, const char* input) { void* smpi_shared_set_call(const char* func, const char* input, void* data) { char* loc = bprintf("%s:%s", func, input); - if(calls==0) { + if (calls==nullptr) calls = xbt_dict_new_homogeneous(nullptr); - } xbt_dict_set(calls, loc, data, nullptr); xbt_free(loc); return data; @@ -642,7 +691,8 @@ void smpi_switch_data_segment(int dest) { * When doing a state restoration, the state of the restored variables might not be consistent with the state of the * virtual memory. In this case, we to change the data segment. */ -void smpi_really_switch_data_segment(int dest) { +void smpi_really_switch_data_segment(int dest) +{ if(smpi_size_data_exe == 0)//no need to switch return; @@ -669,7 +719,8 @@ int smpi_is_privatisation_file(char* file) return strncmp("/dev/shm/my-buffer-", file, std::strlen("/dev/shm/my-buffer-")) == 0; } -void smpi_initialize_global_memory_segments(){ +void smpi_initialize_global_memory_segments() +{ #if !HAVE_PRIVATIZATION smpi_privatize_global_variables=false; @@ -691,11 +742,15 @@ void smpi_initialize_global_memory_segments(){ for (int i=0; i< smpi_process_count(); i++){ //create SIMIX_process_count() mappings of this size with the same data inside + int file_descriptor; void *address = nullptr; - char path[] = "/dev/shm/my-buffer-XXXXXX"; + char path[24]; int status; - int file_descriptor= mkstemp (path); + do { + snprintf(path, sizeof(path), "/smpi-buffer-%06x", rand()%0xffffff); + file_descriptor = shm_open(path, O_RDWR|O_CREAT|O_EXCL, S_IRUSR|S_IWUSR); + } while (file_descriptor == -1 && errno == EEXIST); if (file_descriptor < 0) { if (errno==EMFILE) { xbt_die("Impossible to create temporary file for memory mapping: %s\n\ @@ -716,10 +771,6 @@ Ask the Internet about tutorials on how to increase the files limit such as: htt strerror(errno)); } - status = unlink (path); - if (status) - xbt_die("Impossible to unlink temporary file for memory mapping"); - status = ftruncate(file_descriptor, smpi_size_data_exe); if(status) xbt_die("Impossible to set the size of the temporary file for memory mapping"); @@ -729,6 +780,10 @@ Ask the Internet about tutorials on how to increase the files limit such as: htt if (address == MAP_FAILED) xbt_die("Couldn't find a free region for memory mapping"); + status = shm_unlink(path); + if (status) + xbt_die("Impossible to unlink temporary file for memory mapping"); + //initialize the values memcpy(address, TOPAGE(smpi_start_data_exe), smpi_size_data_exe); @@ -743,11 +798,9 @@ void smpi_destroy_global_memory_segments(){ if (smpi_size_data_exe == 0)//no need to switch return; #if HAVE_PRIVATIZATION - int i; - for (i=0; i< smpi_process_count(); i++){ - if(munmap(smpi_privatisation_regions[i].address, smpi_size_data_exe) < 0) { + for (int i=0; i< smpi_process_count(); i++) { + if (munmap(smpi_privatisation_regions[i].address, smpi_size_data_exe) < 0) XBT_WARN("Unmapping of fd %d failed: %s", smpi_privatisation_regions[i].file_descriptor, strerror(errno)); - } close(smpi_privatisation_regions[i].file_descriptor); } xbt_free(smpi_privatisation_regions);