X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/e566c09daba9ad67b31f3992da9172eb04b807a8..257a3f5cec44f5ff92b998a2cccb4b32113552a1:/src/smpi/smpi_bench.c diff --git a/src/smpi/smpi_bench.c b/src/smpi/smpi_bench.c index c9d2c8b4ab..363381559f 100644 --- a/src/smpi/smpi_bench.c +++ b/src/smpi/smpi_bench.c @@ -10,18 +10,103 @@ #include "xbt/ex.h" #include "surf/surf.h" +#include +#include +#include +#include +#include +#include +#include +#include + XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_bench, smpi, "Logging specific to SMPI (benchmarking)"); -xbt_dict_t allocs = NULL; /* Allocated on first use */ -xbt_dict_t samples = NULL; /* Allocated on first use */ -xbt_dict_t calls = NULL; /* Allocated on first use */ +/* Shared allocations are handled through shared memory segments. + * Associated data and metadata are used as follows: + * + * mmap #1 + * `allocs' dict ---- -. + * ---------- shared_data_t shared_metadata_t / | | | + * .->| | ---> -------------------- <--. ----------------- | | | | + * | ---------- | fd of | | | size of mmap | --| | | | + * | | count (2) | |-- | data | \ | | | + * `----------------- | | | ----------------- ---- | + * -------------------- | ^ | + * | | | + * | | `allocs_metadata' dict | + * | | ---------------------- | + * | `-- | |<-' + * | .-- | |<-. + * | | ---------------------- | + * | | | + * | | | + * | | | + * | | mmap #2 | + * | v ---- -' + * | shared_metadata_t / | | + * | ----------------- | | | + * | | size of mmap | --| | | + * `-- | data | | | | + * ----------------- | | | + * \ | | + * ---- + */ + +#define PTR_STRLEN (2 + 2 * sizeof(void*) + 1) + +xbt_dict_t allocs = NULL; /* Allocated on first use */ +xbt_dict_t allocs_metadata = NULL; /* Allocated on first use */ +xbt_dict_t samples = NULL; /* Allocated on first use */ +xbt_dict_t calls = NULL; /* Allocated on first use */ +__thread int smpi_current_rank = 0; /* Updated after each MPI call */ typedef struct { + int fd; int count; - char data[]; + char* loc; } shared_data_t; +typedef struct { + size_t size; + shared_data_t* data; +} shared_metadata_t; + +static size_t shm_size(int fd) { + struct stat st; + + if(fstat(fd, &st) < 0) { + xbt_die("Could not stat fd %d: %s", fd, strerror(errno)); + } + return (size_t)st.st_size; +} + +static void* shm_map(int fd, size_t size, shared_data_t* data) { + void* mem; + char loc[PTR_STRLEN]; + shared_metadata_t* meta; + + if(size > shm_size(fd)) { + if(ftruncate(fd, (off_t)size) < 0) { + xbt_die("Could not truncate fd %d to %zu: %s", fd, size, strerror(errno)); + } + } + mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if(mem == MAP_FAILED) { + xbt_die("Could not map fd %d: %s", fd, strerror(errno)); + } + if(!allocs_metadata) { + allocs_metadata = xbt_dict_new(); + } + snprintf(loc, PTR_STRLEN, "%p", mem); + meta = xbt_new(shared_metadata_t, 1); + meta->size = size; + meta->data = data; + xbt_dict_set(allocs_metadata, loc, meta, &free); + XBT_DEBUG("MMAP %zu to %p", size, mem); + return mem; +} + typedef struct { int count; double sum; @@ -73,6 +158,7 @@ static void smpi_execute(double duration) void smpi_bench_begin(void) { xbt_os_timer_start(smpi_process_timer()); + smpi_current_rank = smpi_process_index(); } void smpi_bench_end(void) @@ -85,18 +171,22 @@ void smpi_bench_end(void) unsigned int smpi_sleep(unsigned int secs) { + smpi_bench_end(); smpi_execute((double) secs); + smpi_bench_begin(); return secs; } int smpi_gettimeofday(struct timeval *tv, struct timezone *tz) { - double now = SIMIX_get_clock(); - + double now; + smpi_bench_end(); + now = SIMIX_get_clock(); if (tv) { - tv->tv_sec = (time_t) now; - tv->tv_usec = (suseconds_t) (now * 1e6); + tv->tv_sec = (time_t)now; + tv->tv_usec = (suseconds_t)((now - tv->tv_sec) * 1e6); } + smpi_bench_begin(); return 0; } @@ -191,41 +281,83 @@ void smpi_sample_flops(double flops) void *smpi_shared_malloc(size_t size, const char *file, int line) { - char *loc = bprintf("%s:%d:%zu", file, line, size); + char *loc = bprintf("%zu_%s_%d", (size_t)getpid(), file, line); + size_t len = strlen(loc); + size_t i; + int fd; + void* mem; shared_data_t *data; + for(i = 0; i < len; i++) { + /* Make the 'loc' ID be a flat filename */ + if(loc[i] == '/') { + loc[i] = '_'; + } + } if (!allocs) { allocs = xbt_dict_new(); } data = xbt_dict_get_or_null(allocs, loc); - if (!data) { - data = (shared_data_t *) xbt_malloc0(sizeof(int) + size); + if(!data) { + fd = shm_open(loc, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if(fd < 0) { + switch(errno) { + case EEXIST: + xbt_die("Please cleanup /dev/shm/%s", loc); + default: + xbt_die("An unhandled error occured while opening %s: %s", loc, strerror(errno)); + } + } + data = xbt_new(shared_data_t, 1); + data->fd = fd; data->count = 1; + data->loc = loc; + mem = shm_map(fd, size, data); + if(shm_unlink(loc) < 0) { + XBT_WARN("Could not early unlink %s: %s", loc, strerror(errno)); + } xbt_dict_set(allocs, loc, data, &free); + XBT_DEBUG("Mapping %s at %p through %d", loc, mem, fd); } else { + mem = shm_map(data->fd, size, data); data->count++; } - free(loc); - return data->data; + XBT_DEBUG("Malloc %zu in %p (metadata at %p)", size, mem, data); + return mem; } void smpi_shared_free(void *ptr) { - shared_data_t *data = (shared_data_t *) ((int *) ptr - 1); - char *loc; + char loc[PTR_STRLEN]; + shared_metadata_t* meta; + shared_data_t* data; if (!allocs) { XBT_WARN("Cannot free: nothing was allocated"); return; } - loc = xbt_dict_get_key(allocs, data); - if (!loc) { + if(!allocs_metadata) { + XBT_WARN("Cannot free: no metadata was allocated"); + } + snprintf(loc, PTR_STRLEN, "%p", ptr); + meta = (shared_metadata_t*)xbt_dict_get_or_null(allocs_metadata, loc); + if (!meta) { XBT_WARN("Cannot free: %p was not shared-allocated by SMPI", ptr); return; } + data = meta->data; + if(!data) { + XBT_WARN("Cannot free: something is broken in the metadata link"); + return; + } + if(munmap(ptr, meta->size) < 0) { + XBT_WARN("Unmapping of fd %d failed: %s", data->fd, strerror(errno)); + } data->count--; if (data->count <= 0) { - xbt_dict_remove(allocs, loc); + close(data->fd); + xbt_dict_remove(allocs, data->loc); + free(data->loc); } }