1 /* Copyright (c) 2007, 2009-2013. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
9 #include "xbt/sysdep.h"
12 #include "surf/surf.h"
13 #include "simgrid/sg_config.h"
19 #include <sys/types.h>
22 #include <math.h> // sqrt
27 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_bench, smpi,
28 "Logging specific to SMPI (benchmarking)");
30 /* Shared allocations are handled through shared memory segments.
31 * Associated data and metadata are used as follows:
34 * `allocs' dict ---- -.
35 * ---------- shared_data_t shared_metadata_t / | | |
36 * .->| <name> | ---> -------------------- <--. ----------------- | | | |
37 * | ---------- | fd of <name> | | | size of mmap | --| | | |
38 * | | count (2) | |-- | data | \ | | |
39 * `----------------- | <name> | | ----------------- ---- |
40 * -------------------- | ^ |
42 * | | `allocs_metadata' dict |
43 * | | ---------------------- |
44 * | `-- | <addr of mmap #1> |<-'
45 * | .-- | <addr of mmap #2> |<-.
46 * | | ---------------------- |
52 * | shared_metadata_t / | |
53 * | ----------------- | | |
54 * | | size of mmap | --| | |
56 * ----------------- | | |
61 #define PTR_STRLEN (2 + 2 * sizeof(void*) + 1)
63 xbt_dict_t allocs = NULL; /* Allocated on first use */
64 xbt_dict_t allocs_metadata = NULL; /* Allocated on first use */
65 xbt_dict_t samples = NULL; /* Allocated on first use */
66 xbt_dict_t calls = NULL; /* Allocated on first use */
67 __thread int smpi_current_rank = 0; /* Updated after each MPI call */
69 double smpi_cpu_threshold;
70 double smpi_running_power;
83 static size_t shm_size(int fd) {
86 if(fstat(fd, &st) < 0) {
87 xbt_die("Could not stat fd %d: %s", fd, strerror(errno));
89 return (size_t)st.st_size;
93 static void* shm_map(int fd, size_t size, shared_data_t* data) {
96 shared_metadata_t* meta;
98 if(size > shm_size(fd)) {
99 if(ftruncate(fd, (off_t)size) < 0) {
100 xbt_die("Could not truncate fd %d to %zu: %s", fd, size, strerror(errno));
104 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
105 if(mem == MAP_FAILED) {
106 xbt_die("Could not map fd %d: %s", fd, strerror(errno));
108 if(!allocs_metadata) {
109 allocs_metadata = xbt_dict_new_homogeneous(xbt_free);
111 snprintf(loc, PTR_STRLEN, "%p", mem);
112 meta = xbt_new(shared_metadata_t, 1);
115 xbt_dict_set(allocs_metadata, loc, meta, NULL);
116 XBT_DEBUG("MMAP %zu to %p", size, mem);
121 void smpi_bench_destroy(void)
123 xbt_dict_free(&allocs);
124 xbt_dict_free(&allocs_metadata);
125 xbt_dict_free(&samples);
126 xbt_dict_free(&calls);
129 void smpi_execute_flops(double flops) {
132 host = SIMIX_host_self();
133 XBT_DEBUG("Handle real computation time: %f flops", flops);
134 action = simcall_host_execute("computation", host, flops, 1);
136 simcall_set_category (action, TRACE_internal_smpi_get_category());
138 simcall_host_execution_wait(action);
141 static void smpi_execute(double duration)
143 if (duration >= smpi_cpu_threshold) {
144 XBT_DEBUG("Sleep for %f to handle real computation time", duration);
145 double flops = duration * smpi_running_power;
147 int rank = smpi_process_index();
148 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
149 extra->type=TRACING_COMPUTING;
150 extra->comp_size=flops;
151 TRACE_smpi_computing_in(rank, extra);
153 smpi_execute_flops(flops);
156 TRACE_smpi_computing_out(rank);
160 XBT_DEBUG("Real computation took %f while option smpi/cpu_threshold is set to %f => ignore it",
161 duration, smpi_cpu_threshold);
165 void smpi_bench_begin(void)
167 xbt_os_threadtimer_start(smpi_process_timer());
168 smpi_current_rank = smpi_process_index();
171 void smpi_bench_end(void)
173 xbt_os_timer_t timer = smpi_process_timer();
175 xbt_os_threadtimer_stop(timer);
176 smpi_execute(xbt_os_timer_elapsed(timer));
179 unsigned int smpi_sleep(unsigned int secs)
185 double flops = (double) secs*simcall_host_get_speed(SIMIX_host_self());
186 XBT_DEBUG("Sleep for: %f flops", flops);
187 action = simcall_host_execute("computation", SIMIX_host_self(), flops, 1);
189 simcall_set_category (action, TRACE_internal_smpi_get_category());
191 simcall_host_execution_wait(action);
197 int smpi_gettimeofday(struct timeval *tv)
201 now = SIMIX_get_clock();
203 tv->tv_sec = (time_t)now;
205 tv->tv_usec = (useconds_t)((now - tv->tv_sec) * 1e6);
207 tv->tv_usec = (suseconds_t)((now - tv->tv_sec) * 1e6);
214 extern double sg_maxmin_precision;
215 unsigned long long smpi_rastro_resolution (void)
218 double resolution = (1/sg_maxmin_precision);
220 return (unsigned long long)resolution;
223 unsigned long long smpi_rastro_timestamp (void)
226 double now = SIMIX_get_clock();
228 unsigned long long sec = (unsigned long long)now;
229 unsigned long long pre = (now - sec) * smpi_rastro_resolution();
231 return (unsigned long long)sec * smpi_rastro_resolution() + pre;
234 /* ****************************** Functions related to the SMPI_SAMPLE_ macros ************************************/
236 int iters; /* amount of requested iterations */
237 int count; /* amount of iterations done so far */
238 double threshold; /* maximal stderr requested (if positive) */
239 double relstderr; /* observed stderr so far */
240 double mean; /* mean of benched times, to be used if the block is disabled */
241 double sum; /* sum of benched times (to compute the mean and stderr) */
242 double sum_pow2; /* sum of the square of the benched times (to compute the stderr) */
243 int benching; /* 1: we are benchmarking; 0: we have enough data, no bench anymore */
246 int smpi_sample_is_running = 0;
248 static char *sample_location(int global, const char *file, int line) {
250 return bprintf("%s:%d", file, line);
252 return bprintf("%s:%d:%d", file, line, smpi_process_index());
255 static int sample_enough_benchs(local_data_t *data) {
256 int res = data->count >= data->iters;
257 if (data->threshold>0.0) {
259 res = 0; // not enough data
260 if (data->relstderr > data->threshold)
261 res = 0; // stderr too high yet
263 XBT_DEBUG("%s (count:%d iter:%d stderr:%f thres:%f mean:%fs)",
264 (res?"enough benchs":"need more data"),
265 data->count, data->iters, data->relstderr, data->threshold, data->mean);
269 void smpi_sample_1(int global, const char *file, int line, int iters, double threshold)
271 char *loc = sample_location(global, file, line);
274 smpi_bench_end(); /* Take time from previous, unrelated computation into account */
275 smpi_sample_is_running++;
278 samples = xbt_dict_new_homogeneous(free);
280 data = xbt_dict_get_or_null(samples, loc);
282 xbt_assert(threshold>0 || iters>0,
283 "You should provide either a positive amount of iterations to bench, or a positive maximal stderr (or both)");
284 data = (local_data_t *) xbt_new(local_data_t, 1);
287 data->sum_pow2 = 0.0;
289 data->threshold = threshold;
290 data->benching = 1; // If we have no data, we need at least one
292 xbt_dict_set(samples, loc, data, NULL);
293 XBT_DEBUG("XXXXX First time ever on benched nest %s.",loc);
295 if (data->iters != iters || data->threshold != threshold) {
296 XBT_ERROR("Asked to bench block %s with different settings %d, %f is not %d, %f. How did you manage to give two numbers at the same line??",
297 loc, data->iters, data->threshold, iters,threshold);
301 // if we already have some data, check whether sample_2 should get one more bench or whether it should emulate the computation instead
302 data->benching = !sample_enough_benchs(data);
303 XBT_DEBUG("XXXX Re-entering the benched nest %s. %s",loc, (data->benching?"more benching needed":"we have enough data, skip computes"));
308 int smpi_sample_2(int global, const char *file, int line)
310 char *loc = sample_location(global, file, line);
313 xbt_assert(samples, "Y U NO use SMPI_SAMPLE_* macros? Stop messing directly with smpi_sample_* functions!");
314 data = xbt_dict_get(samples, loc);
315 XBT_DEBUG("sample2 %s",loc);
318 if (data->benching==1) {
319 // we need to run a new bench
320 XBT_DEBUG("benchmarking: count:%d iter:%d stderr:%f thres:%f; mean:%f",
321 data->count, data->iters, data->relstderr, data->threshold, data->mean);
325 // Enough data, no more bench (either we got enough data from previous visits to this benched nest, or we just ran one bench and need to bail out now that our job is done).
326 // Just sleep instead
327 XBT_DEBUG("No benchmark (either no need, or just ran one): count >= iter (%d >= %d) or stderr<thres (%f<=%f). apply the %fs delay instead",
328 data->count, data->iters, data->relstderr, data->threshold, data->mean);
329 smpi_execute(data->mean);
331 smpi_sample_is_running--;
332 smpi_bench_begin(); // prepare to capture future, unrelated computations
338 void smpi_sample_3(int global, const char *file, int line)
340 char *loc = sample_location(global, file, line);
343 xbt_assert(samples, "Y U NO use SMPI_SAMPLE_* macros? Stop messing directly with smpi_sample_* functions!");
344 data = xbt_dict_get(samples, loc);
345 XBT_DEBUG("sample3 %s",loc);
348 if (data->benching==0) {
352 // ok, benchmarking this loop is over
353 xbt_os_threadtimer_stop(smpi_process_timer());
358 sample = xbt_os_timer_elapsed(smpi_process_timer());
360 data->sum_pow2 += sample * sample;
361 n = (double)data->count;
362 data->mean = data->sum / n;
363 data->relstderr = sqrt((data->sum_pow2 / n - data->mean * data->mean) / n) / data->mean;
364 if (!sample_enough_benchs(data)) {
365 data->mean = sample; // Still in benching process; We want sample_2 to simulate the exact time of this loop occurrence before leaving, not the mean over the history
367 XBT_DEBUG("Average mean after %d steps is %f, relative standard error is %f (sample was %f)", data->count,
368 data->mean, data->relstderr, sample);
370 // That's enough for now, prevent sample_2 to run the same code over and over
375 static void smpi_shared_alloc_free(void *p)
377 shared_data_t *data = p;
382 static char *smpi_shared_alloc_hash(char *loc)
392 loc = xbt_realloc(loc, 30);
394 for (i = 0; i < 40; i += 6) { /* base64 encode */
395 memcpy(s, hash + i, 6);
396 val = strtoul(s, NULL, 16);
397 for (j = 0; j < 4; j++) {
398 unsigned char x = (val >> (18 - 3 * j)) & 0x3f;
399 loc[1 + 4 * i / 6 + j] =
400 "ABCDEFGHIJKLMNOPQRSTUVZXYZabcdefghijklmnopqrstuvzxyz0123456789-_"[x];
407 void *smpi_shared_malloc(size_t size, const char *file, int line)
410 if (sg_cfg_get_boolean("smpi/use_shared_malloc")){
411 char *loc = bprintf("%zu_%s_%d", (size_t)getpid(), file, line);
414 loc = smpi_shared_alloc_hash(loc); /* hash loc, in order to have something
417 allocs = xbt_dict_new_homogeneous(smpi_shared_alloc_free);
419 data = xbt_dict_get_or_null(allocs, loc);
421 fd = shm_open(loc, O_RDWR | O_CREAT | O_EXCL,
422 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
426 xbt_die("Please cleanup /dev/shm/%s", loc);
428 xbt_die("An unhandled error occured while opening %s. shm_open: %s", loc, strerror(errno));
431 data = xbt_new(shared_data_t, 1);
435 mem = shm_map(fd, size, data);
436 if (shm_unlink(loc) < 0) {
437 XBT_WARN("Could not early unlink %s. shm_unlink: %s", loc, strerror(errno));
439 xbt_dict_set(allocs, loc, data, NULL);
440 XBT_DEBUG("Mapping %s at %p through %d", loc, mem, fd);
443 mem = shm_map(data->fd, size, data);
446 XBT_DEBUG("Shared malloc %zu in %p (metadata at %p)", size, mem, data);
448 mem = xbt_malloc(size);
449 XBT_DEBUG("Classic malloc %zu in %p", size, mem);
454 void smpi_shared_free(void *ptr)
456 char loc[PTR_STRLEN];
457 shared_metadata_t* meta;
459 if (sg_cfg_get_boolean("smpi/use_shared_malloc")){
462 XBT_WARN("Cannot free: nothing was allocated");
465 if(!allocs_metadata) {
466 XBT_WARN("Cannot free: no metadata was allocated");
468 snprintf(loc, PTR_STRLEN, "%p", ptr);
469 meta = (shared_metadata_t*)xbt_dict_get_or_null(allocs_metadata, loc);
471 XBT_WARN("Cannot free: %p was not shared-allocated by SMPI", ptr);
476 XBT_WARN("Cannot free: something is broken in the metadata link");
479 if(munmap(ptr, meta->size) < 0) {
480 XBT_WARN("Unmapping of fd %d failed: %s", data->fd, strerror(errno));
483 XBT_DEBUG("Shared free - no removal - of %p, count = %d", ptr, data->count);
484 if (data->count <= 0) {
486 xbt_dict_remove(allocs, data->loc);
487 XBT_DEBUG("Shared free - with removal - of %p", ptr);
490 XBT_DEBUG("Classic free of %p", ptr);
496 int smpi_shared_known_call(const char* func, const char* input) {
497 char* loc = bprintf("%s:%s", func, input);
502 calls = xbt_dict_new_homogeneous(NULL);
505 xbt_dict_get(calls, loc); /* Succeed or throw */
509 if(ex.category == not_found_error) {
520 void* smpi_shared_get_call(const char* func, const char* input) {
521 char* loc = bprintf("%s:%s", func, input);
525 calls = xbt_dict_new_homogeneous(NULL);
527 data = xbt_dict_get(calls, loc);
532 void* smpi_shared_set_call(const char* func, const char* input, void* data) {
533 char* loc = bprintf("%s:%s", func, input);
536 calls = xbt_dict_new_homogeneous(NULL);
538 xbt_dict_set(calls, loc, data, NULL);