rewrite the MPI execution sampling facility (previous implem was deeply bugged)

diff --git a/include/smpi/smpi.h b/include/smpi/smpi.h
index 9d08843..b2267a7 100644 (file)
--- a/include/smpi/smpi.h
+++ b/include/smpi/smpi.h
@@ -427,7 +427,7 @@ XBT_PUBLIC(unsigned int) smpi_sleep(unsigned int secs);
 XBT_PUBLIC(int) smpi_gettimeofday(struct timeval *tv, struct timezone *tz);
 XBT_PUBLIC(unsigned long long) smpi_rastro_resolution (void);
 XBT_PUBLIC(unsigned long long) smpi_rastro_timestamp (void);
 XBT_PUBLIC(int) smpi_gettimeofday(struct timeval *tv, struct timezone *tz);
 XBT_PUBLIC(unsigned long long) smpi_rastro_resolution (void);
 XBT_PUBLIC(unsigned long long) smpi_rastro_timestamp (void);

-XBT_PUBLIC(int) smpi_sample_1(int global, const char *file, int line,
+XBT_PUBLIC(void) smpi_sample_1(int global, const char *file, int line,

                               int iters, double threshold);
 XBT_PUBLIC(int) smpi_sample_2(int global, const char *file, int line);
 XBT_PUBLIC(void) smpi_sample_3(int global, const char *file, int line);
                               int iters, double threshold);
 XBT_PUBLIC(int) smpi_sample_2(int global, const char *file, int line);
 XBT_PUBLIC(void) smpi_sample_3(int global, const char *file, int line);

diff --git a/src/smpi/smpi_bench.c b/src/smpi/smpi_bench.c
index 87d148a..ad115e1 100644 (file)
--- a/src/smpi/smpi_bench.c
+++ b/src/smpi/smpi_bench.c
@@ -108,17 +108,6 @@ static void* shm_map(int fd, size_t size, shared_data_t* data) {
   return mem;
 }
 
   return mem;
 }
 

-typedef struct {
-  int count;
-  double sum;
-  double sum_pow2;
-  double mean;
-  double relstderr;
-  int iters;
-  double threshold;
-  int running;
-} local_data_t;
-

 void smpi_bench_destroy(void)
 {
   xbt_dict_free(&allocs);
 void smpi_bench_destroy(void)
 {
   xbt_dict_free(&allocs);


@@ -149,7 +138,7 @@ static void smpi_execute(double duration)
                        xbt_cfg_get_double(_surf_cfg_set,
                                           "smpi/running_power"));
   } else {
                        xbt_cfg_get_double(_surf_cfg_set,
                                           "smpi/running_power"));
   } else {

-    XBT_DEBUG("Real computation took %f while threshold is set to %f; ignore it",
+    XBT_DEBUG("Real computation took %f while option smpi/cpu_threshold is set to %f => ignore it",

         duration, xbt_cfg_get_double(_surf_cfg_set, "smpi/cpu_threshold"));
   }
 }
         duration, xbt_cfg_get_double(_surf_cfg_set, "smpi/cpu_threshold"));
   }
 }


@@ -209,24 +198,43 @@ unsigned long long smpi_rastro_timestamp (void)
   return (unsigned long long)sec * smpi_rastro_resolution() + pre;
 }
 
   return (unsigned long long)sec * smpi_rastro_resolution() + pre;
 }
 

-static char *sample_location(int global, const char *file, int line)
-{
+/* ****************************** Functions related to the SMPI_SAMPLE_ macros ************************************/
+typedef struct {
+  int iters;        /* amount of requested iterations */
+  int count;        /* amount of iterations done so far */
+  double threshold; /* maximal stderr requested (if positive) */
+  double relstderr; /* observed stderr so far */
+  double mean;      /* mean of benched times, to be used if the block is disabled */
+  double sum;       /* sum of benched times (to compute the mean and stderr) */
+  double sum_pow2;  /* sum of the square of the benched times (to compute the stderr) */
+  int benching;     /* 1: we are benchmarking; 0: we have enough data, no bench anymore */
+} local_data_t;
+
+static char *sample_location(int global, const char *file, int line) {

   if (global) {
     return bprintf("%s:%d", file, line);
   } else {
     return bprintf("%s:%d:%d", file, line, smpi_process_index());
   }
 }
   if (global) {
     return bprintf("%s:%d", file, line);
   } else {
     return bprintf("%s:%d:%d", file, line, smpi_process_index());
   }
 }

+static int sample_enough_benchs(local_data_t *data) {
+  int res = (data->iters > 0 && data->count >= data->iters)
+      || (data->count > 2 && data->threshold > 0.0 && data->relstderr <= data->threshold);
+  XBT_DEBUG("%s (count:%d iter:%d stderr:%f thres:%f mean:%fs)",
+      (res?"enough benchs":"need more data"),
+      data->count, data->iters, data->relstderr, data->threshold, data->mean);
+  return res;
+}

 
 

-int smpi_sample_1(int global, const char *file, int line, int iters, double threshold)
+void smpi_sample_1(int global, const char *file, int line, int iters, double threshold)

 {
   char *loc = sample_location(global, file, line);
   local_data_t *data;
 
 {
   char *loc = sample_location(global, file, line);
   local_data_t *data;
 

-  smpi_bench_end();     /* Take time from previous MPI call into account */
-  if (!samples) {
+  smpi_bench_end();     /* Take time from previous, unrelated computation into account */
+  if (!samples)

     samples = xbt_dict_new_homogeneous(free);
     samples = xbt_dict_new_homogeneous(free);

-  }
+

   data = xbt_dict_get_or_null(samples, loc);
   if (!data) {
     data = (local_data_t *) xbt_new(local_data_t, 1);
   data = xbt_dict_get_or_null(samples, loc);
   if (!data) {
     data = (local_data_t *) xbt_new(local_data_t, 1);


@@ -235,12 +243,22 @@ int smpi_sample_1(int global, const char *file, int line, int iters, double thre
     data->sum_pow2 = 0.0;
     data->iters = iters;
     data->threshold = threshold;
     data->sum_pow2 = 0.0;
     data->iters = iters;
     data->threshold = threshold;

-    data->running = 0;
+    data->benching = 1; // If we have no data, we need at least one
+    data->mean = 0;

     xbt_dict_set(samples, loc, data, NULL);
     xbt_dict_set(samples, loc, data, NULL);

-    return 0;
+    XBT_DEBUG("XXXXX First time ever on benched nest %s.",loc);
+  } else {
+    if (data->iters != iters || data->threshold != threshold) {
+      XBT_ERROR("Asked to bench block %s with different settings %d, %f is not %d, %f. How did you manage to give two numbers at the same line??",
+          loc, data->iters, data->threshold, iters,threshold);
+      THROW_IMPOSSIBLE;
+    }
+
+    // if we already have some data, check whether sample_2 should get one more bench or whether it should emulate the computation instead
+    data->benching = !sample_enough_benchs(data);
+    XBT_DEBUG("XXXX Re-entering the benched nest %s. %s",loc, (data->benching?"more benching needed":"we have enough data, skip computes"));

   }
   free(loc);
   }
   free(loc);

-  return 1;

 }
 
 int smpi_sample_2(int global, const char *file, int line)
 }
 
 int smpi_sample_2(int global, const char *file, int line)


@@ -250,45 +268,61 @@ int smpi_sample_2(int global, const char *file, int line)
 
   xbt_assert(samples, "Y U NO use SMPI_SAMPLE_* macros? Stop messing directly with smpi_sample_* functions!");
   data = xbt_dict_get(samples, loc);
 
   xbt_assert(samples, "Y U NO use SMPI_SAMPLE_* macros? Stop messing directly with smpi_sample_* functions!");
   data = xbt_dict_get(samples, loc);

+  XBT_DEBUG("sample2 %s",loc);
+  free(loc);

 
 

-  if (!data->running) {
-    if ((data->iters > 0 && data->count >= data->iters)
-        || (data->count > 1 && data->threshold > 0.0 && data->relstderr <= data->threshold)) {
-      XBT_DEBUG("Perform some wait of %f", data->mean);
-      smpi_execute(data->mean);
-    } else {
-      data->running = 1;
-      data->count++;
-    }
+  if (data->benching==1) {
+    // we need to run a new bench
+    XBT_DEBUG("benchmarking: count:%d iter:%d stderr:%f thres:%f; mean:%f",
+        data->count, data->iters, data->relstderr, data->threshold, data->mean);
+    smpi_bench_begin();
+    return 1;

   } else {
   } else {

-    data->running = 0;
+    // Enough data, no more bench (either we got enough data from previous visits to this benched nest, or we just ran one bench and need to bail out now that our job is done).
+    // Just sleep instead
+    XBT_DEBUG("No benchmark (either no need, or just ran one): count >= iter (%d >= %d) or stderr<thres (%f<=%f). apply the %fs delay instead",
+        data->count, data->iters, data->relstderr, data->threshold, data->mean);
+    smpi_execute(data->mean);
+
+    smpi_bench_begin(); // prepare to capture future, unrelated computations
+    return 0;

   }
   }

-  free(loc);
-  smpi_bench_begin();
-  smpi_process_simulated_start();
-  return data->running;

 }
 
 }
 

+

 void smpi_sample_3(int global, const char *file, int line)
 {
   char *loc = sample_location(global, file, line);
   local_data_t *data;
 void smpi_sample_3(int global, const char *file, int line)
 {
   char *loc = sample_location(global, file, line);
   local_data_t *data;

-  double sample, n;

 
   xbt_assert(samples, "Y U NO use SMPI_SAMPLE_* macros? Stop messing directly with smpi_sample_* functions!");
   data = xbt_dict_get(samples, loc);
 
   xbt_assert(samples, "Y U NO use SMPI_SAMPLE_* macros? Stop messing directly with smpi_sample_* functions!");
   data = xbt_dict_get(samples, loc);

-  smpi_bench_end();
-  if(data->running && data->count < data->iters) {
-    sample = smpi_process_simulated_elapsed();
-    data->sum += sample;
-    data->sum_pow2 += sample * sample;
-    n = (double)data->count;
-    data->mean = data->sum / n;
-    data->relstderr = sqrt((data->sum_pow2 / n - data->mean * data->mean) / n) / data->mean;
-    XBT_DEBUG("Average mean after %d steps is %f, relative standard error is %f (sample was %f)", data->count,
-           data->mean, data->relstderr, sample);
+  XBT_DEBUG("sample3 %s",loc);
+
+  if (data->benching==0) {
+    THROW_IMPOSSIBLE;

   }
   }

-  free(loc);
+
+  // ok, benchmarking this loop is over
+  xbt_os_timer_stop(smpi_process_timer());
+
+  // update the stats
+  double sample, n;
+  data->count++;
+  sample = xbt_os_timer_elapsed(smpi_process_timer());
+  data->sum += sample;
+  data->sum_pow2 += sample * sample;
+  n = (double)data->count;
+  data->mean = data->sum / n;
+  data->relstderr = sqrt((data->sum_pow2 / n - data->mean * data->mean) / n) / data->mean;
+  if (!sample_enough_benchs(data)) {
+    data->mean = sample; // Still in benching process; We want sample_2 to simulate the exact time of this loop occurrence before leaving, not the mean over the history
+  }
+  XBT_DEBUG("Average mean after %d steps is %f, relative standard error is %f (sample was %f)", data->count,
+      data->mean, data->relstderr, sample);
+
+  // That's enough for now, prevent sample_2 to run the same code over and over
+  data->benching = 0;

 }
 
 void smpi_sample_flops(double flops)
 }
 
 void smpi_sample_flops(double flops)

diff --git a/src/smpi/smpi_global.c b/src/smpi/smpi_global.c
index f157012..5a2c2ce 100644 (file)
--- a/src/smpi/smpi_global.c
+++ b/src/smpi/smpi_global.c
@@ -150,36 +150,31 @@ smx_rdv_t smpi_process_remote_mailbox(int index) {
   return data->mailbox;
 }
 
   return data->mailbox;
 }
 

-xbt_os_timer_t smpi_process_timer(void)
-{
+xbt_os_timer_t smpi_process_timer(void) {

   smpi_process_data_t data = smpi_process_data();
 
   return data->timer;
 }
 
   smpi_process_data_t data = smpi_process_data();
 
   return data->timer;
 }
 

-void smpi_process_simulated_start(void)
-{
+void smpi_process_simulated_start(void) {

   smpi_process_data_t data = smpi_process_data();
 
   data->simulated = SIMIX_get_clock();
 }
 
   smpi_process_data_t data = smpi_process_data();
 
   data->simulated = SIMIX_get_clock();
 }
 

-double smpi_process_simulated_elapsed(void)
-{
+double smpi_process_simulated_elapsed(void) {

   smpi_process_data_t data = smpi_process_data();
 
   return SIMIX_get_clock() - data->simulated;
 }
 
   smpi_process_data_t data = smpi_process_data();
 
   return SIMIX_get_clock() - data->simulated;
 }
 

-MPI_Comm smpi_process_comm_self(void)
-{
+MPI_Comm smpi_process_comm_self(void) {

   smpi_process_data_t data = smpi_process_data();
 
   return data->comm_self;
 }
 
   smpi_process_data_t data = smpi_process_data();
 
   return data->comm_self;
 }
 

-void print_request(const char *message, MPI_Request request)
-{
+void print_request(const char *message, MPI_Request request) {

   XBT_DEBUG("%s  request %p  [buf = %p, size = %zu, src = %d, dst = %d, tag = %d, flags = %x]",
          message, request, request->buf, request->size,
          request->src, request->dst, request->tag, request->flags);
   XBT_DEBUG("%s  request %p  [buf = %p, size = %zu, src = %d, dst = %d, tag = %d, flags = %x]",
          message, request, request->buf, request->size,
          request->src, request->dst, request->tag, request->flags);