Merge branch 'master' of git+ssh://scm.gforge.inria.fr//gitroot/simgrid/simgrid

[simgrid.git] / src / smpi / smpi_replay.c

diff --git a/src/smpi/smpi_replay.c b/src/smpi/smpi_replay.c
index bf7fc59..2f6bebb 100644 (file)
--- a/src/smpi/smpi_replay.c
+++ b/src/smpi/smpi_replay.c
@@ -18,6 +18,11 @@ xbt_dynar_t *reqq = NULL;
 MPI_Datatype MPI_DEFAULT_TYPE;
 MPI_Datatype MPI_CURRENT_TYPE;
 
+static int sendbuffer_size=0;
+char* sendbuffer=NULL;
+static int recvbuffer_size=0;
+char* recvbuffer=NULL;
+
 static void log_timed_action (const char *const *action, double clock){
   if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
     char *name = xbt_str_join_array(action, " ");
@@ -26,6 +31,32 @@ static void log_timed_action (const char *const *action, double clock){
   }
 }
 
+//allocate a single buffer for all sends, growing it if needed
+void* smpi_get_tmp_sendbuffer(int size){
+  if (!smpi_process_get_replaying())
+       return xbt_malloc(size);
+  if (sendbuffer_size<size){
+    sendbuffer=xbt_realloc(sendbuffer,size);
+    sendbuffer_size=size;
+  }
+  return sendbuffer;
+}
+//allocate a single buffer for all recv
+void* smpi_get_tmp_recvbuffer(int size){
+  if (!smpi_process_get_replaying())
+       return xbt_malloc(size);
+  if (recvbuffer_size<size){
+    recvbuffer=xbt_realloc(recvbuffer,size);
+    recvbuffer_size=size;
+  }
+  return sendbuffer;
+}
+
+void smpi_free_tmp_buffer(void* buf){
+  if (!smpi_process_get_replaying())
+    xbt_free(buf);
+}
+
 /* Helper function */
 static double parse_double(const char *string)
 {
@@ -504,12 +535,11 @@ static void action_bcast(const char *const *action)
   TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__, extra);
 
 #endif
-
-  mpi_coll_bcast_fun(NULL, size, MPI_CURRENT_TYPE, root, MPI_COMM_WORLD);
+    void *sendbuf = smpi_get_tmp_sendbuffer(size* smpi_datatype_size(MPI_CURRENT_TYPE));
+  mpi_coll_bcast_fun(sendbuf, size, MPI_CURRENT_TYPE, root, MPI_COMM_WORLD);
 #ifdef HAVE_TRACING
   TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
 #endif
-
   log_timed_action (action, clock);
 }
 
@@ -527,6 +557,8 @@ static void action_reduce(const char *const *action)
       MPI_CURRENT_TYPE=decode_datatype(action[5]);
     }
   }
+  
+  
 
 #ifdef HAVE_TRACING
   int rank = smpi_process_index();
@@ -540,12 +572,13 @@ static void action_reduce(const char *const *action)
 
   TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__,extra);
 #endif
-   mpi_coll_reduce_fun(NULL, NULL, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, root, MPI_COMM_WORLD);
+    void *recvbuf = smpi_get_tmp_sendbuffer(comm_size* smpi_datatype_size(MPI_CURRENT_TYPE));
+    void *sendbuf = smpi_get_tmp_sendbuffer(comm_size* smpi_datatype_size(MPI_CURRENT_TYPE));
+   mpi_coll_reduce_fun(sendbuf, recvbuf, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, root, MPI_COMM_WORLD);
    smpi_execute_flops(comp_size);
 #ifdef HAVE_TRACING
   TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
 #endif
-
   log_timed_action (action, clock);
 }
 
@@ -567,12 +600,13 @@ static void action_allReduce(const char *const *action) {
 
   TRACE_smpi_collective_in(rank, -1, __FUNCTION__,extra);
 #endif
-  mpi_coll_allreduce_fun(NULL, NULL, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, MPI_COMM_WORLD);
+    void *recvbuf = smpi_get_tmp_sendbuffer(comm_size* smpi_datatype_size(MPI_CURRENT_TYPE));
+    void *sendbuf = smpi_get_tmp_sendbuffer(comm_size* smpi_datatype_size(MPI_CURRENT_TYPE));
+  mpi_coll_allreduce_fun(sendbuf, recvbuf, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, MPI_COMM_WORLD);
   smpi_execute_flops(comp_size);
 #ifdef HAVE_TRACING
   TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
 #endif
-
   log_timed_action (action, clock);
 }
 
@@ -591,8 +625,8 @@ static void action_allToAll(const char *const *action) {
     MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
     MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
   }
-  void *send = xbt_malloc(send_size*comm_size* smpi_datatype_size(MPI_CURRENT_TYPE));  
-  void *recv = xbt_malloc(recv_size*comm_size* smpi_datatype_size(MPI_CURRENT_TYPE2));  
+  void *send = smpi_get_tmp_sendbuffer(send_size*comm_size* smpi_datatype_size(MPI_CURRENT_TYPE));
+  void *recv = smpi_get_tmp_recvbuffer(recv_size*comm_size* smpi_datatype_size(MPI_CURRENT_TYPE2));
 
 #ifdef HAVE_TRACING
   int rank = smpi_process_index();
@@ -611,10 +645,8 @@ static void action_allToAll(const char *const *action) {
 #ifdef HAVE_TRACING
   TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
 #endif
-
   log_timed_action (action, clock);
-  xbt_free(send);
-  xbt_free(recv);
+
 }
 
 
@@ -643,14 +675,14 @@ static void action_gather(const char *const *action) {
     MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
     MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
   }
-  void *send = xbt_malloc(send_size* smpi_datatype_size(MPI_CURRENT_TYPE));
+  void *send = smpi_get_tmp_sendbuffer(send_size* smpi_datatype_size(MPI_CURRENT_TYPE));
   void *recv = NULL;
 
   int root=atoi(action[4]);
-  int rank = smpi_process_index();
+  int rank = smpi_comm_rank(MPI_COMM_WORLD);
 
   if(rank==root)
-    recv = xbt_malloc(recv_size*comm_size* smpi_datatype_size(MPI_CURRENT_TYPE2));
+    recv = smpi_get_tmp_recvbuffer(recv_size*comm_size* smpi_datatype_size(MPI_CURRENT_TYPE2));
 
 #ifdef HAVE_TRACING
   instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
@@ -661,19 +693,17 @@ static void action_gather(const char *const *action) {
   extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
   extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2);
 
-  TRACE_smpi_collective_in(rank, root, __FUNCTION__, extra);
+  TRACE_smpi_collective_in(smpi_process_index(), root, __FUNCTION__, extra);
 #endif
   mpi_coll_gather_fun(send, send_size, MPI_CURRENT_TYPE,
                 recv, recv_size, MPI_CURRENT_TYPE2,
                 root, MPI_COMM_WORLD);
 
 #ifdef HAVE_TRACING
-  TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
+  TRACE_smpi_collective_out(smpi_process_index(), -1, __FUNCTION__);
 #endif
-
   log_timed_action (action, clock);
-  xbt_free(send);
-  xbt_free(recv);
+
 }
 
 
@@ -706,7 +736,7 @@ static void action_gatherv(const char *const *action) {
     MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
     MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
   }
-  void *send = xbt_malloc(send_size* smpi_datatype_size(MPI_CURRENT_TYPE));
+  void *send = smpi_get_tmp_sendbuffer(send_size* smpi_datatype_size(MPI_CURRENT_TYPE));
   void *recv = NULL;
   for(i=0;i<comm_size;i++) {
     recvcounts[i] = atoi(action[i+3]);
@@ -715,10 +745,10 @@ static void action_gatherv(const char *const *action) {
   }
 
   int root=atoi(action[3+comm_size]);
-  int rank = smpi_process_index();
+  int rank = smpi_comm_rank(MPI_COMM_WORLD);;
 
   if(rank==root)
-    recv = xbt_malloc(recv_sum* smpi_datatype_size(MPI_CURRENT_TYPE2));
+    recv = smpi_get_tmp_recvbuffer(recv_sum* smpi_datatype_size(MPI_CURRENT_TYPE2));
 
 #ifdef HAVE_TRACING
   instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
@@ -732,22 +762,19 @@ static void action_gatherv(const char *const *action) {
   extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
   extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2);
 
-  TRACE_smpi_collective_in(rank, root, __FUNCTION__, extra);
+  TRACE_smpi_collective_in(smpi_process_index(), root, __FUNCTION__, extra);
 #endif
 smpi_mpi_gatherv(send, send_size, MPI_CURRENT_TYPE,
                 recv, recvcounts, disps, MPI_CURRENT_TYPE2,
                 root, MPI_COMM_WORLD);
 
 #ifdef HAVE_TRACING
-  TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
+  TRACE_smpi_collective_out(smpi_process_index(), -1, __FUNCTION__);
 #endif
 
   log_timed_action (action, clock);
   xbt_free(recvcounts);
-  xbt_free(send);
-  xbt_free(recv);
   xbt_free(disps);
-
 }
 
 static void action_reducescatter(const char *const *action) {
@@ -773,7 +800,7 @@ static void action_reducescatter(const char *const *action) {
   int *disps = xbt_new0(int, comm_size);  
   int i=0;
   int rank = smpi_process_index();
-
+  int size = 0;
   if(action[3+comm_size])
     MPI_CURRENT_TYPE=decode_datatype(action[3+comm_size]);
   else
@@ -782,6 +809,7 @@ static void action_reducescatter(const char *const *action) {
   for(i=0;i<comm_size;i++) {
     recvcounts[i] = atoi(action[i+2]);
     disps[i] = 0;
+    size+=recvcounts[i];
   }
 
 #ifdef HAVE_TRACING
@@ -795,10 +823,12 @@ static void action_reducescatter(const char *const *action) {
   extra->comp_size = comp_size;
   extra->num_processes = comm_size;
 
-
   TRACE_smpi_collective_in(rank, -1, __FUNCTION__,extra);
 #endif
-   mpi_coll_reduce_scatter_fun(NULL, NULL, recvcounts, MPI_CURRENT_TYPE, MPI_OP_NULL,
+  void *sendbuf = smpi_get_tmp_sendbuffer(size* smpi_datatype_size(MPI_CURRENT_TYPE));
+   void *recvbuf = smpi_get_tmp_recvbuffer(size* smpi_datatype_size(MPI_CURRENT_TYPE));
+   
+   mpi_coll_reduce_scatter_fun(sendbuf, recvbuf, recvcounts, MPI_CURRENT_TYPE, MPI_OP_NULL,
        MPI_COMM_WORLD);
    smpi_execute_flops(comp_size);
 
@@ -844,13 +874,13 @@ static void action_allgatherv(const char *const *action) {
     MPI_CURRENT_TYPE = MPI_DEFAULT_TYPE;
     MPI_CURRENT_TYPE2 = MPI_DEFAULT_TYPE;    
   }
-  void *sendbuf = xbt_malloc(sendcount* smpi_datatype_size(MPI_CURRENT_TYPE));    
+  void *sendbuf = smpi_get_tmp_sendbuffer(sendcount* smpi_datatype_size(MPI_CURRENT_TYPE));
 
   for(i=0;i<comm_size;i++) {
     recvcounts[i] = atoi(action[i+3]);
     recv_sum=recv_sum+recvcounts[i];
   }
-  void *recvbuf = xbt_malloc(recv_sum* smpi_datatype_size(MPI_CURRENT_TYPE2));  
+  void *recvbuf = smpi_get_tmp_recvbuffer(recv_sum* smpi_datatype_size(MPI_CURRENT_TYPE2));
 
 #ifdef HAVE_TRACING
   int rank = smpi_process_index();
@@ -874,8 +904,6 @@ static void action_allgatherv(const char *const *action) {
 #endif
 
   log_timed_action (action, clock);
-  xbt_free(sendbuf);
-  xbt_free(recvbuf);
   xbt_free(recvcounts);
   xbt_free(disps);
 }
@@ -918,8 +946,8 @@ static void action_allToAllv(const char *const *action) {
       MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
   }
 
-  void *sendbuf = xbt_malloc(send_buf_size* smpi_datatype_size(MPI_CURRENT_TYPE));  
-  void *recvbuf = xbt_malloc(recv_buf_size* smpi_datatype_size(MPI_CURRENT_TYPE2));  
+  void *sendbuf = smpi_get_tmp_sendbuffer(send_buf_size* smpi_datatype_size(MPI_CURRENT_TYPE));
+  void *recvbuf  = smpi_get_tmp_recvbuffer(recv_buf_size* smpi_datatype_size(MPI_CURRENT_TYPE2));
 
   for(i=0;i<comm_size;i++) {
     sendcounts[i] = atoi(action[i+3]);
@@ -954,8 +982,6 @@ static void action_allToAllv(const char *const *action) {
 #endif
 
   log_timed_action (action, clock);
-  xbt_free(sendbuf);
-  xbt_free(recvbuf);
   xbt_free(sendcounts);
   xbt_free(recvcounts);
   xbt_free(senddisps);
@@ -965,6 +991,7 @@ static void action_allToAllv(const char *const *action) {
 void smpi_replay_init(int *argc, char***argv){
   smpi_process_init(argc, argv);
   smpi_process_mark_as_initialized();
+  smpi_process_set_replaying(1);
 #ifdef HAVE_TRACING
   int rank = smpi_process_index();
   TRACE_smpi_init(rank);
@@ -1040,6 +1067,8 @@ int smpi_replay_finalize(){
     sim_time = smpi_process_simulated_elapsed();
     XBT_INFO("Simulation time %f", sim_time);
     _xbt_replay_action_exit();
+    xbt_free(sendbuffer);
+    xbt_free(recvbuffer);
     xbt_free(reqq);
     reqq = NULL;
   }