allow replay to handle -1 as a value for the size of a receive.

[simgrid.git] / src / smpi / smpi_replay.c

diff --git a/src/smpi/smpi_replay.c b/src/smpi/smpi_replay.c
index e41a4f2..71db9bf 100644 (file)
--- a/src/smpi/smpi_replay.c
+++ b/src/smpi/smpi_replay.c
@@ -27,11 +27,6 @@ static void log_timed_action (const char *const *action, double clock){
   }
 }
 
-typedef struct {
-  xbt_dynar_t irecvs; /* of MPI_Request */
-} s_smpi_replay_globals_t, *smpi_replay_globals_t;
-
-
 /* Helper function */
 static double parse_double(const char *string)
 {
@@ -108,14 +103,10 @@ static void action_init(const char *const *action)
 {
   int i;
   XBT_DEBUG("Initialize the counters");
-  smpi_replay_globals_t globals =  xbt_new(s_smpi_replay_globals_t, 1);
-  globals->irecvs = xbt_dynar_new(sizeof(MPI_Request),NULL);
 
   if(action[2]) MPI_DEFAULT_TYPE= MPI_DOUBLE; // default MPE dataype 
   else MPI_DEFAULT_TYPE= MPI_BYTE; // default TAU datatype
 
-  smpi_process_set_user_data((void*) globals);
-
   /* start a simulated timer */
   smpi_process_simulated_start();
   /*initialize the number of active processes */
@@ -125,21 +116,13 @@ static void action_init(const char *const *action)
     reqq=xbt_new0(xbt_dynar_t,active_processes);
 
     for(i=0;i<active_processes;i++){
-      reqq[i]=xbt_dynar_new(sizeof(MPI_Request),NULL);
+      reqq[i]=xbt_dynar_new(sizeof(MPI_Request),&xbt_free_ref);
     }
   }
 }
 
 static void action_finalize(const char *const *action)
 {
-  smpi_replay_globals_t globals =
-      (smpi_replay_globals_t) smpi_process_get_user_data();
-  if (globals){
-    XBT_DEBUG("There are %lu irecvs in the dynar",
-         xbt_dynar_length(globals->irecvs));
-    xbt_dynar_free_container(&(globals->irecvs));
-  }
-  free(globals);
 }
 
 static void action_comm_size(const char *const *action)
@@ -170,7 +153,7 @@ static void action_compute(const char *const *action)
   double flops= parse_double(action[2]);
 #ifdef HAVE_TRACING
   int rank = smpi_comm_rank(MPI_COMM_WORLD);
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type=TRACING_COMPUTING;
   extra->comp_size=flops;
   TRACE_smpi_computing_in(rank, extra);
@@ -199,7 +182,7 @@ static void action_send(const char *const *action)
   int rank = smpi_comm_rank(MPI_COMM_WORLD);
 
   int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_SEND;
   extra->send_size = size;
   extra->src = rank;
@@ -232,7 +215,7 @@ static void action_Isend(const char *const *action)
 #ifdef HAVE_TRACING
   int rank = smpi_comm_rank(MPI_COMM_WORLD);
   int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_ISEND;
   extra->send_size = size;
   extra->src = rank;
@@ -267,7 +250,7 @@ static void action_recv(const char *const *action) {
   int rank = smpi_comm_rank(MPI_COMM_WORLD);
   int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
 
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_RECV;
   extra->send_size = size;
   extra->src = src_traced;
@@ -276,6 +259,12 @@ static void action_recv(const char *const *action) {
   TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__, extra);
 #endif
 
+  //unknow size from the receiver pov
+  if(size==-1){
+      smpi_mpi_probe(from, 0, MPI_COMM_WORLD, &status);
+      size=status.count;
+  }
+
   smpi_mpi_recv(NULL, size, MPI_CURRENT_TYPE, from, 0, MPI_COMM_WORLD, &status);
 
 #ifdef HAVE_TRACING
@@ -293,16 +282,13 @@ static void action_Irecv(const char *const *action)
   double clock = smpi_process_simulated_elapsed();
   MPI_Request request;
 
-  smpi_replay_globals_t globals =
-     (smpi_replay_globals_t) smpi_process_get_user_data();
-
   if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
   else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
 
 #ifdef HAVE_TRACING
   int rank = smpi_comm_rank(MPI_COMM_WORLD);
   int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_IRECV;
   extra->send_size = size;
   extra->src = src_traced;
@@ -310,6 +296,12 @@ static void action_Irecv(const char *const *action)
   extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
   TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__, extra);
 #endif
+  MPI_Status status;
+  //unknow size from the receiver pov
+  if(size==-1){
+      smpi_mpi_probe(from, 0, MPI_COMM_WORLD, &status);
+      size=status.count;
+  }
 
   request = smpi_mpi_irecv(NULL, size, MPI_CURRENT_TYPE, from, 0, MPI_COMM_WORLD);
 
@@ -317,7 +309,6 @@ static void action_Irecv(const char *const *action)
   TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
   request->recv = 1;
 #endif
-  xbt_dynar_push(globals->irecvs,&request);
   xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
 
   log_timed_action (action, clock);
@@ -327,15 +318,14 @@ static void action_wait(const char *const *action){
   double clock = smpi_process_simulated_elapsed();
   MPI_Request request;
   MPI_Status status;
-  smpi_replay_globals_t globals =
-      (smpi_replay_globals_t) smpi_process_get_user_data();
 
-  xbt_assert(xbt_dynar_length(globals->irecvs),
-      "action wait not preceded by any irecv: %s",
+  xbt_assert(xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]),
+      "action wait not preceded by any irecv or isend: %s",
       xbt_str_join_array(action," "));
-  request = xbt_dynar_pop_as(globals->irecvs,MPI_Request);
+  request = xbt_dynar_pop_as(reqq[smpi_comm_rank(MPI_COMM_WORLD)],MPI_Request);
+  xbt_assert(request != NULL, "found null request in reqq");
 #ifdef HAVE_TRACING
-  int rank = request && request->comm != MPI_COMM_NULL
+  int rank = request->comm != MPI_COMM_NULL
       ? smpi_comm_rank(request->comm)
       : -1;
 
@@ -343,7 +333,7 @@ static void action_wait(const char *const *action){
   int src_traced = smpi_group_rank(group, request->src);
   int dst_traced = smpi_group_rank(group, request->dst);
   int is_wait_for_receive = request->recv;
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_WAIT;
   TRACE_smpi_ptp_in(rank, src_traced, dst_traced, __FUNCTION__, extra);
 #endif
@@ -402,7 +392,7 @@ static void action_waitall(const char *const *action){
     }
    }
    int rank_traced = smpi_process_index();
-   instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+   instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
    extra->type = TRACING_WAITALL;
    extra->send_size=count_requests;
    TRACE_smpi_ptp_in(rank_traced, -1, -1, __FUNCTION__,extra);
@@ -427,7 +417,7 @@ static void action_waitall(const char *const *action){
    xbt_dynar_free(&recvs);
   #endif
 
-   xbt_dynar_reset(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
+   xbt_dynar_free_container(&(reqq[smpi_comm_rank(MPI_COMM_WORLD)]));
   }
   log_timed_action (action, clock);
 }
@@ -436,7 +426,7 @@ static void action_barrier(const char *const *action){
   double clock = smpi_process_simulated_elapsed();
 #ifdef HAVE_TRACING
   int rank = smpi_comm_rank(MPI_COMM_WORLD);
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_BARRIER;
   TRACE_smpi_collective_in(rank, -1, __FUNCTION__, extra);
 #endif
@@ -471,7 +461,7 @@ static void action_bcast(const char *const *action)
   int rank = smpi_comm_rank(MPI_COMM_WORLD);
   int root_traced = smpi_group_index(smpi_comm_group(MPI_COMM_WORLD), root);
 
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_BCAST;
   extra->send_size = size;
   extra->root = root_traced;
@@ -506,7 +496,7 @@ static void action_reduce(const char *const *action)
 #ifdef HAVE_TRACING
   int rank = smpi_comm_rank(MPI_COMM_WORLD);
   int root_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), root);
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_REDUCE;
   extra->send_size = comm_size;
   extra->comp_size = comp_size;
@@ -534,7 +524,7 @@ static void action_allReduce(const char *const *action) {
   double clock = smpi_process_simulated_elapsed();
 #ifdef HAVE_TRACING
   int rank = smpi_comm_rank(MPI_COMM_WORLD);
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_ALLREDUCE;
   extra->send_size = comm_size;
   extra->comp_size = comp_size;
@@ -572,7 +562,7 @@ static void action_allToAll(const char *const *action) {
 
 #ifdef HAVE_TRACING
   int rank = smpi_process_index();
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_ALLTOALL;
   extra->send_size = send_size;
   extra->recv_size = recv_size;
@@ -620,7 +610,7 @@ static void action_gather(const char *const *action) {
     MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
   }
   void *send = calloc(send_size, smpi_datatype_size(MPI_CURRENT_TYPE));
-  void *recv = calloc(recv_size, smpi_datatype_size(MPI_CURRENT_TYPE2));
+  void *recv = NULL;
 
   int root=atoi(action[4]);
   int rank = smpi_process_index();
@@ -629,7 +619,7 @@ static void action_gather(const char *const *action) {
     recv = calloc(recv_size*comm_size, smpi_datatype_size(MPI_CURRENT_TYPE2));
 
 #ifdef HAVE_TRACING
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_GATHER;
   extra->send_size = send_size;
   extra->recv_size = recv_size;
@@ -653,6 +643,79 @@ smpi_mpi_gather(send, send_size, MPI_CURRENT_TYPE,
 }
 
 
+
+static void action_gatherv(const char *const *action) {
+  /*
+ The structure of the gatherv action for the rank 0 (total 4 processes)
+ is the following:
+ 0 gather 68 68 10 10 10 0 0 0
+
+  where:
+  1) 68 is the sendcount
+  2) 68 10 10 10 is the recvcounts
+  3) 0 is the root node
+  4) 0 is the send datatype id, see decode_datatype()
+  5) 0 is the recv datatype id, see decode_datatype()
+  */
+  double clock = smpi_process_simulated_elapsed();
+  int comm_size = smpi_comm_size(MPI_COMM_WORLD);
+  int send_size = parse_double(action[2]);
+  int *disps = xbt_new0(int, comm_size);
+  int *recvcounts = xbt_new0(int, comm_size);
+  int i=0,recv_sum=0;
+
+  MPI_Datatype MPI_CURRENT_TYPE2;
+  if(action[4+comm_size]) {
+    MPI_CURRENT_TYPE=decode_datatype(action[4+comm_size]);
+    MPI_CURRENT_TYPE2=decode_datatype(action[5+comm_size]);
+  } else {
+    MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
+    MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
+  }
+  void *send = calloc(send_size, smpi_datatype_size(MPI_CURRENT_TYPE));
+  void *recv = NULL;
+  for(i=0;i<comm_size;i++) {
+    recvcounts[i] = atoi(action[i+3]);
+    recv_sum=recv_sum+recvcounts[i];
+    disps[i] = 0;
+  }
+
+  int root=atoi(action[3+comm_size]);
+  int rank = smpi_process_index();
+
+  if(rank==root)
+    recv = calloc(recv_sum, smpi_datatype_size(MPI_CURRENT_TYPE2));
+
+#ifdef HAVE_TRACING
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
+  extra->type = TRACING_GATHERV;
+  extra->send_size = send_size;
+  extra->recvcounts= xbt_malloc(comm_size*sizeof(int));
+  for(i=0; i< comm_size; i++)//copy data to avoid bad free
+    extra->recvcounts[i] = recvcounts[i];
+  extra->root = root;
+  extra->num_processes = comm_size;
+  extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
+  extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2);
+
+  TRACE_smpi_collective_in(rank, root, __FUNCTION__, extra);
+#endif
+smpi_mpi_gatherv(send, send_size, MPI_CURRENT_TYPE,
+                recv, recvcounts, disps, MPI_CURRENT_TYPE2,
+                root, MPI_COMM_WORLD);
+
+#ifdef HAVE_TRACING
+  TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
+#endif
+
+  log_timed_action (action, clock);
+  xbt_free(recvcounts);
+  xbt_free(send);
+  xbt_free(recv);
+  xbt_free(disps);
+
+}
+
 static void action_reducescatter(const char *const *action) {
 
     /*
@@ -690,7 +753,7 @@ static void action_reducescatter(const char *const *action) {
   }
 
 #ifdef HAVE_TRACING
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_REDUCE_SCATTER;
   extra->send_size = 0;
   extra->recvcounts= xbt_malloc(comm_size*sizeof(int));
@@ -713,7 +776,8 @@ static void action_reducescatter(const char *const *action) {
 #ifdef HAVE_TRACING
   TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
 #endif
-
+  xbt_free(recvcounts);
+  xbt_free(disps);
   log_timed_action (action, clock);
 }
 
@@ -760,7 +824,7 @@ static void action_allgatherv(const char *const *action) {
 
 #ifdef HAVE_TRACING
   int rank = smpi_process_index();
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_ALLGATHERV;
   extra->send_size = sendcount;
   extra->recvcounts= xbt_malloc(comm_size*sizeof(int));
@@ -835,11 +899,8 @@ static void action_allToAllv(const char *const *action) {
 
 #ifdef HAVE_TRACING
   int rank = smpi_process_index();
-  int count=0;
-  for(i=0;i<comm_size;i++) count+=sendcounts[i];
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_ALLTOALLV;
-  extra->send_size = count;
   extra->recvcounts= xbt_malloc(comm_size*sizeof(int));
   extra->sendcounts= xbt_malloc(comm_size*sizeof(int));
   extra->num_processes = comm_size;
@@ -847,6 +908,7 @@ static void action_allToAllv(const char *const *action) {
   for(i=0; i< comm_size; i++){//copy data to avoid bad free
     extra->send_size += sendcounts[i];
     extra->sendcounts[i] = sendcounts[i];
+    extra->recv_size += recvcounts[i];
     extra->recvcounts[i] = recvcounts[i];
   }
   extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
@@ -877,7 +939,7 @@ void smpi_replay_init(int *argc, char***argv){
   int rank = smpi_process_index();
   TRACE_smpi_init(rank);
   TRACE_smpi_computing_init(rank);
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_INIT;
   TRACE_smpi_collective_in(rank, -1, __FUNCTION__, extra);
   TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
@@ -903,6 +965,7 @@ void smpi_replay_init(int *argc, char***argv){
     xbt_replay_action_register("allToAll",   action_allToAll);
     xbt_replay_action_register("allToAllV",  action_allToAllv);
     xbt_replay_action_register("gather",  action_gather);
+    xbt_replay_action_register("gatherV",  action_gatherv);
     xbt_replay_action_register("allGatherV",  action_allgatherv);
     xbt_replay_action_register("reduceScatter",  action_reducescatter);
     xbt_replay_action_register("compute",    action_compute);
@@ -914,15 +977,28 @@ void smpi_replay_init(int *argc, char***argv){
 int smpi_replay_finalize(){
   double sim_time= 1.;
   /* One active process will stop. Decrease the counter*/
-  active_processes--;
   XBT_DEBUG("There are %lu elements in reqq[*]",
             xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]));
-  xbt_dynar_free(&reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
+  if (!xbt_dynar_is_empty(reqq[smpi_comm_rank(MPI_COMM_WORLD)])){
+    int count_requests=xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
+    MPI_Request requests[count_requests];
+    MPI_Status status[count_requests];
+    unsigned int i;
+
+    xbt_dynar_foreach(reqq[smpi_comm_rank(MPI_COMM_WORLD)],i,requests[i]);
+    smpi_mpi_waitall(count_requests, requests, status);
+    active_processes--;
+  } else {
+    active_processes--;
+  }
+
+  xbt_dynar_free_container(&(reqq[smpi_comm_rank(MPI_COMM_WORLD)]));
+
   if(!active_processes){
     /* Last process alive speaking */
     /* end the simulated timer */
     sim_time = smpi_process_simulated_elapsed();
-    XBT_INFO("Simulation time %g", sim_time);
+    XBT_INFO("Simulation time %f", sim_time);
     _xbt_replay_action_exit();
     xbt_free(reqq);
     reqq = NULL;
@@ -930,7 +1006,7 @@ int smpi_replay_finalize(){
   smpi_mpi_barrier(MPI_COMM_WORLD);
 #ifdef HAVE_TRACING
   int rank = smpi_process_index();
-  instr_extra_data extra = xbt_new(s_instr_extra_data_t,1);
+  instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
   extra->type = TRACING_FINALIZE;
   TRACE_smpi_collective_in(rank, -1, __FUNCTION__, extra);
 #endif