From d5fb5a821d7fc68f1327118928ba6861bfe8cc59 Mon Sep 17 00:00:00 2001
From: jean-noel quintin <jnquintin@dhcp-892b9b4c.ucd.ie>
Date: Thu, 4 Oct 2012 16:08:14 +0100
Subject: [PATCH] [smpi] reduce the amount of memory used with the detached
 communication and non-contiguous memory

---
 src/smpi/private.h     |  1 +
 src/smpi/smpi_base.c   | 21 +++++++++++----------
 src/smpi/smpi_mpi_dt.c | 17 ++++++++---------
 3 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/src/smpi/private.h b/src/smpi/private.h
index ed59446091..b969e6b9a1 100644
--- a/src/smpi/private.h
+++ b/src/smpi/private.h
@@ -73,6 +73,7 @@ typedef struct s_smpi_mpi_request {
   MPI_Comm comm;
   smx_action_t action;
   unsigned flags;
+  int detached;
 #ifdef HAVE_TRACING
   int send;
   int recv;
diff --git a/src/smpi/smpi_base.c b/src/smpi/smpi_base.c
index d29b237c67..9993cb4151 100644
--- a/src/smpi/smpi_base.c
+++ b/src/smpi/smpi_base.c
@@ -70,6 +70,7 @@ static MPI_Request build_request(void *buf, int count,
   request->comm = comm;
   request->action = NULL;
   request->flags = flags;
+  request->detached = 0;
 #ifdef HAVE_TRACING
   request->send = 0;
   request->recv = 0;
@@ -145,7 +146,6 @@ MPI_Request smpi_mpi_recv_init(void *buf, int count, MPI_Datatype datatype,
 void smpi_mpi_start(MPI_Request request)
 {
   smx_rdv_t mailbox;
-  int detached = 0;
 
   xbt_assert(!request->action,
              "Cannot (re)start a non-finished communication");
@@ -173,11 +173,14 @@ void smpi_mpi_start(MPI_Request request)
       mailbox = smpi_process_remote_mailbox(receiver);
     }
     if (request->size < 64*1024 ) { //(FIXME: this limit should be configurable)
-      void *oldbuf = request->buf;
-      detached = 1;
-      request->buf = malloc(request->size);
-      if (oldbuf)
-        memcpy(request->buf,oldbuf,request->size);
+      void *oldbuf = NULL;
+      if(request->old_type->has_subtype == 0){
+        oldbuf = request->buf;
+        request->detached = 1;
+        request->buf = malloc(request->size);
+        if (oldbuf)
+          memcpy(request->buf,oldbuf,request->size);
+      }
       XBT_DEBUG("Send request %p is detached; buf %p copied into %p",request,oldbuf,request->buf);
     }
 
@@ -188,7 +191,7 @@ void smpi_mpi_start(MPI_Request request)
                          &smpi_mpi_request_free_voidp, // how to free the userdata if a detached send fails
                          request,
                          // detach if msg size < eager/rdv switch limit
-                         detached);
+                         request->detached);
 
 #ifdef HAVE_TRACING
     /* FIXME: detached sends are not traceable (request->action == NULL) */
@@ -325,9 +328,7 @@ static void finish_wait(MPI_Request * request, MPI_Status * status)
     if(req->flags & RECV) {
       subtype->unserialize(req->buf, req->old_buf, req->size/smpi_datatype_size(datatype) , datatype->substruct);
     }
-    //FIXME: I am not sure that if the send is detached we have to free
-    //the sender buffer thus I do it only for the reciever
-    if(req->flags & RECV) free(req->buf);
+    if(req->detached == 0) free(req->buf);
   }
 
   if(req->flags & NON_PERSISTENT) {
diff --git a/src/smpi/smpi_mpi_dt.c b/src/smpi/smpi_mpi_dt.c
index ec3015c460..255e5ca379 100644
--- a/src/smpi/smpi_mpi_dt.c
+++ b/src/smpi/smpi_mpi_dt.c
@@ -201,6 +201,7 @@ void serialize_vector( const void *noncontiguous_vector,
     noncontiguous_vector_char += type_c->block_stride*type_c->size_oldtype;
   }
 }
+
 /*
  *  Copies contiguous data into noncontiguous memory.
  *  @param noncontiguous_vector - output vector
@@ -255,12 +256,12 @@ s_smpi_mpi_vector_t* smpi_datatype_vector_create( int block_stride,
 void smpi_datatype_create(MPI_Datatype* new_type, int size, int has_subtype,
                           void *struct_type, int flags){
   MPI_Datatype new_t= xbt_new(s_smpi_mpi_datatype_t,1);
-  new_t->size=size;
-  new_t->has_subtype=has_subtype;
-  new_t->lb=0;
-  new_t->ub=size;
-  new_t->flags=flags;
-  new_t->substruct=struct_type;
+  new_t->size = size;
+  new_t->has_subtype = has_subtype;
+  new_t->lb = 0;
+  new_t->ub = size;
+  new_t->flags = flags;
+  new_t->substruct = struct_type;
   *new_type = new_t;
 }
 
@@ -384,9 +385,7 @@ int smpi_datatype_hindexed(int count, int* blocklens, MPI_Aint* indices, MPI_Dat
 int smpi_datatype_struct(int count, int* blocklens, MPI_Aint* indices, MPI_Datatype* old_types, MPI_Datatype* new_type)
 {
   int i;
-  size_t size; //Khalid added this
-
-  size = 0;
+  size_t size = 0;
   for(i=0; i< count; i++){
     if (blocklens[i]<=0)
       return MPI_ERR_ARG;
-- 
2.20.1