From bf1edfe6e3a70b02a0032a3fceb7863f0117d069 Mon Sep 17 00:00:00 2001
From: Augustin Degomme <degomme@idpann.imag.fr>
Date: Tue, 9 Jul 2013 14:59:39 +0200
Subject: [PATCH 1/1] Copy the group when doing a comm_split, to avoid to have
 shared objects with reference counters.

---
 src/smpi/private.h    |  1 +
 src/smpi/smpi_comm.c  | 10 +++++++---
 src/smpi/smpi_group.c | 39 ++++++++++++++++++++++++++++++++-------
 src/smpi/smpi_pmpi.c  | 37 ++++++++++++++++++-------------------
 4 files changed, 58 insertions(+), 29 deletions(-)

diff --git a/src/smpi/private.h b/src/smpi/private.h
index 52b9ed8cc2..651841cdd0 100644
--- a/src/smpi/private.h
+++ b/src/smpi/private.h
@@ -157,6 +157,7 @@ void smpi_op_apply(MPI_Op op, void *invec, void *inoutvec, int *len,
                    MPI_Datatype * datatype);
 
 MPI_Group smpi_group_new(int size);
+MPI_Group smpi_group_copy(MPI_Group origin);
 void smpi_group_destroy(MPI_Group group);
 void smpi_group_set_mapping(MPI_Group group, int index, int rank);
 int smpi_group_index(MPI_Group group, int rank);
diff --git a/src/smpi/smpi_comm.c b/src/smpi/smpi_comm.c
index 8e2b9564c7..2f6bf15997 100644
--- a/src/smpi/smpi_comm.c
+++ b/src/smpi/smpi_comm.c
@@ -131,6 +131,8 @@ MPI_Comm smpi_comm_split(MPI_Comm comm, int color, int key)
         group_root = group_out; /* Save root's group */
       }
       for(j = 0; j < count; j++) {
+        //increment refcounter in order to avoid freeing the group too quick before copy
+        smpi_group_use(group_out);
         index = smpi_group_index(group, rankmap[2 * j]);
         smpi_group_set_mapping(group_out, index, j);
       }
@@ -148,23 +150,25 @@ MPI_Comm smpi_comm_split(MPI_Comm comm, int color, int key)
     }
     xbt_free(recvbuf);
     group_out = group_root; /* exit with root's group */
+    if(group_out)smpi_group_unuse(group_out);
   } else {
     if(color != MPI_UNDEFINED) {
       smpi_mpi_recv(&group_out, 1, MPI_PTR, 0, system_tag, comm, MPI_STATUS_IGNORE);
+      if(group_out){
+        group_out=smpi_group_copy(group_out);
+        smpi_group_unuse(group_out);
+      }
     } /* otherwise, exit with group_out == NULL */
   }
-  if(group_out)smpi_group_use(group_out);
   return group_out ? smpi_comm_new(group_out) : MPI_COMM_NULL;
 }
 
 void smpi_comm_use(MPI_Comm comm){
   comm->refcount++;
-  smpi_group_use(comm->group);
 }
 
 void smpi_comm_unuse(MPI_Comm comm){
   comm->refcount--;
-  smpi_group_unuse(comm->group);
   if(comm->refcount==0)
     xbt_free(comm);
 }
diff --git a/src/smpi/smpi_group.c b/src/smpi/smpi_group.c
index 0222b881c3..a88bccad3f 100644
--- a/src/smpi/smpi_group.c
+++ b/src/smpi/smpi_group.c
@@ -35,23 +35,43 @@ MPI_Group smpi_group_new(int size)
   group->size = size;
   group->rank_to_index_map = xbt_new(int, size);
   group->index_to_rank_map = xbt_new(int, count);
-  group->refcount = 0;
+  group->refcount = 1;
   for (i = 0; i < size; i++) {
     group->rank_to_index_map[i] = MPI_UNDEFINED;
   }
   for (i = 0; i < count; i++) {
     group->index_to_rank_map[i] = MPI_UNDEFINED;
   }
+
   return group;
 }
 
-void smpi_group_destroy(MPI_Group group)
+MPI_Group smpi_group_copy(MPI_Group origin)
 {
-  if (smpi_group_unuse(group) <= 0) {
-    xbt_free(group->rank_to_index_map);
-    xbt_free(group->index_to_rank_map);
-    xbt_free(group);
+  MPI_Group group;
+  int i, count;
+
+  count = smpi_process_count();
+  group = xbt_new(s_smpi_mpi_group_t, 1);
+  group->size = origin->size;
+  group->rank_to_index_map = xbt_new(int, group->size);
+  group->index_to_rank_map = xbt_new(int, count);
+  group->refcount = 1;
+  for (i = 0; i < group->size; i++) {
+    group->rank_to_index_map[i] = origin->rank_to_index_map[i];
+  }
+  for (i = 0; i < count; i++) {
+    group->index_to_rank_map[i] = origin->index_to_rank_map[i];
   }
+
+  return group;
+}
+
+
+void smpi_group_destroy(MPI_Group group)
+{
+  XBT_VERB("trying to free group %p, refcount = %d", group, group->refcount);
+  smpi_group_unuse(group);
 }
 
 void smpi_group_set_mapping(MPI_Group group, int index, int rank)
@@ -90,7 +110,12 @@ int smpi_group_use(MPI_Group group)
 
 int smpi_group_unuse(MPI_Group group)
 {
-  group->refcount--;
+  if (group->refcount-- <= 0) {
+    XBT_VERB("freeing group %p", group);
+    xbt_free(group->rank_to_index_map);
+    xbt_free(group->index_to_rank_map);
+    xbt_free(group);
+  }
   return group->refcount;
 }
 
diff --git a/src/smpi/smpi_pmpi.c b/src/smpi/smpi_pmpi.c
index df6074bdbd..26e809b4d2 100644
--- a/src/smpi/smpi_pmpi.c
+++ b/src/smpi/smpi_pmpi.c
@@ -606,26 +606,25 @@ int PMPI_Group_range_incl(MPI_Group group, int n, int ranges[][3],
         }
       }
 
-        *newgroup = smpi_group_new(size);
-        j = 0;
-        for (i = 0; i < n; i++) {
-          for (rank = ranges[i][0];     /* First */
-               rank >= 0; /* Last */
-               ) {
-            index = smpi_group_index(group, rank);
-            smpi_group_set_mapping(*newgroup, index, j);
-            j++;
-            rank += ranges[i][2]; /* Stride */
-	    if (ranges[i][0]<ranges[i][1]){
-	      if(rank > ranges[i][1])
-	        break;
-	    }else{
-	      if(rank < ranges[i][1])
-	        break;
-	    }
-          }
+      *newgroup = smpi_group_new(size);
+      j = 0;
+      for (i = 0; i < n; i++) {
+        for (rank = ranges[i][0];     /* First */
+             rank >= 0; /* Last */
+             ) {
+          index = smpi_group_index(group, rank);
+          smpi_group_set_mapping(*newgroup, index, j);
+          j++;
+          rank += ranges[i][2]; /* Stride */
+	  if (ranges[i][0]<ranges[i][1]){
+	    if(rank > ranges[i][1])
+	      break;
+	  }else{
+	    if(rank < ranges[i][1])
+	      break;
+	  }
         }
-      //}
+      }
     }
     smpi_group_use(*newgroup);
     retval = MPI_SUCCESS;
-- 
2.20.1