Logo AND Algorithmique Numérique Distribuée

Public GIT Repository
Separate boolean from others, as gcc > 6 complains on some operations.
[simgrid.git] / src / smpi / smpi_mpi_dt.cpp
1 /* smpi_mpi_dt.c -- MPI primitives to handle datatypes                      */
2 /* Copyright (c) 2009-2015. The SimGrid Team.
3  * All rights reserved.                                                     */
4
5 /* This program is free software; you can redistribute it and/or modify it
6  * under the terms of the license (GNU LGPL) which comes with this package. */
7
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <limits.h>
12 #include "private.h"
13 #include "smpi_mpi_dt_private.h"
14 #include "mc/mc.h"
15 #include "xbt/replay.h"
16 #include <xbt/ex.hpp>
17 #include "simgrid/modelchecker.h"
18
19 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_mpi_dt, smpi, "Logging specific to SMPI (datatype)");
20
21 xbt_dict_t smpi_type_keyvals = nullptr;
22 int type_keyval_id=0;//avoid collisions
23
24 #define CREATE_MPI_DATATYPE(name, type)               \
25   static s_smpi_mpi_datatype_t mpi_##name = {         \
26     (char*) # name,                                   \
27     sizeof(type),   /* size */                        \
28     0,              /*was 1 sizeof_substruct*/        \
29     0,              /* lb */                          \
30     sizeof(type),   /* ub = lb + size */              \
31     DT_FLAG_BASIC,  /* flags */                       \
32     nullptr,        /* attributes */                  \
33     nullptr,        /* pointer on extended struct*/   \
34     0               /* in_use counter */              \
35   };                                                  \
36 const MPI_Datatype name = &mpi_##name;
37
38 #define CREATE_MPI_DATATYPE_NULL(name)                \
39   static s_smpi_mpi_datatype_t mpi_##name = {         \
40     (char*) # name,                                   \
41     0,              /* size */                        \
42     0,              /* was 1 sizeof_substruct*/       \
43     0,              /* lb */                          \
44     0,              /* ub = lb + size */              \
45     DT_FLAG_BASIC,  /* flags */                       \
46     nullptr,        /* attributes */                  \
47     nullptr,        /* pointer on extended struct*/   \
48     0               /* in_use counter */              \
49   };                                                  \
50 const MPI_Datatype name = &mpi_##name;
51
52 //The following are datatypes for the MPI functions MPI_MAXLOC and MPI_MINLOC.
53 typedef struct {
54   float value;
55   int index;
56 } float_int;
57 typedef struct {
58   float value;
59   float index;
60 } float_float;
61 typedef struct {
62   long value;
63   long index;
64 } long_long;
65 typedef struct {
66   double value;
67   double index;
68 } double_double;
69 typedef struct {
70   long value;
71   int index;
72 } long_int;
73 typedef struct {
74   double value;
75   int index;
76 } double_int;
77 typedef struct {
78   short value;
79   int index;
80 } short_int;
81 typedef struct {
82   int value;
83   int index;
84 } int_int;
85 typedef struct {
86   long double value;
87   int index;
88 } long_double_int;
89 typedef struct {
90   int64_t value;
91   int64_t index;
92 } integer128_t;
93 // Predefined data types
94 CREATE_MPI_DATATYPE(MPI_CHAR, char);
95 CREATE_MPI_DATATYPE(MPI_SHORT, short);
96 CREATE_MPI_DATATYPE(MPI_INT, int);
97 CREATE_MPI_DATATYPE(MPI_LONG, long);
98 CREATE_MPI_DATATYPE(MPI_LONG_LONG, long long);
99 CREATE_MPI_DATATYPE(MPI_SIGNED_CHAR, signed char);
100 CREATE_MPI_DATATYPE(MPI_UNSIGNED_CHAR, unsigned char);
101 CREATE_MPI_DATATYPE(MPI_UNSIGNED_SHORT, unsigned short);
102 CREATE_MPI_DATATYPE(MPI_UNSIGNED, unsigned int);
103 CREATE_MPI_DATATYPE(MPI_UNSIGNED_LONG, unsigned long);
104 CREATE_MPI_DATATYPE(MPI_UNSIGNED_LONG_LONG, unsigned long long);
105 CREATE_MPI_DATATYPE(MPI_FLOAT, float);
106 CREATE_MPI_DATATYPE(MPI_DOUBLE, double);
107 CREATE_MPI_DATATYPE(MPI_LONG_DOUBLE, long double);
108 CREATE_MPI_DATATYPE(MPI_WCHAR, wchar_t);
109 CREATE_MPI_DATATYPE(MPI_C_BOOL, bool);
110 CREATE_MPI_DATATYPE(MPI_BYTE, int8_t);
111 CREATE_MPI_DATATYPE(MPI_INT8_T, int8_t);
112 CREATE_MPI_DATATYPE(MPI_INT16_T, int16_t);
113 CREATE_MPI_DATATYPE(MPI_INT32_T, int32_t);
114 CREATE_MPI_DATATYPE(MPI_INT64_T, int64_t);
115 CREATE_MPI_DATATYPE(MPI_UINT8_T, uint8_t);
116 CREATE_MPI_DATATYPE(MPI_UINT16_T, uint16_t);
117 CREATE_MPI_DATATYPE(MPI_UINT32_T, uint32_t);
118 CREATE_MPI_DATATYPE(MPI_UINT64_T, uint64_t);
119 CREATE_MPI_DATATYPE(MPI_C_FLOAT_COMPLEX, float _Complex);
120 CREATE_MPI_DATATYPE(MPI_C_DOUBLE_COMPLEX, double _Complex);
121 CREATE_MPI_DATATYPE(MPI_C_LONG_DOUBLE_COMPLEX, long double _Complex);
122 CREATE_MPI_DATATYPE(MPI_AINT, MPI_Aint);
123 CREATE_MPI_DATATYPE(MPI_OFFSET, MPI_Offset);
124
125 CREATE_MPI_DATATYPE(MPI_FLOAT_INT, float_int);
126 CREATE_MPI_DATATYPE(MPI_LONG_INT, long_int);
127 CREATE_MPI_DATATYPE(MPI_DOUBLE_INT, double_int);
128 CREATE_MPI_DATATYPE(MPI_SHORT_INT, short_int);
129 CREATE_MPI_DATATYPE(MPI_2INT, int_int);
130 CREATE_MPI_DATATYPE(MPI_2FLOAT, float_float);
131 CREATE_MPI_DATATYPE(MPI_2DOUBLE, double_double);
132 CREATE_MPI_DATATYPE(MPI_2LONG, long_long);
133
134 CREATE_MPI_DATATYPE(MPI_REAL, float);
135 CREATE_MPI_DATATYPE(MPI_REAL4, float);
136 CREATE_MPI_DATATYPE(MPI_REAL8, float);
137 CREATE_MPI_DATATYPE(MPI_REAL16, double);
138 CREATE_MPI_DATATYPE_NULL(MPI_COMPLEX8);
139 CREATE_MPI_DATATYPE_NULL(MPI_COMPLEX16);
140 CREATE_MPI_DATATYPE_NULL(MPI_COMPLEX32);
141 CREATE_MPI_DATATYPE(MPI_INTEGER1, int);
142 CREATE_MPI_DATATYPE(MPI_INTEGER2, int16_t);
143 CREATE_MPI_DATATYPE(MPI_INTEGER4, int32_t);
144 CREATE_MPI_DATATYPE(MPI_INTEGER8, int64_t);
145 CREATE_MPI_DATATYPE(MPI_INTEGER16, integer128_t);
146
147 CREATE_MPI_DATATYPE(MPI_LONG_DOUBLE_INT, long_double_int);
148
149 CREATE_MPI_DATATYPE_NULL(MPI_UB);
150 CREATE_MPI_DATATYPE_NULL(MPI_LB);
151 CREATE_MPI_DATATYPE(MPI_PACKED, char);
152 // Internal use only
153 CREATE_MPI_DATATYPE(MPI_PTR, void*);
154
155 /** Check if the datatype is usable for communications */
156 bool is_datatype_valid(MPI_Datatype datatype) {
157     return datatype != MPI_DATATYPE_NULL && ((datatype->flags & DT_FLAG_COMMITED) != 0);
158 }
159
160 size_t smpi_datatype_size(MPI_Datatype datatype)
161 {
162   return datatype->size;
163 }
164
165 MPI_Aint smpi_datatype_lb(MPI_Datatype datatype)
166 {
167   return datatype->lb;
168 }
169
170 MPI_Aint smpi_datatype_ub(MPI_Datatype datatype)
171 {
172   return datatype->ub;
173 }
174
175 int smpi_datatype_dup(MPI_Datatype datatype, MPI_Datatype* new_t)
176 {
177   int ret=MPI_SUCCESS;
178   *new_t= xbt_new(s_smpi_mpi_datatype_t,1);
179   memcpy(*new_t, datatype, sizeof(s_smpi_mpi_datatype_t));
180   (*new_t)->in_use=1;
181   (*new_t)->flags &= ~DT_FLAG_PREDEFINED;
182   if (datatype->sizeof_substruct){
183     (*new_t)->substruct=xbt_malloc(datatype->sizeof_substruct);
184     memcpy((*new_t)->substruct, datatype->substruct, datatype->sizeof_substruct);
185   }
186   if(datatype->name)
187     (*new_t)->name = xbt_strdup(datatype->name);
188   if(datatype->attributes !=nullptr){
189     (*new_t)->attributes     = xbt_dict_new_homogeneous(nullptr);
190     xbt_dict_cursor_t cursor = nullptr;
191     int* key;
192     int flag;
193     void* value_in;
194     void* value_out;
195     xbt_dict_foreach (datatype->attributes, cursor, key, value_in) {
196       smpi_type_key_elem elem = static_cast<smpi_type_key_elem>(
197           xbt_dict_get_or_null_ext(smpi_type_keyvals, reinterpret_cast<const char*>(key), sizeof(int)));
198       if (elem != nullptr && elem->copy_fn != MPI_NULL_COPY_FN) {
199         ret = elem->copy_fn(datatype, *key, nullptr, value_in, &value_out, &flag);
200         if (ret != MPI_SUCCESS) {
201           smpi_datatype_unuse(*new_t);
202           *new_t = MPI_DATATYPE_NULL;
203           xbt_dict_cursor_free(&cursor);
204           return ret;
205         }
206         if (flag)
207           xbt_dict_set_ext((*new_t)->attributes, reinterpret_cast<const char*>(key), sizeof(int), value_out, nullptr);
208       }
209       }
210     }
211   return ret;
212 }
213
214 int smpi_datatype_extent(MPI_Datatype datatype, MPI_Aint * lb, MPI_Aint * extent)
215 {
216   if(datatype == MPI_DATATYPE_NULL){
217     *lb=0;
218     *extent=0;
219     return MPI_SUCCESS;
220   }
221   *lb = datatype->lb;
222   *extent = datatype->ub - datatype->lb;
223   return MPI_SUCCESS;
224 }
225
226 MPI_Aint smpi_datatype_get_extent(MPI_Datatype datatype){
227   if(datatype == MPI_DATATYPE_NULL){
228     return 0;
229   }
230   return datatype->ub - datatype->lb;
231 }
232
233 void smpi_datatype_get_name(MPI_Datatype datatype, char* name, int* length){
234   *length = strlen(datatype->name);
235   strncpy(name, datatype->name, *length+1);
236 }
237
238 void smpi_datatype_set_name(MPI_Datatype datatype, char* name){
239   if(datatype->name!=nullptr &&  (datatype->flags & DT_FLAG_PREDEFINED) == 0)
240     xbt_free(datatype->name);
241   datatype->name = xbt_strdup(name);
242 }
243
244 int smpi_datatype_copy(void *sendbuf, int sendcount, MPI_Datatype sendtype,
245                        void *recvbuf, int recvcount, MPI_Datatype recvtype)
246 {
247   int count;
248   if(smpi_privatize_global_variables){
249     smpi_switch_data_segment(smpi_process_index());
250   }
251   /* First check if we really have something to do */
252   if (recvcount > 0 && recvbuf != sendbuf) {
253     /* FIXME: treat packed cases */
254     sendcount *= smpi_datatype_size(sendtype);
255     recvcount *= smpi_datatype_size(recvtype);
256     count = sendcount < recvcount ? sendcount : recvcount;
257
258     if(sendtype->sizeof_substruct == 0 && recvtype->sizeof_substruct == 0) {
259       if(!smpi_process_get_replaying()) 
260         memcpy(recvbuf, sendbuf, count);
261     }
262     else if (sendtype->sizeof_substruct == 0)
263     {
264       s_smpi_subtype_t *subtype =  static_cast<s_smpi_subtype_t*>(recvtype->substruct);
265       subtype->unserialize( sendbuf, recvbuf, recvcount/smpi_datatype_size(recvtype), subtype, MPI_REPLACE);
266     }
267     else if (recvtype->sizeof_substruct == 0)
268     {
269       s_smpi_subtype_t *subtype =  static_cast<s_smpi_subtype_t*>(sendtype->substruct);
270       subtype->serialize(sendbuf, recvbuf, sendcount/smpi_datatype_size(sendtype), subtype);
271     }else{
272       s_smpi_subtype_t *subtype =  static_cast<s_smpi_subtype_t*>(sendtype->substruct);
273
274       void * buf_tmp = xbt_malloc(count);
275
276       subtype->serialize( sendbuf, buf_tmp,count/smpi_datatype_size(sendtype), subtype);
277       subtype =  static_cast<s_smpi_subtype_t*>(recvtype->substruct);
278       subtype->unserialize( buf_tmp, recvbuf,count/smpi_datatype_size(recvtype), subtype, MPI_REPLACE);
279
280       xbt_free(buf_tmp);
281     }
282   }
283
284   return sendcount > recvcount ? MPI_ERR_TRUNCATE : MPI_SUCCESS;
285 }
286
287 /*
288  *  Copies noncontiguous data into contiguous memory.
289  *  @param contiguous_vector - output vector
290  *  @param noncontiguous_vector - input vector
291  *  @param type - pointer containing :
292  *      - stride - stride of between noncontiguous data
293  *      - block_length - the width or height of blocked matrix
294  *      - count - the number of rows of matrix
295  */
296 void serialize_vector( const void *noncontiguous_vector, void *contiguous_vector, int count, void *type)
297 {
298   s_smpi_mpi_vector_t* type_c = reinterpret_cast<s_smpi_mpi_vector_t*>(type);
299   int i;
300   char* contiguous_vector_char = static_cast<char*>(contiguous_vector);
301   const char* noncontiguous_vector_char = static_cast<const char*>(noncontiguous_vector);
302
303   for (i = 0; i < type_c->block_count * count; i++) {
304       if (type_c->old_type->sizeof_substruct == 0)
305         memcpy(contiguous_vector_char, noncontiguous_vector_char, type_c->block_length * type_c->size_oldtype);
306       else
307         static_cast<s_smpi_subtype_t*>(type_c->old_type->substruct)->serialize( noncontiguous_vector_char,
308                                                                      contiguous_vector_char,
309                                                                      type_c->block_length, type_c->old_type->substruct);
310
311     contiguous_vector_char += type_c->block_length*type_c->size_oldtype;
312     if((i+1)%type_c->block_count ==0)
313       noncontiguous_vector_char += type_c->block_length*smpi_datatype_get_extent(type_c->old_type);
314     else
315       noncontiguous_vector_char += type_c->block_stride*smpi_datatype_get_extent(type_c->old_type);
316   }
317 }
318
319 /*
320  *  Copies contiguous data into noncontiguous memory.
321  *  @param noncontiguous_vector - output vector
322  *  @param contiguous_vector - input vector
323  *  @param type - pointer contening :
324  *      - stride - stride of between noncontiguous data
325  *      - block_length - the width or height of blocked matrix
326  *      - count - the number of rows of matrix
327  */
328 void unserialize_vector( const void *contiguous_vector, void *noncontiguous_vector, int count, void *type, MPI_Op op)
329 {
330   s_smpi_mpi_vector_t* type_c = reinterpret_cast<s_smpi_mpi_vector_t*>(type);
331   int i;
332
333   const char* contiguous_vector_char = static_cast<const char*>(contiguous_vector);
334   char* noncontiguous_vector_char = static_cast<char*>(noncontiguous_vector);
335
336   for (i = 0; i < type_c->block_count * count; i++) {
337     if (type_c->old_type->sizeof_substruct == 0)
338       smpi_op_apply(op, contiguous_vector_char, noncontiguous_vector_char, &type_c->block_length,
339           &type_c->old_type);
340     else
341       static_cast<s_smpi_subtype_t*>(type_c->old_type->substruct)->unserialize(contiguous_vector_char, noncontiguous_vector_char,
342                                                                     type_c->block_length,type_c->old_type->substruct,
343                                                                     op);
344     contiguous_vector_char += type_c->block_length*type_c->size_oldtype;
345     if((i+1)%type_c->block_count ==0)
346       noncontiguous_vector_char += type_c->block_length*smpi_datatype_get_extent(type_c->old_type);
347     else
348       noncontiguous_vector_char += type_c->block_stride*smpi_datatype_get_extent(type_c->old_type);
349   }
350 }
351
352 /* Create a Sub type vector to be able to serialize and unserialize it the structure s_smpi_mpi_vector_t is derived
353  * from s_smpi_subtype which required the functions unserialize and serialize */
354 s_smpi_mpi_vector_t* smpi_datatype_vector_create( int block_stride, int block_length, int block_count,
355                                                   MPI_Datatype old_type, int size_oldtype){
356   s_smpi_mpi_vector_t *new_t= xbt_new(s_smpi_mpi_vector_t,1);
357   new_t->base.serialize = &serialize_vector;
358   new_t->base.unserialize = &unserialize_vector;
359   new_t->base.subtype_free = &free_vector;
360   new_t->base.subtype_use = &use_vector;
361   new_t->block_stride = block_stride;
362   new_t->block_length = block_length;
363   new_t->block_count = block_count;
364   smpi_datatype_use(old_type);
365   new_t->old_type = old_type;
366   new_t->size_oldtype = size_oldtype;
367   return new_t;
368 }
369
370 void smpi_datatype_create(MPI_Datatype* new_type, int size,int lb, int ub, int sizeof_substruct, void *struct_type,
371                           int flags){
372   MPI_Datatype new_t= xbt_new(s_smpi_mpi_datatype_t,1);
373   new_t->name = nullptr;
374   new_t->size = size;
375   new_t->sizeof_substruct = size>0? sizeof_substruct:0;
376   new_t->lb = lb;
377   new_t->ub = ub;
378   new_t->flags = flags;
379   new_t->substruct = struct_type;
380   new_t->in_use=1;
381   new_t->attributes=nullptr;
382   *new_type = new_t;
383
384 #if HAVE_MC
385   if(MC_is_active())
386     MC_ignore(&(new_t->in_use), sizeof(new_t->in_use));
387 #endif
388 }
389
390 void smpi_datatype_free(MPI_Datatype* type){
391   xbt_assert((*type)->in_use >= 0);
392
393   if((*type)->flags & DT_FLAG_PREDEFINED)
394     return;
395
396   //if still used, mark for deletion
397   if((*type)->in_use!=0){
398       (*type)->flags |=DT_FLAG_DESTROYED;
399       return;
400   }
401
402   if((*type)->attributes !=nullptr){
403     xbt_dict_cursor_t cursor = nullptr;
404     int* key;
405     void * value;
406     int flag;
407     xbt_dict_foreach((*type)->attributes, cursor, key, value){
408       smpi_type_key_elem elem =
409           static_cast<smpi_type_key_elem>(xbt_dict_get_or_null_ext(smpi_type_keyvals, reinterpret_cast<const char*>(key), sizeof(int)));
410       if(elem!=nullptr && elem->delete_fn!=nullptr)
411         elem->delete_fn(*type,*key, value, &flag);
412     }
413     xbt_dict_free(&(*type)->attributes);
414   }
415
416   if ((*type)->sizeof_substruct != 0){
417     //((s_smpi_subtype_t *)(*type)->substruct)->subtype_free(type);  
418     xbt_free((*type)->substruct);
419   }
420   xbt_free((*type)->name);
421   xbt_free(*type);
422   *type = MPI_DATATYPE_NULL;
423 }
424
425 void smpi_datatype_use(MPI_Datatype type){
426
427   if(type != MPI_DATATYPE_NULL)
428     type->in_use++;
429
430   if(type->sizeof_substruct!=0){
431     static_cast<s_smpi_subtype_t *>((type)->substruct)->subtype_use(&type);  
432   }
433 #if HAVE_MC
434   if(MC_is_active())
435     MC_ignore(&(type->in_use), sizeof(type->in_use));
436 #endif
437 }
438
439 void smpi_datatype_unuse(MPI_Datatype type)
440 {
441   if (type == MPI_DATATYPE_NULL)
442     return;
443
444   if (type->in_use > 0)
445     type->in_use--;
446
447   if(type->sizeof_substruct!=0){
448     static_cast<s_smpi_subtype_t *>((type)->substruct)->subtype_free(&type);  
449   }
450
451   if (type->in_use == 0)
452     smpi_datatype_free(&type);
453
454 #if HAVE_MC
455   if(MC_is_active())
456     MC_ignore(&(type->in_use), sizeof(type->in_use));
457 #endif
458 }
459
460 /*Contiguous Implementation*/
461
462 /* Copies noncontiguous data into contiguous memory.
463  *  @param contiguous_hvector - output hvector
464  *  @param noncontiguous_hvector - input hvector
465  *  @param type - pointer contening :
466  *      - stride - stride of between noncontiguous data, in bytes
467  *      - block_length - the width or height of blocked matrix
468  *      - count - the number of rows of matrix
469  */
470 void serialize_contiguous( const void *noncontiguous_hvector, void *contiguous_hvector, int count, void *type)
471 {
472   s_smpi_mpi_contiguous_t* type_c = reinterpret_cast<s_smpi_mpi_contiguous_t*>(type);
473   char* contiguous_vector_char = static_cast<char*>(contiguous_hvector);
474   const char* noncontiguous_vector_char = static_cast<const char*>(noncontiguous_hvector)+type_c->lb;
475   memcpy(contiguous_vector_char, noncontiguous_vector_char, count* type_c->block_count * type_c->size_oldtype);
476 }
477 /* Copies contiguous data into noncontiguous memory.
478  *  @param noncontiguous_vector - output hvector
479  *  @param contiguous_vector - input hvector
480  *  @param type - pointer contening :
481  *      - stride - stride of between noncontiguous data, in bytes
482  *      - block_length - the width or height of blocked matrix
483  *      - count - the number of rows of matrix
484  */
485 void unserialize_contiguous(const void *contiguous_vector, void *noncontiguous_vector, int count, void *type, MPI_Op op)
486 {
487   s_smpi_mpi_contiguous_t* type_c = reinterpret_cast<s_smpi_mpi_contiguous_t*>(type);
488   const char* contiguous_vector_char = static_cast<const char*>(contiguous_vector);
489   char* noncontiguous_vector_char = static_cast<char*>(noncontiguous_vector)+type_c->lb;
490   int n= count* type_c->block_count;
491   smpi_op_apply(op, contiguous_vector_char, noncontiguous_vector_char, &n, &type_c->old_type);
492 }
493
494 void free_contiguous(MPI_Datatype* d){
495   smpi_datatype_unuse(reinterpret_cast<s_smpi_mpi_contiguous_t*>((*d)->substruct)->old_type);
496 }
497
498 void use_contiguous(MPI_Datatype* d){
499   smpi_datatype_use(reinterpret_cast<s_smpi_mpi_contiguous_t*>((*d)->substruct)->old_type);
500 }
501
502 /* Create a Sub type contiguous to be able to serialize and unserialize it the structure s_smpi_mpi_contiguous_t is
503  * derived from s_smpi_subtype which required the functions unserialize and serialize */
504 s_smpi_mpi_contiguous_t* smpi_datatype_contiguous_create( MPI_Aint lb, int block_count, MPI_Datatype old_type,
505                                                   int size_oldtype){
506   if(block_count==0)
507     return nullptr;
508   s_smpi_mpi_contiguous_t *new_t= xbt_new(s_smpi_mpi_contiguous_t,1);
509   new_t->base.serialize = &serialize_contiguous;
510   new_t->base.unserialize = &unserialize_contiguous;
511   new_t->base.subtype_free = &free_contiguous;
512   new_t->base.subtype_use = &use_contiguous;
513   new_t->lb = lb;
514   new_t->block_count = block_count;
515   new_t->old_type = old_type;
516   smpi_datatype_use(old_type);
517   new_t->size_oldtype = size_oldtype;
518   smpi_datatype_use(old_type);
519   return new_t;
520 }
521
522 int smpi_datatype_contiguous(int count, MPI_Datatype old_type, MPI_Datatype* new_type, MPI_Aint lb)
523 {
524   int retval;
525   if(old_type->sizeof_substruct){
526     //handle this case as a hvector with stride equals to the extent of the datatype
527     return smpi_datatype_hvector(count, 1, smpi_datatype_get_extent(old_type), old_type, new_type);
528   }
529   
530   s_smpi_mpi_contiguous_t* subtype = smpi_datatype_contiguous_create( lb, count, old_type,smpi_datatype_size(old_type));
531
532   smpi_datatype_create(new_type, count * smpi_datatype_size(old_type),lb,lb + count * smpi_datatype_size(old_type),
533             sizeof(s_smpi_mpi_contiguous_t),subtype, DT_FLAG_CONTIGUOUS);
534   retval=MPI_SUCCESS;
535   return retval;
536 }
537
538 int smpi_datatype_vector(int count, int blocklen, int stride, MPI_Datatype old_type, MPI_Datatype* new_type)
539 {
540   int retval;
541   if (blocklen<0) 
542     return MPI_ERR_ARG;
543   MPI_Aint lb = 0;
544   MPI_Aint ub = 0;
545   if(count>0){
546     lb=smpi_datatype_lb(old_type);
547     ub=((count-1)*stride+blocklen-1)*smpi_datatype_get_extent(old_type)+smpi_datatype_ub(old_type);
548   }
549   if(old_type->sizeof_substruct != 0 || stride != blocklen){
550
551     s_smpi_mpi_vector_t* subtype = smpi_datatype_vector_create(stride, blocklen, count, old_type,
552                                                                 smpi_datatype_size(old_type));
553     smpi_datatype_create(new_type, count * (blocklen) * smpi_datatype_size(old_type), lb, ub, sizeof(s_smpi_mpi_vector_t), subtype,
554                          DT_FLAG_VECTOR);
555     retval=MPI_SUCCESS;
556   }else{
557     /* in this situation the data are contiguous thus it's not required to serialize and unserialize it*/
558     smpi_datatype_create(new_type, count * blocklen * smpi_datatype_size(old_type), 0, ((count -1) * stride + blocklen)*
559                          smpi_datatype_size(old_type), 0, nullptr, DT_FLAG_VECTOR|DT_FLAG_CONTIGUOUS);
560     retval=MPI_SUCCESS;
561   }
562   return retval;
563 }
564
565 void free_vector(MPI_Datatype* d){
566   smpi_datatype_unuse(reinterpret_cast<s_smpi_mpi_indexed_t*>((*d)->substruct)->old_type);
567 }
568
569 void use_vector(MPI_Datatype* d){
570   smpi_datatype_use(reinterpret_cast<s_smpi_mpi_indexed_t*>((*d)->substruct)->old_type);
571 }
572
573 /* Hvector Implementation - Vector with stride in bytes */
574
575 /* Copies noncontiguous data into contiguous memory.
576  *  @param contiguous_hvector - output hvector
577  *  @param noncontiguous_hvector - input hvector
578  *  @param type - pointer contening :
579  *      - stride - stride of between noncontiguous data, in bytes
580  *      - block_length - the width or height of blocked matrix
581  *      - count - the number of rows of matrix
582  */
583 void serialize_hvector( const void *noncontiguous_hvector, void *contiguous_hvector, int count, void *type)
584 {
585   s_smpi_mpi_hvector_t* type_c = reinterpret_cast<s_smpi_mpi_hvector_t*>(type);
586   int i;
587   char* contiguous_vector_char = static_cast<char*>(contiguous_hvector);
588   const char* noncontiguous_vector_char = static_cast<const char*>(noncontiguous_hvector);
589
590   for (i = 0; i < type_c->block_count * count; i++) {
591     if (type_c->old_type->sizeof_substruct == 0)
592       memcpy(contiguous_vector_char, noncontiguous_vector_char, type_c->block_length * type_c->size_oldtype);
593     else
594       static_cast<s_smpi_subtype_t*>(type_c->old_type->substruct)->serialize( noncontiguous_vector_char,
595                                                                    contiguous_vector_char,
596                                                                    type_c->block_length, type_c->old_type->substruct);
597
598     contiguous_vector_char += type_c->block_length*type_c->size_oldtype;
599     if((i+1)%type_c->block_count ==0)
600       noncontiguous_vector_char += type_c->block_length*type_c->size_oldtype;
601     else
602       noncontiguous_vector_char += type_c->block_stride;
603   }
604 }
605 /* Copies contiguous data into noncontiguous memory.
606  *  @param noncontiguous_vector - output hvector
607  *  @param contiguous_vector - input hvector
608  *  @param type - pointer contening :
609  *      - stride - stride of between noncontiguous data, in bytes
610  *      - block_length - the width or height of blocked matrix
611  *      - count - the number of rows of matrix
612  */
613 void unserialize_hvector( const void *contiguous_vector, void *noncontiguous_vector, int count, void *type, MPI_Op op)
614 {
615   s_smpi_mpi_hvector_t* type_c = reinterpret_cast<s_smpi_mpi_hvector_t*>(type);
616   int i;
617
618   const char* contiguous_vector_char = static_cast<const char*>(contiguous_vector);
619   char* noncontiguous_vector_char = static_cast<char*>(noncontiguous_vector);
620
621   for (i = 0; i < type_c->block_count * count; i++) {
622     if (type_c->old_type->sizeof_substruct == 0)
623       smpi_op_apply(op, contiguous_vector_char, noncontiguous_vector_char, &type_c->block_length, &type_c->old_type);
624     else
625       static_cast<s_smpi_subtype_t*>(type_c->old_type->substruct)->unserialize( contiguous_vector_char, noncontiguous_vector_char,
626                                                                      type_c->block_length, type_c->old_type->substruct,
627                                                                      op);
628     contiguous_vector_char += type_c->block_length*type_c->size_oldtype;
629     if((i+1)%type_c->block_count ==0)
630       noncontiguous_vector_char += type_c->block_length*type_c->size_oldtype;
631     else
632       noncontiguous_vector_char += type_c->block_stride;
633   }
634 }
635
636 /* Create a Sub type vector to be able to serialize and unserialize it the structure s_smpi_mpi_vector_t is derived
637  * from s_smpi_subtype which required the functions unserialize and serialize
638  *
639  */
640 s_smpi_mpi_hvector_t* smpi_datatype_hvector_create( MPI_Aint block_stride, int block_length, int block_count,
641                                                   MPI_Datatype old_type, int size_oldtype){
642   s_smpi_mpi_hvector_t *new_t= xbt_new(s_smpi_mpi_hvector_t,1);
643   new_t->base.serialize = &serialize_hvector;
644   new_t->base.unserialize = &unserialize_hvector;
645   new_t->base.subtype_free = &free_hvector;
646   new_t->base.subtype_use = &use_hvector;
647   new_t->block_stride = block_stride;
648   new_t->block_length = block_length;
649   new_t->block_count = block_count;
650   new_t->old_type = old_type;
651   new_t->size_oldtype = size_oldtype;
652   smpi_datatype_use(old_type);
653   return new_t;
654 }
655
656 //do nothing for vector types
657 void free_hvector(MPI_Datatype* d){
658   smpi_datatype_unuse(reinterpret_cast<s_smpi_mpi_hvector_t*>((*d)->substruct)->old_type);
659 }
660
661 void use_hvector(MPI_Datatype* d){
662   smpi_datatype_use(reinterpret_cast<s_smpi_mpi_hvector_t*>((*d)->substruct)->old_type);
663 }
664
665 int smpi_datatype_hvector(int count, int blocklen, MPI_Aint stride, MPI_Datatype old_type, MPI_Datatype* new_type)
666 {
667   int retval;
668   if (blocklen<0) 
669     return MPI_ERR_ARG;
670   MPI_Aint lb = 0;
671   MPI_Aint ub = 0;
672   if(count>0){
673     lb=smpi_datatype_lb(old_type);
674     ub=((count-1)*stride)+(blocklen-1)*smpi_datatype_get_extent(old_type)+smpi_datatype_ub(old_type);
675   }
676   if(old_type->sizeof_substruct != 0 || stride != blocklen*smpi_datatype_get_extent(old_type)){
677     s_smpi_mpi_hvector_t* subtype = smpi_datatype_hvector_create( stride, blocklen, count, old_type,
678                                                                   smpi_datatype_size(old_type));
679
680     smpi_datatype_create(new_type, count * blocklen * smpi_datatype_size(old_type), lb,ub, sizeof(s_smpi_mpi_hvector_t), subtype, DT_FLAG_VECTOR);
681     retval=MPI_SUCCESS;
682   }else{
683     smpi_datatype_create(new_type, count * blocklen * smpi_datatype_size(old_type),0,count * blocklen *
684                                              smpi_datatype_size(old_type), 0, nullptr, DT_FLAG_VECTOR|DT_FLAG_CONTIGUOUS);
685     retval=MPI_SUCCESS;
686   }
687   return retval;
688 }
689
690 /* Indexed Implementation */
691
692 /* Copies noncontiguous data into contiguous memory.
693  *  @param contiguous_indexed - output indexed
694  *  @param noncontiguous_indexed - input indexed
695  *  @param type - pointer contening :
696  *      - block_lengths - the width or height of blocked matrix
697  *      - block_indices - indices of each data, in element
698  *      - count - the number of rows of matrix
699  */
700 void serialize_indexed( const void *noncontiguous_indexed, void *contiguous_indexed, int count, void *type)
701 {
702   s_smpi_mpi_indexed_t* type_c = reinterpret_cast<s_smpi_mpi_indexed_t*>(type);
703   int i,j;
704   char* contiguous_indexed_char = static_cast<char*>(contiguous_indexed);
705   const char* noncontiguous_indexed_char = static_cast<const char*>(noncontiguous_indexed)+type_c->block_indices[0] * type_c->size_oldtype;
706   for(j=0; j<count;j++){
707     for (i = 0; i < type_c->block_count; i++) {
708       if (type_c->old_type->sizeof_substruct == 0)
709         memcpy(contiguous_indexed_char, noncontiguous_indexed_char, type_c->block_lengths[i] * type_c->size_oldtype);
710       else
711         static_cast<s_smpi_subtype_t*>(type_c->old_type->substruct)->serialize( noncontiguous_indexed_char,
712                                                                      contiguous_indexed_char,
713                                                                      type_c->block_lengths[i],
714                                                                      type_c->old_type->substruct);
715
716       contiguous_indexed_char += type_c->block_lengths[i]*type_c->size_oldtype;
717       if (i<type_c->block_count-1)
718         noncontiguous_indexed_char =
719           static_cast<const char*>(noncontiguous_indexed) + type_c->block_indices[i+1]*smpi_datatype_get_extent(type_c->old_type);
720       else
721         noncontiguous_indexed_char += type_c->block_lengths[i]*smpi_datatype_get_extent(type_c->old_type);
722     }
723     noncontiguous_indexed=static_cast<const void*>(noncontiguous_indexed_char);
724   }
725 }
726 /* Copies contiguous data into noncontiguous memory.
727  *  @param noncontiguous_indexed - output indexed
728  *  @param contiguous_indexed - input indexed
729  *  @param type - pointer contening :
730  *      - block_lengths - the width or height of blocked matrix
731  *      - block_indices - indices of each data, in element
732  *      - count - the number of rows of matrix
733  */
734 void unserialize_indexed( const void *contiguous_indexed, void *noncontiguous_indexed, int count, void *type, MPI_Op op)
735 {
736   s_smpi_mpi_indexed_t* type_c = reinterpret_cast<s_smpi_mpi_indexed_t*>(type);
737   int i,j;
738   const char* contiguous_indexed_char = static_cast<const char*>(contiguous_indexed);
739   char* noncontiguous_indexed_char =
740     static_cast<char*>(noncontiguous_indexed)+type_c->block_indices[0]*smpi_datatype_get_extent(type_c->old_type);
741   for(j=0; j<count;j++){
742     for (i = 0; i < type_c->block_count; i++) {
743       if (type_c->old_type->sizeof_substruct == 0)
744         smpi_op_apply(op, contiguous_indexed_char, noncontiguous_indexed_char, &type_c->block_lengths[i],
745                     &type_c->old_type);
746       else
747         static_cast<s_smpi_subtype_t*>(type_c->old_type->substruct)->unserialize( contiguous_indexed_char,
748                                                                        noncontiguous_indexed_char,
749                                                                        type_c->block_lengths[i],
750                                                                        type_c->old_type->substruct, op);
751
752       contiguous_indexed_char += type_c->block_lengths[i]*type_c->size_oldtype;
753       if (i<type_c->block_count-1)
754         noncontiguous_indexed_char =
755           static_cast<char*>(noncontiguous_indexed) + type_c->block_indices[i+1]*smpi_datatype_get_extent(type_c->old_type);
756       else
757         noncontiguous_indexed_char += type_c->block_lengths[i]*smpi_datatype_get_extent(type_c->old_type);
758     }
759     noncontiguous_indexed=static_cast<void*>(noncontiguous_indexed_char);
760   }
761 }
762
763 void free_indexed(MPI_Datatype* type){
764   if((*type)->in_use==0){
765     xbt_free(reinterpret_cast<s_smpi_mpi_indexed_t*>((*type)->substruct)->block_lengths);
766     xbt_free(reinterpret_cast<s_smpi_mpi_indexed_t*>((*type)->substruct)->block_indices);
767   }
768   smpi_datatype_unuse(reinterpret_cast<s_smpi_mpi_indexed_t*>((*type)->substruct)->old_type);
769 }
770
771 void use_indexed(MPI_Datatype* type){
772   smpi_datatype_use(reinterpret_cast<s_smpi_mpi_indexed_t*>((*type)->substruct)->old_type);
773 }
774
775
776 /* Create a Sub type indexed to be able to serialize and unserialize it the structure s_smpi_mpi_indexed_t is derived
777  * from s_smpi_subtype which required the functions unserialize and serialize */
778 s_smpi_mpi_indexed_t* smpi_datatype_indexed_create( int* block_lengths, int* block_indices, int block_count,
779                                                   MPI_Datatype old_type, int size_oldtype){
780   s_smpi_mpi_indexed_t *new_t= xbt_new(s_smpi_mpi_indexed_t,1);
781   new_t->base.serialize = &serialize_indexed;
782   new_t->base.unserialize = &unserialize_indexed;
783   new_t->base.subtype_free = &free_indexed;
784   new_t->base.subtype_use = &use_indexed;
785   new_t->block_lengths= xbt_new(int, block_count);
786   new_t->block_indices= xbt_new(int, block_count);
787   int i;
788   for(i=0;i<block_count;i++){
789     new_t->block_lengths[i]=block_lengths[i];
790     new_t->block_indices[i]=block_indices[i];
791   }
792   new_t->block_count = block_count;
793   smpi_datatype_use(old_type);
794   new_t->old_type = old_type;
795   new_t->size_oldtype = size_oldtype;
796   return new_t;
797 }
798
799 int smpi_datatype_indexed(int count, int* blocklens, int* indices, MPI_Datatype old_type, MPI_Datatype* new_type)
800 {
801   int i;
802   int retval;
803   int size = 0;
804   bool contiguous=true;
805   MPI_Aint lb = 0;
806   MPI_Aint ub = 0;
807   if(count>0){
808     lb=indices[0]*smpi_datatype_get_extent(old_type);
809     ub=indices[0]*smpi_datatype_get_extent(old_type) + blocklens[0]*smpi_datatype_ub(old_type);
810   }
811
812   for(i=0; i< count; i++){
813     if   (blocklens[i]<0)
814       return MPI_ERR_ARG;
815     size += blocklens[i];
816
817     if(indices[i]*smpi_datatype_get_extent(old_type)+smpi_datatype_lb(old_type)<lb)
818       lb = indices[i]*smpi_datatype_get_extent(old_type)+smpi_datatype_lb(old_type);
819     if(indices[i]*smpi_datatype_get_extent(old_type)+blocklens[i]*smpi_datatype_ub(old_type)>ub)
820       ub = indices[i]*smpi_datatype_get_extent(old_type)+blocklens[i]*smpi_datatype_ub(old_type);
821
822     if ( (i< count -1) && (indices[i]+blocklens[i] != indices[i+1]) )
823       contiguous=false;
824   }
825   if (old_type->sizeof_substruct != 0)
826     contiguous=false;
827
828   if(!contiguous){
829     s_smpi_mpi_indexed_t* subtype = smpi_datatype_indexed_create( blocklens, indices, count, old_type,
830                                                                   smpi_datatype_size(old_type));
831      smpi_datatype_create(new_type,  size * smpi_datatype_size(old_type),lb,ub,sizeof(s_smpi_mpi_indexed_t), subtype, DT_FLAG_DATA);
832   }else{
833     s_smpi_mpi_contiguous_t* subtype = smpi_datatype_contiguous_create( lb, size, old_type,
834                                                                   smpi_datatype_size(old_type));
835     smpi_datatype_create(new_type, size * smpi_datatype_size(old_type), lb, ub, sizeof(s_smpi_mpi_contiguous_t), subtype,
836                          DT_FLAG_DATA|DT_FLAG_CONTIGUOUS);
837   }
838   retval=MPI_SUCCESS;
839   return retval;
840 }
841 /* Hindexed Implementation - Indexed with indices in bytes */
842
843 /* Copies noncontiguous data into contiguous memory.
844  *  @param contiguous_hindexed - output hindexed
845  *  @param noncontiguous_hindexed - input hindexed
846  *  @param type - pointer contening :
847  *      - block_lengths - the width or height of blocked matrix
848  *      - block_indices - indices of each data, in bytes
849  *      - count - the number of rows of matrix
850  */
851 void serialize_hindexed( const void *noncontiguous_hindexed, void *contiguous_hindexed, int count, void *type)
852 {
853   s_smpi_mpi_hindexed_t* type_c = reinterpret_cast<s_smpi_mpi_hindexed_t*>(type);
854   int i,j;
855   char* contiguous_hindexed_char = static_cast<char*>(contiguous_hindexed);
856   const char* noncontiguous_hindexed_char = static_cast<const char*>(noncontiguous_hindexed)+ type_c->block_indices[0];
857   for(j=0; j<count;j++){
858     for (i = 0; i < type_c->block_count; i++) {
859       if (type_c->old_type->sizeof_substruct == 0)
860         memcpy(contiguous_hindexed_char, noncontiguous_hindexed_char, type_c->block_lengths[i] * type_c->size_oldtype);
861       else
862         static_cast<s_smpi_subtype_t*>(type_c->old_type->substruct)->serialize( noncontiguous_hindexed_char,
863                                                                      contiguous_hindexed_char,
864                                                                      type_c->block_lengths[i],
865                                                                      type_c->old_type->substruct);
866
867       contiguous_hindexed_char += type_c->block_lengths[i]*type_c->size_oldtype;
868       if (i<type_c->block_count-1)
869         noncontiguous_hindexed_char = static_cast<const char*>(noncontiguous_hindexed) + type_c->block_indices[i+1];
870       else
871         noncontiguous_hindexed_char += type_c->block_lengths[i]*smpi_datatype_get_extent(type_c->old_type);
872     }
873     noncontiguous_hindexed=reinterpret_cast<const void*>(noncontiguous_hindexed_char);
874   }
875 }
876 /* Copies contiguous data into noncontiguous memory.
877  *  @param noncontiguous_hindexed - output hindexed
878  *  @param contiguous_hindexed - input hindexed
879  *  @param type - pointer contening :
880  *      - block_lengths - the width or height of blocked matrix
881  *      - block_indices - indices of each data, in bytes
882  *      - count - the number of rows of matrix
883  */
884 void unserialize_hindexed( const void *contiguous_hindexed, void *noncontiguous_hindexed, int count, void *type,
885                          MPI_Op op)
886 {
887   s_smpi_mpi_hindexed_t* type_c = reinterpret_cast<s_smpi_mpi_hindexed_t*>(type);
888   int i,j;
889
890   const char* contiguous_hindexed_char = static_cast<const char*>(contiguous_hindexed);
891   char* noncontiguous_hindexed_char = static_cast<char*>(noncontiguous_hindexed)+ type_c->block_indices[0];
892   for(j=0; j<count;j++){
893     for (i = 0; i < type_c->block_count; i++) {
894       if (type_c->old_type->sizeof_substruct == 0)
895         smpi_op_apply(op, contiguous_hindexed_char, noncontiguous_hindexed_char, &type_c->block_lengths[i],
896                             &type_c->old_type);
897       else
898         static_cast<s_smpi_subtype_t*>(type_c->old_type->substruct)->unserialize( contiguous_hindexed_char,
899                                                                        noncontiguous_hindexed_char,
900                                                                        type_c->block_lengths[i],
901                                                                        type_c->old_type->substruct, op);
902
903       contiguous_hindexed_char += type_c->block_lengths[i]*type_c->size_oldtype;
904       if (i<type_c->block_count-1)
905         noncontiguous_hindexed_char = static_cast<char*>(noncontiguous_hindexed) + type_c->block_indices[i+1];
906       else
907         noncontiguous_hindexed_char += type_c->block_lengths[i]*smpi_datatype_get_extent(type_c->old_type);
908     }
909     noncontiguous_hindexed=reinterpret_cast<void*>(noncontiguous_hindexed_char);
910   }
911 }
912
913 void free_hindexed(MPI_Datatype* type){
914   if((*type)->in_use==0){
915     xbt_free(reinterpret_cast<s_smpi_mpi_hindexed_t*>((*type)->substruct)->block_lengths);
916     xbt_free(reinterpret_cast<s_smpi_mpi_hindexed_t*>((*type)->substruct)->block_indices);
917   }
918   smpi_datatype_unuse(reinterpret_cast<s_smpi_mpi_hindexed_t*>((*type)->substruct)->old_type);
919 }
920
921 void use_hindexed(MPI_Datatype* type){
922   smpi_datatype_use(reinterpret_cast<s_smpi_mpi_hindexed_t*>((*type)->substruct)->old_type);
923 }
924
925 /* Create a Sub type hindexed to be able to serialize and unserialize it the structure s_smpi_mpi_hindexed_t is derived
926  * from s_smpi_subtype which required the functions unserialize and serialize
927  */
928 s_smpi_mpi_hindexed_t* smpi_datatype_hindexed_create( int* block_lengths, MPI_Aint* block_indices, int block_count,
929                                                   MPI_Datatype old_type, int size_oldtype){
930   s_smpi_mpi_hindexed_t *new_t= xbt_new(s_smpi_mpi_hindexed_t,1);
931   new_t->base.serialize = &serialize_hindexed;
932   new_t->base.unserialize = &unserialize_hindexed;
933   new_t->base.subtype_free = &free_hindexed;
934   new_t->base.subtype_use = &use_hindexed;
935   new_t->block_lengths= xbt_new(int, block_count);
936   new_t->block_indices= xbt_new(MPI_Aint, block_count);
937   for(int i=0;i<block_count;i++){
938     new_t->block_lengths[i]=block_lengths[i];
939     new_t->block_indices[i]=block_indices[i];
940   }
941   new_t->block_count = block_count;
942   new_t->old_type = old_type;
943   smpi_datatype_use(old_type);
944   new_t->size_oldtype = size_oldtype;
945   return new_t;
946 }
947
948 int smpi_datatype_hindexed(int count, int* blocklens, MPI_Aint* indices, MPI_Datatype old_type, MPI_Datatype* new_type)
949 {
950   int i;
951   int retval;
952   int size = 0;
953   bool contiguous=true;
954   MPI_Aint lb = 0;
955   MPI_Aint ub = 0;
956   if(count>0){
957     lb=indices[0] + smpi_datatype_lb(old_type);
958     ub=indices[0] + blocklens[0]*smpi_datatype_ub(old_type);
959   }
960   for(i=0; i< count; i++){
961     if   (blocklens[i]<0)
962       return MPI_ERR_ARG;
963     size += blocklens[i];
964
965     if(indices[i]+smpi_datatype_lb(old_type)<lb) 
966       lb = indices[i]+smpi_datatype_lb(old_type);
967     if(indices[i]+blocklens[i]*smpi_datatype_ub(old_type)>ub) 
968       ub = indices[i]+blocklens[i]*smpi_datatype_ub(old_type);
969
970     if ( (i< count -1) && (indices[i]+blocklens[i]*(static_cast<int>(smpi_datatype_size(old_type))) != indices[i+1]) )
971       contiguous=false;
972   }
973   if (old_type->sizeof_substruct != 0 || lb!=0)
974     contiguous=false;
975
976   if(!contiguous){
977     s_smpi_mpi_hindexed_t* subtype = smpi_datatype_hindexed_create( blocklens, indices, count, old_type,
978                                                                   smpi_datatype_size(old_type));
979     smpi_datatype_create(new_type,  size * smpi_datatype_size(old_type), lb, ub ,sizeof(s_smpi_mpi_hindexed_t), subtype, DT_FLAG_DATA);
980   }else{
981     s_smpi_mpi_contiguous_t* subtype = smpi_datatype_contiguous_create(lb,size, old_type, smpi_datatype_size(old_type));
982     smpi_datatype_create(new_type,  size * smpi_datatype_size(old_type), 0,size * smpi_datatype_size(old_type),
983                1, subtype, DT_FLAG_DATA|DT_FLAG_CONTIGUOUS);
984   }
985   retval=MPI_SUCCESS;
986   return retval;
987 }
988
989 /* struct Implementation - Indexed with indices in bytes */
990
991 /* Copies noncontiguous data into contiguous memory.
992  *  @param contiguous_struct - output struct
993  *  @param noncontiguous_struct - input struct
994  *  @param type - pointer contening :
995  *      - stride - stride of between noncontiguous data
996  *      - block_length - the width or height of blocked matrix
997  *      - count - the number of rows of matrix
998  */
999 void serialize_struct( const void *noncontiguous_struct, void *contiguous_struct, int count, void *type)
1000 {
1001   s_smpi_mpi_struct_t* type_c = reinterpret_cast<s_smpi_mpi_struct_t*>(type);
1002   int i,j;
1003   char* contiguous_struct_char = static_cast<char*>(contiguous_struct);
1004   const char* noncontiguous_struct_char = static_cast<const char*>(noncontiguous_struct)+ type_c->block_indices[0];
1005   for(j=0; j<count;j++){
1006     for (i = 0; i < type_c->block_count; i++) {
1007       if (type_c->old_types[i]->sizeof_substruct == 0)
1008         memcpy(contiguous_struct_char, noncontiguous_struct_char,
1009                type_c->block_lengths[i] * smpi_datatype_size(type_c->old_types[i]));
1010       else
1011         static_cast<s_smpi_subtype_t*>(type_c->old_types[i]->substruct)->serialize( noncontiguous_struct_char,
1012                                                                          contiguous_struct_char,
1013                                                                          type_c->block_lengths[i],
1014                                                                          type_c->old_types[i]->substruct);
1015
1016
1017       contiguous_struct_char += type_c->block_lengths[i]*smpi_datatype_size(type_c->old_types[i]);
1018       if (i<type_c->block_count-1)
1019         noncontiguous_struct_char = static_cast<const char*>(noncontiguous_struct) + type_c->block_indices[i+1];
1020       else //let's hope this is MPI_UB ?
1021         noncontiguous_struct_char += type_c->block_lengths[i]*smpi_datatype_get_extent(type_c->old_types[i]);
1022     }
1023     noncontiguous_struct=reinterpret_cast<const void*>(noncontiguous_struct_char);
1024   }
1025 }
1026
1027 /* Copies contiguous data into noncontiguous memory.
1028  *  @param noncontiguous_struct - output struct
1029  *  @param contiguous_struct - input struct
1030  *  @param type - pointer contening :
1031  *      - stride - stride of between noncontiguous data
1032  *      - block_length - the width or height of blocked matrix
1033  *      - count - the number of rows of matrix
1034  */
1035 void unserialize_struct( const void *contiguous_struct, void *noncontiguous_struct, int count, void *type, MPI_Op op)
1036 {
1037   s_smpi_mpi_struct_t* type_c = reinterpret_cast<s_smpi_mpi_struct_t*>(type);
1038   int i,j;
1039
1040   const char* contiguous_struct_char = static_cast<const char*>(contiguous_struct);
1041   char* noncontiguous_struct_char = static_cast<char*>(noncontiguous_struct)+ type_c->block_indices[0];
1042   for(j=0; j<count;j++){
1043     for (i = 0; i < type_c->block_count; i++) {
1044       if (type_c->old_types[i]->sizeof_substruct == 0)
1045         smpi_op_apply(op, contiguous_struct_char, noncontiguous_struct_char, &type_c->block_lengths[i],
1046            & type_c->old_types[i]);
1047       else
1048         static_cast<s_smpi_subtype_t*>(type_c->old_types[i]->substruct)->unserialize( contiguous_struct_char,
1049                                                                            noncontiguous_struct_char,
1050                                                                            type_c->block_lengths[i],
1051                                                                            type_c->old_types[i]->substruct, op);
1052
1053       contiguous_struct_char += type_c->block_lengths[i]*smpi_datatype_size(type_c->old_types[i]);
1054       if (i<type_c->block_count-1)
1055         noncontiguous_struct_char =  static_cast<char*>(noncontiguous_struct) + type_c->block_indices[i+1];
1056       else
1057         noncontiguous_struct_char += type_c->block_lengths[i]*smpi_datatype_get_extent(type_c->old_types[i]);
1058     }
1059     noncontiguous_struct=reinterpret_cast<void*>(noncontiguous_struct_char);
1060   }
1061 }
1062
1063 void free_struct(MPI_Datatype* type){
1064   int i=0;
1065   for (i = 0; i < reinterpret_cast<s_smpi_mpi_struct_t*>((*type)->substruct)->block_count; i++)
1066     smpi_datatype_unuse(reinterpret_cast<s_smpi_mpi_struct_t*>((*type)->substruct)->old_types[i]);
1067   if((*type)->in_use==0){
1068     xbt_free(reinterpret_cast<s_smpi_mpi_struct_t*>((*type)->substruct)->block_lengths);
1069     xbt_free(reinterpret_cast<s_smpi_mpi_struct_t*>((*type)->substruct)->block_indices);
1070     xbt_free(reinterpret_cast<s_smpi_mpi_struct_t*>((*type)->substruct)->old_types);
1071   }
1072 }
1073
1074 void use_struct(MPI_Datatype* type){
1075   int i=0;
1076   for (i = 0; i < reinterpret_cast<s_smpi_mpi_struct_t*>((*type)->substruct)->block_count; i++)
1077     smpi_datatype_use(reinterpret_cast<s_smpi_mpi_struct_t*>((*type)->substruct)->old_types[i]);
1078 }
1079
1080 /* Create a Sub type struct to be able to serialize and unserialize it the structure s_smpi_mpi_struct_t is derived
1081  * from s_smpi_subtype which required the functions unserialize and serialize
1082  */
1083 s_smpi_mpi_struct_t* smpi_datatype_struct_create( int* block_lengths, MPI_Aint* block_indices, int block_count,
1084                                                   MPI_Datatype* old_types){
1085   s_smpi_mpi_struct_t *new_t= xbt_new(s_smpi_mpi_struct_t,1);
1086   new_t->base.serialize = &serialize_struct;
1087   new_t->base.unserialize = &unserialize_struct;
1088   new_t->base.subtype_free = &free_struct;
1089   new_t->base.subtype_use = &use_struct;
1090   new_t->block_lengths= xbt_new(int, block_count);
1091   new_t->block_indices= xbt_new(MPI_Aint, block_count);
1092   new_t->old_types=  xbt_new(MPI_Datatype, block_count);
1093   int i;
1094   for(i=0;i<block_count;i++){
1095     new_t->block_lengths[i]=block_lengths[i];
1096     new_t->block_indices[i]=block_indices[i];
1097     new_t->old_types[i]=old_types[i];
1098     smpi_datatype_use(new_t->old_types[i]);
1099   }
1100   new_t->block_count = block_count;
1101   return new_t;
1102 }
1103
1104 int smpi_datatype_struct(int count, int* blocklens, MPI_Aint* indices, MPI_Datatype* old_types, MPI_Datatype* new_type)
1105 {
1106   int i;
1107   size_t size = 0;
1108   bool contiguous=true;
1109   size = 0;
1110   MPI_Aint lb = 0;
1111   MPI_Aint ub = 0;
1112   if(count>0){
1113     lb=indices[0] + smpi_datatype_lb(old_types[0]);
1114     ub=indices[0] + blocklens[0]*smpi_datatype_ub(old_types[0]);
1115   }
1116   bool forced_lb=false;
1117   bool forced_ub=false;
1118   for(i=0; i< count; i++){
1119     if (blocklens[i]<0)
1120       return MPI_ERR_ARG;
1121     if (old_types[i]->sizeof_substruct != 0)
1122       contiguous=false;
1123
1124     size += blocklens[i]*smpi_datatype_size(old_types[i]);
1125     if (old_types[i]==MPI_LB){
1126       lb=indices[i];
1127       forced_lb=true;
1128     }
1129     if (old_types[i]==MPI_UB){
1130       ub=indices[i];
1131       forced_ub=true;
1132     }
1133
1134     if(!forced_lb && indices[i]+smpi_datatype_lb(old_types[i])<lb) 
1135       lb = indices[i];
1136     if(!forced_ub &&  indices[i]+blocklens[i]*smpi_datatype_ub(old_types[i])>ub)
1137       ub = indices[i]+blocklens[i]*smpi_datatype_ub(old_types[i]);
1138
1139     if ( (i< count -1) && (indices[i]+blocklens[i]*static_cast<int>(smpi_datatype_size(old_types[i])) != indices[i+1]) )
1140       contiguous=false;
1141   }
1142
1143   if(!contiguous){
1144     s_smpi_mpi_struct_t* subtype = smpi_datatype_struct_create( blocklens, indices, count, old_types);
1145
1146     smpi_datatype_create(new_type,  size, lb, ub,sizeof(s_smpi_mpi_struct_t), subtype, DT_FLAG_DATA);
1147   }else{
1148     s_smpi_mpi_contiguous_t* subtype = smpi_datatype_contiguous_create( lb, size, MPI_CHAR, 1);
1149     smpi_datatype_create(new_type,  size, lb, ub,1, subtype, DT_FLAG_DATA|DT_FLAG_CONTIGUOUS);
1150   }
1151   return MPI_SUCCESS;
1152 }
1153
1154 void smpi_datatype_commit(MPI_Datatype *datatype)
1155 {
1156   (*datatype)->flags=  ((*datatype)->flags | DT_FLAG_COMMITED);
1157 }
1158
1159 typedef struct s_smpi_mpi_op {
1160   MPI_User_function *func;
1161   bool is_commute;
1162 } s_smpi_mpi_op_t;
1163
1164 #define MAX_OP(a, b)  (b) = (a) < (b) ? (b) : (a)
1165 #define MIN_OP(a, b)  (b) = (a) < (b) ? (a) : (b)
1166 #define SUM_OP(a, b)  (b) += (a)
1167 #define PROD_OP(a, b) (b) *= (a)
1168 #define LAND_OP(a, b) (b) = (a) && (b)
1169 #define LOR_OP(a, b)  (b) = (a) || (b)
1170 #define LXOR_OP(a, b) (b) = (!(a) && (b)) || ((a) && !(b))
1171 #define BAND_OP(a, b) (b) &= (a)
1172 #define BOR_OP(a, b)  (b) |= (a)
1173 #define BXOR_OP(a, b) (b) ^= (a)
1174 #define MAXLOC_OP(a, b)  (b) = (a.value) < (b.value) ? (b) : (a)
1175 #define MINLOC_OP(a, b)  (b) = (a.value) < (b.value) ? (a) : (b)
1176
1177 #define APPLY_FUNC(a, b, length, type, func) \
1178 {                                          \
1179   int i;                                   \
1180   type* x = (type*)(a);                    \
1181   type* y = (type*)(b);                    \
1182   for(i = 0; i < *(length); i++) {         \
1183     func(x[i], y[i]);                      \
1184   }                                        \
1185 }
1186
1187 #define APPLY_OP_LOOP(dtype, type, op) \
1188   if (*datatype == dtype) {\
1189     APPLY_FUNC(a, b, length, type, op)\
1190   } else \
1191
1192
1193 #define APPLY_BASIC_OP_LOOP(op)\
1194 APPLY_OP_LOOP(MPI_CHAR, char,op)\
1195 APPLY_OP_LOOP(MPI_SHORT, short,op)\
1196 APPLY_OP_LOOP(MPI_INT, int,op)\
1197 APPLY_OP_LOOP(MPI_LONG, long,op)\
1198 APPLY_OP_LOOP(MPI_LONG_LONG, long long,op)\
1199 APPLY_OP_LOOP(MPI_SIGNED_CHAR, signed char,op)\
1200 APPLY_OP_LOOP(MPI_UNSIGNED_CHAR, unsigned char,op)\
1201 APPLY_OP_LOOP(MPI_UNSIGNED_SHORT, unsigned short,op)\
1202 APPLY_OP_LOOP(MPI_UNSIGNED, unsigned int,op)\
1203 APPLY_OP_LOOP(MPI_UNSIGNED_LONG, unsigned long,op)\
1204 APPLY_OP_LOOP(MPI_UNSIGNED_LONG_LONG, unsigned long long,op)\
1205 APPLY_OP_LOOP(MPI_WCHAR, wchar_t,op)\
1206 APPLY_OP_LOOP(MPI_BYTE, int8_t,op)\
1207 APPLY_OP_LOOP(MPI_INT8_T, int8_t,op)\
1208 APPLY_OP_LOOP(MPI_INT16_T, int16_t,op)\
1209 APPLY_OP_LOOP(MPI_INT32_T, int32_t,op)\
1210 APPLY_OP_LOOP(MPI_INT64_T, int64_t,op)\
1211 APPLY_OP_LOOP(MPI_UINT8_T, uint8_t,op)\
1212 APPLY_OP_LOOP(MPI_UINT16_T, uint16_t,op)\
1213 APPLY_OP_LOOP(MPI_UINT32_T, uint32_t,op)\
1214 APPLY_OP_LOOP(MPI_UINT64_T, uint64_t,op)\
1215 APPLY_OP_LOOP(MPI_AINT, MPI_Aint,op)\
1216 APPLY_OP_LOOP(MPI_OFFSET, MPI_Offset,op)\
1217 APPLY_OP_LOOP(MPI_INTEGER1, int,op)\
1218 APPLY_OP_LOOP(MPI_INTEGER2, int16_t,op)\
1219 APPLY_OP_LOOP(MPI_INTEGER4, int32_t,op)\
1220 APPLY_OP_LOOP(MPI_INTEGER8, int64_t,op)
1221
1222 #define APPLY_BOOL_OP_LOOP(op)\
1223 APPLY_OP_LOOP(MPI_C_BOOL, bool,op)
1224
1225 #define APPLY_FLOAT_OP_LOOP(op)\
1226 APPLY_OP_LOOP(MPI_FLOAT, float,op)\
1227 APPLY_OP_LOOP(MPI_DOUBLE, double,op)\
1228 APPLY_OP_LOOP(MPI_LONG_DOUBLE, long double,op)\
1229 APPLY_OP_LOOP(MPI_REAL, float,op)\
1230 APPLY_OP_LOOP(MPI_REAL4, float,op)\
1231 APPLY_OP_LOOP(MPI_REAL8, float,op)\
1232 APPLY_OP_LOOP(MPI_REAL16, double,op)
1233
1234 #define APPLY_COMPLEX_OP_LOOP(op)\
1235 APPLY_OP_LOOP(MPI_C_FLOAT_COMPLEX, float _Complex,op)\
1236 APPLY_OP_LOOP(MPI_C_DOUBLE_COMPLEX, double _Complex,op)\
1237 APPLY_OP_LOOP(MPI_C_LONG_DOUBLE_COMPLEX, long double _Complex,op)
1238
1239 #define APPLY_PAIR_OP_LOOP(op)\
1240 APPLY_OP_LOOP(MPI_FLOAT_INT, float_int,op)\
1241 APPLY_OP_LOOP(MPI_LONG_INT, long_int,op)\
1242 APPLY_OP_LOOP(MPI_DOUBLE_INT, double_int,op)\
1243 APPLY_OP_LOOP(MPI_SHORT_INT, short_int,op)\
1244 APPLY_OP_LOOP(MPI_2INT, int_int,op)\
1245 APPLY_OP_LOOP(MPI_2FLOAT, float_float,op)\
1246 APPLY_OP_LOOP(MPI_2DOUBLE, double_double,op)\
1247 APPLY_OP_LOOP(MPI_LONG_DOUBLE_INT, long_double_int,op)\
1248 APPLY_OP_LOOP(MPI_2LONG, long_long,op)
1249
1250 #define APPLY_END_OP_LOOP(op)\
1251   {\
1252     xbt_die("Failed to apply " #op " to type %s", (*datatype)->name);\
1253   }
1254
1255
1256 static void max_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1257 {
1258   APPLY_BASIC_OP_LOOP(MAX_OP)
1259   APPLY_FLOAT_OP_LOOP(MAX_OP)
1260   APPLY_END_OP_LOOP(MAX_OP)
1261 }
1262
1263 static void min_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1264 {
1265   APPLY_BASIC_OP_LOOP(MIN_OP)
1266   APPLY_FLOAT_OP_LOOP(MIN_OP)
1267   APPLY_END_OP_LOOP(MIN_OP)
1268 }
1269
1270 static void sum_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1271 {
1272   APPLY_BASIC_OP_LOOP(SUM_OP)
1273   APPLY_FLOAT_OP_LOOP(SUM_OP)
1274   APPLY_COMPLEX_OP_LOOP(SUM_OP)
1275   APPLY_END_OP_LOOP(SUM_OP)
1276 }
1277
1278 static void prod_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1279 {
1280   APPLY_BASIC_OP_LOOP(PROD_OP)
1281   APPLY_FLOAT_OP_LOOP(PROD_OP)
1282   APPLY_COMPLEX_OP_LOOP(PROD_OP)
1283   APPLY_END_OP_LOOP(PROD_OP)
1284 }
1285
1286 static void land_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1287 {
1288   APPLY_BASIC_OP_LOOP(LAND_OP)
1289   APPLY_BOOL_OP_LOOP(LAND_OP)
1290   APPLY_END_OP_LOOP(LAND_OP)
1291 }
1292
1293 static void lor_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1294 {
1295   APPLY_BASIC_OP_LOOP(LOR_OP)
1296   APPLY_BOOL_OP_LOOP(LOR_OP)
1297   APPLY_END_OP_LOOP(LOR_OP)
1298 }
1299
1300 static void lxor_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1301 {
1302   APPLY_BASIC_OP_LOOP(LXOR_OP)
1303   APPLY_BOOL_OP_LOOP(LXOR_OP)
1304   APPLY_END_OP_LOOP(LXOR_OP)
1305 }
1306
1307 static void band_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1308 {
1309   APPLY_BASIC_OP_LOOP(BAND_OP)
1310   APPLY_BOOL_OP_LOOP(BAND_OP)
1311   APPLY_END_OP_LOOP(BAND_OP)
1312 }
1313
1314 static void bor_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1315 {
1316   APPLY_BASIC_OP_LOOP(BOR_OP)
1317   APPLY_BOOL_OP_LOOP(BOR_OP)
1318   APPLY_END_OP_LOOP(BOR_OP)
1319 }
1320
1321 static void bxor_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1322 {
1323   APPLY_BASIC_OP_LOOP(BXOR_OP)
1324   APPLY_BOOL_OP_LOOP(BXOR_OP)
1325   APPLY_END_OP_LOOP(BXOR_OP)
1326 }
1327
1328 static void minloc_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1329 {
1330   APPLY_PAIR_OP_LOOP(MINLOC_OP)
1331   APPLY_END_OP_LOOP(MINLOC_OP)
1332 }
1333
1334 static void maxloc_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1335 {
1336   APPLY_PAIR_OP_LOOP(MAXLOC_OP)
1337   APPLY_END_OP_LOOP(MAXLOC_OP)
1338 }
1339
1340 static void replace_func(void *a, void *b, int *length, MPI_Datatype * datatype)
1341 {
1342   memcpy(b, a, *length * smpi_datatype_size(*datatype));
1343 }
1344
1345 #define CREATE_MPI_OP(name, func)                             \
1346   static s_smpi_mpi_op_t mpi_##name = { &(func) /* func */, true }; \
1347 MPI_Op name = &mpi_##name;
1348
1349 CREATE_MPI_OP(MPI_MAX, max_func);
1350 CREATE_MPI_OP(MPI_MIN, min_func);
1351 CREATE_MPI_OP(MPI_SUM, sum_func);
1352 CREATE_MPI_OP(MPI_PROD, prod_func);
1353 CREATE_MPI_OP(MPI_LAND, land_func);
1354 CREATE_MPI_OP(MPI_LOR, lor_func);
1355 CREATE_MPI_OP(MPI_LXOR, lxor_func);
1356 CREATE_MPI_OP(MPI_BAND, band_func);
1357 CREATE_MPI_OP(MPI_BOR, bor_func);
1358 CREATE_MPI_OP(MPI_BXOR, bxor_func);
1359 CREATE_MPI_OP(MPI_MAXLOC, maxloc_func);
1360 CREATE_MPI_OP(MPI_MINLOC, minloc_func);
1361 CREATE_MPI_OP(MPI_REPLACE, replace_func);
1362
1363 MPI_Op smpi_op_new(MPI_User_function * function, bool commute)
1364 {
1365   MPI_Op op;
1366   op = xbt_new(s_smpi_mpi_op_t, 1);
1367   op->func = function;
1368   op-> is_commute = commute;
1369   return op;
1370 }
1371
1372 bool smpi_op_is_commute(MPI_Op op)
1373 {
1374   return (op==MPI_OP_NULL) ? true : op-> is_commute;
1375 }
1376
1377 void smpi_op_destroy(MPI_Op op)
1378 {
1379   xbt_free(op);
1380 }
1381
1382 void smpi_op_apply(MPI_Op op, const void *invec, void *inoutvec, int *len, MPI_Datatype * datatype)
1383 {
1384   if(op==MPI_OP_NULL)
1385     return;
1386
1387   if(smpi_privatize_global_variables){//we need to switch as the called function may silently touch global variables
1388     XBT_DEBUG("Applying operation, switch to the right data frame ");
1389     smpi_switch_data_segment(smpi_process_index());
1390   }
1391
1392   if(!smpi_process_get_replaying())
1393     op->func(const_cast<void*>(invec), inoutvec, len, datatype);
1394 }
1395
1396 int smpi_type_attr_delete(MPI_Datatype type, int keyval){
1397   smpi_type_key_elem elem =
1398     static_cast<smpi_type_key_elem>(xbt_dict_get_or_null_ext(smpi_type_keyvals, reinterpret_cast<const char*>(&keyval), sizeof(int)));
1399   if(elem==nullptr)
1400     return MPI_ERR_ARG;
1401   if(elem->delete_fn!=MPI_NULL_DELETE_FN){
1402     void * value = nullptr;
1403     int flag;
1404     if(smpi_type_attr_get(type, keyval, &value, &flag)==MPI_SUCCESS){
1405       int ret = elem->delete_fn(type, keyval, value, &flag);
1406       if(ret!=MPI_SUCCESS) 
1407         return ret;
1408     }
1409   }  
1410   if(type->attributes==nullptr)
1411     return MPI_ERR_ARG;
1412
1413   xbt_dict_remove_ext(type->attributes, reinterpret_cast<const char*>(&keyval), sizeof(int));
1414   return MPI_SUCCESS;
1415 }
1416
1417 int smpi_type_attr_get(MPI_Datatype type, int keyval, void* attr_value, int* flag){
1418   smpi_type_key_elem elem =
1419     static_cast<smpi_type_key_elem>(xbt_dict_get_or_null_ext(smpi_type_keyvals, reinterpret_cast<const char*>(&keyval), sizeof(int)));
1420   if(elem==nullptr)
1421     return MPI_ERR_ARG;
1422   if(type->attributes==nullptr){
1423     *flag=0;
1424     return MPI_SUCCESS;
1425   }
1426   try {
1427     *static_cast<void**>(attr_value) = xbt_dict_get_ext(type->attributes, reinterpret_cast<const char*>(&keyval), sizeof(int));
1428     *flag=1;
1429   }
1430   catch (xbt_ex& ex) {
1431     *flag=0;
1432   }
1433   return MPI_SUCCESS;
1434 }
1435
1436 int smpi_type_attr_put(MPI_Datatype type, int keyval, void* attr_value){
1437   if(smpi_type_keyvals==nullptr)
1438     smpi_type_keyvals = xbt_dict_new_homogeneous(nullptr);
1439   smpi_type_key_elem elem =
1440      static_cast<smpi_type_key_elem>(xbt_dict_get_or_null_ext(smpi_type_keyvals, reinterpret_cast<const char*>(&keyval), sizeof(int)));
1441   if(elem==nullptr)
1442     return MPI_ERR_ARG;
1443   int flag;
1444   void* value = nullptr;
1445   smpi_type_attr_get(type, keyval, &value, &flag);
1446   if(flag!=0 && elem->delete_fn!=MPI_NULL_DELETE_FN){
1447     int ret = elem->delete_fn(type, keyval, value, &flag);
1448     if(ret!=MPI_SUCCESS) 
1449       return ret;
1450   }
1451   if(type->attributes==nullptr)
1452     type->attributes = xbt_dict_new_homogeneous(nullptr);
1453
1454   xbt_dict_set_ext(type->attributes, reinterpret_cast<const char*>(&keyval), sizeof(int), attr_value, nullptr);
1455   return MPI_SUCCESS;
1456 }
1457
1458 int smpi_type_keyval_create(MPI_Type_copy_attr_function* copy_fn, MPI_Type_delete_attr_function* delete_fn, int* keyval,
1459                             void* extra_state){
1460   if(smpi_type_keyvals==nullptr)
1461     smpi_type_keyvals = xbt_dict_new_homogeneous(nullptr);
1462
1463   smpi_type_key_elem value = (smpi_type_key_elem) xbt_new0(s_smpi_mpi_type_key_elem_t,1);
1464
1465   value->copy_fn=copy_fn;
1466   value->delete_fn=delete_fn;
1467
1468   *keyval = type_keyval_id;
1469   xbt_dict_set_ext(smpi_type_keyvals,reinterpret_cast<const char*>(keyval), sizeof(int),reinterpret_cast<void*>(value), nullptr);
1470   type_keyval_id++;
1471   return MPI_SUCCESS;
1472 }
1473
1474 int smpi_type_keyval_free(int* keyval){
1475   smpi_type_key_elem elem =
1476     static_cast<smpi_type_key_elem>(xbt_dict_get_or_null_ext(smpi_type_keyvals, reinterpret_cast<const char*>(keyval), sizeof(int)));
1477   if(elem==0){
1478     return MPI_ERR_ARG;
1479   }
1480   xbt_dict_remove_ext(smpi_type_keyvals, reinterpret_cast<const char*>(keyval), sizeof(int));
1481   xbt_free(elem);
1482   return MPI_SUCCESS;
1483 }
1484
1485 int smpi_mpi_pack(void* inbuf, int incount, MPI_Datatype type, void* outbuf, int outcount, int* position,MPI_Comm comm){
1486   size_t size = smpi_datatype_size(type);
1487   if (outcount - *position < incount*static_cast<int>(size))
1488     return MPI_ERR_BUFFER;
1489   smpi_datatype_copy(inbuf, incount, type, static_cast<char*>(outbuf) + *position, outcount, MPI_CHAR);
1490   *position += incount * size;
1491   return MPI_SUCCESS;
1492 }
1493
1494 int smpi_mpi_unpack(void* inbuf, int insize, int* position, void* outbuf, int outcount, MPI_Datatype type,MPI_Comm comm){
1495   int size = static_cast<int>(smpi_datatype_size(type));
1496   if (outcount*size> insize)
1497     return MPI_ERR_BUFFER;
1498   smpi_datatype_copy(static_cast<char*>(inbuf) + *position, insize, MPI_CHAR, outbuf, outcount, type);
1499   *position += outcount * size;
1500   return MPI_SUCCESS;
1501 }