1 /* Copyright (c) 2007-2022. The SimGrid Team. All rights reserved. */
3 /* This program is free software; you can redistribute it and/or modify it
4 * under the terms of the license (GNU LGPL) which comes with this package. */
7 #include "smpi_coll.hpp"
8 #include "smpi_comm.hpp"
9 #include "smpi_datatype_derived.hpp"
10 #include "smpi_op.hpp"
11 #include "smpi_win.hpp"
12 #include "src/smpi/include/smpi_actor.hpp"
14 XBT_LOG_EXTERNAL_DEFAULT_CATEGORY(smpi_pmpi);
17 CHECK_COUNT(2, origin_count)\
18 CHECK_TYPE(3, origin_datatype)\
19 CHECK_BUFFER(1, origin_addr, origin_count, origin_datatype)\
20 CHECK_PROC_RMA(4, target_rank, win)\
21 CHECK_COUNT(6, target_count)\
22 CHECK_TYPE(7, target_datatype)
24 #define CHECK_TARGET_DISP(num) \
25 if (not win->dynamic()) \
26 CHECK_NEGATIVE((num), MPI_ERR_RMA_RANGE, target_disp)
28 /* PMPI User level calls */
30 int PMPI_Win_create( void *base, MPI_Aint size, int disp_unit, MPI_Info info, MPI_Comm comm, MPI_Win *win){
33 CHECK_BUFFER(1, base, size, MPI_BYTE)
34 CHECK_NEGATIVE(2, MPI_ERR_OTHER, size)
35 CHECK_NEGATIVE(3, MPI_ERR_OTHER, disp_unit)
36 const SmpiBenchGuard suspend_bench;
37 if (base == nullptr && size != 0){
38 retval= MPI_ERR_OTHER;
40 *win = new simgrid::smpi::Win( base, size, disp_unit, info, comm);
46 int PMPI_Win_allocate( MPI_Aint size, int disp_unit, MPI_Info info, MPI_Comm comm, void *base, MPI_Win *win){
48 CHECK_NEGATIVE(2, MPI_ERR_OTHER, size)
49 CHECK_NEGATIVE(3, MPI_ERR_OTHER, disp_unit)
50 void* ptr = xbt_malloc(size);
51 const SmpiBenchGuard suspend_bench;
52 *static_cast<void**>(base) = ptr;
53 *win = new simgrid::smpi::Win(ptr, size, disp_unit, info, comm, true);
57 int PMPI_Win_allocate_shared( MPI_Aint size, int disp_unit, MPI_Info info, MPI_Comm comm, void *base, MPI_Win *win){
59 CHECK_NEGATIVE(2, MPI_ERR_OTHER, size)
60 CHECK_NEGATIVE(3, MPI_ERR_OTHER, disp_unit)
62 int rank = comm->rank();
64 ptr = xbt_malloc(size*comm->size());
66 const SmpiBenchGuard suspend_bench;
67 simgrid::smpi::colls::bcast(&ptr, sizeof(void*), MPI_BYTE, 0, comm);
68 simgrid::smpi::colls::barrier(comm);
69 *static_cast<void**>(base) = (char*)ptr+rank*size;
70 *win = new simgrid::smpi::Win(ptr, size, disp_unit, info, comm, rank == 0);
74 int PMPI_Win_create_dynamic( MPI_Info info, MPI_Comm comm, MPI_Win *win){
76 const SmpiBenchGuard suspend_bench;
77 *win = new simgrid::smpi::Win(info, comm);
81 int PMPI_Win_attach(MPI_Win win, void *base, MPI_Aint size){
83 CHECK_NEGATIVE(3, MPI_ERR_OTHER, size)
84 if (base == nullptr && size != 0)
86 const SmpiBenchGuard suspend_bench;
87 int retval = win->attach(base, size);
91 int PMPI_Win_detach(MPI_Win win, const void* base)
94 CHECK_NULL(2, MPI_ERR_OTHER, base)
95 const SmpiBenchGuard suspend_bench;
96 int retval = win->detach(base);
100 int PMPI_Win_free( MPI_Win* win){
101 CHECK_NULL(1, MPI_ERR_WIN, win)
103 if (_smpi_cfg_pedantic && (*win)->opened() == 1){//only check in pedantic mode, as it's not clear this is illegal
104 XBT_WARN("Attempt to destroy a MPI_Win too early -missing MPI_Win_fence ?");
107 const SmpiBenchGuard suspend_bench;
108 return simgrid::smpi::Win::del(*win);
112 int PMPI_Win_set_name(MPI_Win win, const char * name)
115 CHECK_NULL(2, MPI_ERR_ARG, name)
120 int PMPI_Win_get_name(MPI_Win win, char * name, int* len)
123 CHECK_NULL(2, MPI_ERR_ARG, name)
124 win->get_name(name, len);
128 int PMPI_Win_get_info(MPI_Win win, MPI_Info* info)
131 CHECK_NULL(2, MPI_ERR_ARG, info)
132 *info = new simgrid::smpi::Info(win->info());
136 int PMPI_Win_set_info(MPI_Win win, MPI_Info info)
143 int PMPI_Win_get_group(MPI_Win win, MPI_Group * group){
145 win->get_group(group);
146 if (*group != MPI_COMM_WORLD->group() && *group != MPI_GROUP_NULL && *group != MPI_GROUP_EMPTY)
151 int PMPI_Win_fence( int assert, MPI_Win win){
153 const SmpiBenchGuard suspend_bench;
154 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
155 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_fence"));
156 int retval = win->fence(assert);
157 TRACE_smpi_comm_out(my_proc_id);
161 int PMPI_Get( void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank,
162 MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win){
168 const SmpiBenchGuard suspend_bench;
170 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
172 win->get_group(&group);
173 TRACE_smpi_comm_in(my_proc_id, __func__,
174 new simgrid::instr::Pt2PtTIData("Get", target_rank, origin_count,
175 simgrid::smpi::Datatype::encode(origin_datatype)));
176 retval = win->get( origin_addr, origin_count, origin_datatype, target_rank, target_disp, target_count,
178 TRACE_smpi_comm_out(my_proc_id);
183 int PMPI_Rget( void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank,
184 MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request* request){
185 if(target_rank==MPI_PROC_NULL)
186 *request = MPI_REQUEST_NULL;
190 CHECK_NULL(9, MPI_ERR_ARG, request)
193 const SmpiBenchGuard suspend_bench;
195 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
197 win->get_group(&group);
198 TRACE_smpi_comm_in(my_proc_id, __func__,
199 new simgrid::instr::Pt2PtTIData(
200 "Rget", target_rank, origin_count,
201 simgrid::smpi::Datatype::encode(origin_datatype)));
203 retval = win->get( origin_addr, origin_count, origin_datatype, target_rank, target_disp, target_count,
204 target_datatype, request);
206 TRACE_smpi_comm_out(my_proc_id);
211 int PMPI_Put(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank,
212 MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win){
218 const SmpiBenchGuard suspend_bench;
220 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
222 win->get_group(&group);
223 aid_t dst_traced = group->actor(target_rank);
224 TRACE_smpi_comm_in(my_proc_id, __func__,
225 new simgrid::instr::Pt2PtTIData("Put", target_rank, origin_count,
226 simgrid::smpi::Datatype::encode(origin_datatype)));
227 TRACE_smpi_send(my_proc_id, my_proc_id, dst_traced, SMPI_RMA_TAG, origin_count * origin_datatype->size());
229 retval = win->put( origin_addr, origin_count, origin_datatype, target_rank, target_disp, target_count,
232 TRACE_smpi_comm_out(my_proc_id);
237 int PMPI_Rput(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank,
238 MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request* request){
239 if(target_rank==MPI_PROC_NULL)
240 *request = MPI_REQUEST_NULL;
244 CHECK_NULL(9, MPI_ERR_ARG, request)
246 const SmpiBenchGuard suspend_bench;
248 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
250 win->get_group(&group);
251 aid_t dst_traced = group->actor(target_rank);
252 TRACE_smpi_comm_in(my_proc_id, __func__,
253 new simgrid::instr::Pt2PtTIData(
254 "Rput", target_rank, origin_count,
255 simgrid::smpi::Datatype::encode(origin_datatype)));
256 TRACE_smpi_send(my_proc_id, my_proc_id, dst_traced, SMPI_RMA_TAG, origin_count * origin_datatype->size());
258 retval = win->put( origin_addr, origin_count, origin_datatype, target_rank, target_disp, target_count,
259 target_datatype, request);
261 TRACE_smpi_comm_out(my_proc_id);
266 int PMPI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank,
267 MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win){
270 CHECK_MPI_NULL(8, MPI_OP_NULL, MPI_ERR_OP, op)
275 const SmpiBenchGuard suspend_bench;
276 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
278 win->get_group(&group);
279 TRACE_smpi_comm_in(my_proc_id, __func__,
280 new simgrid::instr::Pt2PtTIData(
281 "Accumulate", target_rank, origin_count,
282 simgrid::smpi::Datatype::encode(origin_datatype)));
283 retval = win->accumulate( origin_addr, origin_count, origin_datatype, target_rank, target_disp, target_count,
284 target_datatype, op);
286 TRACE_smpi_comm_out(my_proc_id);
291 int PMPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank,
292 MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request* request){
293 if(target_rank==MPI_PROC_NULL)
294 *request = MPI_REQUEST_NULL;
297 CHECK_MPI_NULL(8, MPI_OP_NULL, MPI_ERR_OP, op)
299 CHECK_NULL(10, MPI_ERR_ARG, request)
303 const SmpiBenchGuard suspend_bench;
305 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
307 win->get_group(&group);
308 TRACE_smpi_comm_in(my_proc_id, __func__,
309 new simgrid::instr::Pt2PtTIData(
310 "Raccumulate", target_rank, origin_count,
311 simgrid::smpi::Datatype::encode(origin_datatype)));
313 retval = win->accumulate( origin_addr, origin_count, origin_datatype, target_rank, target_disp, target_count,
314 target_datatype, op, request);
316 TRACE_smpi_comm_out(my_proc_id);
321 int PMPI_Get_accumulate(const void* origin_addr, int origin_count, MPI_Datatype origin_datatype, void* result_addr,
322 int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
323 int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win)
325 if (op != MPI_NO_OP){
326 CHECK_BUFFER(1, origin_addr, origin_count, origin_datatype)
327 CHECK_COUNT(2, origin_count)
329 CHECK_TYPE(3, origin_datatype)
331 CHECK_COUNT(5, result_count)
332 CHECK_TYPE(6, result_datatype)
333 CHECK_BUFFER(4, result_addr, result_count, result_datatype)
335 CHECK_PROC_RMA(7, target_rank, win)
336 CHECK_COUNT(9, target_count)
337 CHECK_TYPE(10, target_datatype)
338 CHECK_MPI_NULL(11, MPI_OP_NULL, MPI_ERR_OP, op)
342 const SmpiBenchGuard suspend_bench;
344 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
346 win->get_group(&group);
347 TRACE_smpi_comm_in(my_proc_id, __func__,
348 new simgrid::instr::Pt2PtTIData(
349 "Get_accumulate", target_rank, target_count,
350 simgrid::smpi::Datatype::encode(target_datatype)));
352 retval = win->get_accumulate( origin_addr, origin_count, origin_datatype, result_addr,
353 result_count, result_datatype, target_rank, target_disp,
354 target_count, target_datatype, op);
356 TRACE_smpi_comm_out(my_proc_id);
361 int PMPI_Rget_accumulate(const void* origin_addr, int origin_count, MPI_Datatype origin_datatype, void* result_addr,
362 int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
363 int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request* request)
365 if(target_rank==MPI_PROC_NULL)
366 *request = MPI_REQUEST_NULL;
367 if (op != MPI_NO_OP){
368 CHECK_BUFFER(1, origin_addr, origin_count, origin_datatype)
369 CHECK_COUNT(2, origin_count)
371 CHECK_TYPE(3, origin_datatype)
373 CHECK_COUNT(5, result_count)
374 CHECK_TYPE(6, result_datatype)
375 CHECK_BUFFER(4, result_addr, result_count, result_datatype)
377 CHECK_PROC_RMA(7, target_rank, win)
378 CHECK_COUNT(9, target_count)
379 CHECK_TYPE(10, target_datatype)
380 CHECK_MPI_NULL(11, MPI_OP_NULL, MPI_ERR_OP, op)
382 CHECK_NULL(10, MPI_ERR_ARG, request)
384 const SmpiBenchGuard suspend_bench;
386 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
388 win->get_group(&group);
389 TRACE_smpi_comm_in(my_proc_id, __func__,
390 new simgrid::instr::Pt2PtTIData(
391 "Rget_accumulate", target_rank, target_count,
392 simgrid::smpi::Datatype::encode(target_datatype)));
394 retval = win->get_accumulate( origin_addr, origin_count, origin_datatype, result_addr,
395 result_count, result_datatype, target_rank, target_disp,
396 target_count, target_datatype, op, request);
398 TRACE_smpi_comm_out(my_proc_id);
403 int PMPI_Fetch_and_op(const void *origin_addr, void *result_addr, MPI_Datatype dtype, int target_rank, MPI_Aint target_disp, MPI_Op op, MPI_Win win){
404 return PMPI_Get_accumulate(origin_addr, origin_addr==nullptr?0:1, dtype, result_addr, 1, dtype, target_rank, target_disp, 1, dtype, op, win);
407 int PMPI_Compare_and_swap(const void* origin_addr, void* compare_addr, void* result_addr, MPI_Datatype datatype,
408 int target_rank, MPI_Aint target_disp, MPI_Win win)
410 CHECK_NULL(1, MPI_ERR_BUFFER, origin_addr)
411 CHECK_NULL(2, MPI_ERR_BUFFER, compare_addr)
412 CHECK_NULL(3, MPI_ERR_BUFFER, result_addr)
413 CHECK_TYPE(4, datatype)
415 CHECK_PROC_RMA(5, target_rank, win)
420 const SmpiBenchGuard suspend_bench;
422 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
424 win->get_group(&group);
425 TRACE_smpi_comm_in(my_proc_id, __func__,
426 new simgrid::instr::Pt2PtTIData("Compare_and_swap", target_rank, 1,
427 simgrid::smpi::Datatype::encode(datatype)));
429 retval = win->compare_and_swap(origin_addr, compare_addr, result_addr, datatype, target_rank, target_disp);
431 TRACE_smpi_comm_out(my_proc_id);
436 int PMPI_Win_post(MPI_Group group, int assert, MPI_Win win){
437 CHECK_GROUP(1, group)
439 const SmpiBenchGuard suspend_bench;
440 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
441 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_post"));
442 int retval = win->post(group,assert);
443 TRACE_smpi_comm_out(my_proc_id);
447 int PMPI_Win_start(MPI_Group group, int assert, MPI_Win win){
448 CHECK_GROUP(1, group)
450 const SmpiBenchGuard suspend_bench;
451 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
452 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_start"));
453 int retval = win->start(group,assert);
454 TRACE_smpi_comm_out(my_proc_id);
458 int PMPI_Win_complete(MPI_Win win){
460 const SmpiBenchGuard suspend_bench;
461 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
462 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_complete"));
463 int retval = win->complete();
464 TRACE_smpi_comm_out(my_proc_id);
468 int PMPI_Win_wait(MPI_Win win){
470 const SmpiBenchGuard suspend_bench;
471 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
472 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_wait"));
473 int retval = win->wait();
474 TRACE_smpi_comm_out(my_proc_id);
478 int PMPI_Win_lock(int lock_type, int rank, int assert, MPI_Win win){
480 CHECK_PROC_RMA(2, rank, win)
482 const SmpiBenchGuard suspend_bench;
483 if (lock_type != MPI_LOCK_EXCLUSIVE &&
484 lock_type != MPI_LOCK_SHARED) {
485 retval = MPI_ERR_LOCKTYPE;
487 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
488 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_lock"));
489 retval = win->lock(lock_type,rank,assert);
490 TRACE_smpi_comm_out(my_proc_id);
495 int PMPI_Win_unlock(int rank, MPI_Win win){
497 CHECK_PROC_RMA(1, rank, win)
498 const SmpiBenchGuard suspend_bench;
499 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
500 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_unlock"));
501 int retval = win->unlock(rank);
502 TRACE_smpi_comm_out(my_proc_id);
506 int PMPI_Win_lock_all(int assert, MPI_Win win){
508 const SmpiBenchGuard suspend_bench;
509 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
510 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_lock_all"));
511 int retval = win->lock_all(assert);
512 TRACE_smpi_comm_out(my_proc_id);
516 int PMPI_Win_unlock_all(MPI_Win win){
518 const SmpiBenchGuard suspend_bench;
519 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
520 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_unlock_all"));
521 int retval = win->unlock_all();
522 TRACE_smpi_comm_out(my_proc_id);
526 int PMPI_Win_flush(int rank, MPI_Win win){
528 CHECK_PROC_RMA(1, rank, win)
529 const SmpiBenchGuard suspend_bench;
530 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
531 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_flush"));
532 int retval = win->flush(rank);
533 TRACE_smpi_comm_out(my_proc_id);
537 int PMPI_Win_flush_local(int rank, MPI_Win win){
539 CHECK_PROC_RMA(1, rank, win)
540 const SmpiBenchGuard suspend_bench;
541 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
542 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_flush_local"));
543 int retval = win->flush_local(rank);
544 TRACE_smpi_comm_out(my_proc_id);
548 int PMPI_Win_flush_all(MPI_Win win){
550 const SmpiBenchGuard suspend_bench;
551 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
552 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_flush_all"));
553 int retval = win->flush_all();
554 TRACE_smpi_comm_out(my_proc_id);
558 int PMPI_Win_flush_local_all(MPI_Win win){
560 const SmpiBenchGuard suspend_bench;
561 aid_t my_proc_id = simgrid::s4u::this_actor::get_pid();
562 TRACE_smpi_comm_in(my_proc_id, __func__, new simgrid::instr::NoOpTIData("Win_flush_local_all"));
563 int retval = win->flush_local_all();
564 TRACE_smpi_comm_out(my_proc_id);
568 int PMPI_Win_shared_query (MPI_Win win, int rank, MPI_Aint* size, int* disp_unit, void* baseptr)
571 return win->shared_query(rank, size, disp_unit, baseptr);
574 int PMPI_Win_get_attr (MPI_Win win, int keyval, void *attribute_val, int* flag)
576 static MPI_Aint size;
577 static MPI_Aint disp_unit;
581 *static_cast<void**>(attribute_val) = win->base();
586 *static_cast<MPI_Aint**>(attribute_val) = &size;
589 case MPI_WIN_DISP_UNIT:
590 disp_unit = win->disp_unit();
591 *static_cast<MPI_Aint**>(attribute_val) = &disp_unit;
595 return win->attr_get<simgrid::smpi::Win>(keyval, attribute_val, flag);
599 int PMPI_Win_set_attr (MPI_Win win, int type_keyval, void *attribute_val)
602 return win->attr_put<simgrid::smpi::Win>(type_keyval, attribute_val);
605 int PMPI_Win_delete_attr (MPI_Win win, int type_keyval)
608 return win->attr_delete<simgrid::smpi::Win>(type_keyval);
611 int PMPI_Win_create_keyval(MPI_Win_copy_attr_function* copy_fn, MPI_Win_delete_attr_function* delete_fn, int* keyval,
614 smpi_copy_fn _copy_fn={nullptr, nullptr,copy_fn,nullptr, nullptr,nullptr};
615 smpi_delete_fn _delete_fn={nullptr, nullptr,delete_fn,nullptr, nullptr,nullptr};
616 return simgrid::smpi::Keyval::keyval_create<simgrid::smpi::Win>(_copy_fn, _delete_fn, keyval, extra_state);
619 int PMPI_Win_free_keyval(int* keyval) {
620 return simgrid::smpi::Keyval::keyval_free<simgrid::smpi::Win>(keyval);
623 MPI_Win PMPI_Win_f2c(MPI_Fint win){
626 return simgrid::smpi::Win::f2c(win);
629 MPI_Fint PMPI_Win_c2f(MPI_Win win){
630 if(win==MPI_WIN_NULL)
635 int PMPI_Win_create_errhandler(MPI_Win_errhandler_function* function, MPI_Errhandler* errhandler){
636 *errhandler=new simgrid::smpi::Errhandler(function);
640 int PMPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler* errhandler){
642 if (errhandler==nullptr){
645 *errhandler=win->errhandler();
649 int PMPI_Win_set_errhandler(MPI_Win win, MPI_Errhandler errhandler){
651 if (errhandler==nullptr){
654 win->set_errhandler(errhandler);
658 int PMPI_Win_call_errhandler(MPI_Win win,int errorcode){
660 MPI_Errhandler err = win->errhandler();
661 err->call(win, errorcode);
662 simgrid::smpi::Errhandler::unref(err);