1 /* Copyright (c) 2009, 2010. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
12 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(simix_network, simix,
13 "Logging specific to SIMIX (network)");
15 static xbt_dict_t rdv_points = NULL;
16 unsigned long int smx_total_comms = 0;
18 static XBT_INLINE void SIMIX_comm_start(smx_action_t action);
19 static void SIMIX_waitany_req_remove_from_actions(smx_req_t req);
20 static void SIMIX_comm_copy_data(smx_action_t comm);
21 static smx_action_t SIMIX_comm_new(e_smx_comm_type_t type);
22 static XBT_INLINE void SIMIX_rdv_push(smx_rdv_t rdv, smx_action_t comm);
23 static XBT_INLINE void SIMIX_rdv_remove(smx_rdv_t rdv, smx_action_t comm);
24 static smx_action_t SIMIX_rdv_get_request(smx_rdv_t rdv, e_smx_comm_type_t type,
25 int (*match_fun)(void *, void *), void *);
26 static void SIMIX_rdv_free(void *data);
28 void SIMIX_network_init(void)
30 rdv_points = xbt_dict_new();
33 void SIMIX_network_exit(void)
35 xbt_dict_free(&rdv_points);
38 /******************************************************************************/
39 /* Rendez-Vous Points */
40 /******************************************************************************/
42 smx_rdv_t SIMIX_rdv_create(const char *name)
44 /* two processes may have pushed the same rdv_create request at the same time */
45 smx_rdv_t rdv = name ? xbt_dict_get_or_null(rdv_points, name) : NULL;
48 rdv = xbt_new0(s_smx_rvpoint_t, 1);
49 rdv->name = name ? xbt_strdup(name) : NULL;
50 rdv->comm_fifo = xbt_fifo_new();
53 xbt_dict_set(rdv_points, rdv->name, rdv, SIMIX_rdv_free);
58 void SIMIX_rdv_destroy(smx_rdv_t rdv)
61 xbt_dict_remove(rdv_points, rdv->name);
64 void SIMIX_rdv_free(void *data)
66 smx_rdv_t rdv = (smx_rdv_t) data;
69 xbt_fifo_free(rdv->comm_fifo);
73 smx_rdv_t SIMIX_rdv_get_by_name(const char *name)
75 return xbt_dict_get_or_null(rdv_points, name);
78 int SIMIX_rdv_comm_count_by_host(smx_rdv_t rdv, smx_host_t host)
80 smx_action_t comm = NULL;
81 xbt_fifo_item_t item = NULL;
84 xbt_fifo_foreach(rdv->comm_fifo, item, comm, smx_action_t) {
85 if (comm->comm.src_proc->smx_host == host)
92 smx_action_t SIMIX_rdv_get_head(smx_rdv_t rdv)
94 return xbt_fifo_get_item_content(xbt_fifo_get_first_item(rdv->comm_fifo));
98 * \brief Push a communication request into a rendez-vous point
99 * \param rdv The rendez-vous point
100 * \param comm The communication request
102 static XBT_INLINE void SIMIX_rdv_push(smx_rdv_t rdv, smx_action_t comm)
104 xbt_fifo_push(rdv->comm_fifo, comm);
105 comm->comm.rdv = rdv;
109 * \brief Remove a communication request from a rendez-vous point
110 * \param rdv The rendez-vous point
111 * \param comm The communication request
113 static XBT_INLINE void SIMIX_rdv_remove(smx_rdv_t rdv, smx_action_t comm)
115 xbt_fifo_remove(rdv->comm_fifo, comm);
116 comm->comm.rdv = NULL;
119 smx_action_t SIMIX_comm_get_send_match(smx_rdv_t rdv, int (*match_fun)(void*, void*), void* data) {
120 return SIMIX_rdv_get_request(rdv, SIMIX_COMM_SEND, match_fun, data);
124 * \brief Checks if there is a communication action queued in a rendez-vous matching our needs
125 * \param type The type of communication we are looking for (comm_send, comm_recv)
126 * \return The communication action if found, NULL otherwise
128 smx_action_t SIMIX_rdv_get_request(smx_rdv_t rdv, e_smx_comm_type_t type,
129 int (*match_fun)(void *, void *), void *data)
132 xbt_fifo_item_t item;
133 void* req_data = NULL;
135 xbt_fifo_foreach(rdv->comm_fifo, item, action, smx_action_t){
136 if (action->comm.type == SIMIX_COMM_SEND) {
137 req_data = action->comm.src_data;
138 } else if (action->comm.type == SIMIX_COMM_RECEIVE) {
139 req_data = action->comm.dst_data;
141 if (action->comm.type == type && (!match_fun || match_fun(data, req_data))) {
142 XBT_DEBUG("Found a matching communication action %p", action);
143 xbt_fifo_remove_item(rdv->comm_fifo, item);
144 xbt_fifo_free_item(item);
145 action->comm.refcount++;
146 action->comm.rdv = NULL;
149 XBT_DEBUG("Sorry, communication action %p does not match our needs:"
150 " its type is %d but we are looking for a comm of type %d",
151 action, action->comm.type, type);
153 XBT_DEBUG("No matching communication action found");
157 /******************************************************************************/
158 /* Comunication Actions */
159 /******************************************************************************/
162 * \brief Creates a new comunicate action
163 * \param type The type of request (comm_send, comm_recv)
164 * \return The new comunicate action
166 smx_action_t SIMIX_comm_new(e_smx_comm_type_t type)
170 /* alloc structures */
171 act = xbt_mallocator_get(simix_global->action_mallocator);
172 act->type = SIMIX_ACTION_COMMUNICATE;
173 act->state = SIMIX_WAITING;
175 /* set communication */
176 act->comm.type = type;
177 act->comm.refcount = 1;
179 #ifdef HAVE_LATENCY_BOUND_TRACKING
180 //initialize with unknown value
181 act->latency_limited = -1;
185 act->category = NULL;
188 XBT_DEBUG("Create communicate action %p", act);
195 * \brief Destroy a communicate action
196 * \param action The communicate action to be destroyed
198 void SIMIX_comm_destroy(smx_action_t action)
200 XBT_DEBUG("Destroy action %p (refcount:%d)", action, action->comm.refcount);
202 if (action->comm.refcount <= 0)
203 xbt_die("the refcount of comm %p is already 0 before decreasing it. "
204 "That's a bug!", action);
206 action->comm.refcount--;
207 if (action->comm.refcount > 0)
209 XBT_DEBUG("Really free communication %p; refcount is now %d", action,
210 action->comm.refcount);
212 #ifdef HAVE_LATENCY_BOUND_TRACKING
213 action->latency_limited = SIMIX_comm_is_latency_bounded( action ) ;
217 TRACE_smx_action_destroy(action);
220 xbt_free(action->name);
221 SIMIX_comm_destroy_internal_actions(action);
223 if (action->comm.detached && action->state != SIMIX_DONE) {
224 /* the communication has failed and was detached:
225 * we have to free the buffer */
226 ((void_f_pvoid_t) action->comm.src_data)(action->comm.src_buff);
229 xbt_mallocator_release(simix_global->action_mallocator, action);
232 void SIMIX_comm_destroy_internal_actions(smx_action_t action)
234 if (action->comm.surf_comm){
235 #ifdef HAVE_LATENCY_BOUND_TRACKING
236 action->latency_limited = SIMIX_comm_is_latency_bounded(action);
238 action->comm.surf_comm->model_type->action_unref(action->comm.surf_comm);
239 action->comm.surf_comm = NULL;
242 if (action->comm.src_timeout){
243 action->comm.src_timeout->model_type->action_unref(action->comm.src_timeout);
244 action->comm.src_timeout = NULL;
247 if (action->comm.dst_timeout){
248 action->comm.dst_timeout->model_type->action_unref(action->comm.dst_timeout);
249 action->comm.dst_timeout = NULL;
253 smx_action_t SIMIX_comm_isend(smx_process_t src_proc, smx_rdv_t rdv,
254 double task_size, double rate,
255 void *src_buff, size_t src_buff_size,
256 int (*match_fun)(void *, void *), void *data,
261 /* Look for communication request matching our needs.
262 If it is not found then create it and push it into the rendez-vous point */
263 action = SIMIX_rdv_get_request(rdv, SIMIX_COMM_RECEIVE, match_fun, data);
266 action = SIMIX_comm_new(SIMIX_COMM_SEND);
267 SIMIX_rdv_push(rdv, action);
269 action->state = SIMIX_READY;
270 action->comm.type = SIMIX_COMM_READY;
273 /* If the communication action is detached then decrease the refcount
274 * by one, so it will be eliminated by the receivers destroy call */
276 action->comm.detached = 1;
277 action->comm.refcount--;
280 /* Setup the communication request */
281 action->comm.src_proc = src_proc;
282 action->comm.task_size = task_size;
283 action->comm.rate = rate;
284 action->comm.src_buff = src_buff;
285 action->comm.src_buff_size = src_buff_size;
286 action->comm.src_data = data;
289 action->state = SIMIX_RUNNING;
293 SIMIX_comm_start(action);
297 smx_action_t SIMIX_comm_irecv(smx_process_t dst_proc, smx_rdv_t rdv,
298 void *dst_buff, size_t *dst_buff_size,
299 int (*match_fun)(void *, void *), void *data)
303 /* Look for communication request matching our needs.
304 * If it is not found then create it and push it into the rendez-vous point
306 action = SIMIX_rdv_get_request(rdv, SIMIX_COMM_SEND, match_fun, data);
309 action = SIMIX_comm_new(SIMIX_COMM_RECEIVE);
310 SIMIX_rdv_push(rdv, action);
312 action->state = SIMIX_READY;
313 action->comm.type = SIMIX_COMM_READY;
316 /* Setup communication request */
317 action->comm.dst_proc = dst_proc;
318 action->comm.dst_buff = dst_buff;
319 action->comm.dst_buff_size = dst_buff_size;
320 action->comm.dst_data = data;
323 action->state = SIMIX_RUNNING;
327 SIMIX_comm_start(action);
331 void SIMIX_pre_comm_wait(smx_req_t req, smx_action_t action, double timeout, int idx)
333 /* the request may be a wait, a send or a recv */
336 /* Associate this request to the action */
337 xbt_fifo_push(action->request_list, req);
338 req->issuer->waiting_action = action;
342 action->state = SIMIX_DONE;
344 /* If we reached this point, the wait request must have a timeout */
345 /* Otherwise it shouldn't be enabled and executed by the MC */
349 if (action->comm.src_proc == req->issuer)
350 action->state = SIMIX_SRC_TIMEOUT;
352 action->state = SIMIX_DST_TIMEOUT;
355 SIMIX_comm_finish(action);
359 /* If the action has already finish perform the error handling, */
360 /* otherwise set up a waiting timeout on the right side */
361 if (action->state != SIMIX_WAITING && action->state != SIMIX_RUNNING) {
362 SIMIX_comm_finish(action);
363 } else { /* if (timeout >= 0) { we need a surf sleep action even when there is no timeout, otherwise surf won't tell us when the host fails */
364 sleep = surf_workstation_model->extension.workstation.sleep(req->issuer->smx_host->host, timeout);
365 surf_workstation_model->action_data_set(sleep, action);
367 if (req->issuer == action->comm.src_proc)
368 action->comm.src_timeout = sleep;
370 action->comm.dst_timeout = sleep;
374 void SIMIX_pre_comm_test(smx_req_t req)
376 smx_action_t action = req->comm_test.comm;
379 req->comm_test.result = action->comm.src_proc && action->comm.dst_proc;
380 if(req->comm_test.result){
381 action->state = SIMIX_DONE;
382 xbt_fifo_push(action->request_list, req);
383 SIMIX_comm_finish(action);
385 SIMIX_request_answer(req);
390 req->comm_test.result = (action->state != SIMIX_WAITING && action->state != SIMIX_RUNNING);
391 if (req->comm_test.result) {
392 xbt_fifo_push(action->request_list, req);
393 SIMIX_comm_finish(action);
395 SIMIX_request_answer(req);
399 void SIMIX_pre_comm_testany(smx_req_t req, int idx)
403 xbt_dynar_t actions = req->comm_testany.comms;
404 req->comm_testany.result = -1;
408 SIMIX_request_answer(req);
410 action = xbt_dynar_get_as(actions, idx, smx_action_t);
411 req->comm_testany.result = idx;
412 xbt_fifo_push(action->request_list, req);
413 action->state = SIMIX_DONE;
414 SIMIX_comm_finish(action);
419 xbt_dynar_foreach(req->comm_testany.comms,cursor,action) {
420 if (action->state != SIMIX_WAITING && action->state != SIMIX_RUNNING) {
421 req->comm_testany.result = cursor;
422 xbt_fifo_push(action->request_list, req);
423 SIMIX_comm_finish(action);
427 SIMIX_request_answer(req);
430 void SIMIX_pre_comm_waitany(smx_req_t req, int idx)
433 unsigned int cursor = 0;
434 xbt_dynar_t actions = req->comm_waitany.comms;
437 action = xbt_dynar_get_as(actions, idx, smx_action_t);
438 xbt_fifo_push(action->request_list, req);
439 req->comm_waitany.result = idx;
440 action->state = SIMIX_DONE;
441 SIMIX_comm_finish(action);
445 xbt_dynar_foreach(actions, cursor, action){
446 /* Associate this request to the action */
447 xbt_fifo_push(action->request_list, req);
448 if (action->state != SIMIX_WAITING && action->state != SIMIX_RUNNING){
449 SIMIX_comm_finish(action);
455 void SIMIX_waitany_req_remove_from_actions(smx_req_t req)
458 unsigned int cursor = 0;
459 xbt_dynar_t actions = req->comm_waitany.comms;
461 xbt_dynar_foreach(actions, cursor, action){
462 xbt_fifo_remove(action->request_list, req);
467 * \brief Start the simulation of a communication request
468 * \param action The communication action
470 static XBT_INLINE void SIMIX_comm_start(smx_action_t action)
472 /* If both the sender and the receiver are already there, start the communication */
473 if (action->state == SIMIX_READY) {
474 smx_host_t sender = action->comm.src_proc->smx_host;
475 smx_host_t receiver = action->comm.dst_proc->smx_host;
477 XBT_DEBUG("Starting communication %p from '%s' to '%s'", action,
478 SIMIX_host_get_name(sender), SIMIX_host_get_name(receiver));
480 action->comm.surf_comm = surf_workstation_model->extension.workstation.
481 communicate(sender->host, receiver->host, action->comm.task_size, action->comm.rate);
483 surf_workstation_model->action_data_set(action->comm.surf_comm, action);
485 action->state = SIMIX_RUNNING;
488 TRACE_smx_action_communicate(action, action->comm.src_proc);
491 /* If a link is failed, detect it immediately */
492 if (surf_workstation_model->action_state_get(action->comm.surf_comm) == SURF_ACTION_FAILED) {
493 XBT_DEBUG("Communication from '%s' to '%s' failed to start because of a link failure",
494 SIMIX_host_get_name(sender), SIMIX_host_get_name(receiver));
495 action->state = SIMIX_LINK_FAILURE;
496 SIMIX_comm_destroy_internal_actions(action);
499 /* If any of the process is suspend, create the action but stop its execution,
500 it will be restarted when the sender process resume */
501 if (SIMIX_process_is_suspended(action->comm.src_proc) ||
502 SIMIX_process_is_suspended(action->comm.dst_proc)) {
503 /* FIXME: check what should happen with the action state */
504 surf_workstation_model->suspend(action->comm.surf_comm);
509 void SIMIX_comm_finish(smx_action_t action)
511 unsigned int destroy_count = 0;
514 while ((req = xbt_fifo_shift(action->request_list))) {
516 /* If a waitany request is waiting for this action to finish, then remove
517 it from the other actions in the waitany list. Afterwards, get the
518 position of the actual action in the waitany request's actions dynar and
519 return it as the result of the call */
520 if (req->call == REQ_COMM_WAITANY) {
521 SIMIX_waitany_req_remove_from_actions(req);
523 req->comm_waitany.result = xbt_dynar_search(req->comm_waitany.comms, &action);
526 /* If the action is still in a rendez-vous point then remove from it */
527 if (action->comm.rdv)
528 SIMIX_rdv_remove(action->comm.rdv, action);
530 XBT_DEBUG("SIMIX_comm_finish: action state = %d", action->state);
532 /* Check out for errors */
533 switch (action->state) {
536 XBT_DEBUG("Communication %p complete!", action);
537 SIMIX_comm_copy_data(action);
540 case SIMIX_SRC_TIMEOUT:
542 THROW0(timeout_error, 0, "Communication timeouted because of sender");
544 CATCH(req->issuer->running_ctx->exception) {
545 req->issuer->doexception = 1;
549 case SIMIX_DST_TIMEOUT:
551 THROW0(timeout_error, 0, "Communication timeouted because of receiver");
553 CATCH(req->issuer->running_ctx->exception) {
554 req->issuer->doexception = 1;
558 case SIMIX_SRC_HOST_FAILURE:
560 if (req->issuer == action->comm.src_proc)
561 THROW0(host_error, 0, "Host failed");
563 THROW0(network_error, 0, "Remote peer failed");
565 CATCH(req->issuer->running_ctx->exception) {
566 req->issuer->doexception = 1;
570 case SIMIX_DST_HOST_FAILURE:
572 if (req->issuer == action->comm.dst_proc)
573 THROW0(host_error, 0, "Host failed");
575 THROW0(network_error, 0, "Remote peer failed");
577 CATCH(req->issuer->running_ctx->exception) {
578 req->issuer->doexception = 1;
582 case SIMIX_LINK_FAILURE:
584 XBT_DEBUG("Link failure in action %p between '%s' and '%s': posting an exception to the issuer: %s (%p)",
585 action, action->comm.src_proc->smx_host->name, action->comm.dst_proc->smx_host->name,
586 req->issuer->name, req->issuer);
587 THROW0(network_error, 0, "Link failure");
589 CATCH(req->issuer->running_ctx->exception) {
590 req->issuer->doexception = 1;
598 /* if there is an exception during a waitany or a testany, indicate the position of the failed communication */
599 if (req->issuer->doexception) {
600 if (req->call == REQ_COMM_WAITANY) {
601 req->issuer->running_ctx->exception.value = xbt_dynar_search(req->comm_waitany.comms, &action);
603 else if (req->call == REQ_COMM_TESTANY) {
604 req->issuer->running_ctx->exception.value = xbt_dynar_search(req->comm_testany.comms, &action);
608 req->issuer->waiting_action = NULL;
609 SIMIX_request_answer(req);
613 while (destroy_count-- > 0)
614 SIMIX_comm_destroy(action);
617 void SIMIX_post_comm(smx_action_t action)
619 /* Update action state */
620 if (action->comm.src_timeout &&
621 surf_workstation_model->action_state_get(action->comm.src_timeout) == SURF_ACTION_DONE)
622 action->state = SIMIX_SRC_TIMEOUT;
623 else if (action->comm.dst_timeout &&
624 surf_workstation_model->action_state_get(action->comm.dst_timeout) == SURF_ACTION_DONE)
625 action->state = SIMIX_DST_TIMEOUT;
626 else if (action->comm.src_timeout &&
627 surf_workstation_model->action_state_get(action->comm.src_timeout) == SURF_ACTION_FAILED)
628 action->state = SIMIX_SRC_HOST_FAILURE;
629 else if (action->comm.dst_timeout &&
630 surf_workstation_model->action_state_get(action->comm.dst_timeout) == SURF_ACTION_FAILED)
631 action->state = SIMIX_DST_HOST_FAILURE;
632 else if (action->comm.surf_comm &&
633 surf_workstation_model->action_state_get(action->comm.surf_comm) == SURF_ACTION_FAILED)
634 action->state = SIMIX_LINK_FAILURE;
636 action->state = SIMIX_DONE;
638 XBT_DEBUG("SIMIX_post_comm: action state = %d", action->state);
640 /* After this point the surf actions associated with the simix communicate
641 action are no longer needed, thus we delete them. */
642 SIMIX_comm_destroy_internal_actions(action);
644 /* If there are requests associated with the action, then answer them */
645 if (xbt_fifo_size(action->request_list))
646 SIMIX_comm_finish(action);
649 void SIMIX_comm_cancel(smx_action_t action)
651 /* If the action is a waiting state means that it is still in a rdv */
652 /* so remove from it and delete it */
653 if (action->state == SIMIX_WAITING) {
654 SIMIX_rdv_remove(action->comm.rdv, action);
655 action->state = SIMIX_FAILED;
657 /* When running the MC there are no surf actions */
659 surf_workstation_model->action_cancel(action->comm.surf_comm);
663 void SIMIX_comm_suspend(smx_action_t action)
665 /*FIXME: shall we suspend also the timeout actions? */
666 surf_workstation_model->suspend(action->comm.surf_comm);
669 void SIMIX_comm_resume(smx_action_t action)
671 /*FIXME: check what happen with the timeouts */
672 surf_workstation_model->resume(action->comm.surf_comm);
676 /************* Action Getters **************/
679 * \brief get the amount remaining from the communication
680 * \param action The communication
682 double SIMIX_comm_get_remains(smx_action_t action)
686 switch (action->state) {
689 remains = surf_workstation_model->get_remains(action->comm.surf_comm);
694 remains = 0; /*FIXME: check what should be returned */
698 remains = 0; /*FIXME: is this correct? */
704 e_smx_state_t SIMIX_comm_get_state(smx_action_t action)
706 return action->state;
710 * \brief Return the user data associated to the sender of the communication
711 * \param action The communication
712 * \return the user data
714 void* SIMIX_comm_get_src_data(smx_action_t action)
716 return action->comm.src_data;
720 * \brief Return the user data associated to the receiver of the communication
721 * \param action The communication
722 * \return the user data
724 void* SIMIX_comm_get_dst_data(smx_action_t action)
726 return action->comm.dst_data;
729 smx_process_t SIMIX_comm_get_src_proc(smx_action_t action)
731 return action->comm.src_proc;
734 smx_process_t SIMIX_comm_get_dst_proc(smx_action_t action)
736 return action->comm.dst_proc;
739 #ifdef HAVE_LATENCY_BOUND_TRACKING
741 * \brief verify if communication is latency bounded
742 * \param comm The communication
744 XBT_INLINE int SIMIX_comm_is_latency_bounded(smx_action_t action)
746 if (action->comm.surf_comm){
747 XBT_DEBUG("Getting latency limited for surf_action (%p)", action->comm.surf_comm);
748 action->latency_limited = surf_workstation_model->get_latency_limited(action->comm.surf_comm);
749 XBT_DEBUG("Action limited is %d", action->latency_limited);
751 return action->latency_limited;
755 /******************************************************************************/
756 /* SIMIX_comm_copy_data callbacks */
757 /******************************************************************************/
758 static void (*SIMIX_comm_copy_data_callback) (smx_action_t, size_t) =
759 &SIMIX_comm_copy_pointer_callback;
762 SIMIX_comm_set_copy_data_callback(void (*callback) (smx_action_t, size_t))
764 SIMIX_comm_copy_data_callback = callback;
767 void SIMIX_comm_copy_pointer_callback(smx_action_t comm, size_t buff_size)
769 xbt_assert1((buff_size == sizeof(void *)),
770 "Cannot copy %zu bytes: must be sizeof(void*)", buff_size);
771 *(void **) (comm->comm.dst_buff) = comm->comm.src_buff;
774 void SIMIX_comm_copy_buffer_callback(smx_action_t comm, size_t buff_size)
776 memcpy(comm->comm.dst_buff, comm->comm.src_buff, buff_size);
780 * \brief Copy the communication data from the sender's buffer to the receiver's one
781 * \param comm The communication
783 void SIMIX_comm_copy_data(smx_action_t comm)
785 size_t buff_size = comm->comm.src_buff_size;
786 /* If there is no data to be copy then return */
787 if (!comm->comm.src_buff || !comm->comm.dst_buff || comm->comm.copied == 1)
790 XBT_DEBUG("Copying comm %p data from %s (%p) -> %s (%p) (%zu bytes)",
792 comm->comm.src_proc->smx_host->name, comm->comm.src_buff,
793 comm->comm.dst_proc->smx_host->name, comm->comm.dst_buff, buff_size);
795 /* Copy at most dst_buff_size bytes of the message to receiver's buffer */
796 if (comm->comm.dst_buff_size)
797 buff_size = MIN(buff_size, *(comm->comm.dst_buff_size));
799 /* Update the receiver's buffer size to the copied amount */
800 if (comm->comm.dst_buff_size)
801 *comm->comm.dst_buff_size = buff_size;
806 (*SIMIX_comm_copy_data_callback) (comm, buff_size);
808 /* Set the copied flag so we copy data only once */
809 /* (this function might be called from both communication ends) */
810 comm->comm.copied = 1;