1 /* Copyright (c) 2009-2013. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
11 #include <xbt/replay.h>
13 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_replay,smpi,"Trace Replay with SMPI");
15 int communicator_size = 0;
16 static int active_processes = 0;
17 xbt_dynar_t *reqq = NULL;
19 MPI_Datatype MPI_DEFAULT_TYPE;
20 MPI_Datatype MPI_CURRENT_TYPE;
22 static void log_timed_action (const char *const *action, double clock){
23 if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
24 char *name = xbt_str_join_array(action, " ");
25 XBT_VERB("%s %f", name, smpi_process_simulated_elapsed()-clock);
31 xbt_dynar_t irecvs; /* of MPI_Request */
32 } s_smpi_replay_globals_t, *smpi_replay_globals_t;
36 static double parse_double(const char *string)
40 value = strtod(string, &endptr);
42 THROWF(unknown_error, 0, "%s is not a double", string);
46 static MPI_Datatype decode_datatype(const char *const action)
48 // Declared datatypes,
53 MPI_CURRENT_TYPE=MPI_DOUBLE;
56 MPI_CURRENT_TYPE=MPI_INT;
59 MPI_CURRENT_TYPE=MPI_CHAR;
62 MPI_CURRENT_TYPE=MPI_SHORT;
65 MPI_CURRENT_TYPE=MPI_LONG;
68 MPI_CURRENT_TYPE=MPI_FLOAT;
71 MPI_CURRENT_TYPE=MPI_BYTE;
74 MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
77 return MPI_CURRENT_TYPE;
81 const char* encode_datatype(MPI_Datatype datatype)
84 //default type for output is set to MPI_BYTE
85 // MPI_DEFAULT_TYPE is not set for output, use directly MPI_BYTE
86 if (datatype==MPI_BYTE){
89 if(datatype==MPI_DOUBLE)
93 if(datatype==MPI_CHAR)
95 if(datatype==MPI_SHORT)
97 if(datatype==MPI_LONG)
99 if(datatype==MPI_FLOAT)
102 // default - not implemented.
103 // do not warn here as we pass in this function even for other trace formats
107 static void action_init(const char *const *action)
110 XBT_DEBUG("Initialize the counters");
111 smpi_replay_globals_t globals = xbt_new(s_smpi_replay_globals_t, 1);
112 globals->irecvs = xbt_dynar_new(sizeof(MPI_Request),NULL);
114 if(action[2]) MPI_DEFAULT_TYPE= MPI_DOUBLE; // default MPE dataype
115 else MPI_DEFAULT_TYPE= MPI_BYTE; // default TAU datatype
117 smpi_process_set_user_data((void*) globals);
119 /* start a simulated timer */
120 smpi_process_simulated_start();
121 /*initialize the number of active processes */
122 active_processes = smpi_process_count();
125 reqq=xbt_new0(xbt_dynar_t,active_processes);
127 for(i=0;i<active_processes;i++){
128 reqq[i]=xbt_dynar_new(sizeof(MPI_Request),NULL);
133 static void action_finalize(const char *const *action)
135 smpi_replay_globals_t globals =
136 (smpi_replay_globals_t) smpi_process_get_user_data();
138 XBT_DEBUG("There are %lu irecvs in the dynar",
139 xbt_dynar_length(globals->irecvs));
140 xbt_dynar_free_container(&(globals->irecvs));
145 static void action_comm_size(const char *const *action)
147 double clock = smpi_process_simulated_elapsed();
149 communicator_size = parse_double(action[2]);
150 log_timed_action (action, clock);
153 static void action_comm_split(const char *const *action)
155 double clock = smpi_process_simulated_elapsed();
157 log_timed_action (action, clock);
160 static void action_comm_dup(const char *const *action)
162 double clock = smpi_process_simulated_elapsed();
164 log_timed_action (action, clock);
167 static void action_compute(const char *const *action)
169 double clock = smpi_process_simulated_elapsed();
170 double flops= parse_double(action[2]);
172 int rank = smpi_comm_rank(MPI_COMM_WORLD);
173 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
174 extra->type=TRACING_COMPUTING;
175 extra->comp_size=flops;
176 TRACE_smpi_computing_in(rank, extra);
178 smpi_execute_flops(flops);
180 TRACE_smpi_computing_out(rank);
183 log_timed_action (action, clock);
186 static void action_send(const char *const *action)
188 int to = atoi(action[2]);
189 double size=parse_double(action[3]);
190 double clock = smpi_process_simulated_elapsed();
193 MPI_CURRENT_TYPE=decode_datatype(action[4]);
195 MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
199 int rank = smpi_comm_rank(MPI_COMM_WORLD);
201 int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
202 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
203 extra->type = TRACING_SEND;
204 extra->send_size = size;
206 extra->dst = dst_traced;
207 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
208 TRACE_smpi_ptp_in(rank, rank, dst_traced, __FUNCTION__, extra);
209 TRACE_smpi_send(rank, rank, dst_traced, size*smpi_datatype_size(MPI_CURRENT_TYPE));
212 smpi_mpi_send(NULL, size, MPI_CURRENT_TYPE, to , 0, MPI_COMM_WORLD);
214 log_timed_action (action, clock);
217 TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
222 static void action_Isend(const char *const *action)
224 int to = atoi(action[2]);
225 double size=parse_double(action[3]);
226 double clock = smpi_process_simulated_elapsed();
229 if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
230 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
233 int rank = smpi_comm_rank(MPI_COMM_WORLD);
234 int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
235 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
236 extra->type = TRACING_ISEND;
237 extra->send_size = size;
239 extra->dst = dst_traced;
240 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
241 TRACE_smpi_ptp_in(rank, rank, dst_traced, __FUNCTION__, extra);
242 TRACE_smpi_send(rank, rank, dst_traced, size*smpi_datatype_size(MPI_CURRENT_TYPE));
245 request = smpi_mpi_isend(NULL, size, MPI_CURRENT_TYPE, to, 0,MPI_COMM_WORLD);
248 TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
252 xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
254 log_timed_action (action, clock);
257 static void action_recv(const char *const *action) {
258 int from = atoi(action[2]);
259 double size=parse_double(action[3]);
260 double clock = smpi_process_simulated_elapsed();
263 if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
264 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
267 int rank = smpi_comm_rank(MPI_COMM_WORLD);
268 int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
270 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
271 extra->type = TRACING_RECV;
272 extra->send_size = size;
273 extra->src = src_traced;
275 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
276 TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__, extra);
279 smpi_mpi_recv(NULL, size, MPI_CURRENT_TYPE, from, 0, MPI_COMM_WORLD, &status);
282 TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
283 TRACE_smpi_recv(rank, src_traced, rank);
286 log_timed_action (action, clock);
289 static void action_Irecv(const char *const *action)
291 int from = atoi(action[2]);
292 double size=parse_double(action[3]);
293 double clock = smpi_process_simulated_elapsed();
296 smpi_replay_globals_t globals =
297 (smpi_replay_globals_t) smpi_process_get_user_data();
299 if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
300 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
303 int rank = smpi_comm_rank(MPI_COMM_WORLD);
304 int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
305 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
306 extra->type = TRACING_IRECV;
307 extra->send_size = size;
308 extra->src = src_traced;
310 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
311 TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__, extra);
314 request = smpi_mpi_irecv(NULL, size, MPI_CURRENT_TYPE, from, 0, MPI_COMM_WORLD);
317 TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
320 xbt_dynar_push(globals->irecvs,&request);
321 xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
323 log_timed_action (action, clock);
326 static void action_wait(const char *const *action){
327 double clock = smpi_process_simulated_elapsed();
330 smpi_replay_globals_t globals =
331 (smpi_replay_globals_t) smpi_process_get_user_data();
333 xbt_assert(xbt_dynar_length(globals->irecvs),
334 "action wait not preceded by any irecv: %s",
335 xbt_str_join_array(action," "));
336 request = xbt_dynar_pop_as(globals->irecvs,MPI_Request);
338 int rank = request && request->comm != MPI_COMM_NULL
339 ? smpi_comm_rank(request->comm)
342 MPI_Group group = smpi_comm_group(request->comm);
343 int src_traced = smpi_group_rank(group, request->src);
344 int dst_traced = smpi_group_rank(group, request->dst);
345 int is_wait_for_receive = request->recv;
346 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
347 extra->type = TRACING_WAIT;
348 TRACE_smpi_ptp_in(rank, src_traced, dst_traced, __FUNCTION__, extra);
350 smpi_mpi_wait(&request, &status);
352 TRACE_smpi_ptp_out(rank, src_traced, dst_traced, __FUNCTION__);
353 if (is_wait_for_receive) {
354 TRACE_smpi_recv(rank, src_traced, dst_traced);
358 log_timed_action (action, clock);
361 static void action_waitall(const char *const *action){
362 double clock = smpi_process_simulated_elapsed();
363 int count_requests=0;
366 count_requests=xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
368 if (count_requests>0) {
369 MPI_Request requests[count_requests];
370 MPI_Status status[count_requests];
372 /* The reqq is an array of dynars. Its index corresponds to the rank.
373 Thus each rank saves its own requests to the array request. */
374 xbt_dynar_foreach(reqq[smpi_comm_rank(MPI_COMM_WORLD)],i,requests[i]);
377 //save information from requests
379 xbt_dynar_t srcs = xbt_dynar_new(sizeof(int), NULL);
380 xbt_dynar_t dsts = xbt_dynar_new(sizeof(int), NULL);
381 xbt_dynar_t recvs = xbt_dynar_new(sizeof(int), NULL);
382 for (i = 0; i < count_requests; i++) {
384 int *asrc = xbt_new(int, 1);
385 int *adst = xbt_new(int, 1);
386 int *arecv = xbt_new(int, 1);
387 *asrc = requests[i]->src;
388 *adst = requests[i]->dst;
389 *arecv = requests[i]->recv;
390 xbt_dynar_insert_at(srcs, i, asrc);
391 xbt_dynar_insert_at(dsts, i, adst);
392 xbt_dynar_insert_at(recvs, i, arecv);
397 int *t = xbt_new(int, 1);
398 xbt_dynar_insert_at(srcs, i, t);
399 xbt_dynar_insert_at(dsts, i, t);
400 xbt_dynar_insert_at(recvs, i, t);
404 int rank_traced = smpi_process_index();
405 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
406 extra->type = TRACING_WAITALL;
407 extra->send_size=count_requests;
408 TRACE_smpi_ptp_in(rank_traced, -1, -1, __FUNCTION__,extra);
411 smpi_mpi_waitall(count_requests, requests, status);
414 for (i = 0; i < count_requests; i++) {
415 int src_traced, dst_traced, is_wait_for_receive;
416 xbt_dynar_get_cpy(srcs, i, &src_traced);
417 xbt_dynar_get_cpy(dsts, i, &dst_traced);
418 xbt_dynar_get_cpy(recvs, i, &is_wait_for_receive);
419 if (is_wait_for_receive) {
420 TRACE_smpi_recv(rank_traced, src_traced, dst_traced);
423 TRACE_smpi_ptp_out(rank_traced, -1, -1, __FUNCTION__);
425 xbt_dynar_free(&srcs);
426 xbt_dynar_free(&dsts);
427 xbt_dynar_free(&recvs);
430 xbt_dynar_reset(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
432 log_timed_action (action, clock);
435 static void action_barrier(const char *const *action){
436 double clock = smpi_process_simulated_elapsed();
438 int rank = smpi_comm_rank(MPI_COMM_WORLD);
439 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
440 extra->type = TRACING_BARRIER;
441 TRACE_smpi_collective_in(rank, -1, __FUNCTION__, extra);
443 smpi_mpi_barrier(MPI_COMM_WORLD);
445 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
448 log_timed_action (action, clock);
452 static void action_bcast(const char *const *action)
454 double size = parse_double(action[2]);
455 double clock = smpi_process_simulated_elapsed();
458 * Initialize MPI_CURRENT_TYPE in order to decrease
459 * the number of the checks
461 MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
464 root= atoi(action[3]);
466 MPI_CURRENT_TYPE=decode_datatype(action[4]);
471 int rank = smpi_comm_rank(MPI_COMM_WORLD);
472 int root_traced = smpi_group_index(smpi_comm_group(MPI_COMM_WORLD), root);
474 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
475 extra->type = TRACING_BCAST;
476 extra->send_size = size;
477 extra->root = root_traced;
478 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
479 TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__, extra);
483 mpi_coll_bcast_fun(NULL, size, MPI_CURRENT_TYPE, root, MPI_COMM_WORLD);
485 TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
488 log_timed_action (action, clock);
491 static void action_reduce(const char *const *action)
493 double comm_size = parse_double(action[2]);
494 double comp_size = parse_double(action[3]);
495 double clock = smpi_process_simulated_elapsed();
497 MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
500 root= atoi(action[4]);
502 MPI_CURRENT_TYPE=decode_datatype(action[5]);
507 int rank = smpi_comm_rank(MPI_COMM_WORLD);
508 int root_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), root);
509 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
510 extra->type = TRACING_REDUCE;
511 extra->send_size = comm_size;
512 extra->comp_size = comp_size;
513 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
514 extra->root = root_traced;
516 TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__,extra);
518 mpi_coll_reduce_fun(NULL, NULL, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, root, MPI_COMM_WORLD);
519 smpi_execute_flops(comp_size);
521 TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
524 log_timed_action (action, clock);
527 static void action_allReduce(const char *const *action) {
528 double comm_size = parse_double(action[2]);
529 double comp_size = parse_double(action[3]);
531 if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
532 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
534 double clock = smpi_process_simulated_elapsed();
536 int rank = smpi_comm_rank(MPI_COMM_WORLD);
537 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
538 extra->type = TRACING_ALLREDUCE;
539 extra->send_size = comm_size;
540 extra->comp_size = comp_size;
541 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
543 TRACE_smpi_collective_in(rank, -1, __FUNCTION__,extra);
545 mpi_coll_reduce_fun(NULL, NULL, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, 0, MPI_COMM_WORLD);
546 smpi_execute_flops(comp_size);
547 mpi_coll_bcast_fun(NULL, comm_size, MPI_CURRENT_TYPE, 0, MPI_COMM_WORLD);
549 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
552 log_timed_action (action, clock);
555 static void action_allToAll(const char *const *action) {
556 double clock = smpi_process_simulated_elapsed();
557 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
558 int send_size = parse_double(action[2]);
559 int recv_size = parse_double(action[3]);
560 MPI_Datatype MPI_CURRENT_TYPE2;
563 MPI_CURRENT_TYPE=decode_datatype(action[4]);
564 MPI_CURRENT_TYPE2=decode_datatype(action[5]);
567 MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
568 MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
570 void *send = calloc(send_size*comm_size, smpi_datatype_size(MPI_CURRENT_TYPE));
571 void *recv = calloc(recv_size*comm_size, smpi_datatype_size(MPI_CURRENT_TYPE2));
574 int rank = smpi_process_index();
575 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
576 extra->type = TRACING_ALLTOALL;
577 extra->send_size = send_size;
578 extra->recv_size = recv_size;
579 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
580 extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2);
582 TRACE_smpi_collective_in(rank, -1, __FUNCTION__,extra);
585 mpi_coll_alltoall_fun(send, send_size, MPI_CURRENT_TYPE, recv, recv_size, MPI_CURRENT_TYPE2, MPI_COMM_WORLD);
588 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
591 log_timed_action (action, clock);
597 static void action_gather(const char *const *action) {
599 The structure of the gather action for the rank 0 (total 4 processes)
604 1) 68 is the sendcounts
605 2) 68 is the recvcounts
606 3) 0 is the root node
607 4) 0 is the send datatype id, see decode_datatype()
608 5) 0 is the recv datatype id, see decode_datatype()
610 double clock = smpi_process_simulated_elapsed();
611 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
612 int send_size = parse_double(action[2]);
613 int recv_size = parse_double(action[3]);
614 MPI_Datatype MPI_CURRENT_TYPE2;
616 MPI_CURRENT_TYPE=decode_datatype(action[5]);
617 MPI_CURRENT_TYPE2=decode_datatype(action[6]);
619 MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
620 MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
622 void *send = calloc(send_size, smpi_datatype_size(MPI_CURRENT_TYPE));
625 int root=atoi(action[4]);
626 int rank = smpi_process_index();
629 recv = calloc(recv_size*comm_size, smpi_datatype_size(MPI_CURRENT_TYPE2));
632 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
633 extra->type = TRACING_GATHER;
634 extra->send_size = send_size;
635 extra->recv_size = recv_size;
637 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
638 extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2);
640 TRACE_smpi_collective_in(rank, root, __FUNCTION__, extra);
642 smpi_mpi_gather(send, send_size, MPI_CURRENT_TYPE,
643 recv, recv_size, MPI_CURRENT_TYPE2,
644 root, MPI_COMM_WORLD);
647 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
650 log_timed_action (action, clock);
657 static void action_gatherv(const char *const *action) {
659 The structure of the gatherv action for the rank 0 (total 4 processes)
661 0 gather 68 68 10 10 10 0 0 0
664 1) 68 is the sendcount
665 2) 68 10 10 10 is the recvcounts
666 3) 0 is the root node
667 4) 0 is the send datatype id, see decode_datatype()
668 5) 0 is the recv datatype id, see decode_datatype()
670 double clock = smpi_process_simulated_elapsed();
671 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
672 int send_size = parse_double(action[2]);
673 int *disps = xbt_new0(int, comm_size);
674 int *recvcounts = xbt_new0(int, comm_size);
677 MPI_Datatype MPI_CURRENT_TYPE2;
678 if(action[4+comm_size]) {
679 MPI_CURRENT_TYPE=decode_datatype(action[4+comm_size]);
680 MPI_CURRENT_TYPE2=decode_datatype(action[5+comm_size]);
682 MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
683 MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
685 void *send = calloc(send_size, smpi_datatype_size(MPI_CURRENT_TYPE));
687 for(i=0;i<comm_size;i++) {
688 recvcounts[i] = atoi(action[i+3]);
689 recv_sum=recv_sum+recvcounts[i];
693 int root=atoi(action[3+comm_size]);
694 int rank = smpi_process_index();
697 recv = calloc(recv_sum, smpi_datatype_size(MPI_CURRENT_TYPE2));
700 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
701 extra->type = TRACING_GATHERV;
702 extra->send_size = send_size;
703 extra->recvcounts= xbt_malloc(comm_size*sizeof(int));
704 for(i=0; i< comm_size; i++)//copy data to avoid bad free
705 extra->recvcounts[i] = recvcounts[i];
707 extra->num_processes = comm_size;
708 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
709 extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2);
711 TRACE_smpi_collective_in(rank, root, __FUNCTION__, extra);
713 smpi_mpi_gatherv(send, send_size, MPI_CURRENT_TYPE,
714 recv, recvcounts, disps, MPI_CURRENT_TYPE2,
715 root, MPI_COMM_WORLD);
718 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
721 log_timed_action (action, clock);
722 xbt_free(recvcounts);
729 static void action_reducescatter(const char *const *action) {
732 The structure of the reducescatter action for the rank 0 (total 4 processes)
734 0 reduceScatter 275427 275427 275427 204020 11346849 0
737 1) The first four values after the name of the action declare the recvcounts array
738 2) The value 11346849 is the amount of instructions
739 3) The last value corresponds to the datatype, see decode_datatype().
741 We analyze a MPI_Reduce_scatter call to one MPI_Reduce and one MPI_Scatterv.
745 double clock = smpi_process_simulated_elapsed();
746 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
747 int comp_size = parse_double(action[2+comm_size]);
748 int *recvcounts = xbt_new0(int, comm_size);
749 int *disps = xbt_new0(int, comm_size);
752 int rank = smpi_process_index();
754 if(action[3+comm_size])
755 MPI_CURRENT_TYPE=decode_datatype(action[3+comm_size]);
757 MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
759 for(i=0;i<comm_size;i++) {
760 recvcounts[i] = atoi(action[i+2]);
761 recv_sum=recv_sum+recvcounts[i];
766 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
767 extra->type = TRACING_REDUCE_SCATTER;
768 extra->send_size = 0;
769 extra->recvcounts= xbt_malloc(comm_size*sizeof(int));
770 for(i=0; i< comm_size; i++)//copy data to avoid bad free
771 extra->recvcounts[i] = recvcounts[i];
772 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
773 extra->comp_size = comp_size;
774 extra->num_processes = comm_size;
777 TRACE_smpi_collective_in(rank, -1, __FUNCTION__,extra);
779 mpi_coll_reduce_fun(NULL, NULL, recv_sum, MPI_CURRENT_TYPE, MPI_OP_NULL,
780 root, MPI_COMM_WORLD);
781 smpi_mpi_scatterv(NULL, recvcounts, disps, MPI_CURRENT_TYPE, NULL,
782 recvcounts[rank], MPI_CURRENT_TYPE, 0, MPI_COMM_WORLD);
783 smpi_execute_flops(comp_size);
787 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
789 xbt_free(recvcounts);
791 log_timed_action (action, clock);
795 static void action_allgatherv(const char *const *action) {
798 The structure of the allgatherv action for the rank 0 (total 4 processes)
800 0 allGatherV 275427 275427 275427 275427 204020
803 1) 275427 is the sendcount
804 2) The next four elements declare the recvcounts array
805 3) No more values mean that the datatype for sent and receive buffer
806 is the default one, see decode_datatype().
810 double clock = smpi_process_simulated_elapsed();
812 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
814 int sendcount=atoi(action[2]);
815 int *recvcounts = xbt_new0(int, comm_size);
816 int *disps = xbt_new0(int, comm_size);
818 MPI_Datatype MPI_CURRENT_TYPE2;
820 if(action[3+comm_size]) {
821 MPI_CURRENT_TYPE = decode_datatype(action[3+comm_size]);
822 MPI_CURRENT_TYPE2 = decode_datatype(action[4+comm_size]);
824 MPI_CURRENT_TYPE = MPI_DEFAULT_TYPE;
825 MPI_CURRENT_TYPE2 = MPI_DEFAULT_TYPE;
827 void *sendbuf = calloc(sendcount, smpi_datatype_size(MPI_CURRENT_TYPE));
829 for(i=0;i<comm_size;i++) {
830 recvcounts[i] = atoi(action[i+3]);
831 recv_sum=recv_sum+recvcounts[i];
833 void *recvbuf = calloc(recv_sum, smpi_datatype_size(MPI_CURRENT_TYPE2));
836 int rank = smpi_process_index();
837 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
838 extra->type = TRACING_ALLGATHERV;
839 extra->send_size = sendcount;
840 extra->recvcounts= xbt_malloc(comm_size*sizeof(int));
841 for(i=0; i< comm_size; i++)//copy data to avoid bad free
842 extra->recvcounts[i] = recvcounts[i];
843 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
844 extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2);
845 extra->num_processes = comm_size;
847 TRACE_smpi_collective_in(rank, -1, __FUNCTION__,extra);
850 mpi_coll_allgatherv_fun(sendbuf, sendcount, MPI_CURRENT_TYPE, recvbuf, recvcounts, disps, MPI_CURRENT_TYPE2, MPI_COMM_WORLD);
853 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
856 log_timed_action (action, clock);
859 xbt_free(recvcounts);
864 static void action_allToAllv(const char *const *action) {
866 The structure of the allToAllV action for the rank 0 (total 4 processes)
868 0 allToAllV 100 1 7 10 12 100 1 70 10 5
871 1) 100 is the size of the send buffer *sizeof(int),
872 2) 1 7 10 12 is the sendcounts array
873 3) 100*sizeof(int) is the size of the receiver buffer
874 4) 1 70 10 5 is the recvcounts array
879 double clock = smpi_process_simulated_elapsed();
881 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
882 int send_buf_size=0,recv_buf_size=0,i=0;
883 int *sendcounts = xbt_new0(int, comm_size);
884 int *recvcounts = xbt_new0(int, comm_size);
885 int *senddisps = xbt_new0(int, comm_size);
886 int *recvdisps = xbt_new0(int, comm_size);
888 MPI_Datatype MPI_CURRENT_TYPE2;
890 send_buf_size=parse_double(action[2]);
891 recv_buf_size=parse_double(action[3+comm_size]);
892 if(action[4+2*comm_size]) {
893 MPI_CURRENT_TYPE=decode_datatype(action[4+2*comm_size]);
894 MPI_CURRENT_TYPE2=decode_datatype(action[5+2*comm_size]);
897 MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
898 MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE;
901 void *sendbuf = calloc(send_buf_size, smpi_datatype_size(MPI_CURRENT_TYPE));
902 void *recvbuf = calloc(recv_buf_size, smpi_datatype_size(MPI_CURRENT_TYPE2));
904 for(i=0;i<comm_size;i++) {
905 sendcounts[i] = atoi(action[i+3]);
906 recvcounts[i] = atoi(action[i+4+comm_size]);
911 int rank = smpi_process_index();
912 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
913 extra->type = TRACING_ALLTOALLV;
914 extra->recvcounts= xbt_malloc(comm_size*sizeof(int));
915 extra->sendcounts= xbt_malloc(comm_size*sizeof(int));
916 extra->num_processes = comm_size;
918 for(i=0; i< comm_size; i++){//copy data to avoid bad free
919 extra->send_size += sendcounts[i];
920 extra->sendcounts[i] = sendcounts[i];
921 extra->recv_size += recvcounts[i];
922 extra->recvcounts[i] = recvcounts[i];
924 extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE);
925 extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2);
927 TRACE_smpi_collective_in(rank, -1, __FUNCTION__,extra);
929 mpi_coll_alltoallv_fun(sendbuf, sendcounts, senddisps, MPI_CURRENT_TYPE,
930 recvbuf, recvcounts, recvdisps, MPI_CURRENT_TYPE,
933 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
936 log_timed_action (action, clock);
939 xbt_free(sendcounts);
940 xbt_free(recvcounts);
945 void smpi_replay_init(int *argc, char***argv){
946 smpi_process_init(argc, argv);
947 smpi_process_mark_as_initialized();
949 int rank = smpi_process_index();
950 TRACE_smpi_init(rank);
951 TRACE_smpi_computing_init(rank);
952 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
953 extra->type = TRACING_INIT;
954 TRACE_smpi_collective_in(rank, -1, __FUNCTION__, extra);
955 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
958 if (!smpi_process_index()){
959 _xbt_replay_action_init();
960 xbt_replay_action_register("init", action_init);
961 xbt_replay_action_register("finalize", action_finalize);
962 xbt_replay_action_register("comm_size", action_comm_size);
963 xbt_replay_action_register("comm_split", action_comm_split);
964 xbt_replay_action_register("comm_dup", action_comm_dup);
965 xbt_replay_action_register("send", action_send);
966 xbt_replay_action_register("Isend", action_Isend);
967 xbt_replay_action_register("recv", action_recv);
968 xbt_replay_action_register("Irecv", action_Irecv);
969 xbt_replay_action_register("wait", action_wait);
970 xbt_replay_action_register("waitAll", action_waitall);
971 xbt_replay_action_register("barrier", action_barrier);
972 xbt_replay_action_register("bcast", action_bcast);
973 xbt_replay_action_register("reduce", action_reduce);
974 xbt_replay_action_register("allReduce", action_allReduce);
975 xbt_replay_action_register("allToAll", action_allToAll);
976 xbt_replay_action_register("allToAllV", action_allToAllv);
977 xbt_replay_action_register("gather", action_gather);
978 xbt_replay_action_register("gatherV", action_gatherv);
979 xbt_replay_action_register("allGatherV", action_allgatherv);
980 xbt_replay_action_register("reduceScatter", action_reducescatter);
981 xbt_replay_action_register("compute", action_compute);
984 xbt_replay_action_runner(*argc, *argv);
987 int smpi_replay_finalize(){
989 /* One active process will stop. Decrease the counter*/
991 XBT_DEBUG("There are %lu elements in reqq[*]",
992 xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]));
993 xbt_dynar_free(&reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
994 if(!active_processes){
995 /* Last process alive speaking */
996 /* end the simulated timer */
997 sim_time = smpi_process_simulated_elapsed();
998 XBT_INFO("Simulation time %g", sim_time);
999 _xbt_replay_action_exit();
1003 smpi_mpi_barrier(MPI_COMM_WORLD);
1005 int rank = smpi_process_index();
1006 instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);
1007 extra->type = TRACING_FINALIZE;
1008 TRACE_smpi_collective_in(rank, -1, __FUNCTION__, extra);
1010 smpi_process_finalize();
1012 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
1013 TRACE_smpi_finalize(smpi_process_index());
1015 smpi_process_destroy();