1 /* Copyright (c) 2009, 2010, 2011, 2012. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
11 #include <xbt/replay.h>
13 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_replay,smpi,"Trace Replay with SMPI");
15 int communicator_size = 0;
16 static int active_processes = 0;
17 xbt_dynar_t *reqq = NULL;
19 MPI_Datatype MPI_DEFAULT_TYPE, MPI_CURRENT_TYPE;
21 static void log_timed_action (const char *const *action, double clock){
22 if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
23 char *name = xbt_str_join_array(action, " ");
24 XBT_VERB("%s %f", name, smpi_process_simulated_elapsed()-clock);
30 xbt_dynar_t isends; /* of MPI_Request */
31 xbt_dynar_t irecvs; /* of MPI_Request */
32 } s_smpi_replay_globals_t, *smpi_replay_globals_t;
36 static double parse_double(const char *string)
40 value = strtod(string, &endptr);
42 THROWF(unknown_error, 0, "%s is not a double", string);
46 static MPI_Datatype decode_datatype(const char *const action)
48 // Declared datatypes,
53 MPI_CURRENT_TYPE=MPI_DOUBLE;
56 MPI_CURRENT_TYPE=MPI_INT;
59 MPI_CURRENT_TYPE=MPI_CHAR;
62 MPI_CURRENT_TYPE=MPI_SHORT;
65 MPI_CURRENT_TYPE=MPI_LONG;
68 MPI_CURRENT_TYPE=MPI_FLOAT;
71 MPI_CURRENT_TYPE=MPI_BYTE;
74 MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE;
77 return MPI_CURRENT_TYPE;
80 static void action_init(const char *const *action)
83 XBT_DEBUG("Initialize the counters");
84 smpi_replay_globals_t globals = xbt_new(s_smpi_replay_globals_t, 1);
85 globals->isends = xbt_dynar_new(sizeof(MPI_Request),NULL);
86 globals->irecvs = xbt_dynar_new(sizeof(MPI_Request),NULL);
88 if(action[2]) MPI_DEFAULT_TYPE= MPI_DOUBLE; // default MPE dataype
89 else MPI_DEFAULT_TYPE= MPI_BYTE; // default TAU datatype
91 smpi_process_set_user_data((void*) globals);
93 /* start a simulated timer */
94 smpi_process_simulated_start();
95 /*initialize the number of active processes */
96 active_processes = smpi_process_count();
99 reqq=xbt_new0(xbt_dynar_t,active_processes);
101 for(i=0;i<active_processes;i++){
102 reqq[i]=xbt_dynar_new(sizeof(MPI_Request),NULL);
107 static void action_finalize(const char *const *action)
109 smpi_replay_globals_t globals =
110 (smpi_replay_globals_t) smpi_process_get_user_data();
112 XBT_DEBUG("There are %lu isends and %lu irecvs in the dynars",
113 xbt_dynar_length(globals->isends),xbt_dynar_length(globals->irecvs));
114 xbt_dynar_free_container(&(globals->isends));
115 xbt_dynar_free_container(&(globals->irecvs));
120 static void action_comm_size(const char *const *action)
122 double clock = smpi_process_simulated_elapsed();
124 communicator_size = parse_double(action[2]);
125 log_timed_action (action, clock);
128 static void action_comm_split(const char *const *action)
130 double clock = smpi_process_simulated_elapsed();
132 log_timed_action (action, clock);
135 static void action_comm_dup(const char *const *action)
137 double clock = smpi_process_simulated_elapsed();
139 log_timed_action (action, clock);
142 static void action_compute(const char *const *action)
144 double clock = smpi_process_simulated_elapsed();
145 smpi_execute_flops(parse_double(action[2]));
147 log_timed_action (action, clock);
150 static void action_send(const char *const *action)
152 int to = atoi(action[2]);
153 double size=parse_double(action[3]);
154 double clock = smpi_process_simulated_elapsed();
157 MPI_CURRENT_TYPE=decode_datatype(action[4]);
159 MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
163 int rank = smpi_comm_rank(MPI_COMM_WORLD);
164 TRACE_smpi_computing_out(rank);
165 int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
166 TRACE_smpi_ptp_in(rank, rank, dst_traced, __FUNCTION__);
167 TRACE_smpi_send(rank, rank, dst_traced);
170 smpi_mpi_send(NULL, size, MPI_CURRENT_TYPE, to , 0, MPI_COMM_WORLD);
172 log_timed_action (action, clock);
175 TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
176 TRACE_smpi_computing_in(rank);
181 static void action_Isend(const char *const *action)
183 int to = atoi(action[2]);
184 double size=parse_double(action[3]);
185 double clock = smpi_process_simulated_elapsed();
188 if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
189 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
191 smpi_replay_globals_t globals =
192 (smpi_replay_globals_t) smpi_process_get_user_data();
194 int rank = smpi_comm_rank(MPI_COMM_WORLD);
195 TRACE_smpi_computing_out(rank);
196 int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
197 TRACE_smpi_ptp_in(rank, rank, dst_traced, __FUNCTION__);
198 TRACE_smpi_send(rank, rank, dst_traced);
201 request = smpi_mpi_isend(NULL, size, MPI_CURRENT_TYPE, to, 0,MPI_COMM_WORLD);
204 TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
206 TRACE_smpi_computing_in(rank);
209 xbt_dynar_push(globals->isends,&request);
210 xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
212 log_timed_action (action, clock);
215 static void action_recv(const char *const *action) {
216 int from = atoi(action[2]);
217 double size=parse_double(action[3]);
218 double clock = smpi_process_simulated_elapsed();
221 if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
222 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
225 int rank = smpi_comm_rank(MPI_COMM_WORLD);
226 int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
227 TRACE_smpi_computing_out(rank);
229 TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__);
232 smpi_mpi_recv(NULL, size, MPI_CURRENT_TYPE, from, 0, MPI_COMM_WORLD, &status);
235 TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
236 TRACE_smpi_recv(rank, src_traced, rank);
237 TRACE_smpi_computing_in(rank);
240 log_timed_action (action, clock);
243 static void action_Irecv(const char *const *action)
245 int from = atoi(action[2]);
246 double size=parse_double(action[3]);
247 double clock = smpi_process_simulated_elapsed();
250 smpi_replay_globals_t globals =
251 (smpi_replay_globals_t) smpi_process_get_user_data();
253 if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
254 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
257 int rank = smpi_comm_rank(MPI_COMM_WORLD);
258 int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
259 TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__);
262 request = smpi_mpi_irecv(NULL, size, MPI_CURRENT_TYPE, from, 0, MPI_COMM_WORLD);
265 TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
268 xbt_dynar_push(globals->irecvs,&request);
269 xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
271 log_timed_action (action, clock);
274 static void action_wait(const char *const *action){
275 double clock = smpi_process_simulated_elapsed();
278 smpi_replay_globals_t globals =
279 (smpi_replay_globals_t) smpi_process_get_user_data();
281 xbt_assert(xbt_dynar_length(globals->irecvs),
282 "action wait not preceded by any irecv: %s",
283 xbt_str_join_array(action," "));
284 request = xbt_dynar_pop_as(globals->irecvs,MPI_Request);
286 int rank = request && request->comm != MPI_COMM_NULL
287 ? smpi_comm_rank(request->comm)
289 TRACE_smpi_computing_out(rank);
291 MPI_Group group = smpi_comm_group(request->comm);
292 int src_traced = smpi_group_rank(group, request->src);
293 int dst_traced = smpi_group_rank(group, request->dst);
294 int is_wait_for_receive = request->recv;
295 TRACE_smpi_ptp_in(rank, src_traced, dst_traced, __FUNCTION__);
297 smpi_mpi_wait(&request, &status);
299 TRACE_smpi_ptp_out(rank, src_traced, dst_traced, __FUNCTION__);
300 if (is_wait_for_receive) {
301 TRACE_smpi_recv(rank, src_traced, dst_traced);
303 TRACE_smpi_computing_in(rank);
306 log_timed_action (action, clock);
309 static void action_waitall(const char *const *action){
310 double clock = smpi_process_simulated_elapsed();
311 int count_requests=0;
314 count_requests=xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
316 if (count_requests>0) {
317 MPI_Request requests[count_requests];
318 MPI_Status status[count_requests];
320 /* The reqq is an array of dynars. Its index corresponds to the rank.
321 Thus each rank saves its own requests to the array request. */
322 xbt_dynar_foreach(reqq[smpi_comm_rank(MPI_COMM_WORLD)],i,requests[i]);
325 //save information from requests
327 xbt_dynar_t srcs = xbt_dynar_new(sizeof(int), NULL);
328 xbt_dynar_t dsts = xbt_dynar_new(sizeof(int), NULL);
329 xbt_dynar_t recvs = xbt_dynar_new(sizeof(int), NULL);
330 for (i = 0; i < count_requests; i++) {
332 int *asrc = xbt_new(int, 1);
333 int *adst = xbt_new(int, 1);
334 int *arecv = xbt_new(int, 1);
335 *asrc = requests[i]->src;
336 *adst = requests[i]->dst;
337 *arecv = requests[i]->recv;
338 xbt_dynar_insert_at(srcs, i, asrc);
339 xbt_dynar_insert_at(dsts, i, adst);
340 xbt_dynar_insert_at(recvs, i, arecv);
345 int *t = xbt_new(int, 1);
346 xbt_dynar_insert_at(srcs, i, t);
347 xbt_dynar_insert_at(dsts, i, t);
348 xbt_dynar_insert_at(recvs, i, t);
352 int rank_traced = smpi_process_index();
353 TRACE_smpi_computing_out(rank_traced);
355 TRACE_smpi_ptp_in(rank_traced, -1, -1, __FUNCTION__);
358 smpi_mpi_waitall(count_requests, requests, status);
361 for (i = 0; i < count_requests; i++) {
362 int src_traced, dst_traced, is_wait_for_receive;
363 xbt_dynar_get_cpy(srcs, i, &src_traced);
364 xbt_dynar_get_cpy(dsts, i, &dst_traced);
365 xbt_dynar_get_cpy(recvs, i, &is_wait_for_receive);
366 if (is_wait_for_receive) {
367 TRACE_smpi_recv(rank_traced, src_traced, dst_traced);
370 TRACE_smpi_ptp_out(rank_traced, -1, -1, __FUNCTION__);
372 xbt_dynar_free(&srcs);
373 xbt_dynar_free(&dsts);
374 xbt_dynar_free(&recvs);
375 TRACE_smpi_computing_in(rank_traced);
378 xbt_dynar_reset(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
380 log_timed_action (action, clock);
383 static void action_barrier(const char *const *action){
384 double clock = smpi_process_simulated_elapsed();
386 int rank = smpi_comm_rank(MPI_COMM_WORLD);
387 TRACE_smpi_computing_out(rank);
388 TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
390 smpi_mpi_barrier(MPI_COMM_WORLD);
392 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
393 TRACE_smpi_computing_in(rank);
396 log_timed_action (action, clock);
400 static void action_bcast(const char *const *action)
402 double size = parse_double(action[2]);
403 double clock = smpi_process_simulated_elapsed();
406 * Initialize MPI_CURRENT_TYPE in order to decrease
407 * the number of the checks
409 MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
412 root= atoi(action[3]);
414 MPI_CURRENT_TYPE=decode_datatype(action[4]);
419 int rank = smpi_comm_rank(MPI_COMM_WORLD);
420 TRACE_smpi_computing_out(rank);
421 int root_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), 0);
422 TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__);
425 smpi_mpi_bcast(NULL, size, MPI_CURRENT_TYPE, root, MPI_COMM_WORLD);
427 TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
428 TRACE_smpi_computing_in(rank);
431 log_timed_action (action, clock);
434 static void action_reduce(const char *const *action)
436 double comm_size = parse_double(action[2]);
437 double comp_size = parse_double(action[3]);
438 double clock = smpi_process_simulated_elapsed();
440 MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
443 root= atoi(action[4]);
445 MPI_CURRENT_TYPE=decode_datatype(action[5]);
450 int rank = smpi_comm_rank(MPI_COMM_WORLD);
451 TRACE_smpi_computing_out(rank);
452 int root_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), 0);
453 TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__);
455 mpi_coll_reduce_fun(NULL, NULL, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, root, MPI_COMM_WORLD);
456 smpi_execute_flops(comp_size);
458 TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
459 TRACE_smpi_computing_in(rank);
462 log_timed_action (action, clock);
465 static void action_allReduce(const char *const *action) {
466 double comm_size = parse_double(action[2]);
467 double comp_size = parse_double(action[3]);
469 if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
470 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
472 double clock = smpi_process_simulated_elapsed();
474 int rank = smpi_comm_rank(MPI_COMM_WORLD);
475 TRACE_smpi_computing_out(rank);
476 TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
478 mpi_coll_reduce_fun(NULL, NULL, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, 0, MPI_COMM_WORLD);
479 smpi_execute_flops(comp_size);
480 mpi_coll_bcast_fun(NULL, comm_size, MPI_CURRENT_TYPE, 0, MPI_COMM_WORLD);
482 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
483 TRACE_smpi_computing_in(rank);
486 log_timed_action (action, clock);
489 static void action_allToAll(const char *const *action) {
490 double clock = smpi_process_simulated_elapsed();
491 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
492 int send_size = parse_double(action[2]);
493 int recv_size = parse_double(action[3]);
494 void *send = xbt_new0(int, send_size*comm_size);
495 void *recv = xbt_new0(int, send_size*comm_size);
497 if(action[4]) MPI_CURRENT_TYPE=decode_datatype(action[4]);
498 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
501 int rank = smpi_process_index();
502 TRACE_smpi_computing_out(rank);
503 TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
506 mpi_coll_alltoall_fun(send, send_size, MPI_CURRENT_TYPE, recv, recv_size, MPI_CURRENT_TYPE, MPI_COMM_WORLD);
509 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
510 TRACE_smpi_computing_in(rank);
513 log_timed_action (action, clock);
518 static void action_allToAllv(const char *const *action) {
520 The structure of the allToAllV action for the rank 0 (total 4 processes)
522 0 allToAllV 100 1 7 10 12 5 10 20 45 100 1 70 10 5 1 5 77 90
525 1) 100 is the size of the send buffer *sizeof(int),
526 2) 1 7 10 12 is the sendcounts array
527 3) 5 10 20 45 is the sdispls array
528 4) 100*sizeof(int) is the size of the receiver buffer
529 5) 1 70 10 5 is the recvcounts array
530 6) 1 5 77 90 is the rdispls array
535 double clock = smpi_process_simulated_elapsed();
537 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
538 int send_buf_size=0,recv_buf_size=0,i=0;
539 int *sendcounts = xbt_new0(int, comm_size);
540 int *recvcounts = xbt_new0(int, comm_size);
541 int *senddisps = xbt_new0(int, comm_size);
542 int *recvdisps = xbt_new0(int, comm_size);
544 send_buf_size=parse_double(action[2]);
545 recv_buf_size=parse_double(action[3+2*comm_size]);
547 int *sendbuf = xbt_new0(int, send_buf_size);
548 int *recvbuf = xbt_new0(int, recv_buf_size);
550 if(action[4+4*comm_size]) MPI_CURRENT_TYPE=decode_datatype(action[4+4*comm_size]);
551 else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE;
553 for(i=0;i<comm_size;i++) {
554 sendcounts[i] = atoi(action[i+3]);
555 senddisps[i] = atoi(action[i+3+comm_size]);
556 recvcounts[i] = atoi(action[i+4+2*comm_size]);
557 recvdisps[i] = atoi(action[i+4+3*comm_size]);
562 int rank = MPI_COMM_WORLD != MPI_COMM_NULL ? smpi_process_index() : -1;
563 TRACE_smpi_computing_out(rank);
564 TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
566 mpi_coll_alltoallv_fun(sendbuf, sendcounts, senddisps, MPI_CURRENT_TYPE,
567 recvbuf, recvcounts, recvdisps, MPI_CURRENT_TYPE,
570 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
571 TRACE_smpi_computing_in(rank);
574 log_timed_action (action, clock);
577 xbt_free(sendcounts);
578 xbt_free(recvcounts);
585 void smpi_replay_init(int *argc, char***argv){
586 PMPI_Init(argc, argv);
587 if (!smpi_process_index()){
588 _xbt_replay_action_init();
589 xbt_replay_action_register("init", action_init);
590 xbt_replay_action_register("finalize", action_finalize);
591 xbt_replay_action_register("comm_size", action_comm_size);
592 xbt_replay_action_register("comm_split", action_comm_split);
593 xbt_replay_action_register("comm_dup", action_comm_dup);
594 xbt_replay_action_register("send", action_send);
595 xbt_replay_action_register("Isend", action_Isend);
596 xbt_replay_action_register("recv", action_recv);
597 xbt_replay_action_register("Irecv", action_Irecv);
598 xbt_replay_action_register("wait", action_wait);
599 xbt_replay_action_register("waitAll", action_waitall);
600 xbt_replay_action_register("barrier", action_barrier);
601 xbt_replay_action_register("bcast", action_bcast);
602 xbt_replay_action_register("reduce", action_reduce);
603 xbt_replay_action_register("allReduce", action_allReduce);
604 xbt_replay_action_register("allToAll", action_allToAll);
605 xbt_replay_action_register("allToAllV", action_allToAllv);
606 xbt_replay_action_register("compute", action_compute);
609 xbt_replay_action_runner(*argc, *argv);
612 int smpi_replay_finalize(){
614 /* One active process will stop. Decrease the counter*/
616 XBT_DEBUG("There are %lu elements in reqq[*]",
617 xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]));
618 xbt_dynar_free(&reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
619 if(!active_processes){
620 /* Last process alive speaking */
621 /* end the simulated timer */
622 sim_time = smpi_process_simulated_elapsed();
623 XBT_INFO("Simulation time %g", sim_time);
624 _xbt_replay_action_exit();
628 smpi_mpi_barrier(MPI_COMM_WORLD);
629 return PMPI_Finalize();