1 /* Copyright (c) 2009, 2010, 2011, 2012. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
11 #include <xbt/replay.h>
13 #define MPI_DTYPE MPI_BYTE
15 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_replay,smpi,"Trace Replay with SMPI");
17 int communicator_size = 0;
18 static int active_processes = 0;
21 static void log_timed_action (const char *const *action, double clock){
22 if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
23 char *name = xbt_str_join_array(action, " ");
24 XBT_VERB("%s %f", name, smpi_process_simulated_elapsed()-clock);
31 xbt_dynar_t isends; /* of MPI_Request */
32 xbt_dynar_t irecvs; /* of MPI_Request */
33 } s_smpi_replay_globals_t, *smpi_replay_globals_t;
37 static double parse_double(const char *string)
41 value = strtod(string, &endptr);
43 THROWF(unknown_error, 0, "%s is not a double", string);
47 static void action_init(const char *const *action)
50 XBT_DEBUG("Initialize the counters");
51 smpi_replay_globals_t globals = xbt_new(s_smpi_replay_globals_t, 1);
52 globals->isends = xbt_dynar_new(sizeof(MPI_Request),NULL);
53 globals->irecvs = xbt_dynar_new(sizeof(MPI_Request),NULL);
56 smpi_process_set_user_data((void*) globals);
58 /* start a simulated timer */
59 smpi_process_simulated_start();
60 /*initialize the number of active processes */
61 active_processes = smpi_process_count();
63 reqq=xbt_new0(xbt_dynar_t,active_processes);
65 for(i=0;i<active_processes;i++){
66 reqq[i]=xbt_dynar_new(sizeof(MPI_Request),NULL);
72 static void action_finalize(const char *const *action)
74 smpi_replay_globals_t globals =
75 (smpi_replay_globals_t) smpi_process_get_user_data();
78 XBT_DEBUG("There are %lu isends and %lu irecvs in the dynars",
79 xbt_dynar_length(globals->isends),xbt_dynar_length(globals->irecvs));
80 xbt_dynar_free_container(&(globals->isends));
81 xbt_dynar_free_container(&(globals->irecvs));
86 static void action_comm_size(const char *const *action)
88 double clock = smpi_process_simulated_elapsed();
90 communicator_size = parse_double(action[2]);
91 log_timed_action (action, clock);
94 static void action_comm_split(const char *const *action)
96 double clock = smpi_process_simulated_elapsed();
98 log_timed_action (action, clock);
101 static void action_comm_dup(const char *const *action)
103 double clock = smpi_process_simulated_elapsed();
105 log_timed_action (action, clock);
108 static void action_compute(const char *const *action)
110 double clock = smpi_process_simulated_elapsed();
111 smpi_execute_flops(parse_double(action[2]));
113 log_timed_action (action, clock);
116 static void action_send(const char *const *action)
118 int to = atoi(action[2]);
119 double size=parse_double(action[3]);
120 double clock = smpi_process_simulated_elapsed();
122 int rank = smpi_comm_rank(MPI_COMM_WORLD);
123 TRACE_smpi_computing_out(rank);
124 int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
125 TRACE_smpi_ptp_in(rank, rank, dst_traced, __FUNCTION__);
126 TRACE_smpi_send(rank, rank, dst_traced);
129 smpi_mpi_send(NULL, size, MPI_DTYPE, to , 0, MPI_COMM_WORLD);
131 log_timed_action (action, clock);
134 TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
135 TRACE_smpi_computing_in(rank);
140 static void action_Isend(const char *const *action)
142 int to = atoi(action[2]);
143 double size=parse_double(action[3]);
144 double clock = smpi_process_simulated_elapsed();
146 smpi_replay_globals_t globals =
147 (smpi_replay_globals_t) smpi_process_get_user_data();
149 int rank = smpi_comm_rank(MPI_COMM_WORLD);
150 TRACE_smpi_computing_out(rank);
151 int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to);
152 TRACE_smpi_ptp_in(rank, rank, dst_traced, __FUNCTION__);
153 TRACE_smpi_send(rank, rank, dst_traced);
156 request = smpi_mpi_isend(NULL, size, MPI_DTYPE, to, 0,MPI_COMM_WORLD);
159 TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
161 TRACE_smpi_computing_in(rank);
164 xbt_dynar_push(globals->isends,&request);
165 xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
167 log_timed_action (action, clock);
170 static void action_recv(const char *const *action) {
171 int from = atoi(action[2]);
172 double size=parse_double(action[3]);
173 double clock = smpi_process_simulated_elapsed();
176 int rank = smpi_comm_rank(MPI_COMM_WORLD);
177 int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
178 TRACE_smpi_computing_out(rank);
180 TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__);
183 smpi_mpi_recv(NULL, size, MPI_DTYPE, from, 0, MPI_COMM_WORLD, &status);
186 TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
187 TRACE_smpi_recv(rank, src_traced, rank);
188 TRACE_smpi_computing_in(rank);
191 log_timed_action (action, clock);
194 static void action_Irecv(const char *const *action)
196 int from = atoi(action[2]);
197 double size=parse_double(action[3]);
198 double clock = smpi_process_simulated_elapsed();
200 smpi_replay_globals_t globals =
201 (smpi_replay_globals_t) smpi_process_get_user_data();
204 int rank = smpi_comm_rank(MPI_COMM_WORLD);
205 int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from);
206 TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__);
209 request = smpi_mpi_irecv(NULL, size, MPI_DTYPE, from, 0, MPI_COMM_WORLD);
212 TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
215 xbt_dynar_push(globals->irecvs,&request);
216 xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
218 log_timed_action (action, clock);
221 static void action_wait(const char *const *action){
222 double clock = smpi_process_simulated_elapsed();
225 smpi_replay_globals_t globals =
226 (smpi_replay_globals_t) smpi_process_get_user_data();
228 xbt_assert(xbt_dynar_length(globals->irecvs),
229 "action wait not preceded by any irecv: %s",
230 xbt_str_join_array(action," "));
231 request = xbt_dynar_pop_as(globals->irecvs,MPI_Request);
233 int rank = request && request->comm != MPI_COMM_NULL
234 ? smpi_comm_rank(request->comm)
236 TRACE_smpi_computing_out(rank);
238 MPI_Group group = smpi_comm_group(request->comm);
239 int src_traced = smpi_group_rank(group, request->src);
240 int dst_traced = smpi_group_rank(group, request->dst);
241 int is_wait_for_receive = request->recv;
242 TRACE_smpi_ptp_in(rank, src_traced, dst_traced, __FUNCTION__);
244 smpi_mpi_wait(&request, &status);
246 TRACE_smpi_ptp_out(rank, src_traced, dst_traced, __FUNCTION__);
247 if (is_wait_for_receive) {
248 TRACE_smpi_recv(rank, src_traced, dst_traced);
250 TRACE_smpi_computing_in(rank);
253 log_timed_action (action, clock);
256 static void action_waitall(const char *const *action){
257 double clock = smpi_process_simulated_elapsed();
258 int count_requests=0;
261 count_requests=xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
263 if (count_requests>0) {
264 MPI_Request requests[count_requests];
265 MPI_Status status[count_requests];
267 /* The reqq is an array of dynars. Its index corresponds to the rank.
268 Thus each rank saves its own requests to the array request. */
269 xbt_dynar_foreach(reqq[smpi_comm_rank(MPI_COMM_WORLD)],i,requests[i]);
272 //save information from requests
274 xbt_dynar_t srcs = xbt_dynar_new(sizeof(int), NULL);
275 xbt_dynar_t dsts = xbt_dynar_new(sizeof(int), NULL);
276 xbt_dynar_t recvs = xbt_dynar_new(sizeof(int), NULL);
277 for (i = 0; i < count_requests; i++) {
279 int *asrc = xbt_new(int, 1);
280 int *adst = xbt_new(int, 1);
281 int *arecv = xbt_new(int, 1);
282 *asrc = requests[i]->src;
283 *adst = requests[i]->dst;
284 *arecv = requests[i]->recv;
285 xbt_dynar_insert_at(srcs, i, asrc);
286 xbt_dynar_insert_at(dsts, i, adst);
287 xbt_dynar_insert_at(recvs, i, arecv);
292 int *t = xbt_new(int, 1);
293 xbt_dynar_insert_at(srcs, i, t);
294 xbt_dynar_insert_at(dsts, i, t);
295 xbt_dynar_insert_at(recvs, i, t);
299 int rank_traced = smpi_process_index();
300 TRACE_smpi_computing_out(rank_traced);
302 TRACE_smpi_ptp_in(rank_traced, -1, -1, __FUNCTION__);
305 smpi_mpi_waitall(count_requests, requests, status);
308 for (i = 0; i < count_requests; i++) {
309 int src_traced, dst_traced, is_wait_for_receive;
310 xbt_dynar_get_cpy(srcs, i, &src_traced);
311 xbt_dynar_get_cpy(dsts, i, &dst_traced);
312 xbt_dynar_get_cpy(recvs, i, &is_wait_for_receive);
313 if (is_wait_for_receive) {
314 TRACE_smpi_recv(rank_traced, src_traced, dst_traced);
317 TRACE_smpi_ptp_out(rank_traced, -1, -1, __FUNCTION__);
319 xbt_dynar_free(&srcs);
320 xbt_dynar_free(&dsts);
321 xbt_dynar_free(&recvs);
322 TRACE_smpi_computing_in(rank_traced);
325 xbt_dynar_reset(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
327 log_timed_action (action, clock);
330 static void action_barrier(const char *const *action){
331 double clock = smpi_process_simulated_elapsed();
333 int rank = smpi_comm_rank(MPI_COMM_WORLD);
334 TRACE_smpi_computing_out(rank);
335 TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
337 smpi_mpi_barrier(MPI_COMM_WORLD);
339 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
340 TRACE_smpi_computing_in(rank);
343 log_timed_action (action, clock);
346 static void action_bcast(const char *const *action)
348 double size = parse_double(action[2]);
349 double clock = smpi_process_simulated_elapsed();
351 int rank = smpi_comm_rank(MPI_COMM_WORLD);
352 TRACE_smpi_computing_out(rank);
353 int root_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), 0);
354 TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__);
357 smpi_mpi_bcast(NULL, size, MPI_DTYPE, 0, MPI_COMM_WORLD);
359 TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
360 TRACE_smpi_computing_in(rank);
363 log_timed_action (action, clock);
366 static void action_reduce(const char *const *action)
368 double size = parse_double(action[2]);
369 double clock = smpi_process_simulated_elapsed();
371 int rank = smpi_comm_rank(MPI_COMM_WORLD);
372 TRACE_smpi_computing_out(rank);
373 int root_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), 0);
374 TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__);
376 smpi_mpi_reduce(NULL, NULL, size, MPI_DTYPE, MPI_OP_NULL, 0, MPI_COMM_WORLD);
378 TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
379 TRACE_smpi_computing_in(rank);
382 log_timed_action (action, clock);
385 static void action_allReduce(const char *const *action) {
386 double comm_size = parse_double(action[2]);
387 double comp_size = parse_double(action[3]);
388 double clock = smpi_process_simulated_elapsed();
390 int rank = smpi_comm_rank(MPI_COMM_WORLD);
391 TRACE_smpi_computing_out(rank);
392 TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
394 smpi_mpi_reduce(NULL, NULL, comm_size, MPI_DTYPE, MPI_OP_NULL, 0, MPI_COMM_WORLD);
395 smpi_execute_flops(comp_size);
396 smpi_mpi_bcast(NULL, comm_size, MPI_DTYPE, 0, MPI_COMM_WORLD);
398 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
399 TRACE_smpi_computing_in(rank);
402 log_timed_action (action, clock);
405 static void action_allToAll(const char *const *action) {
406 double clock = smpi_process_simulated_elapsed();
407 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
408 int send_size = atoi(action[2]);
409 int recv_size = atoi(action[3]);
410 void *send = xbt_new0(int, send_size*comm_size);
411 void *recv = xbt_new0(int, send_size*comm_size);
415 int rank = smpi_process_index();
416 TRACE_smpi_computing_out(rank);
417 TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
420 if (send_size < 200 && comm_size > 12) {
421 smpi_coll_tuned_alltoall_bruck(send, send_size, MPI_DTYPE,
422 recv, recv_size, MPI_DTYPE,
424 } else if (send_size < 3000) {
426 smpi_coll_tuned_alltoall_basic_linear(send, send_size, MPI_DTYPE,
427 recv, recv_size, MPI_DTYPE,
430 smpi_coll_tuned_alltoall_pairwise(send, send_size, MPI_DTYPE,
431 recv, recv_size, MPI_DTYPE,
436 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
437 TRACE_smpi_computing_in(rank);
440 log_timed_action (action, clock);
445 static void action_allToAllv(const char *const *action) {
447 The structure of the allToAllV action for the rank 0 (total 4 processes)
449 0 allToAllV 100 1 7 10 12 5 10 20 45 100 1 70 10 5 1 5 77 90
452 1) 100 is the size of the send buffer *sizeof(int),
453 2) 1 7 10 12 is the sendcounts array
454 3) 5 10 20 45 is the sdispls array
455 4) 100*sizeof(int) is the size of the receiver buffer
456 5) 1 70 10 5 is the recvcounts array
457 6) 1 5 77 90 is the rdispls array
462 double clock = smpi_process_simulated_elapsed();
464 int comm_size = smpi_comm_size(MPI_COMM_WORLD);
465 int send_buf_size=0,recv_buf_size=0,i=0;
466 int *sendcounts = xbt_new0(int, comm_size);
467 int *recvcounts = xbt_new0(int, comm_size);
468 int *senddisps = xbt_new0(int, comm_size);
469 int *recvdisps = xbt_new0(int, comm_size);
470 MPI_Datatype sendtype,recvtype;
472 send_buf_size=atoi(action[2]);
473 recv_buf_size=atoi(action[3+2*comm_size]);
475 int *sendbuf = xbt_new0(int, send_buf_size);
476 int *recvbuf = xbt_new0(int, recv_buf_size);
481 for(i=0;i<comm_size;i++) {
482 sendcounts[i] = atoi(action[i+3]);
483 senddisps[i] = atoi(action[i+3+comm_size]);
484 recvcounts[i] = atoi(action[i+4+2*comm_size]);
485 recvdisps[i] = atoi(action[i+4+3*comm_size]);
490 int rank = MPI_COMM_WORLD != MPI_COMM_NULL ? smpi_process_index() : -1;
491 TRACE_smpi_computing_out(rank);
492 TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
494 smpi_coll_basic_alltoallv(sendbuf, sendcounts, senddisps, sendtype,
495 recvbuf, recvcounts, recvdisps, recvtype,
498 TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
499 TRACE_smpi_computing_in(rank);
502 log_timed_action (action, clock);
505 xbt_free(sendcounts);
506 xbt_free(recvcounts);
513 void smpi_replay_init(int *argc, char***argv){
514 PMPI_Init(argc, argv);
515 if (!smpi_process_index()){
516 _xbt_replay_action_init();
517 xbt_replay_action_register("init", action_init);
518 xbt_replay_action_register("finalize", action_finalize);
519 xbt_replay_action_register("comm_size", action_comm_size);
520 xbt_replay_action_register("comm_split", action_comm_split);
521 xbt_replay_action_register("comm_dup", action_comm_dup);
522 xbt_replay_action_register("send", action_send);
523 xbt_replay_action_register("Isend", action_Isend);
524 xbt_replay_action_register("recv", action_recv);
525 xbt_replay_action_register("Irecv", action_Irecv);
526 xbt_replay_action_register("wait", action_wait);
527 xbt_replay_action_register("waitAll", action_waitall);
528 xbt_replay_action_register("barrier", action_barrier);
529 xbt_replay_action_register("bcast", action_bcast);
530 xbt_replay_action_register("reduce", action_reduce);
531 xbt_replay_action_register("allReduce", action_allReduce);
532 xbt_replay_action_register("allToAll", action_allToAll);
533 xbt_replay_action_register("allToAllV", action_allToAllv);
534 xbt_replay_action_register("compute", action_compute);
537 xbt_replay_action_runner(*argc, *argv);
540 int smpi_replay_finalize(){
542 /* One active process will stop. Decrease the counter*/
544 if(!active_processes){
545 /* Last process alive speaking */
546 /* end the simulated timer */
547 xbt_dynar_free(reqq);
548 sim_time = smpi_process_simulated_elapsed();
549 XBT_INFO("Simulation time %g", sim_time);
550 _xbt_replay_action_exit();
552 return PMPI_Finalize();