#include <xbt.h>
#include <xbt/replay.h>
+#define MPI_DTYPE MPI_BYTE
+
XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_replay,smpi,"Trace Replay with SMPI");
int communicator_size = 0;
static int active_processes = 0;
+xbt_dynar_t *reqq;
static void log_timed_action (const char *const *action, double clock){
if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){
}
}
+
typedef struct {
xbt_dynar_t isends; /* of MPI_Request */
xbt_dynar_t irecvs; /* of MPI_Request */
} s_smpi_replay_globals_t, *smpi_replay_globals_t;
+
/* Helper function */
static double parse_double(const char *string)
{
static void action_init(const char *const *action)
{
+ int i;
XBT_DEBUG("Initialize the counters");
smpi_replay_globals_t globals = xbt_new(s_smpi_replay_globals_t, 1);
globals->isends = xbt_dynar_new(sizeof(MPI_Request),NULL);
globals->irecvs = xbt_dynar_new(sizeof(MPI_Request),NULL);
-
+
+
smpi_process_set_user_data((void*) globals);
/* start a simulated timer */
smpi_process_simulated_start();
/*initialize the number of active processes */
active_processes = smpi_process_count();
+
+ reqq=xbt_new0(xbt_dynar_t,active_processes);
+
+ for(i=0;i<active_processes;i++){
+ reqq[i]=xbt_dynar_new(sizeof(MPI_Request),NULL);
+ }
+
+
}
static void action_finalize(const char *const *action)
TRACE_smpi_send(rank, rank, dst_traced);
#endif
- smpi_mpi_send(NULL, size, MPI_BYTE, to , 0, MPI_COMM_WORLD);
+ smpi_mpi_send(NULL, size, MPI_DTYPE, to , 0, MPI_COMM_WORLD);
log_timed_action (action, clock);
int to = atoi(action[2]);
double size=parse_double(action[3]);
double clock = smpi_process_simulated_elapsed();
+ MPI_Request request;
smpi_replay_globals_t globals =
(smpi_replay_globals_t) smpi_process_get_user_data();
#ifdef HAVE_TRACING
TRACE_smpi_send(rank, rank, dst_traced);
#endif
- MPI_Request request = smpi_mpi_isend(NULL, size, MPI_BYTE, to, 0,
- MPI_COMM_WORLD);
+ request = smpi_mpi_isend(NULL, size, MPI_DTYPE, to, 0,MPI_COMM_WORLD);
+
#ifdef HAVE_TRACING
TRACE_smpi_ptp_out(rank, rank, dst_traced, __FUNCTION__);
request->send = 1;
#endif
xbt_dynar_push(globals->isends,&request);
+ xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
log_timed_action (action, clock);
}
TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__);
#endif
- smpi_mpi_recv(NULL, size, MPI_BYTE, from, 0, MPI_COMM_WORLD, &status);
+ smpi_mpi_recv(NULL, size, MPI_DTYPE, from, 0, MPI_COMM_WORLD, &status);
#ifdef HAVE_TRACING
TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
TRACE_smpi_ptp_in(rank, src_traced, rank, __FUNCTION__);
#endif
- request = smpi_mpi_irecv(NULL, size, MPI_BYTE, from, 0, MPI_COMM_WORLD);
+ request = smpi_mpi_irecv(NULL, size, MPI_DTYPE, from, 0, MPI_COMM_WORLD);
+
#ifdef HAVE_TRACING
TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__);
request->recv = 1;
#endif
xbt_dynar_push(globals->irecvs,&request);
+ xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request);
log_timed_action (action, clock);
}
static void action_waitall(const char *const *action){
double clock = smpi_process_simulated_elapsed();
-// smpi_mpi_waitall(count, requests, status);
+ int count_requests=0;
+ unsigned int i=0;
+
+ count_requests=xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
+
+ if (count_requests>0) {
+ MPI_Request requests[count_requests];
+ MPI_Status status[count_requests];
+
+ /* The reqq is an array of dynars. Its index corresponds to the rank.
+ Thus each rank saves its own requests to the array request. */
+ xbt_dynar_foreach(reqq[smpi_comm_rank(MPI_COMM_WORLD)],i,requests[i]);
+
+ #ifdef HAVE_TRACING
+ //save information from requests
+
+ xbt_dynar_t srcs = xbt_dynar_new(sizeof(int), NULL);
+ xbt_dynar_t dsts = xbt_dynar_new(sizeof(int), NULL);
+ xbt_dynar_t recvs = xbt_dynar_new(sizeof(int), NULL);
+ for (i = 0; i < count_requests; i++) {
+ if(requests[i]){
+ int *asrc = xbt_new(int, 1);
+ int *adst = xbt_new(int, 1);
+ int *arecv = xbt_new(int, 1);
+ *asrc = requests[i]->src;
+ *adst = requests[i]->dst;
+ *arecv = requests[i]->recv;
+ xbt_dynar_insert_at(srcs, i, asrc);
+ xbt_dynar_insert_at(dsts, i, adst);
+ xbt_dynar_insert_at(recvs, i, arecv);
+ xbt_free(asrc);
+ xbt_free(adst);
+ xbt_free(arecv);
+ }else {
+ int *t = xbt_new(int, 1);
+ xbt_dynar_insert_at(srcs, i, t);
+ xbt_dynar_insert_at(dsts, i, t);
+ xbt_dynar_insert_at(recvs, i, t);
+ xbt_free(t);
+ }
+ }
+ int rank_traced = smpi_process_index();
+ TRACE_smpi_computing_out(rank_traced);
+
+ TRACE_smpi_ptp_in(rank_traced, -1, -1, __FUNCTION__);
+ #endif
+
+ smpi_mpi_waitall(count_requests, requests, status);
+
+ #ifdef HAVE_TRACING
+ for (i = 0; i < count_requests; i++) {
+ int src_traced, dst_traced, is_wait_for_receive;
+ xbt_dynar_get_cpy(srcs, i, &src_traced);
+ xbt_dynar_get_cpy(dsts, i, &dst_traced);
+ xbt_dynar_get_cpy(recvs, i, &is_wait_for_receive);
+ if (is_wait_for_receive) {
+ TRACE_smpi_recv(rank_traced, src_traced, dst_traced);
+ }
+ }
+ TRACE_smpi_ptp_out(rank_traced, -1, -1, __FUNCTION__);
+ //clean-up of dynars
+ xbt_dynar_free(&srcs);
+ xbt_dynar_free(&dsts);
+ xbt_dynar_free(&recvs);
+ TRACE_smpi_computing_in(rank_traced);
+ #endif
+
+ xbt_dynar_reset(reqq[smpi_comm_rank(MPI_COMM_WORLD)]);
+ }
log_timed_action (action, clock);
}
TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__);
#endif
- smpi_mpi_bcast(NULL, size, MPI_BYTE, 0, MPI_COMM_WORLD);
+ smpi_mpi_bcast(NULL, size, MPI_DTYPE, 0, MPI_COMM_WORLD);
#ifdef HAVE_TRACING
TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
TRACE_smpi_computing_in(rank);
int root_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), 0);
TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__);
#endif
- smpi_mpi_reduce(NULL, NULL, size, MPI_BYTE, MPI_OP_NULL, 0, MPI_COMM_WORLD);
+ smpi_mpi_reduce(NULL, NULL, size, MPI_DTYPE, MPI_OP_NULL, 0, MPI_COMM_WORLD);
#ifdef HAVE_TRACING
TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__);
TRACE_smpi_computing_in(rank);
TRACE_smpi_computing_out(rank);
TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
#endif
- smpi_mpi_reduce(NULL, NULL, comm_size, MPI_BYTE, MPI_OP_NULL, 0, MPI_COMM_WORLD);
+ smpi_mpi_reduce(NULL, NULL, comm_size, MPI_DTYPE, MPI_OP_NULL, 0, MPI_COMM_WORLD);
smpi_execute_flops(comp_size);
- smpi_mpi_bcast(NULL, comm_size, MPI_BYTE, 0, MPI_COMM_WORLD);
+ smpi_mpi_bcast(NULL, comm_size, MPI_DTYPE, 0, MPI_COMM_WORLD);
#ifdef HAVE_TRACING
TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
TRACE_smpi_computing_in(rank);
static void action_allToAll(const char *const *action) {
double clock = smpi_process_simulated_elapsed();
- double comm_size = smpi_comm_size(MPI_COMM_WORLD);
- double send_size = parse_double(action[2]);
- double recv_size = parse_double(action[3]);
+ int comm_size = smpi_comm_size(MPI_COMM_WORLD);
+ int send_size = atoi(action[2]);
+ int recv_size = atoi(action[3]);
+ void *send = xbt_new0(int, send_size*comm_size);
+ void *recv = xbt_new0(int, send_size*comm_size);
+
#ifdef HAVE_TRACING
int rank = smpi_process_index();
#endif
if (send_size < 200 && comm_size > 12) {
- smpi_coll_tuned_alltoall_bruck(NULL, send_size, MPI_BYTE,
- NULL, recv_size, MPI_BYTE,
+ smpi_coll_tuned_alltoall_bruck(send, send_size, MPI_DTYPE,
+ recv, recv_size, MPI_DTYPE,
MPI_COMM_WORLD);
} else if (send_size < 3000) {
- smpi_coll_tuned_alltoall_basic_linear(NULL, send_size, MPI_BYTE,
- NULL, recv_size, MPI_BYTE,
+
+ smpi_coll_tuned_alltoall_basic_linear(send, send_size, MPI_DTYPE,
+ recv, recv_size, MPI_DTYPE,
MPI_COMM_WORLD);
} else {
- smpi_coll_tuned_alltoall_pairwise(NULL, send_size, MPI_BYTE,
- NULL, recv_size, MPI_BYTE,
+ smpi_coll_tuned_alltoall_pairwise(send, send_size, MPI_DTYPE,
+ recv, recv_size, MPI_DTYPE,
MPI_COMM_WORLD);
}
#endif
log_timed_action (action, clock);
+ xbt_free(send);
+ xbt_free(recv);
}
static void action_allToAllv(const char *const *action) {
+ /*
+ The structure of the allToAllV action for the rank 0 (total 4 processes)
+ is the following:
+ 0 allToAllV 100 1 7 10 12 5 10 20 45 100 1 70 10 5 1 5 77 90
+
+ where:
+ 1) 100 is the size of the send buffer *sizeof(int),
+ 2) 1 7 10 12 is the sendcounts array
+ 3) 5 10 20 45 is the sdispls array
+ 4) 100*sizeof(int) is the size of the receiver buffer
+ 5) 1 70 10 5 is the recvcounts array
+ 6) 1 5 77 90 is the rdispls array
+
+ */
+
+
double clock = smpi_process_simulated_elapsed();
+
+ int comm_size = smpi_comm_size(MPI_COMM_WORLD);
+ int send_buf_size=0,recv_buf_size=0,i=0;
+ int *sendcounts = xbt_new0(int, comm_size);
+ int *recvcounts = xbt_new0(int, comm_size);
+ int *senddisps = xbt_new0(int, comm_size);
+ int *recvdisps = xbt_new0(int, comm_size);
+ MPI_Datatype sendtype,recvtype;
+
+ send_buf_size=atoi(action[2]);
+ recv_buf_size=atoi(action[3+2*comm_size]);
+
+ int *sendbuf = xbt_new0(int, send_buf_size);
+ int *recvbuf = xbt_new0(int, recv_buf_size);
+
+ sendtype=MPI_DTYPE;
+ recvtype=MPI_DTYPE;
+
+ for(i=0;i<comm_size;i++) {
+ sendcounts[i] = atoi(action[i+3]);
+ senddisps[i] = atoi(action[i+3+comm_size]);
+ recvcounts[i] = atoi(action[i+4+2*comm_size]);
+ recvdisps[i] = atoi(action[i+4+3*comm_size]);
+ }
+
+#ifdef HAVE_TRACING
+ int rank = MPI_COMM_WORLD != MPI_COMM_NULL ? smpi_process_index() : -1;
+ TRACE_smpi_computing_out(rank);
+ TRACE_smpi_collective_in(rank, -1, __FUNCTION__);
+#endif
+ smpi_coll_basic_alltoallv(sendbuf, sendcounts, senddisps, sendtype,
+ recvbuf, recvcounts, recvdisps, recvtype,
+ MPI_COMM_WORLD);
+#ifdef HAVE_TRACING
+ TRACE_smpi_collective_out(rank, -1, __FUNCTION__);
+ TRACE_smpi_computing_in(rank);
+#endif
+
log_timed_action (action, clock);
+ xbt_free(sendbuf);
+ xbt_free(recvbuf);
+ xbt_free(sendcounts);
+ xbt_free(recvcounts);
+ xbt_free(senddisps);
+ xbt_free(recvdisps);
+
+
}
void smpi_replay_init(int *argc, char***argv){
double sim_time= 1.;
/* One active process will stop. Decrease the counter*/
active_processes--;
-
if(!active_processes){
/* Last process alive speaking */
/* end the simulated timer */
+ xbt_dynar_free(reqq);
sim_time = smpi_process_simulated_elapsed();
XBT_INFO("Simulation time %g", sim_time);
_xbt_replay_action_exit();
