X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/6de03ecc4e630732984a0673512a5d15fd75e270..a6b23c846948f7f89277f75e7c42f3942b31b8d3:/src/smpi/smpi_replay.c diff --git a/src/smpi/smpi_replay.c b/src/smpi/smpi_replay.c index 9e00ae97bd..2f6bebbfb5 100644 --- a/src/smpi/smpi_replay.c +++ b/src/smpi/smpi_replay.c @@ -18,6 +18,11 @@ xbt_dynar_t *reqq = NULL; MPI_Datatype MPI_DEFAULT_TYPE; MPI_Datatype MPI_CURRENT_TYPE; +static int sendbuffer_size=0; +char* sendbuffer=NULL; +static int recvbuffer_size=0; +char* recvbuffer=NULL; + static void log_timed_action (const char *const *action, double clock){ if (XBT_LOG_ISENABLED(smpi_replay, xbt_log_priority_verbose)){ char *name = xbt_str_join_array(action, " "); @@ -26,6 +31,32 @@ static void log_timed_action (const char *const *action, double clock){ } } +//allocate a single buffer for all sends, growing it if needed +void* smpi_get_tmp_sendbuffer(int size){ + if (!smpi_process_get_replaying()) + return xbt_malloc(size); + if (sendbuffer_sizetype=TRACING_COMPUTING; extra->comp_size=flops; @@ -178,7 +209,7 @@ static void action_send(const char *const *action) } #ifdef HAVE_TRACING - int rank = smpi_comm_rank(MPI_COMM_WORLD); + int rank = smpi_process_index(); int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); @@ -212,7 +243,7 @@ static void action_Isend(const char *const *action) else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE; #ifdef HAVE_TRACING - int rank = smpi_comm_rank(MPI_COMM_WORLD); + int rank = smpi_process_index(); int dst_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), to); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); extra->type = TRACING_ISEND; @@ -231,7 +262,7 @@ static void action_Isend(const char *const *action) request->send = 1; #endif - xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request); + xbt_dynar_push(reqq[smpi_process_index()],&request); log_timed_action (action, clock); } @@ -246,7 +277,7 @@ static void action_recv(const char *const *action) { else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE; #ifdef HAVE_TRACING - int rank = smpi_comm_rank(MPI_COMM_WORLD); + int rank = smpi_process_index(); int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); @@ -285,7 +316,7 @@ static void action_Irecv(const char *const *action) else MPI_CURRENT_TYPE= MPI_DEFAULT_TYPE; #ifdef HAVE_TRACING - int rank = smpi_comm_rank(MPI_COMM_WORLD); + int rank = smpi_process_index(); int src_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), from); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); extra->type = TRACING_IRECV; @@ -308,7 +339,36 @@ static void action_Irecv(const char *const *action) TRACE_smpi_ptp_out(rank, src_traced, rank, __FUNCTION__); request->recv = 1; #endif - xbt_dynar_push(reqq[smpi_comm_rank(MPI_COMM_WORLD)],&request); + xbt_dynar_push(reqq[smpi_process_index()],&request); + + log_timed_action (action, clock); +} + +static void action_test(const char *const *action){ + double clock = smpi_process_simulated_elapsed(); + MPI_Request request; + MPI_Status status; + int flag = TRUE; + + request = xbt_dynar_pop_as(reqq[smpi_process_index()],MPI_Request); + xbt_assert(request != NULL, "found null request in reqq"); + +#ifdef HAVE_TRACING + int rank = smpi_process_index(); + instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); + extra->type=TRACING_TEST; + TRACE_smpi_testing_in(rank, extra); +#endif + flag = smpi_mpi_test(&request, &status); + XBT_DEBUG("MPI_Test result: %d", flag); + /* push back request in dynar to be caught by a subsequent wait. if the test + * did succeed, the request is now NULL. + */ + xbt_dynar_push_as(reqq[smpi_process_index()],MPI_Request, request); + +#ifdef HAVE_TRACING + TRACE_smpi_testing_out(rank); +#endif log_timed_action (action, clock); } @@ -318,11 +378,18 @@ static void action_wait(const char *const *action){ MPI_Request request; MPI_Status status; - xbt_assert(xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]), + xbt_assert(xbt_dynar_length(reqq[smpi_process_index()]), "action wait not preceded by any irecv or isend: %s", xbt_str_join_array(action," ")); - request = xbt_dynar_pop_as(reqq[smpi_comm_rank(MPI_COMM_WORLD)],MPI_Request); - xbt_assert(request != NULL, "found null request in reqq"); + request = xbt_dynar_pop_as(reqq[smpi_process_index()],MPI_Request); + + if (!request){ + /* Assuming that the trace is well formed, this mean the comm might have + * been caught by a MPI_test. Then just return. + */ + return; + } + #ifdef HAVE_TRACING int rank = request->comm != MPI_COMM_NULL ? smpi_comm_rank(request->comm) @@ -352,7 +419,7 @@ static void action_waitall(const char *const *action){ int count_requests=0; unsigned int i=0; - count_requests=xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]); + count_requests=xbt_dynar_length(reqq[smpi_process_index()]); if (count_requests>0) { MPI_Request requests[count_requests]; @@ -360,7 +427,7 @@ static void action_waitall(const char *const *action){ /* The reqq is an array of dynars. Its index corresponds to the rank. Thus each rank saves its own requests to the array request. */ - xbt_dynar_foreach(reqq[smpi_comm_rank(MPI_COMM_WORLD)],i,requests[i]); + xbt_dynar_foreach(reqq[smpi_process_index()],i,requests[i]); #ifdef HAVE_TRACING //save information from requests @@ -416,7 +483,7 @@ static void action_waitall(const char *const *action){ xbt_dynar_free(&recvs); #endif - xbt_dynar_free_container(&(reqq[smpi_comm_rank(MPI_COMM_WORLD)])); + xbt_dynar_free_container(&(reqq[smpi_process_index()])); } log_timed_action (action, clock); } @@ -424,12 +491,12 @@ static void action_waitall(const char *const *action){ static void action_barrier(const char *const *action){ double clock = smpi_process_simulated_elapsed(); #ifdef HAVE_TRACING - int rank = smpi_comm_rank(MPI_COMM_WORLD); + int rank = smpi_process_index(); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); extra->type = TRACING_BARRIER; TRACE_smpi_collective_in(rank, -1, __FUNCTION__, extra); #endif - smpi_mpi_barrier(MPI_COMM_WORLD); + mpi_coll_barrier_fun(MPI_COMM_WORLD); #ifdef HAVE_TRACING TRACE_smpi_collective_out(rank, -1, __FUNCTION__); #endif @@ -457,7 +524,7 @@ static void action_bcast(const char *const *action) } #ifdef HAVE_TRACING - int rank = smpi_comm_rank(MPI_COMM_WORLD); + int rank = smpi_process_index(); int root_traced = smpi_group_index(smpi_comm_group(MPI_COMM_WORLD), root); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); @@ -468,12 +535,11 @@ static void action_bcast(const char *const *action) TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__, extra); #endif - - mpi_coll_bcast_fun(NULL, size, MPI_CURRENT_TYPE, root, MPI_COMM_WORLD); + void *sendbuf = smpi_get_tmp_sendbuffer(size* smpi_datatype_size(MPI_CURRENT_TYPE)); + mpi_coll_bcast_fun(sendbuf, size, MPI_CURRENT_TYPE, root, MPI_COMM_WORLD); #ifdef HAVE_TRACING TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__); #endif - log_timed_action (action, clock); } @@ -491,9 +557,11 @@ static void action_reduce(const char *const *action) MPI_CURRENT_TYPE=decode_datatype(action[5]); } } + + #ifdef HAVE_TRACING - int rank = smpi_comm_rank(MPI_COMM_WORLD); + int rank = smpi_process_index(); int root_traced = smpi_group_rank(smpi_comm_group(MPI_COMM_WORLD), root); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); extra->type = TRACING_REDUCE; @@ -504,12 +572,13 @@ static void action_reduce(const char *const *action) TRACE_smpi_collective_in(rank, root_traced, __FUNCTION__,extra); #endif - mpi_coll_reduce_fun(NULL, NULL, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, root, MPI_COMM_WORLD); + void *recvbuf = smpi_get_tmp_sendbuffer(comm_size* smpi_datatype_size(MPI_CURRENT_TYPE)); + void *sendbuf = smpi_get_tmp_sendbuffer(comm_size* smpi_datatype_size(MPI_CURRENT_TYPE)); + mpi_coll_reduce_fun(sendbuf, recvbuf, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, root, MPI_COMM_WORLD); smpi_execute_flops(comp_size); #ifdef HAVE_TRACING TRACE_smpi_collective_out(rank, root_traced, __FUNCTION__); #endif - log_timed_action (action, clock); } @@ -522,7 +591,7 @@ static void action_allReduce(const char *const *action) { double clock = smpi_process_simulated_elapsed(); #ifdef HAVE_TRACING - int rank = smpi_comm_rank(MPI_COMM_WORLD); + int rank = smpi_process_index(); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); extra->type = TRACING_ALLREDUCE; extra->send_size = comm_size; @@ -531,13 +600,13 @@ static void action_allReduce(const char *const *action) { TRACE_smpi_collective_in(rank, -1, __FUNCTION__,extra); #endif - mpi_coll_reduce_fun(NULL, NULL, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, 0, MPI_COMM_WORLD); + void *recvbuf = smpi_get_tmp_sendbuffer(comm_size* smpi_datatype_size(MPI_CURRENT_TYPE)); + void *sendbuf = smpi_get_tmp_sendbuffer(comm_size* smpi_datatype_size(MPI_CURRENT_TYPE)); + mpi_coll_allreduce_fun(sendbuf, recvbuf, comm_size, MPI_CURRENT_TYPE, MPI_OP_NULL, MPI_COMM_WORLD); smpi_execute_flops(comp_size); - mpi_coll_bcast_fun(NULL, comm_size, MPI_CURRENT_TYPE, 0, MPI_COMM_WORLD); #ifdef HAVE_TRACING TRACE_smpi_collective_out(rank, -1, __FUNCTION__); #endif - log_timed_action (action, clock); } @@ -556,8 +625,8 @@ static void action_allToAll(const char *const *action) { MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE; MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE; } - void *send = calloc(send_size*comm_size, smpi_datatype_size(MPI_CURRENT_TYPE)); - void *recv = calloc(recv_size*comm_size, smpi_datatype_size(MPI_CURRENT_TYPE2)); + void *send = smpi_get_tmp_sendbuffer(send_size*comm_size* smpi_datatype_size(MPI_CURRENT_TYPE)); + void *recv = smpi_get_tmp_recvbuffer(recv_size*comm_size* smpi_datatype_size(MPI_CURRENT_TYPE2)); #ifdef HAVE_TRACING int rank = smpi_process_index(); @@ -576,10 +645,8 @@ static void action_allToAll(const char *const *action) { #ifdef HAVE_TRACING TRACE_smpi_collective_out(rank, -1, __FUNCTION__); #endif - log_timed_action (action, clock); - xbt_free(send); - xbt_free(recv); + } @@ -608,14 +675,14 @@ static void action_gather(const char *const *action) { MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE; MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE; } - void *send = calloc(send_size, smpi_datatype_size(MPI_CURRENT_TYPE)); + void *send = smpi_get_tmp_sendbuffer(send_size* smpi_datatype_size(MPI_CURRENT_TYPE)); void *recv = NULL; int root=atoi(action[4]); - int rank = smpi_process_index(); + int rank = smpi_comm_rank(MPI_COMM_WORLD); if(rank==root) - recv = calloc(recv_size*comm_size, smpi_datatype_size(MPI_CURRENT_TYPE2)); + recv = smpi_get_tmp_recvbuffer(recv_size*comm_size* smpi_datatype_size(MPI_CURRENT_TYPE2)); #ifdef HAVE_TRACING instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1); @@ -626,19 +693,17 @@ static void action_gather(const char *const *action) { extra->datatype1 = encode_datatype(MPI_CURRENT_TYPE); extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2); - TRACE_smpi_collective_in(rank, root, __FUNCTION__, extra); + TRACE_smpi_collective_in(smpi_process_index(), root, __FUNCTION__, extra); #endif -smpi_mpi_gather(send, send_size, MPI_CURRENT_TYPE, + mpi_coll_gather_fun(send, send_size, MPI_CURRENT_TYPE, recv, recv_size, MPI_CURRENT_TYPE2, root, MPI_COMM_WORLD); #ifdef HAVE_TRACING - TRACE_smpi_collective_out(rank, -1, __FUNCTION__); + TRACE_smpi_collective_out(smpi_process_index(), -1, __FUNCTION__); #endif - log_timed_action (action, clock); - xbt_free(send); - xbt_free(recv); + } @@ -671,7 +736,7 @@ static void action_gatherv(const char *const *action) { MPI_CURRENT_TYPE=MPI_DEFAULT_TYPE; MPI_CURRENT_TYPE2=MPI_DEFAULT_TYPE; } - void *send = calloc(send_size, smpi_datatype_size(MPI_CURRENT_TYPE)); + void *send = smpi_get_tmp_sendbuffer(send_size* smpi_datatype_size(MPI_CURRENT_TYPE)); void *recv = NULL; for(i=0;idatatype1 = encode_datatype(MPI_CURRENT_TYPE); extra->datatype2 = encode_datatype(MPI_CURRENT_TYPE2); - TRACE_smpi_collective_in(rank, root, __FUNCTION__, extra); + TRACE_smpi_collective_in(smpi_process_index(), root, __FUNCTION__, extra); #endif smpi_mpi_gatherv(send, send_size, MPI_CURRENT_TYPE, recv, recvcounts, disps, MPI_CURRENT_TYPE2, root, MPI_COMM_WORLD); #ifdef HAVE_TRACING - TRACE_smpi_collective_out(rank, -1, __FUNCTION__); + TRACE_smpi_collective_out(smpi_process_index(), -1, __FUNCTION__); #endif log_timed_action (action, clock); xbt_free(recvcounts); - xbt_free(send); - xbt_free(recv); xbt_free(disps); - } static void action_reducescatter(const char *const *action) { @@ -736,10 +798,9 @@ static void action_reducescatter(const char *const *action) { int comp_size = parse_double(action[2+comm_size]); int *recvcounts = xbt_new0(int, comm_size); int *disps = xbt_new0(int, comm_size); - int i=0,recv_sum=0; - int root=0; + int i=0; int rank = smpi_process_index(); - + int size = 0; if(action[3+comm_size]) MPI_CURRENT_TYPE=decode_datatype(action[3+comm_size]); else @@ -747,8 +808,8 @@ static void action_reducescatter(const char *const *action) { for(i=0;icomp_size = comp_size; extra->num_processes = comm_size; - TRACE_smpi_collective_in(rank, -1, __FUNCTION__,extra); #endif - mpi_coll_reduce_fun(NULL, NULL, recv_sum, MPI_CURRENT_TYPE, MPI_OP_NULL, - root, MPI_COMM_WORLD); - smpi_mpi_scatterv(NULL, recvcounts, disps, MPI_CURRENT_TYPE, NULL, - recvcounts[rank], MPI_CURRENT_TYPE, 0, MPI_COMM_WORLD); + void *sendbuf = smpi_get_tmp_sendbuffer(size* smpi_datatype_size(MPI_CURRENT_TYPE)); + void *recvbuf = smpi_get_tmp_recvbuffer(size* smpi_datatype_size(MPI_CURRENT_TYPE)); + + mpi_coll_reduce_scatter_fun(sendbuf, recvbuf, recvcounts, MPI_CURRENT_TYPE, MPI_OP_NULL, + MPI_COMM_WORLD); smpi_execute_flops(comp_size); @@ -813,13 +874,13 @@ static void action_allgatherv(const char *const *action) { MPI_CURRENT_TYPE = MPI_DEFAULT_TYPE; MPI_CURRENT_TYPE2 = MPI_DEFAULT_TYPE; } - void *sendbuf = calloc(sendcount, smpi_datatype_size(MPI_CURRENT_TYPE)); + void *sendbuf = smpi_get_tmp_sendbuffer(sendcount* smpi_datatype_size(MPI_CURRENT_TYPE)); for(i=0;i2){ + char *endptr; + double value = strtod((*argv)[2], &endptr); + if (*endptr != '\0') + THROWF(unknown_error, 0, "%s is not a double", (*argv)[2]); + XBT_VERB("Delayed start for instance - Sleeping for %f flops ",value ); + smpi_execute_flops(value); + } xbt_replay_action_runner(*argc, *argv); } @@ -977,21 +1045,21 @@ int smpi_replay_finalize(){ double sim_time= 1.; /* One active process will stop. Decrease the counter*/ XBT_DEBUG("There are %lu elements in reqq[*]", - xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)])); - if (!xbt_dynar_is_empty(reqq[smpi_comm_rank(MPI_COMM_WORLD)])){ - int count_requests=xbt_dynar_length(reqq[smpi_comm_rank(MPI_COMM_WORLD)]); + xbt_dynar_length(reqq[smpi_process_index()])); + if (!xbt_dynar_is_empty(reqq[smpi_process_index()])){ + int count_requests=xbt_dynar_length(reqq[smpi_process_index()]); MPI_Request requests[count_requests]; MPI_Status status[count_requests]; unsigned int i; - xbt_dynar_foreach(reqq[smpi_comm_rank(MPI_COMM_WORLD)],i,requests[i]); + xbt_dynar_foreach(reqq[smpi_process_index()],i,requests[i]); smpi_mpi_waitall(count_requests, requests, status); active_processes--; } else { active_processes--; } - xbt_dynar_free_container(&(reqq[smpi_comm_rank(MPI_COMM_WORLD)])); + xbt_dynar_free_container(&(reqq[smpi_process_index()])); if(!active_processes){ /* Last process alive speaking */ @@ -999,10 +1067,12 @@ int smpi_replay_finalize(){ sim_time = smpi_process_simulated_elapsed(); XBT_INFO("Simulation time %f", sim_time); _xbt_replay_action_exit(); + xbt_free(sendbuffer); + xbt_free(recvbuffer); xbt_free(reqq); reqq = NULL; } - smpi_mpi_barrier(MPI_COMM_WORLD); + mpi_coll_barrier_fun(MPI_COMM_WORLD); #ifdef HAVE_TRACING int rank = smpi_process_index(); instr_extra_data extra = xbt_new0(s_instr_extra_data_t,1);