From: Tom Cornebize Date: Wed, 12 Apr 2017 21:13:02 +0000 (+0200) Subject: Merge branch 'master' into partial_shared_malloc X-Git-Tag: v3.16~353^2~1^2~2 X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/2dba4579bf4c7f7e8cde2b36ff308645a3cc891b Merge branch 'master' into partial_shared_malloc --- 2dba4579bf4c7f7e8cde2b36ff308645a3cc891b diff --cc .gitignore index 442490262b,466f2bfb94..74adcccb87 --- a/.gitignore +++ b/.gitignore @@@ -986,10 -986,14 +986,16 @@@ teshsuite/smpi/coll-reduce/coll-reduc teshsuite/smpi/coll-reduce-scatter/coll-reduce-scatter teshsuite/smpi/coll-scatter/coll-scatter teshsuite/smpi/macro-shared/macro-shared +teshsuite/smpi/macro-partial-shared/macro-partial-shared +teshsuite/smpi/macro-partial-shared-communication/macro-partial-shared-communication teshsuite/smpi/type-struct/type-struct teshsuite/smpi/type-vector/type-vector + teshsuite/s4u/actor/actor + teshsuite/s4u/concurrent_rw/concurrent_rw + teshsuite/s4u/host_on_off_wait/host_on_off_wait + teshsuite/s4u/listen_async/listen_async + teshsuite/s4u/pid/pid + teshsuite/s4u/storage_client_server/storage_client_server teshsuite/surf/lmm_usage/lmm_usage teshsuite/surf/maxmin_bench/maxmin_bench teshsuite/surf/surf_usage/surf_usage diff --cc src/smpi/smpi_datatype.cpp index 3b36cab2cd,4ce49854c2..2a774f30ac --- a/src/smpi/smpi_datatype.cpp +++ b/src/smpi/smpi_datatype.cpp @@@ -278,9 -276,8 +278,9 @@@ int Datatype::copy(void *sendbuf, int s }else if(smpi_is_shared(recvbuf)){ XBT_DEBUG("Copy output buf %p is shared. Let's ignore it.", recvbuf); } +#endif - if(smpi_privatize_global_variables){ + if(smpi_privatize_global_variables == SMPI_PRIVATIZE_MMAP){ smpi_switch_data_segment(smpi_process()->index()); } /* First check if we really have something to do */ diff --cc src/smpi/smpi_global.cpp index c29f0cadc0,0da7f66a81..3eb02162af --- a/src/smpi/smpi_global.cpp +++ b/src/smpi/smpi_global.cpp @@@ -104,97 -121,49 +122,96 @@@ void smpi_comm_set_copy_data_callback(v smpi_comm_copy_data_callback = callback; } +static void print(std::vector> vec) { + fprintf(stderr, "{"); + for(auto elt: vec) { + fprintf(stderr, "(0x%lx, 0x%lx),", elt.first, elt.second); + } + fprintf(stderr, "}\n"); +} +static void memcpy_private(void *dest, const void *src, size_t n, std::vector> &private_blocks) { + for(auto block : private_blocks) { + memcpy((uint8_t*)dest+block.first, (uint8_t*)src+block.first, block.second-block.first); + } +} + +static void check_blocks(std::vector> &private_blocks, size_t buff_size) { + for(auto block : private_blocks) { + xbt_assert(block.first <= block.second && block.second <= buff_size, "Oops, bug in shared malloc."); + } +} + void smpi_comm_copy_buffer_callback(smx_activity_t synchro, void *buff, size_t buff_size) { - simgrid::kernel::activity::Comm *comm = dynamic_cast(synchro); - + int src_shared=0, dst_shared=0; + size_t src_offset=0, dst_offset=0; + std::vector> src_private_blocks; + std::vector> dst_private_blocks; XBT_DEBUG("Copy the data over"); - if(smpi_is_shared(buff)){ + if((src_shared=smpi_is_shared(buff, src_private_blocks, &src_offset))) { XBT_DEBUG("Sender %p is shared. Let's ignore it.", buff); - }else if(smpi_is_shared((char*)comm->dst_buff)){ + src_private_blocks = shift_and_frame_private_blocks(src_private_blocks, src_offset, buff_size); + } + else { + src_private_blocks.clear(); + src_private_blocks.push_back(std::make_pair(0, buff_size)); + } + if((dst_shared=smpi_is_shared((char*)comm->dst_buff, dst_private_blocks, &dst_offset))) { XBT_DEBUG("Receiver %p is shared. Let's ignore it.", (char*)comm->dst_buff); - }else{ - void* tmpbuff=buff; - if((smpi_privatize_global_variables == SMPI_PRIVATIZE_MMAP) && (static_cast(buff) >= smpi_start_data_exe) - && (static_cast(buff) < smpi_start_data_exe + smpi_size_data_exe ) - ){ - XBT_DEBUG("Privatization : We are copying from a zone inside global memory... Saving data to temp buffer !"); - - smpi_switch_data_segment( - (static_cast((static_cast(comm->src_proc->data)->data))->index())); - tmpbuff = static_cast(xbt_malloc(buff_size)); - memcpy(tmpbuff, buff, buff_size); - } - - if((smpi_privatize_global_variables == SMPI_PRIVATIZE_MMAP) && ((char*)comm->dst_buff >= smpi_start_data_exe) - && ((char*)comm->dst_buff < smpi_start_data_exe + smpi_size_data_exe )){ - XBT_DEBUG("Privatization : We are copying to a zone inside global memory - Switch data segment"); - smpi_switch_data_segment( - (static_cast((static_cast(comm->dst_proc->data)->data))->index())); - } - - XBT_DEBUG("Copying %zu bytes from %p to %p", buff_size, tmpbuff,comm->dst_buff); - memcpy(comm->dst_buff, tmpbuff, buff_size); + dst_private_blocks = shift_and_frame_private_blocks(dst_private_blocks, dst_offset, buff_size); + } + else { + dst_private_blocks.clear(); + dst_private_blocks.push_back(std::make_pair(0, buff_size)); + } +/* + fprintf(stderr, "size: 0x%x\n", buff_size); + fprintf(stderr, "src: "); + print(src_private_blocks); + fprintf(stderr, "src_offset = 0x%x\n", src_offset); + fprintf(stderr, "dst: "); + print(dst_private_blocks); + fprintf(stderr, "dst_offset = 0x%x\n", dst_offset); +*/ + check_blocks(src_private_blocks, buff_size); + check_blocks(dst_private_blocks, buff_size); + auto private_blocks = merge_private_blocks(src_private_blocks, dst_private_blocks); +/* + fprintf(stderr, "Private blocks: "); + print(private_blocks); +*/ + check_blocks(private_blocks, buff_size); + void* tmpbuff=buff; - if((smpi_privatize_global_variables) && (static_cast(buff) >= smpi_start_data_exe) ++ if((smpi_privatize_global_variables == SMPI_PRIVATIZE_MMAP) && (static_cast(buff) >= smpi_start_data_exe) + && (static_cast(buff) < smpi_start_data_exe + smpi_size_data_exe ) + ){ + XBT_DEBUG("Privatization : We are copying from a zone inside global memory... Saving data to temp buffer !"); + + smpi_switch_data_segment( + (static_cast((static_cast(comm->src_proc->data)->data))->index())); + tmpbuff = static_cast(xbt_malloc(buff_size)); + memcpy_private(tmpbuff, buff, buff_size, private_blocks); + } - if((smpi_privatize_global_variables) && ((char*)comm->dst_buff >= smpi_start_data_exe) - if (comm->detached) { - // if this is a detached send, the source buffer was duplicated by SMPI - // sender to make the original buffer available to the application ASAP - xbt_free(buff); - //It seems that the request is used after the call there this should be free somewhere else but where??? - //xbt_free(comm->comm.src_data);// inside SMPI the request is kept inside the user data and should be free - comm->src_buff = nullptr; - } - if(tmpbuff!=buff)xbt_free(tmpbuff); ++ if((smpi_privatize_global_variables == SMPI_PRIVATIZE_MMAP) && ((char*)comm->dst_buff >= smpi_start_data_exe) + && ((char*)comm->dst_buff < smpi_start_data_exe + smpi_size_data_exe )){ + XBT_DEBUG("Privatization : We are copying to a zone inside global memory - Switch data segment"); + smpi_switch_data_segment( + (static_cast((static_cast(comm->dst_proc->data)->data))->index())); + } - + XBT_DEBUG("Copying %zu bytes from %p to %p", buff_size, tmpbuff,comm->dst_buff); + memcpy_private(comm->dst_buff, tmpbuff, buff_size, private_blocks); + + if (comm->detached) { + // if this is a detached send, the source buffer was duplicated by SMPI + // sender to make the original buffer available to the application ASAP + xbt_free(buff); + //It seems that the request is used after the call there this should be free somewhere else but where??? + //xbt_free(comm->comm.src_data);// inside SMPI the request is kept inside the user data and should be free + comm->src_buff = nullptr; } + if(tmpbuff!=buff)xbt_free(tmpbuff); } diff --cc src/smpi/smpi_request.cpp index b3988d883a,2ff10a2790..ed83759424 --- a/src/smpi/smpi_request.cpp +++ b/src/smpi/smpi_request.cpp @@@ -108,13 -108,16 +108,17 @@@ namespace smpi Request::Request(void *buf, int count, MPI_Datatype datatype, int src, int dst, int tag, MPI_Comm comm, unsigned flags) : buf_(buf), old_type_(datatype), src_(src), dst_(dst), tag_(tag), comm_(comm), flags_(flags) { void *old_buf = nullptr; - if(((((flags & RECV) != 0) && ((flags & ACCUMULATE) !=0)) || (datatype->flags() & DT_FLAG_DERIVED)) && (!smpi_is_shared(buf_))){ +// FIXME Handle the case of a partial shared malloc. + if(((((flags & RECV) != 0) && ((flags & ACCUMULATE) !=0)) || (datatype->flags() & DT_FLAG_DERIVED))) { // && (!smpi_is_shared(buf_))){ // This part handles the problem of non-contiguous memory old_buf = buf; - buf_ = count==0 ? nullptr : xbt_malloc(count*datatype->size()); - if ((datatype->flags() & DT_FLAG_DERIVED) && ((flags & SEND) != 0)) { - datatype->serialize(old_buf, buf_, count); + if (count==0){ + buf_ = nullptr; + }else { + buf_ = xbt_malloc(count*datatype->size()); + if ((datatype->flags() & DT_FLAG_DERIVED) && ((flags & SEND) != 0)) { + datatype->serialize(old_buf, buf_, count); + } } } // This part handles the problem of non-contiguous memory (for the unserialisation at the reception) @@@ -759,11 -763,10 +764,11 @@@ void Request::finish_wait(MPI_Request* req->print_request("Finishing"); MPI_Datatype datatype = req->old_type_; - if((((req->flags_ & ACCUMULATE) != 0) || (datatype->flags() & DT_FLAG_DERIVED)) && (!smpi_is_shared(req->old_buf_))){ +// FIXME Handle the case of a partial shared malloc. + if((((req->flags_ & ACCUMULATE) != 0) || (datatype->flags() & DT_FLAG_DERIVED))){// && (!smpi_is_shared(req->old_buf_))){ if (!smpi_process()->replaying()){ - if( smpi_privatize_global_variables != 0 && (static_cast(req->old_buf_) >= smpi_start_data_exe) + if( smpi_privatize_global_variables == SMPI_PRIVATIZE_MMAP && (static_cast(req->old_buf_) >= smpi_start_data_exe) && ((char*)req->old_buf_ < smpi_start_data_exe + smpi_size_data_exe )){ XBT_VERB("Privatization : We are unserializing to a zone in global memory Switch data segment "); smpi_switch_data_segment(smpi_process()->index());