From: jean-noel quintin Date: Wed, 3 Oct 2012 15:24:39 +0000 (+0100) Subject: [smpi] add a gestion of non-contignous data X-Git-Tag: v3_8~146^2~6 X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/aa64b0469b0967780beb549867c2907c14525443 [smpi] add a gestion of non-contignous data --- diff --git a/examples/smpi/CMakeLists.txt b/examples/smpi/CMakeLists.txt index ac183b8eda..7abef72ed9 100644 --- a/examples/smpi/CMakeLists.txt +++ b/examples/smpi/CMakeLists.txt @@ -26,6 +26,7 @@ if(enable_smpi) add_executable(smpi_traced tracing/smpi_traced.c) add_executable(smpi_traced_simple tracing/smpi_traced_simple.c) add_executable(ttest01 ttest01.c) + add_executable(vector_test vector_test.c) add_executable(mc_bugged1 mc_bugged1.c) add_executable(mc_bugged2 mc_bugged2.c) add_executable(smpi_replay replay/replay.c) @@ -49,6 +50,7 @@ if(enable_smpi) target_link_libraries(smpi_traced m simgrid smpi ) target_link_libraries(smpi_traced_simple m simgrid smpi ) target_link_libraries(ttest01 m simgrid smpi ) + target_link_libraries(vector_test m simgrid smpi ) target_link_libraries(mc_bugged1 m simgrid smpi ) target_link_libraries(mc_bugged2 m simgrid smpi ) target_link_libraries(smpi_replay m simgrid smpi ) @@ -91,6 +93,7 @@ set(examples_src ${CMAKE_CURRENT_SOURCE_DIR}/sendtest.c ${CMAKE_CURRENT_SOURCE_DIR}/barrier.c ${CMAKE_CURRENT_SOURCE_DIR}/bcbench.c + ${CMAKE_CURRENT_SOURCE_DIR}/vector_test.c ${CMAKE_CURRENT_SOURCE_DIR}/replay/replay.c ${CMAKE_CURRENT_SOURCE_DIR}/tracing/smpi_traced.c ${CMAKE_CURRENT_SOURCE_DIR}/tracing/smpi_traced_simple.c diff --git a/examples/smpi/vector_test.c b/examples/smpi/vector_test.c new file mode 100644 index 0000000000..f539054d4a --- /dev/null +++ b/examples/smpi/vector_test.c @@ -0,0 +1,42 @@ +#include +#include "mpi.h" +#define SIZE 4 + +int main(int argc, char **argv) { + + int rank, tag=1, i, j; + double a[SIZE][SIZE]; + + MPI_Status stat; + MPI_Datatype columntype; + + MPI_Init(&argc,&argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + MPI_Type_vector(SIZE, 1, SIZE, MPI_DOUBLE, &columntype); + MPI_Type_commit(&columntype); + + + if (rank == 0) { + for(i=0; i #include "surf/surf.h" -XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_base, smpi, - "Logging specific to SMPI (base)"); + +XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_base, smpi, "Logging specific to SMPI (base)"); + static int match_recv(void* a, void* b, smx_action_t ignored) { - MPI_Request ref = (MPI_Request)a; - MPI_Request req = (MPI_Request)b; + MPI_Request ref = (MPI_Request)a; + MPI_Request req = (MPI_Request)b; - xbt_assert(ref, "Cannot match recv against null reference"); - xbt_assert(req, "Cannot match recv against null request"); - return (ref->src == MPI_ANY_SOURCE || req->src == ref->src) - && (ref->tag == MPI_ANY_TAG || req->tag == ref->tag); + xbt_assert(ref, "Cannot match recv against null reference"); + xbt_assert(req, "Cannot match recv against null request"); + return (ref->src == MPI_ANY_SOURCE || req->src == ref->src) + && (ref->tag == MPI_ANY_TAG || req->tag == ref->tag); } static int match_send(void* a, void* b,smx_action_t ignored) { - MPI_Request ref = (MPI_Request)a; - MPI_Request req = (MPI_Request)b; + MPI_Request ref = (MPI_Request)a; + MPI_Request req = (MPI_Request)b; - xbt_assert(ref, "Cannot match send against null reference"); - xbt_assert(req, "Cannot match send against null request"); - return (req->src == MPI_ANY_SOURCE || req->src == ref->src) - && (req->tag == MPI_ANY_TAG || req->tag == ref->tag); + xbt_assert(ref, "Cannot match send against null reference"); + xbt_assert(req, "Cannot match send against null request"); + return (req->src == MPI_ANY_SOURCE || req->src == ref->src) + && (req->tag == MPI_ANY_TAG || req->tag == ref->tag); } static MPI_Request build_request(void *buf, int count, @@ -40,16 +41,28 @@ static MPI_Request build_request(void *buf, int count, { MPI_Request request; + void *old_buf; + request = xbt_new(s_smpi_mpi_request_t, 1); + + s_smpi_subtype_t *subtype = datatype->substruct; + + if(datatype->has_subtype == 1){ + // This part handles the problem of non-contignous memory + old_buf = buf; + buf = malloc(count*smpi_datatype_size(datatype)); + if (flags & SEND) { + subtype->serialize(old_buf, buf, count, datatype->substruct); + } + } + request->buf = buf; - // FIXME: this will have to be changed to support non-contiguous datatypes + // This part handles the problem of non-contignous memory (for the + // unserialisation at the reception) + request->old_buf = old_buf; + request->old_type = datatype; + request->size = smpi_datatype_size(datatype) * count; - request->contiguous=smpi_datatype_contiguous(datatype); - if(request->contiguous != 1){ - request->block_stride = smpi_datatype_block_stride(datatype); - request->block_length = smpi_datatype_block_length(datatype); - request->block_count = smpi_datatype_block_count(datatype)*count; - } request->src = src; request->dst = dst; request->tag = tag; @@ -73,12 +86,12 @@ void smpi_action_trace_run(char *path) if (path) { action_fp = fopen(path, "r"); xbt_assert(action_fp != NULL, "Cannot open %s: %s", path, - strerror(errno)); + strerror(errno)); } if (!xbt_dict_is_empty(action_queues)) { XBT_WARN - ("Not all actions got consumed. If the simulation ended successfully (without deadlock), you may want to add new processes to your deployment file."); + ("Not all actions got consumed. If the simulation ended successfully (without deadlock), you may want to add new processes to your deployment file."); xbt_dict_foreach(action_queues, cursor, name, todo) { @@ -103,8 +116,8 @@ MPI_Request smpi_mpi_send_init(void *buf, int count, MPI_Datatype datatype, int dst, int tag, MPI_Comm comm) { MPI_Request request = - build_request(buf, count, datatype, smpi_comm_rank(comm), dst, tag, - comm, PERSISTENT | SEND); + build_request(buf, count, datatype, smpi_comm_rank(comm), dst, tag, + comm, PERSISTENT | SEND); return request; } @@ -113,8 +126,8 @@ MPI_Request smpi_mpi_recv_init(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm) { MPI_Request request = - build_request(buf, count, datatype, src, smpi_comm_rank(comm), tag, - comm, PERSISTENT | RECV); + build_request(buf, count, datatype, src, smpi_comm_rank(comm), tag, + comm, PERSISTENT | RECV); return request; } @@ -125,13 +138,13 @@ void smpi_mpi_start(MPI_Request request) int detached = 0; xbt_assert(!request->action, - "Cannot (re)start a non-finished communication"); + "Cannot (re)start a non-finished communication"); if(request->flags & RECV) { print_request("New recv", request); if (request->size < xbt_cfg_get_int(_surf_cfg_set, "smpi/async_small_thres")) - mailbox = smpi_process_mailbox_small(); + mailbox = smpi_process_mailbox_small(); else - mailbox = smpi_process_mailbox(); + mailbox = smpi_process_mailbox(); // FIXME: SIMIX does not yet support non-contiguous datatypes request->action = simcall_comm_irecv(mailbox, request->buf, &request->size, &match_recv, request); @@ -140,11 +153,11 @@ void smpi_mpi_start(MPI_Request request) if (request->size < xbt_cfg_get_int(_surf_cfg_set, "smpi/async_small_thres")) { // eager mode => detached send (FIXME: this limit should be configurable) mailbox = smpi_process_remote_mailbox_small( - smpi_group_index(smpi_comm_group(request->comm), request->dst)); + smpi_group_index(smpi_comm_group(request->comm), request->dst)); }else{ XBT_DEBUG("Send request %p is not in the permanent receive mailbox (buf: %p)",request,request->buf); mailbox = smpi_process_remote_mailbox( - smpi_group_index(smpi_comm_group(request->comm), request->dst)); + smpi_group_index(smpi_comm_group(request->comm), request->dst)); } if (request->size < 64*1024 ) { //(FIXME: this limit should be configurable) void *oldbuf = request->buf; @@ -155,28 +168,28 @@ void smpi_mpi_start(MPI_Request request) XBT_DEBUG("Send request %p is detached; buf %p copied into %p",request,oldbuf,request->buf); } - request->action = + request->action = simcall_comm_isend(mailbox, request->size, -1.0, - request->buf, request->size, - &match_send, - &smpi_mpi_request_free_voidp, // how to free the userdata if a detached send fails - request, - // detach if msg size < eager/rdv switch limit - detached); - - #ifdef HAVE_TRACING - /* FIXME: detached sends are not traceable (request->action == NULL) */ - if (request->action) - simcall_set_category(request->action, TRACE_internal_smpi_get_category()); - #endif + request->buf, request->size, + &match_send, + &smpi_mpi_request_free_voidp, // how to free the userdata if a detached send fails + request, + // detach if msg size < eager/rdv switch limit + detached); - } +#ifdef HAVE_TRACING + /* FIXME: detached sends are not traceable (request->action == NULL) */ + if (request->action) + simcall_set_category(request->action, TRACE_internal_smpi_get_category()); +#endif + + } } void smpi_mpi_startall(int count, MPI_Request * requests) { - int i; + int i; for(i = 0; i < count; i++) { smpi_mpi_start(requests[i]); @@ -193,8 +206,8 @@ MPI_Request smpi_isend_init(void *buf, int count, MPI_Datatype datatype, int dst, int tag, MPI_Comm comm) { MPI_Request request = - build_request(buf, count, datatype, smpi_comm_rank(comm), dst, tag, - comm, NON_PERSISTENT | SEND); + build_request(buf, count, datatype, smpi_comm_rank(comm), dst, tag, + comm, NON_PERSISTENT | SEND); return request; } @@ -203,7 +216,7 @@ MPI_Request smpi_mpi_isend(void *buf, int count, MPI_Datatype datatype, int dst, int tag, MPI_Comm comm) { MPI_Request request = - smpi_isend_init(buf, count, datatype, dst, tag, comm); + smpi_isend_init(buf, count, datatype, dst, tag, comm); smpi_mpi_start(request); return request; @@ -213,9 +226,8 @@ MPI_Request smpi_irecv_init(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm) { MPI_Request request = - build_request(buf, count, datatype, src, smpi_comm_rank(comm), tag, - comm, NON_PERSISTENT | RECV); - + build_request(buf, count, datatype, src, smpi_comm_rank(comm), tag, + comm, NON_PERSISTENT | RECV); return request; } @@ -223,7 +235,7 @@ MPI_Request smpi_mpi_irecv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm) { MPI_Request request = - smpi_irecv_init(buf, count, datatype, src, tag, comm); + smpi_irecv_init(buf, count, datatype, src, tag, comm); smpi_mpi_start(request); return request; @@ -233,7 +245,6 @@ void smpi_mpi_recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status * status) { MPI_Request request; - request = smpi_mpi_irecv(buf, count, datatype, src, tag, comm); smpi_mpi_wait(&request, status); } @@ -243,10 +254,10 @@ void smpi_mpi_recv(void *buf, int count, MPI_Datatype datatype, int src, void smpi_mpi_send(void *buf, int count, MPI_Datatype datatype, int dst, int tag, MPI_Comm comm) { - MPI_Request request; + MPI_Request request; - request = smpi_mpi_isend(buf, count, datatype, dst, tag, comm); - smpi_mpi_wait(&request, MPI_STATUS_IGNORE); + request = smpi_mpi_isend(buf, count, datatype, dst, tag, comm); + smpi_mpi_wait(&request, MPI_STATUS_IGNORE); } void smpi_mpi_sendrecv(void *sendbuf, int sendcount, MPI_Datatype sendtype, @@ -258,9 +269,9 @@ void smpi_mpi_sendrecv(void *sendbuf, int sendcount, MPI_Datatype sendtype, MPI_Status stats[2]; requests[0] = - smpi_isend_init(sendbuf, sendcount, sendtype, dst, sendtag, comm); + smpi_isend_init(sendbuf, sendcount, sendtype, dst, sendtag, comm); requests[1] = - smpi_irecv_init(recvbuf, recvcount, recvtype, src, recvtag, comm); + smpi_irecv_init(recvbuf, recvcount, recvtype, src, recvtag, comm); smpi_mpi_startall(2, requests); smpi_mpi_waitall(2, requests, stats); if(status != MPI_STATUS_IGNORE) { @@ -279,7 +290,7 @@ static void finish_wait(MPI_Request * request, MPI_Status * status) MPI_Request req = *request; // if we have a sender, we should use its data, and not the data from the receive if((req->action)&& - (req->src==MPI_ANY_SOURCE || req->tag== MPI_ANY_TAG)) + (req->src==MPI_ANY_SOURCE || req->tag== MPI_ANY_TAG)) req = (MPI_Request)SIMIX_comm_get_src_data((*request)->action); if(status != MPI_STATUS_IGNORE) { @@ -293,6 +304,19 @@ static void finish_wait(MPI_Request * request, MPI_Status * status) req = *request; print_request("Finishing", req); + MPI_Datatype datatype = req->old_type; + if(datatype->has_subtype == 1){ + // This part handles the problem of non-contignous memory + // the unserialization at the reception + s_smpi_subtype_t *subtype = datatype->substruct; + if(req->flags & RECV) { + subtype->unserialize(req->buf, req->old_buf, req->size/smpi_datatype_size(datatype) , datatype->substruct); + } + //FIXME: I am not sure that if the send is detached we have to free + //the sender buffer thus I do it only for the reciever + if(req->flags & RECV) free(req->buf); + } + if(req->flags & NON_PERSISTENT) { smpi_mpi_request_free(request); } else { @@ -301,16 +325,16 @@ static void finish_wait(MPI_Request * request, MPI_Status * status) } int smpi_mpi_test(MPI_Request * request, MPI_Status * status) { -int flag; + int flag; - if ((*request)->action == NULL) - flag = 1; - else + if ((*request)->action == NULL) + flag = 1; + else flag = simcall_comm_test((*request)->action); - if(flag) { - finish_wait(request, status); - } - return flag; + if(flag) { + finish_wait(request, status); + } + return flag; } int smpi_mpi_testany(int count, MPI_Request requests[], int *index, @@ -328,9 +352,9 @@ int smpi_mpi_testany(int count, MPI_Request requests[], int *index, size = 0; for(i = 0; i < count; i++) { if(requests[i]->action) { - xbt_dynar_push(comms, &requests[i]->action); - map[size] = i; - size++; + xbt_dynar_push(comms, &requests[i]->action); + map[size] = i; + size++; } } if(size > 0) { @@ -355,8 +379,8 @@ int smpi_mpi_testall(int count, MPI_Request requests[], int flag=1; int i; for(i=0; i0){ - mailbox = smpi_process_mailbox_small(); - request->action = simcall_comm_iprobe(mailbox, request->src, request->tag, &match_recv, (void*)request); + if (xbt_cfg_get_int(_surf_cfg_set, "smpi/async_small_thres")>0){ + mailbox = smpi_process_mailbox_small(); + request->action = simcall_comm_iprobe(mailbox, request->src, request->tag, &match_recv, (void*)request); - } - if (request->action==NULL){ - mailbox = smpi_process_mailbox(); - request->action = simcall_comm_iprobe(mailbox, request->src, request->tag, &match_recv, (void*)request); - } + } + if (request->action==NULL){ + mailbox = smpi_process_mailbox(); + request->action = simcall_comm_iprobe(mailbox, request->src, request->tag, &match_recv, (void*)request); + } if(request->action){ MPI_Request req = (MPI_Request)SIMIX_comm_get_src_data(request->action); @@ -487,10 +511,10 @@ int smpi_mpi_waitsome(int incount, MPI_Request requests[], int *indices, count = 0; for(i = 0; i < incount; i++) { - if(smpi_mpi_testany(incount, requests, &index, status)) { - indices[count] = index; - count++; - } + if(smpi_mpi_testany(incount, requests, &index, status)) { + indices[count] = index; + count++; + } } return count; } @@ -526,15 +550,15 @@ void smpi_mpi_gather(void *sendbuf, int sendcount, MPI_Datatype sendtype, // FIXME: check for errors smpi_datatype_extent(recvtype, &lb, &recvext); // Local copy from root - smpi_datatype_copy(sendbuf, sendcount, sendtype, - (char *)recvbuf + root * recvcount * recvext, recvcount, recvtype); + smpi_datatype_copy(sendbuf, sendcount, sendtype, + (char *)recvbuf + root * recvcount * recvext, recvcount, recvtype); // Receive buffers from senders requests = xbt_new(MPI_Request, size - 1); index = 0; for(src = 0; src < size; src++) { if(src != root) { - requests[index] = smpi_irecv_init((char *)recvbuf + src * recvcount * recvext, - recvcount, recvtype, + requests[index] = smpi_irecv_init((char *)recvbuf + src * recvcount * recvext, + recvcount, recvtype, src, system_tag, comm); index++; } @@ -564,8 +588,8 @@ void smpi_mpi_gatherv(void *sendbuf, int sendcount, MPI_Datatype sendtype, // FIXME: check for errors smpi_datatype_extent(recvtype, &lb, &recvext); // Local copy from root - smpi_datatype_copy(sendbuf, sendcount, sendtype, - (char *)recvbuf + displs[root] * recvext, + smpi_datatype_copy(sendbuf, sendcount, sendtype, + (char *)recvbuf + displs[root] * recvext, recvcounts[root], recvtype); // Receive buffers from senders requests = xbt_new(MPI_Request, size - 1); @@ -573,8 +597,8 @@ void smpi_mpi_gatherv(void *sendbuf, int sendcount, MPI_Datatype sendtype, for(src = 0; src < size; src++) { if(src != root) { requests[index] = - smpi_irecv_init((char *)recvbuf + displs[src] * recvext, - recvcounts[src], recvtype, src, system_tag, comm); + smpi_irecv_init((char *)recvbuf + displs[src] * recvext, + recvcounts[src], recvtype, src, system_tag, comm); index++; } } @@ -600,8 +624,8 @@ void smpi_mpi_allgather(void *sendbuf, int sendcount, // FIXME: check for errors smpi_datatype_extent(recvtype, &lb, &recvext); // Local copy from self - smpi_datatype_copy(sendbuf, sendcount, sendtype, - (char *)recvbuf + rank * recvcount * recvext, recvcount, + smpi_datatype_copy(sendbuf, sendcount, sendtype, + (char *)recvbuf + rank * recvcount * recvext, recvcount, recvtype); // Send/Recv buffers to/from others; requests = xbt_new(MPI_Request, 2 * (size - 1)); @@ -609,11 +633,11 @@ void smpi_mpi_allgather(void *sendbuf, int sendcount, for(other = 0; other < size; other++) { if(other != rank) { requests[index] = - smpi_isend_init(sendbuf, sendcount, sendtype, other, system_tag, - comm); + smpi_isend_init(sendbuf, sendcount, sendtype, other, system_tag, + comm); index++; - requests[index] = smpi_irecv_init((char *)recvbuf + other * recvcount * recvext, - recvcount, recvtype, other, + requests[index] = smpi_irecv_init((char *)recvbuf + other * recvcount * recvext, + recvcount, recvtype, other, system_tag, comm); index++; } @@ -639,8 +663,8 @@ void smpi_mpi_allgatherv(void *sendbuf, int sendcount, // FIXME: check for errors smpi_datatype_extent(recvtype, &lb, &recvext); // Local copy from self - smpi_datatype_copy(sendbuf, sendcount, sendtype, - (char *)recvbuf + displs[rank] * recvext, + smpi_datatype_copy(sendbuf, sendcount, sendtype, + (char *)recvbuf + displs[rank] * recvext, recvcounts[rank], recvtype); // Send buffers to others; requests = xbt_new(MPI_Request, 2 * (size - 1)); @@ -648,12 +672,12 @@ void smpi_mpi_allgatherv(void *sendbuf, int sendcount, for(other = 0; other < size; other++) { if(other != rank) { requests[index] = - smpi_isend_init(sendbuf, sendcount, sendtype, other, system_tag, - comm); + smpi_isend_init(sendbuf, sendcount, sendtype, other, system_tag, + comm); index++; requests[index] = - smpi_irecv_init((char *)recvbuf + displs[other] * recvext, recvcounts[other], - recvtype, other, system_tag, comm); + smpi_irecv_init((char *)recvbuf + displs[other] * recvext, recvcounts[other], + recvtype, other, system_tag, comm); index++; } } @@ -683,13 +707,13 @@ void smpi_mpi_scatter(void *sendbuf, int sendcount, MPI_Datatype sendtype, smpi_datatype_extent(sendtype, &lb, &sendext); // Local copy from root smpi_datatype_copy((char *)sendbuf + root * sendcount * sendext, - sendcount, sendtype, recvbuf, recvcount, recvtype); + sendcount, sendtype, recvbuf, recvcount, recvtype); // Send buffers to receivers requests = xbt_new(MPI_Request, size - 1); index = 0; for(dst = 0; dst < size; dst++) { if(dst != root) { - requests[index] = smpi_isend_init((char *)sendbuf + dst * sendcount * sendext, + requests[index] = smpi_isend_init((char *)sendbuf + dst * sendcount * sendext, sendcount, sendtype, dst, system_tag, comm); index++; @@ -721,7 +745,7 @@ void smpi_mpi_scatterv(void *sendbuf, int *sendcounts, int *displs, // FIXME: check for errors smpi_datatype_extent(sendtype, &lb, &sendext); // Local copy from root - smpi_datatype_copy((char *)sendbuf + displs[root] * sendext, sendcounts[root], + smpi_datatype_copy((char *)sendbuf + displs[root] * sendext, sendcounts[root], sendtype, recvbuf, recvcount, recvtype); // Send buffers to receivers requests = xbt_new(MPI_Request, size - 1); @@ -729,8 +753,8 @@ void smpi_mpi_scatterv(void *sendbuf, int *sendcounts, int *displs, for(dst = 0; dst < size; dst++) { if(dst != root) { requests[index] = - smpi_isend_init((char *)sendbuf + displs[dst] * sendext, sendcounts[dst], - sendtype, dst, system_tag, comm); + smpi_isend_init((char *)sendbuf + displs[dst] * sendext, sendcounts[dst], + sendtype, dst, system_tag, comm); index++; } } @@ -773,8 +797,8 @@ void smpi_mpi_reduce(void *sendbuf, void *recvbuf, int count, // mapping... tmpbufs[index] = xbt_malloc(count * dataext); requests[index] = - smpi_irecv_init(tmpbufs[index], count, datatype, src, - system_tag, comm); + smpi_irecv_init(tmpbufs[index], count, datatype, src, + system_tag, comm); index++; } } @@ -827,17 +851,17 @@ void smpi_mpi_scan(void *sendbuf, void *recvbuf, int count, tmpbufs = xbt_new(void *, rank); index = 0; for(other = 0; other < rank; other++) { - // FIXME: possibly overkill we we have contiguous/noncontiguous data + // FIXME: possibly overkill we we have contiguous/noncontiguous data // mapping... tmpbufs[index] = xbt_malloc(count * dataext); requests[index] = - smpi_irecv_init(tmpbufs[index], count, datatype, other, system_tag, - comm); + smpi_irecv_init(tmpbufs[index], count, datatype, other, system_tag, + comm); index++; } for(other = rank + 1; other < size; other++) { requests[index] = - smpi_isend_init(sendbuf, count, datatype, other, system_tag, comm); + smpi_isend_init(sendbuf, count, datatype, other, system_tag, comm); index++; } // Wait for completion of all comms. diff --git a/src/smpi/smpi_mpi_dt.c b/src/smpi/smpi_mpi_dt.c index 496152004c..d75b7fa21c 100644 --- a/src/smpi/smpi_mpi_dt.c +++ b/src/smpi/smpi_mpi_dt.c @@ -5,7 +5,7 @@ * All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it - * under the terms of the license (GNU LGPL) which comes with this package. */ + * under the terms of the license (GNU LGPL) which comes with this package. */ #include #include @@ -17,29 +17,16 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_mpi_dt, smpi, "Logging specific to SMPI (datatype)"); -typedef struct s_smpi_mpi_datatype { - size_t size; - size_t contiguous; - size_t block_stride; - size_t block_length; - size_t block_count: - MPI_Aint lb; - MPI_Aint ub; - int flags; -} s_smpi_mpi_datatype_t; - #define CREATE_MPI_DATATYPE(name, type) \ static s_smpi_mpi_datatype_t mpi_##name = { \ sizeof(type), /* size */ \ - 1, /*contiguous*/ \ - 0, /*block_stride*/ \ - 0, /*block_length*/ \ - 0, /*block_count*/ \ + 0, /*was 1 has_subtype*/ \ 0, /* lb */ \ sizeof(type), /* ub = lb + size */ \ - DT_FLAG_BASIC /* flags */ \ + DT_FLAG_BASIC, /* flags */ \ + NULL /* pointer on extended struct*/ \ }; \ - MPI_Datatype name = &mpi_##name; +MPI_Datatype name = &mpi_##name; //The following are datatypes for the MPI functions MPI_MAXLOC and MPI_MINLOC. @@ -115,23 +102,7 @@ size_t smpi_datatype_size(MPI_Datatype datatype) return datatype->size; } -size_t smpi_datatype_contiguous(MPI_Datatype datatype) -{ - return datatype->contiguous; -} -size_t smpi_datatype_block_stride(MPI_Datatype datatype) -{ - return datatype->block_stride; -} -size_t smpi_datatype_block_length(MPI_Datatype datatype) -{ - return datatype->block_length; -} -size_t smpi_datatype_block_count(MPI_Datatype datatype) -{ - return datatype->block_count; -} MPI_Aint smpi_datatype_lb(MPI_Datatype datatype) { @@ -171,18 +142,116 @@ int smpi_datatype_copy(void *sendbuf, int sendcount, MPI_Datatype sendtype, sendcount *= smpi_datatype_size(sendtype); recvcount *= smpi_datatype_size(recvtype); count = sendcount < recvcount ? sendcount : recvcount; - memcpy(recvbuf, sendbuf, count); + + if(sendtype->has_subtype == 0 && recvtype->has_subtype == 0) { + memcpy(recvbuf, sendbuf, count); + } + else if (sendtype->has_subtype == 0) + { + s_smpi_subtype_t *subtype = recvtype->substruct; + subtype->unserialize( sendbuf, recvbuf,1, subtype); + } + else if (recvtype->has_subtype == 0) + { + s_smpi_subtype_t *subtype = sendtype->substruct; + subtype->serialize(sendbuf, recvbuf,1, subtype); + }else{ + s_smpi_subtype_t *subtype = sendtype->substruct; + + s_smpi_mpi_vector_t* type_c = (s_smpi_mpi_vector_t*)sendtype; + + void * buf_tmp = malloc(count * type_c->size_oldtype); + + subtype->serialize( sendbuf, buf_tmp,1, subtype); + subtype = recvtype->substruct; + subtype->unserialize(recvbuf, buf_tmp,1, subtype); + + free(buf_tmp); + } retval = sendcount > recvcount ? MPI_ERR_TRUNCATE : MPI_SUCCESS; } + return retval; } -void smpi_datatype_create(MPI_Datatype* new_type, int size, int flags){ +/* + * Copies noncontiguous data into contiguous memory. + * @param contiguous_vector - output vector + * @param noncontiguous_vector - input vector + * @param type - pointer contening : + * - stride - stride of between noncontiguous data + * - block_length - the width or height of blocked matrix + * - count - the number of rows of matrix + */ +void serialize_vector( const void *noncontiguous_vector, + void *contiguous_vector, + size_t count, + void *type) +{ + s_smpi_mpi_vector_t* type_c = (s_smpi_mpi_vector_t*)type; + int i; + for (i = 0; i < type_c->block_count * count; i++) { + memcpy(contiguous_vector, + noncontiguous_vector, type_c->block_length * type_c->size_oldtype); + contiguous_vector += type_c->block_length*type_c->size_oldtype; + noncontiguous_vector += type_c->block_stride*type_c->size_oldtype; + } +} +/* + * Copies contiguous data into noncontiguous memory. + * @param noncontiguous_vector - output vector + * @param contiguous_vector - input vector + * @param type - pointer contening : + * - stride - stride of between noncontiguous data + * - block_length - the width or height of blocked matrix + * - count - the number of rows of matrix + */ +void unserialize_vector( const void *contiguous_vector, + void *noncontiguous_vector, + size_t count, + void *type) +{ + s_smpi_mpi_vector_t* type_c = (s_smpi_mpi_vector_t*)type; + int i; + for (i = 0; i < type_c->block_count * count; i++) { + memcpy(noncontiguous_vector, + contiguous_vector, type_c->block_length * type_c->size_oldtype); + contiguous_vector += type_c->block_length*type_c->size_oldtype; + noncontiguous_vector += type_c->block_stride*type_c->size_oldtype; + } +} + +/* + * Create a Sub type vector to be able to serialize and unserialize it + * the structre s_smpi_mpi_vector_t is derived from s_smpi_subtype which + * required the functions unserialize and serialize + * + */ +s_smpi_mpi_vector_t* smpi_datatype_vector_create( int block_stride, + int block_length, + int block_count, + MPI_Datatype old_type, + int size_oldtype){ + s_smpi_mpi_vector_t *new_t= xbt_new(s_smpi_mpi_vector_t,1); + new_t->base.serialize = &serialize_vector; + new_t->base.unserialize = &unserialize_vector; + new_t->block_stride = block_stride; + new_t->block_length = block_length; + new_t->block_count = block_count; + new_t->old_type = old_type; + new_t->size_oldtype = size_oldtype; + return new_t; +} + +void smpi_datatype_create(MPI_Datatype* new_type, int size, int has_subtype, + void *struct_type, int flags){ MPI_Datatype new_t= xbt_new(s_smpi_mpi_datatype_t,1); new_t->size=size; + new_t->has_subtype=has_subtype; new_t->lb=0; new_t->ub=size; new_t->flags=flags; + new_t->substruct=struct_type; *new_type = new_t; } @@ -194,9 +263,10 @@ int smpi_datatype_contiguous(int count, MPI_Datatype old_type, MPI_Datatype* new { int retval; if ((old_type->flags & DT_FLAG_COMMITED) != DT_FLAG_COMMITED) { - retval = MPI_ERR_TYPE; + retval = MPI_ERR_TYPE; } else { - smpi_datatype_create(new_type, count * smpi_datatype_size(old_type), DT_FLAG_CONTIGUOUS); + smpi_datatype_create(new_type, count * + smpi_datatype_size(old_type),1,NULL, DT_FLAG_CONTIGUOUS); retval=MPI_SUCCESS; } return retval; @@ -205,12 +275,33 @@ int smpi_datatype_contiguous(int count, MPI_Datatype old_type, MPI_Datatype* new int smpi_datatype_vector(int count, int blocklen, int stride, MPI_Datatype old_type, MPI_Datatype* new_type) { int retval; - if (blocklen<=0)return MPI_ERR_ARG; + if (blocklen<=0) return MPI_ERR_ARG; if ((old_type->flags & DT_FLAG_COMMITED) != DT_FLAG_COMMITED) { - retval = MPI_ERR_TYPE; + retval = MPI_ERR_TYPE; } else { - smpi_datatype_create(new_type, count * (blocklen+stride) * smpi_datatype_size(old_type), DT_FLAG_VECTOR); - retval=MPI_SUCCESS; + if(stride != blocklen){ + s_smpi_mpi_vector_t* subtype = smpi_datatype_vector_create( stride, + blocklen, + count, + old_type, + smpi_datatype_size(old_type)); + + smpi_datatype_create(new_type, count * (blocklen) * + smpi_datatype_size(old_type), + 1, + subtype, + DT_FLAG_VECTOR); + retval=MPI_SUCCESS; + }else{ + /* in this situation the data are contignous thus it's not + * required to serialize and unserialize it*/ + smpi_datatype_create(new_type, count * (blocklen) * + smpi_datatype_size(old_type), + 0, + NULL, + DT_FLAG_VECTOR); + retval=MPI_SUCCESS; + } } return retval; } @@ -218,11 +309,18 @@ int smpi_datatype_vector(int count, int blocklen, int stride, MPI_Datatype old_t int smpi_datatype_hvector(int count, int blocklen, MPI_Aint stride, MPI_Datatype old_type, MPI_Datatype* new_type) { int retval; - if (blocklen<=0)return MPI_ERR_ARG; + if (blocklen<=0) return MPI_ERR_ARG; if ((old_type->flags & DT_FLAG_COMMITED) != DT_FLAG_COMMITED) { - retval = MPI_ERR_TYPE; + retval = MPI_ERR_TYPE; } else { - smpi_datatype_create(new_type, count * ((blocklen * smpi_datatype_size(old_type))+stride), DT_FLAG_VECTOR); + /*FIXME: as for the vector the data should be serialized and + * unserialized moreover a structure derived from s_smpi_subtype should + * be created*/ + smpi_datatype_create(new_type, count * ((blocklen * + smpi_datatype_size(old_type))+stride), + 0, + NULL, + DT_FLAG_VECTOR); retval=MPI_SUCCESS; } return retval; @@ -233,14 +331,20 @@ int smpi_datatype_indexed(int count, int* blocklens, int* indices, MPI_Datatype { int i; int retval; + int size = 0; for(i=0; i< count; i++){ if (blocklens[i]<=0) return MPI_ERR_ARG; + size += blocklens[i]; } if ((old_type->flags & DT_FLAG_COMMITED) != DT_FLAG_COMMITED) { - retval = MPI_ERR_TYPE; + retval = MPI_ERR_TYPE; } else { - smpi_datatype_create(new_type, (blocklens[count-1] + indices[count-1]) * smpi_datatype_size(old_type), DT_FLAG_DATA); + /*FIXME: as for the vector the data should be serialized and + * unserialized moreover a structure derived from s_smpi_subtype should + * be created*/ + smpi_datatype_create(new_type, (size) * + smpi_datatype_size(old_type),0, NULL, DT_FLAG_DATA); retval=MPI_SUCCESS; } return retval; @@ -250,14 +354,19 @@ int smpi_datatype_hindexed(int count, int* blocklens, MPI_Aint* indices, MPI_Dat { int i; int retval; + int size = 0; for(i=0; i< count; i++){ if (blocklens[i]<=0) return MPI_ERR_ARG; + size += blocklens[i]; } if ((old_type->flags & DT_FLAG_COMMITED) != DT_FLAG_COMMITED) { - retval = MPI_ERR_TYPE; + retval = MPI_ERR_TYPE; } else { - smpi_datatype_create(new_type,indices[count-1] + (blocklens[count-1] * smpi_datatype_size(old_type)), DT_FLAG_DATA); + /*FIXME: as for the vector the data should be serialized and + * unserialized moreover a structure derived from s_smpi_subtype should + * be created*/ + smpi_datatype_create(new_type,(size * smpi_datatype_size(old_type)), 0,NULL, DT_FLAG_DATA); retval=MPI_SUCCESS; } return retval; @@ -266,19 +375,27 @@ int smpi_datatype_hindexed(int count, int* blocklens, MPI_Aint* indices, MPI_Dat int smpi_datatype_struct(int count, int* blocklens, MPI_Aint* indices, MPI_Datatype* old_types, MPI_Datatype* new_type) { int i; + size_t size; //Khalid added this + for(i=0; i< count; i++){ if (blocklens[i]<=0) return MPI_ERR_ARG; if ((old_types[i]->flags & DT_FLAG_COMMITED) != DT_FLAG_COMMITED) return MPI_ERR_TYPE; + size += blocklens[i]*smpi_datatype_size(old_types[i]); } - smpi_datatype_create(new_type,indices[count-1] + (blocklens[count-1] * smpi_datatype_size(old_types[count-1])), DT_FLAG_DATA); + /*FIXME: as for the vector the data should be serialized and + * unserialized moreover a structure derived from s_smpi_subtype should + * be created*/ + smpi_datatype_create(new_type, size, + 0, NULL, + DT_FLAG_DATA); return MPI_SUCCESS; } -void smpi_datatype_commit(MPI_Datatype* datatype) +void smpi_datatype_commit(MPI_Datatype *datatype) { - (*datatype)->flags= ( (*datatype)->flags | DT_FLAG_COMMITED); + (*datatype)->flags= ((*datatype)->flags | DT_FLAG_COMMITED); } typedef struct s_smpi_mpi_op { @@ -300,14 +417,14 @@ typedef struct s_smpi_mpi_op { //TODO : MINLOC & MAXLOC #define APPLY_FUNC(a, b, length, type, func) \ - { \ - int i; \ - type* x = (type*)(a); \ - type* y = (type*)(b); \ - for(i = 0; i < *(length); i++) { \ - func(x[i], y[i]); \ - } \ - } +{ \ + int i; \ + type* x = (type*)(a); \ + type* y = (type*)(b); \ + for(i = 0; i < *(length); i++) { \ + func(x[i], y[i]); \ + } \ +} static void max_func(void *a, void *b, int *length, MPI_Datatype * datatype) @@ -597,7 +714,7 @@ static void maxloc_func(void *a, void *b, int *length, #define CREATE_MPI_OP(name, func) \ static s_smpi_mpi_op_t mpi_##name = { &(func) /* func */ }; \ - MPI_Op name = &mpi_##name; +MPI_Op name = &mpi_##name; CREATE_MPI_OP(MPI_MAX, max_func); CREATE_MPI_OP(MPI_MIN, min_func);