From 1f97075c250e334b3d744aadff6dee322c24c408 Mon Sep 17 00:00:00 2001 From: Augustin Degomme Date: Mon, 26 Aug 2013 14:28:04 +0200 Subject: [PATCH] add mpich's allgatherv ring algo, as the one from star-mpi performs terribly --- buildtools/Cmake/AddTests.cmake | 2 +- buildtools/Cmake/DefinePackages.cmake | 1 + src/smpi/colls/allgatherv-mpich-ring.c | 121 +++++++++++++++++++++++++ src/smpi/colls/colls.h | 1 + 4 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 src/smpi/colls/allgatherv-mpich-ring.c diff --git a/buildtools/Cmake/AddTests.cmake b/buildtools/Cmake/AddTests.cmake index b276f09b18..eb4ea1d7a7 100644 --- a/buildtools/Cmake/AddTests.cmake +++ b/buildtools/Cmake/AddTests.cmake @@ -401,7 +401,7 @@ if(NOT enable_memcheck) ADD_TEST(smpi-allgather-coll-${ALLGATHER_COLL} ${CMAKE_BINARY_DIR}/bin/tesh ${TESH_OPTION} --cfg smpi/allgather:${ALLGATHER_COLL} --cd ${CMAKE_BINARY_DIR}/teshsuite/smpi ${CMAKE_HOME_DIRECTORY}/teshsuite/smpi/allgather_coll.tesh) ENDFOREACH() - FOREACH (ALLGATHERV_COLL default GB pair ring ompi mpich ompi_neighborexchange ompi_bruck mpich_rdb) + FOREACH (ALLGATHERV_COLL default GB pair ring ompi mpich ompi_neighborexchange ompi_bruck mpich_rdb mpich_ring) ADD_TEST(smpi-allgatherv-coll-${ALLGATHERV_COLL} ${CMAKE_BINARY_DIR}/bin/tesh ${TESH_OPTION} --cfg smpi/allgatherv:${ALLGATHERV_COLL} --cd ${CMAKE_BINARY_DIR}/teshsuite/smpi ${CMAKE_HOME_DIRECTORY}/teshsuite/smpi/allgatherv_coll.tesh) ENDFOREACH() diff --git a/buildtools/Cmake/DefinePackages.cmake b/buildtools/Cmake/DefinePackages.cmake index dab01cd7bf..a32ce0c907 100644 --- a/buildtools/Cmake/DefinePackages.cmake +++ b/buildtools/Cmake/DefinePackages.cmake @@ -137,6 +137,7 @@ set(SMPI_SRC src/smpi/colls/allgatherv-ompi-neighborexchange.c src/smpi/colls/allgatherv-ompi-bruck.c src/smpi/colls/allgatherv-mpich-rdb.c + src/smpi/colls/allgatherv-mpich-ring.c src/smpi/colls/allreduce-lr.c src/smpi/colls/allreduce-NTS.c src/smpi/colls/allreduce-rab1.c diff --git a/src/smpi/colls/allgatherv-mpich-ring.c b/src/smpi/colls/allgatherv-mpich-ring.c new file mode 100644 index 0000000000..aeacb3f157 --- /dev/null +++ b/src/smpi/colls/allgatherv-mpich-ring.c @@ -0,0 +1,121 @@ +#include "colls_private.h" + +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * + * (C) 2001 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +/***************************************************************************** + * Function: allgather_mpich_ring + * return: int + * inputs: + * send_buff: send input buffer + * send_count: number of elements to send + * send_type: data type of elements being sent + * recv_buff: receive output buffer + * recv_count: number of elements to received + * recv_type: data type of elements being received + * comm: communication + ****************************************************************************/ +int +smpi_coll_tuned_allgatherv_mpich_ring(void *sendbuf, int sendcount, + MPI_Datatype send_type, void *recvbuf, + int *recvcounts, int *displs, MPI_Datatype recvtype, + MPI_Comm comm) +{ + + char * sbuf = NULL, * rbuf = NULL; + int soffset, roffset; + int torecv=0, tosend=0, min, rank, comm_size; + int sendnow, recvnow; + int sidx, ridx; + MPI_Status status; + MPI_Aint recvtype_extent; + int right, left, total_count, i; + rank= smpi_comm_rank(comm); + comm_size=smpi_comm_size(comm); + + recvtype_extent= smpi_datatype_get_extent( recvtype); + total_count = 0; + for (i=0; i recvcounts[i]) + min = recvcounts[i]; + if (min * recvtype_extent < 32768) + min = 32768 / recvtype_extent; + /* Handle the case where the datatype extent is larger than + * the pipeline size. */ + if (!min) + min = 1; + + sidx = rank; + ridx = left; + soffset = 0; + roffset = 0; + while (tosend || torecv) { /* While we have data to send or receive */ + sendnow = ((recvcounts[sidx] - soffset) > min) ? min : (recvcounts[sidx] - soffset); + recvnow = ((recvcounts[ridx] - roffset) > min) ? min : (recvcounts[ridx] - roffset); + sbuf = (char *)recvbuf + ((displs[sidx] + soffset) * recvtype_extent); + rbuf = (char *)recvbuf + ((displs[ridx] + roffset) * recvtype_extent); + + /* Protect against wrap-around of indices */ + if (!tosend) + sendnow = 0; + if (!torecv) + recvnow = 0; + + /* Communicate */ + if (!sendnow && !recvnow) { + /* Don't do anything. This case is possible if two + * consecutive processes contribute 0 bytes each. */ + } + else if (!sendnow) { /* If there's no data to send, just do a recv call */ + smpi_mpi_recv(rbuf, recvnow, recvtype, left, COLL_TAG_ALLGATHERV, comm, &status); + + torecv -= recvnow; + } + else if (!recvnow) { /* If there's no data to receive, just do a send call */ + smpi_mpi_send(sbuf, sendnow, recvtype, right, COLL_TAG_ALLGATHERV, comm); + + tosend -= sendnow; + } + else { /* There's data to be sent and received */ + smpi_mpi_sendrecv(sbuf, sendnow, recvtype, right, COLL_TAG_ALLGATHERV, + rbuf, recvnow, recvtype, left, COLL_TAG_ALLGATHERV, + comm, &status); + tosend -= sendnow; + torecv -= recvnow; + } + + soffset += sendnow; + roffset += recvnow; + if (soffset == recvcounts[sidx]) { + soffset = 0; + sidx = (sidx + comm_size - 1) % comm_size; + } + if (roffset == recvcounts[ridx]) { + roffset = 0; + ridx = (ridx + comm_size - 1) % comm_size; + } + } + + return MPI_SUCCESS; +} diff --git a/src/smpi/colls/colls.h b/src/smpi/colls/colls.h index a2091b10e2..dce6456e73 100644 --- a/src/smpi/colls/colls.h +++ b/src/smpi/colls/colls.h @@ -89,6 +89,7 @@ COLL_APPLY(action, COLL_ALLGATHERV_SIG, ompi_neighborexchange) COLL_sep \ COLL_APPLY(action, COLL_ALLGATHERV_SIG, ompi_bruck) COLL_sep \ COLL_APPLY(action, COLL_ALLGATHERV_SIG, mpich) COLL_sep \ COLL_APPLY(action, COLL_ALLGATHERV_SIG, mpich_rdb) COLL_sep \ +COLL_APPLY(action, COLL_ALLGATHERV_SIG, mpich_ring) COLL_sep \ COLL_APPLY(action, COLL_ALLGATHERV_SIG, automatic) COLL_ALLGATHERVS(COLL_PROTO, COLL_NOsep) -- 2.20.1