src/smpi/colls/reduce/reduce-mvapich-knomial.cpp

   1 /* Copyright (c) 2013-2019. The SimGrid Team.
   2  * All rights reserved.                                                     */
   3
   4 /* This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the license (GNU LGPL) which comes with this package. */
   6
   7 /*
   8  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   9  *                         University Research and Technology
  10  *                         Corporation.  All rights reserved.
  11  * Copyright (c) 2004-2012 The University of Tennessee and The University
  12  *                         of Tennessee Research Foundation.  All rights
  13  *                         reserved.
  14  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  15  *                         University of Stuttgart.  All rights reserved.
  16  * Copyright (c) 2004-2005 The Regents of the University of California.
  17  *                         All rights reserved.
  18  * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
  19  * Copyright (c) 2009      University of Houston. All rights reserved.
  20  *
  21  * Additional copyrights may follow
  22  */
  23
  24 /*
  25  *
  26  *  (C) 2001 by Argonne National Laboratory.
  27  *      See COPYRIGHT in top-level directory.
  28  */
  29 /* Copyright (c) 2001-2014, The Ohio State University. All rights
  30  * reserved.
  31  *
  32  * This file is part of the MVAPICH2 software package developed by the
  33  * team members of The Ohio State University's Network-Based Computing
  34  * Laboratory (NBCL), headed by Professor Dhabaleswar K. (DK) Panda.
  35  *
  36  * For detailed copyright and licensing information, please refer to the
  37  * copyright file COPYRIGHT in the top level MVAPICH2 directory.
  38  *
  39  */
  40
  41 #include "../colls_private.hpp"
  42 #include <algorithm>
  43
  44 extern int mv2_reduce_intra_knomial_factor;
  45 extern int mv2_reduce_inter_knomial_factor;
  46
  47 #define SMPI_DEFAULT_KNOMIAL_FACTOR 4
  48
  49 //        int mv2_reduce_knomial_factor = 2;
  50
  51
  52
  53 static int MPIR_Reduce_knomial_trace(int root, int reduce_knomial_factor,
  54         MPI_Comm comm, int *dst, int *expected_send_count,
  55         int *expected_recv_count, int **src_array)
  56 {
  57     int mask=0x1, k, comm_size, src, rank, relative_rank, lroot=0;
  58     int orig_mask=0x1;
  59     int recv_iter=0, send_iter=0;
  60     int *knomial_reduce_src_array=NULL;
  61     comm_size =  comm->size();
  62     rank = comm->rank();
  63
  64     lroot = root;
  65     relative_rank = (rank - lroot + comm_size) % comm_size;
  66
  67     /* First compute to whom we need to send data */
  68     while (mask < comm_size) {
  69         if (relative_rank % (reduce_knomial_factor*mask)) {
  70             *dst = relative_rank/(reduce_knomial_factor*mask)*
  71                 (reduce_knomial_factor*mask)+root;
  72             if (*dst >= comm_size) {
  73                 *dst -= comm_size;
  74             }
  75             send_iter++;
  76             break;
  77         }
  78         mask *= reduce_knomial_factor;
  79     }
  80     mask /= reduce_knomial_factor;
  81
  82     /* Now compute how many children we have in the knomial-tree */
  83     orig_mask = mask;
  84     while (mask > 0) {
  85         for(k=1;k<reduce_knomial_factor;k++) {
  86             if (relative_rank + mask*k < comm_size) {
  87                 recv_iter++;
  88             }
  89         }
  90         mask /= reduce_knomial_factor;
  91     }
  92
  93     /* Finally, fill up the src array */
  94     if(recv_iter > 0) {
  95         knomial_reduce_src_array = static_cast<int*>(smpi_get_tmp_sendbuffer(sizeof(int)*recv_iter));
  96     }
  97
  98     mask = orig_mask;
  99     recv_iter=0;
 100     while (mask > 0) {
 101         for(k=1;k<reduce_knomial_factor;k++) {
 102             if (relative_rank + mask*k < comm_size) {
 103                 src = rank + mask*k;
 104                 if (src >= comm_size) {
 105                     src -= comm_size;
 106                 }
 107                 knomial_reduce_src_array[recv_iter++] = src;
 108             }
 109         }
 110         mask /= reduce_knomial_factor;
 111     }
 112
 113     *expected_recv_count = recv_iter;
 114     *expected_send_count = send_iter;
 115     *src_array = knomial_reduce_src_array;
 116     return 0;
 117 }
 118
 119 namespace simgrid{
 120 namespace smpi{
 121 int Coll_reduce_mvapich2_knomial::reduce (
 122         void *sendbuf,
 123         void *recvbuf,
 124         int count,
 125         MPI_Datatype datatype,
 126         MPI_Op op,
 127         int root,
 128         MPI_Comm comm)
 129 {
 130     int mpi_errno = MPI_SUCCESS;
 131     int rank, is_commutative;
 132     int src, k;
 133     MPI_Request send_request;
 134     int index=0;
 135     MPI_Aint true_lb, true_extent, extent;
 136     MPI_Status status;
 137     int recv_iter=0, dst=-1, expected_send_count, expected_recv_count;
 138     int *src_array=NULL;
 139     void **tmp_buf=NULL;
 140     MPI_Request *requests=NULL;
 141
 142
 143     if (count == 0) return MPI_SUCCESS;
 144
 145     rank = comm->rank();
 146
 147     /* Create a temporary buffer */
 148
 149     datatype->extent(&true_lb, &true_extent);
 150     extent = datatype->get_extent();
 151
 152     is_commutative =  (op==MPI_OP_NULL || op->is_commutative());
 153
 154     if (rank != root) {
 155         recvbuf = (void*)smpi_get_tmp_recvbuffer(count * std::max(extent, true_extent));
 156         recvbuf = (void *)((char*)recvbuf - true_lb);
 157     }
 158
 159     if ((rank != root) || (sendbuf != MPI_IN_PLACE)) {
 160         mpi_errno = Datatype::copy(sendbuf, count, datatype, recvbuf,
 161                 count, datatype);
 162     }
 163
 164
 165     if(mv2_reduce_intra_knomial_factor<0)
 166       {
 167         mv2_reduce_intra_knomial_factor = SMPI_DEFAULT_KNOMIAL_FACTOR;
 168       }
 169     if(mv2_reduce_inter_knomial_factor<0)
 170       {
 171         mv2_reduce_inter_knomial_factor = SMPI_DEFAULT_KNOMIAL_FACTOR;
 172       }
 173
 174
 175     MPIR_Reduce_knomial_trace(root, mv2_reduce_intra_knomial_factor, comm,
 176            &dst, &expected_send_count, &expected_recv_count, &src_array);
 177
 178     if(expected_recv_count > 0 ) {
 179         tmp_buf  = static_cast<void**>(xbt_malloc(sizeof(void *)*expected_recv_count));
 180         requests = static_cast<MPI_Request*>(xbt_malloc(sizeof(MPI_Request)*expected_recv_count));
 181         for(k=0; k < expected_recv_count; k++ ) {
 182             tmp_buf[k] = smpi_get_tmp_sendbuffer(count * std::max(extent, true_extent));
 183             tmp_buf[k] = (void *)((char*)tmp_buf[k] - true_lb);
 184         }
 185
 186         while(recv_iter  < expected_recv_count) {
 187             src = src_array[expected_recv_count - (recv_iter+1)];
 188
 189             requests[recv_iter]=Request::irecv (tmp_buf[recv_iter], count, datatype ,src,
 190                     COLL_TAG_REDUCE, comm);
 191             recv_iter++;
 192
 193         }
 194
 195         recv_iter=0;
 196         while(recv_iter < expected_recv_count) {
 197             index=Request::waitany(expected_recv_count, requests,
 198                     &status);
 199             recv_iter++;
 200
 201             if (is_commutative) {
 202               if(op!=MPI_OP_NULL) op->apply( tmp_buf[index], recvbuf, &count, datatype);
 203             }
 204         }
 205
 206         for(k=0; k < expected_recv_count; k++ ) {
 207             smpi_free_tmp_buffer(tmp_buf[k]);
 208         }
 209         xbt_free(tmp_buf);
 210         xbt_free(requests);
 211     }
 212
 213     if(src_array != NULL) {
 214         xbt_free(src_array);
 215     }
 216
 217     if(rank != root) {
 218         send_request=Request::isend(recvbuf,count, datatype, dst,
 219                 COLL_TAG_REDUCE,comm);
 220
 221         Request::waitall(1, &send_request, &status);
 222
 223         smpi_free_tmp_buffer((void *)((char*)recvbuf + true_lb));
 224     }
 225
 226     /* --END ERROR HANDLING-- */
 227
 228     return mpi_errno;
 229 }
 230 }
 231 }