X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/3c1eda61bc5b0d9c846abd9bb53b0bef26cb2880..57e7dcd49d4873f73dc0337e4f172bd169f77a44:/src/smpi/smpi_coll.c diff --git a/src/smpi/smpi_coll.c b/src/smpi/smpi_coll.c index 874d32a48d..b3c9b3ae27 100644 --- a/src/smpi/smpi_coll.c +++ b/src/smpi/smpi_coll.c @@ -1,12 +1,10 @@ -/* $Id$tag */ - /* smpi_coll.c -- various optimized routing for collectives */ -/* Copyright (c) 2009 Stephane Genaud. */ -/* All rights reserved. */ +/* Copyright (c) 2009, 2010. The SimGrid Team. + * All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it - * * under the terms of the license (GNU LGPL) which comes with this package. */ + * under the terms of the license (GNU LGPL) which comes with this package. */ #include #include @@ -21,18 +19,19 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_coll, smpi, struct s_proc_tree { int PROCTREE_A; int numChildren; - int * child; + int *child; int parent; int me; int root; int isRoot; }; -typedef struct s_proc_tree * proc_tree_t; +typedef struct s_proc_tree *proc_tree_t; /** * alloc and init **/ -static proc_tree_t alloc_tree(int arity) { +static proc_tree_t alloc_tree(int arity) +{ proc_tree_t tree; int i; @@ -41,7 +40,7 @@ static proc_tree_t alloc_tree(int arity) { tree->isRoot = 0; tree->numChildren = 0; tree->child = xbt_new(int, arity); - for(i = 0; i < arity; i++) { + for (i = 0; i < arity; i++) { tree->child[i] = -1; } tree->root = -1; @@ -52,8 +51,9 @@ static proc_tree_t alloc_tree(int arity) { /** * free **/ -static void free_tree(proc_tree_t tree) { - xbt_free(tree->child ); +static void free_tree(proc_tree_t tree) +{ + xbt_free(tree->child); xbt_free(tree); } @@ -62,22 +62,23 @@ static void free_tree(proc_tree_t tree) { * @param index the rank of the calling process * @param extent the total number of processes **/ -static void build_tree(int index, int extent, proc_tree_t* tree) { +static void build_tree(int index, int extent, proc_tree_t * tree) +{ int places = (*tree)->PROCTREE_A * index; int i, ch, pr; (*tree)->me = index; - (*tree)->root = 0 ; - for(i = 1; i <= (*tree)->PROCTREE_A; i++) { + (*tree)->root = 0; + for (i = 1; i <= (*tree)->PROCTREE_A; i++) { ++places; ch = (*tree)->PROCTREE_A * index + i + (*tree)->root; ch %= extent; - if(places < extent) { + if (places < extent) { (*tree)->child[i - 1] = ch; (*tree)->numChildren++; } } - if(index == (*tree)->root) { + if (index == (*tree)->root) { (*tree)->isRoot = 1; } else { (*tree)->isRoot = 0; @@ -89,27 +90,34 @@ static void build_tree(int index, int extent, proc_tree_t* tree) { /** * bcast **/ -static void tree_bcast(void* buf, int count, MPI_Datatype datatype, int root, MPI_Comm comm, proc_tree_t tree) { - int system_tag = 999; // used negative int but smpi_create_request() declares this illegal (to be checked) +static void tree_bcast(void *buf, int count, MPI_Datatype datatype, + int root, MPI_Comm comm, proc_tree_t tree) +{ + int system_tag = 999; // used negative int but smpi_create_request() declares this illegal (to be checked) int rank, i; - MPI_Request* requests; + MPI_Request *requests; rank = smpi_comm_rank(comm); /* wait for data from my parent in the tree */ - if(!tree->isRoot) { + if (!tree->isRoot) { DEBUG3("<%d> tree_bcast(): i am not root: recv from %d, tag=%d)", rank, tree->parent, system_tag + rank); - smpi_mpi_recv(buf, count, datatype, tree->parent, system_tag + rank, comm, MPI_STATUS_IGNORE); + smpi_mpi_recv(buf, count, datatype, tree->parent, system_tag + rank, + comm, MPI_STATUS_IGNORE); } requests = xbt_new(MPI_Request, tree->numChildren); - DEBUG2("<%d> creates %d requests (1 per child)", rank, tree->numChildren); + DEBUG2("<%d> creates %d requests (1 per child)", rank, + tree->numChildren); /* iniates sends to ranks lower in the tree */ - for(i = 0; i < tree->numChildren; i++) { - if(tree->child[i] == -1) { + for (i = 0; i < tree->numChildren; i++) { + if (tree->child[i] == -1) { requests[i] = MPI_REQUEST_NULL; } else { - DEBUG3("<%d> send to <%d>, tag=%d", rank, tree->child[i], system_tag + tree->child[i]); - requests[i] = smpi_isend_init(buf, count, datatype, tree->child[i], system_tag + tree->child[i], comm); + DEBUG3("<%d> send to <%d>, tag=%d", rank, tree->child[i], + system_tag + tree->child[i]); + requests[i] = + smpi_isend_init(buf, count, datatype, tree->child[i], + system_tag + tree->child[i], comm); } } smpi_mpi_startall(tree->numChildren, requests); @@ -120,27 +128,34 @@ static void tree_bcast(void* buf, int count, MPI_Datatype datatype, int root, MP /** * anti-bcast **/ -static void tree_antibcast(void* buf, int count, MPI_Datatype datatype, int root, MPI_Comm comm, proc_tree_t tree) { - int system_tag = 999; // used negative int but smpi_create_request() declares this illegal (to be checked) +static void tree_antibcast(void *buf, int count, MPI_Datatype datatype, + int root, MPI_Comm comm, proc_tree_t tree) +{ + int system_tag = 999; // used negative int but smpi_create_request() declares this illegal (to be checked) int rank, i; - MPI_Request* requests; + MPI_Request *requests; rank = smpi_comm_rank(comm); // everyone sends to its parent, except root. - if(!tree->isRoot) { + if (!tree->isRoot) { DEBUG3("<%d> tree_antibcast(): i am not root: send to %d, tag=%d)", rank, tree->parent, system_tag + rank); - smpi_mpi_send(buf, count, datatype, tree->parent, system_tag + rank, comm); + smpi_mpi_send(buf, count, datatype, tree->parent, system_tag + rank, + comm); } //every one receives as many messages as it has children requests = xbt_new(MPI_Request, tree->numChildren); - DEBUG2("<%d> creates %d requests (1 per child)", rank, tree->numChildren); - for(i = 0; i < tree->numChildren; i++) { - if(tree->child[i] == -1) { + DEBUG2("<%d> creates %d requests (1 per child)", rank, + tree->numChildren); + for (i = 0; i < tree->numChildren; i++) { + if (tree->child[i] == -1) { requests[i] = MPI_REQUEST_NULL; } else { - DEBUG3("<%d> recv from <%d>, tag=%d", rank, tree->child[i], system_tag + tree->child[i]); - requests[i] = smpi_irecv_init(buf, count, datatype, tree->child[i], system_tag + tree->child[i], comm); + DEBUG3("<%d> recv from <%d>, tag=%d", rank, tree->child[i], + system_tag + tree->child[i]); + requests[i] = + smpi_irecv_init(buf, count, datatype, tree->child[i], + system_tag + tree->child[i], comm); } } smpi_mpi_startall(tree->numChildren, requests); @@ -151,7 +166,9 @@ static void tree_antibcast(void* buf, int count, MPI_Datatype datatype, int root /** * bcast with a binary, ternary, or whatever tree .. **/ -void nary_tree_bcast(void* buf, int count, MPI_Datatype datatype, int root, MPI_Comm comm, int arity) { +void nary_tree_bcast(void *buf, int count, MPI_Datatype datatype, int root, + MPI_Comm comm, int arity) +{ proc_tree_t tree = alloc_tree(arity); int rank, size; @@ -165,10 +182,11 @@ void nary_tree_bcast(void* buf, int count, MPI_Datatype datatype, int root, MPI_ /** * barrier with a binary, ternary, or whatever tree .. **/ -void nary_tree_barrier(MPI_Comm comm, int arity) { - proc_tree_t tree = alloc_tree( arity ); +void nary_tree_barrier(MPI_Comm comm, int arity) +{ + proc_tree_t tree = alloc_tree(arity); int rank, size; - char dummy='$'; + char dummy = '$'; rank = smpi_comm_rank(comm); size = smpi_comm_size(comm); @@ -183,11 +201,17 @@ void nary_tree_barrier(MPI_Comm comm, int arity) { * * Openmpi calls this routine when the message size sent to each rank < 2000 bytes and size < 12 **/ -int smpi_coll_tuned_alltoall_bruck(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) { +int smpi_coll_tuned_alltoall_bruck(void *sendbuf, int sendcount, + MPI_Datatype sendtype, void *recvbuf, + int recvcount, MPI_Datatype recvtype, + MPI_Comm comm) +{ int system_tag = 777; int i, rank, size, err, count; - MPI_Aint lb, sendextent, recvextent; - MPI_Request* requests; + MPI_Aint lb; + MPI_Aint sendextent = 0; + MPI_Aint recvextent = 0; + MPI_Request *requests; // FIXME: check implementation rank = smpi_comm_rank(comm); @@ -196,30 +220,39 @@ int smpi_coll_tuned_alltoall_bruck(void* sendbuf, int sendcount, MPI_Datatype se err = smpi_datatype_extent(sendtype, &lb, &sendextent); err = smpi_datatype_extent(recvtype, &lb, &recvextent); /* Local copy from self */ - err = smpi_datatype_copy(&((char*)sendbuf)[rank * sendextent], sendcount, sendtype, &((char*)recvbuf)[rank * recvextent], recvcount, recvtype); - if(err == MPI_SUCCESS && size > 1) { + err = + smpi_datatype_copy(&((char *) sendbuf)[rank * sendextent], sendcount, + sendtype, &((char *) recvbuf)[rank * recvextent], + recvcount, recvtype); + if (err == MPI_SUCCESS && size > 1) { /* Initiate all send/recv to/from others. */ requests = xbt_new(MPI_Request, 2 * (size - 1)); count = 0; /* Create all receives that will be posted first */ - for(i = 0; i < size; ++i) { - if(i == rank) { - DEBUG3("<%d> skip request creation [src = %d, recvcount = %d]", rank, i, recvcount); + for (i = 0; i < size; ++i) { + if (i == rank) { + DEBUG3("<%d> skip request creation [src = %d, recvcount = %d]", + rank, i, recvcount); continue; } - requests[count] = smpi_irecv_init(&((char*)recvbuf)[i * recvextent], recvcount, recvtype, i, system_tag, comm); + requests[count] = + smpi_irecv_init(&((char *) recvbuf)[i * recvextent], recvcount, + recvtype, i, system_tag, comm); count++; } /* Now create all sends */ - for(i = 0; i < size; ++i) { - if(i == rank) { - DEBUG3("<%d> skip request creation [dst = %d, sendcount = %d]", rank, i, sendcount); + for (i = 0; i < size; ++i) { + if (i == rank) { + DEBUG3("<%d> skip request creation [dst = %d, sendcount = %d]", + rank, i, sendcount); continue; } - requests[count] = smpi_isend_init(&((char*)sendbuf)[i * sendextent], sendcount, sendtype, i, system_tag, comm); + requests[count] = + smpi_isend_init(&((char *) sendbuf)[i * sendextent], sendcount, + sendtype, i, system_tag, comm); count++; } - /* Wait for them all.*/ + /* Wait for them all. */ smpi_mpi_startall(count, requests); DEBUG2("<%d> wait for %d requests", rank, count); smpi_mpi_waitall(count, requests, MPI_STATUS_IGNORE); @@ -231,10 +264,17 @@ int smpi_coll_tuned_alltoall_bruck(void* sendbuf, int sendcount, MPI_Datatype se /** * Alltoall basic_linear **/ -int smpi_coll_tuned_alltoall_basic_linear(void *sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) { +int smpi_coll_tuned_alltoall_basic_linear(void *sendbuf, int sendcount, + MPI_Datatype sendtype, + void *recvbuf, int recvcount, + MPI_Datatype recvtype, + MPI_Comm comm) +{ int system_tag = 888; int i, rank, size, err, count; - MPI_Aint lb, sendinc, recvinc; + MPI_Aint lb; + MPI_Aint sendinc = 0; + MPI_Aint recvinc = 0; MPI_Request *requests; /* Initialize. */ @@ -246,14 +286,19 @@ int smpi_coll_tuned_alltoall_basic_linear(void *sendbuf, int sendcount, MPI_Data sendinc *= sendcount; recvinc *= recvcount; /* simple optimization */ - err = smpi_datatype_copy(&((char*)sendbuf)[rank * sendinc], sendcount, sendtype, &((char*)recvbuf)[rank * recvinc], recvcount, recvtype); - if(err == MPI_SUCCESS && size > 1) { + err = + smpi_datatype_copy(&((char *) sendbuf)[rank * sendinc], sendcount, + sendtype, &((char *) recvbuf)[rank * recvinc], + recvcount, recvtype); + if (err == MPI_SUCCESS && size > 1) { /* Initiate all send/recv to/from others. */ requests = xbt_new(MPI_Request, 2 * (size - 1)); /* Post all receives first -- a simple optimization */ count = 0; - for(i = (rank + 1) % size; i != rank; i = (i + 1) % size) { - requests[count] = smpi_irecv_init(&((char*)recvbuf)[i * recvinc], recvcount, recvtype, i, system_tag, comm); + for (i = (rank + 1) % size; i != rank; i = (i + 1) % size) { + requests[count] = + smpi_irecv_init(&((char *) recvbuf)[i * recvinc], recvcount, + recvtype, i, system_tag, comm); count++; } /* Now post all sends in reverse order @@ -261,11 +306,14 @@ int smpi_coll_tuned_alltoall_basic_linear(void *sendbuf, int sendcount, MPI_Data * when messages actually arrive in the order in which they were posted. * TODO: check the previous assertion */ - for(i = (rank + size - 1) % size; i != rank; i = (i + size - 1) % size ) { - requests[count] = smpi_isend_init(&((char*)sendbuf)[i * sendinc], sendcount, sendtype, i, system_tag, comm); + for (i = (rank + size - 1) % size; i != rank; + i = (i + size - 1) % size) { + requests[count] = + smpi_isend_init(&((char *) sendbuf)[i * sendinc], sendcount, + sendtype, i, system_tag, comm); count++; } - /* Wait for them all.*/ + /* Wait for them all. */ smpi_mpi_startall(count, requests); DEBUG2("<%d> wait for %d requests", rank, count); smpi_mpi_waitall(count, requests, MPI_STATUS_IGNORE); @@ -284,7 +332,11 @@ int smpi_coll_tuned_alltoall_basic_linear(void *sendbuf, int sendcount, MPI_Data * .... * Openmpi calls this routine when the message size sent to each rank is greater than 3000 bytes **/ -int smpi_coll_tuned_alltoall_pairwise(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) { +int smpi_coll_tuned_alltoall_pairwise(void *sendbuf, int sendcount, + MPI_Datatype sendtype, void *recvbuf, + int recvcount, MPI_Datatype recvtype, + MPI_Comm comm) +{ int system_tag = 999; int rank, size, step, sendto, recvfrom, sendsize, recvsize; @@ -294,21 +346,32 @@ int smpi_coll_tuned_alltoall_pairwise(void* sendbuf, int sendcount, MPI_Datatype sendsize = smpi_datatype_size(sendtype); recvsize = smpi_datatype_size(recvtype); /* Perform pairwise exchange - starting from 1 so the local copy is last */ - for(step = 1; step < size + 1; step++) { + for (step = 1; step < size + 1; step++) { /* who do we talk to in this step? */ - sendto = (rank+step)%size; - recvfrom = (rank+size-step)%size; + sendto = (rank + step) % size; + recvfrom = (rank + size - step) % size; /* send and receive */ - smpi_mpi_sendrecv(&((char*)sendbuf)[sendto * sendsize * sendcount], sendcount, sendtype, sendto, system_tag, &((char*)recvbuf)[recvfrom * recvsize * recvcount], recvcount, recvtype, recvfrom, system_tag, comm, MPI_STATUS_IGNORE); + smpi_mpi_sendrecv(&((char *) sendbuf)[sendto * sendsize * sendcount], + sendcount, sendtype, sendto, system_tag, + &((char *) recvbuf)[recvfrom * recvsize * recvcount], + recvcount, recvtype, recvfrom, system_tag, comm, + MPI_STATUS_IGNORE); } return MPI_SUCCESS; } -int smpi_coll_basic_alltoallv(void* sendbuf, int* sendcounts, int* senddisps, MPI_Datatype sendtype, void* recvbuf, int *recvcounts, int* recvdisps, MPI_Datatype recvtype, MPI_Comm comm) { +int smpi_coll_basic_alltoallv(void *sendbuf, int *sendcounts, + int *senddisps, MPI_Datatype sendtype, + void *recvbuf, int *recvcounts, + int *recvdisps, MPI_Datatype recvtype, + MPI_Comm comm) +{ int system_tag = 889; int i, rank, size, err, count; - MPI_Aint lb, sendextent, recvextent; - MPI_Request* requests; + MPI_Aint lb; + MPI_Aint sendextent = 0; + MPI_Aint recvextent = 0; + MPI_Request *requests; /* Initialize. */ rank = smpi_comm_rank(comm); @@ -317,30 +380,42 @@ int smpi_coll_basic_alltoallv(void* sendbuf, int* sendcounts, int* senddisps, MP err = smpi_datatype_extent(sendtype, &lb, &sendextent); err = smpi_datatype_extent(recvtype, &lb, &recvextent); /* Local copy from self */ - err = smpi_datatype_copy(&((char*)sendbuf)[senddisps[rank] * sendextent], sendcounts[rank], sendtype, &((char*)recvbuf)[recvdisps[rank] * recvextent], recvcounts[rank], recvtype); - if(err == MPI_SUCCESS && size > 1) { + err = + smpi_datatype_copy(&((char *) sendbuf)[senddisps[rank] * sendextent], + sendcounts[rank], sendtype, + &((char *) recvbuf)[recvdisps[rank] * recvextent], + recvcounts[rank], recvtype); + if (err == MPI_SUCCESS && size > 1) { /* Initiate all send/recv to/from others. */ requests = xbt_new(MPI_Request, 2 * (size - 1)); count = 0; /* Create all receives that will be posted first */ - for(i = 0; i < size; ++i) { - if(i == rank || recvcounts[i] == 0) { - DEBUG3("<%d> skip request creation [src = %d, recvcounts[src] = %d]", rank, i, recvcounts[i]); + for (i = 0; i < size; ++i) { + if (i == rank || recvcounts[i] == 0) { + DEBUG3 + ("<%d> skip request creation [src = %d, recvcounts[src] = %d]", + rank, i, recvcounts[i]); continue; } - requests[count] = smpi_irecv_init(&((char*)recvbuf)[recvdisps[i] * recvextent], recvcounts[i], recvtype, i, system_tag, comm); + requests[count] = + smpi_irecv_init(&((char *) recvbuf)[recvdisps[i] * recvextent], + recvcounts[i], recvtype, i, system_tag, comm); count++; } /* Now create all sends */ - for(i = 0; i < size; ++i) { - if(i == rank || sendcounts[i] == 0) { - DEBUG3("<%d> skip request creation [dst = %d, sendcounts[dst] = %d]", rank, i, sendcounts[i]); + for (i = 0; i < size; ++i) { + if (i == rank || sendcounts[i] == 0) { + DEBUG3 + ("<%d> skip request creation [dst = %d, sendcounts[dst] = %d]", + rank, i, sendcounts[i]); continue; } - requests[count] = smpi_isend_init(&((char*)sendbuf)[senddisps[i] * sendextent], sendcounts[i], sendtype, i, system_tag, comm); + requests[count] = + smpi_isend_init(&((char *) sendbuf)[senddisps[i] * sendextent], + sendcounts[i], sendtype, i, system_tag, comm); count++; } - /* Wait for them all.*/ + /* Wait for them all. */ smpi_mpi_startall(count, requests); DEBUG2("<%d> wait for %d requests", rank, count); smpi_mpi_waitall(count, requests, MPI_STATUS_IGNORE);