From: genaud Date: Tue, 21 Jul 2009 13:41:23 +0000 (+0000) Subject: * minimum of datatype handling for alltoall tuned X-Git-Tag: SVN~1123 X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/d24c1ddb2db13c24197bb813dc538e480edeb601 * minimum of datatype handling for alltoall tuned => start add files for separate topics, e.g smpi_mpi_dt.{c,h} * started debugging alltoall basic_linear git-svn-id: svn+ssh://scm.gforge.inria.fr/svn/simgrid/simgrid/trunk@6525 48e7efb5-ca39-0410-a469-dd3cf9ba447f --- diff --git a/include/smpi/smpi.h b/include/smpi/smpi.h index f4bc1839a6..2f0df2dc15 100644 --- a/include/smpi/smpi.h +++ b/include/smpi/smpi.h @@ -72,7 +72,7 @@ SG_BEGIN_DECL() #define MPI_COMM_NULL NULL #define MPI_STATUS_IGNORE NULL -#define MPI_Aint ptrdiff_t +#define MPI_Aint ptrdiff_t #define MPI_BYTE (smpi_mpi_global->mpi_byte) #define MPI_CHAR (smpi_mpi_global->mpi_char) @@ -95,6 +95,8 @@ SG_BEGIN_DECL() #define MPI_Comm_rank(a, b) SMPI_MPI_Comm_rank(a, b) #define MPI_Type_size(a, b) SMPI_MPI_Type_size(a, b) #define MPI_Type_get_extent(a, b, c) SMPI_MPI_Type_get_extent(a, b, c) +#define MPI_Type_lb(a, b) SMPI_MPI_Type_lb(a, b) +#define MPI_Type_ub(a, b) SMPI_MPI_Type_ub(a, b) #define MPI_Barrier(a) SMPI_MPI_Barrier(a) #define MPI_Irecv(a, b, c, d, e, f, g) SMPI_MPI_Irecv(a, b, c, d, e, f, g) @@ -120,6 +122,9 @@ XBT_PUBLIC(int) SMPI_MPI_Comm_size(MPI_Comm comm, int *size); XBT_PUBLIC(int) SMPI_MPI_Comm_rank(MPI_Comm comm, int *rank); XBT_PUBLIC(int) SMPI_MPI_Type_size(MPI_Datatype datatype, size_t * size); XBT_PUBLIC(int) SMPI_MPI_Type_get_extent(MPI_Datatype datatype, MPI_Aint* lb, MPI_Aint *extent); +XBT_PUBLIC(int) SMPI_MPI_Type_lb(MPI_Datatype datatype, MPI_Aint* disp); +XBT_PUBLIC(int) SMPI_MPI_Type_ub(MPI_Datatype datatype, MPI_Aint* disp); + XBT_PUBLIC(int) SMPI_MPI_Barrier(MPI_Comm comm); XBT_PUBLIC(int) SMPI_MPI_Irecv(void *buf, int count, MPI_Datatype datatype, diff --git a/src/Makefile.am b/src/Makefile.am index fd675c7a08..3e7d88600b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -82,6 +82,7 @@ EXTRA_DIST= \ \ smpi/private.h \ smpi/smpi_coll_private.h \ + smpi/smpi_mpi_dt_private.h \ smpi/README #LIBRARY_VERSION= 0:0:0 @@ -221,7 +222,8 @@ SMPI_SRC= \ smpi/smpi_sender.c \ smpi/smpi_receiver.c \ smpi/smpi_util.c \ - smpi/smpi_coll.c + smpi/smpi_coll.c \ + smpi/smpi_mpi_dt.c MSG_SRC= msg/msg_config.c \ msg/task.c msg/host.c msg/m_process.c msg/gos.c \ diff --git a/src/smpi/private.h b/src/smpi/private.h index e1f5b9dacc..3ee7f8bcf7 100644 --- a/src/smpi/private.h +++ b/src/smpi/private.h @@ -30,7 +30,7 @@ typedef struct smpi_mpi_datatype_t { ptrdiff_t lb; ptrdiff_t ub; uint16_t flags; /* flags: has it been committed, etc ...*/ - uint16_t id; /* data id, normally the index in the data array. */ + uint16_t id; /* unused so far : data id, normally the index in the data array. */ } s_smpi_mpi_datatype_t; // smpi mpi request diff --git a/src/smpi/smpi_base.c b/src/smpi/smpi_base.c index 7021f117f9..e3b944fd8a 100644 --- a/src/smpi/smpi_base.c +++ b/src/smpi/smpi_base.c @@ -3,11 +3,12 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_base, smpi, "Logging specific to SMPI (base)"); -XBT_LOG_EXTERNAL_CATEGORY(smpi_coll); XBT_LOG_EXTERNAL_CATEGORY(smpi_base); XBT_LOG_EXTERNAL_CATEGORY(smpi_bench); XBT_LOG_EXTERNAL_CATEGORY(smpi_kernel); XBT_LOG_EXTERNAL_CATEGORY(smpi_mpi); +XBT_LOG_EXTERNAL_CATEGORY(smpi_mpi_dt); +XBT_LOG_EXTERNAL_CATEGORY(smpi_coll); XBT_LOG_EXTERNAL_CATEGORY(smpi_receiver); XBT_LOG_EXTERNAL_CATEGORY(smpi_sender); XBT_LOG_EXTERNAL_CATEGORY(smpi_util); @@ -15,21 +16,6 @@ XBT_LOG_EXTERNAL_CATEGORY(smpi_util); smpi_mpi_global_t smpi_mpi_global = NULL; -/** - * Get the lower bound and extent for a Datatype - * - * FIXME: this an incomplete implementation as we do not support yet MPI_Type_commit. - * Hence, this can be called only for primitive type MPI_INT, MPI_DOUBLE, ... - * - * remark: MPI-1 has also the deprecated - * int MPI_Type_extent(MPI_Datatype datatype, *MPI_Aint *extent); - * - **/ -int smpi_mpi_type_get_extent(MPI_Datatype datatype, MPI_Aint *lb, MPI_Aint *extent) { - *extent = datatype->ub - datatype->lb; - return( MPI_SUCCESS ); -} - /** * Operations of MPI_OP : implemented=land,sum,min,max **/ diff --git a/src/smpi/smpi_coll.c b/src/smpi/smpi_coll.c index e5a1ebd2e5..729645386b 100644 --- a/src/smpi/smpi_coll.c +++ b/src/smpi/smpi_coll.c @@ -151,9 +151,7 @@ int tree_bcast( void *buf, int count, MPI_Datatype datatype, int root, /* wait for data from my parent in the tree */ if (!tree->isRoot) { -#ifdef DEBUG_STEPH - printf("[%d] recv(%d from %d, tag=%d)\n",rank,rank, tree->parent, system_tag+rank); -#endif + DEBUG4("[%d] recv(%d from %d, tag=%d)\n",rank,rank, tree->parent, system_tag+rank); retval = smpi_create_request(buf, count, datatype, tree->parent, rank, system_tag + rank, @@ -163,31 +161,23 @@ int tree_bcast( void *buf, int count, MPI_Datatype datatype, int root, rank,retval,__FILE__,__LINE__); } smpi_mpi_irecv(request); -#ifdef DEBUG_STEPH - printf("[%d] waiting on irecv from %d\n",rank , tree->parent); -#endif + DEBUG2("[%d] waiting on irecv from %d\n",rank , tree->parent); smpi_mpi_wait(request, MPI_STATUS_IGNORE); xbt_mallocator_release(smpi_global->request_mallocator, request); } requests = xbt_malloc( tree->numChildren * sizeof(smpi_mpi_request_t)); -#ifdef DEBUG_STEPH - printf("[%d] creates %d requests\n",rank,tree->numChildren); -#endif + DEBUG2("[%d] creates %d requests\n",rank,tree->numChildren); /* iniates sends to ranks lower in the tree */ for (i=0; i < tree->numChildren; i++) { if (tree->child[i] != -1) { -#ifdef DEBUG_STEPH - printf("[%d] send(%d->%d, tag=%d)\n",rank,rank, tree->child[i], system_tag+tree->child[i]); -#endif + DEBUG4("[%d] send(%d->%d, tag=%d)\n",rank,rank, tree->child[i], system_tag+tree->child[i]); retval = smpi_create_request(buf, count, datatype, rank, tree->child[i], system_tag + tree->child[i], comm, &(requests[i])); -#ifdef DEBUG_STEPH - printf("[%d] after create req[%d]=%p req->(src=%d,dst=%d)\n",rank , i, requests[i],requests[i]->src,requests[i]->dst ); -#endif + DEBUG5("[%d] after create req[%d]=%p req->(src=%d,dst=%d)\n",rank,i,requests[i],requests[i]->src,requests[i]->dst ); if (MPI_SUCCESS != retval) { printf("** internal error: smpi_create_request() rank=%d returned retval=%d, %s:%d\n", rank,retval,__FILE__,__LINE__); @@ -345,6 +335,9 @@ int smpi_coll_tuned_alltoall_pairwise (void *sendbuf, int sendcount, MPI_Datatyp void * tmpsend, *tmprecv; rank = smpi_mpi_comm_rank(comm); + INFO1("[%d] algorithm alltoall_pairwise() called.\n",rank); + + /* Perform pairwise exchange - starting from 1 so the local copy is last */ for (step = 1; step < size+1; step++) { @@ -401,7 +394,7 @@ int smpi_coll_tuned_alltoall_basic_linear(void *sbuf, int scount, MPI_Datatype s void* rbuf, int rcount, MPI_Datatype rdtype, MPI_Comm comm) { int i; - int system_tag = 999; + int system_alltoall_tag = 888; int rank; int size = comm->size; int err; @@ -415,6 +408,7 @@ int smpi_coll_tuned_alltoall_basic_linear(void *sbuf, int scount, MPI_Datatype s /* Initialize. */ rank = smpi_mpi_comm_rank(comm); + INFO1("[%d] algorithm alltoall_basic_linear() called.\n",rank); err = smpi_mpi_type_get_extent(sdtype, &lb, &sndinc); err = smpi_mpi_type_get_extent(rdtype, &lb, &rcvinc); @@ -442,7 +436,7 @@ int smpi_coll_tuned_alltoall_basic_linear(void *sbuf, int scount, MPI_Datatype s for (i = (rank + 1) % size; i != rank; i = (i + 1) % size) { err = smpi_create_request( prcv + (i * rcvinc), rcount, rdtype, i, rank, - system_tag + i, + system_alltoall_tag, comm, &(reqs[nreq])); if (MPI_SUCCESS != err) { DEBUG2("[%d] failed to create request for rank %d\n",rank,i); @@ -459,7 +453,7 @@ int smpi_coll_tuned_alltoall_basic_linear(void *sbuf, int scount, MPI_Datatype s for (i = (rank + size - 1) % size; i != rank; i = (i + size - 1) % size ) { err = smpi_create_request (psnd + (i * sndinc), scount, sdtype, rank, i, - system_tag + i, + system_alltoall_tag, comm, &(reqs[nreq])); if (MPI_SUCCESS != err) { DEBUG2("[%d] failed to create request for rank %d\n",rank,i); diff --git a/src/smpi/smpi_global.c b/src/smpi/smpi_global.c index 44a75b37b6..8b626fd63f 100644 --- a/src/smpi/smpi_global.c +++ b/src/smpi/smpi_global.c @@ -1,6 +1,7 @@ #include #include "private.h" +#include "smpi_mpi_dt_private.h" XBT_LOG_NEW_CATEGORY(smpi, "All SMPI categories"); @@ -195,28 +196,33 @@ void smpi_global_init() // mpi datatypes smpi_mpi_global->mpi_byte = xbt_new(s_smpi_mpi_datatype_t, 1); /* we can think of it as a placeholder for value*/ smpi_mpi_global->mpi_byte->size = (size_t) 1; - smpi_mpi_global->mpi_byte->lb = (ptrdiff_t) &(smpi_mpi_global->mpi_byte); + smpi_mpi_global->mpi_byte->lb = (ptrdiff_t) 0; smpi_mpi_global->mpi_byte->ub = smpi_mpi_global->mpi_byte->lb + smpi_mpi_global->mpi_byte->size; + smpi_mpi_global->mpi_byte->flags = DT_FLAG_BASIC; smpi_mpi_global->mpi_char = xbt_new(s_smpi_mpi_datatype_t, 1); smpi_mpi_global->mpi_char->size = (size_t) 1; - smpi_mpi_global->mpi_char->lb = (ptrdiff_t) &(smpi_mpi_global->mpi_char); + smpi_mpi_global->mpi_char->lb = (ptrdiff_t) 0; //&(smpi_mpi_global->mpi_char); smpi_mpi_global->mpi_char->ub = smpi_mpi_global->mpi_char->lb + smpi_mpi_global->mpi_char->size; + smpi_mpi_global->mpi_char->flags = DT_FLAG_BASIC; smpi_mpi_global->mpi_int = xbt_new(s_smpi_mpi_datatype_t, 1); smpi_mpi_global->mpi_int->size = sizeof(int); - smpi_mpi_global->mpi_int->lb = (ptrdiff_t) &(smpi_mpi_global->mpi_int); + smpi_mpi_global->mpi_int->lb = (ptrdiff_t) 0; // &(smpi_mpi_global->mpi_int); smpi_mpi_global->mpi_int->ub = smpi_mpi_global->mpi_int->lb + smpi_mpi_global->mpi_int->size; + smpi_mpi_global->mpi_int->flags = DT_FLAG_BASIC; smpi_mpi_global->mpi_float = xbt_new(s_smpi_mpi_datatype_t, 1); smpi_mpi_global->mpi_float->size = sizeof(float); - smpi_mpi_global->mpi_float->lb = (ptrdiff_t) &(smpi_mpi_global->mpi_float); + smpi_mpi_global->mpi_float->lb = (ptrdiff_t) 0; // &(smpi_mpi_global->mpi_float); smpi_mpi_global->mpi_float->ub = smpi_mpi_global->mpi_float->lb + smpi_mpi_global->mpi_float->size; + smpi_mpi_global->mpi_float->flags = DT_FLAG_BASIC; smpi_mpi_global->mpi_double = xbt_new(s_smpi_mpi_datatype_t, 1); smpi_mpi_global->mpi_double->size = sizeof(double); - smpi_mpi_global->mpi_double->lb = (ptrdiff_t) &(smpi_mpi_global->mpi_float); + smpi_mpi_global->mpi_double->lb = (ptrdiff_t) 0; //&(smpi_mpi_global->mpi_float); smpi_mpi_global->mpi_double->ub = smpi_mpi_global->mpi_double->lb + smpi_mpi_global->mpi_double->size; + smpi_mpi_global->mpi_double->flags = DT_FLAG_BASIC; // mpi operations smpi_mpi_global->mpi_land = xbt_new(s_smpi_mpi_op_t, 1); diff --git a/src/smpi/smpi_mpi.c b/src/smpi/smpi_mpi.c index fb2e86b06c..2b9caebbb1 100644 --- a/src/smpi/smpi_mpi.c +++ b/src/smpi/smpi_mpi.c @@ -2,6 +2,7 @@ #include "private.h" #include "smpi_coll_private.h" +#include "smpi_mpi_dt_private.h" XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_mpi, smpi, "Logging specific to SMPI (mpi)"); @@ -67,41 +68,10 @@ int SMPI_MPI_Comm_rank(MPI_Comm comm, int *rank) } -//------------------------------- Datatypes --------------------------------------- -/** - * query the size of the type - **/ -int SMPI_MPI_Type_size(MPI_Datatype datatype, size_t * size) -{ - int retval = MPI_SUCCESS; - - smpi_bench_end(); - - if (NULL == datatype) { - retval = MPI_ERR_TYPE; - } else if (NULL == size) { - retval = MPI_ERR_ARG; - } else { - *size = datatype->size; - } - - smpi_bench_begin(); - - return retval; -} - /** - * query extent and lower bound of the type + * Barrier **/ -int SMPI_MPI_Type_get_extent( MPI_Datatype datatype, int *lb, int *extent) -{ - return( smpi_mpi_type_get_extent( datatype, lb, extent)); -} - - - - int SMPI_MPI_Barrier(MPI_Comm comm) { int retval = MPI_SUCCESS; @@ -126,6 +96,8 @@ int SMPI_MPI_Barrier(MPI_Comm comm) return retval; } + + int SMPI_MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request * request) { @@ -185,6 +157,9 @@ int SMPI_MPI_Isend(void *buf, int count, MPI_Datatype datatype, int dst, return retval; } +/** + * MPI_Send user level + **/ int SMPI_MPI_Send(void *buf, int count, MPI_Datatype datatype, int dst, int tag, MPI_Comm comm) { @@ -586,6 +561,7 @@ int SMPI_MPI_Alltoall(void *sendbuf, int sendcount, MPI_Datatype datatype, rank = smpi_mpi_comm_rank(comm); block_dsize = datatype->size * sendcount; + INFO2("[%d] optimized alltoall() called. Block size sent to each rank=%d.\n",rank,block_dsize); if ((block_dsize < 200) && (comm->size > 12)) { retval = smpi_coll_tuned_alltoall_bruck(sendbuf, sendcount, datatype, diff --git a/src/smpi/smpi_mpi_dt.c b/src/smpi/smpi_mpi_dt.c new file mode 100644 index 0000000000..7352a0996c --- /dev/null +++ b/src/smpi/smpi_mpi_dt.c @@ -0,0 +1,100 @@ +/* $Id: $tag */ + +/* smpi_mpi_dt.c -- MPI primitives to handle datatypes */ + +/* Note: a very incomplete implementation */ + +/* Copyright (c) 2009 Stephane Genaud. */ +/* All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * * under the terms of the license (GNU LGPL) which comes with this package. */ + + +#include +#include +#include + +#include "private.h" +#include "smpi_mpi_dt_private.h" + +XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_mpi_dt, smpi, + "Logging specific to SMPI (datatype)"); + + +/** + * Get the lower bound and extent for a Datatype + * The extent of a datatype is defined to be the span from the first byte to the last byte + * occupied by entries in this datatype, rounded up to satisfy alignment requirements (epsilon). + * + * For typemap T = {(t_0,disp_0), ..., (t_n-1,disp_n-1)} + * lb(T) = min_j disp_j + * ub(T) = max_j (disp_j+sizeof(t_j)) + epsilon + * extent(T) = ub(T) - lb(T) + * + * FIXME: this an incomplete implementation as we do not support yet MPI_Type_commit. + * Hence, this can be called only for primitive type MPI_INT, MPI_DOUBLE, ... + * + * remark: MPI-1 has also the deprecated + * int MPI_Type_extent(MPI_Datatype datatype, *MPI_Aint *extent); + * + **/ +int smpi_mpi_type_get_extent(MPI_Datatype datatype, MPI_Aint *lb, MPI_Aint *extent) { + + if ( DT_FLAG_COMMITED != (datatype-> flags & DT_FLAG_COMMITED) ) + return( MPI_ERR_TYPE ); + *lb = datatype->lb; + *extent = datatype->ub - datatype->lb; + return( MPI_SUCCESS ); +} +/** + * query the size of the type + **/ +int SMPI_MPI_Type_size(MPI_Datatype datatype, size_t * size) +{ + int retval = MPI_SUCCESS; + + smpi_bench_end(); + + if (NULL == datatype) { + retval = MPI_ERR_TYPE; + } else if (NULL == size) { + retval = MPI_ERR_ARG; + } else { + *size = datatype->size; + } + + smpi_bench_begin(); + + return retval; +} + + + +/** + * query extent and lower bound of the type + **/ +int SMPI_MPI_Type_get_extent( MPI_Datatype datatype, int *lb, int *extent) +{ + return( smpi_mpi_type_get_extent( datatype, lb, extent)); +} + +/* Deprecated Functions. + * The MPI-2 standard deprecated a number of routines because MPI-2 provides better versions. + * This routine is one of those that was deprecated. The routine may continue to be used, but + * new code should use the replacement routine. The replacement for this routine is MPI_Type_Get_extent. + **/ +int SMPI_MPI_Type_ub( MPI_Datatype datatype, MPI_Aint *displacement) +{ + if ( DT_FLAG_COMMITED != (datatype->flags & DT_FLAG_COMMITED) ) + return( MPI_ERR_TYPE ); + *displacement = datatype->ub; + return( MPI_SUCCESS ); +} +int SMPI_MPI_Type_lb( MPI_Datatype datatype, MPI_Aint *displacement) +{ + if ( DT_FLAG_COMMITED != (datatype->flags & DT_FLAG_COMMITED) ) + return( MPI_ERR_TYPE ); + *displacement = datatype->lb; + return( MPI_SUCCESS ); +} diff --git a/src/smpi/smpi_mpi_dt_private.h b/src/smpi/smpi_mpi_dt_private.h new file mode 100644 index 0000000000..47105276d7 --- /dev/null +++ b/src/smpi/smpi_mpi_dt_private.h @@ -0,0 +1,40 @@ +/** + * $Id: $tag + * + * smpi_mpi_dt_private.h -- functions of smpi_mpi_dt.c that are exported to other SMPI modules. + * + **/ +#include "private.h" + +/* flags for the datatypes. */ +#define DT_FLAG_DESTROYED 0x0001 /**< user destroyed but some other layers still have a reference */ +#define DT_FLAG_COMMITED 0x0002 /**< ready to be used for a send/recv operation */ +#define DT_FLAG_CONTIGUOUS 0x0004 /**< contiguous datatype */ +#define DT_FLAG_OVERLAP 0x0008 /**< datatype is unpropper for a recv operation */ +#define DT_FLAG_USER_LB 0x0010 /**< has a user defined LB */ +#define DT_FLAG_USER_UB 0x0020 /**< has a user defined UB */ +#define DT_FLAG_PREDEFINED 0x0040 /**< cannot be removed: initial and predefined datatypes */ +#define DT_FLAG_NO_GAPS 0x0080 /**< no gaps around the datatype */ +#define DT_FLAG_DATA 0x0100 /**< data or control structure */ +#define DT_FLAG_ONE_SIDED 0x0200 /**< datatype can be used for one sided operations */ +#define DT_FLAG_UNAVAILABLE 0x0400 /**< datatypes unavailable on the build (OS or compiler dependant) */ +#define DT_FLAG_VECTOR 0x0800 /**< valid only for loops. The loop contain only one element + **< without extent. It correspond to the vector type. */ +/* + * We should make the difference here between the predefined contiguous and non contiguous + * datatypes. The DT_FLAG_BASIC is held by all predefined contiguous datatypes. + */ +#define DT_FLAG_BASIC (DT_FLAG_PREDEFINED | DT_FLAG_CONTIGUOUS | DT_FLAG_NO_GAPS | DT_FLAG_DATA | DT_FLAG_COMMITED) + + + +int smpi_mpi_type_get_extent(MPI_Datatype datatype, MPI_Aint *lb, MPI_Aint *extent); + +/* Deprecated Functions. + * The MPI-2 standard deprecated a number of routines because MPI-2 provides better versions. + * This routine is one of those that was deprecated. The routine may continue to be used, but + * new code should use the replacement routine. The replacement for this routine is MPI_Type_Get_extent. + **/ +int SMPI_MPI_Type_ub( MPI_Datatype datatype, MPI_Aint *displacement); +int SMPI_MPI_Type_lb( MPI_Datatype datatype, MPI_Aint *displacement); +