1 /* Copyright (c) 2014, 2017. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
7 #include "xbt/sysdep.h"
13 typedef struct s_smpi_mpi_cart_topology {
19 } s_smpi_mpi_cart_topology_t;
21 typedef struct s_smpi_mpi_graph_topology {
26 } s_smpi_mpi_graph_topology_t;
28 typedef struct s_smpi_dist_graph_topology {
36 } s_smpi_mpi_dist_graph_topology_t;
38 typedef struct s_smpi_mpi_topology {
41 MPIR_Graph_Topology graph;
42 MPIR_Cart_Topology cart;
43 MPIR_Dist_Graph_Topology dist_graph;
45 } s_smpi_mpi_topology_t;
47 void smpi_topo_destroy(MPI_Topology topo) {
53 smpi_cart_topo_destroy(topo->topo.cart);
56 // This topology is not supported by SMPI yet
57 smpi_graph_topo_destroy(topo->topo.graph);
60 // This topology is not supported by SMPI yet
61 smpi_dist_graph_topo_destroy(topo->topo.dist_graph);
69 MPI_Topology smpi_topo_create(MPIR_Topo_type kind) {
70 MPI_Topology newTopo = static_cast<MPI_Topology>(xbt_malloc(sizeof(*newTopo)));
72 // Allocate and initialize the right topo should be done by the caller
76 void smpi_graph_topo_destroy(MPIR_Graph_Topology graph) {
78 delete[] graph->index;
79 delete[] graph->edges;
84 void smpi_dist_graph_topo_destroy(MPIR_Dist_Graph_Topology dist_graph) {
86 delete[] dist_graph->in;
87 delete[] dist_graph->in_weights;
88 delete[] dist_graph->out;
89 delete[] dist_graph->out_weights;
94 /*******************************************************************************
95 * Cartesian topologies
96 ******************************************************************************/
97 void smpi_cart_topo_destroy(MPIR_Cart_Topology cart) {
100 delete[] cart->periodic;
101 delete[] cart->position;
106 MPI_Topology smpi_cart_topo_create(int ndims) {
107 MPI_Topology newTopo = smpi_topo_create(MPI_CART);
108 MPIR_Cart_Topology newCart = static_cast<MPIR_Cart_Topology>(xbt_malloc(sizeof(*newCart)));
110 newCart->ndims = ndims;
111 newCart->dims = new int[ndims];
112 newCart->periodic = new int[ndims];
113 newCart->position = new int[ndims];
114 newTopo->topo.cart = newCart;
118 /* reorder is ignored, don't know what would be the consequences of a dumb reordering but neither do I see the point of
120 int smpi_mpi_cart_create(MPI_Comm comm_old, int ndims, int dims[], int periods[], int reorder, MPI_Comm *comm_cart) {
121 int retval = MPI_SUCCESS;
122 MPI_Topology newCart;
127 int rank = smpi_comm_rank(comm_old);
131 for (int i = 0 ; i < ndims ; i++) {
134 if(rank >= newSize) {
135 *comm_cart = MPI_COMM_NULL;
138 newCart = smpi_cart_topo_create(ndims);
139 oldGroup = smpi_comm_group(comm_old);
140 newGroup = smpi_group_new(newSize);
141 for (int i = 0 ; i < newSize ; i++) {
142 smpi_group_set_mapping(newGroup, smpi_group_index(oldGroup, i), i);
145 newCart->topo.cart->nnodes = newSize;
147 // FIXME : code duplication... See smpi_mpi_cart_coords
149 for (int i=0; i<ndims; i++) {
150 newCart->topo.cart->dims[i] = dims[i];
151 newCart->topo.cart->periodic[i] = periods[i];
152 nranks = nranks / dims[i];
153 /* FIXME: nranks could be zero (?) */
154 newCart->topo.cart->position[i] = rank / nranks;
155 rank = rank % nranks;
158 *comm_cart = smpi_comm_new(newGroup, newCart);
161 newCart = smpi_cart_topo_create(ndims);
162 *comm_cart = smpi_comm_new(smpi_group_copy(smpi_comm_group(MPI_COMM_SELF)), newCart);
164 *comm_cart = MPI_COMM_NULL;
170 int smpi_mpi_cart_sub(MPI_Comm comm, const int remain_dims[], MPI_Comm *newcomm) {
171 MPI_Topology oldTopo = smpi_comm_topo(comm);
172 int oldNDims = oldTopo->topo.cart->ndims;
174 int *newDims = nullptr;
175 int *newPeriodic = nullptr;
177 if (remain_dims == nullptr && oldNDims != 0) {
181 for (int i = 0 ; i < oldNDims ; i++) {
187 newDims = xbt_new(int, newNDims);
188 newPeriodic = xbt_new(int, newNDims);
190 // that should not segfault
191 for (int i = 0 ; j < newNDims ; i++) {
193 newDims[j] = oldTopo->topo.cart->dims[i];
194 newPeriodic[j] = oldTopo->topo.cart->periodic[i];
199 return smpi_mpi_cart_create(comm, newNDims, newDims, newPeriodic, 0, newcomm);
202 int smpi_mpi_cart_coords(MPI_Comm comm, int rank, int maxdims, int coords[]) {
203 MPI_Topology topo = smpi_comm_topo(comm);
204 int nnodes = topo->topo.cart->nnodes;
205 for (int i = 0; i< topo->topo.cart->ndims; i++ ) {
206 nnodes = nnodes / topo->topo.cart->dims[i];
207 coords[i] = rank / nnodes;
208 rank = rank % nnodes;
213 int smpi_mpi_cart_get(MPI_Comm comm, int maxdims, int* dims, int* periods, int* coords) {
214 MPI_Topology topo = smpi_comm_topo(comm);
215 int ndims=topo->topo.cart->ndims < maxdims ? topo->topo.cart->ndims : maxdims;
216 for(int i = 0 ; i < ndims ; i++) {
217 dims[i] = topo->topo.cart->dims[i];
218 periods[i] = topo->topo.cart->periodic[i];
219 coords[i] = topo->topo.cart->position[i];
224 int smpi_mpi_cart_rank(MPI_Comm comm, int* coords, int* rank) {
225 MPI_Topology topo = smpi_comm_topo(comm);
226 int ndims = topo->topo.cart->ndims;
231 for (int i=ndims-1; i >=0; i-- ) {
234 /* The user can give us whatever coordinates he wants. If one of them is out of range, either this dimension is
235 * periodic, and we consider the equivalent coordinate inside the bounds, or it's not and then it's an error
237 if (coord >= topo->topo.cart->dims[i]) {
238 if ( topo->topo.cart->periodic[i] ) {
239 coord = coord % topo->topo.cart->dims[i];
241 // Should I do that ?
245 } else if (coord < 0) {
246 if(topo->topo.cart->periodic[i]) {
247 coord = coord % topo->topo.cart->dims[i];
249 coord = topo->topo.cart->dims[i] + coord;
256 *rank += multiplier * coord;
257 multiplier *= topo->topo.cart->dims[i];
262 int smpi_mpi_cart_shift(MPI_Comm comm, int direction, int disp, int *rank_source, int *rank_dest) {
263 MPI_Topology topo = smpi_comm_topo(comm);
264 int position[topo->topo.cart->ndims];
266 if(topo->topo.cart->ndims == 0) {
269 if (topo->topo.cart->ndims < direction) {
273 smpi_mpi_cart_coords(comm, smpi_comm_rank(comm), topo->topo.cart->ndims, position);
274 position[direction] += disp;
276 if(position[direction] < 0 ||
277 position[direction] >= topo->topo.cart->dims[direction]) {
278 if(topo->topo.cart->periodic[direction]) {
279 position[direction] %= topo->topo.cart->dims[direction];
280 smpi_mpi_cart_rank(comm, position, rank_dest);
282 *rank_dest = MPI_PROC_NULL;
285 smpi_mpi_cart_rank(comm, position, rank_dest);
288 position[direction] = topo->topo.cart->position[direction] - disp;
289 if(position[direction] < 0 || position[direction] >= topo->topo.cart->dims[direction]) {
290 if(topo->topo.cart->periodic[direction]) {
291 position[direction] %= topo->topo.cart->dims[direction];
292 smpi_mpi_cart_rank(comm, position, rank_source);
294 *rank_source = MPI_PROC_NULL;
297 smpi_mpi_cart_rank(comm, position, rank_source);
303 int smpi_mpi_cartdim_get(MPI_Comm comm, int *ndims) {
304 MPI_Topology topo = smpi_comm_topo(comm);
306 *ndims = topo->topo.cart->ndims;
310 // Everything below has been taken from ompi, but could be easily rewritten (and partially was to follow sonar rules).
313 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
314 * University Research and Technology
315 * Corporation. All rights reserved.
316 * Copyright (c) 2004-2005 The University of Tennessee and The University
317 * of Tennessee Research Foundation. All rights
319 * Copyright (c) 2004-2014 High Performance Computing Center Stuttgart,
320 * University of Stuttgart. All rights reserved.
321 * Copyright (c) 2004-2005 The Regents of the University of California.
322 * All rights reserved.
323 * Copyright (c) 2012 Los Alamos National Security, LLC. All rights
325 * Copyright (c) 2014 Intel, Inc. All rights reserved
328 * Additional copyrights may follow
333 /* static functions */
334 static int assignnodes(int ndim, int nfactor, int *pfacts,int **pdims);
335 static int getfactors(int num, int *nfators, int **factors);
338 * This is a utility function, no need to have anything in the lower layer for this at all
340 int smpi_mpi_dims_create(int nnodes, int ndims, int dims[])
342 /* Get # of free-to-be-assigned processes and # of free dimensions */
343 int freeprocs = nnodes;
346 for (int i = 0; i < ndims; ++i) {
349 } else if ((*p < 0) || ((nnodes % *p) != 0)) {
359 if (freeprocs == 1) {
365 if (freeprocs == 1) {
366 for (int i = 0; i < ndims; ++i) {
375 /* Factor the number of free processes */
378 int err = getfactors(freeprocs, &nfactors, &factors);
379 if (MPI_SUCCESS != err)
382 /* Assign free processes to free dimensions */
384 err = assignnodes(freedims, nfactors, factors, &procs);
385 if (MPI_SUCCESS != err)
388 /* Return assignment results */
390 for (int i = 0; i < ndims; ++i) {
407 * Function: - assign processes to dimensions
408 * - get "best-balanced" grid
409 * - greedy bin-packing algorithm used
410 * - sort dimensions in decreasing order
411 * - dimensions array dynamically allocated
412 * Accepts: - # of dimensions
413 * - # of prime factors
414 * - array of prime factors
415 * - ptr to array of dimensions (returned value)
416 * Returns: - 0 or ERROR
418 static int assignnodes(int ndim, int nfactor, int *pfacts, int **pdims)
426 /* Allocate and initialize the bins */
427 int *bins = new int[ndim];
432 for (int i = 0 ; i < ndim; ++i) {
436 /* Loop assigning factors from the highest to the lowest */
437 for (int j = nfactor - 1; j >= 0; --j) {
439 /* Assign a factor to the smallest bin */
442 for (int i = 1; i < ndim; ++i) {
451 /* Sort dimensions in decreasing order (O(n^2) for now) */
453 for (int i = 0; i < ndim - 1; ++i) {
455 for (int j = i + 1; j < ndim; ++j) {
472 * Function: - factorize a number
475 * - array of prime factors
476 * Returns: - MPI_SUCCESS or ERROR
478 static int getfactors(int num, int *nfactors, int **factors) {
481 (*factors) = nullptr;
484 /* Allocate the array of prime factors which cannot exceed log_2(num) entries */
485 int sqrtnum = ceil(sqrt(num));
486 int size = ceil(log(num) / log(2));
487 *factors = new int[size];
490 /* determine all occurrences of factor 2 */
491 while((num % 2) == 0) {
495 /* determine all occurrences of uneven prime numbers up to sqrt(num) */
496 for(int d = 3; (num > 1) && (d < sqrtnum); d += 2) {
497 while((num % d) == 0) {
502 /* as we looped only up to sqrt(num) one factor > sqrt(num) may be left over */
504 (*factors)[i++] = num;