1 /* Copyright (c) 2013-2017. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
7 #include "../colls_private.hpp"
9 int bcast_SMP_linear_segment_byte = 8192;
12 int Coll_bcast_SMP_linear::bcast(void *buf, int count,
13 MPI_Datatype datatype, int root,
16 int tag = COLL_TAG_BCAST;
19 MPI_Request *request_array;
20 MPI_Status *status_array;
24 extent = datatype->get_extent();
28 if(comm->get_leaders_comm()==MPI_COMM_NULL){
32 if (comm->is_uniform()){
33 num_core = comm->get_intra_comm()->size();
35 //implementation buggy in this case
36 return Coll_bcast_mpich::bcast( buf , count, datatype,
40 int segment = bcast_SMP_linear_segment_byte / extent;
41 segment = segment == 0 ? 1 :segment;
42 int pipe_length = count / segment;
43 int remainder = count % segment;
44 int increment = segment * extent;
47 /* leader of each SMP do inter-communication
48 and act as a root for intra-communication */
49 int to_inter = (rank + num_core) % size;
50 int to_intra = (rank + 1) % size;
51 int from_inter = (rank - num_core + size) % size;
52 int from_intra = (rank + size - 1) % size;
54 // call native when MPI communication size is too small
55 if (size <= num_core) {
56 XBT_WARN("MPI_bcast_SMP_linear use default MPI_bcast.");
57 Coll_bcast_default::bcast(buf, count, datatype, root, comm);
60 // if root is not zero send to rank zero first
63 Request::send(buf, count, datatype, 0, tag, comm);
65 Request::recv(buf, count, datatype, root, tag, comm, &status);
67 // when a message is smaller than a block size => no pipeline
68 if (count <= segment) {
71 Request::send(buf, count, datatype, to_inter, tag, comm);
72 Request::send(buf, count, datatype, to_intra, tag, comm);
74 // case last ROOT of each SMP
75 else if (rank == (((size - 1) / num_core) * num_core)) {
76 request = Request::irecv(buf, count, datatype, from_inter, tag, comm);
77 Request::wait(&request, &status);
78 Request::send(buf, count, datatype, to_intra, tag, comm);
80 // case intermediate ROOT of each SMP
81 else if (rank % num_core == 0) {
82 request = Request::irecv(buf, count, datatype, from_inter, tag, comm);
83 Request::wait(&request, &status);
84 Request::send(buf, count, datatype, to_inter, tag, comm);
85 Request::send(buf, count, datatype, to_intra, tag, comm);
87 // case last non-ROOT of each SMP
88 else if (((rank + 1) % num_core == 0) || (rank == (size - 1))) {
89 request = Request::irecv(buf, count, datatype, from_intra, tag, comm);
90 Request::wait(&request, &status);
92 // case intermediate non-ROOT of each SMP
94 request = Request::irecv(buf, count, datatype, from_intra, tag, comm);
95 Request::wait(&request, &status);
96 Request::send(buf, count, datatype, to_intra, tag, comm);
103 (MPI_Request *) xbt_malloc((size + pipe_length) * sizeof(MPI_Request));
105 (MPI_Status *) xbt_malloc((size + pipe_length) * sizeof(MPI_Status));
107 // case ROOT of each SMP
108 if (rank % num_core == 0) {
111 for (i = 0; i < pipe_length; i++) {
112 Request::send((char *) buf + (i * increment), segment, datatype, to_inter,
114 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
118 // case last ROOT of each SMP
119 else if (rank == (((size - 1) / num_core) * num_core)) {
120 for (i = 0; i < pipe_length; i++) {
121 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
122 from_inter, (tag + i), comm);
124 for (i = 0; i < pipe_length; i++) {
125 Request::wait(&request_array[i], &status);
126 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
130 // case intermediate ROOT of each SMP
132 for (i = 0; i < pipe_length; i++) {
133 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
134 from_inter, (tag + i), comm);
136 for (i = 0; i < pipe_length; i++) {
137 Request::wait(&request_array[i], &status);
138 Request::send((char *) buf + (i * increment), segment, datatype, to_inter,
140 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
144 } else { // case last non-ROOT of each SMP
145 if (((rank + 1) % num_core == 0) || (rank == (size - 1))) {
146 for (i = 0; i < pipe_length; i++) {
147 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
148 from_intra, (tag + i), comm);
150 for (i = 0; i < pipe_length; i++) {
151 Request::wait(&request_array[i], &status);
154 // case intermediate non-ROOT of each SMP
156 for (i = 0; i < pipe_length; i++) {
157 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
158 from_intra, (tag + i), comm);
160 for (i = 0; i < pipe_length; i++) {
161 Request::wait(&request_array[i], &status);
162 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
171 // when count is not divisible by block size, use default BCAST for the remainder
172 if ((remainder != 0) && (count > segment)) {
173 XBT_WARN("MPI_bcast_SMP_linear use default MPI_bcast.");
174 Colls::bcast((char *) buf + (pipe_length * increment), remainder, datatype,