1 /* Copyright (c) 2013-2019. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
7 #include "../colls_private.hpp"
9 int bcast_SMP_linear_segment_byte = 8192;
12 int Coll_bcast_SMP_linear::bcast(void *buf, int count,
13 MPI_Datatype datatype, int root,
16 int tag = COLL_TAG_BCAST;
22 extent = datatype->get_extent();
26 if(comm->get_leaders_comm()==MPI_COMM_NULL){
30 if (comm->is_uniform()){
31 num_core = comm->get_intra_comm()->size();
33 //implementation buggy in this case
34 return Coll_bcast_mpich::bcast( buf , count, datatype,
38 int segment = bcast_SMP_linear_segment_byte / extent;
39 segment = segment == 0 ? 1 :segment;
40 int pipe_length = count / segment;
41 int remainder = count % segment;
42 int increment = segment * extent;
45 /* leader of each SMP do inter-communication
46 and act as a root for intra-communication */
47 int to_inter = (rank + num_core) % size;
48 int to_intra = (rank + 1) % size;
49 int from_inter = (rank - num_core + size) % size;
50 int from_intra = (rank + size - 1) % size;
52 // call native when MPI communication size is too small
53 if (size <= num_core) {
54 XBT_WARN("MPI_bcast_SMP_linear use default MPI_bcast.");
55 Coll_bcast_default::bcast(buf, count, datatype, root, comm);
58 // if root is not zero send to rank zero first
61 Request::send(buf, count, datatype, 0, tag, comm);
63 Request::recv(buf, count, datatype, root, tag, comm, &status);
65 // when a message is smaller than a block size => no pipeline
66 if (count <= segment) {
69 Request::send(buf, count, datatype, to_inter, tag, comm);
70 Request::send(buf, count, datatype, to_intra, tag, comm);
72 // case last ROOT of each SMP
73 else if (rank == (((size - 1) / num_core) * num_core)) {
74 request = Request::irecv(buf, count, datatype, from_inter, tag, comm);
75 Request::wait(&request, &status);
76 Request::send(buf, count, datatype, to_intra, tag, comm);
78 // case intermediate ROOT of each SMP
79 else if (rank % num_core == 0) {
80 request = Request::irecv(buf, count, datatype, from_inter, tag, comm);
81 Request::wait(&request, &status);
82 Request::send(buf, count, datatype, to_inter, tag, comm);
83 Request::send(buf, count, datatype, to_intra, tag, comm);
85 // case last non-ROOT of each SMP
86 else if (((rank + 1) % num_core == 0) || (rank == (size - 1))) {
87 request = Request::irecv(buf, count, datatype, from_intra, tag, comm);
88 Request::wait(&request, &status);
90 // case intermediate non-ROOT of each SMP
92 request = Request::irecv(buf, count, datatype, from_intra, tag, comm);
93 Request::wait(&request, &status);
94 Request::send(buf, count, datatype, to_intra, tag, comm);
100 MPI_Request* request_array = new MPI_Request[size + pipe_length];
101 MPI_Status* status_array = new MPI_Status[size + pipe_length];
103 // case ROOT of each SMP
104 if (rank % num_core == 0) {
107 for (i = 0; i < pipe_length; i++) {
108 Request::send((char *) buf + (i * increment), segment, datatype, to_inter,
110 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
114 // case last ROOT of each SMP
115 else if (rank == (((size - 1) / num_core) * num_core)) {
116 for (i = 0; i < pipe_length; i++) {
117 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
118 from_inter, (tag + i), comm);
120 for (i = 0; i < pipe_length; i++) {
121 Request::wait(&request_array[i], &status);
122 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
126 // case intermediate ROOT of each SMP
128 for (i = 0; i < pipe_length; i++) {
129 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
130 from_inter, (tag + i), comm);
132 for (i = 0; i < pipe_length; i++) {
133 Request::wait(&request_array[i], &status);
134 Request::send((char *) buf + (i * increment), segment, datatype, to_inter,
136 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
140 } else { // case last non-ROOT of each SMP
141 if (((rank + 1) % num_core == 0) || (rank == (size - 1))) {
142 for (i = 0; i < pipe_length; i++) {
143 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
144 from_intra, (tag + i), comm);
146 for (i = 0; i < pipe_length; i++) {
147 Request::wait(&request_array[i], &status);
150 // case intermediate non-ROOT of each SMP
152 for (i = 0; i < pipe_length; i++) {
153 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
154 from_intra, (tag + i), comm);
156 for (i = 0; i < pipe_length; i++) {
157 Request::wait(&request_array[i], &status);
158 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
163 delete[] request_array;
164 delete[] status_array;
167 // when count is not divisible by block size, use default BCAST for the remainder
168 if ((remainder != 0) && (count > segment)) {
169 XBT_WARN("MPI_bcast_SMP_linear use default MPI_bcast.");
170 Colls::bcast((char *) buf + (pipe_length * increment), remainder, datatype,