1 /* Copyright (c) 2013-2021. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
7 #include "../colls_private.hpp"
9 int bcast_SMP_linear_segment_byte = 8192;
12 int bcast__SMP_linear(void *buf, int count,
13 MPI_Datatype datatype, int root,
16 int tag = COLL_TAG_BCAST;
22 extent = datatype->get_extent();
26 if(comm->get_leaders_comm()==MPI_COMM_NULL){
30 if (comm->is_uniform()){
31 num_core = comm->get_intra_comm()->size();
33 //implementation buggy in this case
34 return bcast__mpich(buf, count, datatype, root, comm);
37 int segment = bcast_SMP_linear_segment_byte / extent;
38 segment = segment == 0 ? 1 :segment;
39 int pipe_length = count / segment;
40 int remainder = count % segment;
41 int increment = segment * extent;
44 /* leader of each SMP do inter-communication
45 and act as a root for intra-communication */
46 int to_inter = (rank + num_core) % size;
47 int to_intra = (rank + 1) % size;
48 int from_inter = (rank - num_core + size) % size;
49 int from_intra = (rank + size - 1) % size;
51 // call native when MPI communication size is too small
52 if (size <= num_core) {
53 XBT_INFO("size <= num_core : MPI_bcast_SMP_linear use default MPI_bcast.");
54 bcast__default(buf, count, datatype, root, comm);
57 // if root is not zero send to rank zero first
60 Request::send(buf, count, datatype, 0, tag, comm);
62 Request::recv(buf, count, datatype, root, tag, comm, &status);
64 // when a message is smaller than a block size => no pipeline
65 if (count <= segment) {
68 Request::send(buf, count, datatype, to_inter, tag, comm);
69 Request::send(buf, count, datatype, to_intra, tag, comm);
71 // case last ROOT of each SMP
72 else if (rank == (((size - 1) / num_core) * num_core)) {
73 request = Request::irecv(buf, count, datatype, from_inter, tag, comm);
74 Request::wait(&request, &status);
75 Request::send(buf, count, datatype, to_intra, tag, comm);
77 // case intermediate ROOT of each SMP
78 else if (rank % num_core == 0) {
79 request = Request::irecv(buf, count, datatype, from_inter, tag, comm);
80 Request::wait(&request, &status);
81 Request::send(buf, count, datatype, to_inter, tag, comm);
82 Request::send(buf, count, datatype, to_intra, tag, comm);
84 // case last non-ROOT of each SMP
85 else if (((rank + 1) % num_core == 0) || (rank == (size - 1))) {
86 request = Request::irecv(buf, count, datatype, from_intra, tag, comm);
87 Request::wait(&request, &status);
89 // case intermediate non-ROOT of each SMP
91 request = Request::irecv(buf, count, datatype, from_intra, tag, comm);
92 Request::wait(&request, &status);
93 Request::send(buf, count, datatype, to_intra, tag, comm);
99 auto* request_array = new MPI_Request[size + pipe_length];
100 auto* status_array = new MPI_Status[size + pipe_length];
102 // case ROOT of each SMP
103 if (rank % num_core == 0) {
106 for (i = 0; i < pipe_length; i++) {
107 Request::send((char *) buf + (i * increment), segment, datatype, to_inter,
109 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
113 // case last ROOT of each SMP
114 else if (rank == (((size - 1) / num_core) * num_core)) {
115 for (i = 0; i < pipe_length; i++) {
116 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
117 from_inter, (tag + i), comm);
119 for (i = 0; i < pipe_length; i++) {
120 Request::wait(&request_array[i], &status);
121 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
125 // case intermediate ROOT of each SMP
127 for (i = 0; i < pipe_length; i++) {
128 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
129 from_inter, (tag + i), comm);
131 for (i = 0; i < pipe_length; i++) {
132 Request::wait(&request_array[i], &status);
133 Request::send((char *) buf + (i * increment), segment, datatype, to_inter,
135 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
139 } else { // case last non-ROOT of each SMP
140 if (((rank + 1) % num_core == 0) || (rank == (size - 1))) {
141 for (i = 0; i < pipe_length; i++) {
142 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
143 from_intra, (tag + i), comm);
145 for (i = 0; i < pipe_length; i++) {
146 Request::wait(&request_array[i], &status);
149 // case intermediate non-ROOT of each SMP
151 for (i = 0; i < pipe_length; i++) {
152 request_array[i] = Request::irecv((char *) buf + (i * increment), segment, datatype,
153 from_intra, (tag + i), comm);
155 for (i = 0; i < pipe_length; i++) {
156 Request::wait(&request_array[i], &status);
157 Request::send((char *) buf + (i * increment), segment, datatype, to_intra,
162 delete[] request_array;
163 delete[] status_array;
166 if ((remainder != 0) && (count > segment)) {
167 XBT_INFO("MPI_bcast_SMP_linear: count is not divisible by block size, use default MPI_bcast for remainder.");
168 colls::bcast((char*)buf + (pipe_length * increment), remainder, datatype, root, comm);