1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2011 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
7 /* A basic test of all 17 nonblocking collective operations specified by the
8 * MPI-3 standard. It only exercises the intracommunicator functionality,
9 * does not use MPI_IN_PLACE, and only transmits/receives simple integer types
10 * with relatively small counts. It does check a few fancier issues, such as
11 * ensuring that "premature user releases" of MPI_Op and MPI_Datatype objects
12 * does not result in an error or segfault. */
21 #define my_assert(cond_) \
24 fprintf(stderr, "assertion (%s) failed, aborting\n", #cond_); \
25 MPI_Abort(MPI_COMM_WORLD, 1); \
29 static void sum_fn(void *invec, void *inoutvec, int *len, MPI_Datatype * datatype)
33 int *inout = inoutvec;
34 for (i = 0; i < *len; ++i) {
35 inout[i] = in[i] + inout[i];
40 int main(int argc, char **argv)
46 int *sendcounts = NULL;
47 int *recvcounts = NULL;
50 int *sendtypes = NULL;
51 int *recvtypes = NULL;
52 signed char *buf_alias = NULL;
55 MPI_Init(&argc, &argv);
56 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
57 MPI_Comm_size(MPI_COMM_WORLD, &size);
59 buf = malloc(COUNT * size * sizeof(int));
60 recvbuf = malloc(COUNT * size * sizeof(int));
61 sendcounts = malloc(size * sizeof(int));
62 recvcounts = malloc(size * sizeof(int));
63 sdispls = malloc(size * sizeof(int));
64 rdispls = malloc(size * sizeof(int));
65 sendtypes = malloc(size * sizeof(MPI_Datatype));
66 recvtypes = malloc(size * sizeof(MPI_Datatype));
69 for (i = 0; i < COUNT; ++i) {
77 MPI_Ibcast(buf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
78 MPI_Wait(&req, MPI_STATUS_IGNORE);
80 for (i = 0; i < COUNT; ++i) {
82 printf("buf[%d]=%d i=%d\n", i, buf[i], i);
83 my_assert(buf[i] == i);
86 /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
87 buf_alias = (signed char *) buf;
88 my_assert(COUNT * size * sizeof(int) > PRIME); /* sanity */
89 for (i = 0; i < PRIME; ++i) {
95 for (i = PRIME; i < COUNT * size * sizeof(int); ++i) {
98 MPI_Ibcast(buf_alias, PRIME, MPI_SIGNED_CHAR, 0, MPI_COMM_WORLD, &req);
99 MPI_Wait(&req, MPI_STATUS_IGNORE);
100 for (i = 0; i < PRIME; ++i) {
101 if (buf_alias[i] != i)
102 printf("buf_alias[%d]=%d i=%d\n", i, buf_alias[i], i);
103 my_assert(buf_alias[i] == i);
107 MPI_Ibarrier(MPI_COMM_WORLD, &req);
108 MPI_Wait(&req, MPI_STATUS_IGNORE);
111 for (i = 0; i < COUNT; ++i) {
113 recvbuf[i] = 0xdeadbeef;
115 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD, &req);
116 MPI_Wait(&req, MPI_STATUS_IGNORE);
118 for (i = 0; i < COUNT; ++i) {
119 if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))
120 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],
121 ((size * (size - 1) / 2) + (i * size)));
122 my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));
126 /* same again, use a user op and free it before the wait */
128 MPI_Op op = MPI_OP_NULL;
129 MPI_Op_create(sum_fn, /*commute= */ 1, &op);
131 for (i = 0; i < COUNT; ++i) {
133 recvbuf[i] = 0xdeadbeef;
135 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, op, 0, MPI_COMM_WORLD, &req);
137 MPI_Wait(&req, MPI_STATUS_IGNORE);
139 for (i = 0; i < COUNT; ++i) {
140 if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))
141 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],
142 ((size * (size - 1) / 2) + (i * size)));
143 my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));
149 for (i = 0; i < COUNT; ++i) {
151 recvbuf[i] = 0xdeadbeef;
153 MPI_Iallreduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
154 MPI_Wait(&req, MPI_STATUS_IGNORE);
155 for (i = 0; i < COUNT; ++i) {
156 if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))
157 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],
158 ((size * (size - 1) / 2) + (i * size)));
159 my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));
162 /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
163 for (i = 0; i < size; ++i) {
164 sendcounts[i] = COUNT;
165 recvcounts[i] = COUNT;
166 sdispls[i] = COUNT * i;
167 rdispls[i] = COUNT * i;
168 for (j = 0; j < COUNT; ++j) {
169 buf[i * COUNT + j] = rank + (i * j);
170 recvbuf[i * COUNT + j] = 0xdeadbeef;
173 MPI_Ialltoallv(buf, sendcounts, sdispls, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT,
174 MPI_COMM_WORLD, &req);
175 MPI_Wait(&req, MPI_STATUS_IGNORE);
176 for (i = 0; i < size; ++i) {
177 for (j = 0; j < COUNT; ++j) {
178 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j))); */
179 my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));
184 for (i = 0; i < size * COUNT; ++i) {
186 recvbuf[i] = 0xdeadbeef;
188 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
189 MPI_Wait(&req, MPI_STATUS_IGNORE);
191 for (i = 0; i < size; ++i) {
192 for (j = 0; j < COUNT; ++j) {
193 my_assert(recvbuf[i * COUNT + j] == i + j);
198 for (i = 0; i < size * COUNT; ++i) {
199 my_assert(recvbuf[i] == 0xdeadbeef);
203 /* same test again, just use a dup'ed datatype and free it before the wait */
205 MPI_Datatype type = MPI_DATATYPE_NULL;
206 MPI_Type_dup(MPI_INT, &type);
208 for (i = 0; i < size * COUNT; ++i) {
210 recvbuf[i] = 0xdeadbeef;
212 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, type, 0, MPI_COMM_WORLD, &req);
213 MPI_Type_free(&type); /* should cause implementations that don't refcount
214 * correctly to blow up or hang in the wait */
215 MPI_Wait(&req, MPI_STATUS_IGNORE);
217 for (i = 0; i < size; ++i) {
218 for (j = 0; j < COUNT; ++j) {
219 my_assert(recvbuf[i * COUNT + j] == i + j);
224 for (i = 0; i < size * COUNT; ++i) {
225 my_assert(recvbuf[i] == 0xdeadbeef);
231 for (i = 0; i < size; ++i) {
232 for (j = 0; j < COUNT; ++j) {
234 buf[i * COUNT + j] = i + j;
236 buf[i * COUNT + j] = 0xdeadbeef;
237 recvbuf[i * COUNT + j] = 0xdeadbeef;
240 MPI_Iscatter(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
241 MPI_Wait(&req, MPI_STATUS_IGNORE);
242 for (j = 0; j < COUNT; ++j) {
243 my_assert(recvbuf[j] == rank + j);
246 for (i = 0; i < size * COUNT; ++i) {
247 /* check we didn't corrupt the sendbuf somehow */
248 my_assert(buf[i] == 0xdeadbeef);
253 for (i = 0; i < size; ++i) {
254 /* weak test, just test the regular case where all counts are equal */
255 sendcounts[i] = COUNT;
256 sdispls[i] = i * COUNT;
257 for (j = 0; j < COUNT; ++j) {
259 buf[i * COUNT + j] = i + j;
261 buf[i * COUNT + j] = 0xdeadbeef;
262 recvbuf[i * COUNT + j] = 0xdeadbeef;
265 MPI_Iscatterv(buf, sendcounts, sdispls, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD,
267 MPI_Wait(&req, MPI_STATUS_IGNORE);
268 for (j = 0; j < COUNT; ++j) {
269 my_assert(recvbuf[j] == rank + j);
272 for (i = 0; i < size * COUNT; ++i) {
273 /* check we didn't corrupt the sendbuf somehow */
274 my_assert(buf[i] == 0xdeadbeef);
277 for (i = 1; i < size; ++i) {
278 for (j = 0; j < COUNT; ++j) {
279 /* check we didn't corrupt the rest of the recvbuf */
280 my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);
284 /* MPI_Ireduce_scatter */
285 for (i = 0; i < size; ++i) {
286 recvcounts[i] = COUNT;
287 for (j = 0; j < COUNT; ++j) {
288 buf[i * COUNT + j] = rank + i;
289 recvbuf[i * COUNT + j] = 0xdeadbeef;
292 MPI_Ireduce_scatter(buf, recvbuf, recvcounts, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
293 MPI_Wait(&req, MPI_STATUS_IGNORE);
294 for (j = 0; j < COUNT; ++j) {
295 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
297 for (i = 1; i < size; ++i) {
298 for (j = 0; j < COUNT; ++j) {
299 /* check we didn't corrupt the rest of the recvbuf */
300 my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);
304 /* MPI_Ireduce_scatter_block */
305 for (i = 0; i < size; ++i) {
306 for (j = 0; j < COUNT; ++j) {
307 buf[i * COUNT + j] = rank + i;
308 recvbuf[i * COUNT + j] = 0xdeadbeef;
311 MPI_Ireduce_scatter_block(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
312 MPI_Wait(&req, MPI_STATUS_IGNORE);
313 for (j = 0; j < COUNT; ++j) {
314 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
316 for (i = 1; i < size; ++i) {
317 for (j = 0; j < COUNT; ++j) {
318 /* check we didn't corrupt the rest of the recvbuf */
319 my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);
324 for (i = 0; i < size * COUNT; ++i) {
326 recvbuf[i] = 0xdeadbeef;
328 for (i = 0; i < COUNT; ++i) {
331 for (i = 0; i < size; ++i) {
332 recvcounts[i] = COUNT;
333 rdispls[i] = i * COUNT;
335 MPI_Igatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, 0, MPI_COMM_WORLD,
337 MPI_Wait(&req, MPI_STATUS_IGNORE);
339 for (i = 0; i < size; ++i) {
340 for (j = 0; j < COUNT; ++j) {
341 my_assert(recvbuf[i * COUNT + j] == i + j);
346 for (i = 0; i < size * COUNT; ++i) {
347 my_assert(recvbuf[i] == 0xdeadbeef);
352 for (i = 0; i < size; ++i) {
353 for (j = 0; j < COUNT; ++j) {
354 buf[i * COUNT + j] = rank + (i * j);
355 recvbuf[i * COUNT + j] = 0xdeadbeef;
358 MPI_Ialltoall(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
359 MPI_Wait(&req, MPI_STATUS_IGNORE);
360 for (i = 0; i < size; ++i) {
361 for (j = 0; j < COUNT; ++j) {
362 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (i * j))); */
363 my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));
368 for (i = 0; i < size * COUNT; ++i) {
370 recvbuf[i] = 0xdeadbeef;
372 MPI_Iallgather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
373 MPI_Wait(&req, MPI_STATUS_IGNORE);
374 for (i = 0; i < size; ++i) {
375 for (j = 0; j < COUNT; ++j) {
376 my_assert(recvbuf[i * COUNT + j] == i + j);
380 /* MPI_Iallgatherv */
381 for (i = 0; i < size; ++i) {
382 for (j = 0; j < COUNT; ++j) {
383 recvbuf[i * COUNT + j] = 0xdeadbeef;
385 recvcounts[i] = COUNT;
386 rdispls[i] = i * COUNT;
388 for (i = 0; i < COUNT; ++i)
390 MPI_Iallgatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, MPI_COMM_WORLD,
392 MPI_Wait(&req, MPI_STATUS_IGNORE);
393 for (i = 0; i < size; ++i) {
394 for (j = 0; j < COUNT; ++j) {
395 my_assert(recvbuf[i * COUNT + j] == i + j);
400 for (i = 0; i < COUNT; ++i) {
402 recvbuf[i] = 0xdeadbeef;
404 MPI_Iscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
405 MPI_Wait(&req, MPI_STATUS_IGNORE);
406 for (i = 0; i < COUNT; ++i) {
407 my_assert(recvbuf[i] == ((rank * (rank + 1) / 2) + (i * (rank + 1))));
411 for (i = 0; i < COUNT; ++i) {
413 recvbuf[i] = 0xdeadbeef;
415 MPI_Iexscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
416 MPI_Wait(&req, MPI_STATUS_IGNORE);
417 for (i = 0; i < COUNT; ++i) {
419 my_assert(recvbuf[i] == 0xdeadbeef);
421 my_assert(recvbuf[i] == ((rank * (rank + 1) / 2) + (i * (rank + 1)) - (rank + i)));
424 /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
425 for (i = 0; i < size; ++i) {
426 sendcounts[i] = COUNT;
427 recvcounts[i] = COUNT;
428 sdispls[i] = COUNT * i * sizeof(int);
429 rdispls[i] = COUNT * i * sizeof(int);
430 sendtypes[i] = MPI_INT;
431 recvtypes[i] = MPI_INT;
432 for (j = 0; j < COUNT; ++j) {
433 buf[i * COUNT + j] = rank + (i * j);
434 recvbuf[i * COUNT + j] = 0xdeadbeef;
437 MPI_Ialltoallw(buf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes,
438 MPI_COMM_WORLD, &req);
439 MPI_Wait(&req, MPI_STATUS_IGNORE);
440 for (i = 0; i < size; ++i) {
441 for (j = 0; j < COUNT; ++j) {
442 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j))); */
443 my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));
448 printf(" No Errors\n");