1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2011 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
7 /* A basic test of all 17 nonblocking collective operations specified by the
8 * draft MPI-3 standard. It only exercises the intracommunicator functionality,
9 * does not use MPI_IN_PLACE, and only transmits/receives simple integer types
10 * with relatively small counts. It does check a few fancier issues, such as
11 * ensuring that "premature user releases" of MPI_Op and MPI_Datatype objects
12 * does not result in an error or segfault. */
17 /* USE_STRICT_MPI may be defined in mpitestconf.h */
18 #include "mpitestconf.h"
23 #define my_assert(cond_) \
26 fprintf(stderr, "assertion (%s) failed, aborting\n", #cond_); \
27 MPI_Abort(MPI_COMM_WORLD, 1); \
31 /* Since MPICH is currently the only NBC implementation in existence, just use
32 * this quick-and-dirty #ifdef to decide whether to test the nonblocking
33 * collectives. Eventually we can add a configure option or configure test, or
34 * the MPI-3 standard will be released and these can be gated on a MPI_VERSION
36 #if !defined(USE_STRICT_MPI) && defined(MPICH)
37 #define TEST_NBC_ROUTINES 1
40 static void sum_fn(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype)
44 int *inout = inoutvec;
45 for (i = 0; i < *len; ++i) {
46 inout[i] = in[i] + inout[i];
51 int main(int argc, char **argv)
56 int *sendcounts = NULL;
57 int *recvcounts = NULL;
60 int *sendtypes = NULL;
61 int *recvtypes = NULL;
62 #if defined(TEST_NBC_ROUTINES)
64 char *buf_alias = NULL;
68 MPI_Init(&argc, &argv);
69 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
70 MPI_Comm_size(MPI_COMM_WORLD, &size);
71 #if defined(TEST_NBC_ROUTINES)
73 buf = malloc(COUNT*size*sizeof(int));
74 recvbuf = malloc(COUNT*size*sizeof(int));
75 sendcounts = malloc(size*sizeof(int));
76 recvcounts = malloc(size*sizeof(int));
77 sdispls = malloc(size*sizeof(int));
78 rdispls = malloc(size*sizeof(int));
79 sendtypes = malloc(size*sizeof(MPI_Datatype));
80 recvtypes = malloc(size*sizeof(MPI_Datatype));
83 for (i = 0; i < COUNT; ++i) {
91 MPI_Ibcast(buf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
92 MPI_Wait(&req, MPI_STATUS_IGNORE);
94 for (i = 0; i < COUNT; ++i) {
96 printf("buf[%d]=%d i=%d\n", i, buf[i], i);
97 my_assert(buf[i] == i);
100 /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
101 buf_alias = (char *)buf;
102 my_assert(COUNT*size*sizeof(int) > PRIME); /* sanity */
103 for (i = 0; i < PRIME; ++i) {
109 for (i = PRIME; i < COUNT * size * sizeof(int); ++i) {
112 MPI_Ibcast(buf, PRIME, MPI_SIGNED_CHAR, 0, MPI_COMM_WORLD, &req);
113 MPI_Wait(&req, MPI_STATUS_IGNORE);
114 for (i = 0; i < PRIME; ++i) {
115 if (buf_alias[i] != i)
116 printf("buf_alias[%d]=%d i=%d\n", i, buf_alias[i], i);
117 my_assert(buf_alias[i] == i);
121 MPI_Ibarrier(MPI_COMM_WORLD, &req);
122 MPI_Wait(&req, MPI_STATUS_IGNORE);
125 for (i = 0; i < COUNT; ++i) {
127 recvbuf[i] = 0xdeadbeef;
129 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD, &req);
130 MPI_Wait(&req, MPI_STATUS_IGNORE);
132 for (i = 0; i < COUNT; ++i) {
133 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
134 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
135 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
139 /* same again, use a user op and free it before the wait */
141 MPI_Op op = MPI_OP_NULL;
142 MPI_Op_create(sum_fn, /*commute=*/1, &op);
144 for (i = 0; i < COUNT; ++i) {
146 recvbuf[i] = 0xdeadbeef;
148 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, op, 0, MPI_COMM_WORLD, &req);
150 MPI_Wait(&req, MPI_STATUS_IGNORE);
152 for (i = 0; i < COUNT; ++i) {
153 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
154 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
155 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
161 for (i = 0; i < COUNT; ++i) {
163 recvbuf[i] = 0xdeadbeef;
165 MPI_Iallreduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
166 MPI_Wait(&req, MPI_STATUS_IGNORE);
167 for (i = 0; i < COUNT; ++i) {
168 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
169 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
170 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
173 /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
174 for (i = 0; i < size; ++i) {
175 sendcounts[i] = COUNT;
176 recvcounts[i] = COUNT;
177 sdispls[i] = COUNT * i;
178 rdispls[i] = COUNT * i;
179 for (j = 0; j < COUNT; ++j) {
180 buf[i*COUNT+j] = rank + (i * j);
181 recvbuf[i*COUNT+j] = 0xdeadbeef;
184 MPI_Ialltoallv(buf, sendcounts, sdispls, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, MPI_COMM_WORLD, &req);
185 MPI_Wait(&req, MPI_STATUS_IGNORE);
186 for (i = 0; i < size; ++i) {
187 for (j = 0; j < COUNT; ++j) {
188 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
189 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
194 for (i = 0; i < size*COUNT; ++i) {
196 recvbuf[i] = 0xdeadbeef;
198 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
199 MPI_Wait(&req, MPI_STATUS_IGNORE);
201 for (i = 0; i < size; ++i) {
202 for (j = 0; j < COUNT; ++j) {
203 my_assert(recvbuf[i*COUNT+j] == i + j);
208 for (i = 0; i < size*COUNT; ++i) {
209 my_assert(recvbuf[i] == 0xdeadbeef);
213 /* same test again, just use a dup'ed datatype and free it before the wait */
215 MPI_Datatype type = MPI_DATATYPE_NULL;
216 MPI_Type_dup(MPI_INT, &type);
218 for (i = 0; i < size*COUNT; ++i) {
220 recvbuf[i] = 0xdeadbeef;
222 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, type, 0, MPI_COMM_WORLD, &req);
223 MPI_Type_free(&type); /* should cause implementations that don't refcount
224 correctly to blow up or hang in the wait */
225 MPI_Wait(&req, MPI_STATUS_IGNORE);
227 for (i = 0; i < size; ++i) {
228 for (j = 0; j < COUNT; ++j) {
229 my_assert(recvbuf[i*COUNT+j] == i + j);
234 for (i = 0; i < size*COUNT; ++i) {
235 my_assert(recvbuf[i] == 0xdeadbeef);
241 for (i = 0; i < size; ++i) {
242 for (j = 0; j < COUNT; ++j) {
244 buf[i*COUNT+j] = i + j;
246 buf[i*COUNT+j] = 0xdeadbeef;
247 recvbuf[i*COUNT+j] = 0xdeadbeef;
250 MPI_Iscatter(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
251 MPI_Wait(&req, MPI_STATUS_IGNORE);
252 for (j = 0; j < COUNT; ++j) {
253 my_assert(recvbuf[j] == rank + j);
256 for (i = 0; i < size*COUNT; ++i) {
257 /* check we didn't corrupt the sendbuf somehow */
258 my_assert(buf[i] == 0xdeadbeef);
263 for (i = 0; i < size; ++i) {
264 /* weak test, just test the regular case where all counts are equal */
265 sendcounts[i] = COUNT;
266 sdispls[i] = i * COUNT;
267 for (j = 0; j < COUNT; ++j) {
269 buf[i*COUNT+j] = i + j;
271 buf[i*COUNT+j] = 0xdeadbeef;
272 recvbuf[i*COUNT+j] = 0xdeadbeef;
275 MPI_Iscatterv(buf, sendcounts, sdispls, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
276 MPI_Wait(&req, MPI_STATUS_IGNORE);
277 for (j = 0; j < COUNT; ++j) {
278 my_assert(recvbuf[j] == rank + j);
281 for (i = 0; i < size*COUNT; ++i) {
282 /* check we didn't corrupt the sendbuf somehow */
283 my_assert(buf[i] == 0xdeadbeef);
286 for (i = 1; i < size; ++i) {
287 for (j = 0; j < COUNT; ++j) {
288 /* check we didn't corrupt the rest of the recvbuf */
289 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
293 /* MPI_Ireduce_scatter */
294 for (i = 0; i < size; ++i) {
295 recvcounts[i] = COUNT;
296 for (j = 0; j < COUNT; ++j) {
297 buf[i*COUNT+j] = rank + i;
298 recvbuf[i*COUNT+j] = 0xdeadbeef;
301 MPI_Ireduce_scatter(buf, recvbuf, recvcounts, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
302 MPI_Wait(&req, MPI_STATUS_IGNORE);
303 for (j = 0; j < COUNT; ++j) {
304 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
306 for (i = 1; i < size; ++i) {
307 for (j = 0; j < COUNT; ++j) {
308 /* check we didn't corrupt the rest of the recvbuf */
309 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
313 /* MPI_Ireduce_scatter_block */
314 for (i = 0; i < size; ++i) {
315 for (j = 0; j < COUNT; ++j) {
316 buf[i*COUNT+j] = rank + i;
317 recvbuf[i*COUNT+j] = 0xdeadbeef;
320 MPI_Ireduce_scatter_block(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
321 MPI_Wait(&req, MPI_STATUS_IGNORE);
322 for (j = 0; j < COUNT; ++j) {
323 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
325 for (i = 1; i < size; ++i) {
326 for (j = 0; j < COUNT; ++j) {
327 /* check we didn't corrupt the rest of the recvbuf */
328 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
333 for (i = 0; i < size*COUNT; ++i) {
335 recvbuf[i] = 0xdeadbeef;
337 for (i = 0; i < COUNT; ++i) {
340 for (i = 0; i < size; ++i) {
341 recvcounts[i] = COUNT;
342 rdispls[i] = i * COUNT;
344 MPI_Igatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, 0, MPI_COMM_WORLD, &req);
345 MPI_Wait(&req, MPI_STATUS_IGNORE);
347 for (i = 0; i < size; ++i) {
348 for (j = 0; j < COUNT; ++j) {
349 my_assert(recvbuf[i*COUNT+j] == i + j);
354 for (i = 0; i < size*COUNT; ++i) {
355 my_assert(recvbuf[i] == 0xdeadbeef);
360 for (i = 0; i < size; ++i) {
361 for (j = 0; j < COUNT; ++j) {
362 buf[i*COUNT+j] = rank + (i * j);
363 recvbuf[i*COUNT+j] = 0xdeadbeef;
366 MPI_Ialltoall(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
367 MPI_Wait(&req, MPI_STATUS_IGNORE);
368 for (i = 0; i < size; ++i) {
369 for (j = 0; j < COUNT; ++j) {
370 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (i * j)));*/
371 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
376 for (i = 0; i < size*COUNT; ++i) {
378 recvbuf[i] = 0xdeadbeef;
380 MPI_Iallgather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
381 MPI_Wait(&req, MPI_STATUS_IGNORE);
382 for (i = 0; i < size; ++i) {
383 for (j = 0; j < COUNT; ++j) {
384 my_assert(recvbuf[i*COUNT+j] == i + j);
388 /* MPI_Iallgatherv */
389 for (i = 0; i < size; ++i) {
390 for (j = 0; j < COUNT; ++j) {
391 recvbuf[i*COUNT+j] = 0xdeadbeef;
393 recvcounts[i] = COUNT;
394 rdispls[i] = i * COUNT;
396 for (i = 0; i < COUNT; ++i)
398 MPI_Iallgatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, MPI_COMM_WORLD, &req);
399 MPI_Wait(&req, MPI_STATUS_IGNORE);
400 for (i = 0; i < size; ++i) {
401 for (j = 0; j < COUNT; ++j) {
402 my_assert(recvbuf[i*COUNT+j] == i + j);
407 for (i = 0; i < COUNT; ++i) {
409 recvbuf[i] = 0xdeadbeef;
411 MPI_Iscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
412 MPI_Wait(&req, MPI_STATUS_IGNORE);
413 for (i = 0; i < COUNT; ++i) {
414 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1))));
418 for (i = 0; i < COUNT; ++i) {
420 recvbuf[i] = 0xdeadbeef;
422 MPI_Iexscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
423 MPI_Wait(&req, MPI_STATUS_IGNORE);
424 for (i = 0; i < COUNT; ++i) {
426 my_assert(recvbuf[i] == 0xdeadbeef);
428 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1)) - (rank + i)));
431 /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
432 for (i = 0; i < size; ++i) {
433 sendcounts[i] = COUNT;
434 recvcounts[i] = COUNT;
435 sdispls[i] = COUNT * i * sizeof(int);
436 rdispls[i] = COUNT * i * sizeof(int);
437 sendtypes[i] = MPI_INT;
438 recvtypes[i] = MPI_INT;
439 for (j = 0; j < COUNT; ++j) {
440 buf[i*COUNT+j] = rank + (i * j);
441 recvbuf[i*COUNT+j] = 0xdeadbeef;
444 MPI_Ialltoallw(buf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, MPI_COMM_WORLD, &req);
445 MPI_Wait(&req, MPI_STATUS_IGNORE);
446 for (i = 0; i < size; ++i) {
447 for (j = 0; j < COUNT; ++j) {
448 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
449 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
453 #endif /* defined(TEST_NBC_ROUTINES) */
456 printf(" No Errors\n");