1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2011 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
7 /* A basic test of all 17 nonblocking collective operations specified by the
8 * draft MPI-3 standard. It only exercises the intracommunicator functionality,
9 * does not use MPI_IN_PLACE, and only transmits/receives simple integer types
10 * with relatively small counts. It does check a few fancier issues, such as
11 * ensuring that "premature user releases" of MPI_Op and MPI_Datatype objects
12 * does not result in an error or segfault. */
17 /* USE_STRICT_MPI may be defined in mpitestconf.h */
18 #include "mpitestconf.h"
23 #define my_assert(cond_) \
26 fprintf(stderr, "assertion (%s) failed, aborting\n", #cond_); \
27 MPI_Abort(MPI_COMM_WORLD, 1); \
32 /* Since MPICH is currently the only NBC implementation in existence, just use
33 * this quick-and-dirty #ifdef to decide whether to test the nonblocking
34 * collectives. Eventually we can add a configure option or configure test, or
35 * the MPI-3 standard will be released and these can be gated on a MPI_VERSION
37 #if !defined(USE_STRICT_MPI) && defined(MPICH)
38 #define TEST_NBC_ROUTINES 1
41 static void sum_fn(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype)
45 int *inout = inoutvec;
46 for (i = 0; i < *len; ++i) {
47 inout[i] = in[i] + inout[i];
52 int main(int argc, char **argv)
57 int *sendcounts = NULL;
58 int *recvcounts = NULL;
61 int *sendtypes = NULL;
62 int *recvtypes = NULL;
63 #if defined(TEST_NBC_ROUTINES)
65 char *buf_alias = NULL;
69 MPI_Init(&argc, &argv);
70 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
71 MPI_Comm_size(MPI_COMM_WORLD, &size);
72 #if defined(TEST_NBC_ROUTINES)
74 buf = malloc(COUNT*size*sizeof(int));
75 recvbuf = malloc(COUNT*size*sizeof(int));
76 sendcounts = malloc(size*sizeof(int));
77 recvcounts = malloc(size*sizeof(int));
78 sdispls = malloc(size*sizeof(int));
79 rdispls = malloc(size*sizeof(int));
80 sendtypes = malloc(size*sizeof(MPI_Datatype));
81 recvtypes = malloc(size*sizeof(MPI_Datatype));
84 for (i = 0; i < COUNT; ++i) {
92 MPI_Ibcast(buf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
93 MPI_Wait(&req, MPI_STATUS_IGNORE);
95 for (i = 0; i < COUNT; ++i) {
97 printf("buf[%d]=%d i=%d\n", i, buf[i], i);
98 my_assert(buf[i] == i);
101 /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
102 buf_alias = (char *)buf;
103 my_assert(COUNT*size*sizeof(int) > PRIME); /* sanity */
104 for (i = 0; i < PRIME; ++i) {
110 for (i = PRIME; i < COUNT * size * sizeof(int); ++i) {
113 MPI_Ibcast(buf, PRIME, MPI_SIGNED_CHAR, 0, MPI_COMM_WORLD, &req);
114 MPI_Wait(&req, MPI_STATUS_IGNORE);
115 for (i = 0; i < PRIME; ++i) {
116 if (buf_alias[i] != i)
117 printf("buf_alias[%d]=%d i=%d\n", i, buf_alias[i], i);
118 my_assert(buf_alias[i] == i);
122 MPI_Ibarrier(MPI_COMM_WORLD, &req);
123 MPI_Wait(&req, MPI_STATUS_IGNORE);
126 for (i = 0; i < COUNT; ++i) {
128 recvbuf[i] = 0xdeadbeef;
130 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD, &req);
131 MPI_Wait(&req, MPI_STATUS_IGNORE);
133 for (i = 0; i < COUNT; ++i) {
134 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
135 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
136 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
140 /* same again, use a user op and free it before the wait */
142 MPI_Op op = MPI_OP_NULL;
143 MPI_Op_create(sum_fn, /*commute=*/1, &op);
145 for (i = 0; i < COUNT; ++i) {
147 recvbuf[i] = 0xdeadbeef;
149 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, op, 0, MPI_COMM_WORLD, &req);
151 MPI_Wait(&req, MPI_STATUS_IGNORE);
153 for (i = 0; i < COUNT; ++i) {
154 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
155 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
156 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
162 for (i = 0; i < COUNT; ++i) {
164 recvbuf[i] = 0xdeadbeef;
166 MPI_Iallreduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
167 MPI_Wait(&req, MPI_STATUS_IGNORE);
168 for (i = 0; i < COUNT; ++i) {
169 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
170 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
171 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
174 /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
175 for (i = 0; i < size; ++i) {
176 sendcounts[i] = COUNT;
177 recvcounts[i] = COUNT;
178 sdispls[i] = COUNT * i;
179 rdispls[i] = COUNT * i;
180 for (j = 0; j < COUNT; ++j) {
181 buf[i*COUNT+j] = rank + (i * j);
182 recvbuf[i*COUNT+j] = 0xdeadbeef;
185 MPI_Ialltoallv(buf, sendcounts, sdispls, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, MPI_COMM_WORLD, &req);
186 MPI_Wait(&req, MPI_STATUS_IGNORE);
187 for (i = 0; i < size; ++i) {
188 for (j = 0; j < COUNT; ++j) {
189 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
190 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
195 for (i = 0; i < size*COUNT; ++i) {
197 recvbuf[i] = 0xdeadbeef;
199 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
200 MPI_Wait(&req, MPI_STATUS_IGNORE);
202 for (i = 0; i < size; ++i) {
203 for (j = 0; j < COUNT; ++j) {
204 my_assert(recvbuf[i*COUNT+j] == i + j);
209 for (i = 0; i < size*COUNT; ++i) {
210 my_assert(recvbuf[i] == 0xdeadbeef);
214 /* same test again, just use a dup'ed datatype and free it before the wait */
216 MPI_Datatype type = MPI_DATATYPE_NULL;
217 MPI_Type_dup(MPI_INT, &type);
219 for (i = 0; i < size*COUNT; ++i) {
221 recvbuf[i] = 0xdeadbeef;
223 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, type, 0, MPI_COMM_WORLD, &req);
224 MPI_Type_free(&type); /* should cause implementations that don't refcount
225 correctly to blow up or hang in the wait */
226 MPI_Wait(&req, MPI_STATUS_IGNORE);
228 for (i = 0; i < size; ++i) {
229 for (j = 0; j < COUNT; ++j) {
230 my_assert(recvbuf[i*COUNT+j] == i + j);
235 for (i = 0; i < size*COUNT; ++i) {
236 my_assert(recvbuf[i] == 0xdeadbeef);
242 for (i = 0; i < size; ++i) {
243 for (j = 0; j < COUNT; ++j) {
245 buf[i*COUNT+j] = i + j;
247 buf[i*COUNT+j] = 0xdeadbeef;
248 recvbuf[i*COUNT+j] = 0xdeadbeef;
251 MPI_Iscatter(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
252 MPI_Wait(&req, MPI_STATUS_IGNORE);
253 for (j = 0; j < COUNT; ++j) {
254 my_assert(recvbuf[j] == rank + j);
257 for (i = 0; i < size*COUNT; ++i) {
258 /* check we didn't corrupt the sendbuf somehow */
259 my_assert(buf[i] == 0xdeadbeef);
264 for (i = 0; i < size; ++i) {
265 /* weak test, just test the regular case where all counts are equal */
266 sendcounts[i] = COUNT;
267 sdispls[i] = i * COUNT;
268 for (j = 0; j < COUNT; ++j) {
270 buf[i*COUNT+j] = i + j;
272 buf[i*COUNT+j] = 0xdeadbeef;
273 recvbuf[i*COUNT+j] = 0xdeadbeef;
276 MPI_Iscatterv(buf, sendcounts, sdispls, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
277 MPI_Wait(&req, MPI_STATUS_IGNORE);
278 for (j = 0; j < COUNT; ++j) {
279 my_assert(recvbuf[j] == rank + j);
282 for (i = 0; i < size*COUNT; ++i) {
283 /* check we didn't corrupt the sendbuf somehow */
284 my_assert(buf[i] == 0xdeadbeef);
287 for (i = 1; i < size; ++i) {
288 for (j = 0; j < COUNT; ++j) {
289 /* check we didn't corrupt the rest of the recvbuf */
290 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
294 /* MPI_Ireduce_scatter */
295 for (i = 0; i < size; ++i) {
296 recvcounts[i] = COUNT;
297 for (j = 0; j < COUNT; ++j) {
298 buf[i*COUNT+j] = rank + i;
299 recvbuf[i*COUNT+j] = 0xdeadbeef;
302 MPI_Ireduce_scatter(buf, recvbuf, recvcounts, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
303 MPI_Wait(&req, MPI_STATUS_IGNORE);
304 for (j = 0; j < COUNT; ++j) {
305 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
307 for (i = 1; i < size; ++i) {
308 for (j = 0; j < COUNT; ++j) {
309 /* check we didn't corrupt the rest of the recvbuf */
310 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
314 /* MPI_Ireduce_scatter_block */
315 for (i = 0; i < size; ++i) {
316 for (j = 0; j < COUNT; ++j) {
317 buf[i*COUNT+j] = rank + i;
318 recvbuf[i*COUNT+j] = 0xdeadbeef;
321 MPI_Ireduce_scatter_block(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
322 MPI_Wait(&req, MPI_STATUS_IGNORE);
323 for (j = 0; j < COUNT; ++j) {
324 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
326 for (i = 1; i < size; ++i) {
327 for (j = 0; j < COUNT; ++j) {
328 /* check we didn't corrupt the rest of the recvbuf */
329 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
334 for (i = 0; i < size*COUNT; ++i) {
336 recvbuf[i] = 0xdeadbeef;
338 for (i = 0; i < COUNT; ++i) {
341 for (i = 0; i < size; ++i) {
342 recvcounts[i] = COUNT;
343 rdispls[i] = i * COUNT;
345 MPI_Igatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, 0, MPI_COMM_WORLD, &req);
346 MPI_Wait(&req, MPI_STATUS_IGNORE);
348 for (i = 0; i < size; ++i) {
349 for (j = 0; j < COUNT; ++j) {
350 my_assert(recvbuf[i*COUNT+j] == i + j);
355 for (i = 0; i < size*COUNT; ++i) {
356 my_assert(recvbuf[i] == 0xdeadbeef);
361 for (i = 0; i < size; ++i) {
362 for (j = 0; j < COUNT; ++j) {
363 buf[i*COUNT+j] = rank + (i * j);
364 recvbuf[i*COUNT+j] = 0xdeadbeef;
367 MPI_Ialltoall(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
368 MPI_Wait(&req, MPI_STATUS_IGNORE);
369 for (i = 0; i < size; ++i) {
370 for (j = 0; j < COUNT; ++j) {
371 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (i * j)));*/
372 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
377 for (i = 0; i < size*COUNT; ++i) {
379 recvbuf[i] = 0xdeadbeef;
381 MPI_Iallgather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
382 MPI_Wait(&req, MPI_STATUS_IGNORE);
383 for (i = 0; i < size; ++i) {
384 for (j = 0; j < COUNT; ++j) {
385 my_assert(recvbuf[i*COUNT+j] == i + j);
389 /* MPI_Iallgatherv */
390 for (i = 0; i < size; ++i) {
391 for (j = 0; j < COUNT; ++j) {
392 recvbuf[i*COUNT+j] = 0xdeadbeef;
394 recvcounts[i] = COUNT;
395 rdispls[i] = i * COUNT;
397 for (i = 0; i < COUNT; ++i)
399 MPI_Iallgatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, MPI_COMM_WORLD, &req);
400 MPI_Wait(&req, MPI_STATUS_IGNORE);
401 for (i = 0; i < size; ++i) {
402 for (j = 0; j < COUNT; ++j) {
403 my_assert(recvbuf[i*COUNT+j] == i + j);
408 for (i = 0; i < COUNT; ++i) {
410 recvbuf[i] = 0xdeadbeef;
412 MPI_Iscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
413 MPI_Wait(&req, MPI_STATUS_IGNORE);
414 for (i = 0; i < COUNT; ++i) {
415 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1))));
419 for (i = 0; i < COUNT; ++i) {
421 recvbuf[i] = 0xdeadbeef;
423 MPI_Iexscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
424 MPI_Wait(&req, MPI_STATUS_IGNORE);
425 for (i = 0; i < COUNT; ++i) {
427 my_assert(recvbuf[i] == 0xdeadbeef);
429 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1)) - (rank + i)));
432 /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
433 for (i = 0; i < size; ++i) {
434 sendcounts[i] = COUNT;
435 recvcounts[i] = COUNT;
436 sdispls[i] = COUNT * i * sizeof(int);
437 rdispls[i] = COUNT * i * sizeof(int);
438 sendtypes[i] = MPI_INT;
439 recvtypes[i] = MPI_INT;
440 for (j = 0; j < COUNT; ++j) {
441 buf[i*COUNT+j] = rank + (i * j);
442 recvbuf[i*COUNT+j] = 0xdeadbeef;
445 MPI_Ialltoallw(buf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, MPI_COMM_WORLD, &req);
446 MPI_Wait(&req, MPI_STATUS_IGNORE);
447 for (i = 0; i < size; ++i) {
448 for (j = 0; j < COUNT; ++j) {
449 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
450 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
454 #endif /* defined(TEST_NBC_ROUTINES) */
457 printf(" No Errors\n");