1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2011 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
7 /* This test attempts to execute multiple simultaneous nonblocking collective
8 * (NBC) MPI routines at the same time, and manages their completion with a
9 * variety of routines (MPI_{Wait,Test}{,_all,_any,_some}). It also throws a
10 * few point-to-point operations into the mix.
12 * Possible improvements:
13 * - post operations on multiple comms from multiple threads
21 /* USE_STRICT_MPI may be defined in mpitestconf.h */
22 #include "mpitestconf.h"
30 /* Constants that control the high level test harness behavior. */
31 /* MAIN_ITERATIONS is how many NBC ops the test will attempt to issue. */
32 #define MAIN_ITERATIONS (100000)
33 /* WINDOW is the maximum number of outstanding NBC requests at any given time */
35 /* we sleep with probability 1/CHANCE_OF_SLEEP */
36 #define CHANCE_OF_SLEEP (1000)
37 /* JITTER_DELAY is denominated in microseconds (us) */
38 #define JITTER_DELAY (50000) /* 0.05 seconds */
39 /* NUM_COMMS is the number of communicators on which ops will be posted */
42 /* Constants that control behavior of the individual testing operations.
43 * Altering these can help to explore the testing space, but increasing them too
44 * much can consume too much memory (often O(n^2) usage). */
45 /* FIXME is COUNT==10 too limiting? should we try a larger count too (~500)? */
49 #define my_assert(cond_) \
54 fprintf(stderr, "assertion (%s) failed on line %d\n", #cond_, __LINE__); \
59 /* Since MPICH is currently the only NBC implementation in existence, just use
60 * this quick-and-dirty #ifdef to decide whether to test the nonblocking
61 * collectives. Eventually we can add a configure option or configure test, or
62 * the MPI-3 standard will be released and these can be gated on a MPI_VERSION
64 #if !defined(USE_STRICT_MPI) && defined(MPICH)
65 #define TEST_NBC_ROUTINES 1
68 #if defined(TEST_NBC_ROUTINES)
69 /* Intended to act like "rand_r", but we can be sure that it will exist and be
70 * consistent across all of comm world. Returns a number in the range
72 #define GEN_PRN_MAX (4294967291-1)
73 static unsigned int gen_prn(unsigned int x)
75 /* a simple "multiplicative congruential method" PRNG, with parameters:
76 * m=4294967291, largest 32-bit prime
77 * a=279470273, good primitive root of m from "TABLES OF LINEAR
78 * CONGRUENTIAL GENERATORS OF DIFFERENT SIZES AND GOOD
79 * LATTICE STRUCTURE", by Pierre L’Ecuyer */
80 return (279470273UL * (unsigned long)x) % 4294967291UL;
83 /* given a random unsigned int value "rndval_" from gen_prn, this evaluates to a
84 * value in the range [min_,max_) */
85 #define rand_range(rndval_,min_,max_) \
86 ((unsigned int)((min_) + ((rndval_) * (1.0 / (GEN_PRN_MAX+1.0)) * ((max_) - (min_)))))
89 static void sum_fn(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype)
93 int *inout = inoutvec;
94 for (i = 0; i < *len; ++i) {
95 inout[i] = in[i] + inout[i];
99 /* used to keep track of buffers that should be freed after the corresponding
100 * operation has completed */
102 int case_num; /* which test case initiated this req/laundry */
114 static void cleanup_laundry(struct laundry *l)
117 l->comm = MPI_COMM_NULL;
118 if (l->buf) free(l->buf);
119 if (l->recvbuf) free(l->recvbuf);
120 if (l->sendcounts) free(l->sendcounts);
121 if (l->recvcounts) free(l->recvcounts);
122 if (l->sdispls) free(l->sdispls);
123 if (l->rdispls) free(l->rdispls);
124 if (l->sendtypes) free(l->sendtypes);
125 if (l->recvtypes) free(l->recvtypes);
128 /* Starts a "random" operation on "comm" corresponding to "rndnum" and returns
129 * in (*req) a request handle corresonding to that operation. This call should
130 * be considered collective over comm (with a consistent value for "rndnum"),
131 * even though the operation may only be a point-to-point request. */
132 static void start_random_nonblocking(MPI_Comm comm, unsigned int rndnum, MPI_Request *req, struct laundry *l)
138 int *sendcounts = NULL;
139 int *recvcounts = NULL;
142 int *sendtypes = NULL;
143 int *recvtypes = NULL;
144 char *buf_alias = NULL;
146 MPI_Comm_rank(comm, &rank);
147 MPI_Comm_size(comm, &size);
149 *req = MPI_REQUEST_NULL;
154 l->buf = buf = malloc(COUNT*size*sizeof(int));
155 l->recvbuf = recvbuf = malloc(COUNT*size*sizeof(int));
156 l->sendcounts = sendcounts = malloc(size*sizeof(int));
157 l->recvcounts = recvcounts = malloc(size*sizeof(int));
158 l->sdispls = sdispls = malloc(size*sizeof(int));
159 l->rdispls = rdispls = malloc(size*sizeof(int));
160 l->sendtypes = sendtypes = malloc(size*sizeof(MPI_Datatype));
161 l->recvtypes = recvtypes = malloc(size*sizeof(MPI_Datatype));
163 #define NUM_CASES (21)
164 l->case_num = rand_range(rndnum, 0, NUM_CASES);
165 switch (l->case_num) {
166 case 0: /* MPI_Ibcast */
167 for (i = 0; i < COUNT; ++i) {
175 MPI_Ibcast(buf, COUNT, MPI_INT, 0, comm, req);
178 case 1: /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
179 /* FIXME fiddle with PRIME and buffer allocation s.t. PRIME is much larger (1021?) */
180 buf_alias = (char *)buf;
181 my_assert(COUNT*size*sizeof(int) > PRIME); /* sanity */
182 for (i = 0; i < PRIME; ++i) {
188 for (i = PRIME; i < COUNT * size * sizeof(int); ++i) {
191 MPI_Ibcast(buf, PRIME, MPI_SIGNED_CHAR, 0, comm, req);
194 case 2: /* MPI_Ibarrier */
195 MPI_Ibarrier(comm, req);
198 case 3: /* MPI_Ireduce */
199 for (i = 0; i < COUNT; ++i) {
201 recvbuf[i] = 0xdeadbeef;
203 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, 0, comm, req);
206 case 4: /* same again, use a user op and free it before the wait */
208 MPI_Op op = MPI_OP_NULL;
209 MPI_Op_create(sum_fn, /*commute=*/1, &op);
210 for (i = 0; i < COUNT; ++i) {
212 recvbuf[i] = 0xdeadbeef;
214 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, op, 0, comm, req);
219 case 5: /* MPI_Iallreduce */
220 for (i = 0; i < COUNT; ++i) {
222 recvbuf[i] = 0xdeadbeef;
224 MPI_Iallreduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
227 case 6: /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
228 for (i = 0; i < size; ++i) {
229 sendcounts[i] = COUNT;
230 recvcounts[i] = COUNT;
231 sdispls[i] = COUNT * i;
232 rdispls[i] = COUNT * i;
233 for (j = 0; j < COUNT; ++j) {
234 buf[i*COUNT+j] = rank + (i * j);
235 recvbuf[i*COUNT+j] = 0xdeadbeef;
238 MPI_Ialltoallv(buf, sendcounts, sdispls, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, comm, req);
241 case 7: /* MPI_Igather */
242 for (i = 0; i < size*COUNT; ++i) {
244 recvbuf[i] = 0xdeadbeef;
246 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);
249 case 8: /* same test again, just use a dup'ed datatype and free it before the wait */
251 MPI_Datatype type = MPI_DATATYPE_NULL;
252 MPI_Type_dup(MPI_INT, &type);
253 for (i = 0; i < size*COUNT; ++i) {
255 recvbuf[i] = 0xdeadbeef;
257 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, type, 0, comm, req);
258 MPI_Type_free(&type); /* should cause implementations that don't refcount
259 correctly to blow up or hang in the wait */
263 case 9: /* MPI_Iscatter */
264 for (i = 0; i < size; ++i) {
265 for (j = 0; j < COUNT; ++j) {
267 buf[i*COUNT+j] = i + j;
269 buf[i*COUNT+j] = 0xdeadbeef;
270 recvbuf[i*COUNT+j] = 0xdeadbeef;
273 MPI_Iscatter(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);
276 case 10: /* MPI_Iscatterv */
277 for (i = 0; i < size; ++i) {
278 /* weak test, just test the regular case where all counts are equal */
279 sendcounts[i] = COUNT;
280 sdispls[i] = i * COUNT;
281 for (j = 0; j < COUNT; ++j) {
283 buf[i*COUNT+j] = i + j;
285 buf[i*COUNT+j] = 0xdeadbeef;
286 recvbuf[i*COUNT+j] = 0xdeadbeef;
289 MPI_Iscatterv(buf, sendcounts, sdispls, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);
292 case 11: /* MPI_Ireduce_scatter */
293 for (i = 0; i < size; ++i) {
294 recvcounts[i] = COUNT;
295 for (j = 0; j < COUNT; ++j) {
296 buf[i*COUNT+j] = rank + i;
297 recvbuf[i*COUNT+j] = 0xdeadbeef;
300 MPI_Ireduce_scatter(buf, recvbuf, recvcounts, MPI_INT, MPI_SUM, comm, req);
303 case 12: /* MPI_Ireduce_scatter_block */
304 for (i = 0; i < size; ++i) {
305 for (j = 0; j < COUNT; ++j) {
306 buf[i*COUNT+j] = rank + i;
307 recvbuf[i*COUNT+j] = 0xdeadbeef;
310 MPI_Ireduce_scatter_block(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
313 case 13: /* MPI_Igatherv */
314 for (i = 0; i < size*COUNT; ++i) {
316 recvbuf[i] = 0xdeadbeef;
318 for (i = 0; i < COUNT; ++i) {
321 for (i = 0; i < size; ++i) {
322 recvcounts[i] = COUNT;
323 rdispls[i] = i * COUNT;
325 MPI_Igatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, 0, comm, req);
328 case 14: /* MPI_Ialltoall */
329 for (i = 0; i < size; ++i) {
330 for (j = 0; j < COUNT; ++j) {
331 buf[i*COUNT+j] = rank + (i * j);
332 recvbuf[i*COUNT+j] = 0xdeadbeef;
335 MPI_Ialltoall(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, comm, req);
338 case 15: /* MPI_Iallgather */
339 for (i = 0; i < size*COUNT; ++i) {
341 recvbuf[i] = 0xdeadbeef;
343 MPI_Iallgather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, comm, req);
346 case 16: /* MPI_Iallgatherv */
347 for (i = 0; i < size; ++i) {
348 for (j = 0; j < COUNT; ++j) {
349 recvbuf[i*COUNT+j] = 0xdeadbeef;
351 recvcounts[i] = COUNT;
352 rdispls[i] = i * COUNT;
354 for (i = 0; i < COUNT; ++i)
356 MPI_Iallgatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, comm, req);
359 case 17: /* MPI_Iscan */
360 for (i = 0; i < COUNT; ++i) {
362 recvbuf[i] = 0xdeadbeef;
364 MPI_Iscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
367 case 18: /* MPI_Iexscan */
368 for (i = 0; i < COUNT; ++i) {
370 recvbuf[i] = 0xdeadbeef;
372 MPI_Iexscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
375 case 19: /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
376 for (i = 0; i < size; ++i) {
377 sendcounts[i] = COUNT;
378 recvcounts[i] = COUNT;
379 sdispls[i] = COUNT * i * sizeof(int);
380 rdispls[i] = COUNT * i * sizeof(int);
381 sendtypes[i] = MPI_INT;
382 recvtypes[i] = MPI_INT;
383 for (j = 0; j < COUNT; ++j) {
384 buf[i*COUNT+j] = rank + (i * j);
385 recvbuf[i*COUNT+j] = 0xdeadbeef;
388 MPI_Ialltoallw(buf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, req);
391 case 20: /* basic pt2pt MPI_Isend/MPI_Irecv pairing */
392 /* even ranks send to odd ranks, but only if we have a full pair */
393 if ((rank % 2 != 0) || (rank != size-1)) {
394 for (j = 0; j < COUNT; ++j) {
396 recvbuf[j] = 0xdeadbeef;
399 MPI_Isend(buf, COUNT, MPI_INT, rank+1, 5, comm, req);
401 MPI_Irecv(recvbuf, COUNT, MPI_INT, rank-1, 5, comm, req);
406 fprintf(stderr, "unexpected value for l->case_num=%d)\n", (l->case_num));
412 static void check_after_completion(struct laundry *l)
416 MPI_Comm comm = l->comm;
418 int *recvbuf = l->recvbuf;
419 int *sendcounts = l->sendcounts;
420 int *recvcounts = l->recvcounts;
421 int *sdispls = l->sdispls;
422 int *rdispls = l->rdispls;
423 int *sendtypes = l->sendtypes;
424 int *recvtypes = l->recvtypes;
425 char *buf_alias = (char *)buf;
427 MPI_Comm_rank(comm, &rank);
428 MPI_Comm_size(comm, &size);
430 /* these cases all correspond to cases in start_random_nonblocking */
431 switch (l->case_num) {
432 case 0: /* MPI_Ibcast */
433 for (i = 0; i < COUNT; ++i) {
435 printf("buf[%d]=%d i=%d\n", i, buf[i], i);
436 my_assert(buf[i] == i);
440 case 1: /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
441 for (i = 0; i < PRIME; ++i) {
442 if (buf_alias[i] != i)
443 printf("buf_alias[%d]=%d i=%d\n", i, buf_alias[i], i);
444 my_assert(buf_alias[i] == i);
448 case 2: /* MPI_Ibarrier */
449 /* nothing to check */
452 case 3: /* MPI_Ireduce */
454 for (i = 0; i < COUNT; ++i) {
455 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
456 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
457 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
462 case 4: /* same again, use a user op and free it before the wait */
464 for (i = 0; i < COUNT; ++i) {
465 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
466 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
467 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
472 case 5: /* MPI_Iallreduce */
473 for (i = 0; i < COUNT; ++i) {
474 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
475 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
476 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
480 case 6: /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
481 for (i = 0; i < size; ++i) {
482 for (j = 0; j < COUNT; ++j) {
483 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
484 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
489 case 7: /* MPI_Igather */
491 for (i = 0; i < size; ++i) {
492 for (j = 0; j < COUNT; ++j) {
493 my_assert(recvbuf[i*COUNT+j] == i + j);
498 for (i = 0; i < size*COUNT; ++i) {
499 my_assert(recvbuf[i] == 0xdeadbeef);
504 case 8: /* same test again, just use a dup'ed datatype and free it before the wait */
506 for (i = 0; i < size; ++i) {
507 for (j = 0; j < COUNT; ++j) {
508 my_assert(recvbuf[i*COUNT+j] == i + j);
513 for (i = 0; i < size*COUNT; ++i) {
514 my_assert(recvbuf[i] == 0xdeadbeef);
519 case 9: /* MPI_Iscatter */
520 for (j = 0; j < COUNT; ++j) {
521 my_assert(recvbuf[j] == rank + j);
524 for (i = 0; i < size*COUNT; ++i) {
525 /* check we didn't corrupt the sendbuf somehow */
526 my_assert(buf[i] == 0xdeadbeef);
531 case 10: /* MPI_Iscatterv */
532 for (j = 0; j < COUNT; ++j) {
533 my_assert(recvbuf[j] == rank + j);
536 for (i = 0; i < size*COUNT; ++i) {
537 /* check we didn't corrupt the sendbuf somehow */
538 my_assert(buf[i] == 0xdeadbeef);
541 for (i = 1; i < size; ++i) {
542 for (j = 0; j < COUNT; ++j) {
543 /* check we didn't corrupt the rest of the recvbuf */
544 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
549 case 11: /* MPI_Ireduce_scatter */
550 for (j = 0; j < COUNT; ++j) {
551 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
553 for (i = 1; i < size; ++i) {
554 for (j = 0; j < COUNT; ++j) {
555 /* check we didn't corrupt the rest of the recvbuf */
556 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
561 case 12: /* MPI_Ireduce_scatter_block */
562 for (j = 0; j < COUNT; ++j) {
563 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
565 for (i = 1; i < size; ++i) {
566 for (j = 0; j < COUNT; ++j) {
567 /* check we didn't corrupt the rest of the recvbuf */
568 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
573 case 13: /* MPI_Igatherv */
575 for (i = 0; i < size; ++i) {
576 for (j = 0; j < COUNT; ++j) {
577 my_assert(recvbuf[i*COUNT+j] == i + j);
582 for (i = 0; i < size*COUNT; ++i) {
583 my_assert(recvbuf[i] == 0xdeadbeef);
588 case 14: /* MPI_Ialltoall */
589 for (i = 0; i < size; ++i) {
590 for (j = 0; j < COUNT; ++j) {
591 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (i * j)));*/
592 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
597 case 15: /* MPI_Iallgather */
598 for (i = 0; i < size; ++i) {
599 for (j = 0; j < COUNT; ++j) {
600 my_assert(recvbuf[i*COUNT+j] == i + j);
605 case 16: /* MPI_Iallgatherv */
606 for (i = 0; i < size; ++i) {
607 for (j = 0; j < COUNT; ++j) {
608 my_assert(recvbuf[i*COUNT+j] == i + j);
613 case 17: /* MPI_Iscan */
614 for (i = 0; i < COUNT; ++i) {
615 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1))));
619 case 18: /* MPI_Iexscan */
620 for (i = 0; i < COUNT; ++i) {
622 my_assert(recvbuf[i] == 0xdeadbeef);
624 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1)) - (rank + i)));
628 case 19: /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
629 for (i = 0; i < size; ++i) {
630 for (j = 0; j < COUNT; ++j) {
631 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
632 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
637 case 20: /* basic pt2pt MPI_Isend/MPI_Irecv pairing */
638 /* even ranks send to odd ranks, but only if we have a full pair */
639 if ((rank % 2 != 0) || (rank != size-1)) {
640 for (j = 0; j < COUNT; ++j) {
641 /* only odd procs did a recv */
643 my_assert(recvbuf[j] == 0xdeadbeef);
646 if (recvbuf[j] != j) printf("recvbuf[%d]=%d j=%d\n", j, recvbuf[j], j);
647 my_assert(recvbuf[j] == j);
654 printf("invalid case_num (%d) detected\n", l->case_num);
661 static void complete_something_somehow(unsigned int rndnum, int numreqs, MPI_Request reqs[], int *outcount, int indices[])
665 #define COMPLETION_CASES (8)
666 switch (rand_range(rndnum, 0, COMPLETION_CASES)) {
668 MPI_Waitall(numreqs, reqs, MPI_STATUSES_IGNORE);
670 for (i = 0; i < numreqs; ++i) {
676 MPI_Testsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);
677 if (*outcount == MPI_UNDEFINED) {
683 MPI_Waitsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);
684 if (*outcount == MPI_UNDEFINED) {
690 MPI_Waitany(numreqs, reqs, &idx, MPI_STATUS_IGNORE);
691 if (idx == MPI_UNDEFINED) {
701 MPI_Testany(numreqs, reqs, &idx, &flag, MPI_STATUS_IGNORE);
702 if (idx == MPI_UNDEFINED) {
712 MPI_Testall(numreqs, reqs, &flag, MPI_STATUSES_IGNORE);
715 for (i = 0; i < numreqs; ++i) {
725 /* select a new random index and wait on it */
726 rndnum = gen_prn(rndnum);
727 idx = rand_range(rndnum, 0, numreqs);
728 MPI_Wait(&reqs[idx], MPI_STATUS_IGNORE);
734 /* select a new random index and wait on it */
735 rndnum = gen_prn(rndnum);
736 idx = rand_range(rndnum, 0, numreqs);
737 MPI_Test(&reqs[idx], &flag, MPI_STATUS_IGNORE);
738 *outcount = (flag ? 1 : 0);
746 #undef COMPLETION_CASES
748 #endif /* defined(TEST_NBC_ROUTINES) */
750 int main(int argc, char **argv)
752 int i, num_posted, num_completed;
754 unsigned int seed = 0x10bc;
755 unsigned int post_seq, complete_seq;
756 #if defined(TEST_NBC_ROUTINES)
757 struct laundry larr[WINDOW];
759 MPI_Request reqs[WINDOW];
762 MPI_Comm comms[NUM_COMMS];
765 MPI_Init(&argc, &argv);
766 MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
767 MPI_Comm_size(MPI_COMM_WORLD, &wsize);
769 #if defined(TEST_NBC_ROUTINES)
771 /* it is critical that all processes in the communicator start with a
772 * consistent value for "post_seq" */
773 post_seq = complete_seq = gen_prn(seed);
778 /* construct all of the communicators, just dups of comm world for now */
779 for (i = 0; i < NUM_COMMS; ++i) {
780 MPI_Comm_dup(MPI_COMM_WORLD, &comms[i]);
783 /* fill the entire window of ops */
784 for (i = 0; i < WINDOW; ++i) {
785 reqs[i] = MPI_REQUEST_NULL;
786 memset(&larr[i], 0, sizeof(struct laundry));
787 larr[i].case_num = -1;
789 /* randomly select a comm, using a new seed to avoid correlating
790 * particular kinds of NBC ops with particular communicators */
791 comm = comms[rand_range(gen_prn(post_seq), 0, NUM_COMMS)];
793 start_random_nonblocking(comm, post_seq, &reqs[i], &larr[i]);
795 post_seq = gen_prn(post_seq);
798 /* now loop repeatedly, completing ops with "random" completion functions,
799 * until we've posted and completed MAIN_ITERATIONS ops */
800 while (num_completed < MAIN_ITERATIONS) {
801 complete_something_somehow(complete_seq, WINDOW, reqs, &outcount, indices);
802 complete_seq = gen_prn(complete_seq);
803 for (i = 0; i < outcount; ++i) {
804 int idx = indices[i];
805 assert(reqs[idx] == MPI_REQUEST_NULL);
806 if (larr[idx].case_num != -1) {
807 check_after_completion(&larr[idx]);
808 cleanup_laundry(&larr[idx]);
810 if (num_posted < MAIN_ITERATIONS) {
811 comm = comms[rand_range(gen_prn(post_seq), 0, NUM_COMMS)];
812 start_random_nonblocking(comm, post_seq, &reqs[idx], &larr[idx]);
814 post_seq = gen_prn(post_seq);
819 /* "randomly" and infrequently introduce some jitter into the system */
820 if (0 == rand_range(gen_prn(complete_seq + wrank), 0, CHANCE_OF_SLEEP)) {
821 usleep(JITTER_DELAY); /* take a short nap */
825 for (i = 0; i < NUM_COMMS; ++i) {
826 MPI_Comm_free(&comms[i]);
829 #endif /* defined(TEST_NBC_ROUTINES) */
833 printf("found %d errors\n", errs);
835 printf(" No errors\n");