1 /* -*- Mode: C; c-basic-offset:4 ; -*- */
4 * (C) 2012 by Argonne National Laboratory.
5 * See COPYRIGHT in top-level directory.
18 #if defined (GACC_TYPE_SHORT)
20 # define TYPE_MPI_BASE MPI_SHORT
21 # define TYPE_FMT "%d"
22 #elif defined (GACC_TYPE_LONG)
24 # define TYPE_MPI_BASE MPI_LONG
25 # define TYPE_FMT "%ld"
26 #elif defined (GACC_TYPE_DOUBLE)
27 # define TYPE_C double
28 # define TYPE_MPI_BASE MPI_DOUBLE
29 # define TYPE_FMT "%f"
32 # define TYPE_MPI_BASE MPI_INT
33 # define TYPE_FMT "%d"
36 #if defined(GACC_TYPE_DERIVED)
37 # define TYPE_MPI derived_type
39 # define TYPE_MPI TYPE_MPI_BASE
42 static void reset_bufs(TYPE_C * win_ptr, TYPE_C * res_ptr, TYPE_C * val_ptr, TYPE_C value,
47 MPI_Barrier(MPI_COMM_WORLD);
49 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
50 MPI_Comm_size(MPI_COMM_WORLD, &nproc);
52 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
53 memset(win_ptr, 0, sizeof(TYPE_C) * nproc * COUNT);
54 MPI_Win_unlock(rank, win);
56 memset(res_ptr, -1, sizeof(TYPE_C) * nproc * COUNT);
58 for (i = 0; i < COUNT; i++)
61 MPI_Barrier(MPI_COMM_WORLD);
64 int main(int argc, char **argv)
67 int errors = 0, all_errors = 0;
68 TYPE_C *win_ptr, *res_ptr, *val_ptr;
70 #if defined (GACC_TYPE_DERIVED)
71 MPI_Datatype derived_type;
74 MPI_Init(&argc, &argv);
76 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
77 MPI_Comm_size(MPI_COMM_WORLD, &nproc);
79 win_ptr = malloc(sizeof(TYPE_C) * nproc * COUNT);
80 res_ptr = malloc(sizeof(TYPE_C) * nproc * COUNT);
81 val_ptr = malloc(sizeof(TYPE_C) * COUNT);
83 #if defined (GACC_TYPE_DERIVED)
84 MPI_Type_contiguous(1, TYPE_MPI_BASE, &derived_type);
85 MPI_Type_commit(&derived_type);
88 MPI_Win_create(win_ptr, sizeof(TYPE_C) * nproc * COUNT, sizeof(TYPE_C),
89 MPI_INFO_NULL, MPI_COMM_WORLD, &win);
91 /* Test self communication */
93 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
95 for (i = 0; i < ITER; i++) {
96 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
97 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
98 rank, 0, COUNT, TYPE_MPI, MPI_SUM, win);
99 MPI_Win_unlock(rank, win);
102 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
103 for (i = 0; i < COUNT; i++) {
104 if (win_ptr[i] != ITER) {
105 SQUELCH(printf("%d->%d -- SELF[%d]: expected " TYPE_FMT ", got " TYPE_FMT "\n",
106 rank, rank, i, (TYPE_C) ITER, win_ptr[i]););
110 MPI_Win_unlock(rank, win);
112 /* Test neighbor communication */
114 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
116 for (i = 0; i < ITER; i++) {
117 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, (rank + 1) % nproc, 0, win);
118 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
119 (rank + 1) % nproc, 0, COUNT, TYPE_MPI, MPI_SUM, win);
120 MPI_Win_unlock((rank + 1) % nproc, win);
123 MPI_Barrier(MPI_COMM_WORLD);
125 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
126 for (i = 0; i < COUNT; i++) {
127 if (win_ptr[i] != ITER) {
128 SQUELCH(printf("%d->%d -- NEIGHBOR[%d]: expected " TYPE_FMT ", got " TYPE_FMT "\n",
129 (rank + 1) % nproc, rank, i, (TYPE_C) ITER, win_ptr[i]););
133 MPI_Win_unlock(rank, win);
135 /* Test contention */
137 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
140 for (i = 0; i < ITER; i++) {
141 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 0, 0, win);
142 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
143 0, 0, COUNT, TYPE_MPI, MPI_SUM, win);
144 MPI_Win_unlock(0, win);
148 MPI_Barrier(MPI_COMM_WORLD);
150 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
151 if (rank == 0 && nproc > 1) {
152 for (i = 0; i < COUNT; i++) {
153 if (win_ptr[i] != ITER * (nproc - 1)) {
154 SQUELCH(printf("*->%d - CONTENTION[%d]: expected=" TYPE_FMT " val=" TYPE_FMT "\n",
155 rank, i, (TYPE_C) ITER * (nproc - 1), win_ptr[i]););
160 MPI_Win_unlock(rank, win);
162 /* Test all-to-all communication (fence) */
164 reset_bufs(win_ptr, res_ptr, val_ptr, rank, win);
166 for (i = 0; i < ITER; i++) {
169 MPI_Win_fence(MPI_MODE_NOPRECEDE, win);
170 for (j = 0; j < nproc; j++) {
171 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, &res_ptr[j * COUNT], COUNT, TYPE_MPI,
172 j, rank * COUNT, COUNT, TYPE_MPI, MPI_SUM, win);
174 MPI_Win_fence(MPI_MODE_NOSUCCEED, win);
175 MPI_Barrier(MPI_COMM_WORLD);
177 for (j = 0; j < nproc; j++) {
179 for (c = 0; c < COUNT; c++) {
180 if (res_ptr[j * COUNT + c] != i * rank) {
182 ("%d->%d -- ALL-TO-ALL (FENCE) [%d]: iter %d, expected result " TYPE_FMT
183 ", got " TYPE_FMT "\n", rank, j, c, i, (TYPE_C) i * rank,
184 res_ptr[j * COUNT + c]););
191 MPI_Barrier(MPI_COMM_WORLD);
192 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
193 for (i = 0; i < nproc; i++) {
195 for (c = 0; c < COUNT; c++) {
196 if (win_ptr[i * COUNT + c] != ITER * i) {
198 ("%d->%d -- ALL-TO-ALL (FENCE): expected " TYPE_FMT ", got " TYPE_FMT "\n",
199 i, rank, (TYPE_C) ITER * i, win_ptr[i * COUNT + c]););
204 MPI_Win_unlock(rank, win);
206 /* Test all-to-all communication (lock-all) */
208 reset_bufs(win_ptr, res_ptr, val_ptr, rank, win);
210 for (i = 0; i < ITER; i++) {
213 MPI_Win_lock_all(0, win);
214 for (j = 0; j < nproc; j++) {
215 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, &res_ptr[j * COUNT], COUNT, TYPE_MPI,
216 j, rank * COUNT, COUNT, TYPE_MPI, MPI_SUM, win);
218 MPI_Win_unlock_all(win);
219 MPI_Barrier(MPI_COMM_WORLD);
221 for (j = 0; j < nproc; j++) {
223 for (c = 0; c < COUNT; c++) {
224 if (res_ptr[j * COUNT + c] != i * rank) {
226 ("%d->%d -- ALL-TO-ALL (LOCK-ALL) [%d]: iter %d, expected result "
227 TYPE_FMT ", got " TYPE_FMT "\n", rank, j, c, i, (TYPE_C) i * rank,
228 res_ptr[j * COUNT + c]););
235 MPI_Barrier(MPI_COMM_WORLD);
236 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
237 for (i = 0; i < nproc; i++) {
239 for (c = 0; c < COUNT; c++) {
240 if (win_ptr[i * COUNT + c] != ITER * i) {
242 ("%d->%d -- ALL-TO-ALL (LOCK-ALL): expected " TYPE_FMT ", got " TYPE_FMT
243 "\n", i, rank, (TYPE_C) ITER * i, win_ptr[i * COUNT + c]););
248 MPI_Win_unlock(rank, win);
250 /* Test all-to-all communication (lock-all+flush) */
252 reset_bufs(win_ptr, res_ptr, val_ptr, rank, win);
254 for (i = 0; i < ITER; i++) {
257 MPI_Win_lock_all(0, win);
258 for (j = 0; j < nproc; j++) {
259 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, &res_ptr[j * COUNT], COUNT, TYPE_MPI,
260 j, rank * COUNT, COUNT, TYPE_MPI, MPI_SUM, win);
261 MPI_Win_flush(j, win);
263 MPI_Win_unlock_all(win);
264 MPI_Barrier(MPI_COMM_WORLD);
266 for (j = 0; j < nproc; j++) {
268 for (c = 0; c < COUNT; c++) {
269 if (res_ptr[j * COUNT + c] != i * rank) {
271 ("%d->%d -- ALL-TO-ALL (LOCK-ALL+FLUSH) [%d]: iter %d, expected result "
272 TYPE_FMT ", got " TYPE_FMT "\n", rank, j, c, i, (TYPE_C) i * rank,
273 res_ptr[j * COUNT + c]););
280 MPI_Barrier(MPI_COMM_WORLD);
281 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
282 for (i = 0; i < nproc; i++) {
284 for (c = 0; c < COUNT; c++) {
285 if (win_ptr[i * COUNT + c] != ITER * i) {
287 ("%d->%d -- ALL-TO-ALL (LOCK-ALL+FLUSH): expected " TYPE_FMT ", got "
288 TYPE_FMT "\n", i, rank, (TYPE_C) ITER * i, win_ptr[i * COUNT + c]););
293 MPI_Win_unlock(rank, win);
295 /* Test NO_OP (neighbor communication) */
297 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
299 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
300 for (i = 0; i < COUNT * nproc; i++)
301 win_ptr[i] = (TYPE_C) rank;
302 MPI_Win_unlock(rank, win);
303 MPI_Barrier(MPI_COMM_WORLD);
305 for (i = 0; i < ITER; i++) {
306 int j, target = (rank + 1) % nproc;
308 /* Test: origin_buf = NULL */
309 memset(res_ptr, -1, sizeof(TYPE_C) * nproc * COUNT); /* reset result buffer. */
311 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
312 MPI_Get_accumulate(NULL, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
313 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
314 MPI_Win_unlock(target, win);
316 for (j = 0; j < COUNT; j++) {
317 if (res_ptr[j] != (TYPE_C) target) {
318 SQUELCH(printf("%d->%d -- NOP(1)[%d]: expected " TYPE_FMT ", got " TYPE_FMT "\n",
319 target, rank, i, (TYPE_C) target, res_ptr[i]););
324 /* Test: origin_buf = NULL, origin_count = 0 */
325 memset(res_ptr, -1, sizeof(TYPE_C) * nproc * COUNT);
327 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
328 MPI_Get_accumulate(NULL, 0, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
329 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
330 MPI_Win_unlock(target, win);
332 for (j = 0; j < COUNT; j++) {
333 if (res_ptr[j] != (TYPE_C) target) {
334 SQUELCH(printf("%d->%d -- NOP(2)[%d]: expected " TYPE_FMT ", got " TYPE_FMT "\n",
335 target, rank, i, (TYPE_C) target, res_ptr[i]););
340 /* Test: origin_buf = NULL, origin_count = 0, origin_dtype = NULL */
341 memset(res_ptr, -1, sizeof(TYPE_C) * nproc * COUNT);
343 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
344 MPI_Get_accumulate(NULL, 0, MPI_DATATYPE_NULL, res_ptr, COUNT, TYPE_MPI,
345 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
346 MPI_Win_unlock(target, win);
348 for (j = 0; j < COUNT; j++) {
349 if (res_ptr[j] != (TYPE_C) target) {
350 SQUELCH(printf("%d->%d -- NOP(2)[%d]: expected " TYPE_FMT ", got " TYPE_FMT "\n",
351 target, rank, i, (TYPE_C) target, res_ptr[i]););
357 /* Test NO_OP (self communication) */
359 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
361 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
362 for (i = 0; i < COUNT * nproc; i++)
363 win_ptr[i] = (TYPE_C) rank;
364 MPI_Win_unlock(rank, win);
365 MPI_Barrier(MPI_COMM_WORLD);
367 for (i = 0; i < ITER; i++) {
368 int j, target = rank;
370 /* Test: origin_buf = NULL */
371 memset(res_ptr, -1, sizeof(TYPE_C) * nproc * COUNT);
373 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
374 MPI_Get_accumulate(NULL, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
375 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
376 MPI_Win_unlock(target, win);
378 for (j = 0; j < COUNT; j++) {
379 if (res_ptr[j] != (TYPE_C) target) {
381 ("%d->%d -- NOP_SELF(1)[%d]: expected " TYPE_FMT ", got " TYPE_FMT "\n",
382 target, rank, i, (TYPE_C) target, res_ptr[i]););
387 /* Test: origin_buf = NULL, origin_count = 0 */
388 memset(res_ptr, -1, sizeof(TYPE_C) * nproc * COUNT);
390 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
391 MPI_Get_accumulate(NULL, 0, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
392 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
393 MPI_Win_unlock(target, win);
395 for (j = 0; j < COUNT; j++) {
396 if (res_ptr[j] != (TYPE_C) target) {
398 ("%d->%d -- NOP_SELF(2)[%d]: expected " TYPE_FMT ", got " TYPE_FMT "\n",
399 target, rank, i, (TYPE_C) target, res_ptr[i]););
404 /* Test: origin_buf = NULL, origin_count = 0, origin_dtype = NULL */
405 memset(res_ptr, -1, sizeof(TYPE_C) * nproc * COUNT);
407 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
408 MPI_Get_accumulate(NULL, 0, MPI_DATATYPE_NULL, res_ptr, COUNT, TYPE_MPI,
409 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
410 MPI_Win_unlock(target, win);
412 for (j = 0; j < COUNT; j++) {
413 if (res_ptr[j] != (TYPE_C) target) {
415 ("%d->%d -- NOP_SELF(2)[%d]: expected " TYPE_FMT ", got " TYPE_FMT "\n",
416 target, rank, i, (TYPE_C) target, res_ptr[i]););
424 MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
426 if (rank == 0 && all_errors == 0)
427 printf(" No Errors\n");
429 #if defined (GACC_TYPE_DERIVED)
430 MPI_Type_free(&derived_type);