1 /* -*- Mode: C; c-basic-offset:4 ; -*- */
4 * (C) 2012 by Argonne National Laboratory.
5 * See COPYRIGHT in top-level directory.
18 #if defined (GACC_TYPE_SHORT)
20 # define TYPE_MPI_BASE MPI_SHORT
21 # define TYPE_FMT "%d"
22 #elif defined (GACC_TYPE_LONG)
24 # define TYPE_MPI_BASE MPI_LONG
25 # define TYPE_FMT "%ld"
26 #elif defined (GACC_TYPE_DOUBLE)
27 # define TYPE_C double
28 # define TYPE_MPI_BASE MPI_DOUBLE
29 # define TYPE_FMT "%f"
32 # define TYPE_MPI_BASE MPI_INT
33 # define TYPE_FMT "%d"
36 #if defined(GACC_TYPE_DERIVED)
37 # define TYPE_MPI derived_type
39 # define TYPE_MPI TYPE_MPI_BASE
42 void reset_bufs(TYPE_C *win_ptr, TYPE_C *res_ptr, TYPE_C *val_ptr, TYPE_C value, MPI_Win win) {
45 MPI_Barrier(MPI_COMM_WORLD);
47 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
48 MPI_Comm_size(MPI_COMM_WORLD, &nproc);
50 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
51 memset(win_ptr, 0, sizeof(TYPE_C)*nproc*COUNT);
52 MPI_Win_unlock(rank, win);
54 memset(res_ptr, -1, sizeof(TYPE_C)*nproc*COUNT);
56 for (i = 0; i < COUNT; i++)
59 MPI_Barrier(MPI_COMM_WORLD);
62 int main(int argc, char **argv) {
64 int errors = 0, all_errors = 0;
65 TYPE_C *win_ptr, *res_ptr, *val_ptr;
67 #if defined (GACC_TYPE_DERIVED)
68 MPI_Datatype derived_type;
71 MPI_Init(&argc, &argv);
73 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
74 MPI_Comm_size(MPI_COMM_WORLD, &nproc);
76 win_ptr = malloc(sizeof(TYPE_C)*nproc*COUNT);
77 res_ptr = malloc(sizeof(TYPE_C)*nproc*COUNT);
78 val_ptr = malloc(sizeof(TYPE_C)*COUNT);
80 #if defined (GACC_TYPE_DERIVED)
81 MPI_Type_contiguous(1, TYPE_MPI_BASE, &derived_type);
82 MPI_Type_commit(&derived_type);
85 MPI_Win_create(win_ptr, sizeof(TYPE_C)*nproc*COUNT, sizeof(TYPE_C),
86 MPI_INFO_NULL, MPI_COMM_WORLD, &win);
88 /* Test self communication */
90 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
92 for (i = 0; i < ITER; i++) {
93 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
94 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
95 rank, 0, COUNT, TYPE_MPI, MPI_SUM, win);
96 MPI_Win_unlock(rank, win);
99 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
100 for (i = 0; i < COUNT; i++) {
101 if (win_ptr[i] != ITER) {
102 SQUELCH( printf("%d->%d -- SELF[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n",
103 rank, rank, i, (TYPE_C) ITER, win_ptr[i]); );
107 MPI_Win_unlock(rank, win);
109 /* Test neighbor communication */
111 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
113 for (i = 0; i < ITER; i++) {
114 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, (rank+1)%nproc, 0, win);
115 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
116 (rank+1)%nproc, 0, COUNT, TYPE_MPI, MPI_SUM, win);
117 MPI_Win_unlock((rank+1)%nproc, win);
120 MPI_Barrier(MPI_COMM_WORLD);
122 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
123 for (i = 0; i < COUNT; i++) {
124 if (win_ptr[i] != ITER) {
125 SQUELCH( printf("%d->%d -- NEIGHBOR[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n",
126 (rank+1)%nproc, rank, i, (TYPE_C) ITER, win_ptr[i]); );
130 MPI_Win_unlock(rank, win);
132 /* Test contention */
134 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
137 for (i = 0; i < ITER; i++) {
138 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 0, 0, win);
139 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
140 0, 0, COUNT, TYPE_MPI, MPI_SUM, win);
141 MPI_Win_unlock(0, win);
145 MPI_Barrier(MPI_COMM_WORLD);
147 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
148 if (rank == 0 && nproc > 1) {
149 for (i = 0; i < COUNT; i++) {
150 if (win_ptr[i] != ITER*(nproc-1)) {
151 SQUELCH( printf("*->%d - CONTENTION[%d]: expected="TYPE_FMT" val="TYPE_FMT"\n",
152 rank, i, (TYPE_C) ITER*(nproc-1), win_ptr[i]); );
157 MPI_Win_unlock(rank, win);
159 /* Test all-to-all communication (fence) */
161 reset_bufs(win_ptr, res_ptr, val_ptr, rank, win);
163 for (i = 0; i < ITER; i++) {
166 MPI_Win_fence(MPI_MODE_NOPRECEDE, win);
167 for (j = 0; j < nproc; j++) {
168 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, &res_ptr[j*COUNT], COUNT, TYPE_MPI,
169 j, rank*COUNT, COUNT, TYPE_MPI, MPI_SUM, win);
171 MPI_Win_fence(MPI_MODE_NOSUCCEED, win);
172 MPI_Barrier(MPI_COMM_WORLD);
174 for (j = 0; j < nproc; j++) {
176 for (c = 0; c < COUNT; c++) {
177 if (res_ptr[j*COUNT+c] != i*rank) {
178 SQUELCH( printf("%d->%d -- ALL-TO-ALL (FENCE) [%d]: iter %d, expected result "TYPE_FMT", got "TYPE_FMT"\n",
179 rank, j, c, i, (TYPE_C) i*rank, res_ptr[j*COUNT+c]); );
186 MPI_Barrier(MPI_COMM_WORLD);
187 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
188 for (i = 0; i < nproc; i++) {
190 for (c = 0; c < COUNT; c++) {
191 if (win_ptr[i*COUNT+c] != ITER*i) {
192 SQUELCH( printf("%d->%d -- ALL-TO-ALL (FENCE): expected "TYPE_FMT", got "TYPE_FMT"\n",
193 i, rank, (TYPE_C) ITER*i, win_ptr[i*COUNT+c]); );
198 MPI_Win_unlock(rank, win);
200 /* Test all-to-all communication (lock-all) */
202 reset_bufs(win_ptr, res_ptr, val_ptr, rank, win);
204 for (i = 0; i < ITER; i++) {
207 MPI_Win_lock_all(0, win);
208 for (j = 0; j < nproc; j++) {
209 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, &res_ptr[j*COUNT], COUNT, TYPE_MPI,
210 j, rank*COUNT, COUNT, TYPE_MPI, MPI_SUM, win);
212 MPI_Win_unlock_all(win);
213 MPI_Barrier(MPI_COMM_WORLD);
215 for (j = 0; j < nproc; j++) {
217 for (c = 0; c < COUNT; c++) {
218 if (res_ptr[j*COUNT+c] != i*rank) {
219 SQUELCH( printf("%d->%d -- ALL-TO-ALL (LOCK-ALL) [%d]: iter %d, expected result "TYPE_FMT", got "TYPE_FMT"\n",
220 rank, j, c, i, (TYPE_C) i*rank, res_ptr[j*COUNT+c]); );
227 MPI_Barrier(MPI_COMM_WORLD);
228 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
229 for (i = 0; i < nproc; i++) {
231 for (c = 0; c < COUNT; c++) {
232 if (win_ptr[i*COUNT+c] != ITER*i) {
233 SQUELCH( printf("%d->%d -- ALL-TO-ALL (LOCK-ALL): expected "TYPE_FMT", got "TYPE_FMT"\n",
234 i, rank, (TYPE_C) ITER*i, win_ptr[i*COUNT+c]); );
239 MPI_Win_unlock(rank, win);
241 /* Test all-to-all communication (lock-all+flush) */
243 reset_bufs(win_ptr, res_ptr, val_ptr, rank, win);
245 for (i = 0; i < ITER; i++) {
248 MPI_Win_lock_all(0, win);
249 for (j = 0; j < nproc; j++) {
250 MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, &res_ptr[j*COUNT], COUNT, TYPE_MPI,
251 j, rank*COUNT, COUNT, TYPE_MPI, MPI_SUM, win);
252 MPI_Win_flush(j, win);
254 MPI_Win_unlock_all(win);
255 MPI_Barrier(MPI_COMM_WORLD);
257 for (j = 0; j < nproc; j++) {
259 for (c = 0; c < COUNT; c++) {
260 if (res_ptr[j*COUNT+c] != i*rank) {
261 SQUELCH( printf("%d->%d -- ALL-TO-ALL (LOCK-ALL+FLUSH) [%d]: iter %d, expected result "TYPE_FMT", got "TYPE_FMT"\n",
262 rank, j, c, i, (TYPE_C) i*rank, res_ptr[j*COUNT+c]); );
269 MPI_Barrier(MPI_COMM_WORLD);
270 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
271 for (i = 0; i < nproc; i++) {
273 for (c = 0; c < COUNT; c++) {
274 if (win_ptr[i*COUNT+c] != ITER*i) {
275 SQUELCH( printf("%d->%d -- ALL-TO-ALL (LOCK-ALL+FLUSH): expected "TYPE_FMT", got "TYPE_FMT"\n",
276 i, rank, (TYPE_C) ITER*i, win_ptr[i*COUNT+c]); );
281 MPI_Win_unlock(rank, win);
283 /* Test NO_OP (neighbor communication) */
285 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
287 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
288 for (i = 0; i < COUNT*nproc; i++)
289 win_ptr[i] = (TYPE_C) rank;
290 MPI_Win_unlock(rank, win);
291 MPI_Barrier(MPI_COMM_WORLD);
293 for (i = 0; i < ITER; i++) {
294 int j, target = (rank+1) % nproc;
296 /* Test: origin_buf = NULL */
297 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
298 MPI_Get_accumulate(NULL, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
299 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
300 MPI_Win_unlock(target, win);
302 for (j = 0; j < COUNT; j++) {
303 if (res_ptr[j] != (TYPE_C) target) {
304 SQUELCH( printf("%d->%d -- NOP(1)[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n",
305 target, rank, i, (TYPE_C) target, res_ptr[i]); );
310 /* Test: origin_buf = NULL, origin_count = 0 */
311 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
312 MPI_Get_accumulate(NULL, 0, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
313 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
314 MPI_Win_unlock(target, win);
316 for (j = 0; j < COUNT; j++) {
317 if (res_ptr[j] != (TYPE_C) target) {
318 SQUELCH( printf("%d->%d -- NOP(2)[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n",
319 target, rank, i, (TYPE_C) target, res_ptr[i]); );
324 /* Test: origin_buf = NULL, origin_count = 0, origin_dtype = NULL */
325 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
326 MPI_Get_accumulate(NULL, 0, MPI_DATATYPE_NULL, res_ptr, COUNT, TYPE_MPI,
327 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
328 MPI_Win_unlock(target, win);
330 for (j = 0; j < COUNT; j++) {
331 if (res_ptr[j] != (TYPE_C) target) {
332 SQUELCH( printf("%d->%d -- NOP(2)[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n",
333 target, rank, i, (TYPE_C) target, res_ptr[i]); );
339 /* Test NO_OP (self communication) */
341 reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
343 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
344 for (i = 0; i < COUNT*nproc; i++)
345 win_ptr[i] = (TYPE_C) rank;
346 MPI_Win_unlock(rank, win);
347 MPI_Barrier(MPI_COMM_WORLD);
349 for (i = 0; i < ITER; i++) {
350 int j, target = rank;
352 /* Test: origin_buf = NULL */
353 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
354 MPI_Get_accumulate(NULL, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
355 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
356 MPI_Win_unlock(target, win);
358 for (j = 0; j < COUNT; j++) {
359 if (res_ptr[j] != (TYPE_C) target) {
360 SQUELCH( printf("%d->%d -- NOP_SELF(1)[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n",
361 target, rank, i, (TYPE_C) target, res_ptr[i]); );
366 /* Test: origin_buf = NULL, origin_count = 0 */
367 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
368 MPI_Get_accumulate(NULL, 0, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
369 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
370 MPI_Win_unlock(target, win);
372 for (j = 0; j < COUNT; j++) {
373 if (res_ptr[j] != (TYPE_C) target) {
374 SQUELCH( printf("%d->%d -- NOP_SELF(2)[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n",
375 target, rank, i, (TYPE_C) target, res_ptr[i]); );
380 /* Test: origin_buf = NULL, origin_count = 0, origin_dtype = NULL */
381 MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, win);
382 MPI_Get_accumulate(NULL, 0, MPI_DATATYPE_NULL, res_ptr, COUNT, TYPE_MPI,
383 target, 0, COUNT, TYPE_MPI, MPI_NO_OP, win);
384 MPI_Win_unlock(target, win);
386 for (j = 0; j < COUNT; j++) {
387 if (res_ptr[j] != (TYPE_C) target) {
388 SQUELCH( printf("%d->%d -- NOP_SELF(2)[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n",
389 target, rank, i, (TYPE_C) target, res_ptr[i]); );
397 MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
399 if (rank == 0 && all_errors == 0)
400 printf(" No Errors\n");
402 #if defined (GACC_TYPE_DERIVED)
403 MPI_Type_free(&derived_type);