1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
4 * (C) 2016 by Argonne National Laboratory.
5 * See COPYRIGHT in top-level directory.
13 /* This test checks the remote completion of flush with RMA write-like operations
14 * (PUT, ACC, GET_ACC, FOP, CAS) concurrently issued from different origin processes
15 * to the same target over overlapping windows (i.e., two windows exposing the same
17 * 1. The first [nprocs-1] processes perform as origin, and the last process
19 * 2. Everyone allocates a buffer and creates [nprocs-1] windows over the same buffer.
20 * 3. Every origin P[i] issues RMA operations and flush to target through
21 * wins[i] respectively, to different location winbuf[i] on target.
22 * 4. Finally, every origin P[i] issues GET and flush to obtain winbuf[i] on
23 * target and checks the correctness. */
29 #define MPI_DATATYPE MPI_INT
31 #define DATATYPE_FORMAT "%d"
33 #define MPI_DATATYPE MPI_DOUBLE
34 #define DATATYPE double
35 #define DATATYPE_FORMAT "%.1f"
38 DATATYPE local_buf[BUF_CNT], result_buf[BUF_CNT], compare_buf[BUF_CNT];
39 DATATYPE exp_target_val = 0.0;
41 const int verbose = 0;
43 int rank = -1, nprocs = 0;
46 int win_size = 0, win_cnt = 0;
48 DATATYPE *winbuf = NULL, *my_base = NULL;
50 #define verbose_print(str,...) { \
52 fprintf(stdout, str, ## __VA_ARGS__); \
56 #define error_print(str,...) { \
57 fprintf(stderr, str, ## __VA_ARGS__); \
61 /* Define operation name for error message */
63 const char *rma_name = "Put";
64 #elif defined(TEST_ACC)
65 const char *rma_name = "Accumulate";
66 #elif defined(TEST_GACC)
67 const char *rma_name = "Get_accumulate";
68 #elif defined(TEST_FOP)
69 const char *rma_name = "Fetch_and_op";
70 #elif defined(TEST_CAS)
71 const char *rma_name = "Compare_and_swap";
73 const char *rma_name = "None";
76 /* Issue functions for different RMA operations */
78 static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr /* NULL */ ,
79 DATATYPE * compare_addr /* NULL */ , int dst, MPI_Aint target_disp,
82 MPI_Put(origin_addr, 1, MPI_DATATYPE, dst, target_disp, 1, MPI_DATATYPE, win);
84 #elif defined(TEST_ACC)
85 static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr /* NULL */ ,
86 DATATYPE * compare_addr /* NULL */ , int dst, MPI_Aint target_disp,
89 MPI_Accumulate(origin_addr, 1, MPI_DATATYPE, dst, target_disp, 1, MPI_DATATYPE, MPI_SUM, win);
91 #elif defined(TEST_GACC)
92 static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr,
93 DATATYPE * compare_addr /* NULL */ , int dst, MPI_Aint target_disp,
96 MPI_Get_accumulate(origin_addr, 1, MPI_DATATYPE, result_addr, 1, MPI_DATATYPE, dst, target_disp,
97 1, MPI_DATATYPE, MPI_SUM, win);
99 #elif defined(TEST_FOP)
100 static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr,
101 DATATYPE * compare_addr /* NULL */ , int dst, MPI_Aint target_disp,
104 MPI_Fetch_and_op(origin_addr, result_addr, MPI_DATATYPE, dst, target_disp, MPI_SUM, win);
106 #elif defined(TEST_CAS)
107 static inline void issue_rma_op(DATATYPE * origin_addr, DATATYPE * result_addr,
108 DATATYPE * compare_addr, int dst, MPI_Aint target_disp, MPI_Win win)
110 MPI_Compare_and_swap(origin_addr, compare_addr, result_addr, MPI_DATATYPE, dst, target_disp,
114 #define issue_rma_op(loc_addr, result_addr, compare_addr, dst, target_disp, win)
117 static inline void set_iteration_data(int x)
121 #if defined(TEST_CAS)
122 for (i = 0; i < BUF_CNT; i++)
123 compare_buf[i] = local_buf[i]; /* always equal, thus swap happens */
126 for (i = 0; i < BUF_CNT; i++) {
127 local_buf[i] = rank + i + x;
129 #if defined(TEST_CAS) || defined(TEST_PUT)
130 exp_target_val = local_buf[i]; /* swap */
132 exp_target_val += local_buf[i]; /* sum */
137 static void print_origin_data(void)
141 printf("[%d] local_buf: ", rank);
142 for (i = 0; i < BUF_CNT; i++)
143 printf(DATATYPE_FORMAT " ", local_buf[i]);
146 printf("[%d] result_buf: ", rank);
147 for (i = 0; i < BUF_CNT; i++)
148 printf(DATATYPE_FORMAT " ", result_buf[i]);
152 static void print_target_data(void)
155 printf("[%d] winbuf: ", rank);
156 for (i = 0; i < win_cnt; i++)
157 printf(DATATYPE_FORMAT " ", winbuf[i]);
162 static int run_test()
166 int dst = 0, target_disp = 0;
167 MPI_Win win = MPI_WIN_NULL;
168 DATATYPE target_val = 0.0;
170 /* 1. Specify working window and displacement.
171 * - Target: no RMA issued, always check results on wins[0].
172 * - Origins: issue RMA on different window and different memory location */
173 if (rank == target) {
182 /* 2. Every one resets local data */
183 memset(local_buf, 0, sizeof(local_buf));
184 memset(result_buf, 0, sizeof(result_buf));
185 memset(compare_buf, 0, sizeof(compare_buf));
187 MPI_Barrier(MPI_COMM_WORLD);
189 if (rank != target) {
191 /* 3. Origins issue RMA to target over its working window */
192 MPI_Win_lock(MPI_LOCK_SHARED, dst, 0, win);
193 verbose_print("[%d] RMA start, test %s (dst=%d, target_disp=%d, win 0x%x) - flush\n",
194 rank, rma_name, dst, target_disp, win);
196 for (x = 0; x < ITER; x++) {
197 /* update local buffers and expected value in every iteration */
198 set_iteration_data(x);
200 for (i = 0; i < BUF_CNT; i++)
201 issue_rma_op(&local_buf[i], &result_buf[i], &compare_buf[i], dst, target_disp, win);
202 MPI_Win_flush(dst, win);
208 /* 4. Check correctness of final target value */
209 MPI_Get(&target_val, 1, MPI_DATATYPE, dst, target_disp, 1, MPI_DATATYPE, win);
210 MPI_Win_flush(dst, win);
211 if (target_val != exp_target_val) {
212 error_print("rank %d (iter %d) - check %s, got target_val = "
213 DATATYPE_FORMAT ", expected " DATATYPE_FORMAT "\n", rank, x,
214 rma_name, target_val, exp_target_val);
218 MPI_Win_unlock(dst, win);
221 MPI_Barrier(MPI_COMM_WORLD);
223 /* 5. Every one prints window buffer */
224 if (verbose && rank == target) {
225 MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win);
227 MPI_Win_unlock(rank, win);
233 static void init_windows(void)
237 /* Everyone creates norigins overlapping windows. */
238 winbuf = malloc(win_size);
239 memset(winbuf, 0, win_size);
241 wins = malloc(norigins * sizeof(MPI_Win));
242 for (i = 0; i < norigins; i++) {
243 wins[i] = MPI_WIN_NULL;
244 MPI_Win_create(winbuf, win_size, sizeof(DATATYPE), MPI_INFO_NULL, MPI_COMM_WORLD, &wins[i]);
248 static void destroy_windows(void)
251 for (i = 0; i < norigins; i++) {
252 if (wins[i] != MPI_WIN_NULL)
253 MPI_Win_free(&wins[i]);
259 int main(int argc, char *argv[])
261 int errors = 0, all_errors = 0;
263 MPI_Init(&argc, &argv);
264 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
265 MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
269 error_print("Error: must use at least 3 processes\n");
271 MPI_Barrier(MPI_COMM_WORLD);
272 MPI_Abort(MPI_COMM_WORLD, 1);
275 /* The last rank performs as target, all others are origin.
276 * Every origin accesses to a different memory location on the target. */
278 norigins = nprocs - 1;
279 win_cnt = nprocs - 1;
280 win_size = sizeof(DATATYPE) * win_cnt;
283 verbose_print("[%d] %d origins, target rank = %d\n", rank, norigins, target);
287 MPI_Barrier(MPI_COMM_WORLD);
292 MPI_Barrier(MPI_COMM_WORLD);
293 MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
295 if (rank == 0 && all_errors == 0) {
296 fprintf(stdout, " No Errors\n");