1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
4 * (C) 2015 by Argonne National Laboratory.
5 * See COPYRIGHT in top-level directory.
15 int local_buf[BUF_CNT], result_addr[BUF_CNT], check_addr[BUF_CNT];
17 int compare_buf[BUF_CNT];
20 const int verbose = 0;
22 /* This test checks the remote completion of flush with RMA write-like operations
23 * (PUT, ACC, GET_ACC, FOP, CAS), and confirms result by GET issued from another
25 * 1. Three processes create window by Win_allocate.
26 * 2. P(origin) issues RMA operations and flush to P(target), then call
27 * send-recv to synchronize with P(checker).
28 * 3. P(checker) then issues get to P(target) to check the results.
31 int rank = -1, nproc = 0;
32 int origin = -1, target = -1, checker = -1;
33 MPI_Win win = MPI_WIN_NULL;
36 /* Define operation name for error message */
38 const char *rma_name = "Put";
39 #elif defined(TEST_ACC)
40 const char *rma_name = "Accumulate";
41 #elif defined(TEST_GACC)
42 const char *rma_name = "Get_accumulate";
43 #elif defined(TEST_FOP)
44 const char *rma_name = "Fetch_and_op";
45 #elif defined(TEST_CAS)
46 const char *rma_name = "Compare_and_swap";
48 const char *rma_name = "None";
51 /* Issue functions for different RMA operations */
53 static inline void issue_rma_op(int i)
55 MPI_Put(&local_buf[i], 1, MPI_INT, target, i, 1, MPI_INT, win);
57 #elif defined(TEST_ACC)
58 static inline void issue_rma_op(int i)
60 MPI_Accumulate(&local_buf[i], 1, MPI_INT, target, i, 1, MPI_INT, MPI_REPLACE, win);
62 #elif defined(TEST_GACC)
63 static inline void issue_rma_op(int i)
65 MPI_Get_accumulate(&local_buf[i], 1, MPI_INT, &result_addr[i], 1, MPI_INT, target, i,
66 1, MPI_INT, MPI_REPLACE, win);
68 #elif defined(TEST_FOP)
69 static inline void issue_rma_op(int i)
71 MPI_Fetch_and_op(&local_buf[i], &result_addr[i], MPI_INT, target, i, MPI_REPLACE, win);
73 #elif defined(TEST_CAS)
74 static inline void issue_rma_op(int i)
76 compare_buf[i] = i; /* always equal to window value, thus swap happens */
77 MPI_Compare_and_swap(&local_buf[i], &compare_buf[i], &result_addr[i], MPI_INT, target, i, win);
82 /* Local check function for GET-like operations */
83 #if defined(TEST_GACC) || defined(TEST_FOP) || defined(TEST_CAS)
85 /* Check local result buffer for GET-like operations */
86 static int check_local_result(int iter)
91 for (i = 0; i < BUF_CNT; i++) {
92 if (result_addr[i] != i) {
93 printf("rank %d (iter %d) - check %s, got result_addr[%d] = %d, expected %d\n",
94 rank, iter, rma_name, i, result_addr[i], i);
102 #define check_local_result(iter) (0)
105 static int run_test()
109 int sbuf = 0, rbuf = 0;
112 for (x = 0; x < ITER; x++) {
113 /* 1. Target resets window data */
114 if (rank == target) {
115 for (i = 0; i < BUF_CNT; i++)
117 MPI_Win_sync(win); /* write is done on window */
120 MPI_Barrier(MPI_COMM_WORLD);
122 /* 2. Every one resets local data */
123 for (i = 0; i < BUF_CNT; i++) {
124 local_buf[i] = BUF_CNT + x * BUF_CNT + i;
128 /* 3. Origin issues RMA operation to target */
129 if (rank == origin) {
130 /* 3-1. Issue RMA. */
131 for (i = 0; i < BUF_CNT; i++) {
134 MPI_Win_flush(target, win);
136 /* 3-2. Sync with checker */
137 MPI_Send(&sbuf, 1, MPI_INT, checker, 999, MPI_COMM_WORLD);
139 /* 3-3. Check local result buffer */
140 errors += check_local_result(x);
143 /* 4. Checker issues GET to target and checks result */
144 if (rank == checker) {
145 /* 4-1. Sync with origin */
146 MPI_Recv(&rbuf, 1, MPI_INT, origin, 999, MPI_COMM_WORLD, &stat);
148 /* 4-2. Get result and check */
149 MPI_Get(check_addr, BUF_CNT, MPI_INT, target, 0, BUF_CNT, MPI_INT, win);
150 MPI_Win_flush(target, win);
152 for (i = 0; i < BUF_CNT; i++) {
153 if (check_addr[i] != local_buf[i]) {
154 printf("rank %d (iter %d) - check %s, got check_addr[%d] = %d, expected %d\n",
155 rank, x, rma_name, i, check_addr[i], local_buf[i]);
161 MPI_Barrier(MPI_COMM_WORLD);
167 int main(int argc, char *argv[])
169 int errors = 0, all_errors = 0;
170 int win_size = sizeof(int) * BUF_CNT;
171 int win_unit = sizeof(int);
173 MPI_Init(&argc, &argv);
174 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
175 MPI_Comm_size(MPI_COMM_WORLD, &nproc);
179 printf("Error: must be run with three processes\n");
180 MPI_Barrier(MPI_COMM_WORLD);
181 MPI_Abort(MPI_COMM_WORLD, 1);
184 #if !defined(TEST_PUT) && !defined(TEST_ACC) && !defined(TEST_GACC) && !defined(TEST_FOP) && !defined(TEST_CAS)
186 printf("Error: must specify operation type at compile time\n");
187 MPI_Barrier(MPI_COMM_WORLD);
188 MPI_Abort(MPI_COMM_WORLD, 1);
191 /* Identify origin, target and checker ranks. */
196 MPI_Win_allocate(win_size, win_unit, MPI_INFO_NULL, MPI_COMM_WORLD, &my_base, &win);
198 /* Start checking. */
199 MPI_Win_lock_all(0, win);
203 MPI_Win_unlock_all(win);
205 MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
206 if (rank == 0 && all_errors == 0)
207 printf(" No Errors\n");
209 if (win != MPI_WIN_NULL)