1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
4 * (C) 2015 by Argonne National Laboratory.
5 * See COPYRIGHT in top-level directory.
15 int local_buf[BUF_CNT], result_addr[BUF_CNT];
17 int compare_buf[BUF_CNT];
20 const int verbose = 0;
22 /* This test checks the remote completion of flush with RMA write-like operations
23 * (PUT, ACC, GET_ACC, FOP, CAS), and confirms result by shm load.
24 * 1. P(target) and P(checker) allocate a shared window, and
25 * then create a global window with P(origin) by using the shared window buffer.
26 * 2. P(origin) issues RMA operations and flush to P(target) through the global
27 * window and then call send-recv to synchronize with P(checker).
28 * 3. P(checker) then checks the result through shm window by local load. */
30 int rank = -1, nproc = 0;
31 int origin = -1, target = -1, checker = -1;
32 MPI_Win win = MPI_WIN_NULL, shm_win = MPI_WIN_NULL;
33 int *shm_target_base = NULL, *my_base = NULL;
35 /* Define operation name for error message */
37 const char *rma_name = "Put";
38 #elif defined(TEST_ACC)
39 const char *rma_name = "Accumulate";
40 #elif defined(TEST_GACC)
41 const char *rma_name = "Get_accumulate";
42 #elif defined(TEST_FOP)
43 const char *rma_name = "Fetch_and_op";
44 #elif defined(TEST_CAS)
45 const char *rma_name = "Compare_and_swap";
47 const char *rma_name = "None";
50 /* Issue functions for different RMA operations */
52 static inline void issue_rma_op(int i)
54 MPI_Put(&local_buf[i], 1, MPI_INT, target, i, 1, MPI_INT, win);
56 #elif defined(TEST_ACC)
57 static inline void issue_rma_op(int i)
59 MPI_Accumulate(&local_buf[i], 1, MPI_INT, target, i, 1, MPI_INT, MPI_REPLACE, win);
61 #elif defined(TEST_GACC)
62 static inline void issue_rma_op(int i)
64 MPI_Get_accumulate(&local_buf[i], 1, MPI_INT, &result_addr[i], 1, MPI_INT, target, i,
65 1, MPI_INT, MPI_REPLACE, win);
67 #elif defined(TEST_FOP)
68 static inline void issue_rma_op(int i)
70 MPI_Fetch_and_op(&local_buf[i], &result_addr[i], MPI_INT, target, i, MPI_REPLACE, win);
72 #elif defined(TEST_CAS)
73 static inline void issue_rma_op(int i)
75 compare_buf[i] = i; /* always equal to window value, thus swap happens */
76 MPI_Compare_and_swap(&local_buf[i], &compare_buf[i], &result_addr[i], MPI_INT, target, i, win);
81 /* Local check function for GET-like operations */
82 #if defined(TEST_GACC) || defined(TEST_FOP) || defined(TEST_CAS)
84 /* Check local result buffer for GET-like operations */
85 static int check_local_result(int iter)
90 for (i = 0; i < BUF_CNT; i++) {
91 if (result_addr[i] != i) {
92 printf("rank %d (iter %d) - check %s, got result_addr[%d] = %d, expected %d\n",
93 rank, iter, rma_name, i, result_addr[i], i);
101 #define check_local_result(iter) (0)
104 static int run_test()
108 int sbuf = 0, rbuf = 0;
111 for (x = 0; x < ITER; x++) {
112 /* 1. Target resets window data */
113 if (rank == target) {
114 for (i = 0; i < BUF_CNT; i++)
116 MPI_Win_sync(shm_win); /* write is done on shm window */
119 MPI_Barrier(MPI_COMM_WORLD);
121 /* 2. Every one resets local data */
122 for (i = 0; i < BUF_CNT; i++) {
123 local_buf[i] = BUF_CNT + x * BUF_CNT + i;
127 /* 3. Origin issues RMA operation to target */
128 if (rank == origin) {
129 /* 3-1. Issue RMA. */
130 for (i = 0; i < BUF_CNT; i++) {
133 MPI_Win_flush(target, win);
135 /* 3-2. Check local result buffer. */
136 errors += check_local_result(x);
138 /* sync with checker */
139 MPI_Send(&sbuf, 1, MPI_INT, checker, 999, MPI_COMM_WORLD);
142 /* 4. Checker confirms result on target */
143 if (rank == checker) {
144 /* sync with origin */
145 MPI_Recv(&rbuf, 1, MPI_INT, origin, 999, MPI_COMM_WORLD, &stat);
147 MPI_Win_sync(shm_win);
149 for (i = 0; i < BUF_CNT; i++) {
150 if (shm_target_base[i] != local_buf[i]) {
151 printf("rank %d (iter %d) - check %s, got shm_target_base[%d] = %d, "
152 "expected %d\n", rank, x, rma_name, i, shm_target_base[i], local_buf[i]);
158 MPI_Barrier(MPI_COMM_WORLD);
164 int main(int argc, char *argv[])
167 int errors = 0, all_errors = 0;
168 MPI_Comm shm_comm = MPI_COMM_NULL;
170 int *shm_ranks = NULL, *shm_root_ranks = NULL;
171 int win_size = sizeof(int) * BUF_CNT;
172 int win_unit = sizeof(int);
173 int shm_root_rank = -1, shm_target = -1, target_shm_root = -1;
175 MPI_Init(&argc, &argv);
176 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
177 MPI_Comm_size(MPI_COMM_WORLD, &nproc);
181 printf("Error: must be run with three processes\n");
182 MPI_Barrier(MPI_COMM_WORLD);
183 MPI_Abort(MPI_COMM_WORLD, 1);
186 #if !defined(TEST_PUT) && !defined(TEST_ACC) && !defined(TEST_GACC) && !defined(TEST_FOP) && !defined(TEST_CAS)
188 printf("Error: must specify operation type at compile time\n");
189 MPI_Barrier(MPI_COMM_WORLD);
190 MPI_Abort(MPI_COMM_WORLD, 1);
193 MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &shm_comm);
194 MPI_Comm_rank(shm_comm, &shm_rank);
196 shm_ranks = (int *) calloc(nproc, sizeof(int));
197 shm_root_ranks = (int *) calloc(nproc, sizeof(int));
199 /* Identify node id */
201 shm_root_rank = rank;
202 MPI_Bcast(&shm_root_rank, 1, MPI_INT, 0, shm_comm);
204 /* Exchange local root rank and local rank */
205 shm_ranks[rank] = shm_rank;
206 shm_root_ranks[rank] = shm_root_rank;
208 MPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, shm_ranks, 1, MPI_INT, MPI_COMM_WORLD);
209 MPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, shm_root_ranks, 1, MPI_INT, MPI_COMM_WORLD);
211 /* Check if there are at least two processes in shared memory. */
212 for (i = 0; i < nproc; i++) {
213 if (shm_ranks[i] != 0) {
214 target_shm_root = shm_root_ranks[i];
219 /* Every process is in separate memory, we cannot create shared window. Just return. */
220 if (target_shm_root < 0)
223 /* Identify origin, target and checker ranks.
224 * the first process in shared memory is target, and the second one is checker;
225 * the last process is origin.*/
227 for (i = 0; i < nproc; i++) {
228 if (shm_root_ranks[i] == target_shm_root) {
229 if (shm_ranks[i] == 0) {
232 else if (shm_ranks[i] == 1) {
236 /* all three processes are in shared memory, origin is the third one. */
241 /* origin is in separate memory. */
247 printf("---- rank %d: origin = %d, checker = %d, target = %d, test %s\n",
248 rank, origin, checker, target, rma_name);
251 /* Allocate shared memory among local processes, then create a global window
252 * with the shared window buffers. */
253 MPI_Win_allocate_shared(win_size, win_unit, MPI_INFO_NULL, shm_comm, &my_base, &shm_win);
254 MPI_Win_create(my_base, win_size, win_unit, MPI_INFO_NULL, MPI_COMM_WORLD, &win);
256 /* Get address of target window on checker process. */
257 if (rank == checker) {
260 MPI_Win_shared_query(shm_win, shm_target, &size, &disp_unit, &shm_target_base);
262 printf("---- I am checker = %d, shm_target_base=%p\n", checker, shm_target_base);
266 /* Start checking. */
267 MPI_Win_lock_all(0, win);
268 MPI_Win_lock_all(0, shm_win);
272 MPI_Win_unlock_all(shm_win);
273 MPI_Win_unlock_all(win);
275 MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
279 if (rank == 0 && all_errors == 0)
280 printf(" No Errors\n");
285 free(shm_root_ranks);
287 if (shm_win != MPI_WIN_NULL)
288 MPI_Win_free(&shm_win);
290 if (win != MPI_WIN_NULL)
293 if (shm_comm != MPI_COMM_NULL)
294 MPI_Comm_free(&shm_comm);