1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
4 * (C) 2014 by Argonne National Laboratory.
5 * See COPYRIGHT in top-level directory.
14 #define ELEM_PER_PROC 4
15 int local_buf[ELEM_PER_PROC];
17 const int verbose = 0;
19 int main(int argc, char **argv)
22 int shm_rank, shm_nproc;
24 int errors = 0, all_errors = 0;
25 int **bases = NULL, *my_base = NULL;
27 MPI_Win shm_win = MPI_WIN_NULL, win = MPI_WIN_NULL;
28 MPI_Comm shm_comm = MPI_COMM_NULL;
29 MPI_Group shm_group = MPI_GROUP_NULL, world_group = MPI_GROUP_NULL;
30 int dst_shm_rank, dst_world_rank;
31 MPI_Info create_info = MPI_INFO_NULL;
33 MPI_Init(&argc, &argv);
35 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
36 MPI_Comm_size(MPI_COMM_WORLD, &nproc);
38 MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &shm_comm);
40 MPI_Comm_rank(shm_comm, &shm_rank);
41 MPI_Comm_size(shm_comm, &shm_nproc);
43 /* Platform does not support shared memory, just return. */
48 /* Specify the last process in the node as the target process */
49 dst_shm_rank = shm_nproc - 1;
50 MPI_Comm_group(shm_comm, &shm_group);
51 MPI_Comm_group(MPI_COMM_WORLD, &world_group);
52 MPI_Group_translate_ranks(shm_group, 1, &dst_shm_rank, world_group, &dst_world_rank);
54 bases = calloc(shm_nproc, sizeof(int *));
56 /* Allocate shm window among local processes, then create a global window with
57 * those shm window buffers */
58 MPI_Win_allocate_shared(sizeof(int) * ELEM_PER_PROC, sizeof(int), MPI_INFO_NULL,
59 shm_comm, &my_base, &shm_win);
61 printf("%d -- allocate shared: my_base = %p, absolute base\n", shm_rank, my_base);
63 for (i = 0; i < shm_nproc; i++) {
64 MPI_Win_shared_query(shm_win, i, &size, &disp_unit, &bases[i]);
66 printf("%d -- shared query: base[%d]=%p, size %ld, unit %d\n",
67 shm_rank, i, bases[i], size, disp_unit);
70 #ifdef USE_INFO_ALLOC_SHM
71 MPI_Info_create(&create_info);
72 MPI_Info_set(create_info, "alloc_shm", "true");
74 create_info = MPI_INFO_NULL;
78 for (i = 0; i < ELEM_PER_PROC; i++) {
83 MPI_Win_create(my_base, sizeof(int) * ELEM_PER_PROC, sizeof(int), create_info, MPI_COMM_WORLD,
86 /* Do RMA through global window, then check value through shared window */
87 MPI_Win_lock_all(0, win);
88 MPI_Win_lock_all(0, shm_win);
91 MPI_Put(&local_buf[0], 1, MPI_INT, dst_world_rank, 0, 1, MPI_INT, win);
92 MPI_Put(&local_buf[ELEM_PER_PROC - 1], 1, MPI_INT, dst_world_rank, ELEM_PER_PROC - 1, 1,
94 MPI_Win_flush(dst_world_rank, win);
97 MPI_Win_sync(shm_win);
98 MPI_Barrier(shm_comm);
99 MPI_Win_sync(shm_win);
101 if (bases[dst_shm_rank][0] != local_buf[0]) {
103 printf("%d -- Got %d at rank %d index %d, expected %d\n", rank,
104 bases[dst_shm_rank][0], dst_shm_rank, 0, local_buf[0]);
106 if (bases[dst_shm_rank][ELEM_PER_PROC - 1] != local_buf[ELEM_PER_PROC - 1]) {
108 printf("%d -- Got %d at rank %d index %d, expected %d\n", rank,
109 bases[dst_shm_rank][ELEM_PER_PROC - 1], dst_shm_rank,
110 ELEM_PER_PROC - 1, local_buf[ELEM_PER_PROC - 1]);
113 MPI_Win_unlock_all(shm_win);
114 MPI_Win_unlock_all(win);
116 MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
119 MPI_Win_free(&shm_win);
122 if (rank == 0 && all_errors == 0)
123 printf(" No Errors\n");
125 if (create_info != MPI_INFO_NULL)
126 MPI_Info_free(&create_info);
127 if (shm_comm != MPI_COMM_NULL)
128 MPI_Comm_free(&shm_comm);
129 if (shm_group != MPI_GROUP_NULL)
130 MPI_Group_free(&shm_group);
131 if (world_group != MPI_GROUP_NULL)
132 MPI_Group_free(&world_group);