src/smpi/colls/reduce-flat-tree.c
src/smpi/colls/reduce-NTSL.c
src/smpi/colls/reduce-scatter-gather.c
- src/smpi/colls/star-reduction.c
)
if(SMPI_F2C)
((rank + size - 1) % size), tag + i, comm, &status);
// compute result to rbuf+recv_offset
- star_reduction(op, (char *)sbuf + recv_offset, (char *)rbuf + recv_offset, &count, &dtype);
+ smpi_op_apply(op, (char *)sbuf + recv_offset, (char *)rbuf + recv_offset, &count, &dtype);
}
// all-gather
((rank + size - 1) % size), tag + i, comm, &status);
// compute result to rbuf+recv_offset
- star_reduction(op, (char *) sbuf + recv_offset, (char *) rbuf + recv_offset,
+ smpi_op_apply(op, (char *) sbuf + recv_offset, (char *) rbuf + recv_offset,
&count, &dtype);
}
// do the reduction on received data. since the
// ordering is right, it doesn't matter whether
// the operation is commutative or not.
- star_reduction(op, tmp_buf, rbuff, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, rbuff, &count, &dtype);
// change the rank
newrank = rank / 2;
// This algorithm is used only for predefined ops
// and predefined ops are always commutative.
- star_reduction(op, (char *) tmp_buf + disps[recv_idx] * extent,
+ smpi_op_apply(op, (char *) tmp_buf + disps[recv_idx] * extent,
(char *) rbuff + disps[recv_idx] * extent,
&recv_cnt, &dtype);
smpi_mpi_sendrecv((char *) recv + send_idx * extent, send_cnt, dtype, dst, tag,
tmp_buf, recv_cnt, dtype, dst, tag, comm, &status);
- star_reduction(op, tmp_buf, (char *) recv + recv_idx * extent, &recv_cnt,
+ smpi_op_apply(op, tmp_buf, (char *) recv + recv_idx * extent, &recv_cnt,
&dtype);
// update send_idx for next iteration
smpi_mpi_sendrecv((char *) rbuff + send_idx * extent, send_cnt, dtype, dst,
tag, tmp_buf, recv_cnt, dtype, dst, tag, comm, &status);
- star_reduction(op, tmp_buf, (char *) rbuff + recv_idx * extent, &recv_cnt,
+ smpi_op_apply(op, tmp_buf, (char *) rbuff + recv_idx * extent, &recv_cnt,
&dtype);
// update send_idx for next iteration
memcpy(tmp, recv, nbytes);
for (i = 1, s_offset = nbytes; i < nprocs; i++, s_offset = i * nbytes)
- star_reduction(op, (char *) recv + s_offset, tmp, &send_size, &dtype);
+ smpi_op_apply(op, (char *) recv + s_offset, tmp, &send_size, &dtype);
mpi_coll_allgather_fun(tmp, send_size, dtype, recv, send_size, dtype, comm);
memcpy(rbuff, recv, count * s_extent);
memcpy((char *) rbuff + r_offset, recv, nbytes);
for (i = 1, s_offset = nbytes; i < nprocs; i++, s_offset = i * nbytes)
- star_reduction(op, (char *) recv + s_offset, (char *) rbuff + r_offset,
+ smpi_op_apply(op, (char *) recv + s_offset, (char *) rbuff + r_offset,
&send_size, &dtype);
mpi_coll_allgather_fun((char *) rbuff + r_offset, send_size, dtype, rbuff, send_size,
// do the reduction on received data. since the
// ordering is right, it doesn't matter whether
// the operation is commutative or not.
- star_reduction(op, tmp_buf, rbuff, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, rbuff, &count, &dtype);
// change the rank
newrank = rank / 2;
// we assume it is commuttive op
// if (op -> op_commute || (dst < rank))
if ((dst < rank)) {
- star_reduction(op, tmp_buf, rbuff, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, rbuff, &count, &dtype);
} else // op is noncommutative and the order is not right
{
- star_reduction(op, rbuff, tmp_buf, &count, &dtype);
+ smpi_op_apply(op, rbuff, tmp_buf, &count, &dtype);
// copy result back into recvbuf
smpi_mpi_sendrecv(tmp_buf, count, dtype, rank, tag, rbuff, count,
src = (inter_rank * num_core) + (intra_rank | mask);
if (src < comm_size) {
smpi_mpi_recv(tmp_buf, count, dtype, src, tag, comm, &status);
- star_reduction(op, tmp_buf, recv_buf, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, recv_buf, &count, &dtype);
}
} else {
dst = (inter_rank * num_core) + (intra_rank & (~mask));
src = (inter_rank | mask) * num_core;
if (src < comm_size) {
smpi_mpi_recv(tmp_buf, count, dtype, src, tag, comm, &status);
- star_reduction(op, tmp_buf, recv_buf, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, recv_buf, &count, &dtype);
}
} else {
dst = (inter_rank & (~mask)) * num_core;
src = (inter_rank * num_core) + (intra_rank | mask);
if (src < comm_size) {
smpi_mpi_recv(tmp_buf, count, dtype, src, tag, comm, &status);
- star_reduction(op, tmp_buf, recv_buf, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, recv_buf, &count, &dtype);
}
} else {
dst = (inter_rank * num_core) + (intra_rank & (~mask));
} else {
src = rank - num_core;
smpi_mpi_recv(tmp_buf, count, dtype, src, tag, comm, &status);
- star_reduction(op, tmp_buf, recv_buf, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, recv_buf, &count, &dtype);
newrank = inter_rank / 2;
}
} else {
/* exchange data in rdb manner */
smpi_mpi_sendrecv(recv_buf, count, dtype, dst, tag, tmp_buf, count, dtype,
dst, tag, comm, &status);
- star_reduction(op, tmp_buf, recv_buf, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, recv_buf, &count, &dtype);
mask <<= 1;
}
}
// if (src < ((inter_rank + 1) * num_core)) {
if (src < comm_size) {
smpi_mpi_recv(tmp_buf, count, dtype, src, tag, comm, &status);
- star_reduction(op, tmp_buf, recv_buf, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, recv_buf, &count, &dtype);
//printf("Node %d recv from node %d when mask is %d\n", rank, src, mask);
}
} else {
&status);
// result is in rbuf
- star_reduction(op, tmp_buf, (char *) recv_buf + recv_offset, &recv_count,
+ smpi_op_apply(op, tmp_buf, (char *) recv_buf + recv_offset, &recv_count,
&dtype);
}
// if (src < ((inter_rank + 1) * num_core)) {
if (src < comm_size) {
smpi_mpi_recv(tmp_buf, count, dtype, src, tag, comm, &status);
- star_reduction(op, tmp_buf, rbuf, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, rbuf, &count, &dtype);
//printf("Node %d recv from node %d when mask is %d\n", rank, src, mask);
}
} else {
tmp_buf, curr_count, dtype, (dst * num_core), tag,
comm, &status);
- star_reduction(op, tmp_buf, (char *)rbuf + recv_offset, &curr_count, &dtype);
+ smpi_op_apply(op, tmp_buf, (char *)rbuf + recv_offset, &curr_count, &dtype);
mask *= 2;
curr_count /= 2;
// if (src < ((inter_rank + 1) * num_core)) {
if (src < comm_size) {
smpi_mpi_recv(tmp_buf, count, dtype, src, tag, comm, &status);
- star_reduction(op, tmp_buf, recv_buf, &count, &dtype);
+ smpi_op_apply(op, tmp_buf, recv_buf, &count, &dtype);
//printf("Node %d recv from node %d when mask is %d\n", rank, src, mask);
}
} else {
&status);
// result is in rbuf
- star_reduction(op, tmp_buf, (char *) recv_buf + recv_offset, &seg_count,
+ smpi_op_apply(op, tmp_buf, (char *) recv_buf + recv_offset, &seg_count,
&dtype);
}
#include "colls.h"
XBT_LOG_EXTERNAL_DEFAULT_CATEGORY(smpi_colls);
-void star_reduction(MPI_Op op, void *src, void *target, int *count, MPI_Datatype *dtype);
#endif
if (count <= segment) {
if (rank == root) {
smpi_mpi_recv(tmp_buf, count, datatype, from, tag, comm, &status);
- star_reduction(op, tmp_buf, rbuf, &count, &datatype);
+ smpi_op_apply(op, tmp_buf, rbuf, &count, &datatype);
} else if (rank == ((root - 1 + size) % size)) {
smpi_mpi_send(rbuf, count, datatype, to, tag, comm);
} else {
smpi_mpi_recv(tmp_buf, count, datatype, from, tag, comm, &status);
- star_reduction(op, tmp_buf, rbuf, &count, &datatype);
+ smpi_op_apply(op, tmp_buf, rbuf, &count, &datatype);
smpi_mpi_send(rbuf, count, datatype, to, tag, comm);
}
free(tmp_buf);
}
for (i = 0; i < pipe_length; i++) {
smpi_mpi_wait(&recv_request_array[i], &status);
- star_reduction(op, tmp_buf + (i * increment), (char *)rbuf + (i * increment),
+ smpi_op_apply(op, tmp_buf + (i * increment), (char *)rbuf + (i * increment),
&segment, &datatype);
}
}
}
for (i = 0; i < pipe_length; i++) {
smpi_mpi_wait(&recv_request_array[i], &status);
- star_reduction(op, tmp_buf + (i * increment), (char *)rbuf + (i * increment),
+ smpi_op_apply(op, tmp_buf + (i * increment), (char *)rbuf + (i * increment),
&segment, &datatype);
send_request_array[i] = smpi_mpi_isend((char *) rbuf + (i * increment), segment, datatype, to,
(tag + i), comm);
smpi_mpi_send(header_buf, HEADER_SIZE, MPI_INT, to, tag, comm);
smpi_mpi_recv(tmp_buf, count, datatype, from, tag, comm, &status);
- star_reduction(op, tmp_buf, rbuf, &count, &datatype);
+ smpi_op_apply(op, tmp_buf, rbuf, &count, &datatype);
}
} /* while loop */
}
from = header_buf[myordering - 1];
smpi_mpi_recv(tmp_buf, count, datatype, header_buf[myordering - 1], tag,
comm, &status);
- star_reduction(op, tmp_buf, rbuf, &count, &datatype);
+ smpi_op_apply(op, tmp_buf, rbuf, &count, &datatype);
smpi_mpi_send(rbuf, count, datatype, to, tag, comm);
}
} /* non-root */
for (i = 0; i < pipe_length; i++) {
smpi_mpi_recv(tmp_buf + (i * increment), segment, datatype, from, tag,
comm, &status);
- star_reduction(op, tmp_buf + (i * increment),
+ smpi_op_apply(op, tmp_buf + (i * increment),
(char *)rbuf + (i * increment), &segment, &datatype);
}
}
}
for (i = 0; i < pipe_length; i++) {
smpi_mpi_wait(&recv_request_array[i], MPI_STATUS_IGNORE);
- star_reduction(op, tmp_buf + (i * increment), (char *)rbuf + (i * increment),
+ smpi_op_apply(op, tmp_buf + (i * increment), (char *)rbuf + (i * increment),
&segment, &datatype);
send_request_array[i]=smpi_mpi_isend((char *)rbuf + (i * increment), segment, datatype, to, tag, comm);
}
if (source < comm_size) {
source = (source + root) % comm_size;
smpi_mpi_recv(tmp_buf, count, datatype, source, tag, comm, &status);
- star_reduction(op, tmp_buf, recvbuf, &count, &datatype);
+ smpi_op_apply(op, tmp_buf, recvbuf, &count, &datatype);
}
} else {
dst = ((relrank & (~mask)) + root) % comm_size;
}
/* Call reduction function. */
- star_reduction(op, inbuf, rbuf, &count, &dtype);
+ smpi_op_apply(op, inbuf, rbuf, &count, &dtype);
}
newrank = -1;
} else {
smpi_mpi_recv(tmp_buf, count, datatype, rank + 1, tag, comm, &status);
- star_reduction(op, tmp_buf, recv_ptr, &new_count, &datatype);
+ smpi_op_apply(op, tmp_buf, recv_ptr, &new_count, &datatype);
newrank = rank / 2;
}
} else /* rank >= 2*rem */
/* tmp_buf contains data received in this step.
recvbuf contains data accumulated so far */
- star_reduction(op, (char *) tmp_buf + disps[recv_idx] * extent,
+ smpi_op_apply(op, (char *) tmp_buf + disps[recv_idx] * extent,
(char *) recv_ptr + disps[recv_idx] * extent,
&recv_cnt, &datatype);
else {
smpi_mpi_recv(tmp_buf, count, datatype, rank + 1, tag, comm, &status);
- star_reduction(op, tmp_buf, recvbuf, &count, &datatype);
+ smpi_op_apply(op, tmp_buf, recvbuf, &count, &datatype);
newrank = rank / 2;
}
} else /* rank >= 2*rem */
/* tmp_buf contains data received in this step.
recvbuf contains data accumulated so far */
- star_reduction(op, (char *) tmp_buf + disps[recv_idx] * extent,
+ smpi_op_apply(op, (char *) tmp_buf + disps[recv_idx] * extent,
(char *) recvbuf + disps[recv_idx] * extent,
&recv_cnt, &datatype);
+++ /dev/null
-#include "colls_private.h"
-
-/*
- * created by Pitch Patarasuk
- * Modified by Xin Yuan
- *
- * realize a subset of MPI predefine operators:
- * MPI_LAND, MPI_BAND: C integer, Fortran integer, Byte
- * MPI_LOR, MPI_BOR: C integer, Fortran integer, Byte
- * MPI_LXOR, MPI_BXOR: C integer, Fortran integer, Byte
- * MPI_SUM, MPI_PROD: C integer, Fortran integer, Floating point
- * MPI_MIN, MPI_MAX: C integer, Fortran integer, Floating point, Byte
- *
- * Types not implemented: MPI_LONG_DOUBLE, MPI_LOGICAL, MPI_COMPLEX
- */
-
-#ifndef STAR_REDUCTION
-#define STAR_REDUCTION
-
-
-#ifdef MPICH2_REDUCTION
-extern MPI_User_function * MPIR_Op_table[];
-#elif defined MVAPICH_REDUCETION
-extern void *MPIR_ToPointer();
-struct MPIR_OP
-{
- MPI_User_function *op;
- int commute;
- int permanent;
-};
-#endif
-
-static void star_generic_reduction(MPI_Op op, void *src, void *target, int *count, MPI_Datatype *dtype){
- int i;
- if ((op == MPI_BOR) || (op == MPI_LOR)) {
- if ((*dtype == MPI_BYTE) || (*dtype == MPI_CHAR)) {
- for (i=0;i<*count;i++) {
- ((char *)target)[i] |= ((char *)src)[i];
- }
- }
- else if ((*dtype == MPI_INT)
- || (*dtype == MPI_LONG)
- || (*dtype == MPI_INT)
- || (*dtype == MPI_UNSIGNED)
- || (*dtype == MPI_UNSIGNED_LONG)) {
- for (i=0;i<*count;i++) {
- ((int *)target)[i] |= ((int *)src)[i];
- }
- }
- else if ((*dtype == MPI_SHORT)
- || (*dtype == MPI_UNSIGNED_SHORT)) {
- for (i=0;i<*count;i++) {
- ((short *)target)[i] |= ((short *)src)[i];
- }
- }
- else {
- printf("reduction operation not supported\n");
- }
- }
-
- else if ((op == MPI_BAND) || (op == MPI_LAND)) {
- if ((*dtype == MPI_BYTE) || (*dtype == MPI_CHAR)) {
- for (i=0;i<*count;i++) {
- ((char *)target)[i] &= ((char *)src)[i];
- }
- }
- else if ((*dtype == MPI_INT)
- || (*dtype == MPI_LONG)
- || (*dtype == MPI_UNSIGNED)
- || (*dtype == MPI_UNSIGNED_LONG)) {
- for (i=0;i<*count;i++) {
- ((int *)target)[i] &= ((int *)src)[i];
- }
- }
- else if ((*dtype == MPI_SHORT)
- || (*dtype == MPI_UNSIGNED_SHORT)) {
- for (i=0;i<*count;i++) {
- ((short *)target)[i] &= ((short *)src)[i];
- }
- }
- else {
- printf("reduction operation not supported\n");
- }
- }
-
-
- else if ((op == MPI_BXOR) || (op == MPI_LXOR)) {
- if ((*dtype == MPI_BYTE) || (*dtype == MPI_CHAR)) {
- for (i=0;i<*count;i++) {
- ((char *)target)[i] ^= ((char *)src)[i];
- }
- }
- else if ((*dtype == MPI_INT)
- || (*dtype == MPI_LONG)
- || (*dtype == MPI_UNSIGNED)
- || (*dtype == MPI_UNSIGNED_LONG)) {
- for (i=0;i<*count;i++) {
- ((int *)target)[i] ^= ((int *)src)[i];
- }
- }
- else if ((*dtype == MPI_SHORT)
- || (*dtype == MPI_UNSIGNED_SHORT)) {
- for (i=0;i<*count;i++) {
- ((short *)target)[i] ^= ((short *)src)[i];
- }
- }
- else {
- printf("reduction operation not supported\n");
- }
- }
-
- else if (op == MPI_MAX) {
- if ((*dtype == MPI_INT)
- || (*dtype == MPI_LONG)) {
- for (i=0;i<*count;i++) {
- if (((int *)src)[i] > ((int *)target)[i]) {
- ((int *)target)[i] = ((int *)src)[i];
- }
- }
- }
- else if ((*dtype == MPI_UNSIGNED)
- || (*dtype == MPI_UNSIGNED_LONG)) {
- for (i=0;i<*count;i++) {
- if (((unsigned int *)src)[i] > ((unsigned int *)target)[i]) {
- ((unsigned int *)target)[i] = ((unsigned int *)src)[i];
- }
- }
- }
- else if (*dtype == MPI_SHORT) {
- for (i=0;i<*count;i++) {
- if (((short *)src)[i] > ((short *)target)[i]) {
- ((short *)target)[i] = ((short *)src)[i];
- }
- }
- }
- else if (*dtype == MPI_UNSIGNED_SHORT) {
- for (i=0;i<*count;i++) {
- if (((unsigned short *)src)[i] > ((unsigned short *)target)[i]) {
- ((unsigned short *)target)[i] = ((unsigned short *)src)[i];
- }
- }
- }
-
- else if (*dtype == MPI_DOUBLE) {
- for (i=0;i<*count;i++) {
- if (((double *)src)[i] > ((double *)target)[i]) {
- ((double *)target)[i] = ((double *)src)[i];
- }
- }
- }
- else if (*dtype == MPI_FLOAT) {
- for (i=0;i<*count;i++) {
- if (((float *)src)[i] > ((float *)target)[i]) {
- ((float *)target)[i] = ((float *)src)[i];
- }
- }
- }
- else if ((*dtype == MPI_CHAR) || (*dtype == MPI_BYTE)) {
- for (i=0;i<*count;i++) {
- if (((char *)src)[i] > ((char *)target)[i]) {
- ((char *)target)[i] = ((char *)src)[i];
- }
- }
- }
- else {
- printf("reduction operation not supported\n");
- }
- }
-
-
-
- else if (op == MPI_MIN) {
- if ((*dtype == MPI_INT)
- || (*dtype == MPI_LONG)) {
- for (i=0;i<*count;i++) {
- if (((int *)src)[i] < ((int *)target)[i]) {
- ((int *)target)[i] = ((int *)src)[i];
- }
- }
- }
- else if ((*dtype == MPI_UNSIGNED)
- || (*dtype == MPI_UNSIGNED_LONG)) {
- for (i=0;i<*count;i++) {
- if (((unsigned int *)src)[i] < ((unsigned int *)target)[i]) {
- ((unsigned int *)target)[i] = ((unsigned int *)src)[i];
- }
- }
- }
- else if (*dtype == MPI_SHORT) {
- for (i=0;i<*count;i++) {
- if (((short *)src)[i] < ((short *)target)[i]) {
- ((short *)target)[i] = ((short *)src)[i];
- }
- }
- }
- else if (*dtype == MPI_UNSIGNED_SHORT) {
- for (i=0;i<*count;i++) {
- if (((unsigned short *)src)[i] < ((unsigned short *)target)[i]) {
- ((unsigned short *)target)[i] = ((unsigned short *)src)[i];
- }
- }
- }
-
- else if (*dtype == MPI_DOUBLE) {
- for (i=0;i<*count;i++) {
- if (((double *)src)[i] < ((double *)target)[i]) {
- ((double *)target)[i] = ((double *)src)[i];
- }
- }
- }
- else if (*dtype == MPI_FLOAT) {
- for (i=0;i<*count;i++) {
- if (((float *)src)[i] < ((float *)target)[i]) {
- ((float *)target)[i] = ((float *)src)[i];
- }
- }
- }
- else if ((*dtype == MPI_CHAR) || (*dtype == MPI_BYTE)) {
- for (i=0;i<*count;i++) {
- if (((char *)src)[i] < ((char *)target)[i]) {
- ((char *)target)[i] = ((char *)src)[i];
- }
- }
- }
- else {
- printf("reduction operation not supported\n");
- }
- }
-
-
- else if (op == MPI_SUM) {
- if ((*dtype == MPI_INT)
- || (*dtype == MPI_LONG)) {
- for (i=0;i<*count;i++) {
- ((int *)target)[i] += ((int *)src)[i];
- }
- }
- else if ((*dtype == MPI_UNSIGNED)
- || (*dtype == MPI_UNSIGNED_LONG)) {
- for (i=0;i<*count;i++) {
- ((unsigned int *)target)[i] += ((unsigned int *)src)[i];
- }
- }
- else if (*dtype == MPI_SHORT) {
- for (i=0;i<*count;i++) {
- ((short *)target)[i] += ((short *)src)[i];
- }
- }
- else if (*dtype == MPI_UNSIGNED_SHORT) {
- for (i=0;i<*count;i++) {
- ((unsigned short *)target)[i] += ((unsigned short *)src)[i];
- }
- }
-
- else if (*dtype == MPI_DOUBLE) {
- for (i=0;i<*count;i++) {
- ((double *)target)[i] += ((double *)src)[i];
- }
- }
- else if (*dtype == MPI_FLOAT) {
- for (i=0;i<*count;i++) {
- ((float *)target)[i] += ((float *)src)[i];
- }
- }
- else {
- printf("reduction operation not supported\n");
- }
- }
-
- else if (op == MPI_PROD) {
- if ((*dtype == MPI_INT)
- || (*dtype == MPI_LONG)) {
- for (i=0;i<*count;i++) {
- ((int *)target)[i] *= ((int *)src)[i];
- }
- }
- else if ((*dtype == MPI_UNSIGNED)
- || (*dtype == MPI_UNSIGNED_LONG)) {
- for (i=0;i<*count;i++) {
- ((unsigned int *)target)[i] *= ((unsigned int *)src)[i];
- }
- }
- else if (*dtype == MPI_SHORT) {
- for (i=0;i<*count;i++) {
- ((short *)target)[i] *= ((short *)src)[i];
- }
- }
- else if (*dtype == MPI_UNSIGNED_SHORT) {
- for (i=0;i<*count;i++) {
- ((unsigned short *)target)[i] *= ((unsigned short *)src)[i];
- }
- }
-
- else if (*dtype == MPI_DOUBLE) {
- for (i=0;i<*count;i++) {
- ((double *)target)[i] *= ((double *)src)[i];
- }
- }
- else if (*dtype == MPI_FLOAT) {
- for (i=0;i<*count;i++) {
- ((float *)target)[i] *= ((float *)src)[i];
- }
- }
- else {
- printf("reduction operation not supported\n");
- }
- }
-
- else {
- printf("reduction operation not supported\n");
- }
-}
-
-void star_reduction(MPI_Op op, void *src, void *target, int *count, MPI_Datatype *dtype){
-
-#ifdef MPICH2_REDUCTION
-MPI_User_function * uop = MPIR_Op_table[op % 16 - 1];
- return (*uop) (src,target,count,dtype);
-#elif defined MVAPICH_REDUCTION
-MPI_User_function *uop;
-struct MPIR_OP *op_ptr;
-op_ptr = MPIR_ToPointer(op);
-uop = op_ptr->op;
- return (*uop) (src,target,count,dtype);
-#else
- return star_generic_reduction(op,src,target,count,dtype);
-#endif
-
-
-
-}
-
-
-#endif