/************ Alltoall variables and initializers */
+#ifndef SMPI_MVAPICH2_SELECTOR_STAMPEDE_HPP
+#define SMPI_MVAPICH2_SELECTOR_STAMPEDE_HPP
+
+#include <algorithm>
+
#define MV2_MAX_NB_THRESHOLDS 32
XBT_PUBLIC(void) smpi_coll_cleanup_mvapich2(void);
-typedef struct {
+struct mv2_alltoall_tuning_element {
int min;
int max;
int (*MV2_pt_Alltoall_function)(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, MPI_Comm comm_ptr);
-} mv2_alltoall_tuning_element;
+};
-typedef struct {
+struct mv2_alltoall_tuning_table {
int numproc;
int size_table;
mv2_alltoall_tuning_element algo_table[MV2_MAX_NB_THRESHOLDS];
mv2_alltoall_tuning_element in_place_algo_table[MV2_MAX_NB_THRESHOLDS];
-} mv2_alltoall_tuning_table;
+};
int (*MV2_Alltoall_function)(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, MPI_Comm comm_ptr) = NULL;
static void init_mv2_alltoall_tables_stampede()
{
- int i;
int agg_table_sum = 0;
mv2_alltoall_tuning_table** table_ptrs = NULL;
mv2_alltoall_num_ppn_conf = 3;
if (simgrid::smpi::Colls::smpi_coll_cleanup_callback == NULL)
simgrid::smpi::Colls::smpi_coll_cleanup_callback = &smpi_coll_cleanup_mvapich2;
- mv2_alltoall_thresholds_table = static_cast<mv2_alltoall_tuning_table**>(
- xbt_malloc(sizeof(mv2_alltoall_tuning_table*) * mv2_alltoall_num_ppn_conf));
- table_ptrs = static_cast<mv2_alltoall_tuning_table**>(
- xbt_malloc(sizeof(mv2_alltoall_tuning_table*) * mv2_alltoall_num_ppn_conf));
- mv2_size_alltoall_tuning_table = static_cast<int*>(xbt_malloc(sizeof(int) * mv2_alltoall_num_ppn_conf));
- mv2_alltoall_table_ppn_conf = static_cast<int*>(xbt_malloc(mv2_alltoall_num_ppn_conf * sizeof(int)));
+ mv2_alltoall_thresholds_table = new mv2_alltoall_tuning_table*[mv2_alltoall_num_ppn_conf];
+ table_ptrs = new mv2_alltoall_tuning_table*[mv2_alltoall_num_ppn_conf];
+ mv2_size_alltoall_tuning_table = new int[mv2_alltoall_num_ppn_conf];
+ mv2_alltoall_table_ppn_conf = new int[mv2_alltoall_num_ppn_conf];
mv2_alltoall_table_ppn_conf[0] = 1;
mv2_size_alltoall_tuning_table[0] = 6;
mv2_alltoall_tuning_table mv2_tmp_alltoall_thresholds_table_1ppn[] = {
};
table_ptrs[2] = mv2_tmp_alltoall_thresholds_table_16ppn;
agg_table_sum = 0;
- for (i = 0; i < mv2_alltoall_num_ppn_conf; i++) {
+ for (int i = 0; i < mv2_alltoall_num_ppn_conf; i++) {
agg_table_sum += mv2_size_alltoall_tuning_table[i];
}
- mv2_alltoall_thresholds_table[0] =
- static_cast<mv2_alltoall_tuning_table*>(xbt_malloc(agg_table_sum * sizeof(mv2_alltoall_tuning_table)));
- memcpy(mv2_alltoall_thresholds_table[0], table_ptrs[0],
- (sizeof(mv2_alltoall_tuning_table) * mv2_size_alltoall_tuning_table[0]));
- for (i = 1; i < mv2_alltoall_num_ppn_conf; i++) {
+ mv2_alltoall_thresholds_table[0] = new mv2_alltoall_tuning_table[agg_table_sum];
+ std::copy_n(table_ptrs[0], mv2_size_alltoall_tuning_table[0], mv2_alltoall_thresholds_table[0]);
+ for (int i = 1; i < mv2_alltoall_num_ppn_conf; i++) {
mv2_alltoall_thresholds_table[i] = mv2_alltoall_thresholds_table[i - 1] + mv2_size_alltoall_tuning_table[i - 1];
- memcpy(mv2_alltoall_thresholds_table[i], table_ptrs[i],
- (sizeof(mv2_alltoall_tuning_table) * mv2_size_alltoall_tuning_table[i]));
+ std::copy_n(table_ptrs[i], mv2_size_alltoall_tuning_table[i], mv2_alltoall_thresholds_table[i]);
}
- xbt_free(table_ptrs);
+ delete[] table_ptrs;
}
/************ Allgather variables and initializers */
-typedef struct {
+struct mv2_allgather_tuning_element {
int min;
int max;
int (*MV2_pt_Allgatherction)(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, MPI_Comm comm_ptr);
-} mv2_allgather_tuning_element;
+};
-typedef struct {
+struct mv2_allgather_tuning_table {
int numproc;
int two_level[MV2_MAX_NB_THRESHOLDS];
int size_inter_table;
mv2_allgather_tuning_element inter_leader[MV2_MAX_NB_THRESHOLDS];
-} mv2_allgather_tuning_table;
+};
int (*MV2_Allgatherction)(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, MPI_Comm comm);
static void init_mv2_allgather_tables_stampede()
{
- int i;
int agg_table_sum = 0;
if (simgrid::smpi::Colls::smpi_coll_cleanup_callback == NULL)
simgrid::smpi::Colls::smpi_coll_cleanup_callback = &smpi_coll_cleanup_mvapich2;
mv2_allgather_tuning_table** table_ptrs = NULL;
mv2_allgather_num_ppn_conf = 3;
- mv2_allgather_thresholds_table = static_cast<mv2_allgather_tuning_table**>(
- xbt_malloc(sizeof(mv2_allgather_tuning_table*) * mv2_allgather_num_ppn_conf));
- table_ptrs = static_cast<mv2_allgather_tuning_table**>(
- xbt_malloc(sizeof(mv2_allgather_tuning_table*) * mv2_allgather_num_ppn_conf));
- mv2_size_allgather_tuning_table = static_cast<int*>(xbt_malloc(sizeof(int) * mv2_allgather_num_ppn_conf));
- mv2_allgather_table_ppn_conf = static_cast<int*>(xbt_malloc(mv2_allgather_num_ppn_conf * sizeof(int)));
+ mv2_allgather_thresholds_table = new mv2_allgather_tuning_table*[mv2_allgather_num_ppn_conf];
+ table_ptrs = new mv2_allgather_tuning_table*[mv2_allgather_num_ppn_conf];
+ mv2_size_allgather_tuning_table = new int[mv2_allgather_num_ppn_conf];
+ mv2_allgather_table_ppn_conf = new int[mv2_allgather_num_ppn_conf];
mv2_allgather_table_ppn_conf[0] = 1;
mv2_size_allgather_tuning_table[0] = 6;
mv2_allgather_tuning_table mv2_tmp_allgather_thresholds_table_1ppn[] = {
};
table_ptrs[2] = mv2_tmp_allgather_thresholds_table_16ppn;
agg_table_sum = 0;
- for (i = 0; i < mv2_allgather_num_ppn_conf; i++) {
+ for (int i = 0; i < mv2_allgather_num_ppn_conf; i++) {
agg_table_sum += mv2_size_allgather_tuning_table[i];
}
- mv2_allgather_thresholds_table[0] =
- static_cast<mv2_allgather_tuning_table*>(xbt_malloc(agg_table_sum * sizeof(mv2_allgather_tuning_table)));
- memcpy(mv2_allgather_thresholds_table[0], table_ptrs[0],
- (sizeof(mv2_allgather_tuning_table) * mv2_size_allgather_tuning_table[0]));
- for (i = 1; i < mv2_allgather_num_ppn_conf; i++) {
+ mv2_allgather_thresholds_table[0] = new mv2_allgather_tuning_table[agg_table_sum];
+ std::copy_n(table_ptrs[0], mv2_size_allgather_tuning_table[0], mv2_allgather_thresholds_table[0]);
+ for (int i = 1; i < mv2_allgather_num_ppn_conf; i++) {
mv2_allgather_thresholds_table[i] = mv2_allgather_thresholds_table[i - 1] + mv2_size_allgather_tuning_table[i - 1];
- memcpy(mv2_allgather_thresholds_table[i], table_ptrs[i],
- (sizeof(mv2_allgather_tuning_table) * mv2_size_allgather_tuning_table[i]));
+ std::copy_n(table_ptrs[i], mv2_size_allgather_tuning_table[i], mv2_allgather_thresholds_table[i]);
}
- xbt_free(table_ptrs);
+ delete[] table_ptrs;
}
/************ Gather variables and initializers */
-typedef struct {
+struct mv2_gather_tuning_element {
int min;
int max;
int (*MV2_pt_Gather_function)(void* sendbuf, int sendcnt, MPI_Datatype sendtype, void* recvbuf, int recvcnt,
MPI_Datatype recvtype, int root, MPI_Comm comm_ptr);
-} mv2_gather_tuning_element;
+};
-typedef struct {
+struct mv2_gather_tuning_table {
int numproc;
int size_inter_table;
mv2_gather_tuning_element inter_leader[MV2_MAX_NB_THRESHOLDS];
int size_intra_table;
mv2_gather_tuning_element intra_node[MV2_MAX_NB_THRESHOLDS];
-} mv2_gather_tuning_table;
+};
int mv2_size_gather_tuning_table = 7;
mv2_gather_tuning_table* mv2_gather_thresholds_table = NULL;
if (simgrid::smpi::Colls::smpi_coll_cleanup_callback == NULL)
simgrid::smpi::Colls::smpi_coll_cleanup_callback = &smpi_coll_cleanup_mvapich2;
mv2_size_gather_tuning_table = 7;
- mv2_gather_thresholds_table =
- static_cast<mv2_gather_tuning_table*>(xbt_malloc(mv2_size_gather_tuning_table * sizeof(mv2_gather_tuning_table)));
+ mv2_gather_thresholds_table = new mv2_gather_tuning_table[mv2_size_gather_tuning_table];
mv2_gather_tuning_table mv2_tmp_gather_thresholds_table[] = {
{16,
2,
{{0, -1, &MPIR_Gather_intra}}},
};
- memcpy(mv2_gather_thresholds_table, mv2_tmp_gather_thresholds_table,
- mv2_size_gather_tuning_table * sizeof(mv2_gather_tuning_table));
+ std::copy_n(mv2_tmp_gather_thresholds_table, mv2_size_gather_tuning_table, mv2_gather_thresholds_table);
}
/************ Allgatherv variables and initializers */
-typedef struct {
+struct mv2_allgatherv_tuning_element {
int min;
int max;
int (*MV2_pt_Allgatherv_function)(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int* recvcounts,
int* displs, MPI_Datatype recvtype, MPI_Comm commg);
-} mv2_allgatherv_tuning_element;
+};
-typedef struct {
+struct mv2_allgatherv_tuning_table {
int numproc;
int size_inter_table;
mv2_allgatherv_tuning_element inter_leader[MV2_MAX_NB_THRESHOLDS];
-} mv2_allgatherv_tuning_table;
+};
int (*MV2_Allgatherv_function)(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int* recvcounts,
int* displs, MPI_Datatype recvtype, MPI_Comm comm);
if (simgrid::smpi::Colls::smpi_coll_cleanup_callback == NULL)
simgrid::smpi::Colls::smpi_coll_cleanup_callback = &smpi_coll_cleanup_mvapich2;
mv2_size_allgatherv_tuning_table = 6;
- mv2_allgatherv_thresholds_table = static_cast<mv2_allgatherv_tuning_table*>(
- xbt_malloc(mv2_size_allgatherv_tuning_table * sizeof(mv2_allgatherv_tuning_table)));
+ mv2_allgatherv_thresholds_table = new mv2_allgatherv_tuning_table[mv2_size_allgatherv_tuning_table];
mv2_allgatherv_tuning_table mv2_tmp_allgatherv_thresholds_table[] = {
{
16,
},
};
- memcpy(mv2_allgatherv_thresholds_table, mv2_tmp_allgatherv_thresholds_table,
- mv2_size_allgatherv_tuning_table * sizeof(mv2_allgatherv_tuning_table));
+ std::copy_n(mv2_tmp_allgatherv_thresholds_table, mv2_size_allgatherv_tuning_table, mv2_allgatherv_thresholds_table);
}
/************ Allreduce variables and initializers */
-typedef struct {
+struct mv2_allreduce_tuning_element {
int min;
int max;
int (*MV2_pt_Allreducection)(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
MPI_Comm comm);
-} mv2_allreduce_tuning_element;
+};
-typedef struct {
+struct mv2_allreduce_tuning_table {
int numproc;
int mcast_enabled;
int is_two_level_allreduce[MV2_MAX_NB_THRESHOLDS];
mv2_allreduce_tuning_element inter_leader[MV2_MAX_NB_THRESHOLDS];
int size_intra_table;
mv2_allreduce_tuning_element intra_node[MV2_MAX_NB_THRESHOLDS];
-} mv2_allreduce_tuning_table;
+};
int (*MV2_Allreducection)(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
MPI_Comm comm) = NULL;
if (simgrid::smpi::Colls::smpi_coll_cleanup_callback == NULL)
simgrid::smpi::Colls::smpi_coll_cleanup_callback = &smpi_coll_cleanup_mvapich2;
mv2_size_allreduce_tuning_table = 8;
- mv2_allreduce_thresholds_table = static_cast<mv2_allreduce_tuning_table*>(
- xbt_malloc(mv2_size_allreduce_tuning_table * sizeof(mv2_allreduce_tuning_table)));
+ mv2_allreduce_thresholds_table = new mv2_allreduce_tuning_table[mv2_size_allreduce_tuning_table];
mv2_allreduce_tuning_table mv2_tmp_allreduce_thresholds_table[] = {
{
16,
},
};
- memcpy(mv2_allreduce_thresholds_table, mv2_tmp_allreduce_thresholds_table,
- mv2_size_allreduce_tuning_table * sizeof(mv2_allreduce_tuning_table));
+ std::copy_n(mv2_tmp_allreduce_thresholds_table, mv2_size_allreduce_tuning_table, mv2_allreduce_thresholds_table);
}
-typedef struct {
+struct mv2_bcast_tuning_element {
int min;
int max;
int (*MV2_pt_Bcast_function)(void* buf, int count, MPI_Datatype datatype, int root, MPI_Comm comm_ptr);
int zcpy_pipelined_knomial_factor;
-} mv2_bcast_tuning_element;
+};
-typedef struct {
+struct mv2_bcast_tuning_table {
int numproc;
int bcast_segment_size;
int intra_node_knomial_factor;
mv2_bcast_tuning_element inter_leader[MV2_MAX_NB_THRESHOLDS];
int size_intra_table;
mv2_bcast_tuning_element intra_node[MV2_MAX_NB_THRESHOLDS];
-} mv2_bcast_tuning_table;
+};
int mv2_size_bcast_tuning_table = 0;
mv2_bcast_tuning_table* mv2_bcast_thresholds_table = NULL;
if (simgrid::smpi::Colls::smpi_coll_cleanup_callback == NULL)
simgrid::smpi::Colls::smpi_coll_cleanup_callback = &smpi_coll_cleanup_mvapich2;
mv2_size_bcast_tuning_table = 8;
- mv2_bcast_thresholds_table =
- static_cast<mv2_bcast_tuning_table*>(xbt_malloc(mv2_size_bcast_tuning_table * sizeof(mv2_bcast_tuning_table)));
+ mv2_bcast_thresholds_table = new mv2_bcast_tuning_table[mv2_size_bcast_tuning_table];
mv2_bcast_tuning_table mv2_tmp_bcast_thresholds_table[] = {
{16,
{32768, 524288, &MPIR_Shmem_Bcast_MV2, -1},
{524288, -1, &MPIR_Shmem_Bcast_MV2, -1}}}};
- memcpy(mv2_bcast_thresholds_table, mv2_tmp_bcast_thresholds_table,
- mv2_size_bcast_tuning_table * sizeof(mv2_bcast_tuning_table));
+ std::copy_n(mv2_tmp_bcast_thresholds_table, mv2_size_bcast_tuning_table, mv2_bcast_thresholds_table);
}
/************ Reduce variables and initializers */
-typedef struct {
+struct mv2_reduce_tuning_element {
int min;
int max;
int (*MV2_pt_Reduce_function)(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root,
MPI_Comm comm_ptr);
-} mv2_reduce_tuning_element;
+};
-typedef struct {
+struct mv2_reduce_tuning_table {
int numproc;
int inter_k_degree;
int intra_k_degree;
mv2_reduce_tuning_element inter_leader[MV2_MAX_NB_THRESHOLDS];
int size_intra_table;
mv2_reduce_tuning_element intra_node[MV2_MAX_NB_THRESHOLDS];
-} mv2_reduce_tuning_table;
+};
int mv2_size_reduce_tuning_table = 0;
mv2_reduce_tuning_table* mv2_reduce_thresholds_table = NULL;
simgrid::smpi::Colls::smpi_coll_cleanup_callback = &smpi_coll_cleanup_mvapich2;
/*Stampede*/
mv2_size_reduce_tuning_table = 8;
- mv2_reduce_thresholds_table =
- static_cast<mv2_reduce_tuning_table*>(xbt_malloc(mv2_size_reduce_tuning_table * sizeof(mv2_reduce_tuning_table)));
+ mv2_reduce_thresholds_table = new mv2_reduce_tuning_table[mv2_size_reduce_tuning_table];
mv2_reduce_tuning_table mv2_tmp_reduce_thresholds_table[] = {
{
16,
},
};
- memcpy(mv2_reduce_thresholds_table, mv2_tmp_reduce_thresholds_table,
- mv2_size_reduce_tuning_table * sizeof(mv2_reduce_tuning_table));
+ std::copy_n(mv2_tmp_reduce_thresholds_table, mv2_size_reduce_tuning_table, mv2_reduce_thresholds_table);
}
/************ Reduce scatter variables and initializers */
-typedef struct {
+struct mv2_red_scat_tuning_element {
int min;
int max;
int (*MV2_pt_Red_scat_function)(void* sendbuf, void* recvbuf, int* recvcnts, MPI_Datatype datatype, MPI_Op op,
MPI_Comm comm_ptr);
-} mv2_red_scat_tuning_element;
+};
-typedef struct {
+struct mv2_red_scat_tuning_table {
int numproc;
int size_inter_table;
mv2_red_scat_tuning_element inter_leader[MV2_MAX_NB_THRESHOLDS];
-} mv2_red_scat_tuning_table;
+};
int mv2_size_red_scat_tuning_table = 0;
mv2_red_scat_tuning_table* mv2_red_scat_thresholds_table = NULL;
if (simgrid::smpi::Colls::smpi_coll_cleanup_callback == NULL)
simgrid::smpi::Colls::smpi_coll_cleanup_callback = &smpi_coll_cleanup_mvapich2;
mv2_size_red_scat_tuning_table = 6;
- mv2_red_scat_thresholds_table = static_cast<mv2_red_scat_tuning_table*>(
- xbt_malloc(mv2_size_red_scat_tuning_table * sizeof(mv2_red_scat_tuning_table)));
+ mv2_red_scat_thresholds_table = new mv2_red_scat_tuning_table[mv2_size_red_scat_tuning_table];
mv2_red_scat_tuning_table mv2_tmp_red_scat_thresholds_table[] = {
{
16,
},
};
- memcpy(mv2_red_scat_thresholds_table, mv2_tmp_red_scat_thresholds_table,
- mv2_size_red_scat_tuning_table * sizeof(mv2_red_scat_tuning_table));
+ std::copy_n(mv2_tmp_red_scat_thresholds_table, mv2_size_red_scat_tuning_table, mv2_red_scat_thresholds_table);
}
/************ Scatter variables and initializers */
-typedef struct {
+struct mv2_scatter_tuning_element {
int min;
int max;
int (*MV2_pt_Scatter_function)(void* sendbuf, int sendcnt, MPI_Datatype sendtype, void* recvbuf, int recvcnt,
MPI_Datatype recvtype, int root, MPI_Comm comm);
-} mv2_scatter_tuning_element;
+};
-typedef struct {
+struct mv2_scatter_tuning_table {
int numproc;
int size_inter_table;
mv2_scatter_tuning_element inter_leader[MV2_MAX_NB_THRESHOLDS];
int size_intra_table;
mv2_scatter_tuning_element intra_node[MV2_MAX_NB_THRESHOLDS];
-} mv2_scatter_tuning_table;
+};
int* mv2_scatter_table_ppn_conf = NULL;
int mv2_scatter_num_ppn_conf = 1;
simgrid::smpi::Colls::smpi_coll_cleanup_callback = &smpi_coll_cleanup_mvapich2;
int agg_table_sum = 0;
- int i;
mv2_scatter_tuning_table** table_ptrs = NULL;
mv2_scatter_num_ppn_conf = 3;
- mv2_scatter_thresholds_table =
- static_cast<mv2_scatter_tuning_table**>(xbt_malloc(sizeof(mv2_scatter_tuning_table*) * mv2_scatter_num_ppn_conf));
- table_ptrs =
- static_cast<mv2_scatter_tuning_table**>(xbt_malloc(sizeof(mv2_scatter_tuning_table*) * mv2_scatter_num_ppn_conf));
- mv2_size_scatter_tuning_table = static_cast<int*>(xbt_malloc(sizeof(int) * mv2_scatter_num_ppn_conf));
- mv2_scatter_table_ppn_conf = static_cast<int*>(xbt_malloc(mv2_scatter_num_ppn_conf * sizeof(int)));
+ mv2_scatter_thresholds_table = new mv2_scatter_tuning_table*[mv2_scatter_num_ppn_conf];
+ table_ptrs = new mv2_scatter_tuning_table*[mv2_scatter_num_ppn_conf];
+ mv2_size_scatter_tuning_table = new int[mv2_scatter_num_ppn_conf];
+ mv2_scatter_table_ppn_conf = new int[mv2_scatter_num_ppn_conf];
mv2_scatter_table_ppn_conf[0] = 1;
mv2_size_scatter_tuning_table[0] = 6;
mv2_scatter_tuning_table mv2_tmp_scatter_thresholds_table_1ppn[] = {
};
table_ptrs[2] = mv2_tmp_scatter_thresholds_table_16ppn;
agg_table_sum = 0;
- for (i = 0; i < mv2_scatter_num_ppn_conf; i++) {
+ for (int i = 0; i < mv2_scatter_num_ppn_conf; i++) {
agg_table_sum += mv2_size_scatter_tuning_table[i];
}
- mv2_scatter_thresholds_table[0] =
- static_cast<mv2_scatter_tuning_table*>(xbt_malloc(agg_table_sum * sizeof(mv2_scatter_tuning_table)));
- memcpy(mv2_scatter_thresholds_table[0], table_ptrs[0],
- (sizeof(mv2_scatter_tuning_table) * mv2_size_scatter_tuning_table[0]));
- for (i = 1; i < mv2_scatter_num_ppn_conf; i++) {
+ mv2_scatter_thresholds_table[0] = new mv2_scatter_tuning_table[agg_table_sum];
+ std::copy_n(table_ptrs[0], mv2_size_scatter_tuning_table[0], mv2_scatter_thresholds_table[0]);
+ for (int i = 1; i < mv2_scatter_num_ppn_conf; i++) {
mv2_scatter_thresholds_table[i] = mv2_scatter_thresholds_table[i - 1] + mv2_size_scatter_tuning_table[i - 1];
- memcpy(mv2_scatter_thresholds_table[i], table_ptrs[i],
- (sizeof(mv2_scatter_tuning_table) * mv2_size_scatter_tuning_table[i]));
+ std::copy_n(table_ptrs[i], mv2_size_scatter_tuning_table[i], mv2_scatter_thresholds_table[i]);
}
- xbt_free(table_ptrs);
+ delete[] table_ptrs;
}
+
+#endif