+
+}
+
+
+
+int smpi_coll_tuned_reduce_mvapich2( void *sendbuf,
+ void *recvbuf,
+ int count,
+ MPI_Datatype datatype,
+ MPI_Op op, int root, MPI_Comm comm)
+{
+ if(mv2_reduce_thresholds_table == NULL)
+ init_mv2_reduce_tables_stampede();
+
+ int mpi_errno = MPI_SUCCESS;
+ int range = 0;
+ int range_threshold = 0;
+ int range_intra_threshold = 0;
+ int is_commutative, pof2;
+ int comm_size = 0;
+ int nbytes = 0;
+ int sendtype_size;
+ int is_two_level = 0;
+
+ comm_size = smpi_comm_size(comm);
+ sendtype_size=smpi_datatype_size(datatype);
+ nbytes = count * sendtype_size;
+
+ if (count == 0)
+ return MPI_SUCCESS;
+
+ is_commutative = smpi_op_is_commute(op);
+
+ /* find nearest power-of-two less than or equal to comm_size */
+ for( pof2 = 1; pof2 <= comm_size; pof2 <<= 1 );
+ pof2 >>=1;
+
+
+ /* Search for the corresponding system size inside the tuning table */
+ while ((range < (mv2_size_reduce_tuning_table - 1)) &&
+ (comm_size > mv2_reduce_thresholds_table[range].numproc)) {
+ range++;
+ }
+ /* Search for corresponding inter-leader function */
+ while ((range_threshold < (mv2_reduce_thresholds_table[range].size_inter_table - 1))
+ && (nbytes >
+ mv2_reduce_thresholds_table[range].inter_leader[range_threshold].max)
+ && (mv2_reduce_thresholds_table[range].inter_leader[range_threshold].max !=
+ -1)) {
+ range_threshold++;
+ }
+
+ /* Search for corresponding intra node function */
+ while ((range_intra_threshold < (mv2_reduce_thresholds_table[range].size_intra_table - 1))
+ && (nbytes >
+ mv2_reduce_thresholds_table[range].intra_node[range_intra_threshold].max)
+ && (mv2_reduce_thresholds_table[range].intra_node[range_intra_threshold].max !=
+ -1)) {
+ range_intra_threshold++;
+ }
+
+ /* Set intra-node function pt for reduce_two_level */
+ MV2_Reduce_intra_function =
+ mv2_reduce_thresholds_table[range].intra_node[range_intra_threshold].
+ MV2_pt_Reduce_function;
+ /* Set inter-leader pt */
+ MV2_Reduce_function =
+ mv2_reduce_thresholds_table[range].inter_leader[range_threshold].
+ MV2_pt_Reduce_function;
+
+ if(mv2_reduce_intra_knomial_factor<0)
+ {
+ mv2_reduce_intra_knomial_factor = mv2_reduce_thresholds_table[range].intra_k_degree;
+ }
+ if(mv2_reduce_inter_knomial_factor<0)
+ {
+ mv2_reduce_inter_knomial_factor = mv2_reduce_thresholds_table[range].inter_k_degree;
+ }
+ if(mv2_reduce_thresholds_table[range].is_two_level_reduce[range_threshold] == 1){
+ is_two_level = 1;
+ }
+ /* We call Reduce function */
+ if(is_two_level == 1)
+ {
+ if (is_commutative == 1) {
+ if(smpi_comm_get_leaders_comm(comm)==MPI_COMM_NULL){
+ smpi_comm_init_smp(comm);
+ }
+ mpi_errno = MPIR_Reduce_two_level_helper_MV2(sendbuf, recvbuf, count,
+ datatype, op, root, comm);
+ } else {
+ mpi_errno = MPIR_Reduce_binomial_MV2(sendbuf, recvbuf, count,
+ datatype, op, root, comm);
+ }
+ } else if(MV2_Reduce_function == &MPIR_Reduce_inter_knomial_wrapper_MV2 ){
+ if(is_commutative ==1)
+ {
+ mpi_errno = MV2_Reduce_function(sendbuf, recvbuf, count,
+ datatype, op, root, comm);
+ } else {
+ mpi_errno = MPIR_Reduce_binomial_MV2(sendbuf, recvbuf, count,
+ datatype, op, root, comm);
+ }
+ } else if(MV2_Reduce_function == &MPIR_Reduce_redscat_gather_MV2){
+ if (/*(HANDLE_GET_KIND(op) == HANDLE_KIND_BUILTIN) &&*/ (count >= pof2))
+ {
+ mpi_errno = MV2_Reduce_function(sendbuf, recvbuf, count,
+ datatype, op, root, comm);
+ } else {
+ mpi_errno = MPIR_Reduce_binomial_MV2(sendbuf, recvbuf, count,
+ datatype, op, root, comm);
+ }
+ } else {
+ mpi_errno = MV2_Reduce_function(sendbuf, recvbuf, count,
+ datatype, op, root, comm);
+ }
+
+
+ return mpi_errno;
+
+}
+
+
+int smpi_coll_tuned_reduce_scatter_mvapich2(void *sendbuf, void *recvbuf, int *recvcnts,
+ MPI_Datatype datatype, MPI_Op op,
+ MPI_Comm comm)
+{
+ int mpi_errno = MPI_SUCCESS;
+ int i = 0, comm_size = smpi_comm_size(comm), total_count = 0, type_size =
+ 0, nbytes = 0;
+ int range = 0;
+ int range_threshold = 0;
+ int is_commutative = 0;
+ int *disps = xbt_malloc(comm_size * sizeof (int));
+
+ if(mv2_red_scat_thresholds_table==NULL)
+ init_mv2_reduce_scatter_tables_stampede();
+
+ is_commutative=smpi_op_is_commute(op);
+ for (i = 0; i < comm_size; i++) {
+ disps[i] = total_count;
+ total_count += recvcnts[i];
+ }
+
+ type_size=smpi_datatype_size(datatype);
+ nbytes = total_count * type_size;
+
+ if (is_commutative) {
+
+ /* Search for the corresponding system size inside the tuning table */
+ while ((range < (mv2_size_red_scat_tuning_table - 1)) &&
+ (comm_size > mv2_red_scat_thresholds_table[range].numproc)) {
+ range++;
+ }
+ /* Search for corresponding inter-leader function */
+ while ((range_threshold < (mv2_red_scat_thresholds_table[range].size_inter_table - 1))
+ && (nbytes >
+ mv2_red_scat_thresholds_table[range].inter_leader[range_threshold].max)
+ && (mv2_red_scat_thresholds_table[range].inter_leader[range_threshold].max !=
+ -1)) {
+ range_threshold++;
+ }
+
+ /* Set inter-leader pt */
+ MV2_Red_scat_function =
+ mv2_red_scat_thresholds_table[range].inter_leader[range_threshold].
+ MV2_pt_Red_scat_function;
+
+ mpi_errno = MV2_Red_scat_function(sendbuf, recvbuf,
+ recvcnts, datatype,
+ op, comm);
+ } else {
+ int is_block_regular = 1;
+ for (i = 0; i < (comm_size - 1); ++i) {
+ if (recvcnts[i] != recvcnts[i+1]) {
+ is_block_regular = 0;
+ break;
+ }
+ }
+ int pof2 = 1;
+ while (pof2 < comm_size) pof2 <<= 1;
+ if (pof2 == comm_size && is_block_regular) {
+ /* noncommutative, pof2 size, and block regular */
+ mpi_errno = MPIR_Reduce_scatter_non_comm_MV2(sendbuf, recvbuf,
+ recvcnts, datatype,
+ op, comm);
+ }
+ mpi_errno = smpi_coll_tuned_reduce_scatter_mpich_rdb(sendbuf, recvbuf,
+ recvcnts, datatype,
+ op, comm);
+ }
+
+ return mpi_errno;
+
+}
+
+
+
+int smpi_coll_tuned_scatter_mvapich2(void *sendbuf,
+ int sendcnt,
+ MPI_Datatype sendtype,
+ void *recvbuf,
+ int recvcnt,
+ MPI_Datatype recvtype,
+ int root, MPI_Comm comm)
+{
+ int range = 0, range_threshold = 0, range_threshold_intra = 0;
+ int mpi_errno = MPI_SUCCESS;
+ // int mpi_errno_ret = MPI_SUCCESS;
+ int rank, nbytes, comm_size;
+ int recvtype_size, sendtype_size;
+ int partial_sub_ok = 0;
+ int conf_index = 0;
+ int local_size = -1;
+ int i;
+ MPI_Comm shmem_comm;
+ // MPID_Comm *shmem_commptr=NULL;
+ if(mv2_scatter_thresholds_table==NULL)
+ init_mv2_scatter_tables_stampede();
+
+ if(smpi_comm_get_leaders_comm(comm)==MPI_COMM_NULL){
+ smpi_comm_init_smp(comm);
+ }
+
+ comm_size = smpi_comm_size(comm);
+
+ rank = smpi_comm_rank(comm);
+
+ if (rank == root) {
+ sendtype_size=smpi_datatype_size(sendtype);
+ nbytes = sendcnt * sendtype_size;
+ } else {
+ recvtype_size=smpi_datatype_size(recvtype);
+ nbytes = recvcnt * recvtype_size;
+ }
+
+ // check if safe to use partial subscription mode
+ if (smpi_comm_is_uniform(comm)) {
+
+ shmem_comm = smpi_comm_get_intra_comm(comm);
+ local_size = smpi_comm_size(shmem_comm);
+ i = 0;
+ if (mv2_scatter_table_ppn_conf[0] == -1) {
+ // Indicating user defined tuning
+ conf_index = 0;
+ }else{
+ do {
+ if (local_size == mv2_scatter_table_ppn_conf[i]) {
+ conf_index = i;
+ partial_sub_ok = 1;
+ break;
+ }
+ i++;
+ } while(i < mv2_scatter_num_ppn_conf);
+ }
+ }
+
+ if (partial_sub_ok != 1) {
+ conf_index = 0;
+ }
+
+ /* Search for the corresponding system size inside the tuning table */
+ while ((range < (mv2_size_scatter_tuning_table[conf_index] - 1)) &&
+ (comm_size > mv2_scatter_thresholds_table[conf_index][range].numproc)) {
+ range++;
+ }
+ /* Search for corresponding inter-leader function */
+ while ((range_threshold < (mv2_scatter_thresholds_table[conf_index][range].size_inter_table - 1))
+ && (nbytes >
+ mv2_scatter_thresholds_table[conf_index][range].inter_leader[range_threshold].max)
+ && (mv2_scatter_thresholds_table[conf_index][range].inter_leader[range_threshold].max != -1)) {
+ range_threshold++;
+ }
+
+ /* Search for corresponding intra-node function */
+ while ((range_threshold_intra <
+ (mv2_scatter_thresholds_table[conf_index][range].size_intra_table - 1))
+ && (nbytes >
+ mv2_scatter_thresholds_table[conf_index][range].intra_node[range_threshold_intra].max)
+ && (mv2_scatter_thresholds_table[conf_index][range].intra_node[range_threshold_intra].max !=
+ -1)) {
+ range_threshold_intra++;
+ }
+
+ MV2_Scatter_function = mv2_scatter_thresholds_table[conf_index][range].inter_leader[range_threshold]
+ .MV2_pt_Scatter_function;
+
+ if(MV2_Scatter_function == &MPIR_Scatter_mcst_wrap_MV2) {
+#if defined(_MCST_SUPPORT_)
+ if(comm->ch.is_mcast_ok == 1
+ && mv2_use_mcast_scatter == 1
+ && comm->ch.shmem_coll_ok == 1) {
+ MV2_Scatter_function = &MPIR_Scatter_mcst_MV2;
+ } else
+#endif /*#if defined(_MCST_SUPPORT_) */
+ {
+ if(mv2_scatter_thresholds_table[conf_index][range].inter_leader[range_threshold + 1].
+ MV2_pt_Scatter_function != NULL) {
+ MV2_Scatter_function = mv2_scatter_thresholds_table[conf_index][range].inter_leader[range_threshold + 1]
+ .MV2_pt_Scatter_function;
+ } else {
+ /* Fallback! */
+ MV2_Scatter_function = &MPIR_Scatter_MV2_Binomial;
+ }
+ }
+ }
+
+ if( (MV2_Scatter_function == &MPIR_Scatter_MV2_two_level_Direct) ||
+ (MV2_Scatter_function == &MPIR_Scatter_MV2_two_level_Binomial)) {
+ if( smpi_comm_is_blocked(comm)) {
+ MV2_Scatter_intra_function = mv2_scatter_thresholds_table[conf_index][range].intra_node[range_threshold_intra]
+ .MV2_pt_Scatter_function;
+
+ mpi_errno =
+ MV2_Scatter_function(sendbuf, sendcnt, sendtype,
+ recvbuf, recvcnt, recvtype, root,
+ comm);
+ } else {
+ mpi_errno = MPIR_Scatter_MV2_Binomial(sendbuf, sendcnt, sendtype,
+ recvbuf, recvcnt, recvtype, root,
+ comm);
+
+ }
+ } else {
+ mpi_errno = MV2_Scatter_function(sendbuf, sendcnt, sendtype,
+ recvbuf, recvcnt, recvtype, root,
+ comm);
+ }
+ return (mpi_errno);