From: Augustin Degomme <augustin.degomme@imag.fr>
Date: Thu, 16 Oct 2014 11:35:26 +0000 (+0200)
Subject: Add a new Infiniband network model, based on the works of Jerome Vienne.
X-Git-Tag: v3_12~732^2~285^2~8
X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/155a1e0df5db6960042e06036b942b9f93378b05?hp=b5bb23bbab9e4dda8dc341c2220b10cd27d2e0d7

Add a new Infiniband network model, based on the works of Jerome Vienne.
This computes penalties on communications to account for the slowdowns experimented on real systems, and modeled by jerome
check http://mescal.imag.fr/membres/jean-marc.vincent/index.html/PhD/Vienne.pdf for details

known issue: doesn't work yet with permanent receive mode, as some surf actions are not released properly
---

diff --git a/src/include/surf/surf.h b/src/include/surf/surf.h
index 1a9d56edea..c9f19e8186 100644
--- a/src/include/surf/surf.h
+++ b/src/include/surf/surf.h
@@ -1008,6 +1008,16 @@ XBT_PUBLIC_DATA(surf_network_model_t) surf_network_model;
  */
 XBT_PUBLIC(void) surf_network_model_init_SMPI(void);
 
+/** \ingroup SURF_models
+ *  \brief Same as network model 'LagrangeVelho', only with different correction factors.
+ *
+ * This model impelments a variant of the contention model on Infinband networks based on
+ * the works of Jérôme Vienne : http://mescal.imag.fr/membres/jean-marc.vincent/index.html/PhD/Vienne.pdf
+ *
+ *  \see surf_workstation_model_init_IB()
+ */
+XBT_PUBLIC(void) surf_network_model_init_IB(void);
+
 /** \ingroup SURF_models
  *  \brief Initializes the platform with the network model 'LegrandVelho'
  *
diff --git a/src/surf/network_ib.cpp b/src/surf/network_ib.cpp
index 29aeb32054..5be0d3ec11 100644
--- a/src/surf/network_ib.cpp
+++ b/src/surf/network_ib.cpp
@@ -3,3 +3,223 @@
 
 /* This program is free software; you can redistribute it and/or modify it
  * under the terms of the license (GNU LGPL) which comes with this package. */
+
+#include "network_ib.hpp"
+#include "simgrid/sg_config.h"
+#include "maxmin_private.hpp"
+
+XBT_LOG_EXTERNAL_DEFAULT_CATEGORY(surf_network);
+
+double Bs=0.925;
+double Be=0.965;
+double ys=1.35;
+
+
+static void IB_create_host_callback(sg_platf_host_cbarg_t t){
+  
+  static int id=0;
+// pour t->id -> rajouter une nouvelle struct dans le dict, pour stocker les comms actives
+  if(((NetworkIBModel*)surf_network_model)->active_nodes==NULL)
+    ((NetworkIBModel*)surf_network_model)->active_nodes=xbt_dict_new();
+  
+  IBNode* act = new IBNode(id);
+
+  id++;
+  xbt_dict_set(((NetworkIBModel*)surf_network_model)->active_nodes, t->id, act, NULL);
+ 
+}
+
+static void IB_action_state_changed_callback(NetworkActionPtr action, e_surf_action_state_t statein, e_surf_action_state_t stateout){
+ if(statein!=SURF_ACTION_RUNNING|| stateout!=SURF_ACTION_DONE)
+    return;
+  std::pair<IBNode*,IBNode*> pair = ((NetworkIBModel*)surf_network_model)->active_comms[action];
+  XBT_VERB("action %p finished", action);
+ 
+ ((NetworkIBModel*)surf_network_model)->updateIBfactors(action, pair.first, pair.second, 1);
+
+  ((NetworkIBModel*)surf_network_model)->active_comms.erase(action);
+  
+}
+
+
+static void IB_action_init_callback(NetworkActionPtr action,RoutingEdgePtr src, RoutingEdgePtr dst, double size, double rate){
+  if(((NetworkIBModel*)surf_network_model)->active_nodes==NULL)
+    xbt_die("IB comm added, without any node connected !");
+  
+  IBNode* act_src= (IBNode*) xbt_dict_get_or_null(((NetworkIBModel*)surf_network_model)->active_nodes, src->getName());
+  if(act_src==NULL)
+    xbt_die("could not find src node active comms !");
+  //act_src->rate=rate;
+  
+  IBNode* act_dst= (IBNode*) xbt_dict_get_or_null(((NetworkIBModel*)surf_network_model)->active_nodes, dst->getName());
+  if(act_dst==NULL)
+    xbt_die("could not find dst node active comms !");  
+ // act_dst->rate=rate;
+  
+  ((NetworkIBModel*)surf_network_model)->active_comms[action]=make_pair<IBNode*,IBNode*>(act_src, act_dst);
+  //post the action in the second dist, to retrieve in the other callback
+  XBT_VERB("action %p init", action);
+
+  ((NetworkIBModel*)surf_network_model)->updateIBfactors(action, act_src, act_dst, 0);
+  
+}
+
+
+
+/*********
+ * Model *
+ *********/
+
+/************************************************************************/
+/* New model based on MPI contention model for Infiniband platforms */
+/************************************************************************/
+/* @Inproceedings{mescal_vienne_phd, */
+/*  author={Jérôme Vienne}, */
+/*  title={prédiction de performances d’applications de calcul haute performance sur réseau Infiniband}, */
+/*  address={Grenoble FRANCE}, */
+/*  month=june, */
+/*  year={2010} */
+/*  } */
+void surf_network_model_init_IB(void)
+{
+
+  if (surf_network_model)
+    return;
+  surf_network_model = new NetworkIBModel();
+  net_define_callbacks();
+  xbt_dynar_push(model_list, &surf_network_model);
+  surf_callback_connect(networkActionStateChangedCallbacks, IB_action_state_changed_callback);
+  surf_callback_connect(networkCommunicateCallbacks, IB_action_init_callback);
+
+  sg_platf_host_add_cb(IB_create_host_callback);
+  xbt_cfg_setdefault_double(_sg_cfg_set, "network/weight_S", 8775);
+}
+
+NetworkIBModel::NetworkIBModel()
+ : NetworkSmpiModel() {
+  m_haveGap=false;
+  active_nodes=NULL;
+}
+
+NetworkIBModel::~NetworkIBModel()
+{
+  xbt_dict_cursor_t cursor = NULL;
+  IBNode* instance = NULL;
+  char *name = NULL;
+  xbt_dict_foreach(active_nodes, cursor, name, instance)
+    delete instance;
+  xbt_dict_free(&active_nodes);
+}
+
+void NetworkIBModel::computeIBfactors(IBNode *root) {
+  double penalized_bw=0.0;
+  double num_comm_out = (double) root->ActiveCommsUp.size();
+  double max_penalty_out=0.0;
+  //first, compute all outbound penalties to get their max
+  for (std::vector<ActiveComm*>::iterator it= root->ActiveCommsUp.begin(); it != root->ActiveCommsUp.end(); ++it) {
+    double my_penalty_out = 1.0;
+
+    if(num_comm_out!=1){
+      if((*it)->destination->nbActiveCommsDown > 2)//number of comms sent to the receiving node
+	my_penalty_out = num_comm_out * Bs * ys;
+      else
+	my_penalty_out = num_comm_out * Bs;
+    }
+
+    max_penalty_out = max(max_penalty_out,my_penalty_out);
+  }
+
+  for (std::vector<ActiveComm*>::iterator it= root->ActiveCommsUp.begin(); it != root->ActiveCommsUp.end(); ++it) {
+
+    //compute inbound penalty
+    double my_penalty_in = 1.0;
+    int nb_comms = (*it)->destination->nbActiveCommsDown;//total number of incoming comms
+    if(nb_comms!=1)
+      my_penalty_in = ((*it)->destination->ActiveCommsDown)[root] //number of comm sent to dest by root node
+		      * Be 
+		      * (*it)->destination->ActiveCommsDown.size();//number of different nodes sending to dest
+    
+    double penalty=max(my_penalty_in,max_penalty_out);
+    
+    double rate_before_update = (*it)->action->getBound();
+    //save initial rate of the action
+    if((*it)->init_rate==-1) 
+      (*it)->init_rate= rate_before_update;
+    
+    penalized_bw= ! num_comm_out ? (*it)->init_rate : (*it)->init_rate /penalty;
+    
+    if (!double_equals(penalized_bw, rate_before_update, sg_surf_precision)){
+     // XBT_VERB("%d->%d action %p penalty updated : in %f, out %f, final %f, before %f , initial rate %f", root->id,(*it)->destination->id,(*it)->action,my_penalty_in,my_penalty_out,penalized_bw, (*it)->action->getBound(), (*it)->init_rate );
+      lmm_update_variable_bound(p_maxminSystem, (*it)->action->getVariable(), penalized_bw);
+    }/*else{
+     // XBT_VERB("%d->%d action %p penalty not updated : in %f, out %f, final %f, initial rate %f", root->id,(*it)->destination->id,(*it)->action,my_penalty_in,my_penalty_out,penalized_bw, (*it)->init_rate );
+    }*/
+
+  }
+  XBT_VERB("Finished computing");
+}
+
+void NetworkIBModel::updateIBfactors_rec(IBNode *root, bool* updatedlist) {
+  if(updatedlist[root->id]==0){
+    XBT_VERB("Updating rec %d", root->id);
+    computeIBfactors(root);
+    updatedlist[root->id]=1;
+    for (std::vector<ActiveComm*>::iterator it= root->ActiveCommsUp.begin(); it != root->ActiveCommsUp.end(); ++it) {
+        if(updatedlist[(*it)->destination->id]!=1)
+          updateIBfactors_rec((*it)->destination, updatedlist);
+    }
+    for (std::map<IBNode*, int>::iterator it= root->ActiveCommsDown.begin(); it != root->ActiveCommsDown.end(); ++it) {
+        if(updatedlist[it->first->id]!=1)
+          updateIBfactors_rec(it->first, updatedlist);
+    }
+  }
+}
+
+
+void NetworkIBModel::updateIBfactors(NetworkActionPtr action, IBNode *from, IBNode * to, int remove) {
+  if (from == to)//disregard local comms (should use loopback)
+    return;
+  
+  bool* updated=(bool*)xbt_malloc0(xbt_dict_size(active_nodes)*sizeof(bool));
+  ActiveComm* comm=NULL;
+  if(remove){
+    if(to->ActiveCommsDown[from]==1)
+      to->ActiveCommsDown.erase(from);
+    else
+      to->ActiveCommsDown[from]-=1;
+
+    to->nbActiveCommsDown--;
+    for (std::vector<ActiveComm*>::iterator it= from->ActiveCommsUp.begin(); 
+	 it != from->ActiveCommsUp.end(); ++it) {
+      XBT_VERB("inside vector");
+      if((*it)->action==action){
+	comm=(*it);
+	from->ActiveCommsUp.erase(it);
+        XBT_VERB("action deleted");
+	break;
+      }
+    }
+    action->unref();
+    /*from->ActiveCommsUp.erase(
+        std::remove(from->ActiveCommsUp.begin(),
+            from->ActiveCommsUp.end(), make_pair(to, action)),
+            from->ActiveCommsUp.end());*/
+
+  }else{
+    //from->ActiveCommsUp.push_back(std::make_pair(to, action));
+    action->ref();
+
+    ActiveComm* comm=new ActiveComm();
+    comm->action=action;
+    comm->destination=to;
+    from->ActiveCommsUp.push_back(comm);
+
+    to->ActiveCommsDown[from]+=1;
+    to->nbActiveCommsDown++;
+  }
+  XBT_VERB("Updating %d", from->id);
+  updateIBfactors_rec(from, updated);
+  XBT_VERB("Finished updating %d", from->id);
+  if(comm)delete comm;
+  xbt_free(updated);
+}
diff --git a/src/surf/network_ib.hpp b/src/surf/network_ib.hpp
index b7d17157c5..8adaf4af6d 100644
--- a/src/surf/network_ib.hpp
+++ b/src/surf/network_ib.hpp
@@ -7,37 +7,46 @@
 #ifndef SURF_NETWORK_IB_HPP_
 #define SURF_NETWORK_IB_HPP_
 
-class NetworkIBModel : public NetworkModel {
-private:
-public:
-  NetworkIBModel();
-  NetworkIBModel(const char *name);
-  ~NetworkModel();
-  virtual ActionPtr communicate(RoutingEdgePtr src, RoutingEdgePtr dst,
-                                double size, double rate);
-  virtual NetworkLinkPtr createNetworkLink(const char *name,
-                                          double bw_initial,
-                                          tmgr_trace_t bw_trace,
-                                          double lat_initial,
-                                          tmgr_trace_t lat_trace,
-                                          e_surf_resource_state_t state_initial,
-                                          tmgr_trace_t state_trace,
-                                          e_surf_link_sharing_policy_t policy,
-                                          xbt_dict_t properties);
+#include "network_smpi.hpp"
+class IBNode;
+
+
+class ActiveComm{
+public :
+  //IBNode* origin;
+  IBNode* destination;
+  NetworkActionPtr action;
+  double init_rate;
+  ActiveComm() : destination(NULL),action(NULL),init_rate(-1){};
+  ~ActiveComm(){};
 };
 
-class NetworkIBLink : public NetworkLink {
+class IBNode{
+public :
+  int id;
+    //store related links, to ease computation of the penalties
+  std::vector<ActiveComm*> ActiveCommsUp;
+  //store the number of comms received from each node
+  std::map<IBNode*, int> ActiveCommsDown;
+  //number of comms the node is receiving
+  int nbActiveCommsDown;
+  IBNode(int id) : id(id),nbActiveCommsDown(0){};
+  ~IBNode(){};
+};
+
+class NetworkIBModel : public NetworkSmpiModel {
 private:
+  void updateIBfactors_rec(IBNode *root, bool* updatedlist);
+  void computeIBfactors(IBNode *root);
 public:
-  NetworkIBLink(NetworkModelPtr model, const char *name, xbt_dict_t props);
-  NetworkIBLink(NetworkModelPtr model, const char *name, xbt_dict_t props,
-                lmm_constraint_t constraint,
-                tmgr_history_t history,
-                tmgr_trace_t state_trace);
-  ~NetworkIBLink();
-  virtual void updateLatency(double value, double date=surf_get_clock());
-  virtual void updateBandwidth(double value, double date=surf_get_clock());
-
+  NetworkIBModel();
+  NetworkIBModel(const char *name);
+  ~NetworkIBModel();
+  void updateIBfactors(NetworkActionPtr action, IBNode *from, IBNode * to, int remove);
+  
+  xbt_dict_t active_nodes;
+  std::map<NetworkActionPtr , std::pair<IBNode*,IBNode*> > active_comms;
 
 };
+
 #endif
diff --git a/src/surf/surf_interface.cpp b/src/surf/surf_interface.cpp
index 6de2b62a46..2e799547f8 100644
--- a/src/surf/surf_interface.cpp
+++ b/src/surf/surf_interface.cpp
@@ -117,6 +117,9 @@ s_surf_model_description_t surf_network_model_description[] = {
   {"SMPI",
    "Realistic network model specifically tailored for HPC settings (accurate modeling of slow start with correction factors on three intervals: < 1KiB, < 64 KiB, >= 64 KiB)",
    surf_network_model_init_SMPI},
+  {"IB",
+   "Realistic network model specifically tailored for HPC settings, with Infiniband contention model",
+   surf_network_model_init_IB},
   {"CM02",
    "Legacy network analytic model (Very similar to LV08, but without corrective factors. The timings of small messages are thus poorly modeled).",
    surf_network_model_init_CM02},
@@ -872,7 +875,7 @@ void Action::setBound(double bound)
 {
   XBT_IN("(%p,%g)", this, bound);
   if (p_variable)
-    lmm_update_variable_bound(getModel()->getMaxminSystem(), getVariable(), bound);
+    lmm_update_variable_bound(getModel()->getMaxminSystem(), p_variable, bound);
 
   if (getModel()->getUpdateMechanism() == UM_LAZY && getLastUpdate()!=surf_get_clock())
     heapRemove(getModel()->getActionHeap());