From: degomme Date: Fri, 17 Jun 2016 00:03:54 +0000 (+0200) Subject: Add Dragonfly topology. Use XC30's Cray description as a basis X-Git-Tag: v3_14~962 X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/3fc07005abb54bbceec067c73455b50f96e5d908?hp=924b94ede191ef9c844da856e535254c764f4a14;ds=sidebyside Add Dragonfly topology. Use XC30's Cray description as a basis For now minimal routing is supported, but non-minimal one will be added soon. We still need to add randomization of the links used at each level No documentation yet --- diff --git a/src/surf/AsClusterDragonfly.cpp b/src/surf/AsClusterDragonfly.cpp new file mode 100644 index 0000000000..3844806341 --- /dev/null +++ b/src/surf/AsClusterDragonfly.cpp @@ -0,0 +1,374 @@ +/* Copyright (c) 2014-2016. The SimGrid Team. All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#include "src/surf/AsClusterDragonfly.hpp" +#include "src/surf/network_interface.hpp" +#include "src/surf/xml/platf.hpp" // FIXME: move that back to the parsing area + +#include +#include + +XBT_LOG_NEW_DEFAULT_SUBCATEGORY(surf_route_cluster_dragonfly, surf_route_cluster, "Dragonfly Routing part of surf"); + + + + + + + +namespace simgrid { +namespace surf { + +AsClusterDragonfly::AsClusterDragonfly(const char*name) + : AsCluster(name) { +} + +AsClusterDragonfly::~AsClusterDragonfly() { + + if(this->routers_!=NULL){ + int i; + for (i=0; inumGroups_*this->numChassisPerGroup_*this->numBladesPerChassis_;i++) + delete(routers_[i]); + xbt_free(routers_); + } +} + +unsigned int *AsClusterDragonfly::rankId_to_coords(int rankId) +{ + + //coords : group, chassis, blade, node + unsigned int *coords = (unsigned int *) malloc(4 * sizeof(unsigned int)); + coords[0] = rankId/ (numChassisPerGroup_*numBladesPerChassis_*numNodesPerBlade_); + rankId=rankId%(numChassisPerGroup_*numBladesPerChassis_*numNodesPerBlade_); + coords[1] = rankId/ (numBladesPerChassis_*numNodesPerBlade_); + rankId=rankId%(numBladesPerChassis_*numNodesPerBlade_); + coords[2] = rankId/ numNodesPerBlade_; + coords[3]=rankId%numNodesPerBlade_; + + return coords; +} + +void AsClusterDragonfly::parse_specific_arguments(sg_platf_cluster_cbarg_t cluster) { + std::vector parameters; + std::vector tmp; + boost::split(parameters, cluster->topo_parameters, boost::is_any_of(";")); + + // TODO : we have to check for zeros and negative numbers, or it might crash + if (parameters.size() != 4){ + surf_parse_error("Dragonfly are defined by the number of groups, chassiss per groups, blades per chassis, nodes per blade"); + } + + // Blue network : number of groups, number of links between each group + boost::split(tmp, parameters[0], boost::is_any_of(",")); + if(tmp.size() != 2) { + surf_parse_error("Dragonfly topologies are defined by 3 levels with 2 elements each, and one with one element"); + } + + this->numGroups_=xbt_str_parse_int(tmp[0].c_str(), "Invalid number of groups: %s"); + this->numLinksBlue_=xbt_str_parse_int(tmp[1].c_str(), "Invalid number of links for the blue level: %s"); + + // Black network : number of chassiss/group, number of links between each router on the black network + boost::split(tmp, parameters[1], boost::is_any_of(",")); + if(tmp.size() != 2) { + surf_parse_error("Dragonfly topologies are defined by 3 levels with 2 elements each, and one with one element"); + } + + this->numChassisPerGroup_=xbt_str_parse_int(tmp[0].c_str(), "Invalid number of groups: %s"); + this->numLinksBlack_=xbt_str_parse_int(tmp[1].c_str(), "Invalid number of links for the black level: %s"); + + + // Green network : number of blades/chassis, number of links between each router on the green network + boost::split(tmp, parameters[2], boost::is_any_of(",")); + if(tmp.size() != 2) { + surf_parse_error("Dragonfly topologies are defined by 3 levels with 2 elements each, and one with one element"); + } + + this->numBladesPerChassis_=xbt_str_parse_int(tmp[0].c_str(), "Invalid number of groups: %s"); + this->numLinksGreen_=xbt_str_parse_int(tmp[1].c_str(), "Invalid number of links for the green level: %s"); + + + // The last part of topo_parameters should be the number of nodes per blade + this->numNodesPerBlade_ = xbt_str_parse_int(parameters[3].c_str(), "Last parameter is not the amount of nodes per blade: %s"); + this->cluster_ = cluster; +} + +/* +* Generate the cluster once every node is created +*/ +void AsClusterDragonfly::seal(){ + if(this->numNodesPerBlade_ == 0) { + return; + } + + this->generateRouters(); + this->generateLinks(); +} + +DragonflyRouter::DragonflyRouter(int group, int chassis, int blade){ + this->group_=group; + this->chassis_=chassis; + this->blade_=blade; +} + +DragonflyRouter::~DragonflyRouter(){ + if(this->myNodes_!=NULL) + xbt_free(myNodes_); + if(this->greenLinks_!=NULL) + xbt_free(greenLinks_); + if(this->blackLinks_!=NULL) + xbt_free(blackLinks_); + if(this->blueLinks_!=NULL) + xbt_free(blueLinks_); +} + + +void AsClusterDragonfly::generateRouters() { + +unsigned int i, j, k; + +this->routers_=(DragonflyRouter**)xbt_malloc0(this->numGroups_*this->numChassisPerGroup_*this->numBladesPerChassis_*sizeof(DragonflyRouter*)); + +for(i=0;inumGroups_;i++){ + for(j=0;jnumChassisPerGroup_;j++){ + for(k=0;knumBladesPerChassis_;k++){ + DragonflyRouter* router = new DragonflyRouter(i,j,k); + this->routers_[i*this->numChassisPerGroup_*this->numBladesPerChassis_+j*this->numBladesPerChassis_+k]=router; + } + } +} + +} + + +void AsClusterDragonfly::createLink(char* id, Link** linkup, Link** linkdown){ + *linkup=NULL; + *linkdown=NULL; + s_sg_platf_link_cbarg_t linkTemplate; + memset(&linkTemplate, 0, sizeof(linkTemplate)); + linkTemplate.bandwidth = this->cluster_->bw; + linkTemplate.latency = this->cluster_->lat; + linkTemplate.policy = this->cluster_->sharing_policy; // sthg to do with that ? + linkTemplate.id = id; + sg_platf_new_link(&linkTemplate); + XBT_DEBUG("Generating link %s", id); + Link* link; + std::string tmpID; + if (this->cluster_->sharing_policy == SURF_LINK_FULLDUPLEX) { + tmpID = std::string(linkTemplate.id) + "_UP"; + link = Link::byName(tmpID.c_str()); + *linkup = link; // check link? + tmpID = std::string(linkTemplate.id) + "_DOWN"; + link = Link::byName(tmpID.c_str()); + *linkdown = link; // check link ? + } + else { + link = Link::byName(linkTemplate.id); + *linkup = link; + *linkdown = link; + } + + free((void*)linkTemplate.id); +} + + +void AsClusterDragonfly::generateLinks() { + + unsigned int i, j, k, l,m; + static int uniqueId = 0; + char* id = NULL; + Link* linkup, *linkdown; + + int numRouters = this->numGroups_*this->numChassisPerGroup_*this->numBladesPerChassis_; + + int numLinksperLink=1; + if (this->cluster_->sharing_policy == SURF_LINK_FULLDUPLEX) + numLinksperLink=2; + + + //Links from routers to their local nodes. + for(i=0; irouters_[i]->myNodes_=(Link**)xbt_malloc0(numLinksperLink*this->numNodesPerBlade_*sizeof(Link*)); + this->routers_[i]->greenLinks_=(Link**)xbt_malloc0(this->numLinksGreen_*this->numBladesPerChassis_*sizeof(Link*)); + this->routers_[i]->blackLinks_=(Link**)xbt_malloc0(this->numLinksBlack_*this->numChassisPerGroup_*sizeof(Link*)); + + for(j=0; j< numLinksperLink*this->numNodesPerBlade_; j+=numLinksperLink){ + id = bprintf("local_link_from_router_%d_to_node_%d_%d", i, j/2, uniqueId); + this->createLink(id, &linkup, &linkdown); + if (this->cluster_->sharing_policy == SURF_LINK_FULLDUPLEX) { + this->routers_[i]->myNodes_[j] = linkup; + this->routers_[i]->myNodes_[j+1] = linkdown; + } + else { + this->routers_[i]->myNodes_[j] = linkup; + } + uniqueId++; + } + } + + //Green links from routers to same chassis routers - alltoall + for(i=0; inumGroups_*this->numChassisPerGroup_;i++){ + for(j=0; jnumBladesPerChassis_;j++){ + for(k=j+1;knumBladesPerChassis_;k++){ + for(l=0;lnumLinksGreen_;l++){ + id = bprintf("green_link_in_chassis_%d_between_routers_%d_and_%d_%d", i%numChassisPerGroup_, j, k, uniqueId); + this->createLink(id, &linkup, &linkdown); + this->routers_[i*numBladesPerChassis_+j]->greenLinks_[k*this->numLinksGreen_+l] = linkup; + this->routers_[i*numBladesPerChassis_+k]->greenLinks_[j*this->numLinksGreen_+l] = linkdown; + uniqueId++; + } + } + } + } + + //Black links from routers to same group routers - alltoall + for(i=0; inumGroups_;i++){ + for(j=0; jnumChassisPerGroup_;j++){ + for(k=j+1;knumChassisPerGroup_;k++){ + for(l=0;lnumBladesPerChassis_;l++){ + for(m=0;mnumLinksBlack_;m++){ + + id = bprintf("black_link_in_group_%d_between_chassis_%d_and_%d_blade_%d_%d", i, j, k,l, uniqueId); + this->createLink(id, &linkup, &linkdown); + this->routers_[i*numBladesPerChassis_*numChassisPerGroup_+j*numBladesPerChassis_+l]->blackLinks_[k*this->numLinksBlack_+m] = linkup; + this->routers_[i*numBladesPerChassis_*numChassisPerGroup_+k*numBladesPerChassis_+l]->blackLinks_[j*this->numLinksBlack_+m] = linkdown; + uniqueId++; + } + } + } + } + } + + + //Blue links betweeen groups - Not all routers involved, only one per group is linked to others. Let's say router n of each group is linked to group n. FIXME: this limits the number of groups + + for(i=0; inumGroups_;i++){ + for(j=i+1; jnumGroups_;j++){ + unsigned int routernumi=i*numBladesPerChassis_*numChassisPerGroup_+j; + unsigned int routernumj=j*numBladesPerChassis_*numChassisPerGroup_+i; + this->routers_[routernumi]->blueLinks_=(Link**)xbt_malloc0(this->numLinksBlue_*sizeof(Link*)); + this->routers_[routernumj]->blueLinks_=(Link**)xbt_malloc0(this->numLinksBlue_*sizeof(Link*)); + for(m=0;mnumLinksBlue_;m++){ + id = bprintf("blue_link_between_group_%d_and_%d_routers_%d_and_%d_%d", i, j, routernumi,routernumj, uniqueId); + this->createLink(id, &linkup, &linkdown); + this->routers_[routernumi]->blueLinks_[m] = linkup; + this->routers_[routernumj]->blueLinks_[m] = linkdown; + uniqueId++; + } + } + } +} + + +void AsClusterDragonfly::getRouteAndLatency(NetCard * src, NetCard * dst, sg_platf_route_cbarg_t route, double *latency) { + + //Minimal routing version. + + if (dst->isRouter() || src->isRouter()) + return; + //TODO:loopback and limiters + XBT_VERB("dragonfly_get_route_and_latency from '%s'[%d] to '%s'[%d]", + src->name(), src->id(), dst->name(), dst->id()); + +// if ((src->id() == dst->id()) && hasLoopback_) { +// s_surf_parsing_link_up_down_t info = xbt_dynar_get_as(privateLinks_, src->id() * linkCountPerNode_, s_surf_parsing_link_up_down_t); + +// route->link_list->push_back(info.linkUp); +// if (lat) +// *lat += info.linkUp->getLatency(); +// return; +// } + + unsigned int *myCoords, *targetCoords; + myCoords = rankId_to_coords(src->id()); + targetCoords = rankId_to_coords(dst->id()); + XBT_DEBUG("src : %u group, %u chassis, %u blade, %u node", myCoords[0], myCoords[1], myCoords[2], myCoords[3]); + XBT_DEBUG("dst : %u group, %u chassis, %u blade, %u node", targetCoords[0], targetCoords[1], targetCoords[2], targetCoords[3]); + + DragonflyRouter* myRouter = routers_[myCoords[0]*(numChassisPerGroup_*numBladesPerChassis_)+myCoords[1] * numBladesPerChassis_+myCoords[2]]; + DragonflyRouter* targetRouter = routers_[targetCoords[0]*(numChassisPerGroup_*numBladesPerChassis_)+targetCoords[1] *numBladesPerChassis_ +targetCoords[2]]; + DragonflyRouter* currentRouter=myRouter; + + //node->router local link + route->link_list->push_back(myRouter->myNodes_[myCoords[3]]); + if(latency) { + *latency += myRouter->myNodes_[myCoords[3]]->getLatency(); + } + +if (src->id() == dst->id()){ +return; +} + if(targetRouter!=myRouter){ + + //are we on a different group ? + if(targetRouter->group_ != currentRouter->group_){ + //go to the router of our group connected to this one. + if(currentRouter->blade_!=targetCoords[0]){ + //go to the nth router in our chassis +//TODO : randomize used green link + route->link_list->push_back(currentRouter->greenLinks_[targetCoords[0]*numLinksGreen_]); + if(latency) { + *latency += currentRouter->greenLinks_[targetCoords[0]*numLinksGreen_]->getLatency(); + } + currentRouter=routers_[myCoords[0]*(numChassisPerGroup_*numBladesPerChassis_)+myCoords[1] * numBladesPerChassis_+targetCoords[0]]; + } + + if(currentRouter->chassis_!=0){ + //go to the first chassis of our group +//TODO : randomize used black link + route->link_list->push_back(currentRouter->blackLinks_[0]); + if(latency) { + *latency += currentRouter->blackLinks_[0]->getLatency(); + } + currentRouter=routers_[myCoords[0]*(numChassisPerGroup_*numBladesPerChassis_)+targetCoords[0]]; + } + +//TODO : randomize used blue link + //go to destination group - the only optical hop + route->link_list->push_back(currentRouter->blueLinks_[0]); + if(latency) { + *latency += currentRouter->blueLinks_[0]->getLatency(); + } + currentRouter=routers_[targetCoords[0]*(numChassisPerGroup_*numBladesPerChassis_)+myCoords[0]]; + } + + + //same group, but same chassis ? + if(targetRouter->chassis_ != currentRouter->chassis_){ +//TODO : randomize used black link + route->link_list->push_back(currentRouter->blackLinks_[targetCoords[1]*numLinksBlack_]); + if(latency) { + *latency += currentRouter->blackLinks_[targetCoords[1]*numLinksBlack_]->getLatency(); + } + currentRouter=routers_[targetCoords[0]*(numChassisPerGroup_*numBladesPerChassis_)+currentRouter->chassis_*numBladesPerChassis_+currentRouter->blade_]; + } + + //same chassis, but same blade ? + if(targetRouter->blade_ != currentRouter->blade_){ +//TODO : randomize used green link + route->link_list->push_back(currentRouter->greenLinks_[targetCoords[2]*numLinksGreen_]); + if(latency) { + *latency += currentRouter->greenLinks_[targetCoords[2]*numLinksGreen_]->getLatency(); + } + currentRouter=routers_[targetCoords[0]*(numChassisPerGroup_*numBladesPerChassis_)+targetCoords[1]*numBladesPerChassis_+targetCoords[2]]; + xbt_assert(currentRouter==targetRouter, "You've got routed into oblivion. Oops"); + } + + } + + + //router->node local link + route->link_list->push_back(targetRouter->myNodes_[targetCoords[3]]); + if(latency) { + *latency += targetRouter->myNodes_[targetCoords[3]]->getLatency(); + } + + xbt_free(myCoords); + xbt_free(targetCoords); + + +} + } +} diff --git a/src/surf/AsClusterDragonfly.hpp b/src/surf/AsClusterDragonfly.hpp new file mode 100644 index 0000000000..8396105b6a --- /dev/null +++ b/src/surf/AsClusterDragonfly.hpp @@ -0,0 +1,54 @@ +/* Copyright (c) 2014-2016. The SimGrid Team. All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#ifndef SURF_ROUTING_CLUSTER_DRAGONFLY_HPP_ +#define SURF_ROUTING_CLUSTER_DRAGONFLY_HPP_ + +#include "src/surf/AsCluster.hpp" + +namespace simgrid { + namespace surf { + + +class XBT_PRIVATE DragonflyRouter { + public: + int group_; + int chassis_; + int blade_; + Link** blueLinks_=NULL; + Link** blackLinks_=NULL; + Link** greenLinks_=NULL; + Link** myNodes_=NULL; + DragonflyRouter(int i, int j, int k); + ~DragonflyRouter(); +}; + + +class XBT_PRIVATE AsClusterDragonfly:public simgrid::surf::AsCluster { + public: + explicit AsClusterDragonfly(const char*name); + ~AsClusterDragonfly() override; +// void create_links_for_node(sg_platf_cluster_cbarg_t cluster, int id, int rank, int position) override; + void getRouteAndLatency(NetCard * src, NetCard * dst, sg_platf_route_cbarg_t into, double *latency) override; + void parse_specific_arguments(sg_platf_cluster_cbarg_t cluster) override; + void seal() override; + void generateRouters(); + void generateLinks(); + void createLink(char* id, Link** linkup, Link** linkdown); + unsigned int * rankId_to_coords(int rankId); + private: + sg_platf_cluster_cbarg_t cluster_; + unsigned int numNodesPerBlade_ = 0; + unsigned int numBladesPerChassis_ = 0; + unsigned int numChassisPerGroup_ = 0; + unsigned int numGroups_ = 0; + unsigned int numLinksGreen_ = 0; + unsigned int numLinksBlack_ = 0; + unsigned int numLinksBlue_ = 0; + DragonflyRouter** routers_=NULL; + }; + + }} +#endif diff --git a/src/surf/sg_platf.cpp b/src/surf/sg_platf.cpp index 23a04956cc..f67e1a6e78 100644 --- a/src/surf/sg_platf.cpp +++ b/src/surf/sg_platf.cpp @@ -27,6 +27,7 @@ #include "src/surf/AsCluster.hpp" #include "src/surf/AsClusterTorus.hpp" #include "src/surf/AsClusterFatTree.hpp" +#include "src/surf/AsClusterDragonfly.hpp" #include "src/surf/AsDijkstra.hpp" #include "src/surf/AsFloyd.hpp" #include "src/surf/AsFull.hpp" @@ -237,6 +238,7 @@ void sg_platf_new_cluster(sg_platf_cluster_cbarg_t cluster) { using simgrid::surf::AsCluster; using simgrid::surf::AsClusterTorus; + using simgrid::surf::AsClusterDragonfly; using simgrid::surf::AsClusterFatTree; int rankId=0; @@ -250,6 +252,9 @@ void sg_platf_new_cluster(sg_platf_cluster_cbarg_t cluster) case SURF_CLUSTER_TORUS: AS.routing = A_surfxml_AS_routing_ClusterTorus; break; + case SURF_CLUSTER_DRAGONFLY: + AS.routing = A_surfxml_AS_routing_ClusterDragonfly; + break; case SURF_CLUSTER_FAT_TREE: AS.routing = A_surfxml_AS_routing_ClusterFatTree; break; @@ -821,6 +826,7 @@ simgrid::s4u::As * sg_platf_new_AS_begin(sg_platf_AS_cbarg_t AS) switch(AS->routing){ case A_surfxml_AS_routing_Cluster: new_as = new simgrid::surf::AsCluster(AS->id); break; case A_surfxml_AS_routing_ClusterTorus: new_as = new simgrid::surf::AsClusterTorus(AS->id); break; + case A_surfxml_AS_routing_ClusterDragonfly: new_as = new simgrid::surf::AsClusterDragonfly(AS->id); break; case A_surfxml_AS_routing_ClusterFatTree: new_as = new simgrid::surf::AsClusterFatTree(AS->id); break; case A_surfxml_AS_routing_Dijkstra: new_as = new simgrid::surf::AsDijkstra(AS->id, 0); break; case A_surfxml_AS_routing_DijkstraCache: new_as = new simgrid::surf::AsDijkstra(AS->id, 1); break; diff --git a/src/surf/xml/platf_private.hpp b/src/surf/xml/platf_private.hpp index 67c7768f3b..a195303c3d 100644 --- a/src/surf/xml/platf_private.hpp +++ b/src/surf/xml/platf_private.hpp @@ -23,6 +23,7 @@ typedef size_t yy_size_t; XBT_PUBLIC(sg_netcard_t) sg_netcard_by_name_or_null(const char *name); typedef enum { + SURF_CLUSTER_DRAGONFLY=3, SURF_CLUSTER_FAT_TREE=2, SURF_CLUSTER_FLAT = 1, SURF_CLUSTER_TORUS = 0 diff --git a/src/surf/xml/simgrid.dtd b/src/surf/xml/simgrid.dtd index 6806909b34..a84cc90480 100644 --- a/src/surf/xml/simgrid.dtd +++ b/src/surf/xml/simgrid.dtd @@ -85,7 +85,7 @@ To upgrade your files, use the tool simgrid_update_xml - + @@ -129,7 +129,7 @@ To upgrade your files, use the tool simgrid_update_xml - + diff --git a/src/surf/xml/surfxml_sax_cb.cpp b/src/surf/xml/surfxml_sax_cb.cpp index a4f90145e7..d42e4297b3 100644 --- a/src/surf/xml/surfxml_sax_cb.cpp +++ b/src/surf/xml/surfxml_sax_cb.cpp @@ -527,6 +527,9 @@ void ETag_surfxml_cluster(void){ case A_surfxml_cluster_topology_FAT___TREE: cluster.topology = SURF_CLUSTER_FAT_TREE; break; + case A_surfxml_cluster_topology_DRAGONFLY: + cluster.topology= SURF_CLUSTER_DRAGONFLY ; + break; default: surf_parse_error("Invalid cluster topology for cluster %s", cluster.id); diff --git a/src/xbt/log.c b/src/xbt/log.c index 7c57168c03..af04c7597a 100644 --- a/src/xbt/log.c +++ b/src/xbt/log.c @@ -242,6 +242,7 @@ static void xbt_log_connect_categories(void) XBT_LOG_CONNECT(surf_routing_generic); XBT_LOG_CONNECT(surf_route_cluster); XBT_LOG_CONNECT(surf_route_cluster_torus); + XBT_LOG_CONNECT(surf_route_cluster_dragonfly); XBT_LOG_CONNECT(surf_route_dijkstra); XBT_LOG_CONNECT(surf_route_fat_tree); XBT_LOG_CONNECT(surf_route_floyd); diff --git a/tools/cmake/DefinePackages.cmake b/tools/cmake/DefinePackages.cmake index 78f127106c..c4ee641d7c 100644 --- a/tools/cmake/DefinePackages.cmake +++ b/tools/cmake/DefinePackages.cmake @@ -67,6 +67,7 @@ set(EXTRA_DIST src/surf/AsCluster.hpp src/surf/AsClusterFatTree.hpp src/surf/AsClusterTorus.hpp + src/surf/AsClusterDragonfly.hpp src/surf/AsDijkstra.hpp src/surf/AsFloyd.hpp src/surf/AsFull.hpp @@ -292,6 +293,7 @@ set(SURF_SRC src/surf/AsCluster.cpp src/surf/AsClusterFatTree.cpp src/surf/AsClusterTorus.cpp + src/surf/AsClusterDragonfly.cpp src/surf/AsDijkstra.cpp src/surf/AsFloyd.cpp src/surf/AsFull.cpp