--- /dev/null
+# Copyright (c) 2006-2021. The SimGrid Team. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the license (GNU LGPL) which comes with this package.
+
+# This example shows how to build a torus cluster with multi-core hosts.
+#
+# However, each leaf in the torus is a StarZone, composed of several CPUs
+#
+# Each actor runs in a specific CPU. One sender broadcasts a message to all receivers.
+
+import simgrid
+import sys
+import typing
+
+
+class Sender:
+ """
+ Send a msg for each host in its host list
+ """
+
+ def __init__(self, hosts, msg_size=int(1e6)):
+ self.hosts = hosts
+ self.msg_size = msg_size
+
+ # Actors that are created as object will execute their __call__ method.
+ # So, the following constitutes the main function of the Sender actor.
+ def __call__(self):
+ pending_comms = []
+ mboxes = []
+
+ for host in self.hosts:
+ msg = "Hello, I'm alive and running on " + simgrid.this_actor.get_host().name
+ mbox = simgrid.Mailbox.by_name(host.name)
+ mboxes.append(mbox)
+ pending_comms.append(mbox.put_async(msg, self.msg_size))
+
+ simgrid.this_actor.info("Done dispatching all messages")
+
+ # Now that all message exchanges were initiated, wait for their completion in one single call
+ simgrid.Comm.wait_all(pending_comms)
+
+ simgrid.this_actor.info("Goodbye now!")
+
+
+class Receiver:
+ """
+ Receiver actor: wait for 1 message on the mailbox identified by the hostname
+ """
+
+ def __call__(self):
+ mbox = simgrid.Mailbox.by_name(simgrid.this_actor.get_host().name)
+ received = mbox.get()
+ simgrid.this_actor.info("I got a '%s'." % received)
+
+#####################################################################################################
+
+
+def create_hostzone(zone: simgrid.NetZone, coord: typing.List[int], id: int) -> typing.Tuple[simgrid.NetPoint, simgrid.NetPoint]:
+ """
+ Callback to set a cluster leaf/element
+
+ In our example, each leaf if a StarZone, composed of 8 CPUs.
+ Each CPU is modeled as a host, connected to the outer world through a high-speed PCI link.
+ Obs.: CPU0 is the gateway for this zone
+
+ (outer world)
+ CPU0 (gateway)
+ up ->| |
+ | |<-down
+ +star+
+ / / \ \
+ / / \ \<-- 100Gbs, 10us link (1 link UP and 1 link DOWN for full-duplex)
+ / / \ \
+ / / \ \
+ CPU1 ... CPU8
+
+ :param zone: Cluster netzone being created (usefull to create the hosts/links inside it)
+ :param coord: Coordinates in the cluster
+ :param id: Internal identifier in the torus (for information)
+ :return netpoint, gateway: the netpoint to the StarZone and CPU0 as gateway
+ """
+ num_cpus = 8 # Number of CPUs in the zone
+ speed = "1Gf" # Speed of each CPU
+ link_bw = "100GBps" # Link bw connecting the CPU
+ link_lat = "1ns" # Link latency
+
+ hostname = "host" + str(id)
+ # create the StarZone
+ host_zone = simgrid.NetZone.create_star_zone(hostname)
+ # setting my Torus parent zone
+ host_zone.set_parent(zone)
+
+ gateway = None
+ # create CPUs
+ for i in range(num_cpus):
+ cpu_name = hostname + "-cpu" + str(i)
+ host = host_zone.create_host(cpu_name, speed).seal()
+ # the first CPU is the gateway
+ if i == 0:
+ gateway = host
+ # create split-duplex link
+ link = host_zone.create_split_duplex_link("link-" + cpu_name, link_bw)
+ link.set_latency(link_lat).seal()
+ # connecting CPU to outer world
+ host_zone.add_route(host.get_netpoint(), None, None, None, [
+ simgrid.LinkInRoute(link, simgrid.LinkInRoute.Direction.UP)], True)
+
+ # seal newly created netzone
+ host_zone.seal()
+ return host_zone.get_netpoint(), gateway.get_netpoint()
+
+#####################################################################################################
+
+
+def create_limiter(zone: simgrid.NetZone, coord: typing.List[int], id: int) -> simgrid.Link:
+ """
+ Callback to create limiter link (1Gbs) for each netpoint
+
+ The coord parameter depends on the cluster being created:
+ - Torus: Direct translation of the Torus' dimensions, e.g. (0, 0, 0) for a 3-D Torus
+ - Fat-Tree: A pair (level in the tree, id), e.g. (0, 0) for first leaf in the tree and (1,0) for the first switch at
+ level 1.
+ - Dragonfly: a tuple (group, chassis, blades/routers, nodes), e.g. (0, 0, 0, 0) for first node in the cluster. To
+ identify the router inside a (group, chassis, blade), we use MAX_UINT in the last parameter (e.g. 0, 0, 0,
+ 4294967295).
+
+ :param zone: Torus netzone being created (usefull to create the hosts/links inside it)
+ :param coord: Coordinates in the cluster
+ :param id: Internal identifier in the torus (for information)
+ :return: Limiter link
+ """
+ return zone.create_link("limiter-" + str(id), [1e9]).seal()
+
+
+def create_torus_cluster():
+ """
+ Creates a TORUS cluster
+
+ Creates a TORUS cluster with dimensions 2x2x2
+
+ The cluster has 8 elements/leaves in total. Each element is a StarZone containing 8 Hosts.
+ Each pair in the torus is connected through 2 links:
+ 1) limiter: a 1Gbs limiter link (set by user through the set_limiter callback)
+ 2) link: 10Gbs link connecting the components (created automatically)
+
+ (Y-axis=2)
+ A
+ |
+ | D (Z-axis=2)
+ + / 10 Gbs
+ | +
+ |/ limiter=1Gps
+ B-----+----C (X-axis=2)
+
+ For example, a communication from A to C goes through:
+ <tt> A->limiter(A)->link(A-B)->limiter(B)->link(B-C)->limiter(C)->C </tt>
+
+ More precisely, considering that A and C are StarZones, a
+ communication from A-CPU-3 to C-CPU-7 goes through:
+ 1) StarZone A: A-CPU-3 -> link-up-A-CPU-3 -> A-CPU-0
+ 2) A-CPU-0->limiter(A)->link(A-B)->limiter(B)->link(B-C)->limiter(C)->C-CPU-0
+ 3) StarZone C: C-CPU-0-> link-down-C-CPU-7 -> C-CPU-7
+
+ Note that we don't have limiter links inside the StarZones(A, B, C),
+ but we have limiters in the Torus that are added to the links in the path (as we can see in "2)")
+
+ More details in: <a href="https://simgrid.org/doc/latest/Platform_examples.html?highlight=torus#torus-cluster">Torus
+ Cluster</a>
+ """
+ # create the torus cluster, 10Gbs link between elements in the cluster
+ simgrid.NetZone.create_torus_zone("cluster", None, [2, 2, 2], simgrid.ClusterCallbacks(create_hostzone, None, create_limiter), 10e9, 10e-6,
+ simgrid.Link.SharingPolicy.SPLITDUPLEX).seal()
+
+#####################################################################################################
+
+
+def create_fatTree_cluster():
+ """
+ Creates a Fat-Tree cluster
+
+ Creates a Fat-Tree cluster with 2 levels and 6 nodes
+ The following parameters are used to create this cluster:
+ - Levels: 2 - two-level of switches in the cluster
+ - Down links: 2, 3 - L2 routers is connected to 2 elements, L1 routers to 3 elements
+ - Up links: 1, 2 - Each node (A-F) is connected to 1 L1 router, L1 routers are connected to 2 L2
+ - Link count: 1, 1 - Use 1 link in each level
+
+ The first parameter describes how many levels we have.
+ The following ones describe the connection between the elements and must have exactly n_levels components.
+
+
+ S3 S4 <-- Level 2 routers
+ link:limiter - / \ / \
+ + ++ +
+ link: 10GBps --> | / \ |
+ (full-duplex) | / \ |
+ + + + +
+ | / \ |
+ S1 S2 <-- Level 1 routers
+ link:limiter -> | |
+ + +
+ link:10GBps --> /|\ /|\
+ / | \ / | \
+ + + + + + +
+ link:limiter -> / | \ / | \
+ A B C D E F <-- level 0 Nodes
+
+ Each element (A to F) is a StarZone containing 8 Hosts.
+ The connection uses 2 links:
+ 1) limiter: a 1Gbs limiter link (set by user through the set_limiter callback)
+ 2) link: 10Gbs link connecting the components (created automatically)
+
+ For example, a communication from A to C goes through:
+ <tt> A->limiter(A)->link(A-S1)->limiter(S1)->link(S1-C)->->limiter(C)->C</tt>
+
+ More precisely, considering that A and C are StarZones, a
+ communication from A-CPU-3 to C-CPU-7 goes through:
+ 1) StarZone A: A-CPU-3 -> link-up-A-CPU-3 -> A-CPU-0
+ 2) A-CPU-0->limiter(A)->link(A-S1)->limiter(S1)->link(S1-C)->limiter(C)->C-CPU-0
+ 3) StarZone C: C-CPU-0-> link-down-C-CPU-7 -> C-CPU-7
+
+ More details in: <a href="https://simgrid.org/doc/latest/Platform_examples.html#fat-tree-cluster">Fat-Tree
+ Cluster</a>
+ """
+ # create the fat tree cluster, 10Gbs link between elements in the cluster
+ simgrid.NetZone.create_fatTree_zone("cluster", None, simgrid.FatTreeParams(2, [2, 3], [1, 2], [1, 1]), simgrid.ClusterCallbacks(create_hostzone, None, create_limiter), 10e9,
+ 10e-6, simgrid.Link.SharingPolicy.SPLITDUPLEX).seal()
+
+#####################################################################################################
+
+
+def create_dragonfly_cluster():
+ """
+ Creates a Dragonfly cluster
+
+ Creates a Dragonfly cluster with 2 groups and 16 nodes
+ The following parameters are used to create this cluster:
+ - Groups: 2 groups, connected with 2 links (blue links)
+ - Chassis: 2 chassis, connected with a single link (black links)
+ - Routers: 2 routers, connected with 2 links (green links)
+ - Nodes: 2 leaves per router, single link
+
+ The diagram below illustrates a group in the dragonfly cluster
+
+ +------------------------------------------------+
+ | black link(1) |
+ | +------------------------+ |
+ | +---|--------------+ +---|--------------+ |
+ | | | green | | | green | |
+ | | | links (2) | | | links (2) | | blue links(2)
+ | | R1 ====== R2 | | R3 ----- R4 ======================> "Group 2"
+ | | / \ / \ | | / \ / \ | |
+ | | A B C D | | E F G H | |
+ | +------------------+ +------------------+ |
+ | Chassis 1 Chassis 2 |
+ +------------------------------------------------+
+ Group 1
+
+ Each element (A, B, C, etc) is a StarZone containing 8 Hosts.
+ The connection between elements (e.g. A->R1) uses 2 links:
+ 1) limiter: a 1Gbs limiter link (set by user through the set_limiter callback)
+ 2) link: 10Gbs link connecting the components (created automatically)
+
+ For example, a communication from A to C goes through:
+ <tt> A->limiter(A)->link(A-R1)->limiter(R1)->link(R1-R2)->limiter(R2)->link(R2-C)limiter(C)->C</tt>
+
+ More details in: <a href="https://simgrid.org/doc/latest/Platform_examples.html#dragonfly-cluster">Dragonfly
+ Cluster</a>
+ """
+ # create the dragonfly cluster, 10Gbs link between elements in the cluster
+ simgrid.NetZone.create_dragonfly_zone("cluster", None, simgrid.DragonflyParams([2, 2], [2, 1], [2, 2], 2), simgrid.ClusterCallbacks(
+ create_hostzone, None, create_limiter), 10e9, 10e-6, simgrid.Link.SharingPolicy.SPLITDUPLEX).seal()
+
+###################################################################################################
+
+
+if __name__ == '__main__':
+ e = simgrid.Engine(sys.argv)
+ platform = sys.argv[1]
+
+ # create platform
+ if platform == "torus":
+ create_torus_cluster()
+ elif platform == "fatTree":
+ create_fatTree_cluster()
+ elif platform == "dragonfly":
+ create_dragonfly_cluster()
+ else:
+ sys.exit("invalid param")
+
+ host_list = e.get_all_hosts()
+ # create the sender actor running on first host
+ simgrid.Actor.create("sender", host_list[0], Sender(host_list))
+ # create receiver in every host
+ for host in host_list:
+ simgrid.Actor.create("receiver-" + host.name, host, Receiver())
+
+ # runs the simulation
+ e.run()
--- /dev/null
+#!/usr/bin/env tesh
+
+$ ${pythoncmd:=python3} ${PYTHON_TOOL_OPTIONS:=} ${srcdir:=.}/clusters-multicpu.py torus "--log=root.fmt:[%10.6r]%e(%i:%a@%h)%e%m%n"
+> [ 0.000000] (1:sender@host0-cpu0) Done dispatching all messages
+> [ 0.000083] (9:receiver-host0-cpu7@host0-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (8:receiver-host0-cpu6@host0-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (7:receiver-host0-cpu5@host0-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (6:receiver-host0-cpu4@host0-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (5:receiver-host0-cpu3@host0-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (4:receiver-host0-cpu2@host0-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (3:receiver-host0-cpu1@host0-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (2:receiver-host0-cpu0@host0-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.045733] (34:receiver-host4-cpu0@host4-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.045733] (18:receiver-host2-cpu0@host2-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.045733] (10:receiver-host1-cpu0@host1-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (41:receiver-host4-cpu7@host4-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (40:receiver-host4-cpu6@host4-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (39:receiver-host4-cpu5@host4-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (38:receiver-host4-cpu4@host4-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (37:receiver-host4-cpu3@host4-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (36:receiver-host4-cpu2@host4-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (35:receiver-host4-cpu1@host4-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (25:receiver-host2-cpu7@host2-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (24:receiver-host2-cpu6@host2-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (23:receiver-host2-cpu5@host2-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (22:receiver-host2-cpu4@host2-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (21:receiver-host2-cpu3@host2-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (20:receiver-host2-cpu2@host2-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (19:receiver-host2-cpu1@host2-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (17:receiver-host1-cpu7@host1-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (16:receiver-host1-cpu6@host1-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (15:receiver-host1-cpu5@host1-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (14:receiver-host1-cpu4@host1-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (13:receiver-host1-cpu3@host1-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (12:receiver-host1-cpu2@host1-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.046063] (11:receiver-host1-cpu1@host1-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058223] (50:receiver-host6-cpu0@host6-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058223] (42:receiver-host5-cpu0@host5-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058223] (26:receiver-host3-cpu0@host3-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (57:receiver-host6-cpu7@host6-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (56:receiver-host6-cpu6@host6-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (55:receiver-host6-cpu5@host6-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (54:receiver-host6-cpu4@host6-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (53:receiver-host6-cpu3@host6-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (52:receiver-host6-cpu2@host6-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (51:receiver-host6-cpu1@host6-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (49:receiver-host5-cpu7@host5-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (48:receiver-host5-cpu6@host5-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (47:receiver-host5-cpu5@host5-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (46:receiver-host5-cpu4@host5-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (45:receiver-host5-cpu3@host5-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (44:receiver-host5-cpu2@host5-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (43:receiver-host5-cpu1@host5-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (33:receiver-host3-cpu7@host3-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (32:receiver-host3-cpu6@host3-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (31:receiver-host3-cpu5@host3-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (30:receiver-host3-cpu4@host3-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (29:receiver-host3-cpu3@host3-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (28:receiver-host3-cpu2@host3-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.058362] (27:receiver-host3-cpu1@host3-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.060722] (58:receiver-host7-cpu0@host7-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.060749] (65:receiver-host7-cpu7@host7-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.060749] (64:receiver-host7-cpu6@host7-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.060749] (63:receiver-host7-cpu5@host7-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.060749] (62:receiver-host7-cpu4@host7-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.060749] (61:receiver-host7-cpu3@host7-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.060749] (60:receiver-host7-cpu2@host7-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.060749] (59:receiver-host7-cpu1@host7-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.060749] (1:sender@host0-cpu0) Goodbye now!
+
+$ ${pythoncmd:=python3} ${PYTHON_TOOL_OPTIONS:=} ${srcdir:=.}/clusters-multicpu.py fatTree "--log=root.fmt:[%10.6r]%e(%i:%a@%h)%e%m%n"
+> [ 0.000000] (1:sender@host0-cpu0) Done dispatching all messages
+> [ 0.000083] (9:receiver-host0-cpu7@host0-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (8:receiver-host0-cpu6@host0-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (7:receiver-host0-cpu5@host0-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (6:receiver-host0-cpu4@host0-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (5:receiver-host0-cpu3@host0-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (4:receiver-host0-cpu2@host0-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (3:receiver-host0-cpu1@host0-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (2:receiver-host0-cpu0@host0-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.027921] (10:receiver-host1-cpu0@host1-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.028052] (17:receiver-host1-cpu7@host1-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.028052] (16:receiver-host1-cpu6@host1-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.028052] (15:receiver-host1-cpu5@host1-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.028052] (14:receiver-host1-cpu4@host1-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.028052] (13:receiver-host1-cpu3@host1-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.028052] (12:receiver-host1-cpu2@host1-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.028052] (11:receiver-host1-cpu1@host1-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043476] (42:receiver-host5-cpu0@host5-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043476] (34:receiver-host4-cpu0@host4-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043476] (26:receiver-host3-cpu0@host3-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043476] (18:receiver-host2-cpu0@host2-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (49:receiver-host5-cpu7@host5-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (48:receiver-host5-cpu6@host5-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (47:receiver-host5-cpu5@host5-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (46:receiver-host5-cpu4@host5-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (45:receiver-host5-cpu3@host5-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (44:receiver-host5-cpu2@host5-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (43:receiver-host5-cpu1@host5-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (41:receiver-host4-cpu7@host4-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (40:receiver-host4-cpu6@host4-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (39:receiver-host4-cpu5@host4-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (38:receiver-host4-cpu4@host4-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (37:receiver-host4-cpu3@host4-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (36:receiver-host4-cpu2@host4-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (35:receiver-host4-cpu1@host4-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (33:receiver-host3-cpu7@host3-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (32:receiver-host3-cpu6@host3-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (31:receiver-host3-cpu5@host3-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (30:receiver-host3-cpu4@host3-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (29:receiver-host3-cpu3@host3-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (28:receiver-host3-cpu2@host3-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (27:receiver-host3-cpu1@host3-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (25:receiver-host2-cpu7@host2-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (24:receiver-host2-cpu6@host2-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (23:receiver-host2-cpu5@host2-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (22:receiver-host2-cpu4@host2-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (21:receiver-host2-cpu3@host2-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (20:receiver-host2-cpu2@host2-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (19:receiver-host2-cpu1@host2-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.043559] (1:sender@host0-cpu0) Goodbye now!
+
+$ ${pythoncmd:=python3} ${PYTHON_TOOL_OPTIONS:=} ${srcdir:=.}/clusters-multicpu.py dragonfly "--log=root.fmt:[%10.6r]%e(%i:%a@%h)%e%m%n"
+> [ 0.000000] (1:sender@host0-cpu0) Done dispatching all messages
+> [ 0.000083] (9:receiver-host0-cpu7@host0-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (8:receiver-host0-cpu6@host0-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (7:receiver-host0-cpu5@host0-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (6:receiver-host0-cpu4@host0-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (5:receiver-host0-cpu3@host0-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (4:receiver-host0-cpu2@host0-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (3:receiver-host0-cpu1@host0-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.000083] (2:receiver-host0-cpu0@host0-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.075914] (10:receiver-host1-cpu0@host1-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.076281] (17:receiver-host1-cpu7@host1-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.076281] (16:receiver-host1-cpu6@host1-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.076281] (15:receiver-host1-cpu5@host1-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.076281] (14:receiver-host1-cpu4@host1-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.076281] (13:receiver-host1-cpu3@host1-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.076281] (12:receiver-host1-cpu2@host1-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.076281] (11:receiver-host1-cpu1@host1-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102132] (74:receiver-host3-cpu0@host3-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102132] (66:receiver-host2-cpu0@host2-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (81:receiver-host3-cpu7@host3-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (80:receiver-host3-cpu6@host3-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (79:receiver-host3-cpu5@host3-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (78:receiver-host3-cpu4@host3-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (77:receiver-host3-cpu3@host3-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (76:receiver-host3-cpu2@host3-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (75:receiver-host3-cpu1@host3-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (73:receiver-host2-cpu7@host2-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (72:receiver-host2-cpu6@host2-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (71:receiver-host2-cpu5@host2-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (70:receiver-host2-cpu4@host2-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (69:receiver-host2-cpu3@host2-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (68:receiver-host2-cpu2@host2-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102454] (67:receiver-host2-cpu1@host2-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102853] (90:receiver-host5-cpu0@host5-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.102853] (82:receiver-host4-cpu0@host4-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (97:receiver-host5-cpu7@host5-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (96:receiver-host5-cpu6@host5-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (95:receiver-host5-cpu5@host5-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (94:receiver-host5-cpu4@host5-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (93:receiver-host5-cpu3@host5-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (92:receiver-host5-cpu2@host5-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (91:receiver-host5-cpu1@host5-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (89:receiver-host4-cpu7@host4-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (88:receiver-host4-cpu6@host4-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (87:receiver-host4-cpu5@host4-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (86:receiver-host4-cpu4@host4-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (85:receiver-host4-cpu3@host4-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (84:receiver-host4-cpu2@host4-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.103113] (83:receiver-host4-cpu1@host4-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118451] (122:receiver-host9-cpu0@host9-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118451] (114:receiver-host8-cpu0@host8-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (129:receiver-host9-cpu7@host9-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (128:receiver-host9-cpu6@host9-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (127:receiver-host9-cpu5@host9-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (126:receiver-host9-cpu4@host9-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (125:receiver-host9-cpu3@host9-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (124:receiver-host9-cpu2@host9-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (123:receiver-host9-cpu1@host9-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (121:receiver-host8-cpu7@host8-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (120:receiver-host8-cpu6@host8-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (119:receiver-host8-cpu5@host8-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (118:receiver-host8-cpu4@host8-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (117:receiver-host8-cpu3@host8-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (116:receiver-host8-cpu2@host8-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118652] (115:receiver-host8-cpu1@host8-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118890] (106:receiver-host7-cpu0@host7-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.118890] (98:receiver-host6-cpu0@host6-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (113:receiver-host7-cpu7@host7-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (112:receiver-host7-cpu6@host7-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (111:receiver-host7-cpu5@host7-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (110:receiver-host7-cpu4@host7-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (109:receiver-host7-cpu3@host7-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (108:receiver-host7-cpu2@host7-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (107:receiver-host7-cpu1@host7-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (105:receiver-host6-cpu7@host6-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (104:receiver-host6-cpu6@host6-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (103:receiver-host6-cpu5@host6-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (102:receiver-host6-cpu4@host6-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (101:receiver-host6-cpu3@host6-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (100:receiver-host6-cpu2@host6-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.119044] (99:receiver-host6-cpu1@host6-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127302] (26:receiver-host11-cpu0@host11-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127302] (18:receiver-host10-cpu0@host10-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (33:receiver-host11-cpu7@host11-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (32:receiver-host11-cpu6@host11-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (31:receiver-host11-cpu5@host11-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (30:receiver-host11-cpu4@host11-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (29:receiver-host11-cpu3@host11-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (28:receiver-host11-cpu2@host11-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (27:receiver-host11-cpu1@host11-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (25:receiver-host10-cpu7@host10-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (24:receiver-host10-cpu6@host10-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (23:receiver-host10-cpu5@host10-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (22:receiver-host10-cpu4@host10-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (21:receiver-host10-cpu3@host10-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (20:receiver-host10-cpu2@host10-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127410] (19:receiver-host10-cpu1@host10-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127518] (42:receiver-host13-cpu0@host13-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127518] (34:receiver-host12-cpu0@host12-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (49:receiver-host13-cpu7@host13-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (48:receiver-host13-cpu6@host13-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (47:receiver-host13-cpu5@host13-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (46:receiver-host13-cpu4@host13-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (45:receiver-host13-cpu3@host13-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (44:receiver-host13-cpu2@host13-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (43:receiver-host13-cpu1@host13-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (41:receiver-host12-cpu7@host12-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (40:receiver-host12-cpu6@host12-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (39:receiver-host12-cpu5@host12-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (38:receiver-host12-cpu4@host12-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (37:receiver-host12-cpu3@host12-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (36:receiver-host12-cpu2@host12-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.127586] (35:receiver-host12-cpu1@host12-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130128] (58:receiver-host15-cpu0@host15-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130128] (50:receiver-host14-cpu0@host14-cpu0) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (65:receiver-host15-cpu7@host15-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (64:receiver-host15-cpu6@host15-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (63:receiver-host15-cpu5@host15-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (62:receiver-host15-cpu4@host15-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (61:receiver-host15-cpu3@host15-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (60:receiver-host15-cpu2@host15-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (59:receiver-host15-cpu1@host15-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (57:receiver-host14-cpu7@host14-cpu7) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (56:receiver-host14-cpu6@host14-cpu6) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (55:receiver-host14-cpu5@host14-cpu5) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (54:receiver-host14-cpu4@host14-cpu4) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (53:receiver-host14-cpu3@host14-cpu3) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (52:receiver-host14-cpu2@host14-cpu2) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (51:receiver-host14-cpu1@host14-cpu1) I got a 'Hello, I'm alive and running on host0-cpu0'.
+> [ 0.130157] (1:sender@host0-cpu0) Goodbye now!
+