From 54ccc949168525ddd88da629da90acd32b178cc0 Mon Sep 17 00:00:00 2001 From: Adrien Gougeon Date: Tue, 30 May 2023 12:57:58 +0200 Subject: [PATCH] add python bindings for operations --- MANIFEST.in | 8 ++ examples/python/CMakeLists.txt | 1 + examples/python/operation-io/operation-io.py | 51 +++++++++++ .../python/operation-io/operation-io.tesh | 12 +++ .../operation-simple/operation-simple.py | 60 +++++++++++++ .../operation-simple/operation-simple.tesh | 9 ++ .../operation-switch-host.py | 90 +++++++++++++++++++ .../operation-switch-host.tesh | 15 ++++ .../operation-variable-load.py | 68 ++++++++++++++ .../operation-variable-load.tesh | 18 ++++ include/simgrid/plugins/operation.hpp | 1 + src/bindings/python/simgrid_python.cpp | 86 ++++++++++++++++++ 12 files changed, 419 insertions(+) create mode 100644 examples/python/operation-io/operation-io.py create mode 100644 examples/python/operation-io/operation-io.tesh create mode 100644 examples/python/operation-simple/operation-simple.py create mode 100644 examples/python/operation-simple/operation-simple.tesh create mode 100644 examples/python/operation-switch-host/operation-switch-host.py create mode 100644 examples/python/operation-switch-host/operation-switch-host.tesh create mode 100644 examples/python/operation-variable-load/operation-variable-load.py create mode 100644 examples/python/operation-variable-load/operation-variable-load.tesh diff --git a/MANIFEST.in b/MANIFEST.in index 8a1999bfac..a43cae6266 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -474,6 +474,14 @@ include examples/python/io-degradation/io-degradation.py include examples/python/io-degradation/io-degradation.tesh include examples/python/network-nonlinear/network-nonlinear.py include examples/python/network-nonlinear/network-nonlinear.tesh +include examples/python/operation-io/operation-io.py +include examples/python/operation-io/operation-io.tesh +include examples/python/operation-simple/operation-simple.py +include examples/python/operation-simple/operation-simple.tesh +include examples/python/operation-switch-host/operation-switch-host.py +include examples/python/operation-switch-host/operation-switch-host.tesh +include examples/python/operation-variable-load/operation-variable-load.py +include examples/python/operation-variable-load/operation-variable-load.tesh include examples/python/platform-comm-serialize/platform-comm-serialize.py include examples/python/platform-comm-serialize/platform-comm-serialize.tesh include examples/python/platform-failures/platform-failures.py diff --git a/examples/python/CMakeLists.txt b/examples/python/CMakeLists.txt index d2bb58a5dc..7d6b742d22 100644 --- a/examples/python/CMakeLists.txt +++ b/examples/python/CMakeLists.txt @@ -3,6 +3,7 @@ foreach(example actor-create actor-daemon actor-join actor-kill actor-migrate ac comm-wait comm-waitall comm-waitallfor comm-waitany comm-failure comm-host2host comm-pingpong comm-ready comm-suspend comm-testany comm-throttling comm-waitallfor comm-waituntil exec-async exec-basic exec-dvfs exec-remote exec-ptask + operation-io operation-simple operation-switch-host operation-variable-load platform-comm-serialize platform-profile platform-failures network-nonlinear clusters-multicpu io-degradation exec-cpu-nonlinear synchro-barrier synchro-mutex synchro-semaphore) diff --git a/examples/python/operation-io/operation-io.py b/examples/python/operation-io/operation-io.py new file mode 100644 index 0000000000..59a07afaad --- /dev/null +++ b/examples/python/operation-io/operation-io.py @@ -0,0 +1,51 @@ +# Copyright (c) 2006-2023. The SimGrid Team. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the license (GNU LGPL) which comes with this package. + +from argparse import ArgumentParser +import sys +from simgrid import Engine, Operation, ExecOp, IoOp, IoOpType + +def parse(): + parser = ArgumentParser() + parser.add_argument( + '--platform', + type=str, + required=True, + help='path to the platform description' + ) + return parser.parse_args() + +def callback(op): + print(f'[{Engine.clock}] Operation {op} finished ({op.count})') + +if __name__ == '__main__': + args = parse() + e = Engine(sys.argv) + e.load_platform(args.platform) + Operation.init() + + # Retrieve hosts + bob = e.host_by_name('bob') + carl = e.host_by_name('carl') + + # Create operations + exec1 = ExecOp.init("exec1", 1e9, bob) + exec2 = ExecOp.init("exec2", 1e9, carl) + write = IoOp.init("write", 1e7, bob.disks[0], IoOpType.WRITE) + read = IoOp.init("read", 1e7, carl.disks[0], IoOpType.READ) + + # Create the graph by defining dependencies between operations + exec1.add_successor(write) + write.add_successor(read) + read.add_successor(exec2) + + # Add a function to be called when operations end for log purpose + Operation.on_end_cb(callback) + + # Enqueue two executions for operation exec1 + exec1.enqueue_execs(2) + + # runs the simulation + e.run() \ No newline at end of file diff --git a/examples/python/operation-io/operation-io.tesh b/examples/python/operation-io/operation-io.tesh new file mode 100644 index 0000000000..d7963639ff --- /dev/null +++ b/examples/python/operation-io/operation-io.tesh @@ -0,0 +1,12 @@ +#!/usr/bin/env tesh + +$ ${pythoncmd:=python3} ${PYTHON_TOOL_OPTIONS:=} ${srcdir:=.}/operation-io.py --platform ${platfdir}/hosts_with_disks.xml +> [1.0] Operation exec1 finished (1) +> [1.25] Operation write finished (1) +> [1.35] Operation read finished (1) +> [2.0] Operation exec1 finished (2) +> [2.25] Operation write finished (2) +> [2.35] Operation exec2 finished (1) +> [2.35] Operation read finished (2) +> [3.35] Operation exec2 finished (2) + diff --git a/examples/python/operation-simple/operation-simple.py b/examples/python/operation-simple/operation-simple.py new file mode 100644 index 0000000000..fcd36ecc9b --- /dev/null +++ b/examples/python/operation-simple/operation-simple.py @@ -0,0 +1,60 @@ +# Copyright (c) 2006-2023. The SimGrid Team. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the license (GNU LGPL) which comes with this package. + +""" +This example demonstrates basic use of the operation plugin. +We model the following graph: + +exec1 -> comm -> exec2 + +exec1 and exec2 are execution operations. +comm is a communication operation. +""" + +from argparse import ArgumentParser +import sys +from simgrid import Engine, Operation, CommOp, ExecOp + +def parse(): + parser = ArgumentParser() + parser.add_argument( + '--platform', + type=str, + required=True, + help='path to the platform description' + ) + return parser.parse_args() + +def callback(op): + print(f'[{Engine.clock}] Operation {op} finished ({op.count})') + +if __name__ == '__main__': + args = parse() + e = Engine(sys.argv) + e.load_platform(args.platform) + Operation.init() + + # Retrieve hosts + tremblay = e.host_by_name('Tremblay') + jupiter = e.host_by_name('Jupiter') + + # Create operations + exec1 = ExecOp.init("exec1", 1e9, tremblay) + exec2 = ExecOp.init("exec2", 1e9, jupiter) + comm = CommOp.init("comm", 1e7, tremblay, jupiter) + + # Create the graph by defining dependencies between operations + exec1.add_successor(comm) + comm.add_successor(exec2) + + # Add a function to be called when operations end for log purpose + Operation.on_end_cb(callback) + + # Enqueue two executions for operation exec1 + exec1.enqueue_execs(2) + + # runs the simulation + e.run() + diff --git a/examples/python/operation-simple/operation-simple.tesh b/examples/python/operation-simple/operation-simple.tesh new file mode 100644 index 0000000000..7cb04993bc --- /dev/null +++ b/examples/python/operation-simple/operation-simple.tesh @@ -0,0 +1,9 @@ +#!/usr/bin/env tesh + +$ ${pythoncmd:=python3} ${PYTHON_TOOL_OPTIONS:=} ${srcdir:=.}/operation-simple.py --platform ${platfdir}/small_platform.xml +> [10.194199500484224] Operation exec1 finished (1) +> [11.714617112501687] Operation comm finished (1) +> [20.388399000968448] Operation exec1 finished (2) +> [21.90881661298591] Operation comm finished (2) +> [24.82146412938331] Operation exec2 finished (1) +> [37.92831114626493] Operation exec2 finished (2) diff --git a/examples/python/operation-switch-host/operation-switch-host.py b/examples/python/operation-switch-host/operation-switch-host.py new file mode 100644 index 0000000000..90a1fae78b --- /dev/null +++ b/examples/python/operation-switch-host/operation-switch-host.py @@ -0,0 +1,90 @@ +# Copyright (c) 2006-2023. The SimGrid Team. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the license (GNU LGPL) which comes with this package. + +""" +/* This example demonstrates how to dynamically modify a graph of operations. + * + * Assuming we have two instances of a service placed on different hosts, + * we want to send data alternatively to thoses instances. + * + * We consider the following graph: + + comm1 + ┌────────────────────────┐ + │ │ + │ Fafard │ + │ ┌───────┐ │ + │ ┌──────►│ exec1 ├─┘ + ▼ │ └───────┘ + Tremblay ──┤comm0 + ▲ │ Jupiter + │ │ ┌───────┐ + │ └──────►│ exec2 ├─┐ + │ └───────┘ │ + │ │ + └────────────────────────┘ + comm2 + */ + """ + +from argparse import ArgumentParser +import sys +from simgrid import Engine, Operation, CommOp, ExecOp + +def parse(): + parser = ArgumentParser() + parser.add_argument( + '--platform', + type=str, + required=True, + help='path to the platform description' + ) + return parser.parse_args() + +def callback(op): + print(f'[{Engine.clock}] Operation {op} finished ({op.count})') + +def switch(op, hosts, execs): + comm0.destination = hosts[op.count % 2] + comm0.remove_successor(execs[op.count % 2 - 1]) + comm0.add_successor(execs[op.count % 2]) + +if __name__ == '__main__': + args = parse() + e = Engine(sys.argv) + e.load_platform(args.platform) + Operation.init() + + # Retrieve hosts + tremblay = e.host_by_name('Tremblay') + jupiter = e.host_by_name('Jupiter') + fafard = e.host_by_name('Fafard') + + # Create operations + comm0 = CommOp.init("comm0") + comm0.bytes = 1e7 + comm0.source = tremblay + exec1 = ExecOp.init("exec1", 1e9, jupiter) + exec2 = ExecOp.init("exec2", 1e9, fafard) + comm1 = CommOp.init("comm1", 1e7, jupiter, tremblay) + comm2 = CommOp.init("comm2", 1e7, fafard, tremblay) + + # Create the initial graph by defining dependencies between operations + exec1.add_successor(comm1) + exec2.add_successor(comm2) + + # Add a function to be called when operations end for log purpose + Operation.on_end_cb(callback) + + # Add a function to be called before each executions of comm0 + # This function modifies the graph of operations by adding or removing + # successors to comm0 + comm0.on_this_start(lambda op: switch(op, [jupiter, fafard], [exec1,exec2])) + + # Enqueue two executions for operation exec1 + comm0.enqueue_execs(4) + + # runs the simulation + e.run() diff --git a/examples/python/operation-switch-host/operation-switch-host.tesh b/examples/python/operation-switch-host/operation-switch-host.tesh new file mode 100644 index 0000000000..a74c06d5bb --- /dev/null +++ b/examples/python/operation-switch-host/operation-switch-host.tesh @@ -0,0 +1,15 @@ +#!/usr/bin/env tesh + +$ ${pythoncmd:=python3} ${PYTHON_TOOL_OPTIONS:=} ${srcdir:=.}/operation-switch-host.py --platform ${platfdir}/small_platform.xml +> [1.5204176120174615] Operation comm0 finished (1) +> [2.873012467069035] Operation comm0 finished (2) +> [4.393430079086497] Operation comm0 finished (3) +> [5.74602493413807] Operation comm0 finished (4) +> [14.62726462889908] Operation exec1 finished (1) +> [15.979859483950655] Operation exec2 finished (1) +> [16.14768224091654] Operation comm1 finished (1) +> [17.33245433900223] Operation comm2 finished (1) +> [27.7341116457807] Operation exec1 finished (2) +> [29.086706500832275] Operation exec2 finished (2) +> [29.25452925779816] Operation comm1 finished (2) +> [30.43930135588385] Operation comm2 finished (2) \ No newline at end of file diff --git a/examples/python/operation-variable-load/operation-variable-load.py b/examples/python/operation-variable-load/operation-variable-load.py new file mode 100644 index 0000000000..800da204bc --- /dev/null +++ b/examples/python/operation-variable-load/operation-variable-load.py @@ -0,0 +1,68 @@ +# Copyright (c) 2006-2023. The SimGrid Team. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the license (GNU LGPL) which comes with this package. + +""" +This example demonstrates how to create a variable load for operations. +We consider the following graph: + +comm -> exec + +With a small load each comm operation is followed by an exec operation. +With a heavy load there is a burst of comm before the exec operation can even finish once. +""" + +from argparse import ArgumentParser +import sys +from simgrid import Engine, Operation, CommOp, ExecOp, Actor, this_actor + +def parse(): + parser = ArgumentParser() + parser.add_argument( + '--platform', + type=str, + required=True, + help='path to the platform description' + ) + return parser.parse_args() + +def callback(op): + print(f'[{Engine.clock}] Operation {op} finished ({op.count})') + +def variable_load(op): + print('--- Small load ---') + for i in range(3): + op.enqueue_execs(1) + this_actor.sleep_for(100) + this_actor.sleep_for(1000) + print('--- Heavy load ---') + for i in range(3): + op.enqueue_execs(1) + this_actor.sleep_for(1) + +if __name__ == '__main__': + args = parse() + e = Engine(sys.argv) + e.load_platform(args.platform) + Operation.init() + + # Retrieve hosts + tremblay = e.host_by_name('Tremblay') + jupiter = e.host_by_name('Jupiter') + + # Create operations + comm = CommOp.init("comm", 1e7, tremblay, jupiter) + exec = ExecOp.init("exec", 1e9, jupiter) + + # Create the graph by defining dependencies between operations + comm.add_successor(exec) + + # Add a function to be called when operations end for log purpose + Operation.on_end_cb(callback) + + # Create the actor that will inject load during the simulation + Actor.create("input", tremblay, variable_load, comm) + + # runs the simulation + e.run() diff --git a/examples/python/operation-variable-load/operation-variable-load.tesh b/examples/python/operation-variable-load/operation-variable-load.tesh new file mode 100644 index 0000000000..4420174dc1 --- /dev/null +++ b/examples/python/operation-variable-load/operation-variable-load.tesh @@ -0,0 +1,18 @@ +#!/usr/bin/env tesh + +$ ${pythoncmd:=python3} ${PYTHON_TOOL_OPTIONS:=} ${srcdir:=.}/operation-variable-load.py --platform ${platfdir}/small_platform.xml +> --- Small load --- +> [1.5204176120174615] Operation comm finished (1) +> [14.62726462889908] Operation exec finished (1) +> [101.52041761201747] Operation comm finished (2) +> [114.62726462889908] Operation exec finished (2) +> [201.52041761201744] Operation comm finished (3) +> [214.62726462889907] Operation exec finished (3) +> --- Heavy load --- +> [1301.5204176120174] Operation comm finished (4) +> [1303.0408352240347] Operation comm finished (5) +> [1304.561252836052] Operation comm finished (6) +> [1314.627264628899] Operation exec finished (4) +> [1327.7341116457806] Operation exec finished (5) +> [1340.8409586626622] Operation exec finished (6) + diff --git a/include/simgrid/plugins/operation.hpp b/include/simgrid/plugins/operation.hpp index ee4b606af2..537d95db6c 100644 --- a/include/simgrid/plugins/operation.hpp +++ b/include/simgrid/plugins/operation.hpp @@ -149,6 +149,7 @@ public: IoOpPtr set_bytes(double bytes); double get_bytes() { return get_amount(); } IoOpPtr set_op_type(s4u::Io::OpType type); + s4u::Io::OpType get_op_type() { return type_; } friend void inline intrusive_ptr_release(IoOp* i) { intrusive_ptr_release(static_cast(i)); } friend void inline intrusive_ptr_add_ref(IoOp* i) { intrusive_ptr_add_ref(static_cast(i)); } diff --git a/src/bindings/python/simgrid_python.cpp b/src/bindings/python/simgrid_python.cpp index ef00ea49ca..db4ec584a5 100644 --- a/src/bindings/python/simgrid_python.cpp +++ b/src/bindings/python/simgrid_python.cpp @@ -11,6 +11,7 @@ #include "simgrid/kernel/ProfileBuilder.hpp" #include "simgrid/kernel/routing/NetPoint.hpp" #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -31,14 +33,24 @@ #include namespace py = pybind11; +using simgrid::plugins::Operation; +using simgrid::plugins::OperationPtr; +using simgrid::plugins::CommOp; +using simgrid::plugins::CommOpPtr; +using simgrid::plugins::ExecOp; +using simgrid::plugins::ExecOpPtr; +using simgrid::plugins::IoOp; +using simgrid::plugins::IoOpPtr; using simgrid::s4u::Actor; using simgrid::s4u::ActorPtr; using simgrid::s4u::Barrier; using simgrid::s4u::BarrierPtr; using simgrid::s4u::Comm; using simgrid::s4u::CommPtr; +using simgrid::s4u::Disk; using simgrid::s4u::Engine; using simgrid::s4u::Host; +using simgrid::s4u::Io; using simgrid::s4u::Link; using simgrid::s4u::Mailbox; using simgrid::s4u::Mutex; @@ -403,6 +415,7 @@ PYBIND11_MODULE(simgrid, m) return self.attr("netpoint"); }) .def_property_readonly("netpoint", &Host::get_netpoint, "Retrieve the netpoint associated to this zone") + .def_property_readonly("disks", &Host::get_disks, "The list of disks on this host (read-only).") .def("get_disks", &Host::get_disks, "Retrieve the list of disks in this host") .def("set_core_count", [](py::object self, double count) // XBT_ATTRIB_DEPRECATED_v334 @@ -887,4 +900,77 @@ PYBIND11_MODULE(simgrid, m) "Resume that actor, that was previously suspend()ed.") .def_static("kill_all", &Actor::kill_all, py::call_guard(), "Kill all actors but the caller."); + + /* Enum Class IoOpType */ + py::enum_(m, "IoOpType") + .value("READ", simgrid::s4u::Io::OpType::READ) + .value("WRITE", simgrid::s4u::Io::OpType::WRITE); + + /* Class Operation */ + py::class_(m, "Operation", + "Operation. See the C++ documentation for details.") + .def_static("init", &Operation::init) + .def_static("on_start_cb", [](py::object cb) { + cb.inc_ref(); // keep alive after return + const py::gil_scoped_release gil_release; + Operation::on_start_cb([cb](Operation* op) { + const py::gil_scoped_acquire py_context; // need a new context for callback + py::reinterpret_borrow(cb.ptr())(op); + }); + }, + "Add a callback called when each operation starts.") + .def_static("on_end_cb", [](py::object cb) { + cb.inc_ref(); // keep alive after return + const py::gil_scoped_release gil_release; + Operation::on_end_cb([cb](Operation* op) { + const py::gil_scoped_acquire py_context; // need a new context for callback + py::reinterpret_borrow(cb.ptr())(op); + }); + }, + "Add a callback called when each operation ends.") + .def_property_readonly("name", &Operation::get_name, "The name of this operation (read-only).") + .def_property_readonly("count", &Operation::get_count, "The execution count of this operation (read-only).") + .def_property_readonly("successors", &Operation::get_successors, "The successors of this operation (read-only).") + .def_property("amount", &Operation::get_amount, &Operation::set_amount, "The amount of work to do for this operation.") + .def("enqueue_execs", py::overload_cast(&Operation::enqueue_execs), py::call_guard(), py::arg("n"), "Enqueue executions for this operation.") + .def("add_successor", py::overload_cast(&Operation::add_successor), py::call_guard(), py::arg("op"), "Add a successor to this operation.") + .def("remove_successor", py::overload_cast(&Operation::remove_successor), py::call_guard(), py::arg("op"), "Remove a successor of this operation.") + .def("remove_all_successors", &Operation::remove_all_successors, py::call_guard(), "Remove all successors of this operation.") + .def("on_this_start", py::overload_cast&>(&Operation::on_this_start), py::arg("func"), "Add a callback called when this operation starts.") + .def("on_this_end", py::overload_cast&>(&Operation::on_this_end), py::arg("func"), "Add a callback called when this operation ends.") + .def("__repr__", [](const OperationPtr op) { + return op->get_name(); + }); + + /* Class CommOp */ + py::class_(m, "CommOp", + "Communication Operation. See the C++ documentation for details.") + .def_static("init", py::overload_cast(&CommOp::init), py::call_guard(), + py::arg("name"), "CommOp constructor") + .def_static("init", py::overload_cast(&CommOp::init), py::call_guard(), + py::arg("name"), py::arg("bytes"), py::arg("source"), py::arg("destination"), "CommOp constructor") + .def_property("source", &CommOp::get_source, &CommOp::set_source, "The source of the communication.") + .def_property("destination", &CommOp::get_destination, &CommOp::set_destination, "The destination of the communication.") + .def_property("bytes", &CommOp::get_bytes, &CommOp::set_bytes, "The amount of bytes to send."); + + /* Class ExecOp */ + py::class_(m, "ExecOp", + "Execution Operation. See the C++ documentation for details.") + .def_static("init", py::overload_cast(&ExecOp::init), py::call_guard(), + py::arg("name"), "ExecOp constructor") + .def_static("init", py::overload_cast(&ExecOp::init), py::call_guard(), + py::arg("name"), py::arg("flops"), py::arg("host"), "CommOp constructor.") + .def_property("host", &ExecOp::get_host, &ExecOp::set_host, "The host of the execution.") + .def_property("flops", &ExecOp::get_flops, &ExecOp::set_flops, "The amount of flops to execute."); + + /* Class IoOp */ + py::class_(m, "IoOp", + "IO Operation. See the C++ documentation for details.") + .def_static("init", py::overload_cast(&IoOp::init), py::call_guard(), + py::arg("name"), "IoOp constructor") + .def_static("init", py::overload_cast(&IoOp::init), py::call_guard(), + py::arg("name"), py::arg("bytes"), py::arg("disk"), py::arg("type"), "IoOp constructor.") + .def_property("disk", &IoOp::get_disk, &IoOp::set_disk, "The disk of the IO.") + .def_property("bytes", &IoOp::get_bytes, &IoOp::set_bytes, "The amount of bytes to process.") + .def_property("type", &IoOp::get_bytes, &IoOp::set_bytes, "The type of IO."); } -- 2.20.1