X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/e39ef0ce987f5528306f439591bb028fe85ebd4e..3d07c492fcabdf883adce049d7ce605709a4fcb9:/src/simdag/sd_daxloader.cpp?ds=sidebyside diff --git a/src/simdag/sd_daxloader.cpp b/src/simdag/sd_daxloader.cpp index 5872efa65c..d471fe25e0 100644 --- a/src/simdag/sd_daxloader.cpp +++ b/src/simdag/sd_daxloader.cpp @@ -1,160 +1,106 @@ -/* Copyright (c) 2009-2019. The SimGrid Team. +/* Copyright (c) 2009-2021. The SimGrid Team. * All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ -#include "simdag_private.hpp" -#include "simgrid/simdag.h" -#include "xbt/file.hpp" -#include "xbt/log.h" -#include "xbt/misc.h" +#include "src/internal_config.h" #include #include +#include +#include +#include #include +#include +#include +#include +#include #include "dax_dtd.h" #include "dax_dtd.c" -XBT_LOG_NEW_DEFAULT_SUBCATEGORY(sd_daxparse, sd, "Parsing DAX files"); +XBT_LOG_NEW_DEFAULT_CATEGORY(dag_parsing, "Generation DAGs from files"); + +#if HAVE_GRAPHVIZ +#include +#endif /* Ensure that transfer tasks have unique names even though a file is used several times */ -void uniq_transfer_task_name(SD_task_t task) +static void uniq_transfer_task_name(simgrid::s4u::Comm* comm) { - SD_task_t child = *(task->successors->begin()); - SD_task_t parent = *(task->predecessors->begin()); - - std::string new_name = - std::string(SD_task_get_name(parent)) + "_" + SD_task_get_name(task) + "_" + SD_task_get_name(child); + const auto& child = comm->get_successors().front(); + const auto& parent = *(comm->get_dependencies().begin()); - SD_task_set_name(task, new_name.c_str()); -} + std::string new_name = parent->get_name() + "_" + comm->get_name() + "_" + child->get_name(); -static bool children_are_marked(SD_task_t task){ - return std::none_of(task->successors->begin(), task->successors->end(), - [](const SD_task_t& elm) { return not elm->marked; }) && - std::none_of(task->outputs->begin(), task->outputs->end(), - [](const SD_task_t& elm) { return not elm->marked; }); + comm->set_name(new_name)->vetoable_start(); } -static bool parents_are_marked(SD_task_t task){ - return std::none_of(task->predecessors->begin(), task->predecessors->end(), - [](const SD_task_t& elm) { return not elm->marked; }) && - std::none_of(task->inputs->begin(), task->inputs->end(), [](const SD_task_t& elm) { return not elm->marked; }); -} +static bool check_for_cycle(const std::vector& dag) +{ + std::vector current; -bool acyclic_graph_detail(xbt_dynar_t dag){ - unsigned int count; - bool all_marked = true; - SD_task_t task = nullptr; - std::vector current; - xbt_dynar_foreach (dag, count, task) - if (task->kind != SD_TASK_COMM_E2E && task->successors->empty() && task->outputs->empty()) - current.push_back(task); + for (const auto& a : dag) + if (dynamic_cast(a.get()) != nullptr && a->is_waited_by() == 0) + current.push_back(a); while (not current.empty()) { - std::vector next; - for (auto const& t : current) { - //Mark task - t->marked = true; - for (SD_task_t const& input : *t->inputs) { - input->marked = true; - // Inputs are communication, hence they can have only one predecessor - SD_task_t input_pred = *(input->predecessors->begin()); - if (children_are_marked(input_pred)) - next.push_back(input_pred); - } - for (SD_task_t const& pred : *t->predecessors) { - if (children_are_marked(pred)) - next.push_back(pred); + std::vector next; + for (auto const& a : current) { + a->mark(); + for (auto const& pred : a->get_dependencies()) { + if (dynamic_cast(pred.get()) != nullptr) { + pred->mark(); + // Comms have only one predecessor + auto pred_pred = *(pred->get_dependencies().begin()); + if (std::none_of(pred_pred->get_successors().begin(), pred_pred->get_successors().end(), + [](const simgrid::s4u::ActivityPtr& act) { return not act->is_marked(); })) + next.push_back(pred_pred); + } else { + if (std::none_of(pred->get_successors().begin(), pred->get_successors().end(), + [](const simgrid::s4u::ActivityPtr& act) { return not act->is_marked(); })) + next.push_back(pred); + } } } current.clear(); current = next; } - all_marked = true; - //test if all tasks are marked - xbt_dynar_foreach(dag,count,task){ - if (task->kind != SD_TASK_COMM_E2E && not task->marked) { - XBT_WARN("the task %s is not marked",task->name); - all_marked = false; - break; - } - } - - if (not all_marked) { - XBT_VERB("there is at least one cycle in your task graph"); - xbt_dynar_foreach(dag,count,task){ - if(task->kind != SD_TASK_COMM_E2E && task->predecessors->empty() && task->inputs->empty()){ - task->marked = true; - current.push_back(task); - } - } - //test if something has to be done for the next iteration - while (not current.empty()) { - std::vector next; - //test if the current iteration is done - for (auto const& t : current) { - t->marked = true; - for (SD_task_t const& output : *t->outputs) { - output->marked = true; - // outputs are communication, hence they can have only one successor - SD_task_t output_succ = *(output->successors->begin()); - if (parents_are_marked(output_succ)) - next.push_back(output_succ); - } - for (SD_task_t const& succ : *t->successors) { - if (parents_are_marked(succ)) - next.push_back(succ); - } - } - current.clear(); - current = next; - } - - all_marked = true; - xbt_dynar_foreach(dag,count,task){ - if (task->kind != SD_TASK_COMM_E2E && not task->marked) { - XBT_WARN("the task %s is in a cycle",task->name); - all_marked = false; - } - } - } - return all_marked; + return not std::any_of(dag.begin(), dag.end(), [](const simgrid::s4u::ActivityPtr& a) { return not a->is_marked(); }); } static YY_BUFFER_STATE input_buffer; -static xbt_dynar_t result; -static std::map jobs; -static std::map files; -static SD_task_t current_job; +namespace simgrid { +namespace s4u { + +static std::vector result; +static std::map> jobs; +static std::map> files; +static ExecPtr current_job; /** @brief loads a DAX file describing a DAG * * See https://confluence.pegasus.isi.edu/display/pegasus/WorkflowGenerator for more details. */ -xbt_dynar_t SD_daxload(const char *filename) +std::vector create_DAG_from_DAX(const std::string& filename) { - SD_task_t file; - FILE* in_file = fopen(filename, "r"); - xbt_assert(in_file, "Unable to open \"%s\"\n", filename); + FILE* in_file = fopen(filename.c_str(), "r"); + xbt_assert(in_file, "Unable to open \"%s\"\n", filename.c_str()); input_buffer = dax__create_buffer(in_file, 10); dax__switch_to_buffer(input_buffer); dax_lineno = 1; - result = xbt_dynar_new(sizeof(SD_task_t), nullptr); - SD_task_t root_task = SD_task_create_comp_seq("root", nullptr, 0); - /* by design the root task is always SCHEDULABLE */ - SD_task_set_state(root_task, SD_SCHEDULABLE); + auto root_task = Exec::init()->set_name("root")->set_flops_amount(0); + root_task->vetoable_start(); + + result.push_back(root_task); - xbt_dynar_push(result, &root_task); - SD_task_t end_task = SD_task_create_comp_seq("end", nullptr, 0); + auto end_task = Exec::init()->set_name("end")->set_flops_amount(0); + end_task->vetoable_start(); - int res = dax_lex(); - if (res != 0) - xbt_die("Parse error in %s: %s", filename, dax__parse_err_msg()); + xbt_assert(dax_lex() == 0, "Parse error in %s: %s", filename.c_str(), dax__parse_err_msg()); dax__delete_buffer(input_buffer); fclose(in_file); dax_lex_destroy(); @@ -164,75 +110,190 @@ xbt_dynar_t SD_daxload(const char *filename) * Files not produced in the system are said to be produced by root task (top of DAG). * Files not consumed in the system are said to be consumed by end task (bottom of DAG). */ + CommPtr file; for (auto const& elm : files) { file = elm.second; - SD_task_t newfile; - if (file->predecessors->empty()) { - for (SD_task_t const& it : *file->successors) { - newfile = SD_task_create_comm_e2e(file->name, nullptr, file->amount); - SD_task_dependency_add(root_task, newfile); - SD_task_dependency_add(newfile, it); - xbt_dynar_push(result, &newfile); + CommPtr newfile; + if (file->dependencies_solved()) { + for (auto const& it : file->get_successors()) { + newfile = Comm::sendto_init()->set_name(file->get_name())->set_payload_size(file->get_remaining()); + root_task->add_successor(newfile); + newfile->add_successor(it); + result.push_back(newfile); } } - if (file->successors->empty()) { - for (SD_task_t const& it : *file->predecessors) { - newfile = SD_task_create_comm_e2e(file->name, nullptr, file->amount); - SD_task_dependency_add(it, newfile); - SD_task_dependency_add(newfile, end_task); - xbt_dynar_push(result, &newfile); + if (file->is_waited_by() == 0) { + for (auto const& it : file->get_dependencies()) { + newfile = Comm::sendto_init()->set_name(file->get_name())->set_payload_size(file->get_remaining()); + it->add_successor(newfile); + newfile->add_successor(end_task); + result.push_back(newfile); } } - for (SD_task_t const& it : *file->predecessors) { - for (SD_task_t const& it2 : *file->successors) { + for (auto const& it : file->get_dependencies()) { + for (auto const& it2 : file->get_successors()) { if (it == it2) { XBT_WARN("File %s is produced and consumed by task %s." "This loop dependency will prevent the execution of the task.", - file->name, it->name); + file->get_cname(), it->get_cname()); } - newfile = SD_task_create_comm_e2e(file->name, nullptr, file->amount); - SD_task_dependency_add(it, newfile); - SD_task_dependency_add(newfile, it2); - xbt_dynar_push(result, &newfile); + newfile = Comm::sendto_init()->set_name(file->get_name())->set_payload_size(file->get_remaining()); + it->add_successor(newfile); + newfile->add_successor(it2); + result.push_back(newfile); } } /* Free previous copy of the files */ - SD_task_destroy(file); + file->destroy(); } /* Push end task last */ - xbt_dynar_push(result, &end_task); + result.push_back(end_task); - unsigned int cpt; - xbt_dynar_foreach(result, cpt, file) { - if (SD_task_get_kind(file) == SD_TASK_COMM_E2E) { - uniq_transfer_task_name(file); + for (const auto& a : result) { + auto* comm = dynamic_cast(a.get()); + if (comm != nullptr) { + uniq_transfer_task_name(comm); } else { /* If some tasks do not take files as input, connect them to the root * if they don't produce files, connect them to the end node. */ - if ((file != root_task) && (file != end_task)) { - if (file->inputs->empty()) - SD_task_dependency_add(root_task, file); - if (file->outputs->empty()) - SD_task_dependency_add(file, end_task); + if ((a != root_task) && (a != end_task)) { + if (a->dependencies_solved()) + root_task->add_successor(a); + if (a->is_waited_by() == 0) + a->add_successor(end_task); } } } - if (not acyclic_graph_detail(result)) { + if (not check_for_cycle(result)) { XBT_ERROR("The DAX described in %s is not a DAG. It contains a cycle.", simgrid::xbt::Path(filename).get_base_name().c_str()); - xbt_dynar_foreach(result, cpt, file) - SD_task_destroy(file); - xbt_dynar_free_container(&result); - result = nullptr; + for (const auto& a : result) + a->destroy(); + result.clear(); } return result; } +#if HAVE_GRAPHVIZ +std::vector create_DAG_from_dot(const std::string& filename) +{ + FILE* in_file = fopen(filename.c_str(), "r"); + xbt_assert(in_file != nullptr, "Failed to open file: %s", filename.c_str()); + + Agraph_t* dag_dot = agread(in_file, NIL(Agdisc_t*)); + + std::unordered_map activities; + std::vector dag; + + ActivityPtr root; + ActivityPtr end; + ActivityPtr act; + /* Create all the nodes */ + Agnode_t* node = nullptr; + for (node = agfstnode(dag_dot); node; node = agnxtnode(dag_dot, node)) { + char* name = agnameof(node); + double amount = atof(agget(node, (char*)"size")); + + if (activities.find(name) == activities.end()) { + XBT_DEBUG("See ", name, amount); + act = Exec::init()->set_name(name)->set_flops_amount(amount)->vetoable_start(); + activities.insert({std::string(name), act}); + if (strcmp(name, "root") && strcmp(name, "end")) + dag.push_back(act); + } else { + XBT_WARN("Exec '%s' is defined more than once", name); + } + } + /*Check if 'root' and 'end' nodes have been explicitly declared. If not, create them. */ + if (activities.find("root") == activities.end()) + root = Exec::init()->set_name("root")->set_flops_amount(0)->vetoable_start(); + else + root = activities.at("root"); + + if (activities.find("end") == activities.end()) + end = Exec::init()->set_name("end")->set_flops_amount(0)->vetoable_start(); + else + end = activities.at("end"); + + /* Create edges */ + std::vector edges; + for (node = agfstnode(dag_dot); node; node = agnxtnode(dag_dot, node)) { + edges.clear(); + for (Agedge_t* edge = agfstout(dag_dot, node); edge; edge = agnxtout(dag_dot, edge)) + edges.push_back(edge); + + /* Be sure edges are sorted */ + std::sort(edges.begin(), edges.end(), [](const Agedge_t* a, const Agedge_t* b) { return AGSEQ(a) < AGSEQ(b); }); + + for (Agedge_t* edge : edges) { + const char* src_name = agnameof(agtail(edge)); + const char* dst_name = agnameof(aghead(edge)); + double size = atof(agget(edge, (char*)"size")); + + ActivityPtr src = activities.at(src_name); + ActivityPtr dst = activities.at(dst_name); + if (size > 0) { + std::string name = std::string(src_name) + "->" + dst_name; + XBT_DEBUG("See ", name.c_str(), size); + if (activities.find(name) == activities.end()) { + act = Comm::sendto_init()->set_name(name)->set_payload_size(size)->vetoable_start(); + src->add_successor(act); + act->add_successor(dst); + activities.insert({name, act}); + dag.push_back(act); + } else { + XBT_WARN("Comm '%s' is defined more than once", name.c_str()); + } + } else { + src->add_successor(dst); + } + } + } + + XBT_DEBUG("All activities have been created, put %s at the beginning and %s at the end", root->get_cname(), + end->get_cname()); + dag.insert(dag.begin(), root); + dag.push_back(end); + + /* Connect entry tasks to 'root', and exit tasks to 'end'*/ + for (const auto& a : dag) { + if (a->dependencies_solved() && a != root) { + XBT_DEBUG("Activity '%s' has no dependencies. Add dependency from 'root'", a->get_cname()); + root->add_successor(a); + } + + if (a->is_waited_by() == 0 && a != end) { + XBT_DEBUG("Activity '%s' has no successors. Add dependency to 'end'", a->get_cname()); + a->add_successor(end); + } + } + agclose(dag_dot); + fclose(in_file); + + if (not check_for_cycle(dag)) { + std::string base = simgrid::xbt::Path(filename).get_base_name(); + XBT_ERROR("The DOT described in %s is not a DAG. It contains a cycle.", base.c_str()); + for (const auto& a : dag) + a->destroy(); + dag.clear(); + } + + return dag; +} +#else +std::vector create_DAG_from_dot(const std::string& filename) xbt_die( + "create_DAG_from_dot() is not usable because graphviz was not found.\n" + "Please install graphviz, graphviz-dev, and libgraphviz-dev (and erase CMakeCache.txt) before recompiling."); +} +#endif +} // namespace s4u +} // namespace simgrid + void STag_dax__adag() { try { @@ -251,9 +312,9 @@ void STag_dax__job() std::string name = std::string(A_dax__job_id) + "@" + A_dax__job_name; runtime *= 4200000000.; /* Assume that timings were done on a 4.2GFlops machine. I mean, why not? */ XBT_DEBUG("See ", A_dax__job_id, A_dax__job_runtime, runtime); - current_job = SD_task_create_comp_seq(name.c_str(), nullptr, runtime); - jobs.insert({A_dax__job_id, current_job}); - xbt_dynar_push(result, ¤t_job); + simgrid::s4u::current_job = simgrid::s4u::Exec::init()->set_name(name)->set_flops_amount(runtime)->vetoable_start(); + simgrid::s4u::jobs.insert({A_dax__job_id, simgrid::s4u::current_job}); + simgrid::s4u::result.push_back(simgrid::s4u::current_job); } catch (const std::invalid_argument&) { throw std::invalid_argument(std::string("Parse error: ") + A_dax__job_runtime + " is not a double"); } @@ -270,33 +331,32 @@ void STag_dax__uses() bool is_input = (A_dax__uses_link == A_dax__uses_link_input); XBT_DEBUG("See ",A_dax__uses_file,(is_input?"in":"out")); - auto it = files.find(A_dax__uses_file); - SD_task_t file; - if (it == files.end()) { - file = SD_task_create_comm_e2e(A_dax__uses_file, nullptr, size); - sd_global->initial_tasks.erase(file); - files[A_dax__uses_file] = file; + auto it = simgrid::s4u::files.find(A_dax__uses_file); + simgrid::s4u::CommPtr file; + if (it == simgrid::s4u::files.end()) { + file = simgrid::s4u::Comm::sendto_init()->set_name(A_dax__uses_file)->set_payload_size(size); + simgrid::s4u::files[A_dax__uses_file] = file.get(); } else { file = it->second; - if (file->amount < size || file->amount > size) { - XBT_WARN("Ignore file %s size redefinition from %.0f to %.0f", A_dax__uses_file, SD_task_get_amount(file), size); + if (file->get_remaining() < size || file->get_remaining() > size) { + XBT_WARN("Ignore file %s size redefinition from %.0f to %.0f", A_dax__uses_file, file->get_remaining(), size); } } if (is_input) { - SD_task_dependency_add(file, current_job); + file->add_successor(simgrid::s4u::current_job); } else { - SD_task_dependency_add(current_job, file); - if ((file->predecessors->size() + file->inputs->size()) > 1) { - XBT_WARN("File %s created at more than one location...", file->name); + simgrid::s4u::current_job->add_successor(file); + if (file->get_dependencies().size() > 1) { + XBT_WARN("File %s created at more than one location...", file->get_cname()); } } } -static SD_task_t current_child; +static simgrid::s4u::ExecPtr current_child; void STag_dax__child() { - auto job = jobs.find(A_dax__child_ref); - if (job != jobs.end()) { + auto job = simgrid::s4u::jobs.find(A_dax__child_ref); + if (job != simgrid::s4u::jobs.end()) { current_child = job->second; } else { throw std::out_of_range(std::string("Parse error on line ") + std::to_string(dax_lineno) + @@ -311,15 +371,15 @@ void ETag_dax__child() void STag_dax__parent() { - auto job = jobs.find(A_dax__parent_ref); - if (job != jobs.end()) { - SD_task_t parent = job->second; - SD_task_dependency_add(parent, current_child); - XBT_DEBUG("Control-flow dependency from %s to %s", current_child->name, parent->name); + auto job = simgrid::s4u::jobs.find(A_dax__parent_ref); + if (job != simgrid::s4u::jobs.end()) { + auto parent = job->second; + parent->add_successor(current_child); + XBT_DEBUG("Control-flow dependency from %s to %s", current_child->get_cname(), parent->get_cname()); } else { throw std::out_of_range(std::string("Parse error on line ") + std::to_string(dax_lineno) + - ": Asked to add a dependency from " + current_child->name + " to " + A_dax__parent_ref + - ", but " + A_dax__parent_ref + " does not exist"); + ": Asked to add a dependency from " + current_child->get_name() + " to " + + A_dax__parent_ref + ", but " + A_dax__parent_ref + " does not exist"); } } @@ -330,7 +390,7 @@ void ETag_dax__adag() void ETag_dax__job() { - current_job = nullptr; + simgrid::s4u::current_job = nullptr; XBT_DEBUG("See "); }