1 /* cunit - A little C Unit facility */
3 /* Copyright (c) 2005-2019. The SimGrid Team. All rights reserved. */
5 /* This program is free software; you can redistribute it and/or modify it
6 * under the terms of the license (GNU LGPL) which comes with this package. */
8 /* This is partially inspirated from the OSSP ts (Test Suite Library) */
9 /* At some point we should use https://github.com/google/googletest instead */
11 #include "src/internal_config.h"
17 #include "simgrid/Exception.hpp"
18 #include <xbt/cunit.h>
19 #include <xbt/string.hpp>
21 /* output stream to use everywhere */
22 static std::ostream& _xbt_test_out = std::cerr;
24 /* collection of all suites */
25 static std::vector<xbt_test_suite_t> _xbt_test_suites;
26 /* global statistics */
27 static int _xbt_test_nb_tests = 0;
28 static int _xbt_test_test_failed = 0;
29 static int _xbt_test_test_ignore = 0;
30 static int _xbt_test_test_expect = 0;
32 static int _xbt_test_nb_units = 0;
33 static int _xbt_test_unit_failed = 0;
34 static int _xbt_test_unit_ignore = 0;
35 static int _xbt_test_unit_disabled = 0;
37 static int _xbt_test_nb_suites = 0;
38 static int _xbt_test_suite_failed = 0;
39 static int _xbt_test_suite_ignore = 0;
40 static int _xbt_test_suite_disabled = 0;
43 xbt_test_unit_t _xbt_test_current_unit = nullptr;
45 /* test suite test log */
46 class s_xbt_test_log {
48 s_xbt_test_log(std::string text, std::string file, int line)
49 : text_(std::move(text)), file_(std::move(file)), line_(line)
59 void s_xbt_test_log::dump() const
61 _xbt_test_out << " log " << this << "(" << file_ << ":" << line_ << ")=" << text_ << "\n";
64 /* test suite test check */
65 class s_xbt_test_test {
67 s_xbt_test_test(std::string title, std::string file, int line)
68 : title_(std::move(title)), file_(std::move(file)), line_(line)
75 bool expected_failure_ = false;
76 bool ignored_ = false;
79 std::vector<s_xbt_test_log> logs_;
82 void s_xbt_test_test::dump() const
84 _xbt_test_out << " test " << this << "(" << file_ << ":" << line_ << ")=" << title_ << " ("
85 << (failed_ ? "failed" : "not failed") << ")\n";
86 for (s_xbt_test_log const& log : this->logs_)
90 /* test suite test unit */
91 class s_xbt_test_unit {
93 s_xbt_test_unit(std::string name, std::string title, ts_test_cb_t func)
94 : name_(std::move(name)), title_(std::move(title)), func_(func)
102 std::vector<s_xbt_test_test> tests_;
104 bool enabled_ = true;
106 int test_failed_ = 0;
107 int test_ignore_ = 0;
108 int test_expect_ = 0;
111 void s_xbt_test_unit::dump() const
113 _xbt_test_out << " UNIT " << name_ << ": " << title_ << " (" << (this->enabled_ ? "enabled" : "disabled") << ")\n";
114 if (this->enabled_) {
115 for (s_xbt_test_test const& test : this->tests_)
121 class s_xbt_test_suite {
123 s_xbt_test_suite(std::string name, std::string title) : name_(std::move(name)), title_(std::move(title)) {}
125 void push(s_xbt_test_unit unit) { units_.emplace_back(std::move(unit)); }
126 int run(int verbosity);
130 std::vector<s_xbt_test_unit> units_;
132 bool enabled_ = true;
135 int test_failed_ = 0;
136 int test_ignore_ = 0;
137 int test_expect_ = 0;
138 int unit_failed_ = 0;
139 int unit_ignore_ = 0;
140 int unit_disabled_ = 0;
143 /** @brief retrieve a testsuite from name, or create a new one */
144 xbt_test_suite_t xbt_test_suite_by_name(const char *name, const char *fmt, ...)
146 auto res = std::find_if(begin(_xbt_test_suites), end(_xbt_test_suites),
147 [&name](xbt_test_suite_t const& suite) { return suite->name_ == name; });
148 if (res != end(_xbt_test_suites))
153 xbt_test_suite_t suite = new s_xbt_test_suite(name, simgrid::xbt::string_vprintf(fmt, ap));
156 _xbt_test_suites.push_back(suite);
161 void s_xbt_test_suite::dump() const
163 _xbt_test_out << "TESTSUITE " << name_ << ": " << title_ << " (" << (this->enabled_ ? "enabled" : "disabled")
165 if (this->enabled_) {
166 for (s_xbt_test_unit const& unit : this->units_)
171 /* add test case to test suite */
172 void xbt_test_suite_push(xbt_test_suite_t suite, const char *name, ts_test_cb_t func, const char *fmt, ...)
180 s_xbt_test_unit unit(name, simgrid::xbt::string_vprintf(fmt, ap), func);
185 /* run test one suite */
186 int s_xbt_test_suite::run(int verbosity)
188 /* suite title pretty-printing */
189 int suite_len = this->title_.length();
190 xbt_assert(suite_len < 68, "suite title \"%s\" too long (%d should be less than 68", this->title_.c_str(), suite_len);
192 std::string suite_title = " ";
193 suite_title.resize(40 - (suite_len + 4) / 2, '=');
194 suite_title += std::string("[ ") + this->title_ + " ]";
195 suite_title.resize(79, '=');
196 if (not this->enabled_)
197 suite_title.replace(70, std::string::npos, " DISABLED");
198 _xbt_test_out << "\n" << suite_title << "\n";
200 if (this->enabled_) {
201 /* iterate through all tests */
202 for (s_xbt_test_unit& unit : this->units_) {
203 /* init unit case counters */
205 unit.test_ignore_ = 0;
206 unit.test_failed_ = 0;
207 unit.test_expect_ = 0;
209 /* display unit title */
210 std::string cp = std::string(" Unit: ") + unit.title_ + " ";
214 /* run the test case function */
215 _xbt_test_current_unit = &unit;
219 /* iterate through all performed tests to determine status */
220 for (s_xbt_test_test const& test : unit.tests_) {
226 if ((test.failed_ && not test.expected_failure_) || (not test.failed_ && test.expected_failure_))
228 if (test.expected_failure_)
232 /* Display whether this unit went well */
233 if (unit.test_failed_ > 0 || unit.test_expect_ || (verbosity && unit.nb_tests_ > 0)) {
234 /* some tests failed (or were supposed to), so do detailed reporting of test case */
235 if (unit.test_failed_ > 0) {
236 _xbt_test_out << ".. failed\n";
237 } else if (unit.nb_tests_) {
238 _xbt_test_out << "...... ok\n"; /* successful, but show about expected */
240 _xbt_test_out << ".... skip\n"; /* shouldn't happen, but I'm a bit lost with this logic */
242 for (s_xbt_test_test const& test : unit.tests_) {
243 std::string file = test.file_;
244 int line = test.line_;
248 else if (test.expected_failure_) {
259 _xbt_test_out << " " << resname << ": " << test.title_ << " [" << file << ":" << line << "]\n";
261 if ((test.expected_failure_ && not test.failed_) || (not test.expected_failure_ && test.failed_)) {
262 for (s_xbt_test_log const& log : test.logs_) {
263 file = (log.file_.empty() ? file : log.file_);
264 line = (log.line_ == 0 ? line : log.line_);
265 _xbt_test_out << " " << file << ":" << line << ": " << log.text_ << "\n";
269 _xbt_test_out << " Summary: " << unit.test_failed_ << " of " << unit.nb_tests_ << " tests failed";
270 if (unit.test_ignore_) {
271 _xbt_test_out << " (" << unit.test_ignore_ << " tests ignored)\n";
273 _xbt_test_out << "\n";
275 } else if (not unit.enabled_) {
276 _xbt_test_out << " disabled\n"; /* no test were run */
277 } else if (unit.nb_tests_) {
278 _xbt_test_out << "...... ok\n"; /* successful */
280 _xbt_test_out << ".... skip\n"; /* no test were run */
283 /* Accumulate test counts into the suite */
284 this->nb_tests_ += unit.nb_tests_;
285 this->test_failed_ += unit.test_failed_;
286 this->test_ignore_ += unit.test_ignore_;
287 this->test_expect_ += unit.test_expect_;
289 _xbt_test_nb_tests += unit.nb_tests_;
290 _xbt_test_test_failed += unit.test_failed_;
291 _xbt_test_test_ignore += unit.test_ignore_;
292 _xbt_test_test_expect += unit.test_expect_;
294 /* What's the conclusion of this test anyway? */
295 if (unit.nb_tests_) {
297 if (unit.test_failed_)
298 this->unit_failed_++;
299 } else if (not unit.enabled_) {
300 this->unit_disabled_++;
302 this->unit_ignore_++;
306 _xbt_test_nb_units += this->nb_units_;
307 _xbt_test_unit_failed += this->unit_failed_;
308 _xbt_test_unit_ignore += this->unit_ignore_;
309 _xbt_test_unit_disabled += this->unit_disabled_;
311 if (this->nb_units_) {
312 _xbt_test_nb_suites++;
313 if (this->test_failed_)
314 _xbt_test_suite_failed++;
315 } else if (not this->enabled_) {
316 _xbt_test_suite_disabled++;
318 _xbt_test_suite_ignore++;
321 /* print test suite summary */
322 if (this->enabled_) {
323 bool first = true; /* for result pretty printing */
325 _xbt_test_out << " =====================================================================";
327 _xbt_test_out << (this->unit_failed_ ? "== FAILED\n" : "====== OK\n");
329 _xbt_test_out << (this->unit_disabled_ ? " DISABLED\n" : "==== SKIP\n");
330 _xbt_test_out.setf(std::ios::fixed);
331 _xbt_test_out.precision(0);
332 _xbt_test_out << " Summary: Units: "
333 << (this->nb_units_ ? ((1 - (double)this->unit_failed_ / (double)this->nb_units_) * 100.0) : 100.0)
334 << "% ok (" << this->nb_units_ << " units: ";
335 if (this->nb_units_ != this->unit_failed_) {
336 _xbt_test_out << (first ? "" : ", ") << (this->nb_units_ - this->unit_failed_) << " ok";
339 if (this->unit_failed_) {
340 _xbt_test_out << (first ? "" : ", ") << this->unit_failed_ << " failed";
343 if (this->unit_ignore_) {
344 _xbt_test_out << (first ? "" : ", ") << this->unit_ignore_ << " ignored";
347 if (this->unit_disabled_) {
348 _xbt_test_out << (first ? "" : ", ") << this->unit_disabled_ << " disabled";
350 _xbt_test_out << ")\n Tests: "
351 << (this->nb_tests_ ? ((1 - (double)this->test_failed_ / (double)this->nb_tests_) * 100.0) : 100.0)
352 << "% ok (" << this->nb_tests_ << " tests: ";
354 if (this->nb_tests_ != this->test_failed_) {
355 _xbt_test_out << (first ? "" : ", ") << (this->nb_tests_ - this->test_failed_) << " ok";
358 if (this->test_failed_) {
359 _xbt_test_out << (first ? "" : ", ") << this->test_failed_ << " failed";
362 if (this->test_ignore_) {
363 _xbt_test_out << (first ? "" : "; ") << this->test_ignore_ << " ignored";
366 if (this->test_expect_) {
367 _xbt_test_out << (first ? "" : "; ") << this->test_expect_ << " expected to fail";
369 _xbt_test_out << ")\n";
371 return this->unit_failed_;
374 static void apply_selection(char *selection)
376 if (not selection || selection[0] == '\0')
379 /* for the parsing */
380 std::string sel = selection;
382 std::string dir; /* the directive */
383 std::string suitename;
384 std::string unitname;
386 /* First apply the selection */
389 bool enabling = true;
391 size_t p = sel.find(',', p0);
392 if (p != std::string::npos) {
393 dir = sel.substr(p0, p - p0);
396 dir = sel.substr(p0);
410 if (p != std::string::npos) {
411 suitename = dir.substr(0, p);
412 unitname = dir.substr(p + 1);
418 /* Deal with the specific case of 'all' pseudo serie */
419 if (suitename == "all") {
420 xbt_assert(unitname.empty(), "The 'all' pseudo-suite does not accept any unit specification\n");
422 for (xbt_test_suite_t& suite : _xbt_test_suites) {
423 for (s_xbt_test_unit& unit : suite->units_) {
424 unit.enabled_ = enabling;
426 suite->enabled_ = enabling;
429 bool suitefound = false;
430 for (xbt_test_suite_t& thissuite : _xbt_test_suites) {
431 if (suitename == thissuite->name_) {
432 /* Do not disable the whole suite when we just want to disable a child */
433 if (enabling || unitname.empty())
434 thissuite->enabled_ = enabling;
436 if (unitname.empty()) {
437 for (s_xbt_test_unit& unit : thissuite->units_) {
438 unit.enabled_ = enabling;
440 } else { /* act on one child only */
441 /* search relevant unit */
442 auto unit = std::find_if(begin(thissuite->units_), end(thissuite->units_),
443 [&unitname](s_xbt_test_unit const& unit) { return unit.name_ == unitname; });
444 if (unit == end(thissuite->units_))
445 xbt_die("Suite '%s' has no unit of name '%s'. Cannot apply the selection\n", suitename.c_str(),
447 unit->enabled_ = enabling;
448 } /* act on childs (either all or one) */
450 break; /* found the relevant serie. We are happy */
452 } /* search relevant series */
453 xbt_assert(suitefound, "No suite of name '%s' found. Cannot apply the selection\n", suitename.c_str());
458 void xbt_test_dump(char *selection)
460 apply_selection(selection);
462 if (not _xbt_test_suites.empty()) {
463 for (xbt_test_suite_t suite : _xbt_test_suites)
466 _xbt_test_out << " No suite defined.";
470 int xbt_test_run(char *selection, int verbosity)
472 apply_selection(selection);
474 if (not _xbt_test_suites.empty()) {
477 /* Run all the suites */
478 for (xbt_test_suite_t& suite : _xbt_test_suites)
480 suite->run(verbosity);
482 /* Display some more statistics */
483 _xbt_test_out.setf(std::ios::fixed);
484 _xbt_test_out.precision(0);
485 _xbt_test_out << "\n\n TOTAL: Suites: "
486 << (_xbt_test_nb_suites ? ((1 - (double)_xbt_test_suite_failed / (double)_xbt_test_nb_suites) * 100.0)
488 << "% ok (" << _xbt_test_nb_suites << " suites: ";
489 if (_xbt_test_nb_suites != _xbt_test_suite_failed) {
490 _xbt_test_out << (_xbt_test_nb_suites - _xbt_test_suite_failed) << " ok";
493 if (_xbt_test_suite_failed) {
494 _xbt_test_out << (first ? "" : ", ") << _xbt_test_suite_failed << " failed";
498 if (_xbt_test_suite_ignore) {
499 _xbt_test_out << (first ? "" : ", ") << _xbt_test_suite_ignore << " ignored";
501 _xbt_test_out << ")\n Units: "
502 << (_xbt_test_nb_units ? ((1 - (double)_xbt_test_unit_failed / (double)_xbt_test_nb_units) * 100.0)
504 << "% ok (" << _xbt_test_nb_units << " units: ";
506 if (_xbt_test_nb_units != _xbt_test_unit_failed) {
507 _xbt_test_out << (_xbt_test_nb_units - _xbt_test_unit_failed) << " ok";
510 if (_xbt_test_unit_failed) {
511 _xbt_test_out << (first ? "" : ", ") << _xbt_test_unit_failed << " failed";
514 if (_xbt_test_unit_ignore) {
515 _xbt_test_out << (first ? "" : ", ") << _xbt_test_unit_ignore << " ignored";
517 _xbt_test_out << ")\n Tests: "
518 << (_xbt_test_nb_tests ? ((1 - (double)_xbt_test_test_failed / (double)_xbt_test_nb_tests) * 100.0)
520 << "% ok (" << _xbt_test_nb_tests << " tests: ";
522 if (_xbt_test_nb_tests != _xbt_test_test_failed) {
523 _xbt_test_out << (_xbt_test_nb_tests - _xbt_test_test_failed) << " ok";
526 if (_xbt_test_test_failed) {
527 _xbt_test_out << (first ? "" : ", ") << _xbt_test_test_failed << " failed";
530 if (_xbt_test_test_ignore) {
531 _xbt_test_out << (first ? "" : ", ") << _xbt_test_test_ignore << " ignored";
534 if (_xbt_test_test_expect) {
535 _xbt_test_out << (first ? "" : ", ") << _xbt_test_test_expect << " expected to fail";
538 _xbt_test_out << ")\n";
540 _xbt_test_out << "No unit to run!\n";
541 _xbt_test_unit_failed++;
543 return _xbt_test_unit_failed;
548 for (xbt_test_suite_t suite : _xbt_test_suites)
550 _xbt_test_suites.clear();
553 /* annotate test case with test */
554 void _xbt_test_add(const char *file, int line, const char *fmt, ...)
556 xbt_test_unit_t unit = _xbt_test_current_unit;
561 unit->tests_.emplace_back(simgrid::xbt::string_vprintf(fmt, ap), file, line);
565 /* annotate test case with log message and failure */
566 void _xbt_test_fail(const char *file, int line, const char *fmt, ...)
568 xbt_test_unit_t unit = _xbt_test_current_unit;
570 xbt_assert(not _xbt_test_current_unit->tests_.empty(), "Test failed even before being declared (broken unit: %s)",
571 unit->title_.c_str());
573 s_xbt_test_test& test = unit->tests_.back();
576 test.logs_.emplace_back(simgrid::xbt::string_vprintf(fmt, ap), file, line);
582 void xbt_test_exception(xbt_ex e)
584 _xbt_test_fail(e.throw_point().file_, e.throw_point().line_, "Exception %s raised: %s", xbt_ex_catname(e.category),
588 void xbt_test_expect_failure()
590 xbt_assert(not _xbt_test_current_unit->tests_.empty(),
591 "Cannot expect the failure of a test before declaring it (broken unit: %s)",
592 _xbt_test_current_unit->title_.c_str());
593 _xbt_test_current_unit->tests_.back().expected_failure_ = true;
598 xbt_assert(not _xbt_test_current_unit->tests_.empty(), "Test skipped even before being declared (broken unit: %s)",
599 _xbt_test_current_unit->title_.c_str());
600 _xbt_test_current_unit->tests_.back().ignored_ = true;
603 /* annotate test case with log message only */
604 void _xbt_test_log(const char *file, int line, const char *fmt, ...)
606 xbt_test_unit_t unit = _xbt_test_current_unit;
608 xbt_assert(not _xbt_test_current_unit->tests_.empty(),
609 "Test logged into even before being declared (broken test unit: %s)", unit->title_.c_str());
613 unit->tests_.back().logs_.emplace_back(simgrid::xbt::string_vprintf(fmt, ap), file, line);
618 XBT_TEST_SUITE("cunit", "Testsuite mechanism autotest");
620 XBT_TEST_UNIT("expect", test_expected_failure, "expected failures")
622 xbt_test_add("Skipped test");
625 xbt_test_add("%s %s", "EXPECTED", "FAILURE");
626 xbt_test_expect_failure();
627 xbt_test_log("%s %s", "Test", "log");
628 xbt_test_fail("EXPECTED FAILURE");
630 #endif /* SIMGRID_TEST */