From: Martin Quinson Date: Mon, 1 Feb 2016 16:32:45 +0000 (+0100) Subject: rewrite the doc around the tests X-Git-Tag: v3_13~1002 X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/09b888098e2c4e1c0af0bbae4ad034806f37d95f rewrite the doc around the tests --- diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 06557c19b3..fed3b985af 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -659,7 +659,7 @@ INPUT = doxygen/index.doc \ doxygen/inside_doxygen.doc \ doxygen/inside_extending.doc \ doxygen/inside_cmake.doc \ - doxygen/inside_ci.doc \ + doxygen/inside_tests.doc \ doxygen/inside_release.doc \ doxygen/contributing.doc \ doxygen/tracing.doc \ diff --git a/doc/doxygen/inside_ci.doc b/doc/doxygen/inside_ci.doc deleted file mode 100644 index 7c5bba77eb..0000000000 --- a/doc/doxygen/inside_ci.doc +++ /dev/null @@ -1,50 +0,0 @@ -/*! -\page inside_ci Continous Integration (with Jenkins) - -\section ci_jenkins Jenkins Interface - -\subsection inside_jenkins_basics Where can I find Jenkins? - -The SimGrid team currently uses Jenkins to automate testing. Our Jenkins -interface can be found here: https://ci.inria.fr/simgrid/ - -If you need an account, talk to Martin Quinson. - -\subsection inside_jenkins_add_host How can I add a new host? - -You have to login to the CI interface of INRIA: https://ci.inria.fr -There you can manage the project and add new nodes. - -\subsection inside_jenkins_reboot_host How can I restart / reboot a host? - -Exactly the same as in \ref inside_jenkins_add_host. The only exception -is that you have to click on "restart" of the host you want to restart. - - -\subsection inside_jenkins_config_matrix Disable a certain build in the configuration matrix - -Jenkins uses a configuration matrix, i.e., a matrix consisting of configurations -on the one axis and hosts on the other. Normally, every host will try to build -every configuration but this may not be desirable. - -In order to disable a single configuration for a specific host (but leave -other configurations for the same host enabled), go to your Project and click -on "Configuration". There, find the field "combination filter" (if your interface -language is English) and tick the checkbox; then add a groovy-expression to -disable a specific configuration. For example, in order to disable the "ModelChecker" -build on host "small-freebsd-64-clang", use: - -\verbatim -(label=="small-freebsd-64-clang").implies(build_mode!="ModelChecker") -\endverbatim - -\section ci_servers CI Servers - -\subsection ci_servers_build_dir Where is SimGrid built? - -SimGrid gets built in /builds/workspace/$PROJECT/build_mode/$CONFIG/label/$SERVER/build - -Here, $PROJECT could be for instance "SimGrid-Multi", $CONFIG "DEBUG" or "ModelChecker" -and $SERVER for instance "simgrid-fedora20-64-clang". - -*/ diff --git a/doc/doxygen/inside_cmake.doc b/doc/doxygen/inside_cmake.doc index bfcf98144b..b2de29f530 100644 --- a/doc/doxygen/inside_cmake.doc +++ b/doc/doxygen/inside_cmake.doc @@ -68,29 +68,26 @@ Don't forget to run the "make distcheck" target after any modification to the cmake files: it checks whether all necessary files are present in the distribution. -\section cmake_dev_guide_ex How to add examples? - -First of all, are you sure that you want to create an example, or is -it merely a new test case? The examples located in examples/ must be -interesting to the users. It is expected that the users will take one -of these examples and start editing it to make it fit their needs. If -what you are about to write is merly a test, exercising a specific -part of the tool suite but not really interesting to the users, then -you want to add it to the teshsuite/ directory. - -Actually, the examples are also used as regresion tests by adding tesh -files and registering them to the testing infrastructure (for that, -don't forget to add a tesh file and follow the instructions of -section \ref inside_cmake_addtest). The main difference is that -examples must be interesting to the users in addition. - -In both cases, you have to create a CMakeList.txt in the chosen source +\section inside_cmake_examples How to add an example? + +The first rule is that the content of examples/ must be interesting to +the users. It is expected that the users will take one of these +examples and start editing it to make it fit their needs. +So, it should be self-contained, informative and should use only the +public APIs. + +To ensure that all examples actually work as expected, every examples +are also used as integration test (see \ref inside_tests), but you +should still strive to keep the code under examples/ as informative as +possible for the users. In particular, torture test cases should be +placed in teshsuite/, not examples/, so that the users don't stumble +upon them by error. + +To add a new example, create a CMakeList.txt in the chosen source directory. It must specify where to create the executable, the source list, dependencies and the name of the binary. \verbatim -cmake_minimum_required(VERSION 2.6) - set(EXECUTABLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}") add_executable(Hello Hello.c) @@ -138,118 +135,12 @@ set(CMAKE_SOURCE_FILES ) \endverbatim -\li if you add tesh files (and you should), please refer to the -following section to register them to the testing infrastructure. +\li Add the tesh file and register your example to the testing + infrastructure. See \ref inside_tests_add_integration for more + details. -Once you're done, you should check with "make distcheck" that you did +Once you're done, you must run "make distcheck" to ensure that you did not forget to add any file to the distributed archives. -\section inside_cmake_addtest How to add tests? - -\subsection inside_cmake_addtest_unit Unit testing in SimGrid - -If you want to test a specific function or set of functions, you need -a unit test. Edit -/tools/cmake/UnitTesting.cmake to add your -source file to the TEST_CFILES list, and add the corresponding unit -file to the TEST_UNITS list. For example, if your file is toto.c, -your unit file will be toto_unit.c. The full path to your file must be -provided, but the unit file will always be in src/ directly. - -Then, you want to actually add your tests in the source file. All the -tests must be protected by "#ifdef SIMGRID_TEST" so that they don't -get included in the regular build. Then, you want to add a test suite -that will contain a bunch of tests (in Junit, that would be a test -unit) with the macro #XBT_TEST_SUITE, and populate it with a bunch of -actual tests with the macro #XBT_TEST_UNIT (sorry for the mischosen -names if you are used to junit). Just look at the dynar example (or -any other) to see how it works in practice. Do not hesitate to stress -test your code this way, but make sure that it runs reasonably fast, -or nobody will run "ctest" before commiting code. - -If you are interested in the mechanic turning this into an actual -test, check the /tools/sg_unit_extractor.pl script. - -To actually run your tests once you're done, run "make testall", that -builds the binary containing all our unit tests and run it. This -binary allows you to chose which suite you want to test: - -\verbatim -$ testall --help # revise how it goes if you forgot -$ testall --dump-only # learn about all existing test suites -$ testall --tests=-all # run no test at all -$ testall --tests=-all,+foo # run only the foo test suite. -$ testall --tests=-all,+foo:bar # run only the bar test from the foo suite. -$ testall --tests=-foo:bar # run all tests but the bar test from the foo suite. -\endverbatim - -\subsection inside_cmake_addtest_integration Integration testing in SimGrid - -If you want to test a full binary (such as an example), then you -probably want to use the tesh binary that ensures that the output -produced by a command perfectly matches the expected output. In -particular, this is very precious to ensure that no change modifies -the timings computed by the models without notice. - -The first step is to write a tesh file for your test, containing the -command to run, the provided input (if any, but almost no SimGrid test -provide such an input) and the expected output. Check the tesh man -page for more details. - -Tesh is sometimes annoying as you have to ensure that the expected -output will always be exactly the same. In particular, your should not -output machine dependent informations, nor memory adresses as they -would change on each run. Several steps can be used here, such as the -obfucation of the memory adresses unless the verbose logs are -displayed (using the #XBT_LOG_ISENABLED() macro), or the modification -of the log formats to hide the timings when they depend on the host -machine. - -Then you have to request cmake to run your test when "ctest" is -launched. For that, you have to modify source -/tools/cmake/Tests.cmake. Make sure to pick -a wise name for your test. It is often useful to check a category of -tests together. The only way to do so in ctest is to use the -R -argument that specifies a regular expression that the test names must -match. For example, you can run all MSG test with "ctest -R msg" That -explains the importance of the test names. - -Once the name is chosen, create a new test by adding a line similar to -the following (assuming that you use tesh as expected). - -\verbatim -# ADD_TEST(test-name ${CMAKE_BINARY_DIR}/bin/tesh ) -# option --setenv bindir set the directory containing the binary -# --setenv srcdir set the directory containing the source file -# --cd set the working directory -ADD_TEST(my-test-name ${CMAKE_BINARY_DIR}/bin/tesh - --setenv bindir=${CMAKE_BINARY_DIR}/examples/my-test/ - --setenv srcdir=${CMAKE_HOME_DIRECTORY}/examples/my-test/ - --cd ${CMAKE_HOME_DIRECTORY}/examples/my-test/ - ${CMAKE_HOME_DIRECTORY}/examples/msg/io/io.tesh -) -\endverbatim - -If you prefer to not use tesh for some reasons, prefer the following -form: - -\verbatim -# ADD_TEST(NAME ] -# [WORKING_DIRECTORY dir] -# COMMAND [arg1 [arg2 ...]]) -ADD_TEST(NAME my-test-name - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/examples/my-test/ - COMMAND Hello -) -\endverbatim - -As usual, you must run "make distcheck" after modifying the cmake files, -to ensure that you did not forget any files in the distributed archive. - -\subsection inside_cmake_ci Continous Integration - -We are using Continous Integration to help us provide a stable build -across as many platforms as possible. %As this is not related to cmake, -you have to head over to Section \ref inside_ci. */ diff --git a/doc/doxygen/inside_tests.doc b/doc/doxygen/inside_tests.doc new file mode 100644 index 0000000000..bd1c5f89ef --- /dev/null +++ b/doc/doxygen/inside_tests.doc @@ -0,0 +1,240 @@ +/*! +@page inside_tests Testing SimGrid + +This page will teach you how to run the tests, selecting the ones you +want, and how to add new tests to the archive. + +\tableofcontents + +SimGrid code coverage is usually between 70% and 80%, which is much +more than most projects out there. This is because we consider SimGrid +to be a rather complex project, and we want to modify it with less fear. + +We have two sets of tests in SimGrid: Each of the 10,000+ unit tests +check one specific case for one specific function, while the 500+ +integration tests run a given simulation specifically intended to +exercise a larger amount of functions together. Every example provided +in examples/ is used as an integration test, while some other torture +tests and corner cases integration tests are located in teshsuite/. +For each integration test, we ensure that the output exactly matches +the defined expectations. Since SimGrid displays the timestamp of +every loggued line, this ensures that every change of the models' +prediction will be noticed. All these tests should ensure that SimGrid +is safe to use and to depend on. + +\section inside_tests_runintegration Running the tests + +Running the tests is done using the ctest binary that comes with +cmake. These tests are run for every commit and the result is publicly +available. + +\verbatim +ctest # Launch all tests +ctest -R msg # Launch only the tests which name match the string "msg" +ctest -j4 # Launch all tests in parallel, at most 4 at the same time +ctest --verbose # Display all details on what's going on +ctest --output-on-failure # Only get verbose for the tests that fail + +ctest -R msg- -j5 --output-on-failure # You changed MSG and want to check that you didn't break anything, huh? + # That's fine, I do so all the time myself. +\endverbatim + +\section inside_tests_rununit Running the unit tests + +All unit tests are packed into the testall binary, that lives in src/. +These tests are run when you launch ctest, don't worry. + +\verbatim +make testall # Rebuild the test runner on need +./src/testall # Launch all tests +./src/testall --help # revise how it goes if you forgot +./src/testall --tests=-all # run no test at all (yeah, that's useless) +./src/testall --dump-only # Display all existing test suite +./src/testall --tests=-all,+dict # Only launch the tests from the dict testsuite +./src/testall --tests=-all,+foo:bar # run only the bar test from the foo suite. +\endverbatim + + +\section inside_tests_add_units Adding unit tests + +If you want to test a specific function or set of functions, you need +a unit test. Edit +/tools/cmake/UnitTesting.cmake to add your +source file to the TEST_CFILES list, and add the corresponding unit +file to the TEST_UNITS list. For example, if your file is toto.c, +your unit file will be toto_unit.c. The full path to your file must be +provided, but the unit file will always be in src/ directly. + +If you want to create unit tests in the file src/xbt/toto.c, your +changes should look similar to: + +\verbatim +--- a/tools/cmake/UnitTesting.cmake ++++ b/tools/cmake/UnitTesting.cmake +@@ -11,6 +11,7 @@ set(TEST_CFILES + src/xbt/xbt_strbuff.c + src/xbt/xbt_sha.c + src/xbt/config.c ++ src/xbt/toto.c + ) + set(TEST_UNITS + ${CMAKE_CURRENT_BINARY_DIR}/src/cunit_unit.c +@@ -22,6 +23,7 @@ set(TEST_UNITS + ${CMAKE_CURRENT_BINARY_DIR}/src/xbt_strbuff_unit.c + ${CMAKE_CURRENT_BINARY_DIR}/src/xbt_sha_unit.c + ${CMAKE_CURRENT_BINARY_DIR}/src/config_unit.c ++ ${CMAKE_CURRENT_BINARY_DIR}/src/toto_unit.c + + ${CMAKE_CURRENT_BINARY_DIR}/src/simgrid_units_main.c + ) +\endverbatim + +Then, you want to actually add your tests in the source file. All the +tests must be protected by "#ifdef SIMGRID_TEST" so that they don't +get included in the regular build. Then, you want to add a test suite +that will contain a bunch of tests (in Junit, that would be a test +unit) with the macro #XBT_TEST_SUITE, and populate it with a bunch of +actual tests with the macro #XBT_TEST_UNIT (sorry for the mischosen +names if you are used to junit). Just look at the dynar example (or +any other) to see how it works in practice. Do not hesitate to stress +test your code this way, but make sure that it runs reasonably fast, +or nobody will run "ctest" before commiting code. + +For more details on how the tests are extracted from the module +source, check the tools/sg_unit_extractor.pl script directly. + + +\section inside_tests_add_integration Adding integration tests + +TESH (the TEsting SHell) is the test runner that we wrote for our +integration tests. It is distributed with the SimGrid source file, and +even comes with a man page. TESH ensures that the output produced by a +command perfectly matches the expected output. This is very precious +to ensure that no change modifies the timings computed by the models +without notice. + +To add a new integration test, you thus have 3 things to do: + + - Write the code exercising the feature you target. You should + strive to make this code clear, well documented and informative for + the users. If you manage to do so, put this somewhere under + examples/ and modify the cmake files as explained on this page: + \ref inside_cmake_examples. If you feel like you should write a + torture test that is not interesting to the users (because nobody + would sainly write something similar in user code), then put it under + teshsuite/ somewhere. + - Write the tesh file, containing the command to run, the + provided input (if any, but almost no SimGrid test provide such an + input) and the expected output. Check the tesh man page for more + details. \n + Tesh is sometimes annoying as you have to ensure that the expected + output will always be exactly the same. In particular, your should + not output machine dependent informations, nor memory adresses as + they would change on each run. Several steps can be used here, such + as the obfucation of the memory adresses unless the verbose logs + are displayed (using the #XBT_LOG_ISENABLED() macro), or the + modification of the log formats to hide the timings when they + depend on the host machine. + - Add your test in the cmake infrastructure. For that, modify + the file /tools/cmake/Tests.cmake. Make sure to + pick a wise name for your test. It is often useful to check a + category of tests together. The only way to do so in ctest is to + use the -R argument that specifies a regular expression that the + test names must match. For example, you can run all MSG test with + "ctest -R msg". That explains the importance of the test names. + +Once the name is chosen, create a new test by adding a line similar to +the following (assuming that you use tesh as expected). + +\verbatim +# Usage: ADD_TEST(test-name ${CMAKE_BINARY_DIR}/bin/tesh ) +# option --setenv bindir set the directory containing the binary +# --setenv srcdir set the directory containing the source file +# --cd set the working directory +ADD_TEST(my-test-name ${CMAKE_BINARY_DIR}/bin/tesh + --setenv bindir=${CMAKE_BINARY_DIR}/examples/my-test/ + --setenv srcdir=${CMAKE_HOME_DIRECTORY}/examples/my-test/ + --cd ${CMAKE_HOME_DIRECTORY}/examples/my-test/ + ${CMAKE_HOME_DIRECTORY}/examples/msg/io/io.tesh +) +\endverbatim + +As usual, you must run "make distcheck" after modifying the cmake files, +to ensure that you did not forget any files in the distributed archive. + +\section inside_tests_ci Continous Integration + +We use several systems to automatically test SimGrid with a large set +of parameters, across as many platforms as possible. +We use Jenkins on Inria +servers as a workhorse: it runs all of our tests for many +configurations. It takes a long time to answer, and it often reports +issues but when it's green, then you know that SimGrid is very fit! +We use Travis to +quickly run some tests on Linux and Mac. It answers quickly but may +miss issues. And we use AppVeyor +to build and somehow test SimGrid on windows. + +\subsection inside_tests_jenkins Jenkins on the Inria CI servers + +You should not have to change the configuration of the Jenkins tool +yourself, although you could have to change the slaves' configuration +using the CI interface of INRIA -- +refer to the CI documentation. + +The result can be seen here: https://ci.inria.fr/simgrid/ + +We have 3 projects on Jenkins: +\li SimGrid-Multi + is the main project, running the tests that we spoke about.\n It is + configured (on Jenkins) to run the script tools/jenkins/build.sh +\li SimGrid-DynamicAnalysis + runs the tests both under valgrind to find the memory errors and + under gcovr to report the achieved test coverage.\n It is configured + (on Jenkins) to run the script tools/jenkins/DynamicAnalysis.sh +\li SimGrid-Windows + is an ongoing attempt to get Windows tested on Jenkins too. + +In each case, SimGrid gets built in +/builds/workspace/$PROJECT/build_mode/$CONFIG/label/$SERVER/build +with $PROJECT being for instance "SimGrid-Multi", $CONFIG "DEBUG" or +"ModelChecker" and $SERVER for instance "simgrid-fedora20-64-clang". + +If some configurations are known to fail on some systems (such as +model-checking on non-linux systems), go to your Project and click on +"Configuration". There, find the field "combination filter" (if your +interface language is English) and tick the checkbox; then add a +groovy-expression to disable a specific configuration. For example, in +order to disable the "ModelChecker" build on host +"small-freebsd-64-clang", use: + +\verbatim +(label=="small-freebsd-64-clang").implies(build_mode!="ModelChecker") +\endverbatim + +\subsection inside_tests_travis Travis + +Travis is a free (as in free beer) Continuous Integration system that +open-sourced project can use freely. It is very well integrated in the +GitHub ecosystem. There is a plenty of documentation out there. Our +configuration is in the file .travis.yml as it should be, and the +result is here: https://travis-ci.org/mquinson/simgrid + +\subsection inside_tests_appveyor AppVeyor + +AppVeyor aims at becoming the Travis of Windows. It is maybe less +mature than Travis, or maybe it is just that I'm less trained in +Windows. Our configuration is in the file appveyor.yml as it should +be, and the result is here: https://ci.appveyor.com/project/mquinson/simgrid + +It should be noted that I miserably failed to use the environment +provided by AppVeyor, since SimGrid does not build with Microsoft +Visual Studio. Instead, we download a whole development environment +from the internet at each build. That's an archive of already compiled +binaries that are unpacked on the appveyor systems each time we start. +We re-use the ones from the +symengine +project. Thanks to them for compiling sane tools and constituting that +archive, it saved my mind! + +*/ diff --git a/doc/doxygen/internals.doc b/doc/doxygen/internals.doc index e3ecacd81f..d6462287f9 100644 --- a/doc/doxygen/internals.doc +++ b/doc/doxygen/internals.doc @@ -18,11 +18,11 @@ SimGrid project, ie how to extend the framework in any way, how the automatic tests are run, and so on. These informations are split on several pages, as follows: + - @subpage inside_tests - @subpage inside_doxygen - @subpage inside_extending - @subpage inside_cmake - @subpage inside_release - - @subpage inside_ci \htmlonly diff --git a/tools/cmake/DefinePackages.cmake b/tools/cmake/DefinePackages.cmake index 4a08a383d8..b5af8fe838 100644 --- a/tools/cmake/DefinePackages.cmake +++ b/tools/cmake/DefinePackages.cmake @@ -877,7 +877,7 @@ set(DOC_SOURCES doc/doxygen/header.html doc/doxygen/help.doc doc/doxygen/index.doc - doc/doxygen/inside_ci.doc + doc/doxygen/inside_tests.doc doc/doxygen/inside_cmake.doc doc/doxygen/inside_doxygen.doc doc/doxygen/inside_extending.doc diff --git a/tools/sg_unit_extractor.pl b/tools/sg_unit_extractor.pl index 6502ccfa73..9cbc2f5aa5 100755 --- a/tools/sg_unit_extractor.pl +++ b/tools/sg_unit_extractor.pl @@ -1,11 +1,12 @@ #! /usr/bin/env perl -# Copyright (c) 2005-2012, 2014. The SimGrid Team. -# All rights reserved. +# Copyright (c) 2005-2012, 2014. The SimGrid Team. All rights reserved. # This program is free software; you can redistribute it and/or modify it # under the terms of the license (GNU LGPL) which comes with this package. + + use strict; use strict; @@ -16,11 +17,12 @@ my $progname="sg_unit_extractor"; sub usage($) { my $ret; - print "USAGE: $progname [--root=part/to/cut] [--path=where/to/search NOT WORKING] [--outdir=where/to/generate/files] infile [infile+]\n"; + print "USAGE: $progname [--root=part/to/cut] [--outdir=where/to/generate/files] infile [infile+]\n\n"; + print "This program is in charge of extracting the unit tests out of the SimGrid source code.\n"; + print "See http://simgrid.gforge.inria.fr/doc/latest/inside_tests.html for more details.\n"; exit $ret; } -my $path=undef; my $outdir=undef; my $root; my $help; @@ -29,7 +31,6 @@ Getopt::Long::config('permute','no_getopt_compat', 'no_auto_abbrev'); GetOptions( 'help|h' => sub {usage(0)}, 'root=s' =>\$root, - 'path=s' =>\$path, 'outdir=s' =>\$outdir) or usage(1); usage(1) if (scalar @ARGV == 0);