From: David Glesser Date: Thu, 1 Sep 2016 16:19:32 +0000 (+0200) Subject: Replace tesh.pl by pesh.py a much better version of tesh in python. X-Git-Tag: v3_14~449^2~1^2~2 X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/1951629e2f269636b75d56de2e73f6cd048a21e8?ds=sidebyside Replace tesh.pl by pesh.py a much better version of tesh in python. --- diff --git a/examples/msg/maestro-set/maestro-set.tesh b/examples/msg/maestro-set/maestro-set.tesh index 08ab8262e8..c5da2fa305 100644 --- a/examples/msg/maestro-set/maestro-set.tesh +++ b/examples/msg/maestro-set/maestro-set.tesh @@ -1,6 +1,7 @@ #! ./tesh -! output sort 19 $ $SG_TEST_EXENV maestro-set/maestro-set$EXEEXT ${srcdir:=.}/../platforms/small_platform.xml ${srcdir:=.}/maestro-set/maestro-set_d.xml "--log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n" +! output sort 19 +$ $SG_TEST_EXENV maestro-set/maestro-set$EXEEXT ${srcdir:=.}/../platforms/small_platform.xml ${srcdir:=.}/maestro-set/maestro-set_d.xml "--log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n" > [ 0.000000] (1:sender@Tremblay) sender > [ 0.000000] (1:sender@Tremblay) host = Jupiter > [ 0.000000] (1:sender@Tremblay) task_la->data = 0.000000e+00 diff --git a/tools/cmake/Documentation.cmake b/tools/cmake/Documentation.cmake index 431c28ae32..43b5375e2c 100644 --- a/tools/cmake/Documentation.cmake +++ b/tools/cmake/Documentation.cmake @@ -118,7 +118,7 @@ endif() add_custom_target(manpages ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${MANPAGE_DIR} COMMAND pod2man ${CMAKE_HOME_DIRECTORY}/tools/simgrid_update_xml.pl > ${MANPAGE_DIR}/simgrid_update_xml.1 - COMMAND pod2man ${CMAKE_HOME_DIRECTORY}/tools/tesh/tesh.pl > ${MANPAGE_DIR}/tesh.1 + COMMAND pod2man ${CMAKE_HOME_DIRECTORY}/tools/tesh/tesh.pod > ${MANPAGE_DIR}/tesh.1 COMMENT "Generating manpages" ) install(FILES diff --git a/tools/cmake/Tests.cmake b/tools/cmake/Tests.cmake index 953fabdfbc..f15edfd240 100644 --- a/tools/cmake/Tests.cmake +++ b/tools/cmake/Tests.cmake @@ -6,7 +6,7 @@ IF(enable_smpi AND NOT WIN32) execute_process(COMMAND chmod a=rwx ${CMAKE_BINARY_DIR}/bin/smpirun) ENDIF() -SET(TESH_COMMAND ${PERL_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/tesh) +SET(TESH_COMMAND ${CMAKE_BINARY_DIR}/bin/tesh) IF(CMAKE_HOST_WIN32) SET(TESH_OPTION ${TESH_OPTION} --timeout 50) ENDIF() diff --git a/tools/tesh/CMakeLists.txt b/tools/tesh/CMakeLists.txt index e4fdb3d041..c9fb49590b 100644 --- a/tools/tesh/CMakeLists.txt +++ b/tools/tesh/CMakeLists.txt @@ -1,11 +1,6 @@ -configure_file("${CMAKE_HOME_DIRECTORY}/tools/tesh/tesh.pl" +configure_file("${CMAKE_HOME_DIRECTORY}/tools/tesh/tesh.py" "${CMAKE_BINARY_DIR}/bin/tesh" @ONLY IMMEDIATE) - file(COPY ${CMAKE_HOME_DIRECTORY}/tools/cmake/scripts/Diff.pm - DESTINATION ${CMAKE_BINARY_DIR}/bin - FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ) - file(COPY ${CMAKE_HOME_DIRECTORY}/tools/cmake/scripts/IPC - DESTINATION ${CMAKE_BINARY_DIR}/bin) foreach(x setenv set-output-ignore set-output-sort set-return set-timeout background basic bg-basic bg-set-signal catch-return catch-signal catch-timeout catch-wrong-output cd IO-bigsize IO-broken-pipe IO-orders) @@ -16,4 +11,4 @@ foreach(x setenv set-output-ignore set-output-sort set-return set-timeout backgr endforeach() set(tesh_files ${tesh_files} PARENT_SCOPE) -set(bin_files ${bin_files} ${CMAKE_CURRENT_SOURCE_DIR}/tesh.pl ${CMAKE_CURRENT_SOURCE_DIR}/generate_tesh PARENT_SCOPE) +set(bin_files ${bin_files} ${CMAKE_CURRENT_SOURCE_DIR}/tesh.py ${CMAKE_CURRENT_SOURCE_DIR}/generate_tesh PARENT_SCOPE) diff --git a/tools/tesh/IO-orders.tesh b/tools/tesh/IO-orders.tesh index 7453ff88c8..e2c0a25d71 100644 --- a/tools/tesh/IO-orders.tesh +++ b/tools/tesh/IO-orders.tesh @@ -8,7 +8,7 @@ p Order: in, out, cmd > Test suite from stdin > [(stdin):3] cat > Test suite from stdin OK -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh p Order: out, in, cmd < > TOTO @@ -17,7 +17,7 @@ p Order: out, in, cmd > Test suite from stdin > [(stdin):3] cat > Test suite from stdin OK -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh p Order: out, cmd, in < > TOTO @@ -26,7 +26,7 @@ p Order: out, cmd, in > Test suite from stdin > [(stdin):2] cat > Test suite from stdin OK -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh p Order: in, cmd, out < < TOTO @@ -35,7 +35,7 @@ p Order: in, cmd, out > Test suite from stdin > [(stdin):2] cat > Test suite from stdin OK -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh p Order: cmd, out, in < $ cat @@ -44,7 +44,7 @@ p Order: cmd, out, in > Test suite from stdin > [(stdin):1] cat > Test suite from stdin OK -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh p Order: cmd, in, out < $ cat @@ -53,4 +53,4 @@ p Order: cmd, in, out > Test suite from stdin > [(stdin):1] cat > Test suite from stdin OK -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh diff --git a/tools/tesh/basic.tesh b/tools/tesh/basic.tesh index 7f3d8c6672..db3d8f7e1e 100644 --- a/tools/tesh/basic.tesh +++ b/tools/tesh/basic.tesh @@ -13,7 +13,7 @@ $ cat tmp_fich ! output ignore < $ cat tmp_fich < > TUTU TOTO -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh $ rm tmp_fich @@ -77,14 +77,14 @@ $ sed 's/_/ /' < < _x < $ sed 's/_/ /' < > x -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh ! expect return 2 ! output ignore < < x < $ cat < > x -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh p * trailing spaces... < x_ @@ -96,14 +96,14 @@ $ sed 's/_/ /' < < x_ < $ sed 's/_/ /' < > x -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh ! expect return 2 ! output ignore < < x < $ cat < > x -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh p * empty lines... < a @@ -122,7 +122,7 @@ $ cat < $ cat < > a < > c -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh ! expect return 2 ! output ignore @@ -132,7 +132,7 @@ $ perl ${bindir:=.}/tesh < > a < > < > c -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh < < b @@ -150,7 +150,7 @@ $ cat < $ cat < > b < > c -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh ! expect return 2 ! output ignore @@ -160,4 +160,4 @@ $ perl ${bindir:=.}/tesh < > < > b < > c -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh diff --git a/tools/tesh/bg-set-signal.tesh b/tools/tesh/bg-set-signal.tesh index d6e9796326..069496dde1 100644 --- a/tools/tesh/bg-set-signal.tesh +++ b/tools/tesh/bg-set-signal.tesh @@ -6,6 +6,7 @@ $ cmake -E remove_directory temp_testdir-bg-set-signal $ mkdir temp_testdir-bg-set-signal $ cd temp_testdir-bg-set-signal + < kill 'SEGV', $$; $ mkfile segfault.pl diff --git a/tools/tesh/catch-return.tesh b/tools/tesh/catch-return.tesh index 3a96bf0fd1..b72c2d8c7b 100644 --- a/tools/tesh/catch-return.tesh +++ b/tools/tesh/catch-return.tesh @@ -2,10 +2,9 @@ # This suite builds and uses a program returning 1. # tesh must detect this condition and report the issue. -! expect return 41 +! expect return 2 < $ perl -e "exit 1" -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh > Test suite from stdin > [(stdin):1] perl -e "exit 1" > Test suite `(stdin)': NOK (<(stdin):1> returned code 1) -> <(stdin):1> No output so far. diff --git a/tools/tesh/catch-signal.tesh b/tools/tesh/catch-signal.tesh index be19f344c8..e835b01c64 100644 --- a/tools/tesh/catch-signal.tesh +++ b/tools/tesh/catch-signal.tesh @@ -6,6 +6,7 @@ $ cmake -E remove_directory temp_testdir-catch-signal $ mkdir temp_testdir-catch-signal $ cd temp_testdir-catch-signal + < kill 'SEGV', $$; $ mkfile segfault.pl @@ -14,13 +15,12 @@ p Check that we notice when SEGV is raised $ perl segfault.pl p Check that we return the expected return value on SEGV -! expect return 15 +! expect return 11 < $ perl segfault.pl -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh > Test suite from stdin > [(stdin):1] perl segfault.pl > Test suite `(stdin)': NOK (<(stdin):1> got signal SIGSEGV) -> <(stdin):1> No output so far. $ cd .. $ cmake -E remove_directory temp_testdir-catch-signal diff --git a/tools/tesh/catch-timeout.tesh b/tools/tesh/catch-timeout.tesh index 00c5ff660d..5cde56c2d3 100644 --- a/tools/tesh/catch-timeout.tesh +++ b/tools/tesh/catch-timeout.tesh @@ -9,5 +9,4 @@ > Test suite from stdin > [(stdin):2] sleep 6 > Test suite `(stdin)': NOK (<(stdin):2> timeout after 1 sec) -> <(stdin):2> No output so far. -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh diff --git a/tools/tesh/catch-wrong-output.tesh b/tools/tesh/catch-wrong-output.tesh index 1dcd26d6dd..05ccc57ef0 100644 --- a/tools/tesh/catch-wrong-output.tesh +++ b/tools/tesh/catch-wrong-output.tesh @@ -6,10 +6,13 @@ p This tests whether TESH detects wrong outputs < > TOTO < < TUTU < $ cat -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh > Test suite from stdin > [(stdin):3] cat > Output of <(stdin):3> mismatch: -> - TOTO -> + TUTU +>--- expected +>+++ obtained +>@@ -1 +1 @@ +> -TOTO +> +TUTU > Test suite `(stdin)': NOK (<(stdin):3> output mismatch) diff --git a/tools/tesh/cd.tesh b/tools/tesh/cd.tesh index db9137ee5e..de606a32c1 100644 --- a/tools/tesh/cd.tesh +++ b/tools/tesh/cd.tesh @@ -15,7 +15,7 @@ $ ls > Test suite from stdin > Chdir to toto failed: No such file or directory > Test suite `(stdin)': NOK (system error) -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh # The next command checks that there is a testdir_temp-cd in the upper directory, # ie that mkdir and cd both worked. diff --git a/tools/tesh/set-output-ignore.tesh b/tools/tesh/set-output-ignore.tesh index 120b620c5e..04da086a7a 100644 --- a/tools/tesh/set-output-ignore.tesh +++ b/tools/tesh/set-output-ignore.tesh @@ -6,7 +6,7 @@ p This tests whether TESH accepts to ignore command output < > TOTO < < TUTU < $ cat -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh > Test suite from stdin > [(stdin):4] cat > (ignoring the output of <(stdin):4> as requested) diff --git a/tools/tesh/set-output-sort.tesh b/tools/tesh/set-output-sort.tesh index f0abb75614..beb475c99c 100644 --- a/tools/tesh/set-output-sort.tesh +++ b/tools/tesh/set-output-sort.tesh @@ -9,85 +9,85 @@ p This tests whether TESH correctly sorts command output < ! output sort < $ true < -< $ printf 'profiling: foo\\n' +< $ printf 'profiling: foo\n' < < $ printf 'profiling: foo' < < ! output sort -< $ printf 'profiling: foo\\n' +< $ printf 'profiling: foo\n' < < ! output sort < $ printf 'profiling: foo' < -< $ printf 'a\\nb\\nc\\nd\\n' +< $ printf 'a\nb\nc\nd\n' < > a < > b < > c < > d < -< $ printf 'a\\nb\\nc\\nd' +< $ printf 'a\nb\nc\nd' < > a < > b < > c < > d < < ! output sort -< $ printf 'c\\nd\\nb\\na\\n' +< $ printf 'c\nd\nb\na\n' < > a < > b < > c < > d < < ! output sort -< $ printf 'c\\nd\\nb\\na' +< $ printf 'c\nd\nb\na' < > a < > b < > c < > d < -< $ printf 'a\\nprofiling: foo\\nprofiling: bar\\nb\\nc\\nd\\nprofiling: baz\\n' +< $ printf 'a\nprofiling: foo\nprofiling: bar\nb\nc\nd\nprofiling: baz\n' < > a < > b < > c < > d < -< $ printf 'a\\nprofiling: foo\\nprofiling: bar\\nb\\nc\\nd\\nprofiling: baz' +< $ printf 'a\nprofiling: foo\nprofiling: bar\nb\nc\nd\nprofiling: baz' < > a < > b < > c < > d < < ! output sort -< $ printf 'c\\nprofiling: foo\\nprofiling: bar\\nd\\nb\\na\\nprofiling: baz\\n' +< $ printf 'c\nprofiling: foo\nprofiling: bar\nd\nb\na\nprofiling: baz\n' < > a < > b < > c < > d < < ! output sort -< $ printf 'c\\nprofiling: foo\\nprofiling: bar\\nd\\nb\\na\\nprofiling: baz' +< $ printf 'c\nprofiling: foo\nprofiling: bar\nd\nb\na\nprofiling: baz' < > a < > b < > c < > d -$ perl ${bindir:=.}/tesh --enable-coverage +$ ${bindir:=.}/tesh --enable-coverage > Enable coverage > Test suite from stdin > [(stdin):1] Test sorting and filtering of output > [(stdin):3] true > [(stdin):6] true -> [(stdin):8] printf 'profiling: foo\\n' +> [(stdin):8] printf 'profiling: foo\n' > [(stdin):10] printf 'profiling: foo' -> [(stdin):13] printf 'profiling: foo\\n' +> [(stdin):13] printf 'profiling: foo\n' > [(stdin):16] printf 'profiling: foo' -> [(stdin):18] printf 'a\\nb\\nc\\nd\\n' -> [(stdin):24] printf 'a\\nb\\nc\\nd' -> [(stdin):31] printf 'c\\nd\\nb\\na\\n' -> [(stdin):38] printf 'c\\nd\\nb\\na' -> [(stdin):44] printf 'a\\nprofiling: foo\\nprofiling: bar\\nb\\nc\\nd\\nprofiling: baz\\n' -> [(stdin):50] printf 'a\\nprofiling: foo\\nprofiling: bar\\nb\\nc\\nd\\nprofiling: baz' -> [(stdin):57] printf 'c\\nprofiling: foo\\nprofiling: bar\\nd\\nb\\na\\nprofiling: baz\\n' -> [(stdin):64] printf 'c\\nprofiling: foo\\nprofiling: bar\\nd\\nb\\na\\nprofiling: baz' +> [(stdin):18] printf 'a\nb\nc\nd\n' +> [(stdin):24] printf 'a\nb\nc\nd' +> [(stdin):31] printf 'c\nd\nb\na\n' +> [(stdin):38] printf 'c\nd\nb\na' +> [(stdin):44] printf 'a\nprofiling: foo\nprofiling: bar\nb\nc\nd\nprofiling: baz\n' +> [(stdin):50] printf 'a\nprofiling: foo\nprofiling: bar\nb\nc\nd\nprofiling: baz' +> [(stdin):57] printf 'c\nprofiling: foo\nprofiling: bar\nd\nb\na\nprofiling: baz\n' +> [(stdin):64] printf 'c\nprofiling: foo\nprofiling: bar\nd\nb\na\nprofiling: baz' > Test suite from stdin OK p Check the Right Prefix Length (19) for "output sort" @@ -113,7 +113,7 @@ $ cat < $ cat < > 123456789012345678 A line < > 123456789012345678 B line -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh p Check user-defined prefix length for "output sort" ! output sort 5 @@ -138,4 +138,4 @@ $ cat < $ cat < > 000 A line < > 000 B line -$ perl ${bindir:=.}/tesh +$ ${bindir:=.}/tesh diff --git a/tools/tesh/tesh.pl b/tools/tesh/tesh.pl deleted file mode 100755 index fe1d60dbf0..0000000000 --- a/tools/tesh/tesh.pl +++ /dev/null @@ -1,814 +0,0 @@ -#! /usr/bin/env perl - -# Copyright (c) 2012-2015. The SimGrid Team. -# All rights reserved. - -# This program is free software; you can redistribute it and/or modify it -# under the terms of the license (GNU LGPL) which comes with this package. -eval 'exec perl -S $0 ${1+"$@"}' - if $running_under_some_shell; - -# If you change this file, please stick to the formatting you got with: -# perltidy --backup-and-modify-in-place --maximum-line-length=180 --output-line-ending=unix --cuddled-else - -=encoding UTF-8 - -=head1 NAME - -tesh -- testing shell - -=head1 SYNOPSIS - -B [I]... I - -=head1 DESCRIPTION - -Tesh is the testing shell, a specialized shell for running tests. It -provides the specified input to the tested commands, and check that -they produce the expected output and return the expected value. - -=head1 OPTIONS - - --cd some/directory : ask tesh to switch the working directory before - launching the tests - --setenv var=value : set a specific environment variable - --cfg arg : add parameter --cfg=arg to each command line - --log arg : add parameter --log=arg to each command line - --enable-coverage : ignore output lines starting with "profiling:" - --enable-sanitizers : ignore output lines starting with containing warnings - -=head1 TEST SUITE FILE SYTAX - -A test suite is composed of one or several I separated -by empty lines, each of them being composed of a command to run, its -input text and the expected output. - -The first char of each line specifies the type of line according to -the following list. The second char of each line is ignored. - - `$' command to run in foreground - `&' command to run in background - - `<' input to pass to the command - `>' output expected from the command - - `!' metacommand, which can be one of: - `timeout' |no - `expect signal' - `expect return' - `output' - `setenv =' - - `p' an informative message to print - -If the expected output do not match the produced output, or if the -command did not end as expected, Tesh provides an error message (see -the OUTPUT section below) and stops. - -=head2 Command blocks examples - -In a given command block, you can declare the command, its input and -its expected output in the order that you see fit. - - $ cat - < TOTO - > TOTO - - > TOTO - $ cat - < TOTO - - > TOTO - < TOTO - $ cat - -You can group several commands together, provided that they don't have -any input nor output. - - $ mkdir testdir - $ cd testdir - -=head2 Enforcing the command return code - -By default, Tesh enforces that the tested command returns 0. If not, -it fails with an appropriate message and returns I itself. - -You specify that a given command block is expected to return another -code as follows: - - # This command MUST return 42 - ! expect return 42 - $ sh -e "exit 42" - -The I construct applies only to the next command block. - -=head2 Commands that are expected to raise signals - -By default, Tesh detects when the command is killed by a signal (such -as SEGV on segfaults). This is usually unexpected and unfortunate. But -if not, you can specify that a given command block is expected to fail -with a signal as follows: - - # This command MUST raise a segfault - ! expect signal SIGSEGV - $ ./some_failing_code - -The I construct applies only to the next command block. - -=head2 Timeouts - -By default, no command is allowed to run more than 5 seconds. You can -change this value as follows: - - # Allow some more time to the command - ! timeout 60 - $ ./some_longer_command - -You can also disable the timeout completely by passing "no" as a value: - - # This command will never timeout - ! timeout no - $ ./some_very_long_but_safe_command - -=head2 Setting environment variables - -You can modify the environment of the tested commands as follows: - - ! setenv PATH=/bin - $ my_command - -=head2 Not enforcing the expected output - -By default, the commands output is matched against the one expected, -and an error is raised on discrepancy. Metacommands to change this: - -=over 4 - -=item output ignore - -The output is completely discarded. - -=item output display - -The output is displayed, but no error is issued if it differs from the -expected output. - -=item output sort - -The output is sorted before comparison (see next section). - -=back - -=head2 Sorting output - -If the order of the command output changes between runs, you want to -sort it before enforcing that it is exactly what you expect. In -SimGrid for example, this happens when parallel execution is -activated: User processes are run in parallel at each timestamp, and -the output is not reproducible anymore. Until you sort the lines. - -You can sort the command output as follows: - - ! output sort - $ ./some_multithreaded_command - -Sorting lines this ways often makes the tesh output very intricate, -complicating the error analysis: the process logical order is defeated -by the lexicographical sort. - -The solution is to prefix each line of your output with temporal -information so that lines can be grouped by timestamps. The -lexicographical sort then only applies to lines that occurred at the -same timestamp. Here is a SimGrid example: - - # Sort only lines depending on the first 19 chars - ! output sort 19 - $ ./some_simgrid_simulator --log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n - -This approach may seem surprizing at the first glance but it does its job: - -=over 4 - -=item Every timestamps remain separated, as it should; - -=item In each timestamp, the output order of processes become - reproducible: that's the lexicographical order of their name; - -=item For each process, the order of its execution is preserved: its - messages within a given timestamp are not reordered. - -=back - -That way, tesh can do its job (no false positive, no false negative) -despite the unpredictable order of executions of processes within a -timestamp, and reported errors remain easy to analyze (execution of a -given process preserved). - -This example is very SimGrid oriented, but the feature could even be -usable by others, who knows? - - -=head1 BUILTIN COMMANDS - -=head2 mkfile: creating a file - -This command creates a file of the name provided as argument, and adds -the content it gets as input. - - $ mkfile myFile - > some content - > to the file - -It is not possible to use the cat command, as one would expect, -because stream redirections are currently not implemented in Tesh. - -=head1 BUGS, LIMITATIONS AND POSSIBLE IMPROVEMENTS - -The main limitation is the lack of stream redirections in the commands -(">", "<" and "|" shell constructs and friends). The B builtin -command makes this situation bearable. - -It would be nice if we could replace the tesh file completely with -command line flags when the output is not to be verified. - -=cut - -BEGIN { - # Disabling IPC::Run::Debug saves tons of useless calls. - $ENV{'IPCRUNDEBUG'} = 'none' - unless exists $ENV{'IPCRUNDEBUG'}; -} - -my $enable_coverage = 0; -my $enable_sanitizers = 0; -my $diff_tool = 0; -my $diff_tool_tmp_fh = 0; -my $diff_tool_tmp_filename = 0; -my $sort_prefix = -1; -my $tesh_file; -my $tesh_name; -my $error = 0; -my $exitcode = 0; -my @bg_cmds; -my (%environ); -$SIG{'PIPE'} = 'IGNORE'; - -my $path = $0; -$path =~ s|[^/]*$||; -push @INC, $path; - -use lib "@CMAKE_BINARY_DIR@/bin"; - -use Diff qw(diff); # postpone a bit to have time to change INC - -use Getopt::Long qw(GetOptions); -use strict; -use Text::ParseWords; -use IPC::Run qw(start run timeout finish); -use IO::File; -use English; - -#### -#### Portability bits for windows -#### - -use constant RUNNING_ON_WINDOWS => ( $OSNAME =~ /^(?:mswin|dos|os2)/oi ); -use POSIX qw(:sys_wait_h WIFEXITED WIFSIGNALED WIFSTOPPED WEXITSTATUS WTERMSIG WSTOPSIG - :signal_h SIGINT SIGTERM SIGKILL SIGABRT SIGSEGV); - -BEGIN { - if (RUNNING_ON_WINDOWS) { # Missing on windows - *WIFEXITED = sub { not $_[0] & 127 }; - *WEXITSTATUS = sub { $_[0] >> 8 }; - *WIFSIGNALED = sub { ( $_[0] & 127 ) && ( $_[0] & 127 != 127 ) }; - *WTERMSIG = sub { $_[0] & 127 }; - - # used on the command lines - $environ{'EXEEXT'} = ".exe"; - } -} - - -#### -#### Command line option handling -#### - -my %opts = ( "debug" => 0, - "timeout" => 5, # No command should run any longer than 5 seconds by default - ); - -Getopt::Long::config( 'bundling', 'no_getopt_compat', 'no_auto_abbrev' ); -GetOptions( - 'debug|d' => \$opts{"debug"}, - - 'difftool=s' => \$diff_tool, - - 'cd=s' => sub { cd_cmd( $_[1] ) }, - 'timeout=s' => \$opts{'timeout'}, - 'setenv=s' => sub { setenv_cmd( $_[1] ) }, - 'cfg=s' => sub { $opts{'cfg'} .= " --cfg=$_[1]" }, - 'log=s' => sub { $opts{'log'} .= " --log=$_[1]" }, - 'enable-coverage+' => \$enable_coverage, - 'enable-sanitizers+' => \$enable_sanitizers, -); - -$tesh_file = pop @ARGV; -$tesh_name = $tesh_file; -$tesh_name =~ s|^.*?/([^/]*)$|$1|; - -print "Enable coverage\n" if ($enable_coverage); -print "Enable sanitizers\n" if ($enable_sanitizers); - -if ($diff_tool) { - use File::Temp qw/ tempfile /; - ( $diff_tool_tmp_fh, $diff_tool_tmp_filename ) = tempfile(); - print "New tesh: $diff_tool_tmp_filename\n"; -} - -if ( $tesh_file =~ m/(.*)\.tesh/ ) { - my $fullname = $tesh_file; - if (not ($fullname =~ m|^/|)) { # not absolute path - my $dir = qx,pwd,; - chomp($dir); - $fullname = "$dir/$fullname" - } - print "Test suite '$tesh_file'\n"; -} else { - $tesh_name = "(stdin)"; - print "Test suite from stdin\n"; -} - -########################################################################### - -sub exec_cmd { - my %cmd = %{ $_[0] }; - if ( $opts{'debug'} ) { - map { print "IN: $_\n" } @{ $cmd{'in'} }; - map { print "OUT: $_\n" } @{ $cmd{'out'} }; - print "CMD: $cmd{'cmd'}\n"; - } - - # substitute environment variables - foreach my $key ( keys %environ ) { - $cmd{'cmd'} = var_subst( $cmd{'cmd'}, $key, $environ{$key} ); - } - - # substitute remaining variables, if any - while ( $cmd{'cmd'} =~ /\$\{(\w+)(?::[=-][^}]*)?\}/ ) { - $cmd{'cmd'} = var_subst( $cmd{'cmd'}, $1, "" ); - } - while ( $cmd{'cmd'} =~ /\$(\w+)/ ) { - $cmd{'cmd'} = var_subst( $cmd{'cmd'}, $1, "" ); - } - - # add cfg and log options - $cmd{'cmd'} .= " $opts{'cfg'}" - if ( defined( $opts{'cfg'} ) && length( $opts{'cfg'} ) ); - $cmd{'cmd'} .= " $opts{'log'}" - if ( defined( $opts{'log'} ) && length( $opts{'log'} ) ); - - # finally trim any remaining space chars - $cmd{'cmd'} =~ s/^\s+//; - $cmd{'cmd'} =~ s/\s+$//; - - print "[$tesh_name:$cmd{'line'}] $cmd{'cmd'}\n"; - - $cmd{'return'} ||= 0; - $cmd{'timeout'} ||= $opts{'timeout'}; - - - ### - # exec the command line - - my @cmdline; - if(defined $ENV{VALGRIND_COMMAND}) { - push @cmdline, $ENV{VALGRIND_COMMAND}; - push @cmdline, split(" ", $ENV{VALGRIND_OPTIONS}); - if($cmd{'timeout'} ne 'no'){ - $cmd{'timeout'}=$cmd{'timeout'}*20 - } - } - push @cmdline, quotewords( '\s+', 0, $cmd{'cmd'} ); - my $input = defined($cmd{'in'})? join("\n",@{$cmd{'in'}}) : ""; - my $output = " " x 10240; $output = ""; # Preallocate 10kB, and reset length to 0 - $cmd{'got'} = \$output; - $cmd{'job'} = start \@cmdline, '<', \$input, '>&', \$output, - ($cmd{'timeout'} eq 'no' ? () : timeout($cmd{'timeout'})); - - if ( $cmd{'background'} ) { - # Just enqueue the job. It will be dealed with at the end - push @bg_cmds, \%cmd; - } else { - # Deal with its ending conditions right away - analyze_result( \%cmd ); - } -} - -sub analyze_result { - my %cmd = %{ $_[0] }; - $cmd{'timeouted'} = 0; # initialization - - # Wait for the end of the child process - ##### - eval { - finish( $cmd{'job'} ); - }; - if ($@) { # deal with the errors that occurred in the child process - if ($@ =~ /timeout/) { - $cmd{'job'}->kill_kill; - $cmd{'timeouted'} = 1; - } elsif ($@ =~ /^ack / and $@ =~ /pipe/) { # IPC::Run is not very expressive about the pipes that it gets :( - print STDERR "Tesh: Broken pipe (ignored).\n"; - } else { - die $@; # Don't know what it is, so let it go. - } - } - - # Gather information - #### - - # pop all output from executing child - my @got; - map { print "GOT: $_\n" } ${$cmd{'got'}} if $opts{'debug'}; - foreach my $got ( split("\n", ${$cmd{'got'}}) ) { - $got =~ s/\r//g; - chomp $got; - print $diff_tool_tmp_fh "> $got\n" if ($diff_tool); - - unless (( $enable_coverage and $got =~ /^profiling:/ ) or - ( $enable_sanitizers and $got =~ m/WARNING: ASan doesn't fully support/) or - ( $got =~ m/Unable to clean temporary file C:/)) # Crude hack to ignore cruft from Java on Windows - { - push @got, $got; - } - } - - # How did the child process terminate? - my $status = $?; - $cmd{'gotret'} = "Unparsable status. Please report this tesh bug."; - if ( $cmd{'timeouted'} ) { - $cmd{'gotret'} = "timeout after $cmd{'timeout'} sec"; - $error = 1; - $exitcode = 3; - } elsif ( WIFEXITED($status) ) { - $exitcode = WEXITSTATUS($status) + 40; - $cmd{'gotret'} = "returned code " . WEXITSTATUS($status); - } elsif ( WIFSIGNALED($status) ) { - my $code; - if ( WTERMSIG($status) == SIGINT ) { $code = "SIGINT"; } - elsif ( WTERMSIG($status) == SIGTERM ) { $code = "SIGTERM"; } - elsif ( WTERMSIG($status) == SIGKILL ) { $code = "SIGKILL"; } - elsif ( WTERMSIG($status) == SIGABRT ) { $code = "SIGABRT"; } - elsif ( WTERMSIG($status) == SIGSEGV ) { $code = "SIGSEGV"; } - $exitcode = WTERMSIG($status) + 4; - $cmd{'gotret'} = "got signal $code"; - } - - # How was it supposed to terminate? - my $wantret; - if ( defined( $cmd{'expect'} ) and ( $cmd{'expect'} ne "" ) ) { - $wantret = "got signal $cmd{'expect'}"; - } else { - $wantret = "returned code " . ( defined( $cmd{'return'} ) ? $cmd{'return'} : 0 ); - } - - # Enforce the outcome - #### - - # Did it end as expected? - if ( $cmd{'gotret'} ne $wantret ) { - $error = 1; - my $msg = "Test suite `$tesh_name': NOK (<$tesh_name:$cmd{'line'}> $cmd{'gotret'})\n"; - if ( scalar @got ) { - $msg = $msg . "Output of <$tesh_name:$cmd{'line'}> so far:\n"; - map { $msg .= "|| $_\n" } @got; - } else { - $msg .= "<$tesh_name:$cmd{'line'}> No output so far.\n"; - } - print STDERR "$msg"; - } - - # Does the output match? - if ( $cmd{'sort'} ) { - sub mysort { - substr( $a, 0, $sort_prefix ) cmp substr( $b, 0, $sort_prefix ); - } - use sort 'stable'; - if ( $sort_prefix > 0 ) { - @got = sort mysort @got; - } else { - @got = sort @got; - } - while ( @got and $got[0] eq "" ) { - shift @got; - } - - # Sort the expected output too, to make tesh files easier to write for humans - if ( defined( $cmd{'out'} ) ) { - if ( $sort_prefix > 0 ) { - @{ $cmd{'out'} } = sort mysort @{ $cmd{'out'} }; - } else { - @{ $cmd{'out'} } = sort @{ $cmd{'out'} }; - } - while ( @{ $cmd{'out'} } and ${ $cmd{'out'} }[0] eq "" ) { - shift @{ $cmd{'out'} }; - } - } - } - - # Report the output if asked so or if it differs - if ( defined( $cmd{'output display'} ) ) { - print "[Tesh/INFO] Here is the (ignored) command output:\n"; - map { print "||$_\n" } @got; - } elsif ( defined( $cmd{'output ignore'} ) ) { - print "(ignoring the output of <$tesh_name:$cmd{'line'}> as requested)\n"; - } else { - my $diff = build_diff( \@{ $cmd{'out'} }, \@got ); - - if ( length $diff ) { - print "Output of <$tesh_name:$cmd{'line'}> mismatch" . ( $cmd{'sort'} ? " (even after sorting)" : "" ) . ":\n"; - map { print "$_\n" } split( /\n/, $diff ); - if ( $cmd{'sort'} ) { - print "WARNING: Both the observed output and expected output were sorted as requested.\n"; - print "WARNING: Output were only sorted using the $sort_prefix first chars.\n" - if ( $sort_prefix > 0 ); - print "WARNING: Use to sort by simulated date and process ID only.\n"; - - # print "----8<--------------- Begin of unprocessed observed output (as it should appear in file):\n"; - # map {print "> $_\n"} @{$cmd{'unsorted got'}}; - # print "--------------->8---- End of the unprocessed observed output.\n"; - } - - print "Test suite `$tesh_name': NOK (<$tesh_name:$cmd{'line'}> output mismatch)\n"; - exit 2; - } - } -} - -# parse tesh file -my $infh; # The file descriptor from which we should read the teshfile -if ( $tesh_name eq "(stdin)" ) { - $infh = *STDIN; -} else { - open $infh, $tesh_file - or die "[Tesh/CRITICAL] Unable to open $tesh_file: $!\n"; -} - -my %cmd; # everything about the next command to run -my $line_num = 0; -LINE: while ( not $error and defined( my $line = <$infh> )) { - chomp $line; - $line =~ s/\r//g; - - $line_num++; - print "[TESH/debug] $line_num: $line\n" if $opts{'debug'}; - - # deal with line continuations - while ( $line =~ /^(.*?)\\$/ ) { - my $next = <$infh>; - die "[TESH/CRITICAL] Continued line at end of file\n" - unless defined($next); - $line_num++; - chomp $next; - print "[TESH/debug] $line_num: $next\n" if $opts{'debug'}; - $line = $1 . $next; - } - - # If the line is empty, run any previously defined block and proceed to next line - unless ( $line =~ m/^(.)(.*)$/ ) { - if ( defined( $cmd{'cmd'} ) ) { - exec_cmd( \%cmd ); - %cmd = (); - } - print $diff_tool_tmp_fh "$line\n" if ($diff_tool); - next LINE; - } - - my ( $cmd, $arg ) = ( $1, $2 ); - print $diff_tool_tmp_fh "$line\n" if ( $diff_tool and $cmd ne '>' ); - $arg =~ s/^ //g; - $arg =~ s/\r//g; - $arg =~ s/\\\\/\\/g; - - # Deal with the lines that can contribute to the current command block - if ( $cmd =~ /^#/ ) { # comment - next LINE; - } elsif ( $cmd eq '>' ) { # expected result line - print "[TESH/debug] push expected result\n" if $opts{'debug'}; - push @{ $cmd{'out'} }, $arg; - next LINE; - - } elsif ( $cmd eq '<' ) { # provided input - print "[TESH/debug] push provided input\n" if $opts{'debug'}; - push @{ $cmd{'in'} }, $arg; - next LINE; - - } elsif ( $cmd eq 'p' ) { # comment - print "[$tesh_name:$line_num] $arg\n"; - next LINE; - - } - - # We dealt with all sort of lines that can contribute to a command block, so we have something else here. - # If we have something buffered, run it now and start a new block - if ( defined( $cmd{'cmd'} ) ) { - exec_cmd( \%cmd ); - %cmd = (); - } - - # Deal with the lines that must be placed before a command block - if ( $cmd eq '$' ) { # Command - if ( $arg =~ /^mkfile / ) { # "mkfile" command line - die "[TESH/CRITICAL] Output expected from mkfile command!\n" - if scalar @{ cmd { 'out' } }; - - $cmd{'arg'} = $arg; - $cmd{'arg'} =~ s/mkfile //; - mkfile_cmd( \%cmd ); - %cmd = (); - - } elsif ( $arg =~ /^\s*cd / ) { - die "[TESH/CRITICAL] Input provided to cd command!\n" - if scalar @{ cmd { 'in' } }; - die "[TESH/CRITICAL] Output expected from cd command!\n" - if scalar @{ cmd { 'out' } }; - - $arg =~ s/^ *cd //; - cd_cmd($arg); - %cmd = (); - - } else { # regular command - $cmd{'cmd'} = $arg; - $cmd{'line'} = $line_num; - } - - } elsif ( $cmd eq '&' ) { # background command line - die "[TESH/CRITICAL] mkfile cannot be run in background\n" - if ($arg =~ /^mkfile/); - die "[TESH/CRITICAL] cd cannot be run in background\n" - if ($arg =~ /^cd/); - - $cmd{'background'} = 1; - $cmd{'cmd'} = $arg; - $cmd{'line'} = $line_num; - - # Deal with the meta-commands - } elsif ( $line =~ /^! (.*)/) { - $line = $1; - - if ( $line =~ /^output sort/ ) { - $cmd{'sort'} = 1; - if ( $line =~ /^output sort\s+(\d+)/ ) { - $sort_prefix = $1; - } - } elsif ($line =~ /^output ignore/ ) { - $cmd{'output ignore'} = 1; - } elsif ( $line =~ /^output display/ ) { - $cmd{'output display'} = 1; - } elsif ( $line =~ /^expect signal (\w*)/ ) { - $cmd{'expect'} = $1; - } elsif ( $line =~ /^expect return/ ) { - $line =~ s/^expect return //g; - $line =~ s/\r//g; - $cmd{'return'} = $line; - } elsif ( $line =~ /^setenv/ ) { - $line =~ s/^setenv //g; - $line =~ s/\r//g; - setenv_cmd($line); - } elsif ( $line =~ /^timeout/ ) { - $line =~ s/^timeout //; - $line =~ s/\r//g; - $cmd{'timeout'} = $line; - } - } else { - die "[TESH/CRITICAL] parse error: $line\n"; - } -} - -# We are done reading the input file -close $infh unless ( $tesh_name eq "(stdin)" ); - -# Deal with last command, if any -if ( defined( $cmd{'cmd'} ) ) { - exec_cmd( \%cmd ); - %cmd = (); -} - -foreach (@bg_cmds) { - my %test = %{$_}; - analyze_result( \%test ); -} - -if ($diff_tool) { - close $diff_tool_tmp_fh; - system("$diff_tool $diff_tool_tmp_filename $tesh_file"); - unlink $diff_tool_tmp_filename; -} - -if ( $error != 0 ) { - exit $exitcode; -} elsif ( $tesh_name eq "(stdin)" ) { - print "Test suite from stdin OK\n"; -} else { - print "Test suite `$tesh_name' OK\n"; -} - -exit 0; - -#### -#### Helper functions -#### - -sub build_diff { - my $res; - my $diff = Diff->new(@_); - - $diff->Base(1); # Return line numbers, not indices - my $chunk_count = $diff->Next(-1); # Compute the amount of chuncks - return "" if ( $chunk_count == 1 && $diff->Same() ); - $diff->Reset(); - while ( $diff->Next() ) { - my @same = $diff->Same(); - if ( $diff->Same() ) { - if ( $diff->Next(0) > 1 ) { # not first chunk: print 2 first lines - $res .= ' ' . $same[0] . "\n"; - $res .= ' ' . $same[1] . "\n" if ( scalar @same > 1 ); - } - $res .= "...\n" if ( scalar @same > 2 ); - - # $res .= $diff->Next(0)."/$chunk_count\n"; - if ( $diff->Next(0) < $chunk_count ) { # not last chunk: print 2 last lines - $res .= ' ' . $same[ scalar @same - 2 ] . "\n" - if ( scalar @same > 1 ); - $res .= ' ' . $same[ scalar @same - 1 ] . "\n"; - } - } - next if $diff->Same(); - map { $res .= "- $_\n" } $diff->Items(1); - map { $res .= "+ $_\n" } $diff->Items(2); - } - return $res; -} - -# Helper function replacing any occurence of variable '$name' by its '$value' -# As in Bash, ${$value:=BLABLA} is rewritten to $value if set or to BLABLA if $value is not set -sub var_subst { - my ( $text, $name, $value ) = @_; - if ($value) { - $text =~ s/\$\{$name(?::[=-][^}]*)?\}/$value/g; - $text =~ s/\$$name(\W|$)/$value$1/g; - } else { - $text =~ s/\$\{$name:=([^}]*)\}/$1/g; - $text =~ s/\$\{$name\}//g; - $text =~ s/\$$name(\W|$)/$1/g; - } - return $text; -} - -################################ The possible commands ################################ - -sub mkfile_cmd($) { - my %cmd = %{ $_[0] }; - my $file = $cmd{'arg'}; - print STDERR "[Tesh/INFO] mkfile $file. Ctn: >>".join( '\n', @{ $cmd{'in'} })."<<\n" - if $opts{'debug'}; - - unlink($file); - open( FILE, ">$file" ) - or die "[Tesh/CRITICAL] Unable to create file $file: $!\n"; - print FILE join( "\n", @{ $cmd{'in'} } ); - print FILE "\n" if ( scalar @{ $cmd{'in'} } > 0 ); - close(FILE); -} - -# Command CD. Just change to the provided directory -sub cd_cmd($) { - my $directory = shift; - my $failure = 1; - if ( -e $directory && -d $directory ) { - chdir("$directory"); - print "[Tesh/INFO] change directory to $directory\n"; - $failure = 0; - } elsif ( -e $directory ) { - print "Cannot change directory to '$directory': it is not a directory\n"; - } else { - print "Chdir to $directory failed: No such file or directory\n"; - } - if ( $failure == 1 ) { - print "Test suite `$tesh_name': NOK (system error)\n"; - exit 4; - } -} - -# Command setenv. Gets "variable=content", and update the environment accordingly -sub setenv_cmd($) { - my $arg = shift; - if ( $arg =~ /^(.*?)=(.*)$/ ) { - my ( $var, $ctn ) = ( $1, $2 ); - print "[Tesh/INFO] setenv $var=$ctn\n"; - $environ{$var} = $ctn; - $ENV{$var} = $ctn; - } else { - die "[Tesh/CRITICAL] Malformed argument to setenv: expected 'name=value' but got '$arg'\n"; - } -} diff --git a/tools/tesh/tesh.pod b/tools/tesh/tesh.pod new file mode 100755 index 0000000000..0ada1ab44d --- /dev/null +++ b/tools/tesh/tesh.pod @@ -0,0 +1,257 @@ +=encoding UTF-8 + +=head1 NAME + +tesh -- testing shell + +=head1 SYNOPSIS + +B [I]... I + +=head1 DESCRIPTION + +Tesh is the testing shell, a specialized shell for running tests. It +provides the specified input to the tested commands, and check that +they produce the expected output and return the expected value. + +=head1 OPTIONS + + --help : display help + --cd some/directory : ask tesh to switch the working directory before + launching the tests + --setenv var=value : set a specific environment variable + --cfg arg : add parameter --cfg=arg to each command line + --log arg : add parameter --log=arg to each command line + --enable-coverage : ignore output lines starting with "profiling:" + + +=head1 TEST SUITE FILE SYTAX + +A test suite is composed of one or several I separated +by empty lines, each of them being composed of a command to run, its +input text and the expected output. + +The first char of each line specifies the type of line according to +the following list. The second char of each line is ignored. + + `$' command to run in foreground + `&' command to run in background + + `<' input to pass to the command + `>' output expected from the command + + `!' metacommand, which can be one of: + `timeout' |no + `expect signal' + `expect return' + `output' + `output sort' [integer] + `setenv =' + `ignore' + + `p' an informative message to print + `#' a comment + +If the expected output do not match the produced output, or if the +command did not end as expected, Tesh provides an error message (see +the OUTPUT section below) and stops. + +=head2 Command blocks examples + +In a given command block, you can declare the command, its input and +its expected output in the order that you see fit. + + $ cat + < TOTO + > TOTO + + > TOTO + $ cat + < TOTO + + > TOTO + < TOTO + $ cat + +You can group several commands together, provided that they don't have +any input nor output. + + $ mkdir testdir + $ cd testdir + +=head2 Enforcing the command return code + +By default, Tesh enforces that the tested command returns 0. If not, +it fails with an appropriate message and returns I itself. + +You specify that a given command block is expected to return another +code as follows: + + # This command MUST return 42 + ! expect return 42 + $ sh -e "exit 42" + +The I construct applies only to the next command block. + +=head2 Commands that are expected to raise signals + +By default, Tesh detects when the command is killed by a signal (such +as SEGV on segfaults). This is usually unexpected and unfortunate. But +if not, you can specify that a given command block is expected to fail +with a signal as follows: + + # This command MUST raise a segfault + ! expect signal SIGSEGV + $ ./some_failing_code + +The I construct applies only to the next command block. + +=head2 Timeouts + +By default, no command is allowed to run more than 5 seconds. You can +change this value as follows: + + # Allow some more time to the command + ! timeout 60 + $ ./some_longer_command + +You can also disable the timeout completely by passing "no" as a value: + + # This command will never timeout + ! timeout no + $ ./some_very_long_but_safe_command + +=head2 Setting environment variables + +You can modify the environment of the tested commands as follows: + + ! setenv PATH=/bin + $ my_command + +You can also set an envirmnent variable from the command line: + + tesh --setenv bindir=/opt/bin/ + +And then use it within the tesh file: + + $ ${bindir}/myprogram + +Tesh also supports perl default value for undefined variables: + + $ ${bindir:=/usr/bin}/myprogram + +=head2 Not enforcing the expected output + +By default, the commands output is matched against the one expected, +and an error is raised on discrepancy. Metacommands to change this: + +=over 4 + +=item output ignore + +The output is completely discarded. + +=item output display + +The output is displayed, but no error is issued if it differs from the +expected output. + +=item output sort + +The output is sorted before comparison (see next section). + +=back + +=head2 Sorting output + +If the order of the command output changes between runs, you want to +sort it before enforcing that it is exactly what you expect. In +SimGrid for example, this happens when parallel execution is +activated: User processes are run in parallel at each timestamp, and +the output is not reproducible anymore. Until you sort the lines. + +You can sort the command output as follows: + + ! output sort + $ ./some_multithreaded_command + +Sorting lines this ways often makes the tesh output very intricate, +complicating the error analysis: the process logical order is defeated +by the lexicographical sort. + +The solution is to prefix each line of your output with temporal +information so that lines can be grouped by timestamps. The +lexicographical sort then only applies to lines that occured at the +same timestamp. Here is a SimGrid example: + + # Sort only lines depending on the first 19 chars + ! output sort 19 + $ ./some_simgrid_simulator --log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n + +This approach may seem surprizing at the first glance but it does its job: + +=over 4 + +=item Every timestamps remain separated, as it should; + +=item In each timestamp, the output order of processes become + reproducible: that's the lexicographical order of their name; + +=item For each process, the order of its execution is preserved: its + messages within a given timestamp are not reordered. + +=back + +That way, tesh can do its job (no false positive, no false negative) +despite the unpredictable order of executions of processes within a +timestamp, and reported errors remain easy to analyze (execution of a +given process preserved). + +This example is very SimGrid oriented, but the feature could even be +usable by others, who knows? + +=head2 Ignoring some output + +Some outputed lines can be ignored by setting the ignore command followed +by a regular expression: + + ! ignore .*0x[0-9A-F]+\. + $ printf 'word\nMemory address: 0x42AA42.\nanotherword\n' + > word + > anotherword + + +=head2 Colored and formatted text + +Tesh removes ANSI/VT100 control sequences from outputed text to make easier the writing of tests. + + $ printf "I \033[0;31mlove\033[0m tesh\n" + > I love tesh + + + +=head1 BUILTIN COMMANDS + +=head2 mkfile: creating a file + +This command creates a file of the name provided as argument, and adds +the content it gets as input. + + $ mkfile myFile + > some content + > to the file + +It is not possible to use the cat command, as one would expect, +because stream redirections are currently not implemented in Tesh. + +=head1 BUGS, LIMITATIONS AND POSSIBLE IMPROVEMENTS + +The main limitation is the lack of stream redirections in the commands +(">", "<" and "|" shell constructs and friends). The B builtin +command makes this situation bearable. + +It would be nice if we could replace the tesh file completely with +command line flags when the output is not to be verified. + + +=cut diff --git a/tools/tesh/tesh.py b/tools/tesh/tesh.py new file mode 100755 index 0000000000..ef69904fb3 --- /dev/null +++ b/tools/tesh/tesh.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + +tesh -- testing shell +======================== + +Copyright (c) 2012-2015. The SimGrid Team. +All rights reserved. + +This program is free software; you can redistribute it and/or modify it +under the terms of the license (GNU LGPL) which comes with this package. + +#TODO: emptty: does "! output display" also disable the comparaison with the output ? yes + +#TODO: child of child of child printf => tester dans rsg... + +#TODO: regular expression in output +#TODO: linked regular expression in output + + +""" + + +import sys, os +import shlex +import re +import difflib +import signal +import argparse + +if sys.version_info[0] == 3: + import subprocess + import _thread +elif sys.version_info[0] < 3: + import subprocess32 as subprocess + import thread as _thread +else: + raise "This program has not been made to exist this long" + + + +############## +# +# Utilities +# +# + + +# Singleton metaclass that works in Python 2 & 3 +# http://stackoverflow.com/questions/6760685/creating-a-singleton-in-python +class _Singleton(type): + """ A metaclass that creates a Singleton base class when called. """ + _instances = {} + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(_Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] +class Singleton(_Singleton('SingletonMeta', (object,), {})): pass + +SIGNALS_TO_NAMES_DICT = dict((getattr(signal, n), n) \ + for n in dir(signal) if n.startswith('SIG') and '_' not in n ) + + + +#exit correctly +def exit(errcode): + #If you do not flush some prints are skipped + sys.stdout.flush() + #os._exit exit even when executed within a thread + os._exit(errcode) + + +def fatal_error(msg): + print("[Tesh/CRITICAL] "+str(msg)) + exit(1) + + +#Set an environment variable. +# arg must be a string with the format "variable=value" +def setenv(arg): + print("[Tesh/INFO] setenv "+arg) + t = arg.split("=") + os.environ[t[0]] = t[1] + #os.putenv(t[0], t[1]) does not work + #see http://stackoverflow.com/questions/17705419/python-os-environ-os-putenv-usr-bin-env + + +#http://stackoverflow.com/questions/30734967/how-to-expand-environment-variables-in-python-as-bash-does +def expandvars2(path): + return re.sub(r'(? 1 and line[-2] == "\\": + txt = txt[0:-1] + line = next(self.f) + self.linenumber += 1 + txt += line[0:-1] + return txt + + +#keep the state of tesh (mostly configuration values) +class TeshState(Singleton): + def __init__(self): + self.threads = [] + self.args_suffix = "" + self.ignore_regexps_common = [] + + def add_thread(self, thread): + self.threads.append(thread) + + def join_all_threads(self): + for t in self.threads: + t.acquire() + t.release() + + + + +#Command line object +class Cmd(object): + def __init__(self): + self.input_pipe = [] + self.output_pipe_stdout = [] + self.output_pipe_stderr = [] + self.timeout = 5 + self.args = None + self.linenumber = -1 + + self.background = False + self.cwd = None + + self.ignore_output = False + self.expect_return = 0 + + self.output_display = False + + self.sort = -1 + + self.ignore_regexps = TeshState().ignore_regexps_common + + def add_input_pipe(self, l): + self.input_pipe.append(l) + + def add_output_pipe_stdout(self, l): + self.output_pipe_stdout.append(l) + + def add_output_pipe_stderr(self, l): + self.output_pipe_stderr.append(l) + + def set_cmd(self, args, linenumber): + self.args = args + self.linenumber = linenumber + + def add_ignore(self, txt): + self.ignore_regexps.append(re.compile(txt)) + + def remove_ignored_lines(self, lines): + for ign in self.ignore_regexps: + lines = [l for l in lines if not ign.match(l)] + return lines + + + def _cmd_mkfile(self, argline): + filename = argline[len("mkfile "):] + file = open(filename, "w") + if file is None: + fatal_error("Unable to create file "+filename) + file.write("\n".join(self.input_pipe)) + file.close() + + def _cmd_cd(self, argline): + args = shlex.split(argline) + if len(args) != 2: + fatal_error("Too many arguments to cd") + try: + os.chdir(args[1]) + print("[Tesh/INFO] change directory to "+args[1]) + except FileNotFoundError: + print("Chdir to "+args[1]+" failed: No such file or directory") + print("Test suite `"+FileReader().filename+"': NOK (system error)") + exit(4) + + + #Run the Cmd if possible. + # Return False if nothing has been ran. + def run_if_possible(self): + if self.can_run(): + if self.background: + #Python threads loose the cwd + self.cwd = os.getcwd() + lock = _thread.allocate_lock() + lock.acquire() + TeshState().add_thread(lock) + _thread.start_new_thread( Cmd._run, (self, lock) ) + else: + self._run() + return True + else: + return False + + + def _run(self, lock=None): + #Python threads loose the cwd + if self.cwd is not None: + os.chdir(self.cwd) + self.cwd = None + + #retrocompatibility: support ${aaa:=.} variable format + def replace_perl_variables(m): + vname = m.group(1) + vdefault = m.group(2) + if vname in os.environ: + return "$"+vname + else: + return vdefault + self.args = re.sub(r"\${(\w+):=([^}]*)}", replace_perl_variables, self.args) + + #replace bash environment variables ($THINGS) to their values + self.args = expandvars2(self.args) + + if re.match("^mkfile ", self.args) is not None: + self._cmd_mkfile(self.args) + if lock is not None: lock.release() + return + + if re.match("^cd ", self.args) is not None: + self._cmd_cd(self.args) + if lock is not None: lock.release() + return + + self.args += TeshState().args_suffix + + print("["+FileReader().filename+":"+str(self.linenumber)+"] "+self.args) + + args = shlex.split(self.args) + #print (args) + try: + proc = subprocess.Popen(args, bufsize=1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + except OSError as e: + if e.errno == 8: + e.strerror += "\nOSError: [Errno 8] Executed scripts should start with shebang line (like #!/bin/sh)" + raise e + + try: + (stdout_data, stderr_data) = proc.communicate("\n".join(self.input_pipe), self.timeout) + except subprocess.TimeoutExpired: + print("Test suite `"+FileReader().filename+"': NOK (<"+FileReader().filename+":"+str(self.linenumber)+"> timeout after "+str(self.timeout)+" sec)") + exit(3) + + if self.output_display: + print(stdout_data) + + #remove text colors + ansi_escape = re.compile(r'\x1b[^m]*m') + stdout_data = ansi_escape.sub('', stdout_data) + + #print ((stdout_data, stderr_data)) + + if self.ignore_output: + print("(ignoring the output of <"+FileReader().filename+":"+str(self.linenumber)+"> as requested)") + else: + stdouta = stdout_data.split("\n") + while len(stdouta) > 0 and stdouta[-1] == "": + del stdouta[-1] + stdouta = self.remove_ignored_lines(stdouta) + + #the "sort" bash command is case unsensitive, + # we mimic its behaviour + if self.sort == 0: + stdouta.sort(key=lambda x: x.lower()) + self.output_pipe_stdout.sort(key=lambda x: x.lower()) + elif self.sort > 0: + stdouta.sort(key=lambda x: x[:self.sort].lower()) + self.output_pipe_stdout.sort(key=lambda x: x[:self.sort].lower()) + + diff = list(difflib.unified_diff(self.output_pipe_stdout, stdouta,lineterm="",fromfile='expected', tofile='obtained')) + if len(diff) > 0: + print("Output of <"+FileReader().filename+":"+str(self.linenumber)+"> mismatch:") + for line in diff: + print(line) + print("Test suite `"+FileReader().filename+"': NOK (<"+str(FileReader())+"> output mismatch)") + if lock is not None: lock.release() + exit(2) + + #print ((proc.returncode, self.expect_return)) + + if proc.returncode != self.expect_return: + if proc.returncode >= 0: + print("Test suite `"+FileReader().filename+"': NOK (<"+str(FileReader())+"> returned code "+str(proc.returncode)+")") + if lock is not None: lock.release() + exit(2) + else: + print("Test suite `"+FileReader().filename+"': NOK (<"+str(FileReader())+"> got signal "+SIGNALS_TO_NAMES_DICT[-proc.returncode]+")") + if lock is not None: lock.release() + exit(-proc.returncode) + + if lock is not None: lock.release() + + + + def can_run(self): + return self.args is not None + + + + + + + + + + +############## +# +# Main +# +# + + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='tesh -- testing shell', add_help=True) + group1 = parser.add_argument_group('Options') + group1.add_argument('teshfile', nargs='?', help='Name of teshfile, stdin if omitted') + group1.add_argument('--cd', metavar='some/directory', help='ask tesh to switch the working directory before launching the tests') + group1.add_argument('--setenv', metavar='var=value', action='append', help='set a specific environment variable') + group1.add_argument('--cfg', metavar='arg', help='add parameter --cfg=arg to each command line') + group1.add_argument('--log', metavar='arg', help='add parameter --log=arg to each command line') + group1.add_argument('--enable-coverage', action='store_true', help='ignore output lines starting with "profiling:"') + + try: + options = parser.parse_args() + except: + exit(1) + + if options.cd is not None: + os.chdir(options.cd) + + if options.enable_coverage: + print("Enable coverage") + TeshState().ignore_regexps_common = [re.compile("^profiling:")] + + if options.teshfile is None: + f = FileReader(None) + print("Test suite from stdin") + else: + f = FileReader(options.teshfile) + print("Test suite '"+f.filename+"'") + + if options.setenv is not None: + for e in options.setenv: + setenv(e) + + if options.cfg is not None: + TeshState().args_suffix += " --cfg="+options.cfg + if options.log is not None: + TeshState().args_suffix += " --log="+options.log + + + #cmd holds the current command line + # tech commands will add some parameters to it + # when ready, we execute it. + cmd = Cmd() + + line = f.readfullline() + while line is not None: + #print(">>============="+line+"==<<") + if len(line) == 0: + #print ("END CMD block") + if cmd.run_if_possible(): + cmd = Cmd() + + elif line[0] == "#": + pass + + elif line[0:2] == "p ": + print("["+str(FileReader())+"] "+line[2:]) + + elif line[0:2] == "< ": + cmd.add_input_pipe(line[2:]) + elif line[0:1] == "<": + cmd.add_input_pipe(line[1:]) + + elif line[0:2] == "> ": + cmd.add_output_pipe_stdout(line[2:]) + elif line[0:1] == ">": + cmd.add_output_pipe_stdout(line[1:]) + + elif line[0:2] == "$ ": + if cmd.run_if_possible(): + cmd = Cmd() + cmd.set_cmd(line[2:], f.linenumber) + + elif line[0:2] == "& ": + if cmd.run_if_possible(): + cmd = Cmd() + cmd.set_cmd(line[2:], f.linenumber) + cmd.background = True + + elif line[0:15] == "! output ignore": + cmd.ignore_output = True + #print("cmd.ignore_output = True") + elif line[0:16] == "! output display": + cmd.output_display = True + cmd.ignore_output = True + elif line[0:15] == "! expect return": + cmd.expect_return = int(line[16:]) + #print("expect return "+str(int(line[16:]))) + elif line[0:15] == "! expect signal": + sig = line[16:] + #get the signal integer value from the signal module + if sig not in signal.__dict__: + fatal_error("unrecognized signal '"+sig+"'") + sig = int(signal.__dict__[sig]) + #popen return -signal when a process ends with a signal + cmd.expect_return = -sig + elif line[0:len("! timeout ")] == "! timeout ": + if "no" in line[len("! timeout "):]: + cmd.timeout = None + else: + cmd.timeout = int(line[len("! timeout "):]) + + elif line[0:len("! output sort")] == "! output sort": + if len(line) >= len("! output sort "): + sort = int(line[len("! output sort "):]) + else: + sort = 0 + cmd.sort = sort + elif line[0:len("! setenv ")] == "! setenv ": + setenv(line[len("! setenv "):]) + + elif line[0:len("! ignore ")] == "! ignore ": + cmd.add_ignore(line[len("! ignore "):]) + + else: + fatal_error("UNRECOGNIZED OPTION") + + + line = f.readfullline() + + cmd.run_if_possible() + + TeshState().join_all_threads() + + if f.filename == "(stdin)": + print("Test suite from stdin OK") + else: + print("Test suite `"+f.filename+"' OK") + + + + +