X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/7e4e539e69efb17211745fbb57727937e001cf60..8a210e1091c499a7354ac4ff08c19b29c1143093:/tools/tesh/tesh.pl diff --git a/tools/tesh/tesh.pl b/tools/tesh/tesh.pl index b36605206d..f25be34489 100755 --- a/tools/tesh/tesh.pl +++ b/tools/tesh/tesh.pl @@ -19,13 +19,221 @@ tesh -- testing shell =head1 SYNOPSIS -B [I] I +B [I]... I + +=head1 DESCRIPTION + +Tesh is the testing shell, a specialized shell for running tests. It +provides the specified input to the tested commands, and check that +they produce the expected output and return the expected value. + +=head1 OPTIONS + + --cd some/directory : ask tesh to switch the working directory before + launching the tests + --setenv var=value : set a specific environment variable + --cfg arg : add parameter --cfg=arg to each command line + --enable-coverage : ignore output lines starting with "profiling:" + +=head1 TEST SUITE FILE SYTAX + +A test suite is composed of one or several I separated +by empty lines, each of them being composed of a command to run, its +input text and the expected output. + +The first char of each line specifies the type of line according to +the following list. The second char of each line is ignored. + + `$' command to run in foreground + `&' command to run in background + + `<' input to pass to the command + `>' output expected from the command + + `!' metacommand, which can be one of: + `timeout' |no + `expect signal' + `expect return' + `output' + `setenv =' + + `p' an informative message to print + +If the expected output do not match the produced output, or if the +command did not end as expected, Tesh provides an error message (see +the OUTPUT section below) and stops. + +=head2 Command blocks examples + +In a given command block, you can declare the command, its input and +its expected output in the order that you see fit. + + $ cat + < TOTO + > TOTO + + > TOTO + $ cat + < TOTO + + > TOTO + < TOTO + $ cat + +You can group several commands together, provided that they don't have +any input nor output. + + $ mkdir testdir + $ cd testdir + +=head2 Enforcing the command return code + +By default, Tesh enforces that the tested command returns 0. If not, +it fails with an appropriate message and returns I itself. + +You specify that a given command block is expected to return another +code as follows: + + # This command MUST return 42 + ! expect return 42 + $ sh -e "exit 42" + +The I construct applies only to the next command block. + +=head2 Commands that are expected to raise signals + +By default, Tesh detects when the command is killed by a signal (such +as SEGV on segfaults). This is usually unexpected and unfortunate. But +if not, you can specify that a given command block is expected to fail +with a signal as follows: + + # This command MUST raise a segfault + ! expect signal SIGSEGV + $ ./some_failing_code + +The I construct applies only to the next command block. + +=head2 Timeouts + +By default, no command is allowed to run more than 5 seconds. You can +change this value as follows: + + # Allow some more time to the command + ! timeout 60 + $ ./some_longer_command + +You can also disable the timeout completely by passing "no" as a value: + + # This command will never timeout + ! timeout no + $ ./some_very_long_but_safe_command + +=head2 Setting environment variables + +You can modify the environment of the tested commands as follows: + + ! setenv PATH=/bin + $ my_command + +=head2 Not enforcing the expected output + +By default, the commands output is matched against the one expected, +and an error is raised on discrepancy. Metacommands to change this: + +=over 4 + +=item output ignore + +The output is completely discarded. + +=item output display + +The output is displayed, but no error is issued if it differs from the +expected output. + +=item output sort + +The output is sorted before comparison (see next section). + +=back + +=head2 Sorting output + +If the order of the command output changes between runs, you want to +sort it before enforcing that it is exactly what you expect. In +SimGrid for example, this happens when parallel execution is +activated: User processes are run in parallel at each timestamp, and +the output is not reproducible anymore. Until you sort the lines. + +You can sort the command output as follows: + + ! output sort + $ ./some_multithreaded_command + +Sorting lines this ways often makes the tesh output very intricate, +complicating the error analysis: the process logical order is defeated +by the lexicographical sort. + +The solution is to prefix each line of your output with temporal +information so that lines can be grouped by timestamps. The +lexicographical sort then only applies to lines that occured at the +same timestamp. Here is a SimGrid example: + + # Sort only lines depending on the first 19 chars + ! output sort 19 + $ ./some_simgrid_simulator --log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n + +This approach may seem surprizing at the first glance but it does its job: + +=over 4 + +=item Every timestamps remain separated, as it should; + +=item In each timestamp, the output order of processes become + reproducible: that's the lexicographical order of their name; + +=item For each process, the order of its execution is preserved: its + messages within a given timestamp are not reordered. + +=back + +That way, tesh can do its job (no false positive, no false negative) +despite the unpredictable order of executions of processes within a +timestamp, and reported errors remain easy to analyze (execution of a +given process preserved). + +This example is very SimGrid oriented, but the feature could even be +usable by others, who knows? + + +=head1 BUILTIN COMMANDS + +=head2 mkfile: creating a file + +This command creates a file of the name provided as argument, and adds +the content it gets as input. + + $ mkfile myFile + > some content + > to the file + +It is not possible to use the cat command, as one would expect, +because stream redirections are currently not implemented in Tesh. + +=head1 BUGS AND LIMITATIONS + +The main limitation is the lack of stream redirections in the commands +(">", "<" and "|" shell constructs and friends). The B builtin +command makes this situation bearable. =cut -my ($timeout) = 0; -my ($time_to_wait) = 0; -my $path = $0; +BEGIN { + # Disabling IPC::Run::Debug saves tons of useless calls. + $ENV{'IPCRUNDEBUG'} = 'none' + unless exists $ENV{'IPCRUNDEBUG'}; +} + my $enable_coverage = 0; my $diff_tool = 0; my $diff_tool_tmp_fh = 0; @@ -38,6 +246,8 @@ my $exitcode = 0; my @bg_cmds; my (%environ); $SIG{'PIPE'} = 'IGNORE'; + +my $path = $0; $path =~ s|[^/]*$||; push @INC, $path; @@ -48,7 +258,7 @@ use Diff qw(diff); # postpone a bit to have time to change INC use Getopt::Long qw(GetOptions); use strict; use Text::ParseWords; -use IPC::Open3; +use IPC::Run qw(start run timeout finish); use IO::File; use English; @@ -74,22 +284,9 @@ BEGIN { #### Command line option handling #### -if ( $ARGV[0] eq "--internal-killer-process" ) { - - # We fork+exec a waiter process in charge of killing the command after timeout - # If the command stops earlier, that's fine: the killer sends a signal to an already stopped process, fails, and quits. - # Nobody cares about the killer issues. - # The only problem could arise if another process is given the same PID than cmd. We bet it won't happen :) - my $time_to_wait = $ARGV[1]; - my $pid = $ARGV[2]; - sleep $time_to_wait; - kill( 'TERM', $pid ); - sleep 1; - kill( 'KILL', $pid ); - exit $time_to_wait; -} - -my %opts = ( "debug" => 0 ); +my %opts = ( "debug" => 0, + "timeout" => 5, # No command should run any longer than 5 seconds by default + ); Getopt::Long::config( 'bundling', 'no_getopt_compat', 'no_auto_abbrev' ); GetOptions( @@ -106,9 +303,7 @@ GetOptions( $tesh_file = pop @ARGV; -if ($enable_coverage) { - print "Enable coverage\n"; -} +print "Enable coverage\n" if ($enable_coverage); if ($diff_tool) { use File::Temp qw/ tempfile /; @@ -125,14 +320,6 @@ if ( $tesh_file =~ m/(.*)\.tesh/ ) { print "Test suite from stdin\n"; } -## -## File parsing -## -my ($return) = -1; -my ($forked); -my ($config) = ""; -my (@buffer_tesh) = (); - ########################################################################### sub exit_status { @@ -156,12 +343,8 @@ sub exit_status { sub exec_cmd { my %cmd = %{ $_[0] }; if ( $opts{'debug'} ) { - print "IN BEGIN\n"; - map { print " $_" } @{ $cmd{'in'} }; - print "IN END\n"; - print "OUT BEGIN\n"; - map { print " $_" } @{ $cmd{'out'} }; - print "OUT END\n"; + map { print "IN: $_\n" } @{ $cmd{'in'} }; + map { print "OUT: $_\n" } @{ $cmd{'out'} }; print "CMD: $cmd{'cmd'}\n"; } @@ -189,76 +372,72 @@ sub exec_cmd { $cmd{'cmd'} .= " $opts{'cfg'}" if ( defined( $opts{'cfg'} ) && length( $opts{'cfg'} ) ); - # final cleanup + # finally trim any remaining space chars $cmd{'cmd'} =~ s/^\s+//; $cmd{'cmd'} =~ s/\s+$//; print "[$tesh_name:$cmd{'line'}] $cmd{'cmd'}\n"; + $cmd{'return'} ||= 0; + $cmd{'timeout'} ||= $opts{'timeout'}; + + ### # exec the command line - $cmd{'got'} = IO::File->new_tmpfile; - $cmd{'got'}->autoflush(1); - local *E = $cmd{'got'}; - $cmd{'pid'} = - open3( \*CHILD_IN, ">&E", ">&E", quotewords( '\s+', 0, $cmd{'cmd'} ) ); - - # push all provided input to executing child - map { print CHILD_IN "$_\n"; } @{ $cmd{'in'} }; - close CHILD_IN; - - # if timeout specified, fork and kill executing child at the end of timeout - if ( not $cmd{'background'} - and ( defined( $cmd{'timeout'} ) or defined( $opts{'timeout'} ) ) ) - { - $time_to_wait = - defined( $cmd{'timeout'} ) ? $cmd{'timeout'} : $opts{'timeout'}; - $forked = fork(); - $timeout = -1; - die "fork() failed: $!" unless defined $forked; - if ( $forked == 0 ) { # child - exec("$PROGRAM_NAME --internal-killer-process $time_to_wait $cmd{'pid'}"); - } - } - - # Cleanup the executing child, and kill the timeouter brother on need - $cmd{'return'} = 0 unless defined( $cmd{'return'} ); - if ( $cmd{'background'} != 1 ) { - waitpid( $cmd{'pid'}, 0 ); - $cmd{'gotret'} = exit_status($?); - parse_result( \%cmd ); - } else { + my @cmdline = quotewords( '\s+', 0, $cmd{'cmd'} ); + my $input = defined($cmd{'in'})? join("\n",@{$cmd{'in'}}) : ""; + my $output = " " x 10240; $output = ""; # Preallocate 10kB, and reset length to 0 + $cmd{'got'} = \$output; + $cmd{'job'} = start \@cmdline, '<', \$input, '>&', \$output, + ($cmd{'timeout'} eq 'no' ? () : timeout($cmd{'timeout'})); - # & commands, which will be handled at the end + if ( $cmd{'background'} ) { + # Just enqueue the job. It will be dealed with at the end push @bg_cmds, \%cmd; + } else { + # Deal with its ending conditions right away + analyze_result( \%cmd ); } } -sub parse_result { +sub analyze_result { my %cmd = %{ $_[0] }; - my $gotret = $cmd{'gotret'}; + + eval { + finish( $cmd{'job'} ); + }; + if ($@) { + if ($@ =~ /timeout/) { + $cmd{'job'}->kill_kill; + $cmd{'timeouted'} = 1; + } elsif ($@ =~ /^ack / and $@ =~ /pipe/) { + print STDERR "Tesh: Broken pipe (ignored).\n"; + } else { + die $@; # Don't know what it is, so let it go. + } + } + $cmd{'timeouted'} ||= 0; + + my $gotret = $cmd{'gotret'} = exit_status($?); my $wantret; if ( defined( $cmd{'expect'} ) and ( $cmd{'expect'} ne "" ) ) { $wantret = "got signal $cmd{'expect'}"; } else { - $wantret = - "returned code " . ( defined( $cmd{'return'} ) ? $cmd{'return'} : 0 ); + $wantret = "returned code " . ( defined( $cmd{'return'} ) ? $cmd{'return'} : 0 ); } - local *got = $cmd{'got'}; - seek( got, 0, 0 ); - # pop all output from executing child my @got; - while ( defined( my $got = ) ) { + map { print "GOT: $_\n" } ${$cmd{'got'}} if $opts{'debug'}; + foreach my $got ( split("\n", ${$cmd{'got'}}) ) { $got =~ s/\r//g; chomp $got; print $diff_tool_tmp_fh "> $got\n" if ($diff_tool); - if ( !( $enable_coverage and $got =~ /^profiling:/ ) ) { + unless ( $enable_coverage and $got =~ /^profiling:/ ) { push @got, $got; } } @@ -294,41 +473,27 @@ sub parse_result { } } - # Did we timeout ? If yes, handle it. If not, kill the forked process. + # Did we timeout? - if ( $timeout == -1 - and ( $gotret eq "got signal SIGTERM" or $gotret eq "got signal SIGKILL" ) ) - { - $gotret = "return code 0"; - $timeout = 1; - $gotret = "timeout after $time_to_wait sec"; + if ( $cmd{'timeouted'} ) { + $gotret = "timeout after $cmd{'timeout'} sec"; $error = 1; $exitcode = 3; print STDERR "<$cmd{'file'}:$cmd{'line'}> timeouted. Kill the process.\n"; - } else { - $timeout = 0; } if ( $gotret ne $wantret ) { $error = 1; my $msg = "Test suite `$cmd{'file'}': NOK (<$cmd{'file'}:$cmd{'line'}> $gotret)\n"; - if ( $timeout != 1 ) { + if ( scalar @got ) { $msg = $msg . "Output of <$cmd{'file'}:$cmd{'line'}> so far:\n"; - } - map { $msg .= "|| $_\n" } @got; - if ( !@got ) { - if ( $timeout == 1 ) { - print STDERR "<$cmd{'file'}:$cmd{'line'}> No output before timeout\n"; - } else { - $msg .= "||\n"; - } - } - $timeout = 0; + map { $msg .= "|| $_\n" } @got; + } else { + $msg .= "<$cmd{'file'}:$cmd{'line'}> No output so far.\n"; + } print STDERR "$msg"; } - ### - # Check the result of execution - ### + # Does the output match? my $diff; if ( defined( $cmd{'output display'} ) ) { print "[Tesh/INFO] Here is the (ignored) command output:\n"; @@ -374,11 +539,10 @@ LINE: while ( defined( my $line = <$infh> ) and not $error ) { $line_num++; print "[TESH/debug] $line_num: $line\n" if $opts{'debug'}; - my $next; # deal with line continuations while ( $line =~ /^(.*?)\\$/ ) { - $next = <$infh>; + my $next = <$infh>; die "[TESH/CRITICAL] Continued line at end of file\n" unless defined($next); $line_num++; @@ -483,7 +647,6 @@ LINE: while ( defined( my $line = <$infh> ) and not $error ) { exec_cmd( \%cmd ); %cmd = (); } - print "hey\n"; $cmd{'expect'} = "$1"; } elsif ( $line =~ /^!\s*expect return/ ) { #expect return if ( defined( $cmd{'cmd'} ) ) { @@ -512,10 +675,6 @@ LINE: while ( defined( my $line = <$infh> ) and not $error ) { } else { die "[TESH/CRITICAL] parse error: $line\n"; } - if ($forked) { - kill( 'KILL', $forked ); - $timeout = 0; - } } # We're done reading the input file @@ -527,16 +686,9 @@ if ( defined( $cmd{'cmd'} ) ) { %cmd = (); } -if ($forked) { - kill( 'KILL', $forked ); - $timeout = 0; -} - foreach (@bg_cmds) { my %test = %{$_}; - waitpid( $test{'pid'}, 0 ); - $test{'gotret'} = exit_status($?); - parse_result( \%test ); + analyze_result( \%test ); } if ($diff_tool) { @@ -553,6 +705,8 @@ if ( $error != 0 ) { print "Test suite `$tesh_name' OK\n"; } +exit 0; + #### #### Helper functions #### @@ -608,7 +762,8 @@ sub var_subst { sub mkfile_cmd($) { my %cmd = %{ $_[0] }; my $file = $cmd{'arg'}; - print "[Tesh/INFO] mkfile $file\n"; + print STDERR "[Tesh/INFO] mkfile $file. Ctn: >>".join( '\n', @{ $cmd{'in'} })."<<\n" + if $opts{'debug'}; unlink($file); open( FILE, ">$file" ) @@ -644,6 +799,7 @@ sub setenv_cmd($) { my ( $var, $ctn ) = ( $1, $2 ); print "[Tesh/INFO] setenv $var=$ctn\n"; $environ{$var} = $ctn; + $ENV{$var} = $ctn; } else { die "[Tesh/CRITICAL] Malformed argument to setenv: expected 'name=value' but got '$arg'\n"; }