#! ./tesh
-! output sort 19 $ $SG_TEST_EXENV maestro-set/maestro-set$EXEEXT ${srcdir:=.}/../platforms/small_platform.xml ${srcdir:=.}/maestro-set/maestro-set_d.xml "--log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n"
+! output sort 19
+$ $SG_TEST_EXENV maestro-set/maestro-set$EXEEXT ${srcdir:=.}/../platforms/small_platform.xml ${srcdir:=.}/maestro-set/maestro-set_d.xml "--log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n"
> [ 0.000000] (1:sender@Tremblay) sender
> [ 0.000000] (1:sender@Tremblay) host = Jupiter
> [ 0.000000] (1:sender@Tremblay) task_la->data = 0.000000e+00
add_custom_target(manpages ALL
COMMAND ${CMAKE_COMMAND} -E make_directory ${MANPAGE_DIR}
COMMAND pod2man ${CMAKE_HOME_DIRECTORY}/tools/simgrid_update_xml.pl > ${MANPAGE_DIR}/simgrid_update_xml.1
- COMMAND pod2man ${CMAKE_HOME_DIRECTORY}/tools/tesh/tesh.pl > ${MANPAGE_DIR}/tesh.1
+ COMMAND pod2man ${CMAKE_HOME_DIRECTORY}/tools/tesh/tesh.pod > ${MANPAGE_DIR}/tesh.1
COMMENT "Generating manpages"
)
install(FILES
execute_process(COMMAND chmod a=rwx ${CMAKE_BINARY_DIR}/bin/smpirun)
ENDIF()
-SET(TESH_COMMAND ${PERL_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/tesh)
+SET(TESH_COMMAND ${CMAKE_BINARY_DIR}/bin/tesh)
IF(CMAKE_HOST_WIN32)
SET(TESH_OPTION ${TESH_OPTION} --timeout 50)
ENDIF()
-configure_file("${CMAKE_HOME_DIRECTORY}/tools/tesh/tesh.pl"
+configure_file("${CMAKE_HOME_DIRECTORY}/tools/tesh/tesh.py"
"${CMAKE_BINARY_DIR}/bin/tesh" @ONLY IMMEDIATE)
- file(COPY ${CMAKE_HOME_DIRECTORY}/tools/cmake/scripts/Diff.pm
- DESTINATION ${CMAKE_BINARY_DIR}/bin
- FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ)
- file(COPY ${CMAKE_HOME_DIRECTORY}/tools/cmake/scripts/IPC
- DESTINATION ${CMAKE_BINARY_DIR}/bin)
foreach(x setenv set-output-ignore set-output-sort set-return set-timeout background basic bg-basic bg-set-signal
catch-return catch-signal catch-timeout catch-wrong-output cd IO-bigsize IO-broken-pipe IO-orders)
endforeach()
set(tesh_files ${tesh_files} PARENT_SCOPE)
-set(bin_files ${bin_files} ${CMAKE_CURRENT_SOURCE_DIR}/tesh.pl ${CMAKE_CURRENT_SOURCE_DIR}/generate_tesh PARENT_SCOPE)
+set(bin_files ${bin_files} ${CMAKE_CURRENT_SOURCE_DIR}/tesh.py ${CMAKE_CURRENT_SOURCE_DIR}/generate_tesh PARENT_SCOPE)
> Test suite from stdin
> [(stdin):3] cat
> Test suite from stdin OK
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
p Order: out, in, cmd
< > TOTO
> Test suite from stdin
> [(stdin):3] cat
> Test suite from stdin OK
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
p Order: out, cmd, in
< > TOTO
> Test suite from stdin
> [(stdin):2] cat
> Test suite from stdin OK
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
p Order: in, cmd, out
< < TOTO
> Test suite from stdin
> [(stdin):2] cat
> Test suite from stdin OK
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
p Order: cmd, out, in
< $ cat
> Test suite from stdin
> [(stdin):1] cat
> Test suite from stdin OK
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
p Order: cmd, in, out
< $ cat
> Test suite from stdin
> [(stdin):1] cat
> Test suite from stdin OK
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
! output ignore
< $ cat tmp_fich
< > TUTU TOTO
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
$ rm tmp_fich
< < _x
< $ sed 's/_/ /'
< > x
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
! expect return 2
! output ignore
< < x
< $ cat
< > x
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
p * trailing spaces...
< x_
< < x_
< $ sed 's/_/ /'
< > x
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
! expect return 2
! output ignore
< < x
< $ cat
< > x
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
p * empty lines...
< a
< $ cat
< > a
< > c
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
! expect return 2
! output ignore
< > a
< >
< > c
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
<
< b
< $ cat
< > b
< > c
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
! expect return 2
! output ignore
< >
< > b
< > c
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
$ mkdir temp_testdir-bg-set-signal
$ cd temp_testdir-bg-set-signal
+
< kill 'SEGV', $$;
$ mkfile segfault.pl
# This suite builds and uses a program returning 1.
# tesh must detect this condition and report the issue.
-! expect return 41
+! expect return 2
< $ perl -e "exit 1"
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
> Test suite from stdin
> [(stdin):1] perl -e "exit 1"
> Test suite `(stdin)': NOK (<(stdin):1> returned code 1)
-> <(stdin):1> No output so far.
$ mkdir temp_testdir-catch-signal
$ cd temp_testdir-catch-signal
+
< kill 'SEGV', $$;
$ mkfile segfault.pl
$ perl segfault.pl
p Check that we return the expected return value on SEGV
-! expect return 15
+! expect return 11
< $ perl segfault.pl
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
> Test suite from stdin
> [(stdin):1] perl segfault.pl
> Test suite `(stdin)': NOK (<(stdin):1> got signal SIGSEGV)
-> <(stdin):1> No output so far.
$ cd ..
$ cmake -E remove_directory temp_testdir-catch-signal
> Test suite from stdin
> [(stdin):2] sleep 6
> Test suite `(stdin)': NOK (<(stdin):2> timeout after 1 sec)
-> <(stdin):2> No output so far.
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
< > TOTO
< < TUTU
< $ cat
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
> Test suite from stdin
> [(stdin):3] cat
> Output of <(stdin):3> mismatch:
-> - TOTO
-> + TUTU
+>--- expected
+>+++ obtained
+>@@ -1 +1 @@
+> -TOTO
+> +TUTU
> Test suite `(stdin)': NOK (<(stdin):3> output mismatch)
> Test suite from stdin
> Chdir to toto failed: No such file or directory
> Test suite `(stdin)': NOK (system error)
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
# The next command checks that there is a testdir_temp-cd in the upper directory,
# ie that mkdir and cd both worked.
< > TOTO
< < TUTU
< $ cat
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
> Test suite from stdin
> [(stdin):4] cat
> (ignoring the output of <(stdin):4> as requested)
< ! output sort
< $ true
<
-< $ printf 'profiling: foo\\n'
+< $ printf 'profiling: foo\n'
<
< $ printf 'profiling: foo'
<
< ! output sort
-< $ printf 'profiling: foo\\n'
+< $ printf 'profiling: foo\n'
<
< ! output sort
< $ printf 'profiling: foo'
<
-< $ printf 'a\\nb\\nc\\nd\\n'
+< $ printf 'a\nb\nc\nd\n'
< > a
< > b
< > c
< > d
<
-< $ printf 'a\\nb\\nc\\nd'
+< $ printf 'a\nb\nc\nd'
< > a
< > b
< > c
< > d
<
< ! output sort
-< $ printf 'c\\nd\\nb\\na\\n'
+< $ printf 'c\nd\nb\na\n'
< > a
< > b
< > c
< > d
<
< ! output sort
-< $ printf 'c\\nd\\nb\\na'
+< $ printf 'c\nd\nb\na'
< > a
< > b
< > c
< > d
<
-< $ printf 'a\\nprofiling: foo\\nprofiling: bar\\nb\\nc\\nd\\nprofiling: baz\\n'
+< $ printf 'a\nprofiling: foo\nprofiling: bar\nb\nc\nd\nprofiling: baz\n'
< > a
< > b
< > c
< > d
<
-< $ printf 'a\\nprofiling: foo\\nprofiling: bar\\nb\\nc\\nd\\nprofiling: baz'
+< $ printf 'a\nprofiling: foo\nprofiling: bar\nb\nc\nd\nprofiling: baz'
< > a
< > b
< > c
< > d
<
< ! output sort
-< $ printf 'c\\nprofiling: foo\\nprofiling: bar\\nd\\nb\\na\\nprofiling: baz\\n'
+< $ printf 'c\nprofiling: foo\nprofiling: bar\nd\nb\na\nprofiling: baz\n'
< > a
< > b
< > c
< > d
<
< ! output sort
-< $ printf 'c\\nprofiling: foo\\nprofiling: bar\\nd\\nb\\na\\nprofiling: baz'
+< $ printf 'c\nprofiling: foo\nprofiling: bar\nd\nb\na\nprofiling: baz'
< > a
< > b
< > c
< > d
-$ perl ${bindir:=.}/tesh --enable-coverage
+$ ${bindir:=.}/tesh --enable-coverage
> Enable coverage
> Test suite from stdin
> [(stdin):1] Test sorting and filtering of output
> [(stdin):3] true
> [(stdin):6] true
-> [(stdin):8] printf 'profiling: foo\\n'
+> [(stdin):8] printf 'profiling: foo\n'
> [(stdin):10] printf 'profiling: foo'
-> [(stdin):13] printf 'profiling: foo\\n'
+> [(stdin):13] printf 'profiling: foo\n'
> [(stdin):16] printf 'profiling: foo'
-> [(stdin):18] printf 'a\\nb\\nc\\nd\\n'
-> [(stdin):24] printf 'a\\nb\\nc\\nd'
-> [(stdin):31] printf 'c\\nd\\nb\\na\\n'
-> [(stdin):38] printf 'c\\nd\\nb\\na'
-> [(stdin):44] printf 'a\\nprofiling: foo\\nprofiling: bar\\nb\\nc\\nd\\nprofiling: baz\\n'
-> [(stdin):50] printf 'a\\nprofiling: foo\\nprofiling: bar\\nb\\nc\\nd\\nprofiling: baz'
-> [(stdin):57] printf 'c\\nprofiling: foo\\nprofiling: bar\\nd\\nb\\na\\nprofiling: baz\\n'
-> [(stdin):64] printf 'c\\nprofiling: foo\\nprofiling: bar\\nd\\nb\\na\\nprofiling: baz'
+> [(stdin):18] printf 'a\nb\nc\nd\n'
+> [(stdin):24] printf 'a\nb\nc\nd'
+> [(stdin):31] printf 'c\nd\nb\na\n'
+> [(stdin):38] printf 'c\nd\nb\na'
+> [(stdin):44] printf 'a\nprofiling: foo\nprofiling: bar\nb\nc\nd\nprofiling: baz\n'
+> [(stdin):50] printf 'a\nprofiling: foo\nprofiling: bar\nb\nc\nd\nprofiling: baz'
+> [(stdin):57] printf 'c\nprofiling: foo\nprofiling: bar\nd\nb\na\nprofiling: baz\n'
+> [(stdin):64] printf 'c\nprofiling: foo\nprofiling: bar\nd\nb\na\nprofiling: baz'
> Test suite from stdin OK
p Check the Right Prefix Length (19) for "output sort"
< $ cat
< > 123456789012345678 A line
< > 123456789012345678 B line
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
p Check user-defined prefix length for "output sort"
! output sort 5
< $ cat
< > 000 A line
< > 000 B line
-$ perl ${bindir:=.}/tesh
+$ ${bindir:=.}/tesh
+++ /dev/null
-#! /usr/bin/env perl
-
-# Copyright (c) 2012-2015. The SimGrid Team.
-# All rights reserved.
-
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the license (GNU LGPL) which comes with this package.
-eval 'exec perl -S $0 ${1+"$@"}'
- if $running_under_some_shell;
-
-# If you change this file, please stick to the formatting you got with:
-# perltidy --backup-and-modify-in-place --maximum-line-length=180 --output-line-ending=unix --cuddled-else
-
-=encoding UTF-8
-
-=head1 NAME
-
-tesh -- testing shell
-
-=head1 SYNOPSIS
-
-B<tesh> [I<options>]... I<testsuite>
-
-=head1 DESCRIPTION
-
-Tesh is the testing shell, a specialized shell for running tests. It
-provides the specified input to the tested commands, and check that
-they produce the expected output and return the expected value.
-
-=head1 OPTIONS
-
- --cd some/directory : ask tesh to switch the working directory before
- launching the tests
- --setenv var=value : set a specific environment variable
- --cfg arg : add parameter --cfg=arg to each command line
- --log arg : add parameter --log=arg to each command line
- --enable-coverage : ignore output lines starting with "profiling:"
- --enable-sanitizers : ignore output lines starting with containing warnings
-
-=head1 TEST SUITE FILE SYTAX
-
-A test suite is composed of one or several I<command blocks> separated
-by empty lines, each of them being composed of a command to run, its
-input text and the expected output.
-
-The first char of each line specifies the type of line according to
-the following list. The second char of each line is ignored.
-
- `$' command to run in foreground
- `&' command to run in background
-
- `<' input to pass to the command
- `>' output expected from the command
-
- `!' metacommand, which can be one of:
- `timeout' <integer>|no
- `expect signal' <signal name>
- `expect return' <integer>
- `output' <ignore|display>
- `setenv <key>=<val>'
-
- `p' an informative message to print
-
-If the expected output do not match the produced output, or if the
-command did not end as expected, Tesh provides an error message (see
-the OUTPUT section below) and stops.
-
-=head2 Command blocks examples
-
-In a given command block, you can declare the command, its input and
-its expected output in the order that you see fit.
-
- $ cat
- < TOTO
- > TOTO
-
- > TOTO
- $ cat
- < TOTO
-
- > TOTO
- < TOTO
- $ cat
-
-You can group several commands together, provided that they don't have
-any input nor output.
-
- $ mkdir testdir
- $ cd testdir
-
-=head2 Enforcing the command return code
-
-By default, Tesh enforces that the tested command returns 0. If not,
-it fails with an appropriate message and returns I<code+40> itself.
-
-You specify that a given command block is expected to return another
-code as follows:
-
- # This command MUST return 42
- ! expect return 42
- $ sh -e "exit 42"
-
-The I<expect return> construct applies only to the next command block.
-
-=head2 Commands that are expected to raise signals
-
-By default, Tesh detects when the command is killed by a signal (such
-as SEGV on segfaults). This is usually unexpected and unfortunate. But
-if not, you can specify that a given command block is expected to fail
-with a signal as follows:
-
- # This command MUST raise a segfault
- ! expect signal SIGSEGV
- $ ./some_failing_code
-
-The I<expect signal> construct applies only to the next command block.
-
-=head2 Timeouts
-
-By default, no command is allowed to run more than 5 seconds. You can
-change this value as follows:
-
- # Allow some more time to the command
- ! timeout 60
- $ ./some_longer_command
-
-You can also disable the timeout completely by passing "no" as a value:
-
- # This command will never timeout
- ! timeout no
- $ ./some_very_long_but_safe_command
-
-=head2 Setting environment variables
-
-You can modify the environment of the tested commands as follows:
-
- ! setenv PATH=/bin
- $ my_command
-
-=head2 Not enforcing the expected output
-
-By default, the commands output is matched against the one expected,
-and an error is raised on discrepancy. Metacommands to change this:
-
-=over 4
-
-=item output ignore
-
-The output is completely discarded.
-
-=item output display
-
-The output is displayed, but no error is issued if it differs from the
-expected output.
-
-=item output sort
-
-The output is sorted before comparison (see next section).
-
-=back
-
-=head2 Sorting output
-
-If the order of the command output changes between runs, you want to
-sort it before enforcing that it is exactly what you expect. In
-SimGrid for example, this happens when parallel execution is
-activated: User processes are run in parallel at each timestamp, and
-the output is not reproducible anymore. Until you sort the lines.
-
-You can sort the command output as follows:
-
- ! output sort
- $ ./some_multithreaded_command
-
-Sorting lines this ways often makes the tesh output very intricate,
-complicating the error analysis: the process logical order is defeated
-by the lexicographical sort.
-
-The solution is to prefix each line of your output with temporal
-information so that lines can be grouped by timestamps. The
-lexicographical sort then only applies to lines that occurred at the
-same timestamp. Here is a SimGrid example:
-
- # Sort only lines depending on the first 19 chars
- ! output sort 19
- $ ./some_simgrid_simulator --log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n
-
-This approach may seem surprizing at the first glance but it does its job:
-
-=over 4
-
-=item Every timestamps remain separated, as it should;
-
-=item In each timestamp, the output order of processes become
- reproducible: that's the lexicographical order of their name;
-
-=item For each process, the order of its execution is preserved: its
- messages within a given timestamp are not reordered.
-
-=back
-
-That way, tesh can do its job (no false positive, no false negative)
-despite the unpredictable order of executions of processes within a
-timestamp, and reported errors remain easy to analyze (execution of a
-given process preserved).
-
-This example is very SimGrid oriented, but the feature could even be
-usable by others, who knows?
-
-
-=head1 BUILTIN COMMANDS
-
-=head2 mkfile: creating a file
-
-This command creates a file of the name provided as argument, and adds
-the content it gets as input.
-
- $ mkfile myFile
- > some content
- > to the file
-
-It is not possible to use the cat command, as one would expect,
-because stream redirections are currently not implemented in Tesh.
-
-=head1 BUGS, LIMITATIONS AND POSSIBLE IMPROVEMENTS
-
-The main limitation is the lack of stream redirections in the commands
-(">", "<" and "|" shell constructs and friends). The B<mkfile> builtin
-command makes this situation bearable.
-
-It would be nice if we could replace the tesh file completely with
-command line flags when the output is not to be verified.
-
-=cut
-
-BEGIN {
- # Disabling IPC::Run::Debug saves tons of useless calls.
- $ENV{'IPCRUNDEBUG'} = 'none'
- unless exists $ENV{'IPCRUNDEBUG'};
-}
-
-my $enable_coverage = 0;
-my $enable_sanitizers = 0;
-my $diff_tool = 0;
-my $diff_tool_tmp_fh = 0;
-my $diff_tool_tmp_filename = 0;
-my $sort_prefix = -1;
-my $tesh_file;
-my $tesh_name;
-my $error = 0;
-my $exitcode = 0;
-my @bg_cmds;
-my (%environ);
-$SIG{'PIPE'} = 'IGNORE';
-
-my $path = $0;
-$path =~ s|[^/]*$||;
-push @INC, $path;
-
-use lib "@CMAKE_BINARY_DIR@/bin";
-
-use Diff qw(diff); # postpone a bit to have time to change INC
-
-use Getopt::Long qw(GetOptions);
-use strict;
-use Text::ParseWords;
-use IPC::Run qw(start run timeout finish);
-use IO::File;
-use English;
-
-####
-#### Portability bits for windows
-####
-
-use constant RUNNING_ON_WINDOWS => ( $OSNAME =~ /^(?:mswin|dos|os2)/oi );
-use POSIX qw(:sys_wait_h WIFEXITED WIFSIGNALED WIFSTOPPED WEXITSTATUS WTERMSIG WSTOPSIG
- :signal_h SIGINT SIGTERM SIGKILL SIGABRT SIGSEGV);
-
-BEGIN {
- if (RUNNING_ON_WINDOWS) { # Missing on windows
- *WIFEXITED = sub { not $_[0] & 127 };
- *WEXITSTATUS = sub { $_[0] >> 8 };
- *WIFSIGNALED = sub { ( $_[0] & 127 ) && ( $_[0] & 127 != 127 ) };
- *WTERMSIG = sub { $_[0] & 127 };
-
- # used on the command lines
- $environ{'EXEEXT'} = ".exe";
- }
-}
-
-
-####
-#### Command line option handling
-####
-
-my %opts = ( "debug" => 0,
- "timeout" => 5, # No command should run any longer than 5 seconds by default
- );
-
-Getopt::Long::config( 'bundling', 'no_getopt_compat', 'no_auto_abbrev' );
-GetOptions(
- 'debug|d' => \$opts{"debug"},
-
- 'difftool=s' => \$diff_tool,
-
- 'cd=s' => sub { cd_cmd( $_[1] ) },
- 'timeout=s' => \$opts{'timeout'},
- 'setenv=s' => sub { setenv_cmd( $_[1] ) },
- 'cfg=s' => sub { $opts{'cfg'} .= " --cfg=$_[1]" },
- 'log=s' => sub { $opts{'log'} .= " --log=$_[1]" },
- 'enable-coverage+' => \$enable_coverage,
- 'enable-sanitizers+' => \$enable_sanitizers,
-);
-
-$tesh_file = pop @ARGV;
-$tesh_name = $tesh_file;
-$tesh_name =~ s|^.*?/([^/]*)$|$1|;
-
-print "Enable coverage\n" if ($enable_coverage);
-print "Enable sanitizers\n" if ($enable_sanitizers);
-
-if ($diff_tool) {
- use File::Temp qw/ tempfile /;
- ( $diff_tool_tmp_fh, $diff_tool_tmp_filename ) = tempfile();
- print "New tesh: $diff_tool_tmp_filename\n";
-}
-
-if ( $tesh_file =~ m/(.*)\.tesh/ ) {
- my $fullname = $tesh_file;
- if (not ($fullname =~ m|^/|)) { # not absolute path
- my $dir = qx,pwd,;
- chomp($dir);
- $fullname = "$dir/$fullname"
- }
- print "Test suite '$tesh_file'\n";
-} else {
- $tesh_name = "(stdin)";
- print "Test suite from stdin\n";
-}
-
-###########################################################################
-
-sub exec_cmd {
- my %cmd = %{ $_[0] };
- if ( $opts{'debug'} ) {
- map { print "IN: $_\n" } @{ $cmd{'in'} };
- map { print "OUT: $_\n" } @{ $cmd{'out'} };
- print "CMD: $cmd{'cmd'}\n";
- }
-
- # substitute environment variables
- foreach my $key ( keys %environ ) {
- $cmd{'cmd'} = var_subst( $cmd{'cmd'}, $key, $environ{$key} );
- }
-
- # substitute remaining variables, if any
- while ( $cmd{'cmd'} =~ /\$\{(\w+)(?::[=-][^}]*)?\}/ ) {
- $cmd{'cmd'} = var_subst( $cmd{'cmd'}, $1, "" );
- }
- while ( $cmd{'cmd'} =~ /\$(\w+)/ ) {
- $cmd{'cmd'} = var_subst( $cmd{'cmd'}, $1, "" );
- }
-
- # add cfg and log options
- $cmd{'cmd'} .= " $opts{'cfg'}"
- if ( defined( $opts{'cfg'} ) && length( $opts{'cfg'} ) );
- $cmd{'cmd'} .= " $opts{'log'}"
- if ( defined( $opts{'log'} ) && length( $opts{'log'} ) );
-
- # finally trim any remaining space chars
- $cmd{'cmd'} =~ s/^\s+//;
- $cmd{'cmd'} =~ s/\s+$//;
-
- print "[$tesh_name:$cmd{'line'}] $cmd{'cmd'}\n";
-
- $cmd{'return'} ||= 0;
- $cmd{'timeout'} ||= $opts{'timeout'};
-
-
- ###
- # exec the command line
-
- my @cmdline;
- if(defined $ENV{VALGRIND_COMMAND}) {
- push @cmdline, $ENV{VALGRIND_COMMAND};
- push @cmdline, split(" ", $ENV{VALGRIND_OPTIONS});
- if($cmd{'timeout'} ne 'no'){
- $cmd{'timeout'}=$cmd{'timeout'}*20
- }
- }
- push @cmdline, quotewords( '\s+', 0, $cmd{'cmd'} );
- my $input = defined($cmd{'in'})? join("\n",@{$cmd{'in'}}) : "";
- my $output = " " x 10240; $output = ""; # Preallocate 10kB, and reset length to 0
- $cmd{'got'} = \$output;
- $cmd{'job'} = start \@cmdline, '<', \$input, '>&', \$output,
- ($cmd{'timeout'} eq 'no' ? () : timeout($cmd{'timeout'}));
-
- if ( $cmd{'background'} ) {
- # Just enqueue the job. It will be dealed with at the end
- push @bg_cmds, \%cmd;
- } else {
- # Deal with its ending conditions right away
- analyze_result( \%cmd );
- }
-}
-
-sub analyze_result {
- my %cmd = %{ $_[0] };
- $cmd{'timeouted'} = 0; # initialization
-
- # Wait for the end of the child process
- #####
- eval {
- finish( $cmd{'job'} );
- };
- if ($@) { # deal with the errors that occurred in the child process
- if ($@ =~ /timeout/) {
- $cmd{'job'}->kill_kill;
- $cmd{'timeouted'} = 1;
- } elsif ($@ =~ /^ack / and $@ =~ /pipe/) { # IPC::Run is not very expressive about the pipes that it gets :(
- print STDERR "Tesh: Broken pipe (ignored).\n";
- } else {
- die $@; # Don't know what it is, so let it go.
- }
- }
-
- # Gather information
- ####
-
- # pop all output from executing child
- my @got;
- map { print "GOT: $_\n" } ${$cmd{'got'}} if $opts{'debug'};
- foreach my $got ( split("\n", ${$cmd{'got'}}) ) {
- $got =~ s/\r//g;
- chomp $got;
- print $diff_tool_tmp_fh "> $got\n" if ($diff_tool);
-
- unless (( $enable_coverage and $got =~ /^profiling:/ ) or
- ( $enable_sanitizers and $got =~ m/WARNING: ASan doesn't fully support/) or
- ( $got =~ m/Unable to clean temporary file C:/)) # Crude hack to ignore cruft from Java on Windows
- {
- push @got, $got;
- }
- }
-
- # How did the child process terminate?
- my $status = $?;
- $cmd{'gotret'} = "Unparsable status. Please report this tesh bug.";
- if ( $cmd{'timeouted'} ) {
- $cmd{'gotret'} = "timeout after $cmd{'timeout'} sec";
- $error = 1;
- $exitcode = 3;
- } elsif ( WIFEXITED($status) ) {
- $exitcode = WEXITSTATUS($status) + 40;
- $cmd{'gotret'} = "returned code " . WEXITSTATUS($status);
- } elsif ( WIFSIGNALED($status) ) {
- my $code;
- if ( WTERMSIG($status) == SIGINT ) { $code = "SIGINT"; }
- elsif ( WTERMSIG($status) == SIGTERM ) { $code = "SIGTERM"; }
- elsif ( WTERMSIG($status) == SIGKILL ) { $code = "SIGKILL"; }
- elsif ( WTERMSIG($status) == SIGABRT ) { $code = "SIGABRT"; }
- elsif ( WTERMSIG($status) == SIGSEGV ) { $code = "SIGSEGV"; }
- $exitcode = WTERMSIG($status) + 4;
- $cmd{'gotret'} = "got signal $code";
- }
-
- # How was it supposed to terminate?
- my $wantret;
- if ( defined( $cmd{'expect'} ) and ( $cmd{'expect'} ne "" ) ) {
- $wantret = "got signal $cmd{'expect'}";
- } else {
- $wantret = "returned code " . ( defined( $cmd{'return'} ) ? $cmd{'return'} : 0 );
- }
-
- # Enforce the outcome
- ####
-
- # Did it end as expected?
- if ( $cmd{'gotret'} ne $wantret ) {
- $error = 1;
- my $msg = "Test suite `$tesh_name': NOK (<$tesh_name:$cmd{'line'}> $cmd{'gotret'})\n";
- if ( scalar @got ) {
- $msg = $msg . "Output of <$tesh_name:$cmd{'line'}> so far:\n";
- map { $msg .= "|| $_\n" } @got;
- } else {
- $msg .= "<$tesh_name:$cmd{'line'}> No output so far.\n";
- }
- print STDERR "$msg";
- }
-
- # Does the output match?
- if ( $cmd{'sort'} ) {
- sub mysort {
- substr( $a, 0, $sort_prefix ) cmp substr( $b, 0, $sort_prefix );
- }
- use sort 'stable';
- if ( $sort_prefix > 0 ) {
- @got = sort mysort @got;
- } else {
- @got = sort @got;
- }
- while ( @got and $got[0] eq "" ) {
- shift @got;
- }
-
- # Sort the expected output too, to make tesh files easier to write for humans
- if ( defined( $cmd{'out'} ) ) {
- if ( $sort_prefix > 0 ) {
- @{ $cmd{'out'} } = sort mysort @{ $cmd{'out'} };
- } else {
- @{ $cmd{'out'} } = sort @{ $cmd{'out'} };
- }
- while ( @{ $cmd{'out'} } and ${ $cmd{'out'} }[0] eq "" ) {
- shift @{ $cmd{'out'} };
- }
- }
- }
-
- # Report the output if asked so or if it differs
- if ( defined( $cmd{'output display'} ) ) {
- print "[Tesh/INFO] Here is the (ignored) command output:\n";
- map { print "||$_\n" } @got;
- } elsif ( defined( $cmd{'output ignore'} ) ) {
- print "(ignoring the output of <$tesh_name:$cmd{'line'}> as requested)\n";
- } else {
- my $diff = build_diff( \@{ $cmd{'out'} }, \@got );
-
- if ( length $diff ) {
- print "Output of <$tesh_name:$cmd{'line'}> mismatch" . ( $cmd{'sort'} ? " (even after sorting)" : "" ) . ":\n";
- map { print "$_\n" } split( /\n/, $diff );
- if ( $cmd{'sort'} ) {
- print "WARNING: Both the observed output and expected output were sorted as requested.\n";
- print "WARNING: Output were only sorted using the $sort_prefix first chars.\n"
- if ( $sort_prefix > 0 );
- print "WARNING: Use <! output sort 19> to sort by simulated date and process ID only.\n";
-
- # print "----8<--------------- Begin of unprocessed observed output (as it should appear in file):\n";
- # map {print "> $_\n"} @{$cmd{'unsorted got'}};
- # print "--------------->8---- End of the unprocessed observed output.\n";
- }
-
- print "Test suite `$tesh_name': NOK (<$tesh_name:$cmd{'line'}> output mismatch)\n";
- exit 2;
- }
- }
-}
-
-# parse tesh file
-my $infh; # The file descriptor from which we should read the teshfile
-if ( $tesh_name eq "(stdin)" ) {
- $infh = *STDIN;
-} else {
- open $infh, $tesh_file
- or die "[Tesh/CRITICAL] Unable to open $tesh_file: $!\n";
-}
-
-my %cmd; # everything about the next command to run
-my $line_num = 0;
-LINE: while ( not $error and defined( my $line = <$infh> )) {
- chomp $line;
- $line =~ s/\r//g;
-
- $line_num++;
- print "[TESH/debug] $line_num: $line\n" if $opts{'debug'};
-
- # deal with line continuations
- while ( $line =~ /^(.*?)\\$/ ) {
- my $next = <$infh>;
- die "[TESH/CRITICAL] Continued line at end of file\n"
- unless defined($next);
- $line_num++;
- chomp $next;
- print "[TESH/debug] $line_num: $next\n" if $opts{'debug'};
- $line = $1 . $next;
- }
-
- # If the line is empty, run any previously defined block and proceed to next line
- unless ( $line =~ m/^(.)(.*)$/ ) {
- if ( defined( $cmd{'cmd'} ) ) {
- exec_cmd( \%cmd );
- %cmd = ();
- }
- print $diff_tool_tmp_fh "$line\n" if ($diff_tool);
- next LINE;
- }
-
- my ( $cmd, $arg ) = ( $1, $2 );
- print $diff_tool_tmp_fh "$line\n" if ( $diff_tool and $cmd ne '>' );
- $arg =~ s/^ //g;
- $arg =~ s/\r//g;
- $arg =~ s/\\\\/\\/g;
-
- # Deal with the lines that can contribute to the current command block
- if ( $cmd =~ /^#/ ) { # comment
- next LINE;
- } elsif ( $cmd eq '>' ) { # expected result line
- print "[TESH/debug] push expected result\n" if $opts{'debug'};
- push @{ $cmd{'out'} }, $arg;
- next LINE;
-
- } elsif ( $cmd eq '<' ) { # provided input
- print "[TESH/debug] push provided input\n" if $opts{'debug'};
- push @{ $cmd{'in'} }, $arg;
- next LINE;
-
- } elsif ( $cmd eq 'p' ) { # comment
- print "[$tesh_name:$line_num] $arg\n";
- next LINE;
-
- }
-
- # We dealt with all sort of lines that can contribute to a command block, so we have something else here.
- # If we have something buffered, run it now and start a new block
- if ( defined( $cmd{'cmd'} ) ) {
- exec_cmd( \%cmd );
- %cmd = ();
- }
-
- # Deal with the lines that must be placed before a command block
- if ( $cmd eq '$' ) { # Command
- if ( $arg =~ /^mkfile / ) { # "mkfile" command line
- die "[TESH/CRITICAL] Output expected from mkfile command!\n"
- if scalar @{ cmd { 'out' } };
-
- $cmd{'arg'} = $arg;
- $cmd{'arg'} =~ s/mkfile //;
- mkfile_cmd( \%cmd );
- %cmd = ();
-
- } elsif ( $arg =~ /^\s*cd / ) {
- die "[TESH/CRITICAL] Input provided to cd command!\n"
- if scalar @{ cmd { 'in' } };
- die "[TESH/CRITICAL] Output expected from cd command!\n"
- if scalar @{ cmd { 'out' } };
-
- $arg =~ s/^ *cd //;
- cd_cmd($arg);
- %cmd = ();
-
- } else { # regular command
- $cmd{'cmd'} = $arg;
- $cmd{'line'} = $line_num;
- }
-
- } elsif ( $cmd eq '&' ) { # background command line
- die "[TESH/CRITICAL] mkfile cannot be run in background\n"
- if ($arg =~ /^mkfile/);
- die "[TESH/CRITICAL] cd cannot be run in background\n"
- if ($arg =~ /^cd/);
-
- $cmd{'background'} = 1;
- $cmd{'cmd'} = $arg;
- $cmd{'line'} = $line_num;
-
- # Deal with the meta-commands
- } elsif ( $line =~ /^! (.*)/) {
- $line = $1;
-
- if ( $line =~ /^output sort/ ) {
- $cmd{'sort'} = 1;
- if ( $line =~ /^output sort\s+(\d+)/ ) {
- $sort_prefix = $1;
- }
- } elsif ($line =~ /^output ignore/ ) {
- $cmd{'output ignore'} = 1;
- } elsif ( $line =~ /^output display/ ) {
- $cmd{'output display'} = 1;
- } elsif ( $line =~ /^expect signal (\w*)/ ) {
- $cmd{'expect'} = $1;
- } elsif ( $line =~ /^expect return/ ) {
- $line =~ s/^expect return //g;
- $line =~ s/\r//g;
- $cmd{'return'} = $line;
- } elsif ( $line =~ /^setenv/ ) {
- $line =~ s/^setenv //g;
- $line =~ s/\r//g;
- setenv_cmd($line);
- } elsif ( $line =~ /^timeout/ ) {
- $line =~ s/^timeout //;
- $line =~ s/\r//g;
- $cmd{'timeout'} = $line;
- }
- } else {
- die "[TESH/CRITICAL] parse error: $line\n";
- }
-}
-
-# We are done reading the input file
-close $infh unless ( $tesh_name eq "(stdin)" );
-
-# Deal with last command, if any
-if ( defined( $cmd{'cmd'} ) ) {
- exec_cmd( \%cmd );
- %cmd = ();
-}
-
-foreach (@bg_cmds) {
- my %test = %{$_};
- analyze_result( \%test );
-}
-
-if ($diff_tool) {
- close $diff_tool_tmp_fh;
- system("$diff_tool $diff_tool_tmp_filename $tesh_file");
- unlink $diff_tool_tmp_filename;
-}
-
-if ( $error != 0 ) {
- exit $exitcode;
-} elsif ( $tesh_name eq "(stdin)" ) {
- print "Test suite from stdin OK\n";
-} else {
- print "Test suite `$tesh_name' OK\n";
-}
-
-exit 0;
-
-####
-#### Helper functions
-####
-
-sub build_diff {
- my $res;
- my $diff = Diff->new(@_);
-
- $diff->Base(1); # Return line numbers, not indices
- my $chunk_count = $diff->Next(-1); # Compute the amount of chuncks
- return "" if ( $chunk_count == 1 && $diff->Same() );
- $diff->Reset();
- while ( $diff->Next() ) {
- my @same = $diff->Same();
- if ( $diff->Same() ) {
- if ( $diff->Next(0) > 1 ) { # not first chunk: print 2 first lines
- $res .= ' ' . $same[0] . "\n";
- $res .= ' ' . $same[1] . "\n" if ( scalar @same > 1 );
- }
- $res .= "...\n" if ( scalar @same > 2 );
-
- # $res .= $diff->Next(0)."/$chunk_count\n";
- if ( $diff->Next(0) < $chunk_count ) { # not last chunk: print 2 last lines
- $res .= ' ' . $same[ scalar @same - 2 ] . "\n"
- if ( scalar @same > 1 );
- $res .= ' ' . $same[ scalar @same - 1 ] . "\n";
- }
- }
- next if $diff->Same();
- map { $res .= "- $_\n" } $diff->Items(1);
- map { $res .= "+ $_\n" } $diff->Items(2);
- }
- return $res;
-}
-
-# Helper function replacing any occurence of variable '$name' by its '$value'
-# As in Bash, ${$value:=BLABLA} is rewritten to $value if set or to BLABLA if $value is not set
-sub var_subst {
- my ( $text, $name, $value ) = @_;
- if ($value) {
- $text =~ s/\$\{$name(?::[=-][^}]*)?\}/$value/g;
- $text =~ s/\$$name(\W|$)/$value$1/g;
- } else {
- $text =~ s/\$\{$name:=([^}]*)\}/$1/g;
- $text =~ s/\$\{$name\}//g;
- $text =~ s/\$$name(\W|$)/$1/g;
- }
- return $text;
-}
-
-################################ The possible commands ################################
-
-sub mkfile_cmd($) {
- my %cmd = %{ $_[0] };
- my $file = $cmd{'arg'};
- print STDERR "[Tesh/INFO] mkfile $file. Ctn: >>".join( '\n', @{ $cmd{'in'} })."<<\n"
- if $opts{'debug'};
-
- unlink($file);
- open( FILE, ">$file" )
- or die "[Tesh/CRITICAL] Unable to create file $file: $!\n";
- print FILE join( "\n", @{ $cmd{'in'} } );
- print FILE "\n" if ( scalar @{ $cmd{'in'} } > 0 );
- close(FILE);
-}
-
-# Command CD. Just change to the provided directory
-sub cd_cmd($) {
- my $directory = shift;
- my $failure = 1;
- if ( -e $directory && -d $directory ) {
- chdir("$directory");
- print "[Tesh/INFO] change directory to $directory\n";
- $failure = 0;
- } elsif ( -e $directory ) {
- print "Cannot change directory to '$directory': it is not a directory\n";
- } else {
- print "Chdir to $directory failed: No such file or directory\n";
- }
- if ( $failure == 1 ) {
- print "Test suite `$tesh_name': NOK (system error)\n";
- exit 4;
- }
-}
-
-# Command setenv. Gets "variable=content", and update the environment accordingly
-sub setenv_cmd($) {
- my $arg = shift;
- if ( $arg =~ /^(.*?)=(.*)$/ ) {
- my ( $var, $ctn ) = ( $1, $2 );
- print "[Tesh/INFO] setenv $var=$ctn\n";
- $environ{$var} = $ctn;
- $ENV{$var} = $ctn;
- } else {
- die "[Tesh/CRITICAL] Malformed argument to setenv: expected 'name=value' but got '$arg'\n";
- }
-}
--- /dev/null
+=encoding UTF-8
+
+=head1 NAME
+
+tesh -- testing shell
+
+=head1 SYNOPSIS
+
+B<tesh> [I<options>]... I<testsuite>
+
+=head1 DESCRIPTION
+
+Tesh is the testing shell, a specialized shell for running tests. It
+provides the specified input to the tested commands, and check that
+they produce the expected output and return the expected value.
+
+=head1 OPTIONS
+
+ --help : display help
+ --cd some/directory : ask tesh to switch the working directory before
+ launching the tests
+ --setenv var=value : set a specific environment variable
+ --cfg arg : add parameter --cfg=arg to each command line
+ --log arg : add parameter --log=arg to each command line
+ --enable-coverage : ignore output lines starting with "profiling:"
+
+
+=head1 TEST SUITE FILE SYTAX
+
+A test suite is composed of one or several I<command blocks> separated
+by empty lines, each of them being composed of a command to run, its
+input text and the expected output.
+
+The first char of each line specifies the type of line according to
+the following list. The second char of each line is ignored.
+
+ `$' command to run in foreground
+ `&' command to run in background
+
+ `<' input to pass to the command
+ `>' output expected from the command
+
+ `!' metacommand, which can be one of:
+ `timeout' <integer>|no
+ `expect signal' <signal name>
+ `expect return' <integer>
+ `output' <ignore|display>
+ `output sort' [integer]
+ `setenv <key>=<val>'
+ `ignore' <regexp>
+
+ `p' an informative message to print
+ `#' a comment
+
+If the expected output do not match the produced output, or if the
+command did not end as expected, Tesh provides an error message (see
+the OUTPUT section below) and stops.
+
+=head2 Command blocks examples
+
+In a given command block, you can declare the command, its input and
+its expected output in the order that you see fit.
+
+ $ cat
+ < TOTO
+ > TOTO
+
+ > TOTO
+ $ cat
+ < TOTO
+
+ > TOTO
+ < TOTO
+ $ cat
+
+You can group several commands together, provided that they don't have
+any input nor output.
+
+ $ mkdir testdir
+ $ cd testdir
+
+=head2 Enforcing the command return code
+
+By default, Tesh enforces that the tested command returns 0. If not,
+it fails with an appropriate message and returns I<code+40> itself.
+
+You specify that a given command block is expected to return another
+code as follows:
+
+ # This command MUST return 42
+ ! expect return 42
+ $ sh -e "exit 42"
+
+The I<expect return> construct applies only to the next command block.
+
+=head2 Commands that are expected to raise signals
+
+By default, Tesh detects when the command is killed by a signal (such
+as SEGV on segfaults). This is usually unexpected and unfortunate. But
+if not, you can specify that a given command block is expected to fail
+with a signal as follows:
+
+ # This command MUST raise a segfault
+ ! expect signal SIGSEGV
+ $ ./some_failing_code
+
+The I<expect signal> construct applies only to the next command block.
+
+=head2 Timeouts
+
+By default, no command is allowed to run more than 5 seconds. You can
+change this value as follows:
+
+ # Allow some more time to the command
+ ! timeout 60
+ $ ./some_longer_command
+
+You can also disable the timeout completely by passing "no" as a value:
+
+ # This command will never timeout
+ ! timeout no
+ $ ./some_very_long_but_safe_command
+
+=head2 Setting environment variables
+
+You can modify the environment of the tested commands as follows:
+
+ ! setenv PATH=/bin
+ $ my_command
+
+You can also set an envirmnent variable from the command line:
+
+ tesh --setenv bindir=/opt/bin/
+
+And then use it within the tesh file:
+
+ $ ${bindir}/myprogram
+
+Tesh also supports perl default value for undefined variables:
+
+ $ ${bindir:=/usr/bin}/myprogram
+
+=head2 Not enforcing the expected output
+
+By default, the commands output is matched against the one expected,
+and an error is raised on discrepancy. Metacommands to change this:
+
+=over 4
+
+=item output ignore
+
+The output is completely discarded.
+
+=item output display
+
+The output is displayed, but no error is issued if it differs from the
+expected output.
+
+=item output sort
+
+The output is sorted before comparison (see next section).
+
+=back
+
+=head2 Sorting output
+
+If the order of the command output changes between runs, you want to
+sort it before enforcing that it is exactly what you expect. In
+SimGrid for example, this happens when parallel execution is
+activated: User processes are run in parallel at each timestamp, and
+the output is not reproducible anymore. Until you sort the lines.
+
+You can sort the command output as follows:
+
+ ! output sort
+ $ ./some_multithreaded_command
+
+Sorting lines this ways often makes the tesh output very intricate,
+complicating the error analysis: the process logical order is defeated
+by the lexicographical sort.
+
+The solution is to prefix each line of your output with temporal
+information so that lines can be grouped by timestamps. The
+lexicographical sort then only applies to lines that occured at the
+same timestamp. Here is a SimGrid example:
+
+ # Sort only lines depending on the first 19 chars
+ ! output sort 19
+ $ ./some_simgrid_simulator --log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n
+
+This approach may seem surprizing at the first glance but it does its job:
+
+=over 4
+
+=item Every timestamps remain separated, as it should;
+
+=item In each timestamp, the output order of processes become
+ reproducible: that's the lexicographical order of their name;
+
+=item For each process, the order of its execution is preserved: its
+ messages within a given timestamp are not reordered.
+
+=back
+
+That way, tesh can do its job (no false positive, no false negative)
+despite the unpredictable order of executions of processes within a
+timestamp, and reported errors remain easy to analyze (execution of a
+given process preserved).
+
+This example is very SimGrid oriented, but the feature could even be
+usable by others, who knows?
+
+=head2 Ignoring some output
+
+Some outputed lines can be ignored by setting the ignore command followed
+by a regular expression:
+
+ ! ignore .*0x[0-9A-F]+\.
+ $ printf 'word\nMemory address: 0x42AA42.\nanotherword\n'
+ > word
+ > anotherword
+
+
+=head2 Colored and formatted text
+
+Tesh removes ANSI/VT100 control sequences from outputed text to make easier the writing of tests.
+
+ $ printf "I \033[0;31mlove\033[0m tesh\n"
+ > I love tesh
+
+
+
+=head1 BUILTIN COMMANDS
+
+=head2 mkfile: creating a file
+
+This command creates a file of the name provided as argument, and adds
+the content it gets as input.
+
+ $ mkfile myFile
+ > some content
+ > to the file
+
+It is not possible to use the cat command, as one would expect,
+because stream redirections are currently not implemented in Tesh.
+
+=head1 BUGS, LIMITATIONS AND POSSIBLE IMPROVEMENTS
+
+The main limitation is the lack of stream redirections in the commands
+(">", "<" and "|" shell constructs and friends). The B<mkfile> builtin
+command makes this situation bearable.
+
+It would be nice if we could replace the tesh file completely with
+command line flags when the output is not to be verified.
+
+
+=cut
--- /dev/null
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+
+tesh -- testing shell
+========================
+
+Copyright (c) 2012-2015. The SimGrid Team.
+All rights reserved.
+
+This program is free software; you can redistribute it and/or modify it
+under the terms of the license (GNU LGPL) which comes with this package.
+
+#TODO: <glesserd> emptty: does "! output display" also disable the comparaison with the output ? yes
+
+#TODO: child of child of child printf => tester dans rsg...
+
+#TODO: regular expression in output
+#TODO: linked regular expression in output
+
+
+"""
+
+
+import sys, os
+import shlex
+import re
+import difflib
+import signal
+import argparse
+
+if sys.version_info[0] == 3:
+ import subprocess
+ import _thread
+elif sys.version_info[0] < 3:
+ import subprocess32 as subprocess
+ import thread as _thread
+else:
+ raise "This program has not been made to exist this long"
+
+
+
+##############
+#
+# Utilities
+#
+#
+
+
+# Singleton metaclass that works in Python 2 & 3
+# http://stackoverflow.com/questions/6760685/creating-a-singleton-in-python
+class _Singleton(type):
+ """ A metaclass that creates a Singleton base class when called. """
+ _instances = {}
+ def __call__(cls, *args, **kwargs):
+ if cls not in cls._instances:
+ cls._instances[cls] = super(_Singleton, cls).__call__(*args, **kwargs)
+ return cls._instances[cls]
+class Singleton(_Singleton('SingletonMeta', (object,), {})): pass
+
+SIGNALS_TO_NAMES_DICT = dict((getattr(signal, n), n) \
+ for n in dir(signal) if n.startswith('SIG') and '_' not in n )
+
+
+
+#exit correctly
+def exit(errcode):
+ #If you do not flush some prints are skipped
+ sys.stdout.flush()
+ #os._exit exit even when executed within a thread
+ os._exit(errcode)
+
+
+def fatal_error(msg):
+ print("[Tesh/CRITICAL] "+str(msg))
+ exit(1)
+
+
+#Set an environment variable.
+# arg must be a string with the format "variable=value"
+def setenv(arg):
+ print("[Tesh/INFO] setenv "+arg)
+ t = arg.split("=")
+ os.environ[t[0]] = t[1]
+ #os.putenv(t[0], t[1]) does not work
+ #see http://stackoverflow.com/questions/17705419/python-os-environ-os-putenv-usr-bin-env
+
+
+#http://stackoverflow.com/questions/30734967/how-to-expand-environment-variables-in-python-as-bash-does
+def expandvars2(path):
+ return re.sub(r'(?<!\\)\$[A-Za-z_][A-Za-z0-9_]*', '', os.path.expandvars(path))
+
+# https://github.com/Cadair/jupyter_environment_kernels/issues/10
+try:
+ FileNotFoundError
+except NameError:
+ #py2
+ FileNotFoundError = OSError
+
+
+##############
+#
+# Classes
+#
+#
+
+
+
+# read file line per line (and concat line that ends with "\")
+class FileReader(Singleton):
+ def __init__(self, filename):
+ if filename is None:
+ self.filename = "(stdin)"
+ self.f = sys.stdin
+ else:
+ self.filename_raw = filename
+ self.filename = os.path.basename(filename)
+ self.f = open(self.filename_raw)
+
+ self.linenumber = 0
+
+ def linenumber(self):
+ return self.linenumber
+
+ def __repr__(self):
+ return self.filename+":"+str(self.linenumber)
+
+ def readfullline(self):
+ try:
+ line = next(self.f)
+ self.linenumber += 1
+ except StopIteration:
+ return None
+ if line[-1] == "\n":
+ txt = line[0:-1]
+ else:
+ txt = line
+ while len(line) > 1 and line[-2] == "\\":
+ txt = txt[0:-1]
+ line = next(self.f)
+ self.linenumber += 1
+ txt += line[0:-1]
+ return txt
+
+
+#keep the state of tesh (mostly configuration values)
+class TeshState(Singleton):
+ def __init__(self):
+ self.threads = []
+ self.args_suffix = ""
+ self.ignore_regexps_common = []
+
+ def add_thread(self, thread):
+ self.threads.append(thread)
+
+ def join_all_threads(self):
+ for t in self.threads:
+ t.acquire()
+ t.release()
+
+
+
+
+#Command line object
+class Cmd(object):
+ def __init__(self):
+ self.input_pipe = []
+ self.output_pipe_stdout = []
+ self.output_pipe_stderr = []
+ self.timeout = 5
+ self.args = None
+ self.linenumber = -1
+
+ self.background = False
+ self.cwd = None
+
+ self.ignore_output = False
+ self.expect_return = 0
+
+ self.output_display = False
+
+ self.sort = -1
+
+ self.ignore_regexps = TeshState().ignore_regexps_common
+
+ def add_input_pipe(self, l):
+ self.input_pipe.append(l)
+
+ def add_output_pipe_stdout(self, l):
+ self.output_pipe_stdout.append(l)
+
+ def add_output_pipe_stderr(self, l):
+ self.output_pipe_stderr.append(l)
+
+ def set_cmd(self, args, linenumber):
+ self.args = args
+ self.linenumber = linenumber
+
+ def add_ignore(self, txt):
+ self.ignore_regexps.append(re.compile(txt))
+
+ def remove_ignored_lines(self, lines):
+ for ign in self.ignore_regexps:
+ lines = [l for l in lines if not ign.match(l)]
+ return lines
+
+
+ def _cmd_mkfile(self, argline):
+ filename = argline[len("mkfile "):]
+ file = open(filename, "w")
+ if file is None:
+ fatal_error("Unable to create file "+filename)
+ file.write("\n".join(self.input_pipe))
+ file.close()
+
+ def _cmd_cd(self, argline):
+ args = shlex.split(argline)
+ if len(args) != 2:
+ fatal_error("Too many arguments to cd")
+ try:
+ os.chdir(args[1])
+ print("[Tesh/INFO] change directory to "+args[1])
+ except FileNotFoundError:
+ print("Chdir to "+args[1]+" failed: No such file or directory")
+ print("Test suite `"+FileReader().filename+"': NOK (system error)")
+ exit(4)
+
+
+ #Run the Cmd if possible.
+ # Return False if nothing has been ran.
+ def run_if_possible(self):
+ if self.can_run():
+ if self.background:
+ #Python threads loose the cwd
+ self.cwd = os.getcwd()
+ lock = _thread.allocate_lock()
+ lock.acquire()
+ TeshState().add_thread(lock)
+ _thread.start_new_thread( Cmd._run, (self, lock) )
+ else:
+ self._run()
+ return True
+ else:
+ return False
+
+
+ def _run(self, lock=None):
+ #Python threads loose the cwd
+ if self.cwd is not None:
+ os.chdir(self.cwd)
+ self.cwd = None
+
+ #retrocompatibility: support ${aaa:=.} variable format
+ def replace_perl_variables(m):
+ vname = m.group(1)
+ vdefault = m.group(2)
+ if vname in os.environ:
+ return "$"+vname
+ else:
+ return vdefault
+ self.args = re.sub(r"\${(\w+):=([^}]*)}", replace_perl_variables, self.args)
+
+ #replace bash environment variables ($THINGS) to their values
+ self.args = expandvars2(self.args)
+
+ if re.match("^mkfile ", self.args) is not None:
+ self._cmd_mkfile(self.args)
+ if lock is not None: lock.release()
+ return
+
+ if re.match("^cd ", self.args) is not None:
+ self._cmd_cd(self.args)
+ if lock is not None: lock.release()
+ return
+
+ self.args += TeshState().args_suffix
+
+ print("["+FileReader().filename+":"+str(self.linenumber)+"] "+self.args)
+
+ args = shlex.split(self.args)
+ #print (args)
+ try:
+ proc = subprocess.Popen(args, bufsize=1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+ except OSError as e:
+ if e.errno == 8:
+ e.strerror += "\nOSError: [Errno 8] Executed scripts should start with shebang line (like #!/bin/sh)"
+ raise e
+
+ try:
+ (stdout_data, stderr_data) = proc.communicate("\n".join(self.input_pipe), self.timeout)
+ except subprocess.TimeoutExpired:
+ print("Test suite `"+FileReader().filename+"': NOK (<"+FileReader().filename+":"+str(self.linenumber)+"> timeout after "+str(self.timeout)+" sec)")
+ exit(3)
+
+ if self.output_display:
+ print(stdout_data)
+
+ #remove text colors
+ ansi_escape = re.compile(r'\x1b[^m]*m')
+ stdout_data = ansi_escape.sub('', stdout_data)
+
+ #print ((stdout_data, stderr_data))
+
+ if self.ignore_output:
+ print("(ignoring the output of <"+FileReader().filename+":"+str(self.linenumber)+"> as requested)")
+ else:
+ stdouta = stdout_data.split("\n")
+ while len(stdouta) > 0 and stdouta[-1] == "":
+ del stdouta[-1]
+ stdouta = self.remove_ignored_lines(stdouta)
+
+ #the "sort" bash command is case unsensitive,
+ # we mimic its behaviour
+ if self.sort == 0:
+ stdouta.sort(key=lambda x: x.lower())
+ self.output_pipe_stdout.sort(key=lambda x: x.lower())
+ elif self.sort > 0:
+ stdouta.sort(key=lambda x: x[:self.sort].lower())
+ self.output_pipe_stdout.sort(key=lambda x: x[:self.sort].lower())
+
+ diff = list(difflib.unified_diff(self.output_pipe_stdout, stdouta,lineterm="",fromfile='expected', tofile='obtained'))
+ if len(diff) > 0:
+ print("Output of <"+FileReader().filename+":"+str(self.linenumber)+"> mismatch:")
+ for line in diff:
+ print(line)
+ print("Test suite `"+FileReader().filename+"': NOK (<"+str(FileReader())+"> output mismatch)")
+ if lock is not None: lock.release()
+ exit(2)
+
+ #print ((proc.returncode, self.expect_return))
+
+ if proc.returncode != self.expect_return:
+ if proc.returncode >= 0:
+ print("Test suite `"+FileReader().filename+"': NOK (<"+str(FileReader())+"> returned code "+str(proc.returncode)+")")
+ if lock is not None: lock.release()
+ exit(2)
+ else:
+ print("Test suite `"+FileReader().filename+"': NOK (<"+str(FileReader())+"> got signal "+SIGNALS_TO_NAMES_DICT[-proc.returncode]+")")
+ if lock is not None: lock.release()
+ exit(-proc.returncode)
+
+ if lock is not None: lock.release()
+
+
+
+ def can_run(self):
+ return self.args is not None
+
+
+
+
+
+
+
+
+
+
+##############
+#
+# Main
+#
+#
+
+
+
+if __name__ == '__main__':
+
+ parser = argparse.ArgumentParser(description='tesh -- testing shell', add_help=True)
+ group1 = parser.add_argument_group('Options')
+ group1.add_argument('teshfile', nargs='?', help='Name of teshfile, stdin if omitted')
+ group1.add_argument('--cd', metavar='some/directory', help='ask tesh to switch the working directory before launching the tests')
+ group1.add_argument('--setenv', metavar='var=value', action='append', help='set a specific environment variable')
+ group1.add_argument('--cfg', metavar='arg', help='add parameter --cfg=arg to each command line')
+ group1.add_argument('--log', metavar='arg', help='add parameter --log=arg to each command line')
+ group1.add_argument('--enable-coverage', action='store_true', help='ignore output lines starting with "profiling:"')
+
+ try:
+ options = parser.parse_args()
+ except:
+ exit(1)
+
+ if options.cd is not None:
+ os.chdir(options.cd)
+
+ if options.enable_coverage:
+ print("Enable coverage")
+ TeshState().ignore_regexps_common = [re.compile("^profiling:")]
+
+ if options.teshfile is None:
+ f = FileReader(None)
+ print("Test suite from stdin")
+ else:
+ f = FileReader(options.teshfile)
+ print("Test suite '"+f.filename+"'")
+
+ if options.setenv is not None:
+ for e in options.setenv:
+ setenv(e)
+
+ if options.cfg is not None:
+ TeshState().args_suffix += " --cfg="+options.cfg
+ if options.log is not None:
+ TeshState().args_suffix += " --log="+options.log
+
+
+ #cmd holds the current command line
+ # tech commands will add some parameters to it
+ # when ready, we execute it.
+ cmd = Cmd()
+
+ line = f.readfullline()
+ while line is not None:
+ #print(">>============="+line+"==<<")
+ if len(line) == 0:
+ #print ("END CMD block")
+ if cmd.run_if_possible():
+ cmd = Cmd()
+
+ elif line[0] == "#":
+ pass
+
+ elif line[0:2] == "p ":
+ print("["+str(FileReader())+"] "+line[2:])
+
+ elif line[0:2] == "< ":
+ cmd.add_input_pipe(line[2:])
+ elif line[0:1] == "<":
+ cmd.add_input_pipe(line[1:])
+
+ elif line[0:2] == "> ":
+ cmd.add_output_pipe_stdout(line[2:])
+ elif line[0:1] == ">":
+ cmd.add_output_pipe_stdout(line[1:])
+
+ elif line[0:2] == "$ ":
+ if cmd.run_if_possible():
+ cmd = Cmd()
+ cmd.set_cmd(line[2:], f.linenumber)
+
+ elif line[0:2] == "& ":
+ if cmd.run_if_possible():
+ cmd = Cmd()
+ cmd.set_cmd(line[2:], f.linenumber)
+ cmd.background = True
+
+ elif line[0:15] == "! output ignore":
+ cmd.ignore_output = True
+ #print("cmd.ignore_output = True")
+ elif line[0:16] == "! output display":
+ cmd.output_display = True
+ cmd.ignore_output = True
+ elif line[0:15] == "! expect return":
+ cmd.expect_return = int(line[16:])
+ #print("expect return "+str(int(line[16:])))
+ elif line[0:15] == "! expect signal":
+ sig = line[16:]
+ #get the signal integer value from the signal module
+ if sig not in signal.__dict__:
+ fatal_error("unrecognized signal '"+sig+"'")
+ sig = int(signal.__dict__[sig])
+ #popen return -signal when a process ends with a signal
+ cmd.expect_return = -sig
+ elif line[0:len("! timeout ")] == "! timeout ":
+ if "no" in line[len("! timeout "):]:
+ cmd.timeout = None
+ else:
+ cmd.timeout = int(line[len("! timeout "):])
+
+ elif line[0:len("! output sort")] == "! output sort":
+ if len(line) >= len("! output sort "):
+ sort = int(line[len("! output sort "):])
+ else:
+ sort = 0
+ cmd.sort = sort
+ elif line[0:len("! setenv ")] == "! setenv ":
+ setenv(line[len("! setenv "):])
+
+ elif line[0:len("! ignore ")] == "! ignore ":
+ cmd.add_ignore(line[len("! ignore "):])
+
+ else:
+ fatal_error("UNRECOGNIZED OPTION")
+
+
+ line = f.readfullline()
+
+ cmd.run_if_possible()
+
+ TeshState().join_all_threads()
+
+ if f.filename == "(stdin)":
+ print("Test suite from stdin OK")
+ else:
+ print("Test suite `"+f.filename+"' OK")
+
+
+
+
+
