3 # Copyright (c) 2012-2015. The SimGrid Team.
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the license (GNU LGPL) which comes with this package.
8 eval 'exec perl -S $0 ${1+"$@"}'
9 if $running_under_some_shell;
11 # If you change this file, please stick to the formatting you got with:
12 # perltidy --backup-and-modify-in-place --maximum-line-length=180 --output-line-ending=unix --cuddled-else
22 B<tesh> [I<options>]... I<testsuite>
26 Tesh is the testing shell, a specialized shell for running tests. It
27 provides the specified input to the tested commands, and check that
28 they produce the expected output and return the expected value.
32 --cd some/directory : ask tesh to switch the working directory before
34 --setenv var=value : set a specific environment variable
35 --cfg arg : add parameter --cfg=arg to each command line
36 --log arg : add parameter --log=arg to each command line
37 --enable-coverage : ignore output lines starting with "profiling:"
39 =head1 TEST SUITE FILE SYTAX
41 A test suite is composed of one or several I<command blocks> separated
42 by empty lines, each of them being composed of a command to run, its
43 input text and the expected output.
45 The first char of each line specifies the type of line according to
46 the following list. The second char of each line is ignored.
48 `$' command to run in foreground
49 `&' command to run in background
51 `<' input to pass to the command
52 `>' output expected from the command
54 `!' metacommand, which can be one of:
55 `timeout' <integer>|no
56 `expect signal' <signal name>
57 `expect return' <integer>
58 `output' <ignore|display>
61 `p' an informative message to print
63 If the expected output do not match the produced output, or if the
64 command did not end as expected, Tesh provides an error message (see
65 the OUTPUT section below) and stops.
67 =head2 Command blocks examples
69 In a given command block, you can declare the command, its input and
70 its expected output in the order that you see fit.
84 You can group several commands together, provided that they don't have
90 =head2 Enforcing the command return code
92 By default, Tesh enforces that the tested command returns 0. If not,
93 it fails with an appropriate message and returns I<code+40> itself.
95 You specify that a given command block is expected to return another
98 # This command MUST return 42
102 The I<expect return> construct applies only to the next command block.
104 =head2 Commands that are expected to raise signals
106 By default, Tesh detects when the command is killed by a signal (such
107 as SEGV on segfaults). This is usually unexpected and unfortunate. But
108 if not, you can specify that a given command block is expected to fail
109 with a signal as follows:
111 # This command MUST raise a segfault
112 ! expect signal SIGSEGV
113 $ ./some_failing_code
115 The I<expect signal> construct applies only to the next command block.
119 By default, no command is allowed to run more than 5 seconds. You can
120 change this value as follows:
122 # Allow some more time to the command
124 $ ./some_longer_command
126 You can also disable the timeout completely by passing "no" as a value:
128 # This command will never timeout
130 $ ./some_very_long_but_safe_command
132 =head2 Setting environment variables
134 You can modify the environment of the tested commands as follows:
139 =head2 Not enforcing the expected output
141 By default, the commands output is matched against the one expected,
142 and an error is raised on discrepancy. Metacommands to change this:
148 The output is completely discarded.
152 The output is displayed, but no error is issued if it differs from the
157 The output is sorted before comparison (see next section).
161 =head2 Sorting output
163 If the order of the command output changes between runs, you want to
164 sort it before enforcing that it is exactly what you expect. In
165 SimGrid for example, this happens when parallel execution is
166 activated: User processes are run in parallel at each timestamp, and
167 the output is not reproducible anymore. Until you sort the lines.
169 You can sort the command output as follows:
172 $ ./some_multithreaded_command
174 Sorting lines this ways often makes the tesh output very intricate,
175 complicating the error analysis: the process logical order is defeated
176 by the lexicographical sort.
178 The solution is to prefix each line of your output with temporal
179 information so that lines can be grouped by timestamps. The
180 lexicographical sort then only applies to lines that occured at the
181 same timestamp. Here is a SimGrid example:
183 # Sort only lines depending on the first 19 chars
185 $ ./some_simgrid_simulator --log=root.fmt:[%10.6r]%e(%i:%P@%h)%e%m%n
187 This approach may seem surprizing at the first glance but it does its job:
191 =item Every timestamps remain separated, as it should;
193 =item In each timestamp, the output order of processes become
194 reproducible: that's the lexicographical order of their name;
196 =item For each process, the order of its execution is preserved: its
197 messages within a given timestamp are not reordered.
201 That way, tesh can do its job (no false positive, no false negative)
202 despite the unpredictable order of executions of processes within a
203 timestamp, and reported errors remain easy to analyze (execution of a
204 given process preserved).
206 This example is very SimGrid oriented, but the feature could even be
207 usable by others, who knows?
210 =head1 BUILTIN COMMANDS
212 =head2 mkfile: creating a file
214 This command creates a file of the name provided as argument, and adds
215 the content it gets as input.
221 It is not possible to use the cat command, as one would expect,
222 because stream redirections are currently not implemented in Tesh.
224 =head1 BUGS, LIMITATIONS AND POSSIBLE IMPROVEMENTS
226 The main limitation is the lack of stream redirections in the commands
227 (">", "<" and "|" shell constructs and friends). The B<mkfile> builtin
228 command makes this situation bearable.
230 It would be nice if we could replace the tesh file completely with
231 command line flags when the output is not to be verified.
236 # Disabling IPC::Run::Debug saves tons of useless calls.
237 $ENV{'IPCRUNDEBUG'} = 'none'
238 unless exists $ENV{'IPCRUNDEBUG'};
241 my $enable_coverage = 0;
243 my $diff_tool_tmp_fh = 0;
244 my $diff_tool_tmp_filename = 0;
245 my $sort_prefix = -1;
252 $SIG{'PIPE'} = 'IGNORE';
258 use lib "@CMAKE_BINARY_DIR@/bin";
260 use Diff qw(diff); # postpone a bit to have time to change INC
262 use Getopt::Long qw(GetOptions);
264 use Text::ParseWords;
265 use IPC::Run qw(start run timeout finish);
270 #### Portability bits for windows
273 use constant RUNNING_ON_WINDOWS => ( $OSNAME =~ /^(?:mswin|dos|os2)/oi );
274 use POSIX qw(:sys_wait_h WIFEXITED WIFSIGNALED WIFSTOPPED WEXITSTATUS WTERMSIG WSTOPSIG
275 :signal_h SIGINT SIGTERM SIGKILL SIGABRT SIGSEGV);
278 if (RUNNING_ON_WINDOWS) { # Missing on windows
279 *WIFEXITED = sub { not $_[0] & 127 };
280 *WEXITSTATUS = sub { $_[0] >> 8 };
281 *WIFSIGNALED = sub { ( $_[0] & 127 ) && ( $_[0] & 127 != 127 ) };
282 *WTERMSIG = sub { $_[0] & 127 };
284 # used on the command lines
285 $environ{'EXEEXT'} = ".exe";
291 #### Command line option handling
294 my %opts = ( "debug" => 0,
295 "timeout" => 5, # No command should run any longer than 5 seconds by default
298 Getopt::Long::config( 'bundling', 'no_getopt_compat', 'no_auto_abbrev' );
300 'debug|d' => \$opts{"debug"},
302 'difftool=s' => \$diff_tool,
304 'cd=s' => sub { cd_cmd( $_[1] ) },
305 'timeout=s' => \$opts{'timeout'},
306 'setenv=s' => sub { setenv_cmd( $_[1] ) },
307 'cfg=s' => sub { $opts{'cfg'} .= " --cfg=$_[1]" },
308 'log=s' => sub { $opts{'log'} .= " --log=$_[1]" },
309 'enable-coverage+' => \$enable_coverage,
312 $tesh_file = pop @ARGV;
313 $tesh_name = $tesh_file;
314 $tesh_name =~ s|^.*?/([^/]*)$|$1|;
316 print "Enable coverage\n" if ($enable_coverage);
319 use File::Temp qw/ tempfile /;
320 ( $diff_tool_tmp_fh, $diff_tool_tmp_filename ) = tempfile();
321 print "New tesh: $diff_tool_tmp_filename\n";
324 if ( $tesh_file =~ m/(.*)\.tesh/ ) {
325 print "Test suite `$tesh_file'\n";
327 $tesh_name = "(stdin)";
328 print "Test suite from stdin\n";
331 ###########################################################################
334 my %cmd = %{ $_[0] };
335 if ( $opts{'debug'} ) {
336 map { print "IN: $_\n" } @{ $cmd{'in'} };
337 map { print "OUT: $_\n" } @{ $cmd{'out'} };
338 print "CMD: $cmd{'cmd'}\n";
341 # substitute environment variables
342 foreach my $key ( keys %environ ) {
343 $cmd{'cmd'} = var_subst( $cmd{'cmd'}, $key, $environ{$key} );
346 # substitute remaining variables, if any
347 while ( $cmd{'cmd'} =~ /\$\{(\w+)(?::[=-][^}]*)?\}/ ) {
348 $cmd{'cmd'} = var_subst( $cmd{'cmd'}, $1, "" );
350 while ( $cmd{'cmd'} =~ /\$(\w+)/ ) {
351 $cmd{'cmd'} = var_subst( $cmd{'cmd'}, $1, "" );
354 # add cfg and log options
355 $cmd{'cmd'} .= " $opts{'cfg'}"
356 if ( defined( $opts{'cfg'} ) && length( $opts{'cfg'} ) );
357 $cmd{'cmd'} .= " $opts{'log'}"
358 if ( defined( $opts{'log'} ) && length( $opts{'log'} ) );
360 # finally trim any remaining space chars
361 $cmd{'cmd'} =~ s/^\s+//;
362 $cmd{'cmd'} =~ s/\s+$//;
364 print "[$tesh_name:$cmd{'line'}] $cmd{'cmd'}\n";
366 $cmd{'return'} ||= 0;
367 $cmd{'timeout'} ||= $opts{'timeout'};
371 # exec the command line
373 my @cmdline = quotewords( '\s+', 0, $cmd{'cmd'} );
374 my $input = defined($cmd{'in'})? join("\n",@{$cmd{'in'}}) : "";
375 my $output = " " x 10240; $output = ""; # Preallocate 10kB, and reset length to 0
376 $cmd{'got'} = \$output;
377 $cmd{'job'} = start \@cmdline, '<', \$input, '>&', \$output,
378 ($cmd{'timeout'} eq 'no' ? () : timeout($cmd{'timeout'}));
380 if ( $cmd{'background'} ) {
381 # Just enqueue the job. It will be dealed with at the end
382 push @bg_cmds, \%cmd;
384 # Deal with its ending conditions right away
385 analyze_result( \%cmd );
390 my %cmd = %{ $_[0] };
391 $cmd{'timeouted'} = 0; # initialization
393 # Wait for the end of the child process
396 finish( $cmd{'job'} );
398 if ($@) { # deal with the errors that occured in the child process
399 if ($@ =~ /timeout/) {
400 $cmd{'job'}->kill_kill;
401 $cmd{'timeouted'} = 1;
402 } elsif ($@ =~ /^ack / and $@ =~ /pipe/) { # IPC::Run is not very expressive about the pipes that it gets :(
403 print STDERR "Tesh: Broken pipe (ignored).\n";
405 die $@; # Don't know what it is, so let it go.
412 # pop all output from executing child
414 map { print "GOT: $_\n" } ${$cmd{'got'}} if $opts{'debug'};
415 foreach my $got ( split("\n", ${$cmd{'got'}}) ) {
418 print $diff_tool_tmp_fh "> $got\n" if ($diff_tool);
420 unless ( $enable_coverage and $got =~ /^profiling:/ ) {
425 # How did the child process terminate?
427 $cmd{'gotret'} = "Unparsable status. Please report this tesh bug.";
428 if ( $cmd{'timeouted'} ) {
429 $cmd{'gotret'} = "timeout after $cmd{'timeout'} sec";
432 } elsif ( WIFEXITED($status) ) {
433 $exitcode = WEXITSTATUS($status) + 40;
434 $cmd{'gotret'} = "returned code " . WEXITSTATUS($status);
435 } elsif ( WIFSIGNALED($status) ) {
437 if ( WTERMSIG($status) == SIGINT ) { $code = "SIGINT"; }
438 elsif ( WTERMSIG($status) == SIGTERM ) { $code = "SIGTERM"; }
439 elsif ( WTERMSIG($status) == SIGKILL ) { $code = "SIGKILL"; }
440 elsif ( WTERMSIG($status) == SIGABRT ) { $code = "SIGABRT"; }
441 elsif ( WTERMSIG($status) == SIGSEGV ) { $code = "SIGSEGV"; }
442 $exitcode = WTERMSIG($status) + 4;
443 $cmd{'gotret'} = "got signal $code";
446 # How was it supposed to terminate?
448 if ( defined( $cmd{'expect'} ) and ( $cmd{'expect'} ne "" ) ) {
449 $wantret = "got signal $cmd{'expect'}";
451 $wantret = "returned code " . ( defined( $cmd{'return'} ) ? $cmd{'return'} : 0 );
454 # Enforce the outcome
457 # Did it end as expected?
458 if ( $cmd{'gotret'} ne $wantret ) {
460 my $msg = "Test suite `$tesh_name': NOK (<$tesh_name:$cmd{'line'}> $cmd{'gotret'})\n";
462 $msg = $msg . "Output of <$tesh_name:$cmd{'line'}> so far:\n";
463 map { $msg .= "|| $_\n" } @got;
465 $msg .= "<$tesh_name:$cmd{'line'}> No output so far.\n";
470 # Does the output match?
471 if ( $cmd{'sort'} ) {
473 substr( $a, 0, $sort_prefix ) cmp substr( $b, 0, $sort_prefix );
476 if ( $sort_prefix > 0 ) {
477 @got = sort mysort @got;
481 while ( @got and $got[0] eq "" ) {
485 # Sort the expected output too, to make tesh files easier to write for humans
486 if ( defined( $cmd{'out'} ) ) {
487 if ( $sort_prefix > 0 ) {
488 @{ $cmd{'out'} } = sort mysort @{ $cmd{'out'} };
490 @{ $cmd{'out'} } = sort @{ $cmd{'out'} };
492 while ( @{ $cmd{'out'} } and ${ $cmd{'out'} }[0] eq "" ) {
493 shift @{ $cmd{'out'} };
498 # Report the output if asked so or if it differs
499 if ( defined( $cmd{'output display'} ) ) {
500 print "[Tesh/INFO] Here is the (ignored) command output:\n";
501 map { print "||$_\n" } @got;
502 } elsif ( defined( $cmd{'output ignore'} ) ) {
503 print "(ignoring the output of <$tesh_name:$cmd{'line'}> as requested)\n";
505 my $diff = build_diff( \@{ $cmd{'out'} }, \@got );
507 if ( length $diff ) {
508 print "Output of <$tesh_name:$cmd{'line'}> mismatch" . ( $cmd{'sort'} ? " (even after sorting)" : "" ) . ":\n";
509 map { print "$_\n" } split( /\n/, $diff );
510 if ( $cmd{'sort'} ) {
511 print "WARNING: Both the observed output and expected output were sorted as requested.\n";
512 print "WARNING: Output were only sorted using the $sort_prefix first chars.\n"
513 if ( $sort_prefix > 0 );
514 print "WARNING: Use <! output sort 19> to sort by simulated date and process ID only.\n";
516 # print "----8<--------------- Begin of unprocessed observed output (as it should appear in file):\n";
517 # map {print "> $_\n"} @{$cmd{'unsorted got'}};
518 # print "--------------->8---- End of the unprocessed observed output.\n";
521 print "Test suite `$tesh_name': NOK (<$tesh_name:$cmd{'line'}> output mismatch)\n";
528 my $infh; # The file descriptor from which we should read the teshfile
529 if ( $tesh_name eq "(stdin)" ) {
532 open $infh, $tesh_file
533 or die "[Tesh/CRITICAL] Unable to open $tesh_file: $!\n";
536 my %cmd; # everything about the next command to run
538 LINE: while ( not $error and defined( my $line = <$infh> )) {
543 print "[TESH/debug] $line_num: $line\n" if $opts{'debug'};
545 # deal with line continuations
546 while ( $line =~ /^(.*?)\\$/ ) {
548 die "[TESH/CRITICAL] Continued line at end of file\n"
549 unless defined($next);
552 print "[TESH/debug] $line_num: $next\n" if $opts{'debug'};
556 # If the line is empty, run any previously defined block and proceed to next line
557 unless ( $line =~ m/^(.)(.*)$/ ) {
558 if ( defined( $cmd{'cmd'} ) ) {
562 print $diff_tool_tmp_fh "$line\n" if ($diff_tool);
566 my ( $cmd, $arg ) = ( $1, $2 );
567 print $diff_tool_tmp_fh "$line\n" if ( $diff_tool and $cmd ne '>' );
572 # Deal with the lines that can contribute to the current command block
573 if ( $cmd =~ /^#/ ) { # comment
575 } elsif ( $cmd eq '>' ) { # expected result line
576 print "[TESH/debug] push expected result\n" if $opts{'debug'};
577 push @{ $cmd{'out'} }, $arg;
580 } elsif ( $cmd eq '<' ) { # provided input
581 print "[TESH/debug] push provided input\n" if $opts{'debug'};
582 push @{ $cmd{'in'} }, $arg;
585 } elsif ( $cmd eq 'p' ) { # comment
586 print "[$tesh_name:$line_num] $arg\n";
591 # We dealt with all sort of lines that can contribute to a command block, so we have something else here.
592 # If we have something buffered, run it now and start a new block
593 if ( defined( $cmd{'cmd'} ) ) {
598 # Deal with the lines that must be placed before a command block
599 if ( $cmd eq '$' ) { # Command
600 if ( $arg =~ /^mkfile / ) { # "mkfile" command line
601 die "[TESH/CRITICAL] Output expected from mkfile command!\n"
602 if scalar @{ cmd { 'out' } };
605 $cmd{'arg'} =~ s/mkfile //;
609 } elsif ( $arg =~ /^\s*cd / ) {
610 die "[TESH/CRITICAL] Input provided to cd command!\n"
611 if scalar @{ cmd { 'in' } };
612 die "[TESH/CRITICAL] Output expected from cd command!\n"
613 if scalar @{ cmd { 'out' } };
619 } else { # regular command
621 $cmd{'line'} = $line_num;
624 } elsif ( $cmd eq '&' ) { # background command line
625 die "[TESH/CRITICAL] mkfile cannot be run in background\n"
626 if ($arg =~ /^mkfile/);
627 die "[TESH/CRITICAL] cd cannot be run in background\n"
630 $cmd{'background'} = 1;
632 $cmd{'line'} = $line_num;
634 # Deal with the meta-commands
635 } elsif ( $line =~ /^! (.*)/) {
638 if ( $line =~ /^output sort/ ) {
640 if ( $line =~ /^output sort\s+(\d+)/ ) {
643 } elsif ($line =~ /^output ignore/ ) {
644 $cmd{'output ignore'} = 1;
645 } elsif ( $line =~ /^output display/ ) {
646 $cmd{'output display'} = 1;
647 } elsif ( $line =~ /^expect signal (\w*)/ ) {
649 } elsif ( $line =~ /^expect return/ ) {
650 $line =~ s/^expect return //g;
652 $cmd{'return'} = $line;
653 } elsif ( $line =~ /^setenv/ ) {
654 $line =~ s/^setenv //g;
657 } elsif ( $line =~ /^timeout/ ) {
658 $line =~ s/^timeout //;
660 $cmd{'timeout'} = $line;
663 die "[TESH/CRITICAL] parse error: $line\n";
667 # We are done reading the input file
668 close $infh unless ( $tesh_name eq "(stdin)" );
670 # Deal with last command, if any
671 if ( defined( $cmd{'cmd'} ) ) {
678 analyze_result( \%test );
682 close $diff_tool_tmp_fh;
683 system("$diff_tool $diff_tool_tmp_filename $tesh_file");
684 unlink $diff_tool_tmp_filename;
689 } elsif ( $tesh_name eq "(stdin)" ) {
690 print "Test suite from stdin OK\n";
692 print "Test suite `$tesh_name' OK\n";
698 #### Helper functions
703 my $diff = Diff->new(@_);
705 $diff->Base(1); # Return line numbers, not indices
706 my $chunk_count = $diff->Next(-1); # Compute the amount of chuncks
707 return "" if ( $chunk_count == 1 && $diff->Same() );
709 while ( $diff->Next() ) {
710 my @same = $diff->Same();
711 if ( $diff->Same() ) {
712 if ( $diff->Next(0) > 1 ) { # not first chunk: print 2 first lines
713 $res .= ' ' . $same[0] . "\n";
714 $res .= ' ' . $same[1] . "\n" if ( scalar @same > 1 );
716 $res .= "...\n" if ( scalar @same > 2 );
718 # $res .= $diff->Next(0)."/$chunk_count\n";
719 if ( $diff->Next(0) < $chunk_count ) { # not last chunk: print 2 last lines
720 $res .= ' ' . $same[ scalar @same - 2 ] . "\n"
721 if ( scalar @same > 1 );
722 $res .= ' ' . $same[ scalar @same - 1 ] . "\n";
725 next if $diff->Same();
726 map { $res .= "- $_\n" } $diff->Items(1);
727 map { $res .= "+ $_\n" } $diff->Items(2);
732 # Helper function replacing any occurence of variable '$name' by its '$value'
733 # As in Bash, ${$value:=BLABLA} is rewritten to $value if set or to BLABLA if $value is not set
735 my ( $text, $name, $value ) = @_;
737 $text =~ s/\$\{$name(?::[=-][^}]*)?\}/$value/g;
738 $text =~ s/\$$name(\W|$)/$value$1/g;
740 $text =~ s/\$\{$name:=([^}]*)\}/$1/g;
741 $text =~ s/\$\{$name\}//g;
742 $text =~ s/\$$name(\W|$)/$1/g;
747 ################################ The possible commands ################################
750 my %cmd = %{ $_[0] };
751 my $file = $cmd{'arg'};
752 print STDERR "[Tesh/INFO] mkfile $file. Ctn: >>".join( '\n', @{ $cmd{'in'} })."<<\n"
756 open( FILE, ">$file" )
757 or die "[Tesh/CRITICAL] Unable to create file $file: $!\n";
758 print FILE join( "\n", @{ $cmd{'in'} } );
759 print FILE "\n" if ( scalar @{ $cmd{'in'} } > 0 );
763 # Command CD. Just change to the provided directory
765 my $directory = shift;
767 if ( -e $directory && -d $directory ) {
769 print "[Tesh/INFO] change directory to $directory\n";
771 } elsif ( -e $directory ) {
772 print "Cannot change directory to '$directory': it is not a directory\n";
774 print "Chdir to $directory failed: No such file or directory\n";
776 if ( $failure == 1 ) {
777 print "Test suite `$tesh_name': NOK (system error)\n";
782 # Command setenv. Gets "variable=content", and update the environment accordingly
785 if ( $arg =~ /^(.*)=(.*)$/ ) {
786 my ( $var, $ctn ) = ( $1, $2 );
787 print "[Tesh/INFO] setenv $var=$ctn\n";
788 $environ{$var} = $ctn;
791 die "[Tesh/CRITICAL] Malformed argument to setenv: expected 'name=value' but got '$arg'\n";