#! /usr/bin/env perl
# -*- Mode: perl; -*-
#
# This script is the beginnings of a script to run a sequence of test
# programs.  See the MPICH document for a description of the test
# strategy and requirements.
#
# Description
#   Tests are controlled by a file listing test programs; if the file is
#   a directory, then all of the programs in the directory and subdirectories
#   are run
#
#   To run a test, the following steps are executed
#   Build the executable:
#      make programname
#   Run the executable
#      mpiexec -n <np> ./programname >out 2>err
#   Check the return code (non zero is failure)
#   Check the stderr output (non empty is failure)
#   Check the stdout output (No Errors or Test passed are the only valid
#      output)
#   Remove executable, out, err files
#
# The format of a list file is
# programname number-of-processes
# If number-of-processes is missing, $np_default is used (this is 2 but can
# be overridden with -np=new-value)
#
# Special feature:
# Because these tests can take a long time to run, there is an
# option to cause the tests to stop is a "stopfile" is found.
# The stopfile can be created by a separate, watchdog process, to ensure that
# tests end at a certain time.
# The name of this file is (by default) .stoptest
# in the  top-level run directory.  The environment variable
#    MPITEST_STOPTEST
# can specify a different file name.
#
# Import the mkpath command
use File::Path;

# Global variables
$MPIMajorVersion = "2";
$MPIMinorVersion = "2";
$mpiexec = "smpirun";    # Name of mpiexec program (including path, if necessary)
$testIsStrict = "true";
$MPIhasMPIX   = "no";
$np_arg  = "-np";         # Name of argument to specify the number of processes
$err_count = 0;          # Number of programs that failed.
$total_run = 0;          # Number of programs tested
$total_seen = 0;         # Number of programs considered for testing
$np_default = 2;         # Default number of processes to use
$np_max     = -1;        # Maximum number of processes to use (overrides any
                         # value in the test list files.  -1 is Infinity
$defaultTimeLimit = 180; # default timeout

$srcdir = ".";           # Used to set the source dir for testlist files

$curdir = ".";           # used to track the relative current directory

# Output forms
$xmloutput = 0;          # Set to true to get xml output (also specify file)
$closeXMLOutput = 1;     # Set to false to leave XML output file open to
                         # accept additional data
$verbose = 1;            # Set to true to get more output
$showProgress = 0;       # Set to true to get a "." with each run program.
$newline = "\r\n";       # Set to \r\n for Windows-friendly, \n for Unix only
$batchRun = 0;           # Set to true to batch the execution of the tests
                         # (i.e., run them together, then test output,
                         # rather than build/run/check for each test)
$testCount = 0;          # Used with batchRun to count tests.
$batrundir = ".";        # Set to the directory into which to run the examples

$execarg="";
$wrapparg="";

$enabled_privatization = 1; # disable tests that need SMPI privatization to run
# TAP (Test Anything Protocol) output
my $tapoutput = 0;
my $tapfile = '';
my $tapfullfile = '';

$debug = 1;

$depth = 0;              # This is used to manage multiple open list files

# Build flags
$remove_this_pgm = 0;
$clean_pgms      = 0;

my $program_wrapper = '';

#---------------------------------------------------------------------------
# Get some arguments from the environment
#   Currently, only the following are understood:
#   VERBOSE
#   RUNTESTS_VERBOSE  (an alias for VERBOSE in case you want to
#                      reserve VERBOSE)
#   RUNTESTS_SHOWPROGRESS
#   MPITEST_STOPTEST
#   MPITEST_TIMEOUT
#   MPITEST_PROGRAM_WRAPPER (Value is added after -np but before test
#                            executable.  Tools like valgrind may be inserted
#                            this way.)
#---------------------------------------------------------------------------
if ( defined($ENV{"VERBOSE"}) || defined($ENV{"V"}) || defined($ENV{"RUNTESTS_VERBOSE"}) ) {
    $verbose = 1;
}
if ( defined($ENV{"RUNTESTS_SHOWPROGRESS"} ) ) {
    $showProgress = 1;
}
if (defined($ENV{"MPITEST_STOPTEST"})) {
    $stopfile = $ENV{"MPITEST_STOPTEST"};
}
else {
    $stopfile = `pwd` . "/.stoptest";
    $stopfile =~ s/\r*\n*//g;    # Remove any newlines (from pwd)
}

if (defined($ENV{"MPITEST_TIMEOUT"})) {
    $defaultTimeLimit = $ENV{"MPITEST_TIMEOUT"};
}

# Define this to leave the XML output file open to receive additional data
if (defined($ENV{'NOXMLCLOSE'}) && $ENV{'NOXMLCLOSE'} eq 'YES') {
    $closeXMLOutput = 0;
}

if (defined($ENV{'MPITEST_PROGRAM_WRAPPER'})) {
    $program_wrapper = $ENV{'MPITEST_PROGRAM_WRAPPER'};
}

if (defined($ENV{'MPITEST_BATCH'})) {
    if ($ENV{'MPITEST_BATCH'} eq 'YES' || $ENV{'MPITEST_BATCH'} eq 'yes') {
	$batchRun = 1;
    } elsif ($ENV{'MPITEST_BATCH'} eq 'NO' || $ENV{'MPITEST_BATCH'} eq 'no') {
	$batchRun = 0;
    }
    else {
	print STDERR "Unrecognized value for MPITEST_BATCH = $ENV{'MPITEST_BATCH'}\n";
    }
}
if (defined($ENV{'MPITEST_BATCHDIR'})) {
    $batrundir = $ENV{'MPITEST_BATCHDIR'};
}

#---------------------------------------------------------------------------
# Process arguments and override any defaults
#---------------------------------------------------------------------------
foreach $_ (@ARGV) {
    if (/--?mpiexec=(.*)/) {
	# Use mpiexec as given - it may be in the path, and
	# we don't want to bother to try and find it.
	$mpiexec = $1;
    }
    elsif (/--?np=(.*)/)   { $np_default = $1; }
    elsif (/--?maxnp=(.*)/) { $np_max = $1; }
    elsif (/--?tests=(.*)/) { $listfiles = $1; }
    elsif (/--?srcdir=(.*)/) { $srcdir = $1;
	$mpiexec="$mpiexec  -platform ${srcdir}/../../../../examples/platforms/small_platform_with_routers.xml -hostfile ${srcdir}/../../hostfile_coll --log=root.thr:critical --cfg=smpi/host-speed:1e9  --cfg=smpi/async-small-thresh:65536"; }
    elsif (/--?verbose/) { $verbose = 1; }
    elsif (/--?showprogress/) { $showProgress = 1; }
    elsif (/--?debug/) { $debug = 1; }
    elsif (/--?batch/) { $batchRun = 1; }
    elsif (/--?batchdir=(.*)/) { $batrundir = $1; }
    elsif (/--?timeoutarg=(.*)/) { $timeoutArgPattern = $1; }
    elsif (/--?execarg=(.*)/) { $execarg = "$execarg $1"; }
    elsif (/--?privatization=(.*)/) { 
print STDERR "privatization called\n";
$enabled_privatization = $1; }
    elsif (/--?wrapper=(.*)/) {
         $wrapparg = "-wrapper \"$1\"" if $1 ne "";
    }
    elsif (/--?xmlfile=(.*)/) {
	$xmlfile   = $1;
	if (! ($xmlfile =~ /^\//)) {
	    $thisdir = `pwd`;
	    chop $thisdir;
	    $xmlfullfile = $thisdir . "/" . $xmlfile ;
	}
	else {
	    $xmlfullfile = $xmlfile;
	}
	$xmloutput = 1;
	open( XMLOUT, ">$xmlfile" ) || die "Cannot open $xmlfile\n";
	my $date = `date "+%Y-%m-%d-%H-%M"`;
	$date =~ s/\r?\n//;
	# MPISOURCE can be used to describe the source of MPI for this
	# test.
	print XMLOUT "<?xml version='1.0' ?>$newline";
	print XMLOUT "<?xml-stylesheet href=\"TestResults.xsl\" type=\"text/xsl\" ?>$newline";
	print XMLOUT "<MPITESTRESULTS>$newline";
	print XMLOUT "<DATE>$date</DATE>$newline";
	print XMLOUT "<MPISOURCE></MPISOURCE>$newline";
    }
    elsif (/--?noxmlclose/) {
	$closeXMLOutput = 0;
    }
    elsif (/--?tapfile=(.*)/) {
        $tapfile = $1;
        if ($tapfile !~ m|^/|) {
            $thisdir = `pwd`;
            chomp $thisdir;
            $tapfullfile = $thisdir . "/" . $tapfile ;
        }
        else {
            $tapfullfile = $tapfile;
        }
        $tapoutput = 1;
        open( TAPOUT, ">$tapfile" ) || die "Cannot open $tapfile\n";
        my $date = `date "+%Y-%m-%d-%H-%M"`;
        $date =~ s/\r?\n//;
        print TAPOUT "TAP version 13\n";
        print TAPOUT "# MPICH test suite results (TAP format)\n";
        print TAPOUT "# date ${date}\n";
        # we do not know at this point how many tests will be run, so do
        # not print a test plan line like "1..450" until the very end
    }
}

# Perform any post argument processing
if ($batchRun) {
    if (! -d $batrundir) {
	mkpath $batrundir || die "Could not create $batrundir\n";
    }
    open( BATOUT, ">$batrundir/runtests.batch" ) || die "Could not open $batrundir/runtests.batch\n";
}
else {
    # We must have mpiexec
    if ("$mpiexec" eq "") {
	print STDERR "No mpiexec found!\n";
	exit(1);
    }
}

#
# Process any files
if ($listfiles eq "") {
    if ($batchRun) {
	print STDERR "An implicit list of tests is not permitted in batch mode\n";
	exit(1);
    }
    else {
	&ProcessImplicitList;
    }
}
elsif (-d $listfiles) {
    print STDERR "Testing by directories not yet supported\n";
}
else {
    &RunList( $listfiles );
}

if ($xmloutput && $closeXMLOutput) {
    print XMLOUT "</MPITESTRESULTS>$newline";
    close XMLOUT;
}

if ($tapoutput) {
    print TAPOUT "1..$total_seen\n";
    close TAPOUT;
}

# Output a summary:
if ($batchRun) {
    print "Programs created along with a runtest.batch file in $batrundir\n";
    print "Run that script and then use checktests to summarize the results\n";
}
else {
    if ($err_count) {
	print "$err_count tests failed out of $total_run\n";
	print "Failing tests : $failed_tests\n";
	if ($xmloutput) {
	    print "Details in $xmlfullfile\n";
	}
    }
    else {
	print " All $total_run tests passed!\n";
    }
    if ($tapoutput) {
        print "TAP formatted results in $tapfullfile\n";
    }
}
exit ($err_count > 0);
#
# ---------------------------------------------------------------------------
# Routines
#
# Enter a new directory and process a list file.
#  ProcessDir( directory-name, list-file-name )
sub ProcessDir {
    my $dir = $_[0]; $dir =~ s/\/$//;
    my $listfile = $_[1];
    my $savedir = `pwd`;
    my $savecurdir = $curdir;
    my $savesrcdir = $srcdir;

    chop $savedir;
    if (substr($srcdir,0,3) eq "../") {
      $srcdir = "../$srcdir";
    }

    print "Processing directory $dir\n" if ($verbose || $debug);
    chdir $dir;
    if ($dir =~ /\//) {
	print STDERR "only direct subdirectories allowed in list files";
    }
    $curdir .= "/$dir";

    &RunList( $listfile );
    print "\n" if $showProgress; # Terminate line from progress output
    chdir $savedir;
    $curdir = $savecurdir;
    $srcdir = $savesrcdir;
}
# ---------------------------------------------------------------------------
# Run the programs listed in the file given as the argument.
# This file describes the tests in the format
#  programname number-of-processes [ key=value ... ]
# If the second value is not given, the default value is used.
#
sub RunList {
    my $LIST = "LIST$depth"; $depth++;
    my $listfile = $_[0];
    my $ResultTest = "";
    my $InitForRun = "";
    my $listfileSource = $listfile;

    print "Looking in $curdir/$listfile\n" if $debug;
    if (! -s "$listfile" && -s "$srcdir/$curdir/$listfile" ) {
	$listfileSource = "$srcdir/$curdir/$listfile";
    }
    open( $LIST, "<$listfileSource" ) ||
	die "Could not open $listfileSource\n";
    while (<$LIST>) {
	# Check for stop file
	if (-s $stopfile) {
	    # Exit because we found a stopfile
	    print STDERR "Terminating test because stopfile $stopfile found\n";
	    last;
	}
	# Skip comments
	s/#.*//g;
	# Remove any trailing newlines/returns
	s/\r?\n//;
        # Remove any leading whitespace
        s/^\s*//;
	# Some tests require that support routines are built first
	# This is specified with !<dir>:<target>
	if (/^\s*\!([^:]*):(.*)/) {
	    # Hack: just execute in a subshell.  This discards any
	    # output.
	    `cd $1 && make $2`;
	    next;
	}
	# List file entries have the form:
	# program [ np [ name=value ... ] ]
	# See files errhan/testlist, init/testlist, and spawn/testlist
	# for examples of using the key=value form
	my @args = split(/\s+/,$_);
	my $programname = $args[0];
	my $np = "";
	my $ResultTest = "";
	my $InitForRun = "";
	my $timeLimit  = "";
	my $progArgs   = "";
	my $mpiexecArgs = "$execarg";
	my $requiresStrict = "";
	my $requiresMPIX   = "";
	my $progEnv    = "";
	my $mpiVersion = "";
        my $needs_privatization = 0;
        my $xfail = "";
	if ($#args >= 1) { $np = $args[1]; }
	# Process the key=value arguments
	for (my $i=2; $i <= $#args; $i++) {
	    if ($args[$i] =~ /([^=]+)=(.*)/) {
		my $key = $1;
		my $value = $2;
		if ($key eq "resultTest") {
		    $ResultTest = $value;
		}
		elsif ($key eq "init") {
		    $InitForRun = $value;
		}
		elsif ($key eq "timeLimit") {
		    $timeLimit = $value;
		}
		elsif ($key eq "arg") {
		    $progArgs = "$progArgs $value";
		}
		elsif ($key eq "mpiexecarg") {
		    $mpiexecArgs = "$mpiexecArgs $value";
		}
		elsif ($key eq "env") {
		    $progEnv = "$progEnv $value";
		}
		elsif ($key eq "mpiversion") {
		    $mpiVersion = $value;
		}
		elsif ($key eq "needs_privatization") {
		    $needs_privatization = $value;
		}
		elsif ($key eq "strict") {
		    $requiresStrict = $value
		}
		elsif ($key eq "mpix") {
		    $requiresMPIX = $value
		}
                elsif ($key eq "xfail") {
                    if ($value eq "") {
                        print STDERR "\"xfail=\" requires an argument\n";
                    }
                    $xfail = $value;
                }
		else {
		    print STDERR "Unrecognized key $key in $listfileSource\n";
		}
	    }
	}

	# skip empty lines
	if ($programname eq "") { next; }

	# if privatization is disabled, and if the test needs it, ignore it
	if ($needs_privatization == 1 &&
		$enabled_privatization != 1) {
                SkippedTest($programname, $np, $workdir, "requires SMPI privatization");
	        next; 
	}

	if ($np eq "") { $np = $np_default; }
	if ($np_max > 0 && $np > $np_max) { $np = $np_max; }

        # allows us to accurately output TAP test numbers without disturbing the
        # original totals that have traditionally been reported
        #
        # These "unless" blocks are ugly, but permit us to honor skipping
        # criteria for directories as well without counting directories as tests
        # in our XML/TAP output.
        unless (-d $programname) {
            $total_seen++;
        }

	# If a minimum MPI version is specified, check against the
	# available MPI.  If the version is unknown, we ignore this
	# test (thus, all tests will be run).
	if ($mpiVersion ne "" && $MPIMajorVersion ne "unknown" &&
	    $MPIMinorVersion ne "unknown") {
	    my ($majorReq,$minorReq) = split(/\./,$mpiVersion);
            if ($majorReq > $MPIMajorVersion or
                ($majorReq == $MPIMajorVersion && $minorReq > $MPIMinorVersion))
            {
                unless (-d $programname) {
                    SkippedTest($programname, $np, $workdir, "requires MPI version $mpiVersion");
                }
                next;
            }
	}
	# Check whether strict is required by MPI but not by the
	# test (use strict=false for tests that use non-standard extensions)
        if (lc($requiresStrict) eq "false" && lc($testIsStrict) eq "true") {
            unless (-d $programname) {
                SkippedTest($programname, $np, $workdir, "non-strict test, strict MPI mode requested");
            }
            next;
        }

        if (lc($testIsStrict) eq "true") {
            # Strict MPI testing was requested, so assume that a non-MPICH MPI
            # implementation is being tested and the "xfail" implementation
            # assumptions do not hold.
            $xfail = '';
        }

        if (lc($requiresMPIX) eq "true" && lc($MPIHasMPIX) eq "no") {
            unless (-d $programname) {
                SkippedTest($programname, $np, $workdir, "tests MPIX extensions, MPIX testing disabled");
            }
            next;
        }

	if (-d $programname) {
	    # If a directory, go into the that directory and
	    # look for a new list file
	    &ProcessDir( $programname, $listfile );
	}
	else {
	    $total_run++;
	    if (&BuildMPIProgram( $programname, $xfail ) == 0) {
		if ($batchRun == 1) {
		    &AddMPIProgram( $programname, $np, $ResultTest,
				    $InitForRun, $timeLimit, $progArgs,
				    $progEnv, $mpiexecArgs, $xfail );
		}
		else {
		    &RunMPIProgram( $programname, $np, $ResultTest,
				    $InitForRun, $timeLimit, $progArgs,
				    $progEnv, $mpiexecArgs, $xfail );
		}
	    }
	    elsif ($xfail ne '') {
		# We expected to run this program, so failure to build
		# is an error
		$found_error = 1;
		$err_count++;
	    }
	    if ($batchRun == 0) {
		&CleanUpAfterRun( $programname );
	    }
	}
    }
    close( $LIST );
}
#
# This routine tries to run all of the files in the current
# directory
sub ProcessImplicitList {
    # The default is to run every file in the current directory.
    # If there are no built programs, build and run every file
    # WARNING: This assumes that anything executable should be run as
    # an MPI test.
    $found_exec = 0;
    $found_src  = 0;
    open (PGMS, "ls -1 |" ) || die "Cannot list directory\n";
    while (<PGMS>) {
	s/\r?\n//;
	$programname = $_;
	if (-d $programname) { next; }  # Ignore directories
	if ($programname eq "runtests") { next; } # Ignore self
	if ($programname eq "checktests") { next; } # Ignore helper
	if ($programname eq "configure") { next; } # Ignore configure script
	if ($programname eq "config.status") { next; } # Ignore configure helper
	if (-x $programname) { $found_exec++; }
	if ($programname =~ /\.[cf]$/) { $found_src++; }
    }
    close PGMS;

    if ($found_exec) {
	print "Found executables\n" if $debug;
	open (PGMS, "ls -1 |" ) || die "Cannot list programs\n";
	while (<PGMS>) {
	    # Check for stop file
	    if (-s $stopfile) {
		# Exit because we found a stopfile
		print STDERR "Terminating test because stopfile $stopfile found\n";
		last;
	    }
	    s/\r?\n//;
	    $programname = $_;
	    if (-d $programname) { next; }  # Ignore directories
	    if ($programname eq "runtests") { next; } # Ignore self
	    if (-x $programname) {
		$total_run++;
		&RunMPIProgram( $programname, $np_default, "", "", "", "", "", "", "" );
	    }
	}
	close PGMS;
    }
    elsif ($found_src) {
	print "Found source files\n" if $debug;
	open (PGMS, "ls -1 *.c |" ) || die "Cannot list programs\n";
	while (<PGMS>) {
	    if (-s $stopfile) {
		# Exit because we found a stopfile
		print STDERR "Terminating test because stopfile $stopfile found\n";
		last;
	    }
	    s/\r?\n//;
	    $programname = $_;
	    # Skip messages from ls about no files
	    if (! -s $programname) { next; }
	    $programname =~ s/\.c//;
	    $total_run++;
	    if (&BuildMPIProgram( $programname, "") == 0) {
		&RunMPIProgram( $programname, $np_default, "", "", "", "", "", "", "" );
	    }
	    else {
		# We expected to run this program, so failure to build
		# is an error
		$found_error = 1;
		$err_count++;
	    }
	    &CleanUpAfterRun( $programname );
	}
	close PGMS;
    }
}
# Run the program.
# ToDo: Add a way to limit the time that any particular program may run.
# The arguments are
#    name of program, number of processes, name of routine to check results
#    init for testing, timelimit, and any additional program arguments
# If the 3rd arg is not present, the a default that simply checks that the
# return status is 0 and that the output is " No Errors" is used.
sub RunMPIProgram {
    my ($programname,$np,$ResultTest,$InitForTest,$timeLimit,$progArgs,$progEnv,$mpiexecArgs,$xfail) = @_;
    my $found_error   = 0;
    my $found_noerror = 0;
    my $inline = "";

    &RunPreMsg( $programname, $np, $curdir );

    unlink "err";

    # Set a default timeout on tests (3 minutes for now)
    my $timeout = $defaultTimeLimit;
    if (defined($timeLimit) && $timeLimit =~ /^\d+$/) {
	$timeout = $timeLimit;
    }
    $ENV{"MPIEXEC_TIMEOUT"} = $timeout;

    # Run the optional setup routine. For example, the timeout tests could
    # be set to a shorter timeout.
    if ($InitForTest ne "") {
	&$InitForTest();
    }
    print STDOUT "Env includes $progEnv\n" if $verbose;
    print STDOUT "$mpiexec $wrapparg $mpiexecArgs $np_arg $np $program_wrapper ./$programname $progArgs\n" if $verbose;
    print STDOUT "." if $showProgress;
    # Save and restore the environment if necessary before running mpiexec.
    if ($progEnv ne "") {
	%saveEnv = %ENV;
	foreach $val (split(/\s+/, $progEnv)) {
	    if ($val =~ /([^=]+)=(.*)/) {
		$ENV{$1} = $2;
	    }
	    else {
		print STDERR "Environment variable/value $val not in a=b form\n";
	    }
	}
    }
    open ( MPIOUT, "$mpiexec $wrapparg $np_arg $np $mpiexecArgs $program_wrapper ./$programname $progArgs 2>&1 |" ) ||
	die "Could not run ./$programname\n";
    if ($progEnv ne "") {
	%ENV = %saveEnv;
    }
    if ($ResultTest ne "") {
	# Read and process the output
	($found_error, $inline) = &$ResultTest( MPIOUT, $programname );
    }
    else {
	if ($verbose) {
	    $inline = "$mpiexec $wrapparg $np_arg $np $program_wrapper ./$programname\n";
	}
	else {
	    $inline = "";
	}
	while (<MPIOUT>) {
	    print STDOUT $_ if $verbose;
	    # Skip FORTRAN STOP
	    if (/FORTRAN STOP/) { next; }
	    $inline .= $_;
	    if (/^==[0-9]+== ?WARNING: ASan doesn't fully support/) {
		next;
	    }
	    if (/^\s*No [Ee]rrors\s*$/ && $found_noerror == 0) {
		$found_noerror = 1;
	    }
	    if (! /^\s*No [Ee]rrors\s*$/ && !/^\s*Test Passed\s*$/) {
		print STDERR "Unexpected output in $programname: $_";
		if (!$found_error) {
		    $found_error = 1;
		    $failed_tests .= $programname;
		    $failed_tests .= " ";
		    $err_count ++;
		}
	    }
	}
	if ($found_noerror == 0) {
	    print STDERR "Program $programname exited without No Errors\n";
	    if (!$found_error) {
		$found_error = 1;
		$failed_tests .= $programname;
		$failed_tests .= " ";
		$err_count ++;
	    }
	}
	$rc = close ( MPIOUT );
	if ($rc == 0) {
	    # Only generate a message if we think that the program
	    # passed the test.
	    if (!$found_error) {
		$run_status = $?;
		$signal_num = $run_status & 127;
		if ($run_status > 255) { $run_status >>= 8; }
		print STDERR "Program $programname exited with non-zero status $run_status\n";
		if ($signal_num != 0) {
		    print STDERR "Program $programname exited with signal $signal_num\n";
		}
		$failed_tests .= $programname;
		$failed_tests .= " ";
		$found_error = 1;
		$err_count ++;
	    }
	}
    }
    if ($found_error) {
	&RunTestFailed( $programname, $np, $curdir, $inline, $xfail );
    }
    else {
	&RunTestPassed( $programname, $np, $curdir, $xfail );
    }
    &RunPostMsg( $programname, $np, $curdir );
}

# This version simply writes the mpiexec command out, with the output going
# into a file, and recording the output status of the run.
sub AddMPIProgram {
    my ($programname,$np,$ResultTest,$InitForTest,$timeLimit,$progArgs,$progEnv,$mpiexecArgs, $xfail) = @_;

    if (! -x $programname) {
	print STDERR "Could not find $programname!";
	return;
    }

    if ($ResultTest ne "") {
	# This test really needs to be run manually, with this test
	# Eventually, we can update this to include handleing in checktests.
	print STDERR "Run $curdir/$programname with $np processes and use $ResultTest to check the results\n";
	return;
    }

    # Set a default timeout on tests (3 minutes for now)
    my $timeout = $defaultTimeLimit;
    if (defined($timeLimit) && $timeLimit =~ /^\d+$/) {
	# On some systems, there is no effective time limit on
	# individual mpi program runs.  In that case, we may
	# want to treat these also as "run manually".
	$timeout = $timeLimit;
    }
    print BATOUT "export MPIEXEC_TIMEOUT=$timeout\n";

    # Run the optional setup routine. For example, the timeout tests could
    # be set to a shorter timeout.
    if ($InitForTest ne "") {
	&$InitForTest();
    }

    # For non-MPICH versions of mpiexec, a timeout may require a different
    # environment variable or command line option (e.g., for Cray aprun,
    # the option -t <sec> must be given, there is no environment variable
    # to set the timeout.
    $extraArgs = "";
    if (defined($timeoutArgPattern) && $timeoutArgPattern ne "") {
	my $timeArg = $timeoutArgPattern;
	$timeoutArg =~ s/<SEC>/$timeout/;
	$extraArgs .= $timeoutArg
    }

    print STDOUT "Env includes $progEnv\n" if $verbose;
    print STDOUT "$mpiexec $np_arg $np $extraArgs $program_wrapper ./$programname $progArgs\n" if $verbose;
    print STDOUT "." if $showProgress;
    # Save and restore the environment if necessary before running mpiexec.
    if ($progEnv ne "") {
	# Need to fix:
	# save_NAME_is_set=is old name set
	# save_NAME=oldValue
	# export NAME=newvalue
	# (run)
	# export NAME=oldValue (if set!)
	print STDERR "Batch output does not permit changes to environment\n";
    }
    # The approach here is to move the test codes to a single directory from
    # which they can be run; this avoids complex code to change directories
    # and ensure that the output goes "into the right place".
    $testCount++;
    rename $programname, "$batrundir/$programname";
    print BATOUT "echo \"# $mpiexec $np_arg $np $extraArgs $mpiexecArgs $program_wrapper $curdir/$programname $progArgs\" > runtests.$testCount.out\n";
    # Some programs expect to run in the same directory as the executable
    print BATOUT "$mpiexec $np_arg $np $extraArgs $mpiexecArgs $program_wrapper ./$programname $progArgs >> runtests.$testCount.out 2>&1\n";
    print BATOUT "echo \$? > runtests.$testCount.status\n";
}

#
# Return value is 0 on success, non zero on failure
sub BuildMPIProgram {
    my $programname = shift;
    if (! -x $programname) {
        die "Could not find $programname. Aborting.\n";
    }
    return 0;
    # THE FOLLOWING IS DISABLED.
    my $xfail = shift;
    my $rc = 0;
    if ($verbose) { print STDERR "making $programname\n"; }
    if (! -x $programname) { $remove_this_pgm = 1; }
    else { $remove_this_pgm = 0; }
    my $output = `make $programname 2>&1`;
    $rc = $?;
    if ($rc > 255) { $rc >>= 8; }
    if (! -x $programname) {
	print STDERR "Failed to build $programname; $output\n";
	if ($rc == 0) {
	    $rc = 1;
	}
	# Add a line to the summary file describing the failure
	# This will ensure that failures to build will end up
	# in the summary file (which is otherwise written by the
	# RunMPIProgram step)
	&RunPreMsg( $programname, $np, $curdir );
	&RunTestFailed( $programname, $np, $curdir, "Failed to build $programname; $output", $xfail );
	&RunPostMsg( $programname, $np, $curdir );
    }
    return $rc;
}

sub CleanUpAfterRun {
    my $programname = $_[0];

    # Check for that this program has exited.  If it is still running,
    # issue a warning and leave the application.  Of course, this
    # check is complicated by the lack of a standard access to the
    # running processes for this user in Unix.
    @stillRunning = &FindRunning( $programname );

    if ($#stillRunning > -1) {
	print STDERR "Some programs ($programname) may still be running:\npids = ";
	for (my $i=0; $i <= $#stillRunning; $i++ ) {
	    print STDERR $stillRunning[$i] . " ";
	}
	print STDERR "\n";
	# Remind the user that the executable remains; we leave it around
	# to allow the programmer to debug the running program, for which
	# the executable is needed.
	print STDERR "The executable ($programname) will not be removed.\n";
    }
    else {
	if ($remove_this_pgm && $clean_pgms) {
	    unlink $programname, "$programname.o";
	}
	$remove_this_pgm = 0;
    }
}
# ----------------------------------------------------------------------------
sub FindRunning {
    my $programname = $_[0];
    my @pids = ();

    my $logname = $ENV{'USER'};
    my $pidloc = 1;
    my $rc = open PSFD, "ps auxw -U $logname 2>&1 |";

    if ($rc == 0) {
	$rc = open PSFD, "ps -fu $logname 2>&1 |";
    }
    if ($rc == 0) {
	print STDERR "Could not execute ps command\n";
	return @pids;
    }

    while (<PSFD>) {
	if (/$programname/) {
	    @fields = split(/\s+/);
	    my $pid = $fields[$pidloc];
	    # Check that we've found a numeric pid
	    if ($pid =~ /^\d+$/) {
		$pids[$#pids + 1] = $pid;
	    }
	}
    }
    close PSFD;

    return @pids;
}
# ----------------------------------------------------------------------------
#
# TestStatus is a special test that reports success *only* when the
# status return is NONZERO
sub TestStatus {
    my $MPIOUT = $_[0];
    my $programname = $_[1];
    my $found_error = 0;

    my $inline = "";
    while (<$MPIOUT>) {
	#print STDOUT $_ if $verbose;
	# Skip FORTRAN STOP
	if (/FORTRAN STOP/) { next; }
	$inline .= $_;
	# ANY output is an error. We have the following output
	# exception for the Hydra process manager.
	if (/=*/) { last; }
	if (! /^\s*$/) {
	    print STDERR "Unexpected output in $programname: $_";
	    if (!$found_error) {
		$found_error = 1;
		$err_count ++;
	    }
	}
    }
    $rc = close ( MPIOUT );
    if ($rc == 0) {
	$run_status = $?;
	$signal_num = $run_status & 127;
	if ($run_status > 255) { $run_status >>= 8; }
    }
    else {
	# This test *requires* non-zero return codes
        if (!$found_error) {
	    $found_error = 1;
	    $err_count ++;
        }
	$inline .= "$mpiexec returned a zero status but the program returned a nonzero status\n";
    }
    return ($found_error,$inline);
}
#
# TestTimeout is a special test that reports success *only* when the
# status return is NONZERO and there are no processes left over.
# This test currently checks only for the return status.
sub TestTimeout {
    my $MPIOUT = $_[0];
    my $programname = $_[1];
    my $found_error = 0;

    my $inline = "";
    while (<$MPIOUT>) {
	#print STDOUT $_ if $verbose;
	# Skip FORTRAN STOP
	if (/FORTRAN STOP/) { next; }
	$inline .= $_;
	if (/[Tt]imeout/) { next; }
	# Allow 'signaled with Interrupt' (see gforker mpiexec)
	if (/signaled with Interrupt/) { next; }
	# Allow 'job ending due to env var MPIEXEC_TIMEOUT' (mpd)
	if (/job ending due to env var MPIEXEC_TIMEOUT/) { next; }
	# Allow 'APPLICATION TIMED OUT' (hydra)
	if (/\[mpiexec@.*\] APPLICATION TIMED OUT/) { last; }
	# ANY output is an error (other than timeout)
	if (! /^\s*$/) {
	    print STDERR "Unexpected output in $programname: $_";
	    if (!$found_error) {
		$found_error = 1;
		$err_count ++;
	    }
	}
    }
    $rc = close ( MPIOUT );
    if ($rc == 0) {
	$run_status = $?;
	$signal_num = $run_status & 127;
	if ($run_status > 255) { $run_status >>= 8; }
    }
    else {
	# This test *requires* non-zero return codes
	if (!$found_error) {
	    $found_error = 1;
	    $err_count ++;
        }
	$inline .= "$mpiexec returned a zero status but the program returned a nonzero status\n";
    }
    #
    # Here should go a check of the processes
    # open( PFD, "ps -fu $LOGNAME | grep -v grep | grep $programname |" );
    # while (<PFD>) {
    #
    # }
    # close PFD;
    return ($found_error,$inline);
}
#
# TestErrFatal is a special test that reports success *only* when the
# status return is NONZERO; it ignores error messages
sub TestErrFatal {
    my $MPIOUT = $_[0];
    my $programname = $_[1];
    my $found_error = 0;

    my $inline = "";
    while (<$MPIOUT>) {
	#print STDOUT $_ if $verbose;
	# Skip FORTRAN STOP
	if (/FORTRAN STOP/) { next; }
	$inline .= $_;
	# ALL output is allowed.
    }
    $rc = close ( MPIOUT );
    if ($rc == 0) {
	$run_status = $?;
	$signal_num = $run_status & 127;
	if ($run_status > 255) { $run_status >>= 8; }
    }
    else {
	# This test *requires* non-zero return codes
	if (!$found_error) {
	    $found_error = 1;
	    $err_count ++;
	}
	$inline .= "$mpiexec returned a zero status but the program returned a nonzero status\n";
    }
    return ($found_error,$inline);
}

# ----------------------------------------------------------------------------
# Output routines:
#  RunPreMsg( programname, np, workdir ) - Call before running a program
#  RunTestFailed, RunTestPassed - Call after test
#  RunPostMsg               - Call at end of each test
#
sub RunPreMsg {
    my ($programname,$np,$workdir) = @_;
    if ($xmloutput) {
	print XMLOUT "<MPITEST>$newline<NAME>$programname</NAME>$newline";
	print XMLOUT "<NP>$np</NP>$newline";
	print XMLOUT "<WORKDIR>$workdir</WORKDIR>$newline";
    }
}
sub RunPostMsg {
    my ($programname, $np, $workdir) = @_;
    if ($xmloutput) {
	print XMLOUT "</MPITEST>$newline";
    }
}
sub RunTestPassed {
    my ($programname, $np, $workdir, $xfail) = @_;
    if ($xmloutput) {
	print XMLOUT "<STATUS>pass</STATUS>$newline";
    }
    if ($tapoutput) {
        my $xfailstr = '';
        if ($xfail ne '') {
            $xfailstr = " # TODO $xfail";
        }
        print TAPOUT "ok ${total_run} - $workdir/$programname ${np}${xfailstr}\n";
    }
}
sub RunTestFailed {
    my $programname = shift;
    my $np = shift;
    my $workdir = shift;
    my $output = shift;
    my $xfail = shift;

    if ($xmloutput) {
        my $xout = $output;
        # basic escapes that wreck the XML output
        $xout =~ s/</\*AMP\*lt;/g;
        $xout =~ s/>/\*AMP\*gt;/g;
        $xout =~ s/&/\*AMP\*amp;/g;
        $xout =~ s/\*AMP\*/&/g;
        # TODO: Also capture any non-printing characters (XML doesn't like them
        # either).
	print XMLOUT "<STATUS>fail</STATUS>$newline";
	print XMLOUT "<TESTDIFF>$newline$xout</TESTDIFF>$newline";
    }

    if ($tapoutput) {
        my $xfailstr = '';
        if ($xfail ne '') {
            $xfailstr = " # TODO $xfail";
        }
        print TAPOUT "not ok ${total_run} - $workdir/$programname ${np}${xfailstr}\n";
        print TAPOUT "  ---\n";
        print TAPOUT "  Directory: $workdir\n";
        print TAPOUT "  File: $programname\n";
        print TAPOUT "  Num-procs: $np\n";
        print TAPOUT "  Date: \"" . localtime . "\"\n";

        # The following would be nice, but it leads to unfortunate formatting in
        # the Jenkins web output for now.  Using comment lines instead, since
        # they are easier to read/find in a browser.
##        print TAPOUT "  Output: |\n";
##        # using block literal format, requires that all chars are printable
##        # UTF-8 (or UTF-16, but we won't encounter that)
##        foreach my $line (split m/\r?\n/, $output) {
##            chomp $line;
##            # 4 spaces, 2 for TAP indent, 2 more for YAML block indent
##            print TAPOUT "    $line\n";
##        }

        print TAPOUT "  ...\n";

        # Alternative to the "Output:" YAML block literal above.  Do not put any
        # spaces before the '#', this causes some TAP parsers (including Perl's
        # TAP::Parser) to treat the line as "unknown" instead of a proper
        # comment.
        print TAPOUT "## Test output (expected 'No Errors'):\n";
        foreach my $line (split m/\r?\n/, $output) {
            chomp $line;
            print TAPOUT "## $line\n";
        }
    }
}

sub SkippedTest {
    my $programname = shift;
    my $np = shift;
    my $workdir = shift;
    my $reason = shift;

    # simply omit from the XML output

    if ($tapoutput) {
        print TAPOUT "ok ${total_seen} - $workdir/$programname $np  # SKIP $reason\n";
    }
}

# ----------------------------------------------------------------------------
# Alternate init routines
sub InitQuickTimeout {
    $ENV{"MPIEXEC_TIMEOUT"} = 10;
}