MBI: display the expected diagnostic on error

[simgrid.git] / teshsuite / smpi / MBI / MBIutils.py

diff --git a/teshsuite/smpi/MBI/MBIutils.py b/teshsuite/smpi/MBI/MBIutils.py
index 070d321..c422cca 100644 (file)
--- a/teshsuite/smpi/MBI/MBIutils.py
+++ b/teshsuite/smpi/MBI/MBIutils.py
@@ -1,4 +1,4 @@
-# Copyright 2021-2022. The MBI project. All rights reserved. 
+# Copyright 2021-2022. The MBI project. All rights reserved.

 # This program is free software; you can redistribute it and/or modify it under the terms of the license (GNU GPL).
 
 import os
 # This program is free software; you can redistribute it and/or modify it under the terms of the license (GNU GPL).
 
 import os


@@ -12,37 +12,37 @@ import signal
 import hashlib
 
 class AbstractTool:
 import hashlib
 
 class AbstractTool:

-    def ensure_image(self, params=""):
+    def ensure_image(self, params="", dockerparams=""):

         """Verify that this is executed from the right docker image, and complain if not."""
         if os.path.exists("/MBI") or os.path.exists("trust_the_installation"):
             print("This seems to be a MBI docker image. Good.")
         else:
             print("Please run this script in a MBI docker image. Run these commands:")
             print("  docker build -f Dockerfile -t mpi-bugs-initiative:latest . # Only the first time")
         """Verify that this is executed from the right docker image, and complain if not."""
         if os.path.exists("/MBI") or os.path.exists("trust_the_installation"):
             print("This seems to be a MBI docker image. Good.")
         else:
             print("Please run this script in a MBI docker image. Run these commands:")
             print("  docker build -f Dockerfile -t mpi-bugs-initiative:latest . # Only the first time")

-            print(f"  docker run -it --rm --name MIB --volume $(pwd):/MBI mpi-bugs-initiative /MBI/MBI.py {params}")
+            print(f"  docker run -it --rm --name MIB --volume $(pwd):/MBI {dockerparams}mpi-bugs-initiative /MBI/MBI.py {params}")

             sys.exit(1)
 
     def build(self, rootdir, cached=True):
         """Rebuilds the tool binaries. By default, we try to reuse the existing build."""
             sys.exit(1)
 
     def build(self, rootdir, cached=True):
         """Rebuilds the tool binaries. By default, we try to reuse the existing build."""

-        print ("Nothing to do to rebuild the tool binaries.")
+        print("Nothing to do to rebuild the tool binaries.")

 
     def setup(self, rootdir):
         """
         Ensure that this tool (previously built) is usable in this environment: setup the PATH, etc.
         This is called only once for all tests, from the logs directory.
         """
 
     def setup(self, rootdir):
         """
         Ensure that this tool (previously built) is usable in this environment: setup the PATH, etc.
         This is called only once for all tests, from the logs directory.
         """

-        pass
+        # pass

 
 

-    def run(execcmd, filename, binary, id, timeout):
+    def run(self, execcmd, filename, binary, num_id, timeout, batchinfo):

         """Compile that test code and anaylse it with the Tool if needed (a cache system should be used)"""
         """Compile that test code and anaylse it with the Tool if needed (a cache system should be used)"""

-        pass
+        # pass

 
     def teardown(self):
         """
         Clean the results of all test runs: remove temp files and binaries.
         This is called only once for all tests, from the logs directory.
         """
 
     def teardown(self):
         """
         Clean the results of all test runs: remove temp files and binaries.
         This is called only once for all tests, from the logs directory.
         """

-        pass
+        # pass

 
     def parse(self, cachefile):
         """Read the result of a previous run from the cache, and compute the test outcome"""
 
     def parse(self, cachefile):
         """Read the result of a previous run from the cache, and compute the test outcome"""


@@ -52,20 +52,21 @@ class AbstractTool:
 possible_details = {
     # scope limited to one call
     'InvalidBuffer':'AInvalidParam', 'InvalidCommunicator':'AInvalidParam', 'InvalidDatatype':'AInvalidParam', 'InvalidRoot':'AInvalidParam', 'InvalidTag':'AInvalidParam', 'InvalidWindow':'AInvalidParam', 'InvalidOperator':'AInvalidParam', 'InvalidOtherArg':'AInvalidParam', 'ActualDatatype':'AInvalidParam',
 possible_details = {
     # scope limited to one call
     'InvalidBuffer':'AInvalidParam', 'InvalidCommunicator':'AInvalidParam', 'InvalidDatatype':'AInvalidParam', 'InvalidRoot':'AInvalidParam', 'InvalidTag':'AInvalidParam', 'InvalidWindow':'AInvalidParam', 'InvalidOperator':'AInvalidParam', 'InvalidOtherArg':'AInvalidParam', 'ActualDatatype':'AInvalidParam',

-    'InvalidSrcDest':'AInvalidParam', 
+    'InvalidSrcDest':'AInvalidParam',

     # scope: Process-wide
     # scope: Process-wide

-#    'OutOfInitFini':'BInitFini', 
+#    'OutOfInitFini':'BInitFini',

     'CommunicatorLeak':'BResLeak', 'DatatypeLeak':'BResLeak', 'GroupLeak':'BResLeak', 'OperatorLeak':'BResLeak', 'TypeLeak':'BResLeak', 'RequestLeak':'BResLeak',
     'MissingStart':'BReqLifecycle', 'MissingWait':'BReqLifecycle',
     'CommunicatorLeak':'BResLeak', 'DatatypeLeak':'BResLeak', 'GroupLeak':'BResLeak', 'OperatorLeak':'BResLeak', 'TypeLeak':'BResLeak', 'RequestLeak':'BResLeak',
     'MissingStart':'BReqLifecycle', 'MissingWait':'BReqLifecycle',

+    'MissingEpoch':'BEpochLifecycle','DoubleEpoch':'BEpochLifecycle',

     'LocalConcurrency':'BLocalConcurrency',
     # scope: communicator
     'LocalConcurrency':'BLocalConcurrency',
     # scope: communicator

-    'CallMatching':'DMatch', 
+    'CallMatching':'DMatch',

     'CommunicatorMatching':'CMatch', 'DatatypeMatching':'CMatch', 'OperatorMatching':'CMatch', 'RootMatching':'CMatch', 'TagMatching':'CMatch',
     'CommunicatorMatching':'CMatch', 'DatatypeMatching':'CMatch', 'OperatorMatching':'CMatch', 'RootMatching':'CMatch', 'TagMatching':'CMatch',

-    'MessageRace':'DRace', 
-    
+    'MessageRace':'DRace',
+

     'GlobalConcurrency':'DGlobalConcurrency',
     # larger scope
     'GlobalConcurrency':'DGlobalConcurrency',
     # larger scope

-#    'BufferingHazard':'EBufferingHazard',
+    'BufferingHazard':'EBufferingHazard',

     'OK':'FOK'}
 
 error_scope = {
     'OK':'FOK'}
 
 error_scope = {


@@ -73,12 +74,13 @@ error_scope = {
     'BResLeak':'single process',
 #    'BInitFini':'single process',
     'BReqLifecycle':'single process',
     'BResLeak':'single process',
 #    'BInitFini':'single process',
     'BReqLifecycle':'single process',

+    'BEpochLifecycle':'single process',

     'BLocalConcurrency':'single process',
     'CMatch':'multi-processes',
     'DRace':'multi-processes',
     'DMatch':'multi-processes',
     'DGlobalConcurrency':'multi-processes',
     'BLocalConcurrency':'single process',
     'CMatch':'multi-processes',
     'DRace':'multi-processes',
     'DMatch':'multi-processes',
     'DGlobalConcurrency':'multi-processes',

-#    'EBufferingHazard':'system',
+    'EBufferingHazard':'system',

     'FOK':'correct executions'
 }
 
     'FOK':'correct executions'
 }
 


@@ -95,7 +97,7 @@ displayed_name = {
     'EBufferingHazard':'Buffering hazard',
     'FOK':"Correct execution",
 
     'EBufferingHazard':'Buffering hazard',
     'FOK':"Correct execution",
 

-    'aislinn':'Aislinn','civl':'CIVL','hermes':'Hermes', 'isp':'ISP','itac':'ITAC', 'simgrid':'Mc SimGrid', 'smpi':'SMPI','smpivg':'SMPI+VG', 'mpisv':'MPI-SV', 'must':'MUST', 'parcoach':'PARCOACH'
+    'aislinn':'Aislinn', 'civl':'CIVL', 'hermes':'Hermes', 'isp':'ISP', 'itac':'ITAC', 'simgrid':'Mc SimGrid', 'smpi':'SMPI', 'smpivg':'SMPI+VG', 'mpisv':'MPI-SV', 'must':'MUST', 'parcoach':'PARCOACH'

 }
 
 def parse_one_code(filename):
 }
 
 def parse_one_code(filename):


@@ -105,10 +107,10 @@ def parse_one_code(filename):
     """
     res = []
     test_num = 0
     """
     res = []
     test_num = 0

-    with open(filename, "r") as input:
+    with open(filename, "r") as input_file:

         state = 0  # 0: before header; 1: in header; 2; after header
         line_num = 1
         state = 0  # 0: before header; 1: in header; 2; after header
         line_num = 1

-        for line in input:
+        for line in input_file:

             if re.match(".*BEGIN_MBI_TESTS.*", line):
                 if state == 0:
                     state = 1
             if re.match(".*BEGIN_MBI_TESTS.*", line):
                 if state == 0:
                     state = 1


@@ -119,10 +121,10 @@ def parse_one_code(filename):
                     state = 2
                 else:
                     raise ValueError(f"Unexpected end of MBI_TESTS header at line {line_num}: \n{line}")
                     state = 2
                 else:
                     raise ValueError(f"Unexpected end of MBI_TESTS header at line {line_num}: \n{line}")

-            if state == 1 and re.match("\s+\$ ?.*", line):
-                m = re.match('\s+\$ ?(.*)', line)
+            if state == 1 and re.match(r'\s+\$ ?.*', line):
+                m = re.match(r'\s+\$ ?(.*)', line)

                 cmd = m.group(1)
                 cmd = m.group(1)

-                nextline = next(input)
+                nextline = next(input_file)

                 detail = 'OK'
                 if re.match('[ |]*OK *', nextline):
                     expect = 'OK'
                 detail = 'OK'
                 if re.match('[ |]*OK *', nextline):
                     expect = 'OK'


@@ -136,7 +138,14 @@ def parse_one_code(filename):
                     if detail not in possible_details:
                         raise ValueError(
                             f"\n{filename}:{line_num}: MBI parse error: Detailled outcome {detail} is not one of the allowed ones.")
                     if detail not in possible_details:
                         raise ValueError(
                             f"\n{filename}:{line_num}: MBI parse error: Detailled outcome {detail} is not one of the allowed ones.")

-                test = {'filename': filename, 'id': test_num, 'cmd': cmd, 'expect': expect, 'detail': detail}
+
+                nextline = next(input_file)
+                m = re.match('[ |]*(.*)', nextline)
+                if not m:
+                    raise ValueError(f"\n{filename}:{line_num}: MBI parse error: Expected diagnostic of the test not found.\n")
+                diagnostic = m.group(1)
+
+                test = {'filename': filename, 'id': test_num, 'cmd': cmd, 'expect': expect, 'detail': detail, 'diagnostic': diagnostic}

                 res.append(test.copy())
                 test_num += 1
                 line_num += 1
                 res.append(test.copy())
                 test_num += 1
                 line_num += 1


@@ -166,32 +175,32 @@ def categorize(tool, toolname, test_id, expected):
     if outcome == 'timeout':
         res_category = 'timeout'
         if elapsed is None:
     if outcome == 'timeout':
         res_category = 'timeout'
         if elapsed is None:

-            diagnostic = f'hard timeout'
+            diagnostic = 'hard timeout'

         else:
             diagnostic = f'timeout after {elapsed} sec'
         else:
             diagnostic = f'timeout after {elapsed} sec'

-    elif outcome == 'failure':
+    elif outcome == 'failure' or outcome == 'segfault':

         res_category = 'failure'
         res_category = 'failure'

-        diagnostic = f'tool error, or test not run'
+        diagnostic = 'tool error, or test not run'

     elif outcome == 'UNIMPLEMENTED':
         res_category = 'unimplemented'
     elif outcome == 'UNIMPLEMENTED':
         res_category = 'unimplemented'

-        diagnostic = f'coverage issue'
+        diagnostic = 'coverage issue'

     elif outcome == 'other':
         res_category = 'other'
     elif outcome == 'other':
         res_category = 'other'

-        diagnostic = f'inconclusive run'
+        diagnostic = 'inconclusive run'

     elif expected == 'OK':
         if outcome == 'OK':
             res_category = 'TRUE_NEG'
     elif expected == 'OK':
         if outcome == 'OK':
             res_category = 'TRUE_NEG'

-            diagnostic = f'correctly reported no error'
+            diagnostic = 'correctly reported no error'

         else:
             res_category = 'FALSE_POS'
         else:
             res_category = 'FALSE_POS'

-            diagnostic = f'reported an error in a correct code'
+            diagnostic = 'reported an error in a correct code'

     elif expected == 'ERROR':
         if outcome == 'OK':
             res_category = 'FALSE_NEG'
     elif expected == 'ERROR':
         if outcome == 'OK':
             res_category = 'FALSE_NEG'

-            diagnostic = f'failed to detect an error'
+            diagnostic = 'failed to detect an error'

         else:
             res_category = 'TRUE_POS'
         else:
             res_category = 'TRUE_POS'

-            diagnostic =  f'correctly detected an error'
+            diagnostic = 'correctly detected an error'

     else:
         raise ValueError(f"Unexpected expectation: {expected} (must be OK or ERROR)")
 
     else:
         raise ValueError(f"Unexpected expectation: {expected} (must be OK or ERROR)")
 


@@ -201,11 +210,11 @@ def categorize(tool, toolname, test_id, expected):
 def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo, read_line_lambda=None):
     """
     Runs the test on need. Returns True if the test was ran, and False if it was cached.
 def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo, read_line_lambda=None):
     """
     Runs the test on need. Returns True if the test was ran, and False if it was cached.

-    
+

     The result is cached if possible, and the test is rerun only if the `test.txt` (containing the tool output) or the `test.elapsed` (containing the timing info) do not exist, or if `test.md5sum` (containing the md5sum of the code to compile) does not match.
 
     Parameters:
     The result is cached if possible, and the test is rerun only if the `test.txt` (containing the tool output) or the `test.elapsed` (containing the timing info) do not exist, or if `test.md5sum` (containing the md5sum of the code to compile) does not match.
 
     Parameters:

-     - buildcmd and execcmd are shell commands to run. buildcmd can be any shell line (incuding && groups), but execcmd must be a single binary to run. 
+     - buildcmd and execcmd are shell commands to run. buildcmd can be any shell line (incuding && groups), but execcmd must be a single binary to run.

      - cachefile is the name of the test
      - filename is the source file containing the code
      - binary the file name in which to compile the code
      - cachefile is the name of the test
      - filename is the source file containing the code
      - binary the file name in which to compile the code


@@ -214,7 +223,7 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo,
     """
     if os.path.exists(f'{cachefile}.txt') and os.path.exists(f'{cachefile}.elapsed') and os.path.exists(f'{cachefile}.md5sum'):
         hash_md5 = hashlib.md5()
     """
     if os.path.exists(f'{cachefile}.txt') and os.path.exists(f'{cachefile}.elapsed') and os.path.exists(f'{cachefile}.md5sum'):
         hash_md5 = hashlib.md5()

-        with open(filename, 'rb') as sourcefile :
+        with open(filename, 'rb') as sourcefile:

             for chunk in iter(lambda: sourcefile.read(4096), b""):
                 hash_md5.update(chunk)
         newdigest = hash_md5.hexdigest()
             for chunk in iter(lambda: sourcefile.read(4096), b""):
                 hash_md5.update(chunk)
         newdigest = hash_md5.hexdigest()


@@ -224,19 +233,18 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo,
         if olddigest == newdigest:
             print(f" (result cached -- digest: {olddigest})")
             return False
         if olddigest == newdigest:
             print(f" (result cached -- digest: {olddigest})")
             return False

-        else:
-            os.remove(f'{cachefile}.txt')
+        os.remove(f'{cachefile}.txt')

 
     print(f"Wait up to {timeout} seconds")
 
     start_time = time.time()
 
     print(f"Wait up to {timeout} seconds")
 
     start_time = time.time()

-    if buildcmd == None:
+    if buildcmd is None:

         output = f"No need to compile {binary}.c (batchinfo:{batchinfo})\n\n"
     else:
         output = f"Compiling {binary}.c (batchinfo:{batchinfo})\n\n"
         output += f"$ {buildcmd}\n"
 
         output = f"No need to compile {binary}.c (batchinfo:{batchinfo})\n\n"
     else:
         output = f"Compiling {binary}.c (batchinfo:{batchinfo})\n\n"
         output += f"$ {buildcmd}\n"
 

-        compil = subprocess.run(buildcmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        compil = subprocess.run(buildcmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

         if compil.stdout is not None:
             output += str(compil.stdout, errors='replace')
         if compil.returncode != 0:
         if compil.stdout is not None:
             output += str(compil.stdout, errors='replace')
         if compil.returncode != 0:


@@ -261,7 +269,6 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo,
 
     pid = process.pid
     pgid = os.getpgid(pid)  # We need that to forcefully kill subprocesses when leaving
 
     pid = process.pid
     pgid = os.getpgid(pid)  # We need that to forcefully kill subprocesses when leaving

-    outcome = None

     while True:
         if poll_obj.poll(5):  # Something to read? Do check the timeout status every 5 sec if not
             line = process.stdout.readline()
     while True:
         if poll_obj.poll(5):  # Something to read? Do check the timeout status every 5 sec if not
             line = process.stdout.readline()


@@ -272,7 +279,6 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo,
             if read_line_lambda != None:
                 read_line_lambda(line, process)
         if time.time() - start_time > timeout:
             if read_line_lambda != None:
                 read_line_lambda(line, process)
         if time.time() - start_time > timeout:

-            outcome = 'timeout'

             with open(f'{cachefile}.timeout', 'w') as outfile:
                 outfile.write(f'{time.time() - start_time} seconds')
             break
             with open(f'{cachefile}.timeout', 'w') as outfile:
                 outfile.write(f'{time.time() - start_time} seconds')
             break


@@ -315,9 +321,9 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo,
         outfile.write(output)
     with open(f'{cachefile}.md5sum', 'w') as outfile:
         hashed = hashlib.md5()
         outfile.write(output)
     with open(f'{cachefile}.md5sum', 'w') as outfile:
         hashed = hashlib.md5()

-        with open(filename, 'rb') as sourcefile :
+        with open(filename, 'rb') as sourcefile:

             for chunk in iter(lambda: sourcefile.read(4096), b""):
                 hashed.update(chunk)
         outfile.write(hashed.hexdigest())
             for chunk in iter(lambda: sourcefile.read(4096), b""):
                 hashed.update(chunk)
         outfile.write(hashed.hexdigest())

-    
+

     return True
     return True