X-Git-Url: http://info.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/992c3a8621f205aedc022a0d5a5d3fc7bb75e6a1..a46b20e2007bf3a1a675ea590511672f051d38da:/teshsuite/smpi/MBI/MBIutils.py diff --git a/teshsuite/smpi/MBI/MBIutils.py b/teshsuite/smpi/MBI/MBIutils.py index 070d3218a8..c422cca055 100644 --- a/teshsuite/smpi/MBI/MBIutils.py +++ b/teshsuite/smpi/MBI/MBIutils.py @@ -1,4 +1,4 @@ -# Copyright 2021-2022. The MBI project. All rights reserved. +# Copyright 2021-2022. The MBI project. All rights reserved. # This program is free software; you can redistribute it and/or modify it under the terms of the license (GNU GPL). import os @@ -12,37 +12,37 @@ import signal import hashlib class AbstractTool: - def ensure_image(self, params=""): + def ensure_image(self, params="", dockerparams=""): """Verify that this is executed from the right docker image, and complain if not.""" if os.path.exists("/MBI") or os.path.exists("trust_the_installation"): print("This seems to be a MBI docker image. Good.") else: print("Please run this script in a MBI docker image. Run these commands:") print(" docker build -f Dockerfile -t mpi-bugs-initiative:latest . # Only the first time") - print(f" docker run -it --rm --name MIB --volume $(pwd):/MBI mpi-bugs-initiative /MBI/MBI.py {params}") + print(f" docker run -it --rm --name MIB --volume $(pwd):/MBI {dockerparams}mpi-bugs-initiative /MBI/MBI.py {params}") sys.exit(1) def build(self, rootdir, cached=True): """Rebuilds the tool binaries. By default, we try to reuse the existing build.""" - print ("Nothing to do to rebuild the tool binaries.") + print("Nothing to do to rebuild the tool binaries.") def setup(self, rootdir): """ Ensure that this tool (previously built) is usable in this environment: setup the PATH, etc. This is called only once for all tests, from the logs directory. """ - pass + # pass - def run(execcmd, filename, binary, id, timeout): + def run(self, execcmd, filename, binary, num_id, timeout, batchinfo): """Compile that test code and anaylse it with the Tool if needed (a cache system should be used)""" - pass + # pass def teardown(self): """ Clean the results of all test runs: remove temp files and binaries. This is called only once for all tests, from the logs directory. """ - pass + # pass def parse(self, cachefile): """Read the result of a previous run from the cache, and compute the test outcome""" @@ -52,20 +52,21 @@ class AbstractTool: possible_details = { # scope limited to one call 'InvalidBuffer':'AInvalidParam', 'InvalidCommunicator':'AInvalidParam', 'InvalidDatatype':'AInvalidParam', 'InvalidRoot':'AInvalidParam', 'InvalidTag':'AInvalidParam', 'InvalidWindow':'AInvalidParam', 'InvalidOperator':'AInvalidParam', 'InvalidOtherArg':'AInvalidParam', 'ActualDatatype':'AInvalidParam', - 'InvalidSrcDest':'AInvalidParam', + 'InvalidSrcDest':'AInvalidParam', # scope: Process-wide -# 'OutOfInitFini':'BInitFini', +# 'OutOfInitFini':'BInitFini', 'CommunicatorLeak':'BResLeak', 'DatatypeLeak':'BResLeak', 'GroupLeak':'BResLeak', 'OperatorLeak':'BResLeak', 'TypeLeak':'BResLeak', 'RequestLeak':'BResLeak', 'MissingStart':'BReqLifecycle', 'MissingWait':'BReqLifecycle', + 'MissingEpoch':'BEpochLifecycle','DoubleEpoch':'BEpochLifecycle', 'LocalConcurrency':'BLocalConcurrency', # scope: communicator - 'CallMatching':'DMatch', + 'CallMatching':'DMatch', 'CommunicatorMatching':'CMatch', 'DatatypeMatching':'CMatch', 'OperatorMatching':'CMatch', 'RootMatching':'CMatch', 'TagMatching':'CMatch', - 'MessageRace':'DRace', - + 'MessageRace':'DRace', + 'GlobalConcurrency':'DGlobalConcurrency', # larger scope -# 'BufferingHazard':'EBufferingHazard', + 'BufferingHazard':'EBufferingHazard', 'OK':'FOK'} error_scope = { @@ -73,12 +74,13 @@ error_scope = { 'BResLeak':'single process', # 'BInitFini':'single process', 'BReqLifecycle':'single process', + 'BEpochLifecycle':'single process', 'BLocalConcurrency':'single process', 'CMatch':'multi-processes', 'DRace':'multi-processes', 'DMatch':'multi-processes', 'DGlobalConcurrency':'multi-processes', -# 'EBufferingHazard':'system', + 'EBufferingHazard':'system', 'FOK':'correct executions' } @@ -95,7 +97,7 @@ displayed_name = { 'EBufferingHazard':'Buffering hazard', 'FOK':"Correct execution", - 'aislinn':'Aislinn','civl':'CIVL','hermes':'Hermes', 'isp':'ISP','itac':'ITAC', 'simgrid':'Mc SimGrid', 'smpi':'SMPI','smpivg':'SMPI+VG', 'mpisv':'MPI-SV', 'must':'MUST', 'parcoach':'PARCOACH' + 'aislinn':'Aislinn', 'civl':'CIVL', 'hermes':'Hermes', 'isp':'ISP', 'itac':'ITAC', 'simgrid':'Mc SimGrid', 'smpi':'SMPI', 'smpivg':'SMPI+VG', 'mpisv':'MPI-SV', 'must':'MUST', 'parcoach':'PARCOACH' } def parse_one_code(filename): @@ -105,10 +107,10 @@ def parse_one_code(filename): """ res = [] test_num = 0 - with open(filename, "r") as input: + with open(filename, "r") as input_file: state = 0 # 0: before header; 1: in header; 2; after header line_num = 1 - for line in input: + for line in input_file: if re.match(".*BEGIN_MBI_TESTS.*", line): if state == 0: state = 1 @@ -119,10 +121,10 @@ def parse_one_code(filename): state = 2 else: raise ValueError(f"Unexpected end of MBI_TESTS header at line {line_num}: \n{line}") - if state == 1 and re.match("\s+\$ ?.*", line): - m = re.match('\s+\$ ?(.*)', line) + if state == 1 and re.match(r'\s+\$ ?.*', line): + m = re.match(r'\s+\$ ?(.*)', line) cmd = m.group(1) - nextline = next(input) + nextline = next(input_file) detail = 'OK' if re.match('[ |]*OK *', nextline): expect = 'OK' @@ -136,7 +138,14 @@ def parse_one_code(filename): if detail not in possible_details: raise ValueError( f"\n{filename}:{line_num}: MBI parse error: Detailled outcome {detail} is not one of the allowed ones.") - test = {'filename': filename, 'id': test_num, 'cmd': cmd, 'expect': expect, 'detail': detail} + + nextline = next(input_file) + m = re.match('[ |]*(.*)', nextline) + if not m: + raise ValueError(f"\n{filename}:{line_num}: MBI parse error: Expected diagnostic of the test not found.\n") + diagnostic = m.group(1) + + test = {'filename': filename, 'id': test_num, 'cmd': cmd, 'expect': expect, 'detail': detail, 'diagnostic': diagnostic} res.append(test.copy()) test_num += 1 line_num += 1 @@ -166,32 +175,32 @@ def categorize(tool, toolname, test_id, expected): if outcome == 'timeout': res_category = 'timeout' if elapsed is None: - diagnostic = f'hard timeout' + diagnostic = 'hard timeout' else: diagnostic = f'timeout after {elapsed} sec' - elif outcome == 'failure': + elif outcome == 'failure' or outcome == 'segfault': res_category = 'failure' - diagnostic = f'tool error, or test not run' + diagnostic = 'tool error, or test not run' elif outcome == 'UNIMPLEMENTED': res_category = 'unimplemented' - diagnostic = f'coverage issue' + diagnostic = 'coverage issue' elif outcome == 'other': res_category = 'other' - diagnostic = f'inconclusive run' + diagnostic = 'inconclusive run' elif expected == 'OK': if outcome == 'OK': res_category = 'TRUE_NEG' - diagnostic = f'correctly reported no error' + diagnostic = 'correctly reported no error' else: res_category = 'FALSE_POS' - diagnostic = f'reported an error in a correct code' + diagnostic = 'reported an error in a correct code' elif expected == 'ERROR': if outcome == 'OK': res_category = 'FALSE_NEG' - diagnostic = f'failed to detect an error' + diagnostic = 'failed to detect an error' else: res_category = 'TRUE_POS' - diagnostic = f'correctly detected an error' + diagnostic = 'correctly detected an error' else: raise ValueError(f"Unexpected expectation: {expected} (must be OK or ERROR)") @@ -201,11 +210,11 @@ def categorize(tool, toolname, test_id, expected): def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo, read_line_lambda=None): """ Runs the test on need. Returns True if the test was ran, and False if it was cached. - + The result is cached if possible, and the test is rerun only if the `test.txt` (containing the tool output) or the `test.elapsed` (containing the timing info) do not exist, or if `test.md5sum` (containing the md5sum of the code to compile) does not match. Parameters: - - buildcmd and execcmd are shell commands to run. buildcmd can be any shell line (incuding && groups), but execcmd must be a single binary to run. + - buildcmd and execcmd are shell commands to run. buildcmd can be any shell line (incuding && groups), but execcmd must be a single binary to run. - cachefile is the name of the test - filename is the source file containing the code - binary the file name in which to compile the code @@ -214,7 +223,7 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo, """ if os.path.exists(f'{cachefile}.txt') and os.path.exists(f'{cachefile}.elapsed') and os.path.exists(f'{cachefile}.md5sum'): hash_md5 = hashlib.md5() - with open(filename, 'rb') as sourcefile : + with open(filename, 'rb') as sourcefile: for chunk in iter(lambda: sourcefile.read(4096), b""): hash_md5.update(chunk) newdigest = hash_md5.hexdigest() @@ -224,19 +233,18 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo, if olddigest == newdigest: print(f" (result cached -- digest: {olddigest})") return False - else: - os.remove(f'{cachefile}.txt') + os.remove(f'{cachefile}.txt') print(f"Wait up to {timeout} seconds") start_time = time.time() - if buildcmd == None: + if buildcmd is None: output = f"No need to compile {binary}.c (batchinfo:{batchinfo})\n\n" else: output = f"Compiling {binary}.c (batchinfo:{batchinfo})\n\n" output += f"$ {buildcmd}\n" - compil = subprocess.run(buildcmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + compil = subprocess.run(buildcmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if compil.stdout is not None: output += str(compil.stdout, errors='replace') if compil.returncode != 0: @@ -261,7 +269,6 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo, pid = process.pid pgid = os.getpgid(pid) # We need that to forcefully kill subprocesses when leaving - outcome = None while True: if poll_obj.poll(5): # Something to read? Do check the timeout status every 5 sec if not line = process.stdout.readline() @@ -272,7 +279,6 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo, if read_line_lambda != None: read_line_lambda(line, process) if time.time() - start_time > timeout: - outcome = 'timeout' with open(f'{cachefile}.timeout', 'w') as outfile: outfile.write(f'{time.time() - start_time} seconds') break @@ -315,9 +321,9 @@ def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo, outfile.write(output) with open(f'{cachefile}.md5sum', 'w') as outfile: hashed = hashlib.md5() - with open(filename, 'rb') as sourcefile : + with open(filename, 'rb') as sourcefile: for chunk in iter(lambda: sourcefile.read(4096), b""): hashed.update(chunk) outfile.write(hashed.hexdigest()) - + return True