'''Test runner and result class for the regression test suite. ''' import functools import io import sys import time import traceback import unittest from test import support from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, verbosity=2 if verbosity else 0) self.buffer = True if self.USE_XML: from xml.etree import ElementTree as ET from datetime import datetime, UTC self.__ET = ET self.__suite = ET.Element('testsuite') self.__suite.set('start', datetime.now(UTC) .replace(tzinfo=None) .isoformat(' ')) self.__e = None self.__start_time = None @classmethod def __getId(cls, test): try: test_id = test.id except AttributeError: return str(test) try: return test_id() except TypeError: return str(test_id) return repr(test) def startTest(self, test): super().startTest(test) if self.USE_XML: self.__e = e = self.__ET.SubElement(self.__suite, 'testcase') self.__start_time = time.perf_counter() def _add_result(self, test, capture=False, **args): if not self.USE_XML: return e = self.__e self.__e = None if e is None: return ET = self.__ET e.set('name', args.pop('name', self.__getId(test))) e.set('status', args.pop('status', 'run')) e.set('result', args.pop('result', 'completed')) if self.__start_time: e.set('time', f'{time.perf_counter() - self.__start_time:0.6f}') if capture: if self._stdout_buffer is not None: stdout = self._stdout_buffer.getvalue().rstrip() ET.SubElement(e, 'system-out').text = sanitize_xml(stdout) if self._stderr_buffer is not None: stderr = self._stderr_buffer.getvalue().rstrip() ET.SubElement(e, 'system-err').text = sanitize_xml(stderr) for k, v in args.items(): if not k or not v: continue e2 = ET.SubElement(e, k) if hasattr(v, 'items'): for k2, v2 in v.items(): if k2: e2.set(k2, sanitize_xml(str(v2))) else: e2.text = sanitize_xml(str(v2)) else: e2.text = sanitize_xml(str(v)) @classmethod def __makeErrorDict(cls, err_type, err_value, err_tb): if isinstance(err_type, type): if err_type.__module__ == 'builtins': typename = err_type.__name__ else: typename = f'{err_type.__module__}.{err_type.__name__}' else: typename = repr(err_type) msg = traceback.format_exception(err_type, err_value, None) tb = traceback.format_exception(err_type, err_value, err_tb) return { 'type': typename, 'message': ''.join(msg), '': ''.join(tb), } def addError(self, test, err): self._add_result(test, True, error=self.__makeErrorDict(*err)) super().addError(test, err) def addExpectedFailure(self, test, err): self._add_result(test, True, output=self.__makeErrorDict(*err)) super().addExpectedFailure(test, err) def addFailure(self, test, err): self._add_result(test, True, failure=self.__makeErrorDict(*err)) super().addFailure(test, err) if support.failfast: self.stop() def addSkip(self, test, reason): self._add_result(test, skipped=reason) super().addSkip(test, reason) def addSuccess(self, test): self._add_result(test) super().addSuccess(test) def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") e = self.__suite e.set('tests', str(self.testsRun)) e.set('errors', str(len(self.errors))) e.set('failures', str(len(self.failures))) return e class QuietRegressionTestRunner: def __init__(self, stream, buffer=False): self.result = RegressionTestResult(stream, None, 0) self.result.buffer = buffer def run(self, test): test(self.result) return self.result def get_test_runner_class(verbosity, buffer=False): if verbosity: return functools.partial(unittest.TextTestRunner, resultclass=RegressionTestResult, buffer=buffer, verbosity=verbosity) return functools.partial(QuietRegressionTestRunner, buffer=buffer) def get_test_runner(stream, verbosity, capture_output=False): return get_test_runner_class(verbosity, capture_output)(stream) if __name__ == '__main__': import xml.etree.ElementTree as ET RegressionTestResult.USE_XML = True class TestTests(unittest.TestCase): def test_pass(self): pass def test_pass_slow(self): time.sleep(1.0) def test_fail(self): print('stdout', file=sys.stdout) print('stderr', file=sys.stderr) self.fail('failure message') def test_error(self): print('stdout', file=sys.stdout) print('stderr', file=sys.stderr) raise RuntimeError('error message') suite = unittest.TestSuite() suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestTests)) stream = io.StringIO() runner_cls = get_test_runner_class(sum(a == '-v' for a in sys.argv)) runner = runner_cls(sys.stdout) result = runner.run(suite) print('Output:', stream.getvalue()) print('XML: ', end='') for s in ET.tostringlist(result.get_xml_element()): print(s.decode(), end='') print()