Issue #25220: Add functional tests to test_regrtest
* test all available ways to run the Python test suite * test many regrtest options: --slow, --coverage, -r, -u, etc. Note: python -m test --coverage doesn't work on Windows.
This commit is contained in:
Victor Stinner
parent
174d059248
commit
6b415101a2
|
|
@ -1,18 +1,33 @@
|
|||
"""
|
||||
Tests of regrtest.py.
|
||||
|
||||
Note: test_regrtest cannot be run twice in parallel.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import faulthandler
|
||||
import getopt
|
||||
import os.path
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import unittest
|
||||
from test import libregrtest
|
||||
from test import support
|
||||
from test.support import script_helper
|
||||
|
||||
|
||||
Py_DEBUG = hasattr(sys, 'getobjects')
|
||||
ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
|
||||
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
|
||||
|
||||
|
||||
class ParseArgsTestCase(unittest.TestCase):
|
||||
|
||||
"""Test regrtest's argument parsing."""
|
||||
"""
|
||||
Test regrtest's argument parsing, function _parse_args().
|
||||
"""
|
||||
|
||||
def checkError(self, args, msg):
|
||||
with support.captured_stderr() as err, self.assertRaises(SystemExit):
|
||||
|
|
@ -272,5 +287,279 @@ class ParseArgsTestCase(unittest.TestCase):
|
|||
self.assertEqual(ns.args, ['foo'])
|
||||
|
||||
|
||||
class BaseTestCase(unittest.TestCase):
|
||||
TEST_UNIQUE_ID = 1
|
||||
TESTNAME_PREFIX = 'test_regrtest_'
|
||||
TESTNAME_REGEX = r'test_[a-z0-9_]+'
|
||||
|
||||
def setUp(self):
|
||||
self.testdir = os.path.join(ROOT_DIR, 'Lib', 'test')
|
||||
|
||||
# When test_regrtest is interrupted by CTRL+c, it can leave
|
||||
# temporary test files
|
||||
remove = [entry.path
|
||||
for entry in os.scandir(self.testdir)
|
||||
if (entry.name.startswith(self.TESTNAME_PREFIX)
|
||||
and entry.name.endswith(".py"))]
|
||||
for path in remove:
|
||||
print("WARNING: test_regrtest: remove %s" % path)
|
||||
support.unlink(path)
|
||||
|
||||
def create_test(self, name=None, code=''):
|
||||
if not name:
|
||||
name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID
|
||||
BaseTestCase.TEST_UNIQUE_ID += 1
|
||||
|
||||
# test_regrtest cannot be run twice in parallel because
|
||||
# of setUp() and create_test()
|
||||
name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name)
|
||||
path = os.path.join(self.testdir, name + '.py')
|
||||
|
||||
self.addCleanup(support.unlink, path)
|
||||
# Use 'x' mode to ensure that we do not override existing tests
|
||||
with open(path, 'x', encoding='utf-8') as fp:
|
||||
fp.write(code)
|
||||
return name
|
||||
|
||||
def regex_search(self, regex, output):
|
||||
match = re.search(regex, output, re.MULTILINE)
|
||||
if not match:
|
||||
self.fail("%r not found in %r" % (regex, output))
|
||||
return match
|
||||
|
||||
def check_line(self, output, regex):
|
||||
regex = re.compile(r'^' + regex, re.MULTILINE)
|
||||
self.assertRegex(output, regex)
|
||||
|
||||
def parse_executed_tests(self, output):
|
||||
parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX,
|
||||
output,
|
||||
re.MULTILINE)
|
||||
return set(match.group(1) for match in parser)
|
||||
|
||||
def check_executed_tests(self, output, tests, skipped=None):
|
||||
if isinstance(tests, str):
|
||||
tests = [tests]
|
||||
executed = self.parse_executed_tests(output)
|
||||
self.assertEqual(executed, set(tests), output)
|
||||
ntest = len(tests)
|
||||
if skipped:
|
||||
if isinstance(skipped, str):
|
||||
skipped = [skipped]
|
||||
nskipped = len(skipped)
|
||||
|
||||
plural = 's' if nskipped != 1 else ''
|
||||
names = ' '.join(sorted(skipped))
|
||||
expected = (r'%s test%s skipped:\n %s$'
|
||||
% (nskipped, plural, names))
|
||||
self.check_line(output, expected)
|
||||
|
||||
ok = ntest - nskipped
|
||||
if ok:
|
||||
self.check_line(output, r'%s test OK\.$' % ok)
|
||||
else:
|
||||
self.check_line(output, r'All %s tests OK\.$' % ntest)
|
||||
|
||||
def parse_random_seed(self, output):
|
||||
match = self.regex_search(r'Using random seed ([0-9]+)', output)
|
||||
randseed = int(match.group(1))
|
||||
self.assertTrue(0 <= randseed <= 10000000, randseed)
|
||||
return randseed
|
||||
|
||||
|
||||
class ProgramsTestCase(BaseTestCase):
|
||||
"""
|
||||
Test various ways to run the Python test suite. Use options close
|
||||
to options used on the buildbot.
|
||||
"""
|
||||
|
||||
NTEST = 4
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
# Create NTEST tests doing nothing
|
||||
self.tests = [self.create_test() for index in range(self.NTEST)]
|
||||
|
||||
self.python_args = ['-Wd', '-E', '-bb']
|
||||
self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4']
|
||||
if sys.platform == 'win32':
|
||||
self.regrtest_args.append('-n')
|
||||
|
||||
def check_output(self, output):
|
||||
self.parse_random_seed(output)
|
||||
self.check_executed_tests(output, self.tests)
|
||||
|
||||
def run_tests(self, args):
|
||||
res = script_helper.assert_python_ok(*args)
|
||||
output = os.fsdecode(res.out)
|
||||
self.check_output(output)
|
||||
|
||||
def test_script_regrtest(self):
|
||||
# Lib/test/regrtest.py
|
||||
script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py')
|
||||
|
||||
args = [*self.python_args, script, *self.regrtest_args, *self.tests]
|
||||
self.run_tests(args)
|
||||
|
||||
def test_module_test(self):
|
||||
# -m test
|
||||
args = [*self.python_args, '-m', 'test',
|
||||
*self.regrtest_args, *self.tests]
|
||||
self.run_tests(args)
|
||||
|
||||
def test_module_regrtest(self):
|
||||
# -m test.regrtest
|
||||
args = [*self.python_args, '-m', 'test.regrtest',
|
||||
*self.regrtest_args, *self.tests]
|
||||
self.run_tests(args)
|
||||
|
||||
def test_module_autotest(self):
|
||||
# -m test.autotest
|
||||
args = [*self.python_args, '-m', 'test.autotest',
|
||||
*self.regrtest_args, *self.tests]
|
||||
self.run_tests(args)
|
||||
|
||||
def test_module_from_test_autotest(self):
|
||||
# from test import autotest
|
||||
code = 'from test import autotest'
|
||||
args = [*self.python_args, '-c', code,
|
||||
*self.regrtest_args, *self.tests]
|
||||
self.run_tests(args)
|
||||
|
||||
def test_script_autotest(self):
|
||||
# Lib/test/autotest.py
|
||||
script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py')
|
||||
args = [*self.python_args, script, *self.regrtest_args, *self.tests]
|
||||
self.run_tests(args)
|
||||
|
||||
def test_tools_script_run_tests(self):
|
||||
# Tools/scripts/run_tests.py
|
||||
script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py')
|
||||
self.run_tests([script, *self.tests])
|
||||
|
||||
def run_rt_bat(self, script, *args):
|
||||
rt_args = []
|
||||
if platform.architecture()[0] == '64bit':
|
||||
rt_args.append('-x64') # 64-bit build
|
||||
if Py_DEBUG:
|
||||
rt_args.append('-d') # Debug build
|
||||
|
||||
args = [script, *rt_args, *args]
|
||||
proc = subprocess.run(args,
|
||||
check=True, universal_newlines=True,
|
||||
input='',
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
self.check_output(proc.stdout)
|
||||
|
||||
@unittest.skipUnless(sys.platform == 'win32', 'Windows only')
|
||||
def test_tools_buildbot_test(self):
|
||||
# Tools\buildbot\test.bat
|
||||
script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat')
|
||||
self.run_rt_bat(script, *self.tests)
|
||||
|
||||
@unittest.skipUnless(sys.platform == 'win32', 'Windows only')
|
||||
def test_pcbuild_rt(self):
|
||||
# PCbuild\rt.bat
|
||||
script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat')
|
||||
# -q: quick, don't run tests twice
|
||||
rt_args = ["-q"]
|
||||
self.run_rt_bat(script, *rt_args, *self.regrtest_args, *self.tests)
|
||||
|
||||
|
||||
class ArgsTestCase(BaseTestCase):
|
||||
"""
|
||||
Test arguments of the Python test suite.
|
||||
"""
|
||||
|
||||
def run_tests(self, *args):
|
||||
args = ['-m', 'test', *args]
|
||||
res = script_helper.assert_python_ok(*args)
|
||||
return os.fsdecode(res.out)
|
||||
|
||||
def test_resources(self):
|
||||
# test -u command line option
|
||||
tests = {}
|
||||
for resource in ('audio', 'network'):
|
||||
code = 'from test import support\nsupport.requires(%r)' % resource
|
||||
tests[resource] = self.create_test(resource, code)
|
||||
test_names = sorted(tests.values())
|
||||
|
||||
# -u all: 2 resources enabled
|
||||
output = self.run_tests('-u', 'all', *test_names)
|
||||
self.check_executed_tests(output, test_names)
|
||||
|
||||
# -u audio: 1 resource enabled
|
||||
output = self.run_tests('-uaudio', *test_names)
|
||||
self.check_executed_tests(output, test_names,
|
||||
skipped=tests['network'])
|
||||
|
||||
# no option: 0 resources enabled
|
||||
output = self.run_tests(*test_names)
|
||||
self.check_executed_tests(output, test_names,
|
||||
skipped=test_names)
|
||||
|
||||
def test_random(self):
|
||||
# test -r and --randseed command line option
|
||||
code = textwrap.dedent("""
|
||||
import random
|
||||
print("TESTRANDOM: %s" % random.randint(1, 1000))
|
||||
""")
|
||||
test = self.create_test('random', code)
|
||||
|
||||
# first run to get the output with the random seed
|
||||
output = self.run_tests('-r', test)
|
||||
randseed = self.parse_random_seed(output)
|
||||
match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
|
||||
test_random = int(match.group(1))
|
||||
|
||||
# try to reproduce with the random seed
|
||||
output = self.run_tests('-r', '--randseed=%s' % randseed, test)
|
||||
randseed2 = self.parse_random_seed(output)
|
||||
self.assertEqual(randseed2, randseed)
|
||||
|
||||
match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
|
||||
test_random2 = int(match.group(1))
|
||||
self.assertEqual(test_random2, test_random)
|
||||
|
||||
def test_fromfile(self):
|
||||
# test --fromfile
|
||||
tests = [self.create_test() for index in range(5)]
|
||||
|
||||
# Write the list of files using a format similar to regrtest output:
|
||||
# [1/2] test_1
|
||||
# [2/2] test_2
|
||||
filename = support.TESTFN
|
||||
self.addCleanup(support.unlink, filename)
|
||||
with open(filename, "w") as fp:
|
||||
for index, name in enumerate(tests, 1):
|
||||
print("[%s/%s] %s" % (index, len(tests), name), file=fp)
|
||||
|
||||
output = self.run_tests('--fromfile', filename)
|
||||
self.check_executed_tests(output, tests)
|
||||
|
||||
def test_slow(self):
|
||||
# test --slow
|
||||
tests = [self.create_test() for index in range(3)]
|
||||
output = self.run_tests("--slow", *tests)
|
||||
self.check_executed_tests(output, tests)
|
||||
regex = ('10 slowest tests:\n'
|
||||
'(?:%s: [0-9]+\.[0-9]+s\n){%s}'
|
||||
% (self.TESTNAME_REGEX, len(tests)))
|
||||
self.check_line(output, regex)
|
||||
|
||||
@unittest.skipIf(sys.platform == 'win32',
|
||||
"FIXME: coverage doesn't work on Windows")
|
||||
def test_coverage(self):
|
||||
# test --coverage
|
||||
test = self.create_test()
|
||||
output = self.run_tests("--coverage", test)
|
||||
executed = self.parse_executed_tests(output)
|
||||
self.assertEqual(executed, {test}, output)
|
||||
regex = ('lines +cov% +module +\(path\)\n'
|
||||
'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+')
|
||||
self.check_line(output, regex)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue