Issue #25220: Add functional tests to test_regrtest
* test all available ways to run the Python test suite * test many regrtest options: --slow, --coverage, -r, -u, etc. Note: python -m test --coverage doesn't work on Windows.
This commit is contained in:
Victor Stinner
parent
174d059248
commit
6b415101a2
|
|
@ -1,18 +1,33 @@
|
||||||
"""
|
"""
|
||||||
Tests of regrtest.py.
|
Tests of regrtest.py.
|
||||||
|
|
||||||
|
Note: test_regrtest cannot be run twice in parallel.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import faulthandler
|
import faulthandler
|
||||||
import getopt
|
import getopt
|
||||||
import os.path
|
import os.path
|
||||||
|
import platform
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import textwrap
|
||||||
import unittest
|
import unittest
|
||||||
from test import libregrtest
|
from test import libregrtest
|
||||||
from test import support
|
from test import support
|
||||||
|
from test.support import script_helper
|
||||||
|
|
||||||
|
|
||||||
|
Py_DEBUG = hasattr(sys, 'getobjects')
|
||||||
|
ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
|
||||||
|
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
|
||||||
|
|
||||||
|
|
||||||
class ParseArgsTestCase(unittest.TestCase):
|
class ParseArgsTestCase(unittest.TestCase):
|
||||||
|
"""
|
||||||
"""Test regrtest's argument parsing."""
|
Test regrtest's argument parsing, function _parse_args().
|
||||||
|
"""
|
||||||
|
|
||||||
def checkError(self, args, msg):
|
def checkError(self, args, msg):
|
||||||
with support.captured_stderr() as err, self.assertRaises(SystemExit):
|
with support.captured_stderr() as err, self.assertRaises(SystemExit):
|
||||||
|
|
@ -272,5 +287,279 @@ class ParseArgsTestCase(unittest.TestCase):
|
||||||
self.assertEqual(ns.args, ['foo'])
|
self.assertEqual(ns.args, ['foo'])
|
||||||
|
|
||||||
|
|
||||||
|
class BaseTestCase(unittest.TestCase):
|
||||||
|
TEST_UNIQUE_ID = 1
|
||||||
|
TESTNAME_PREFIX = 'test_regrtest_'
|
||||||
|
TESTNAME_REGEX = r'test_[a-z0-9_]+'
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.testdir = os.path.join(ROOT_DIR, 'Lib', 'test')
|
||||||
|
|
||||||
|
# When test_regrtest is interrupted by CTRL+c, it can leave
|
||||||
|
# temporary test files
|
||||||
|
remove = [entry.path
|
||||||
|
for entry in os.scandir(self.testdir)
|
||||||
|
if (entry.name.startswith(self.TESTNAME_PREFIX)
|
||||||
|
and entry.name.endswith(".py"))]
|
||||||
|
for path in remove:
|
||||||
|
print("WARNING: test_regrtest: remove %s" % path)
|
||||||
|
support.unlink(path)
|
||||||
|
|
||||||
|
def create_test(self, name=None, code=''):
|
||||||
|
if not name:
|
||||||
|
name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID
|
||||||
|
BaseTestCase.TEST_UNIQUE_ID += 1
|
||||||
|
|
||||||
|
# test_regrtest cannot be run twice in parallel because
|
||||||
|
# of setUp() and create_test()
|
||||||
|
name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name)
|
||||||
|
path = os.path.join(self.testdir, name + '.py')
|
||||||
|
|
||||||
|
self.addCleanup(support.unlink, path)
|
||||||
|
# Use 'x' mode to ensure that we do not override existing tests
|
||||||
|
with open(path, 'x', encoding='utf-8') as fp:
|
||||||
|
fp.write(code)
|
||||||
|
return name
|
||||||
|
|
||||||
|
def regex_search(self, regex, output):
|
||||||
|
match = re.search(regex, output, re.MULTILINE)
|
||||||
|
if not match:
|
||||||
|
self.fail("%r not found in %r" % (regex, output))
|
||||||
|
return match
|
||||||
|
|
||||||
|
def check_line(self, output, regex):
|
||||||
|
regex = re.compile(r'^' + regex, re.MULTILINE)
|
||||||
|
self.assertRegex(output, regex)
|
||||||
|
|
||||||
|
def parse_executed_tests(self, output):
|
||||||
|
parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX,
|
||||||
|
output,
|
||||||
|
re.MULTILINE)
|
||||||
|
return set(match.group(1) for match in parser)
|
||||||
|
|
||||||
|
def check_executed_tests(self, output, tests, skipped=None):
|
||||||
|
if isinstance(tests, str):
|
||||||
|
tests = [tests]
|
||||||
|
executed = self.parse_executed_tests(output)
|
||||||
|
self.assertEqual(executed, set(tests), output)
|
||||||
|
ntest = len(tests)
|
||||||
|
if skipped:
|
||||||
|
if isinstance(skipped, str):
|
||||||
|
skipped = [skipped]
|
||||||
|
nskipped = len(skipped)
|
||||||
|
|
||||||
|
plural = 's' if nskipped != 1 else ''
|
||||||
|
names = ' '.join(sorted(skipped))
|
||||||
|
expected = (r'%s test%s skipped:\n %s$'
|
||||||
|
% (nskipped, plural, names))
|
||||||
|
self.check_line(output, expected)
|
||||||
|
|
||||||
|
ok = ntest - nskipped
|
||||||
|
if ok:
|
||||||
|
self.check_line(output, r'%s test OK\.$' % ok)
|
||||||
|
else:
|
||||||
|
self.check_line(output, r'All %s tests OK\.$' % ntest)
|
||||||
|
|
||||||
|
def parse_random_seed(self, output):
|
||||||
|
match = self.regex_search(r'Using random seed ([0-9]+)', output)
|
||||||
|
randseed = int(match.group(1))
|
||||||
|
self.assertTrue(0 <= randseed <= 10000000, randseed)
|
||||||
|
return randseed
|
||||||
|
|
||||||
|
|
||||||
|
class ProgramsTestCase(BaseTestCase):
|
||||||
|
"""
|
||||||
|
Test various ways to run the Python test suite. Use options close
|
||||||
|
to options used on the buildbot.
|
||||||
|
"""
|
||||||
|
|
||||||
|
NTEST = 4
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super().setUp()
|
||||||
|
|
||||||
|
# Create NTEST tests doing nothing
|
||||||
|
self.tests = [self.create_test() for index in range(self.NTEST)]
|
||||||
|
|
||||||
|
self.python_args = ['-Wd', '-E', '-bb']
|
||||||
|
self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4']
|
||||||
|
if sys.platform == 'win32':
|
||||||
|
self.regrtest_args.append('-n')
|
||||||
|
|
||||||
|
def check_output(self, output):
|
||||||
|
self.parse_random_seed(output)
|
||||||
|
self.check_executed_tests(output, self.tests)
|
||||||
|
|
||||||
|
def run_tests(self, args):
|
||||||
|
res = script_helper.assert_python_ok(*args)
|
||||||
|
output = os.fsdecode(res.out)
|
||||||
|
self.check_output(output)
|
||||||
|
|
||||||
|
def test_script_regrtest(self):
|
||||||
|
# Lib/test/regrtest.py
|
||||||
|
script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py')
|
||||||
|
|
||||||
|
args = [*self.python_args, script, *self.regrtest_args, *self.tests]
|
||||||
|
self.run_tests(args)
|
||||||
|
|
||||||
|
def test_module_test(self):
|
||||||
|
# -m test
|
||||||
|
args = [*self.python_args, '-m', 'test',
|
||||||
|
*self.regrtest_args, *self.tests]
|
||||||
|
self.run_tests(args)
|
||||||
|
|
||||||
|
def test_module_regrtest(self):
|
||||||
|
# -m test.regrtest
|
||||||
|
args = [*self.python_args, '-m', 'test.regrtest',
|
||||||
|
*self.regrtest_args, *self.tests]
|
||||||
|
self.run_tests(args)
|
||||||
|
|
||||||
|
def test_module_autotest(self):
|
||||||
|
# -m test.autotest
|
||||||
|
args = [*self.python_args, '-m', 'test.autotest',
|
||||||
|
*self.regrtest_args, *self.tests]
|
||||||
|
self.run_tests(args)
|
||||||
|
|
||||||
|
def test_module_from_test_autotest(self):
|
||||||
|
# from test import autotest
|
||||||
|
code = 'from test import autotest'
|
||||||
|
args = [*self.python_args, '-c', code,
|
||||||
|
*self.regrtest_args, *self.tests]
|
||||||
|
self.run_tests(args)
|
||||||
|
|
||||||
|
def test_script_autotest(self):
|
||||||
|
# Lib/test/autotest.py
|
||||||
|
script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py')
|
||||||
|
args = [*self.python_args, script, *self.regrtest_args, *self.tests]
|
||||||
|
self.run_tests(args)
|
||||||
|
|
||||||
|
def test_tools_script_run_tests(self):
|
||||||
|
# Tools/scripts/run_tests.py
|
||||||
|
script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py')
|
||||||
|
self.run_tests([script, *self.tests])
|
||||||
|
|
||||||
|
def run_rt_bat(self, script, *args):
|
||||||
|
rt_args = []
|
||||||
|
if platform.architecture()[0] == '64bit':
|
||||||
|
rt_args.append('-x64') # 64-bit build
|
||||||
|
if Py_DEBUG:
|
||||||
|
rt_args.append('-d') # Debug build
|
||||||
|
|
||||||
|
args = [script, *rt_args, *args]
|
||||||
|
proc = subprocess.run(args,
|
||||||
|
check=True, universal_newlines=True,
|
||||||
|
input='',
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
self.check_output(proc.stdout)
|
||||||
|
|
||||||
|
@unittest.skipUnless(sys.platform == 'win32', 'Windows only')
|
||||||
|
def test_tools_buildbot_test(self):
|
||||||
|
# Tools\buildbot\test.bat
|
||||||
|
script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat')
|
||||||
|
self.run_rt_bat(script, *self.tests)
|
||||||
|
|
||||||
|
@unittest.skipUnless(sys.platform == 'win32', 'Windows only')
|
||||||
|
def test_pcbuild_rt(self):
|
||||||
|
# PCbuild\rt.bat
|
||||||
|
script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat')
|
||||||
|
# -q: quick, don't run tests twice
|
||||||
|
rt_args = ["-q"]
|
||||||
|
self.run_rt_bat(script, *rt_args, *self.regrtest_args, *self.tests)
|
||||||
|
|
||||||
|
|
||||||
|
class ArgsTestCase(BaseTestCase):
|
||||||
|
"""
|
||||||
|
Test arguments of the Python test suite.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def run_tests(self, *args):
|
||||||
|
args = ['-m', 'test', *args]
|
||||||
|
res = script_helper.assert_python_ok(*args)
|
||||||
|
return os.fsdecode(res.out)
|
||||||
|
|
||||||
|
def test_resources(self):
|
||||||
|
# test -u command line option
|
||||||
|
tests = {}
|
||||||
|
for resource in ('audio', 'network'):
|
||||||
|
code = 'from test import support\nsupport.requires(%r)' % resource
|
||||||
|
tests[resource] = self.create_test(resource, code)
|
||||||
|
test_names = sorted(tests.values())
|
||||||
|
|
||||||
|
# -u all: 2 resources enabled
|
||||||
|
output = self.run_tests('-u', 'all', *test_names)
|
||||||
|
self.check_executed_tests(output, test_names)
|
||||||
|
|
||||||
|
# -u audio: 1 resource enabled
|
||||||
|
output = self.run_tests('-uaudio', *test_names)
|
||||||
|
self.check_executed_tests(output, test_names,
|
||||||
|
skipped=tests['network'])
|
||||||
|
|
||||||
|
# no option: 0 resources enabled
|
||||||
|
output = self.run_tests(*test_names)
|
||||||
|
self.check_executed_tests(output, test_names,
|
||||||
|
skipped=test_names)
|
||||||
|
|
||||||
|
def test_random(self):
|
||||||
|
# test -r and --randseed command line option
|
||||||
|
code = textwrap.dedent("""
|
||||||
|
import random
|
||||||
|
print("TESTRANDOM: %s" % random.randint(1, 1000))
|
||||||
|
""")
|
||||||
|
test = self.create_test('random', code)
|
||||||
|
|
||||||
|
# first run to get the output with the random seed
|
||||||
|
output = self.run_tests('-r', test)
|
||||||
|
randseed = self.parse_random_seed(output)
|
||||||
|
match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
|
||||||
|
test_random = int(match.group(1))
|
||||||
|
|
||||||
|
# try to reproduce with the random seed
|
||||||
|
output = self.run_tests('-r', '--randseed=%s' % randseed, test)
|
||||||
|
randseed2 = self.parse_random_seed(output)
|
||||||
|
self.assertEqual(randseed2, randseed)
|
||||||
|
|
||||||
|
match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
|
||||||
|
test_random2 = int(match.group(1))
|
||||||
|
self.assertEqual(test_random2, test_random)
|
||||||
|
|
||||||
|
def test_fromfile(self):
|
||||||
|
# test --fromfile
|
||||||
|
tests = [self.create_test() for index in range(5)]
|
||||||
|
|
||||||
|
# Write the list of files using a format similar to regrtest output:
|
||||||
|
# [1/2] test_1
|
||||||
|
# [2/2] test_2
|
||||||
|
filename = support.TESTFN
|
||||||
|
self.addCleanup(support.unlink, filename)
|
||||||
|
with open(filename, "w") as fp:
|
||||||
|
for index, name in enumerate(tests, 1):
|
||||||
|
print("[%s/%s] %s" % (index, len(tests), name), file=fp)
|
||||||
|
|
||||||
|
output = self.run_tests('--fromfile', filename)
|
||||||
|
self.check_executed_tests(output, tests)
|
||||||
|
|
||||||
|
def test_slow(self):
|
||||||
|
# test --slow
|
||||||
|
tests = [self.create_test() for index in range(3)]
|
||||||
|
output = self.run_tests("--slow", *tests)
|
||||||
|
self.check_executed_tests(output, tests)
|
||||||
|
regex = ('10 slowest tests:\n'
|
||||||
|
'(?:%s: [0-9]+\.[0-9]+s\n){%s}'
|
||||||
|
% (self.TESTNAME_REGEX, len(tests)))
|
||||||
|
self.check_line(output, regex)
|
||||||
|
|
||||||
|
@unittest.skipIf(sys.platform == 'win32',
|
||||||
|
"FIXME: coverage doesn't work on Windows")
|
||||||
|
def test_coverage(self):
|
||||||
|
# test --coverage
|
||||||
|
test = self.create_test()
|
||||||
|
output = self.run_tests("--coverage", test)
|
||||||
|
executed = self.parse_executed_tests(output)
|
||||||
|
self.assertEqual(executed, {test}, output)
|
||||||
|
regex = ('lines +cov% +module +\(path\)\n'
|
||||||
|
'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+')
|
||||||
|
self.check_line(output, regex)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue