Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-109566, regrtest: Add --fast-ci and --slow-ci options (#109570) 

* Add --fast-ci and --slow-ci options to libregrtest:

  * --fast-ci uses a default timeout of 10 minutes and "-u all,-cpu"
    (skip slowest tests).
  * --slow-ci uses a default timeout of 20 minues and "-u all" (run
    all tests).

* regrtest header now lists test resources.
* Makefile changes:

  * "make test", "make hostrunnertest" and "make coverage-report" now
    use --fast-ci option and TESTTIMEOUT variable.
  * "make buildbottest" now uses "--slow-ci". Remove options which
    became redundant with "--slow-ci".
  * "make testall" and "make testuniversal" now use --slow-ci option
    and TESTTIMEOUT variable.
  * "make testall" now uses "find -exec rm ..." instead of
    "find ... -print|xargs rm ...", same as "make clean".

* GitHub Actions workflow:

  * Ubuntu and Address Sanitizer jobs now use "make test". Remove
    options which became redundant with "--fast-ci".
  * Windows jobs now use --fast-ci option.
  * Use -j0 to detect the number of CPUs.

* Set Makefile TESTTIMEOUT default to an empty string, since
  --slow-ci and --fast-ci use different default timeout. It's now
  accepted to pass "--timeout=" to regrtest: treated as not timeout.
* Tools/scripts/run_tests.py now uses --fast-ci option.
* Tools/buildbot/test.bat now uses --slow-ci option. Remove
  --timeout=1200 option, redundant with --slow-ci.
This commit is contained in:
Victor Stinner 2023-09-26 17:22:50 +02:00 committed by GitHub
parent 19bf398695
commit 859618c8cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 161 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/build.yml vendored
View File

@ -182,7 +182,7 @@ jobs:
- name: Display build info
run: .\python.bat -m test.pythoninfo
- name: Tests
run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci
build_win_amd64:
name: 'Windows (x64)'
@ -201,7 +201,7 @@ jobs:
- name: Display build info
run: .\python.bat -m test.pythoninfo
- name: Tests
run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci
build_win_arm64:
name: 'Windows (arm64)'
@ -252,7 +252,7 @@ jobs:
- name: Display build info
run: make pythoninfo
- name: Tests
run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
run: make test
build_ubuntu:
name: 'Ubuntu'
@ -319,7 +319,7 @@ jobs:
run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
- name: Tests
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
run: xvfb-run make test
build_ubuntu_ssltests:
name: 'Ubuntu SSL tests with OpenSSL'
@ -535,7 +535,7 @@ jobs:
- name: Display build info
run: make pythoninfo
- name: Tests
run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
run: xvfb-run make test
all-required-green: # This job does nothing and is only used for the branch protection
name: All required checks pass

15
Doc/using/configure.rst
View File

@ -964,9 +964,18 @@ Main Makefile targets
You can use the configure :option:`--enable-optimizations` option to make
this the default target of the ``make`` command (``make all`` or just
``make``).
* ``make buildbottest``: Build Python and run the Python test suite, the same
way than buildbots test Python. Set ``TESTTIMEOUT`` variable (in seconds)
to change the test timeout (1200 by default: 20 minutes).
* ``make test``: Build Python and run the Python test suite with ``--slow-ci``
option. Variables:
* ``TESTOPTS``: additional regrtest command line options.
* ``TESTPYTHONOPTS``: additional Python command line options.
* ``TESTTIMEOUT``: timeout in seconds (default: 20 minutes).
* ``make buildbottest``: Similar to ``make test``, but use ``--slow-ci``
option and default timeout of 20 minutes, instead of ``--fast-ci`` option
and a default timeout of 10 minutes.
* ``make install``: Build and install Python.
* ``make regen-all``: Regenerate (almost) all generated files;
``make regen-stdlib-module-names`` and ``autoconf`` must be run separately

55
Lib/test/libregrtest/cmdline.py
View File

@ -4,6 +4,8 @@ import shlex
import sys
from test.support import os_helper
from .utils import MS_WINDOWS
USAGE = """\
python -m test [options] [test_name1 [test_name2 ...]]
@ -145,6 +147,7 @@ RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata')
class Namespace(argparse.Namespace):
def __init__(self, **kwargs) -> None:
self.ci = False
self.testdir = None
self.verbose = 0
self.quiet = False
@ -209,7 +212,13 @@ def _create_parser():
# We add help explicitly to control what argument group it renders under.
group.add_argument('-h', '--help', action='help',
help='show this help message and exit')
group.add_argument('--timeout', metavar='TIMEOUT', type=float,
group.add_argument('--fast-ci', action='store_true',
help='Fast Continuous Integration (CI) mode used by '
'GitHub Actions')
group.add_argument('--slow-ci', action='store_true',
help='Slow Continuous Integration (CI) mode used by '
'buildbot workers')
group.add_argument('--timeout', metavar='TIMEOUT',
help='dump the traceback and exit if a test takes '
'more than TIMEOUT seconds; disabled if TIMEOUT '
'is negative or equals to zero')
@ -384,7 +393,49 @@ def _parse_args(args, **kwargs):
for arg in ns.args:
if arg.startswith('-'):
parser.error("unrecognized arguments: %s" % arg)
sys.exit(1)
if ns.timeout is not None:
# Support "--timeout=" (no value) so Makefile.pre.pre TESTTIMEOUT
# can be used by "make buildbottest" and "make test".
if ns.timeout != "":
try:
ns.timeout = float(ns.timeout)
except ValueError:
parser.error(f"invalid timeout value: {ns.timeout!r}")
else:
ns.timeout = None
# Continuous Integration (CI): common options for fast/slow CI modes
if ns.slow_ci or ns.fast_ci:
# Similar to options:
#
# -j0 --randomize --fail-env-changed --fail-rerun --rerun
# --slowest --verbose3 --nowindows
if ns.use_mp is None:
ns.use_mp = 0
ns.randomize = True
ns.fail_env_changed = True
ns.fail_rerun = True
ns.rerun = True
ns.print_slow = True
ns.verbose3 = True
if MS_WINDOWS:
ns.nowindows = True # Silence alerts under Windows
# When both --slow-ci and --fast-ci options are present,
# --slow-ci has the priority
if ns.slow_ci:
# Similar to: -u "all" --timeout=1200
if not ns.use:
ns.use = [['all']]
if ns.timeout is None:
ns.timeout = 1200 # 20 minutes
elif ns.fast_ci:
# Similar to: -u "all,-cpu" --timeout=600
if not ns.use:
ns.use = [['all', '-cpu']]
if ns.timeout is None:
ns.timeout = 600 # 10 minutes
if ns.single and ns.fromfile:
parser.error("-s and -f don't go together!")

2
Lib/test/libregrtest/main.py
View File

@ -425,7 +425,7 @@ class Regrtest:
if (self.want_header
or not(self.pgo or self.quiet or self.single_test_run
or tests or self.cmdline_args)):
display_header()
display_header(self.use_resources)
if self.randomize:
print("Using random seed", self.random_seed)

4
Lib/test/libregrtest/results.py
View File

@ -8,11 +8,13 @@ from .utils import (
printlist, count, format_duration)
# Python uses exit code 1 when an exception is not catched
# argparse.ArgumentParser.error() uses exit code 2
EXITCODE_BAD_TEST = 2
EXITCODE_ENV_CHANGED = 3
EXITCODE_NO_TESTS_RAN = 4
EXITCODE_RERUN_FAIL = 5
EXITCODE_INTERRUPTED = 130
EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2
class TestResults:

9
Lib/test/libregrtest/utils.py
View File

@ -547,7 +547,7 @@ def adjust_rlimit_nofile():
f"{new_fd_limit}: {err}.")
def display_header():
def display_header(use_resources: tuple[str, ...]):
encoding = sys.stdout.encoding
# Print basic platform information
@ -569,6 +569,13 @@ def display_header():
print("== encodings: locale=%s, FS=%s"
% (locale.getencoding(), sys.getfilesystemencoding()))
if use_resources:
print(f"== resources ({len(use_resources)}): "
f"{', '.join(sorted(use_resources))}")
else:
print(f"== resources: (all disabled, use -u option)")
# This makes it easier to remember what to set in your local
# environment when trying to reproduce a sanitizer failure.
asan = support.check_sanitizer(address=True)

50
Lib/test/test_regrtest.py
View File

@ -23,8 +23,9 @@ import unittest
from test import support
from test.support import os_helper, TestStats, without_optimizer
from test.libregrtest import cmdline
from test.libregrtest import utils
from test.libregrtest import main
from test.libregrtest import setup
from test.libregrtest import utils
from test.libregrtest.utils import normalize_test_name
if not support.has_subprocess_support:
@ -75,8 +76,15 @@ class ParseArgsTestCase(unittest.TestCase):
def test_timeout(self):
ns = self.parse_args(['--timeout', '4.2'])
self.assertEqual(ns.timeout, 4.2)
# negative, zero and empty string are treated as "no timeout"
for value in ('-1', '0', ''):
with self.subTest(value=value):
ns = self.parse_args([f'--timeout={value}'])
self.assertEqual(ns.timeout, None)
self.checkError(['--timeout'], 'expected one argument')
self.checkError(['--timeout', 'foo'], 'invalid float value')
self.checkError(['--timeout', 'foo'], 'invalid timeout value:')
def test_wait(self):
ns = self.parse_args(['--wait'])
@ -366,6 +374,44 @@ class ParseArgsTestCase(unittest.TestCase):
self.checkError(['--unknown-option'],
'unrecognized arguments: --unknown-option')
def check_ci_mode(self, args, use_resources):
ns = cmdline._parse_args(args)
if utils.MS_WINDOWS:
self.assertTrue(ns.nowindows)
# Check Regrtest attributes which are more reliable than Namespace
# which has an unclear API
regrtest = main.Regrtest(ns)
self.assertNotEqual(regrtest.num_workers, 0)
self.assertTrue(regrtest.want_rerun)
self.assertTrue(regrtest.randomize)
self.assertIsNone(regrtest.random_seed)
self.assertTrue(regrtest.fail_env_changed)
self.assertTrue(regrtest.fail_rerun)
self.assertTrue(regrtest.print_slowest)
self.assertTrue(regrtest.output_on_failure)
self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources))
return regrtest
def test_fast_ci(self):
args = ['--fast-ci']
use_resources = sorted(cmdline.ALL_RESOURCES)
use_resources.remove('cpu')
regrtest = self.check_ci_mode(args, use_resources)
self.assertEqual(regrtest.timeout, 10 * 60)
def test_fast_ci_resource(self):
# it should be possible to override resources
args = ['--fast-ci', '-u', 'network']
use_resources = ['network']
self.check_ci_mode(args, use_resources)
def test_slow_ci(self):
args = ['--slow-ci']
use_resources = sorted(cmdline.ALL_RESOURCES)
regrtest = self.check_ci_mode(args, use_resources)
self.assertEqual(regrtest.timeout, 20 * 60)
@dataclasses.dataclass(slots=True)
class Rerun:

45
Makefile.pre.in
View File

@ -771,7 +771,7 @@ coverage-report: regen-token regen-frozen
@ # build with coverage info
$(MAKE) coverage
@ # run tests, ignore failures
$(TESTRUNNER) $(TESTOPTS) || true
$(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) || true
@ # build lcov report
$(MAKE) coverage-lcov
@ -1844,7 +1844,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS)
TESTOPTS= $(EXTRATESTOPTS)
TESTPYTHON= $(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS)
TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py
TESTTIMEOUT= 1200
TESTTIMEOUT=
# Remove "test_python_*" directories of previous failed test jobs.
# Pass TESTOPTS options because it can contain --tempdir option.
@ -1854,9 +1854,10 @@ cleantest: all
# Run a basic set of regression tests.
# This excludes some tests that are particularly resource-intensive.
# Similar to buildbottest, but use --fast-ci option, instead of --slow-ci.
.PHONY: test
test: all
$(TESTRUNNER) $(TESTOPTS)
$(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
# Run the full test suite twice - once without .pyc files, and once with.
# In the past, we've had problems where bugs in the marshalling or
@ -1867,43 +1868,43 @@ test: all
# sample data.
.PHONY: testall
testall: all
-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
$(TESTPYTHON) -E $(srcdir)/Lib/compileall.py
-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-$(TESTRUNNER) -u all $(TESTOPTS)
$(TESTRUNNER) -u all $(TESTOPTS)
-find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true
$(TESTPYTHON) -E $(srcdir)/Lib/compileall.py
-find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true
$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
# Run the test suite for both architectures in a Universal build on OSX.
# Must be run on an Intel box.
.PHONY: testuniversal
testuniversal: all
@if [ `arch` != 'i386' ]; then \
echo "This can only be used on OSX/i386" ;\
exit 1 ;\
fi
$(TESTRUNNER) -u all $(TESTOPTS)
$(RUNSHARED) /usr/libexec/oah/translate \
./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS)
@if [ `arch` != 'i386' ]; then \
echo "This can only be used on OSX/i386" ;\
exit 1 ;\
fi
$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
$(RUNSHARED) /usr/libexec/oah/translate \
./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS)
# Like testall, but with only one pass and without multiple processes.
# Run an optional script to include information about the build environment.
.PHONY: buildbottest
buildbottest: all
-@if which pybuildbot.identify >/dev/null 2>&1; then \
pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
fi
$(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --fail-rerun --timeout=$(TESTTIMEOUT) $(TESTOPTS)
-@if which pybuildbot.identify >/dev/null 2>&1; then \
pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
fi
$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
# Like testall, but run Python tests with HOSTRUNNER directly.
.PHONY: hostrunnertest
hostrunnertest: all
$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS)
$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
.PHONY: pythoninfo
pythoninfo: all
$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo
QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io \
QUICKTESTOPTS= -x test_subprocess test_io \
test_multibytecodec test_urllib2_localnet test_itertools \
test_multiprocessing_fork test_multiprocessing_spawn \
test_multiprocessing_forkserver \
@ -1912,7 +1913,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io \
.PHONY: quicktest
quicktest: all
$(TESTRUNNER) $(QUICKTESTOPTS)
$(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) $(QUICKTESTOPTS)
# SSL tests
.PHONY: multisslcompile

4
Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst Normal file
View File

@ -0,0 +1,4 @@
regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a
default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests).
``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all
tests). Patch by Victor Stinner.

6
Tools/buildbot/test.bat
View File

@ -5,7 +5,7 @@ setlocal
set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH;%SystemRoot%\System32\OpenSSH
set here=%~dp0
set rt_opts=-q -d
set regrtest_args=-j1
set regrtest_args=
set arm32_ssh=
:CheckOpts
@ -23,7 +23,7 @@ if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto NativeExecuti
if "%arm32_ssh%"=="true" goto :Arm32Ssh
:NativeExecution
call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 %regrtest_args%
call "%here%..\..\PCbuild\rt.bat" %rt_opts% --slow-ci %regrtest_args%
exit /b %ERRORLEVEL%
:Arm32Ssh
@ -35,7 +35,7 @@ if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DI
set TEMP_ARGS=--temp %REMOTE_PYTHON_DIR%temp
set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 %regrtest_args% %TEMP_ARGS%
set rt_args=%rt_opts% --slow-ci %dashU% %regrtest_args% %TEMP_ARGS%
ssh %SSH_SERVER% "set TEMP=%REMOTE_PYTHON_DIR%temp& cd %REMOTE_PYTHON_DIR% & %REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args%
set ERR=%ERRORLEVEL%
scp %SSH_SERVER%:"%REMOTE_PYTHON_DIR%test-results.xml" "%PYTHON_SOURCE%\test-results.xml"

12
Tools/scripts/run_tests.py
View File

@ -18,9 +18,6 @@ def is_multiprocess_flag(arg):
return arg.startswith('-j') or arg.startswith('--multiprocess')
def is_resource_use_flag(arg):
return arg.startswith('-u') or arg.startswith('--use')
def is_python_flag(arg):
return arg.startswith('-p') or arg.startswith('--python')
@ -56,20 +53,13 @@ def main(regrtest_args):
args.extend(test.support.args_from_interpreter_flags())
args.extend(['-m', 'test', # Run the test suite
'-r', # Randomize test order
'-w', # Re-run failed tests in verbose mode
'--fast-ci', # Fast Continuous Integration mode
])
if sys.platform == 'win32':
args.append('-n') # Silence alerts under Windows
if not any(is_multiprocess_flag(arg) for arg in regrtest_args):
if cross_compile and hostrunner:
# For now use only two cores for cross-compiled builds;
# hostrunner can be expensive.
args.extend(['-j', '2'])
else:
args.extend(['-j', '0']) # Use all CPU cores
if not any(is_resource_use_flag(arg) for arg in regrtest_args):
args.extend(['-u', 'all,-largefile,-audio,-gui'])
if cross_compile and hostrunner:
# If HOSTRUNNER is set and -p/--python option is not given, then