From 3c01d16ed91fa77a8e70d9fa3a6c33d6290fd9c4 Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Mon, 1 Aug 2011 23:48:26 +0200 Subject: [PATCH] Issue #11651: Move options for running tests into a Python script. This will be particularly useful to Windows users. run_tests.py originally written by Brett Cannon. --- Lib/test/regrtest.py | 7 ++++++ Makefile.pre.in | 32 +++++++++++++-------------- Misc/NEWS | 7 ++++++ Tools/scripts/run_tests.py | 45 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 16 deletions(-) create mode 100755 Tools/scripts/run_tests.py diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index ef8ea6d918d..77c089c348d 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -133,6 +133,8 @@ resources to test. Currently only the following are defined: all - Enable all special resources. + none - Disable all special resources (this is the default). + audio - Tests that use the audio device. (There are known cases of broken audio drivers that can crash Python or even the Linux kernel.) @@ -387,6 +389,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if r == 'all': use_resources[:] = RESOURCE_NAMES continue + if r == 'none': + del use_resources[:] + continue remove = False if r[0] == '-': remove = True @@ -424,6 +429,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, use_mp = 2 + multiprocessing.cpu_count() except (ImportError, NotImplementedError): use_mp = 3 + if use_mp == 1: + use_mp = None elif o == '--header': header = True elif o == '--slaveargs': diff --git a/Makefile.pre.in b/Makefile.pre.in index 0c7b180d904..c97c7ca5605 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -747,14 +747,15 @@ $(LIBRARY_OBJS) $(MODOBJS) Modules/python.o: $(PYTHON_HEADERS) ###################################################################### +TESTOPTS= $(EXTRATESTOPTS) +TESTPYTHON= $(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS) +TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py +TESTTIMEOUT= 3600 + # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. -TESTOPTS= $(EXTRATESTOPTS) -TESTPROG= $(srcdir)/Lib/test/regrtest.py -TESTPYTHON= $(RUNSHARED) ./$(BUILDPYTHON) -Wd -E -bb $(TESTPYTHONOPTS) -TESTTIMEOUT= 3600 test: all platform - $(TESTPYTHON) $(TESTPROG) -j0 $(TESTOPTS) + $(TESTRUNNER) $(TESTOPTS) # Run the full test suite twice - once without .pyc files, and once with. # In the past, we've had problems where bugs in the marshalling or @@ -765,10 +766,10 @@ test: all platform # sample data. testall: all platform -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - $(TESTPYTHON) $(srcdir)/Lib/compileall.py + $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - -$(TESTPYTHON) $(TESTPROG) -j0 -uall $(TESTOPTS) - $(TESTPYTHON) $(TESTPROG) -j0 -uall $(TESTOPTS) + -$(TESTRUNNER) -u all $(TESTOPTS) + $(TESTRUNNER) -u all $(TESTOPTS) # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. @@ -777,25 +778,24 @@ testuniversal: all platform echo "This can only be used on OSX/i386" ;\ exit 1 ;\ fi - $(TESTPYTHON) $(TESTPROG) -j0 -uall $(TESTOPTS) - $(RUNSHARED) /usr/libexec/oah/translate ./$(BUILDPYTHON) -E $(TESTPROG) -j0 -uall $(TESTOPTS) + $(TESTRUNNER) -u all $(TESTOPTS) + $(RUNSHARED) /usr/libexec/oah/translate \ + ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS) -# Like testall, but with only one pass. +# Like testall, but with only one pass and without multiple processes. # Run an optional script to include information about the build environment. buildbottest: all platform -@if which pybuildbot.identify >/dev/null 2>&1; then \ pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \ fi - $(TESTPYTHON) $(TESTPROG) -uall -rwW --timeout=$(TESTTIMEOUT) $(TESTOPTS) + $(TESTRUNNER) -j 1 -u all -W --timeout=$(TESTTIMEOUT) $(TESTOPTS) QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multibytecodec test_urllib2_localnet test_itertools \ test_multiprocessing test_mailbox test_socket test_poll \ - test_select test_zipfile + test_select test_zipfile test_concurrent_futures quicktest: all platform - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - -$(TESTPYTHON) $(TESTPROG) -j0 $(QUICKTESTOPTS) - $(TESTPYTHON) $(TESTPROG) -j0 $(QUICKTESTOPTS) + $(TESTRUNNER) $(QUICKTESTOPTS) install: altinstall bininstall diff --git a/Misc/NEWS b/Misc/NEWS index afe4894d29c..8000ca0ac8d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1147,6 +1147,13 @@ Extension Modules Tests ----- +- Issue #11651: Improve the Makefile test targets to run more of the test suite + more quickly. The --multiprocess option is now enabled by default, reducing + the amount of time needed to run the tests. "make test" and "make quicktest" + now include some resource-intensive tests, but no longer run the test suite + twice to check for bugs in .pyc generation. Tools/scripts/run_test.py provides + as an easy platform-independent way to run test suite with sensible defaults. + - Issue #12331: The test suite for the packaging module can now run from an installed Python. diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py new file mode 100755 index 00000000000..543afe7a067 --- /dev/null +++ b/Tools/scripts/run_tests.py @@ -0,0 +1,45 @@ +"""Run Python's test suite in a fast, rigorous way. + +The defaults are meant to be thorough but to skip certain resources are not +used (by default) which can consume a lot of time and resources (e.g., +largefile) or can be distracting (e.g., audio and gui). These defaults +can be overridden by simply passing a -u option to this script. + +""" + +import os +import sys +import test.support + + +def is_multiprocess_flag(arg): + return arg.startswith('-j') or arg.startswith('--multiprocess') + + +def is_resource_use_flag(arg): + return arg.startswith('-u') or arg.startswith('--use') + + +def main(regrtest_args): + args = [sys.executable, + '-W', 'default', # Warnings set to 'default' + '-bb', # Warnings about bytes/bytearray + '-E', # Ignore environment variables + ] + # Allow user-specified interpreter options to override our defaults. + args.extend(test.support.args_from_interpreter_flags()) + args.extend(['-m', 'test', # Run the test suite + '-r', # Randomize test order + '-w', # Re-run failed tests in verbose mode + ]) + if not any(is_multiprocess_flag(arg) for arg in regrtest_args): + args.extend(['-j', '0']) # Use all CPU cores + if not any(is_resource_use_flag(arg) for arg in regrtest_args): + args.extend(['-u', 'all,-largefile,-audio,-gui']) + args.extend(regrtest_args) + print(' '.join(args)) + os.execv(sys.executable, args) + + +if __name__ == '__main__': + main(sys.argv[1:])