Issue #11651: Move options for running tests into a Python script.

This will be particularly useful to Windows users.

run_tests.py originally written by Brett Cannon.

Lib/test/regrtest.py

@@ -133,6 +133,8 @@ resources to test.  Currently only the following are defined:
 
     all -       Enable all special resources.
 
+    none -      Disable all special resources (this is the default).
+
     audio -     Tests that use the audio device.  (There are known
                 cases of broken audio drivers that can crash Python or
                 even the Linux kernel.)
@@ -387,6 +389,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
                 if r == 'all':
                     use_resources[:] = RESOURCE_NAMES
                     continue
+                if r == 'none':
+                    del use_resources[:]
+                    continue
                 remove = False
                 if r[0] == '-':
                     remove = True
@@ -424,6 +429,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
                     use_mp = 2 + multiprocessing.cpu_count()
                 except (ImportError, NotImplementedError):
                     use_mp = 3
+            if use_mp == 1:
+                use_mp = None
         elif o == '--header':
             header = True
         elif o == '--slaveargs':

Makefile.pre.in

@@ -747,14 +747,15 @@ $(LIBRARY_OBJS) $(MODOBJS) Modules/python.o: $(PYTHON_HEADERS)
 
 ######################################################################
 
+TESTOPTS=	$(EXTRATESTOPTS)
+TESTPYTHON=	$(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS)
+TESTRUNNER=	$(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py
+TESTTIMEOUT=	3600
+
 # Run a basic set of regression tests.
 # This excludes some tests that are particularly resource-intensive.
-TESTOPTS=	$(EXTRATESTOPTS)
-TESTPROG=	$(srcdir)/Lib/test/regrtest.py
-TESTPYTHON=	$(RUNSHARED) ./$(BUILDPYTHON) -Wd -E -bb $(TESTPYTHONOPTS)
-TESTTIMEOUT=	3600
 test:		all platform
-		$(TESTPYTHON) $(TESTPROG) -j0 $(TESTOPTS)
+		$(TESTRUNNER) $(TESTOPTS)
 
 # Run the full test suite twice - once without .pyc files, and once with.
 # In the past, we've had problems where bugs in the marshalling or
@@ -765,10 +766,10 @@ test:		all platform
 # sample data.
 testall:	all platform
 		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-		$(TESTPYTHON) $(srcdir)/Lib/compileall.py
+		$(TESTPYTHON) -E $(srcdir)/Lib/compileall.py
 		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-		-$(TESTPYTHON) $(TESTPROG) -j0 -uall $(TESTOPTS)
+		-$(TESTRUNNER) -u all $(TESTOPTS)
-		$(TESTPYTHON) $(TESTPROG) -j0 -uall $(TESTOPTS)
+		$(TESTRUNNER) -u all $(TESTOPTS)
 
 # Run the test suite for both architectures in a Universal build on OSX.
 # Must be run on an Intel box.
@@ -777,25 +778,24 @@ testuniversal:	all platform
 			echo "This can only be used on OSX/i386" ;\
 			exit 1 ;\
 		fi
-		$(TESTPYTHON) $(TESTPROG) -j0 -uall $(TESTOPTS)
+		$(TESTRUNNER) -u all $(TESTOPTS)
-		$(RUNSHARED) /usr/libexec/oah/translate ./$(BUILDPYTHON) -E $(TESTPROG) -j0 -uall $(TESTOPTS)
+		$(RUNSHARED) /usr/libexec/oah/translate \
+			./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS)
 
-# Like testall, but with only one pass.
+# Like testall, but with only one pass and without multiple processes.
 # Run an optional script to include information about the build environment.
 buildbottest:	all platform
 		-@if which pybuildbot.identify >/dev/null 2>&1; then \
 			pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
 		fi
-		$(TESTPYTHON) $(TESTPROG) -uall -rwW --timeout=$(TESTTIMEOUT) $(TESTOPTS)
+		$(TESTRUNNER) -j 1 -u all -W --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
 QUICKTESTOPTS=	$(TESTOPTS) -x test_subprocess test_io test_lib2to3 \
 		test_multibytecodec test_urllib2_localnet test_itertools \
 		test_multiprocessing test_mailbox test_socket test_poll \
-		test_select test_zipfile
+		test_select test_zipfile test_concurrent_futures
 quicktest:	all platform
-		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
+		$(TESTRUNNER) $(QUICKTESTOPTS)
-		-$(TESTPYTHON) $(TESTPROG) -j0 $(QUICKTESTOPTS)
-		$(TESTPYTHON) $(TESTPROG) -j0 $(QUICKTESTOPTS)
 
 
 install: altinstall bininstall

Misc/NEWS

@@ -1147,6 +1147,13 @@ Extension Modules
 Tests
 -----
 
+- Issue #11651: Improve the Makefile test targets to run more of the test suite
+  more quickly. The --multiprocess option is now enabled by default, reducing
+  the amount of time needed to run the tests. "make test" and "make quicktest"
+  now include some resource-intensive tests, but no longer run the test suite
+  twice to check for bugs in .pyc generation. Tools/scripts/run_test.py provides
+  as an easy platform-independent way to run test suite with sensible defaults.
+
 - Issue #12331: The test suite for the packaging module can now run from an
   installed Python.
 

Tools/scripts/run_tests.py

@@ -0,0 +1,45 @@
+"""Run Python's test suite in a fast, rigorous way.
+
+The defaults are meant to be thorough but to skip certain resources are not
+used (by default) which can consume a lot of time and resources (e.g.,
+largefile) or can be distracting (e.g., audio and gui). These defaults
+can be overridden by simply passing a -u option to this script.
+
+"""
+
+import os
+import sys
+import test.support
+
+
+def is_multiprocess_flag(arg):
+    return arg.startswith('-j') or arg.startswith('--multiprocess')
+
+
+def is_resource_use_flag(arg):
+    return arg.startswith('-u') or arg.startswith('--use')
+
+
+def main(regrtest_args):
+    args = [sys.executable,
+            '-W', 'default',      # Warnings set to 'default'
+            '-bb',                # Warnings about bytes/bytearray
+            '-E',                 # Ignore environment variables
+            ]
+    # Allow user-specified interpreter options to override our defaults.
+    args.extend(test.support.args_from_interpreter_flags())
+    args.extend(['-m', 'test',    # Run the test suite
+                 '-r',            # Randomize test order
+                 '-w',            # Re-run failed tests in verbose mode
+                 ])
+    if not any(is_multiprocess_flag(arg) for arg in regrtest_args):
+        args.extend(['-j', '0'])  # Use all CPU cores
+    if not any(is_resource_use_flag(arg) for arg in regrtest_args):
+        args.extend(['-u', 'all,-largefile,-audio,-gui'])
+    args.extend(regrtest_args)
+    print(' '.join(args))
+    os.execv(sys.executable, args)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])