Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Issue #6152: New option '-j'/'--multiprocess' for regrtest allows running 

regression tests in parallel, shortening the total runtime.
This commit is contained in:
Antoine Pitrou 2009-05-31 14:20:14 +00:00
parent d198b76d36
commit 4698d9928e
4 changed files with 153 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Doc/whatsnew/2.7.rst
View File

@ -654,6 +654,12 @@ Other Changes and Fixes
The :option:`-r` option also now reports the seed that was used The :option:`-r` option also now reports the seed that was used
(Added by Collin Winter.) (Added by Collin Winter.)
* The :file:`regrtest.py` script now takes a :option:`-j` switch
that takes an integer specifying how many tests run in parallel. This
allows to shorten the total runtime on multi-core machines.
This option is compatible with several other options, including the
:option:`-R` switch which is known to produce long runtimes.
(Added by Antoine Pitrou, :issue:`6152`.)
.. ====================================================================== .. ======================================================================

199
Lib/test/regrtest.py
View File

@ -26,6 +26,7 @@ Command line options:
-L: runleaks -- run the leaks(1) command just before exit -L: runleaks -- run the leaks(1) command just before exit
-R: huntrleaks -- search for reference leaks (needs debug build, v. slow) -R: huntrleaks -- search for reference leaks (needs debug build, v. slow)
-M: memlimit -- run very large memory-consuming tests -M: memlimit -- run very large memory-consuming tests
-j: multiprocess -- run several processes at once
If non-option arguments are present, they are names for tests to run, If non-option arguments are present, they are names for tests to run,
unless -x is given, in which case they are names for tests not to run. unless -x is given, in which case they are names for tests not to run.
@ -133,6 +134,7 @@ option '-uall,-bsddb'.
import cStringIO import cStringIO
import getopt import getopt
import json
import os import os
import random import random
import re import re
@ -193,7 +195,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
exclude=False, single=False, randomize=False, fromfile=None, exclude=False, single=False, randomize=False, fromfile=None,
findleaks=False, use_resources=None, trace=False, coverdir='coverage', findleaks=False, use_resources=None, trace=False, coverdir='coverage',
runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
random_seed=None): random_seed=None, use_mp=None):
"""Execute a test suite. """Execute a test suite.
This also parses command-line options and modifies its behavior This also parses command-line options and modifies its behavior
@ -218,13 +220,13 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
test_support.record_original_stdout(sys.stdout) test_support.record_original_stdout(sys.stdout)
try: try:
opts, args = getopt.getopt(sys.argv[1:], 'hvgqxsSrf:lu:t:TD:NLR:wM:', opts, args = getopt.getopt(sys.argv[1:], 'hvgqxsSrf:lu:t:TD:NLR:wM:j:',
['help', 'verbose', 'quiet', 'exclude', ['help', 'verbose', 'quiet', 'exclude',
'single', 'slow', 'random', 'fromfile', 'single', 'slow', 'random', 'fromfile',
'findleaks', 'use=', 'threshold=', 'trace', 'findleaks', 'use=', 'threshold=', 'trace',
'coverdir=', 'nocoverdir', 'runleaks', 'coverdir=', 'nocoverdir', 'runleaks',
'huntrleaks=', 'verbose2', 'memlimit=', 'huntrleaks=', 'verbose2', 'memlimit=',
'randseed=' 'randseed=', 'multiprocess=', 'slaveargs=',
]) ])
except getopt.error, msg: except getopt.error, msg:
usage(2, msg) usage(2, msg)
@ -303,8 +305,23 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
use_resources.remove(r) use_resources.remove(r)
elif r not in use_resources: elif r not in use_resources:
use_resources.append(r) use_resources.append(r)
elif o in ('-j', '--multiprocess'):
use_mp = int(a)
elif o == '--slaveargs':
args, kwargs = json.loads(a)
try:
result = runtest(*args, **kwargs)
except BaseException, e:
result = -3, e.__class__.__name__
print # Force a newline (just in case)
print json.dumps(result)
sys.exit(0)
if single and fromfile: if single and fromfile:
usage(2, "-s and -f don't go together!") usage(2, "-s and -f don't go together!")
if use_mp and trace:
usage(2, "-T and -j don't go together!")
if use_mp and findleaks:
usage(2, "-l and -j don't go together!")
good = [] good = []
bad = [] bad = []
@ -370,50 +387,111 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix], tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix],
trace=False, count=True) trace=False, count=True)
test_times = [] test_times = []
test_support.verbose = verbose # Tell tests to be moderately quiet
test_support.use_resources = use_resources test_support.use_resources = use_resources
save_modules = sys.modules.keys() save_modules = sys.modules.keys()
for test in tests:
if not quiet: def accumulate_result(test, result):
print test ok, test_time = result
sys.stdout.flush() test_times.append((test_time, test))
if trace: if ok > 0:
# If we're tracing code coverage, then we don't exit with status good.append(test)
# if on a false return value from main. elif ok == 0:
tracer.runctx('runtest(test, verbose, quiet,' bad.append(test)
' test_times, testdir)',
globals=globals(), locals=vars())
else: else:
skipped.append(test)
if ok == -2:
resource_denieds.append(test)
if use_mp:
from threading import Thread
from Queue import Queue, Empty
from subprocess import Popen, PIPE, STDOUT
from collections import deque
debug_output_pat = re.compile(r"\[\d+ refs\]$")
pending = deque()
output = Queue()
for test in tests:
args_tuple = (
(test, verbose, quiet, testdir),
dict(huntrleaks=huntrleaks, use_resources=use_resources)
)
pending.append((test, args_tuple))
def work():
# A worker thread.
try: try:
ok = runtest(test, verbose, quiet, test_times, while True:
testdir, huntrleaks) try:
except KeyboardInterrupt: test, args_tuple = pending.popleft()
# print a newline separate from the ^C except IndexError:
print output.put((None, None, None))
break return
except: if not quiet:
print test
sys.stdout.flush()
popen = Popen([sys.executable, '-m', 'test.regrtest',
'--slaveargs', json.dumps(args_tuple)],
stdout=PIPE, stderr=STDOUT,
universal_newlines=True, close_fds=True)
out = popen.communicate()[0].strip()
out = debug_output_pat.sub("", out)
out, _, result = out.strip().rpartition("\n")
result = json.loads(result)
output.put((test, out.strip(), result))
except BaseException:
output.put((None, None, None))
raise raise
if ok > 0: workers = [Thread(target=work) for i in range(use_mp)]
good.append(test) for worker in workers:
elif ok == 0: worker.start()
bad.append(test) finished = 0
while finished < use_mp:
test, out, result = output.get()
if test is None:
finished += 1
continue
if out:
print out
if result[0] == -3:
assert result[1] == 'KeyboardInterrupt'
pending.clear()
raise KeyboardInterrupt # What else?
accumulate_result(test, result)
for worker in workers:
worker.join()
else:
for test in tests:
if not quiet:
print test
sys.stdout.flush()
if trace:
# If we're tracing code coverage, then we don't exit with status
# if on a false return value from main.
tracer.runctx('runtest(test, verbose, quiet, testdir)',
globals=globals(), locals=vars())
else: else:
skipped.append(test) try:
if ok == -2: result = runtest(test, verbose, quiet,
resource_denieds.append(test) testdir, huntrleaks)
if findleaks: accumulate_result(test, result)
gc.collect() except KeyboardInterrupt:
if gc.garbage: # print a newline separate from the ^C
print "Warning: test created", len(gc.garbage), print
print "uncollectable object(s)." break
# move the uncollectable objects somewhere so we don't see except:
# them again raise
found_garbage.extend(gc.garbage) if findleaks:
del gc.garbage[:] gc.collect()
# Unload the newly imported modules (best effort finalization) if gc.garbage:
for module in sys.modules.keys(): print "Warning: test created", len(gc.garbage),
if module not in save_modules and module.startswith("test."): print "uncollectable object(s)."
test_support.unload(module) # move the uncollectable objects somewhere so we don't see
# them again
found_garbage.extend(gc.garbage)
del gc.garbage[:]
# Unload the newly imported modules (best effort finalization)
for module in sys.modules.keys():
if module not in save_modules and module.startswith("test."):
test_support.unload(module)
# The lists won't be sorted if running with -r # The lists won't be sorted if running with -r
good.sort() good.sort()
@ -457,7 +535,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
sys.stdout.flush() sys.stdout.flush()
try: try:
test_support.verbose = True test_support.verbose = True
ok = runtest(test, True, quiet, test_times, testdir, ok = runtest(test, True, quiet, testdir,
huntrleaks) huntrleaks)
except KeyboardInterrupt: except KeyboardInterrupt:
# print a newline separate from the ^C # print a newline separate from the ^C
@ -521,8 +599,8 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
tests.sort() tests.sort()
return stdtests + tests return stdtests + tests
def runtest(test, verbose, quiet, test_times, def runtest(test, verbose, quiet,
testdir=None, huntrleaks=False): testdir=None, huntrleaks=False, use_resources=None):
"""Run a single test. """Run a single test.
test -- the name of the test test -- the name of the test
@ -539,13 +617,16 @@ def runtest(test, verbose, quiet, test_times,
1 test passed 1 test passed
""" """
test_support.verbose = verbose # Tell tests to be moderately quiet
if use_resources is not None:
test_support.use_resources = use_resources
try: try:
return runtest_inner(test, verbose, quiet, test_times, return runtest_inner(test, verbose, quiet,
testdir, huntrleaks) testdir, huntrleaks)
finally: finally:
cleanup_test_droppings(test, verbose) cleanup_test_droppings(test, verbose)
def runtest_inner(test, verbose, quiet, test_times, def runtest_inner(test, verbose, quiet,
testdir=None, huntrleaks=False): testdir=None, huntrleaks=False):
test_support.unload(test) test_support.unload(test)
if not testdir: if not testdir:
@ -555,6 +636,7 @@ def runtest_inner(test, verbose, quiet, test_times,
else: else:
capture_stdout = cStringIO.StringIO() capture_stdout = cStringIO.StringIO()
test_time = 0.0
refleak = False # True if the test leaked references. refleak = False # True if the test leaked references.
try: try:
save_stdout = sys.stdout save_stdout = sys.stdout
@ -578,25 +660,24 @@ def runtest_inner(test, verbose, quiet, test_times,
if huntrleaks: if huntrleaks:
refleak = dash_R(the_module, test, indirect_test, huntrleaks) refleak = dash_R(the_module, test, indirect_test, huntrleaks)
test_time = time.time() - start_time test_time = time.time() - start_time
test_times.append((test_time, test))
finally: finally:
sys.stdout = save_stdout sys.stdout = save_stdout
except test_support.ResourceDenied, msg: except test_support.ResourceDenied, msg:
if not quiet: if not quiet:
print test, "skipped --", msg print test, "skipped --", msg
sys.stdout.flush() sys.stdout.flush()
return -2 return -2, test_time
except unittest.SkipTest, msg: except unittest.SkipTest, msg:
if not quiet: if not quiet:
print test, "skipped --", msg print test, "skipped --", msg
sys.stdout.flush() sys.stdout.flush()
return -1 return -1, test_time
except KeyboardInterrupt: except KeyboardInterrupt:
raise raise
except test_support.TestFailed, msg: except test_support.TestFailed, msg:
print "test", test, "failed --", msg print "test", test, "failed --", msg
sys.stdout.flush() sys.stdout.flush()
return 0 return 0, test_time
except: except:
type, value = sys.exc_info()[:2] type, value = sys.exc_info()[:2]
print "test", test, "crashed --", str(type) + ":", value print "test", test, "crashed --", str(type) + ":", value
@ -604,22 +685,22 @@ def runtest_inner(test, verbose, quiet, test_times,
if verbose: if verbose:
traceback.print_exc(file=sys.stdout) traceback.print_exc(file=sys.stdout)
sys.stdout.flush() sys.stdout.flush()
return 0 return 0, test_time
else: else:
if refleak: if refleak:
return 0 return 0, test_time
# Except in verbose mode, tests should not print anything # Except in verbose mode, tests should not print anything
if verbose or huntrleaks: if verbose or huntrleaks:
return 1 return 1, test_time
output = capture_stdout.getvalue() output = capture_stdout.getvalue()
if not output: if not output:
return 1 return 1, test_time
print "test", test, "produced unexpected output:" print "test", test, "produced unexpected output:"
print "*" * 70 print "*" * 70
print output print output
print "*" * 70 print "*" * 70
sys.stdout.flush() sys.stdout.flush()
return 0 return 0, test_time
def cleanup_test_droppings(testname, verbose): def cleanup_test_droppings(testname, verbose):
import shutil import shutil
@ -707,9 +788,9 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
if any(deltas): if any(deltas):
msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas)) msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas))
print >> sys.stderr, msg print >> sys.stderr, msg
refrep = open(fname, "a") with open(fname, "a") as refrep:
print >> refrep, msg print >> refrep, msg
refrep.close() refrep.flush()
return True return True
return False return False
@ -1227,6 +1308,6 @@ if __name__ == '__main__':
i -= 1 i -= 1
if os.path.abspath(os.path.normpath(sys.path[i])) == mydir: if os.path.abspath(os.path.normpath(sys.path[i])) == mydir:
del sys.path[i] del sys.path[i]
if len(sys.path) == pathlen: if '--slaveargs' not in sys.argv and len(sys.path) == pathlen:
print 'Could not find %r in sys.path to remove it' % mydir print 'Could not find %r in sys.path to remove it' % mydir
main() main()

4
Lib/test/test_support.py
View File

@ -378,6 +378,10 @@ else:
'Unicode filename tests may not be effective' \ 'Unicode filename tests may not be effective' \
% TESTFN_UNICODE_UNENCODEABLE % TESTFN_UNICODE_UNENCODEABLE
# Disambiguate TESTFN for parallel testing, while letting it remain a valid
# module name.
TESTFN = "{0}_{1}_tmp".format(TESTFN, os.getpid())
# Make sure we can write to TESTFN, try in /tmp if we can't # Make sure we can write to TESTFN, try in /tmp if we can't
fp = None fp = None
try: try:

3
Misc/NEWS
View File

@ -1104,6 +1104,9 @@ Extension Modules
Tests Tests
----- -----
- Issue #6152: New option '-j'/'--multiprocess' for regrtest allows running
regression tests in parallel, shortening the total runtime.
- Issue #5354: New test support function import_fresh_module() makes - Issue #5354: New test support function import_fresh_module() makes
it easy to import both normal and optimised versions of modules. it easy to import both normal and optimised versions of modules.
test_heapq and test_warnings have been adjusted to use it, tests for test_heapq and test_warnings have been adjusted to use it, tests for