traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Enhanced Issue 7058 patch, which will not be backported. Refactors the 

code, adds checks for stdin/out/err, cwd, and sys.path, and adds a new
section in the summary for tests that modify the environment (thanks to
Ezio Melotti for that suggestion).
This commit is contained in:
R. David Murray 2009-10-14 13:58:07 +00:00
parent 5c92d4301d
commit 14dcd43d0b
2 changed files with 131 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
Lib/test/regrtest.py
View File

@ -312,7 +312,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
try:
result = runtest(*args, **kwargs)
except BaseException, e:
result = -3, e.__class__.__name__
result = -4, e.__class__.__name__
print # Force a newline (just in case)
print json.dumps(result)
sys.exit(0)
@ -327,6 +327,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
bad = []
skipped = []
resource_denieds = []
environment_changed = []
if findleaks:
try:
@ -395,11 +396,13 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
test_times.append((test_time, test))
if ok > 0:
good.append(test)
elif ok == 0:
elif -2 < ok <= 0:
bad.append(test)
if ok == -1:
environment_changed.append(test)
else:
skipped.append(test)
if ok == -2:
if ok == -3:
resource_denieds.append(test)
if use_mp:
@ -452,7 +455,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
continue
if out:
print out
if result[0] == -3:
if result[0] == -4:
assert result[1] == 'KeyboardInterrupt'
pending.clear()
raise KeyboardInterrupt # What else?
@ -498,6 +501,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
good.sort()
bad.sort()
skipped.sort()
environment_changed.sort()
if good and not quiet:
if not bad and not skipped and len(good) > 1:
@ -508,9 +512,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
print "10 slowest tests:"
for time, test in test_times[:10]:
print "%s: %.1fs" % (test, time)
if bad:
bad = set(bad) - set(environment_changed)
if bad:
print count(len(bad), "test"), "failed:"
printlist(bad)
if environment_changed:
print "{} altered the execution environment:".format(
count(len(environment_changed), "test"))
printlist(environment_changed)
if skipped and not quiet:
print count(len(skipped), "test"), "skipped:"
printlist(skipped)
@ -612,8 +622,10 @@ def runtest(test, verbose, quiet,
huntrleaks -- run multiple times to test for leaks; requires a debug
build; a triple corresponding to -R's three arguments
Return:
-2 test skipped because resource denied
-1 test skipped for some other reason
-4 KeyboardInterrupt when run under -j
-3 test skipped because resource denied
-2 test skipped for some other reason
-1 test failed because it changed the execution environment
0 test failed
1 test passed
"""
@ -627,6 +639,96 @@ def runtest(test, verbose, quiet,
finally:
cleanup_test_droppings(test, verbose)
# Unit tests are supposed to leave the execution environment unchanged
# once they complete. But sometimes tests have bugs, especially when
# tests fail, and the changes to environment go on to mess up other
# tests. This can cause issues with buildbot stability, since tests
# are run in random order and so problems may appear to come and go.
# There are a few things we can save and restore to mitigate this, and
# the following context manager handles this task.
class saved_test_environment:
"""Save bits of the test environment and restore them at block exit.
with saved_test_environment(testname, quiet):
#stuff
Unless quiet is True, a warning is printed to stderr if any of
the saved items was changed by the test. The attribute 'changed'
is initially False, but is set to True if a change is detected.
"""
changed = False
def __init__(self, testname, quiet=False):
self.testname = testname
self.quiet = quiet
# To add things to save and restore, add a name XXX to the resources list
# and add corresponding get_XXX/restore_XXX functions. get_XXX should
# return the value to be saved and compared against a second call to the
# get function when test execution completes. restore_XXX should accept
# the saved value and restore the resource using it. It will be called if
# and only if a change in the value is detected. XXX will have any '_'
# replaced with '.' characters and will then be used in the error messages
# as the name of the resource that changed.
resources = ('sys_argv', 'cwd', 'sys_stdin', 'sys_stdout', 'sys_stderr',
'os_environ', 'sys_path')
def get_sys_argv(self):
return sys.argv[:]
def restore_sys_argv(self, saved_argv):
sys.argv[:] = saved_argv
def get_cwd(self):
return os.getcwd()
def restore_cwd(self, saved_cwd):
os.chdir(saved_cwd)
def get_sys_stdout(self):
return sys.stdout
def restore_sys_stdout(self, saved_stdout):
sys.stdout = saved_stdout
def get_sys_stderr(self):
return sys.stderr
def restore_sys_stderr(self, saved_stderr):
sys.stderr = saved_stderr
def get_sys_stdin(self):
return sys.stdin
def restore_sys_stdin(self, saved_stdin):
sys.stdin = saved_stdin
def get_os_environ(self):
return dict(os.environ)
def restore_os_environ(self, saved_environ):
os.environ.clear()
os.environ.update(saved_environ)
def get_sys_path(self):
return sys.path[:]
def restore_sys_path(self, saved_path):
sys.path[:] = saved_path
def __enter__(self):
self.saved_values = dict((name, getattr(self, 'get_'+name)())
for name in self.resources)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
for name in self.resources:
if not getattr(self, 'get_'+name)() == self.saved_values[name]:
self.changed = True
getattr(self, 'restore_'+name)(self.saved_values[name])
if not self.quiet:
print >>sys.stderr, ("Warning -- {} was modified "
"by {}").format(name.replace('_', '.'), self.testname)
return False
def runtest_inner(test, verbose, quiet,
testdir=None, huntrleaks=False):
test_support.unload(test)
@ -641,10 +743,6 @@ def runtest_inner(test, verbose, quiet,
refleak = False # True if the test leaked references.
try:
save_stdout = sys.stdout
# Save various things that tests may mess up so we can restore
# them afterward.
save_environ = dict(os.environ)
save_argv = sys.argv[:]
try:
if capture_stdout:
sys.stdout = capture_stdout
@ -653,6 +751,7 @@ def runtest_inner(test, verbose, quiet,
else:
# Always import it from the test package
abstest = 'test.' + test
with saved_test_environment(test, quiet) as environment:
start_time = time.time()
the_package = __import__(abstest, globals(), locals(), [])
the_module = getattr(the_package, test)
@ -663,31 +762,21 @@ def runtest_inner(test, verbose, quiet,
if indirect_test is not None:
indirect_test()
if huntrleaks:
refleak = dash_R(the_module, test, indirect_test, huntrleaks)
refleak = dash_R(the_module, test, indirect_test,
huntrleaks)
test_time = time.time() - start_time
finally:
sys.stdout = save_stdout
# Restore what we saved if needed, but also complain if the test
# changed it so that the test may eventually get fixed.
if not os.environ == save_environ:
if not quiet:
print "Warning: os.environ was modified by", test
os.environ.clear()
os.environ.update(save_environ)
if not sys.argv == save_argv:
if not quiet:
print "Warning: argv was modified by", test
sys.argv[:] = save_argv
except test_support.ResourceDenied, msg:
if not quiet:
print test, "skipped --", msg
sys.stdout.flush()
return -2, test_time
return -3, test_time
except unittest.SkipTest, msg:
if not quiet:
print test, "skipped --", msg
sys.stdout.flush()
return -1, test_time
return -2, test_time
except KeyboardInterrupt:
raise
except test_support.TestFailed, msg:
@ -705,6 +794,8 @@ def runtest_inner(test, verbose, quiet,
else:
if refleak:
return 0, test_time
if environment.changed:
return -1, test_time
# Except in verbose mode, tests should not print anything
if verbose or huntrleaks:
return 1, test_time

5
Misc/NEWS
View File

@ -1460,8 +1460,9 @@ Tests
- Issue #7055: test___all__ now greedily detects all modules which have an
__all__ attribute, rather than using a hardcoded and incomplete list.
- Issue #7058: Added save/restore for argv and os.environ to runtest_inner
in regrtest, with warnings if the called test modifies them.
- Issue #7058: Added save/restore for things like sys.argv and cwd to
runtest_inner in regrtest, with warnings if the called test modifies them,
and a new section in the summary report at the end.
- Issue #7042: Fix test_signal (test_itimer_virtual) failure on OS X 10.6.