From a0a62225094a890c1b07ebc4f89e4c7c45c66b13 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sun, 9 Sep 2001 06:12:01 +0000 Subject: [PATCH] Teach regrtest how to pass on doctest failure msgs. This is done via a horridly inefficient hack in regrtest's Compare class, but it's about as clean as can be: regrtest has to set up the Compare instance before importing a test module, and by the time the module *is* imported it's too late to change that decision. The good news is that the more tests we convert to unittest and doctest, the less the inefficiency here matters. Even now there are few tests with large expected-output files (the new cost here is a Python-level call per .write() when there's an expected- output file). --- Lib/test/README | 28 ++++++++++++++++++---- Lib/test/regrtest.py | 8 +++++-- Lib/test/test_cookie.py | 6 ++--- Lib/test/test_descrtut.py | 13 ++++++---- Lib/test/test_difflib.py | 5 ++-- Lib/test/test_doctest.py | 5 ++-- Lib/test/test_generators.py | 10 ++++---- Lib/test/test_support.py | 47 ++++++++++++++++++++++++++++++++++++- 8 files changed, 95 insertions(+), 27 deletions(-) diff --git a/Lib/test/README b/Lib/test/README index 0a663e0f16e..485095764ab 100644 --- a/Lib/test/README +++ b/Lib/test/README @@ -55,14 +55,32 @@ The module in the test package is simply a wrapper that causes doctest to run over the tests in the module. The test for the difflib module provides a convenient example: - from test_support import verbose - import doctest, difflib - doctest.testmod(difflib, verbose=verbose) + import difflib, test_support + test_support.run_doctest(difflib) If the test is successful, nothing is written to stdout (so you should not create a corresponding output/test_difflib file), but running regrtest -with -v will give a detailed report, the same as if passing -v to doctest -(that's what importing verbose from test_support accomplishes). +with -v will give a detailed report, the same as if passing -v to doctest. + +A second argument can be passed to run_doctest to tell doctest to search +sys.argv for -v instead of using test_support's idea of verbosity. This +is useful for writing doctest-based tests that aren't simply running a +doctest'ed Lib module, but contain the doctests themselves. Then at +times you may want to run such a test directly as a doctest, independent +of the regrtest framework. The tail end of test_descrtut.py is a good +example: + + def test_main(verbose=None): + import test_support, test.test_descrtut + test_support.run_doctest(test.test_descrtut, verbose) + + if __name__ == "__main__": + test_main(1) + +If run via regrtest, test_main() is called (by regrtest) without specifying +verbose, and then test_supprot's idea of verbosity is used. But when +run directly, test_main(1) is called, and then doctest's idea of verbosity +is used. See the documentation for the doctest module for information on writing tests using the doctest framework. diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 2c09f63fdf4..16d51ea913a 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -288,7 +288,7 @@ def runtest(test, generate, verbose, quiet, testdir = None): elif verbose: cfp = sys.stdout else: - cfp = Compare(outputfile) + cfp = Compare(outputfile, sys.stdout) except IOError: cfp = None print "Warning: can't open", outputfile @@ -386,7 +386,8 @@ def printlist(x, width=70, indent=4): print line class Compare: - def __init__(self, filename): + def __init__(self, filename, origstdout): + self.origstdout = origstdout if os.path.exists(filename): self.fp = open(filename, 'r') else: @@ -395,6 +396,9 @@ class Compare: self.stuffthatmatched = [] def write(self, data): + if test_support.suppress_output_comparison(): + self.origstdout.write(data) + return expected = self.fp.read(len(data)) if data == expected: self.stuffthatmatched.append(expected) diff --git a/Lib/test/test_cookie.py b/Lib/test/test_cookie.py index 40c881ff167..7d452f5e0c2 100644 --- a/Lib/test/test_cookie.py +++ b/Lib/test/test_cookie.py @@ -1,9 +1,7 @@ # Simple test suite for Cookie.py -from test_support import verify +from test_support import verify, verbose, run_doctest import Cookie -from test_support import verify, verbose -import doctest # Currently this only tests SimpleCookie @@ -46,4 +44,4 @@ verify(C['Customer']['version'] == '1') verify(C['Customer']['path'] == '/acme') print "If anything blows up after this line, it's from Cookie's doctest." -doctest.testmod(Cookie) +run_doctest(Cookie) diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 1f8f5c8802a..a0fe0f1b495 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -484,10 +484,15 @@ __test__ = {"tut1": test_1, # This worms around a bootstrap problem. # Note that doctest and regrtest both look in sys.argv for a "-v" argument, # so this works as expected in both ways of running regrtest. -def test_main(): - import doctest, test.test_descrtut - doctest.testmod(test.test_descrtut) +def test_main(verbose=None): + # Obscure: import this module as test.test_descrtut instead of as + # plain test_descrtut because the name of this module works its way + # into the doctest examples, and unless the full test.test_descrtut + # business is used the name can change depending on how the test is + # invoked. + import test_support, test.test_descrtut + test_support.run_doctest(test.test_descrtut, verbose) # This part isn't needed for regrtest, but for running the test directly. if __name__ == "__main__": - test_main() + test_main(1) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 3b2f8213bc1..9c63d9ad1d4 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -1,3 +1,2 @@ -from test_support import verbose -import doctest, difflib -doctest.testmod(difflib, verbose=verbose) +import difflib, test_support +test_support.run_doctest(difflib) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 00c09284462..82ab3980b07 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -1,3 +1,2 @@ -from test_support import verbose -import doctest -doctest.testmod(doctest, verbose=verbose) +import doctest, test_support +test_support.run_doctest(doctest) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 0e9d0607399..118b1d9fce1 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1351,16 +1351,16 @@ __test__ = {"tut": tutorial_tests, # This worms around a bootstrap problem. # Note that doctest and regrtest both look in sys.argv for a "-v" argument, # so this works as expected in both ways of running regrtest. -def test_main(): - import doctest, test_generators +def test_main(verbose=None): + import doctest, test_support, test_generators if 0: # change to 1 to run forever (to check for leaks) while 1: doctest.master = None - doctest.testmod(test_generators) + test_support.run_doctest(test_generators, verbose) print ".", else: - doctest.testmod(test_generators) + test_support.run_doctest(test_generators, verbose) # This part isn't needed for regrtest, but for running the test directly. if __name__ == "__main__": - test_main() + test_main(1) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 0e87131cfbe..53e8ae76e6f 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -2,7 +2,6 @@ import sys - class Error(Exception): """Base class for regression test exceptions.""" @@ -22,6 +21,26 @@ class TestSkipped(Error): verbose = 1 # Flag set to 0 by regrtest.py use_resources = None # Flag set to [] by regrtest.py +# _output_comparison controls whether regrtest will try to compare stdout +# with an expected-output file. For straight regrtests, it should. +# The doctest driver should set_output_comparison(0) for the duration, and +# restore the old value when it's done. +# Note that this control is in addition to verbose mode: output will be +# compared if and only if _output_comparison is true and verbose mode is +# not in effect. +_output_comparison = 1 + +def set_output_comparison(newvalue): + global _output_comparison + oldvalue = _output_comparison + _output_comparison = newvalue + return oldvalue + +# regrtest's interface to _output_comparison. +def suppress_output_comparison(): + return not _output_comparison + + def unload(name): try: del sys.modules[name] @@ -156,3 +175,29 @@ def run_unittest(testclass): raise TestFailed("errors occurred in %s.%s" % (testclass.__module__, testclass.__name__)) raise TestFailed(err) + +#======================================================================= +# doctest driver. + +def run_doctest(module, verbosity=None): + """Run doctest on the given module. + + If optional argument verbosity is not specified (or is None), pass + test_support's belief about verbosity on to doctest. Else doctest + sys.argv for -v. + """ + + import doctest + + if verbosity is None: + verbosity = verbose + else: + verbosity = None + + oldvalue = set_output_comparison(0) + try: + f, t = doctest.testmod(module, verbose=verbosity) + if f: + raise TestFailed("%d of %d doctests failed" % (f, t)) + finally: + set_output_comparison(oldvalue)