From d7955b8196578306e9d86f6c61c9cb3ee72edab0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jul 2017 15:07:53 +0200 Subject: [PATCH] [2.7] bpo-29512, bpo-30764: Backport regrtest enhancements from 3.5 to 2.7 (#2541) * bpo-29512, bpo-30764: Backport regrtest enhancements from 3.5 to 2.7 * bpo-29512: Add test.bisect, bisect failing tests (#2452) Add a new "python3 -m test.bisect" tool to bisect failing tests. It can be used to find which test method(s) leak references, leak files, etc. * bpo-30764: Fix regrtest --fail-env-changed --forever (#2536) (#2539) --forever now stops if a fail changes the environment. * Fix test_bisect: use absolute import --- Lib/test/bisect.py | 181 ++++++++++++++++++++++++++++++++++++++++ Lib/test/regrtest.py | 4 +- Lib/test/test_bisect.py | 4 + 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100755 Lib/test/bisect.py diff --git a/Lib/test/bisect.py b/Lib/test/bisect.py new file mode 100755 index 00000000000..6fc56189026 --- /dev/null +++ b/Lib/test/bisect.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +""" +Command line tool to bisect failing CPython tests. + +Find the test_os test method which alters the environment: + + ./python -m test.bisect --fail-env-changed test_os + +Find a reference leak in "test_os", write the list of failing tests into the +"bisect" file: + + ./python -m test.bisect -o bisect -R 3:3 test_os + +Load an existing list of tests from a file using -i option: + + ./python -m test --list-cases -m FileTests test_os > tests + ./python -m test.bisect -i tests test_os +""" +from __future__ import print_function + +import argparse +import datetime +import os.path +import math +import random +import subprocess +import sys +import tempfile +import time + + +def write_tests(filename, tests): + with open(filename, "w") as fp: + for name in tests: + print(name, file=fp) + fp.flush() + + +def write_output(filename, tests): + if not filename: + return + print("Write %s tests into %s" % (len(tests), filename)) + write_tests(filename, tests) + return filename + + +def format_shell_args(args): + return ' '.join(args) + + +def list_cases(args): + cmd = [sys.executable, '-m', 'test', '--list-cases'] + cmd.extend(args.test_args) + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + universal_newlines=True) + try: + stdout = proc.communicate()[0] + except: + proc.stdout.close() + proc.kill() + proc.wait() + raise + exitcode = proc.wait() + if exitcode: + cmd = format_shell_args(cmd) + print("Failed to list tests: %s failed with exit code %s" + % (cmd, exitcode)) + sys.exit(exitcode) + tests = stdout.splitlines() + return tests + + +def run_tests(args, tests, huntrleaks=None): + tmp = tempfile.mktemp() + try: + write_tests(tmp, tests) + + cmd = [sys.executable, '-m', 'test', '--matchfile', tmp] + cmd.extend(args.test_args) + print("+ %s" % format_shell_args(cmd)) + proc = subprocess.Popen(cmd) + try: + exitcode = proc.wait() + except: + proc.kill() + proc.wait() + raise + return exitcode + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input', + help='Test names produced by --list-tests written ' + 'into a file. If not set, run --list-tests') + parser.add_argument('-o', '--output', + help='Result of the bisection') + parser.add_argument('-n', '--max-tests', type=int, default=1, + help='Maximum number of tests to stop the bisection ' + '(default: 1)') + parser.add_argument('-N', '--max-iter', type=int, default=100, + help='Maximum number of bisection iterations ' + '(default: 100)') + # FIXME: document that following arguments are test arguments + + args, test_args = parser.parse_known_args() + args.test_args = test_args + return args + + +def main(): + args = parse_args() + + if args.input: + with open(args.input) as fp: + tests = [line.strip() for line in fp] + else: + tests = list_cases(args) + + print("Start bisection with %s tests" % len(tests)) + print("Test arguments: %s" % format_shell_args(args.test_args)) + print("Bisection will stop when getting %s or less tests " + "(-n/--max-tests option), or after %s iterations " + "(-N/--max-iter option)" + % (args.max_tests, args.max_iter)) + output = write_output(args.output, tests) + print() + + start_time = time.time() + iteration = 1 + try: + while len(tests) > args.max_tests and iteration <= args.max_iter: + ntest = len(tests) + ntest = max(ntest // 2, 1) + subtests = random.sample(tests, ntest) + + print("[+] Iteration %s: run %s tests/%s" + % (iteration, len(subtests), len(tests))) + print() + + exitcode = run_tests(args, subtests) + + print("ran %s tests/%s" % (ntest, len(tests))) + print("exit", exitcode) + if exitcode: + print("Tests failed: use this new subtest") + tests = subtests + output = write_output(args.output, tests) + else: + print("Tests succeeded: skip this subtest, try a new subbset") + print() + iteration += 1 + except KeyboardInterrupt: + print() + print("Bisection interrupted!") + print() + + print("Tests (%s):" % len(tests)) + for test in tests: + print("* %s" % test) + print() + + if output: + print("Output written into %s" % output) + + dt = math.ceil(time.time() - start_time) + if len(tests) <= args.max_tests: + print("Bisection completed in %s iterations and %s" + % (iteration, datetime.timedelta(seconds=dt))) + sys.exit(1) + else: + print("Bisection failed after %s iterations and %s" + % (iteration, datetime.timedelta(seconds=dt))) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index aadbf3b7f0d..6852860887c 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -599,6 +599,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, yield test if bad: return + if fail_env_changed and environment_changed: + return tests = test_forever() test_count = '' test_count_width = 3 @@ -913,7 +915,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, result = "FAILURE" elif interrupted: result = "INTERRUPTED" - elif environment_changed and fail_env_changed: + elif fail_env_changed and environment_changed: result = "ENV CHANGED" else: result = "SUCCESS" diff --git a/Lib/test/test_bisect.py b/Lib/test/test_bisect.py index 5c3330b4e41..47b5ebdd685 100644 --- a/Lib/test/test_bisect.py +++ b/Lib/test/test_bisect.py @@ -1,3 +1,7 @@ +# Use absolute import to prevent importing Lib/test/bisect.py, +# instead of Lib/bisect.py +from __future__ import absolute_import + import sys import unittest from test import test_support