[3.8] Update libregrtest from master (GH-19516)

* bpo-37531: regrtest now catchs ProcessLookupError (GH-16827)

Fix a warning on a race condition on TestWorkerProcess.kill(): ignore
silently ProcessLookupError rather than logging an useless warning.

(cherry picked from commit a661392f8f)

* bpo-38502: regrtest uses process groups if available (GH-16829)

test.regrtest now uses process groups in the multiprocessing mode
(-jN command line option) if process groups are available: if
os.setsid() and os.killpg() functions are available.

(cherry picked from commit ecb035cd14)

* bpo-37957: Allow regrtest to receive a file with test (and subtests) to ignore (GH-16989)

When building Python in some uncommon platforms there are some known tests that will fail. Right now, the test suite has the ability to ignore entire tests using the -x option and to receive a filter file using the --matchfile filter. The problem with the --matchfile option is that it receives a file with patterns to accept and when you want to ignore a couple of tests and subtests, is too cumbersome to lists ALL tests that are not the ones that you want to accept and he problem with -x is that is not easy to ignore just a subtests that fail and the whole test needs to be ignored.

For these reasons, add a new option to allow to ignore a list of test and subtests for these situations.

(cherry picked from commit e0cd8aa70a)

* regrtest: log timeout at startup (GH-19514)

Reduce also worker timeout.

(cherry picked from commit 4cf65a630a)

Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>

Lib/test/libregrtest/cmdline.py

@@ -207,10 +207,17 @@ def _create_parser():
     group.add_argument('-m', '--match', metavar='PAT',
                        dest='match_tests', action='append',
                        help='match test cases and methods with glob pattern PAT')
+    group.add_argument('-i', '--ignore', metavar='PAT',
+                       dest='ignore_tests', action='append',
+                       help='ignore test cases and methods with glob pattern PAT')
     group.add_argument('--matchfile', metavar='FILENAME',
                        dest='match_filename',
                        help='similar to --match but get patterns from a '
                             'text file, one pattern per line')
+    group.add_argument('--ignorefile', metavar='FILENAME',
+                       dest='ignore_filename',
+                       help='similar to --matchfile but it receives patterns '
+                            'from text file to ignore')
     group.add_argument('-G', '--failfast', action='store_true',
                        help='fail as soon as a test fails (only with -v or -W)')
     group.add_argument('-u', '--use', metavar='RES1,RES2,...',
@@ -317,7 +324,8 @@ def _parse_args(args, **kwargs):
          findleaks=1, use_resources=None, trace=False, coverdir='coverage',
          runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
          random_seed=None, use_mp=None, verbose3=False, forever=False,
-         header=False, failfast=False, match_tests=None, pgo=False)
+         header=False, failfast=False, match_tests=None, ignore_tests=None,
+         pgo=False)
     for k, v in kwargs.items():
         if not hasattr(ns, k):
             raise TypeError('%r is an invalid keyword argument '
@@ -395,6 +403,12 @@ def _parse_args(args, **kwargs):
         with open(ns.match_filename) as fp:
             for line in fp:
                 ns.match_tests.append(line.strip())
+    if ns.ignore_filename:
+        if ns.ignore_tests is None:
+            ns.ignore_tests = []
+        with open(ns.ignore_filename) as fp:
+            for line in fp:
+                ns.ignore_tests.append(line.strip())
     if ns.forever:
         # --forever implies --failfast
         ns.failfast = True

Lib/test/libregrtest/main.py

@@ -287,7 +287,7 @@ class Regrtest:
 
     def list_cases(self):
         support.verbose = False
-        support.set_match_tests(self.ns.match_tests)
+        support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests)
 
         for test_name in self.selected:
             abstest = get_abs_module(self.ns, test_name)
@@ -394,7 +394,10 @@ class Regrtest:
 
         save_modules = sys.modules.keys()
 
-        self.log("Run tests sequentially")
+        msg = "Run tests sequentially"
+        if self.ns.timeout:
+            msg += " (timeout: %s)" % format_duration(self.ns.timeout)
+        self.log(msg)
 
         previous_test = None
         for test_index, test_name in enumerate(self.tests, 1):

Lib/test/libregrtest/runtest.py

@@ -123,7 +123,7 @@ def _runtest(ns, test_name):
 
     start_time = time.perf_counter()
     try:
-        support.set_match_tests(ns.match_tests)
+        support.set_match_tests(ns.match_tests, ns.ignore_tests)
         support.junit_xml_list = xml_list = [] if ns.xmlpath else None
         if ns.failfast:
             support.failfast = True

Lib/test/libregrtest/runtest_mp.py

@@ -3,6 +3,7 @@ import faulthandler
 import json
 import os
 import queue
+import signal
 import subprocess
 import sys
 import threading
@@ -31,6 +32,8 @@ assert MAIN_PROCESS_TIMEOUT >= PROGRESS_UPDATE
 # Time to wait until a worker completes: should be immediate
 JOIN_TIMEOUT = 30.0   # seconds
 
+USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg"))
+
 
 def must_stop(result, ns):
     if result.result == INTERRUPTED:
@@ -59,12 +62,16 @@ def run_test_in_subprocess(testname, ns):
     # Running the child from the same working directory as regrtest's original
     # invocation ensures that TEMPDIR for the child is the same when
     # sysconfig.is_python_build() is true. See issue 15300.
+    kw = {}
+    if USE_PROCESS_GROUP:
+        kw['start_new_session'] = True
     return subprocess.Popen(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             universal_newlines=True,
                             close_fds=(os.name != 'nt'),
-                            cwd=support.SAVEDCWD)
+                            cwd=support.SAVEDCWD,
+                            **kw)
 
 
 def run_tests_worker(ns, test_name):
@@ -149,11 +156,24 @@ class TestWorkerProcess(threading.Thread):
             return
         self._killed = True
 
-        print(f"Kill {self}", file=sys.stderr, flush=True)
+        if USE_PROCESS_GROUP:
+            what = f"{self} process group"
+        else:
+            what = f"{self}"
+
+        print(f"Kill {what}", file=sys.stderr, flush=True)
         try:
-            popen.kill()
+            if USE_PROCESS_GROUP:
+                os.killpg(popen.pid, signal.SIGKILL)
+            else:
+                popen.kill()
+        except ProcessLookupError:
+            # popen.kill(): the process completed, the TestWorkerProcess thread
+            # read its exit status, but Popen.send_signal() read the returncode
+            # just before Popen.wait() set returncode.
+            pass
         except OSError as exc:
-            print_warning(f"Failed to kill {self}: {exc!r}")
+            print_warning(f"Failed to kill {what}: {exc!r}")
 
     def stop(self):
         # Method called from a different thread to stop this thread
@@ -332,7 +352,11 @@ class MultiprocessTestRunner:
         self.output = queue.Queue()
         self.pending = MultiprocessIterator(self.regrtest.tests)
         if self.ns.timeout is not None:
-            self.worker_timeout = self.ns.timeout * 1.5
+            # Rely on faulthandler to kill a worker process. This timouet is
+            # when faulthandler fails to kill a worker process. Give a maximum
+            # of 5 minutes to faulthandler to kill the worker.
+            self.worker_timeout = min(self.ns.timeout * 1.5,
+                                      self.ns.timeout + 5 * 60)
         else:
             self.worker_timeout = None
         self.workers = None
@@ -340,8 +364,12 @@ class MultiprocessTestRunner:
     def start_workers(self):
         self.workers = [TestWorkerProcess(index, self)
                         for index in range(1, self.ns.use_mp + 1)]
-        self.log("Run tests in parallel using %s child processes"
-                 % len(self.workers))
+        msg = f"Run tests in parallel using {len(self.workers)} child processes"
+        if self.ns.timeout:
+            msg += (" (timeout: %s, worker timeout: %s)"
+                    % (format_duration(self.ns.timeout),
+                       format_duration(self.worker_timeout)))
+        self.log(msg)
         for worker in self.workers:
             worker.start()
 

Lib/test/support/__init__.py

@@ -2047,7 +2047,9 @@ def _run_suite(suite):
 
 # By default, don't filter tests
 _match_test_func = None
-_match_test_patterns = None
+
+_accept_test_patterns = None
+_ignore_test_patterns = None
 
 
 def match_test(test):
@@ -2063,18 +2065,45 @@ def _is_full_match_test(pattern):
     # as a full test identifier.
     # Example: 'test.test_os.FileTests.test_access'.
     #
-    # Reject patterns which contain fnmatch patterns: '*', '?', '[...]'
-    # or '[!...]'. For example, reject 'test_access*'.
+    # ignore patterns which contain fnmatch patterns: '*', '?', '[...]'
+    # or '[!...]'. For example, ignore 'test_access*'.
     return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern))
 
 
-def set_match_tests(patterns):
-    global _match_test_func, _match_test_patterns
+def set_match_tests(accept_patterns=None, ignore_patterns=None):
+    global _match_test_func, _accept_test_patterns, _ignore_test_patterns
 
-    if patterns == _match_test_patterns:
-        # No change: no need to recompile patterns.
-        return
 
+    if accept_patterns is None:
+        accept_patterns = ()
+    if ignore_patterns is None:
+        ignore_patterns = ()
+
+    accept_func = ignore_func = None
+
+    if accept_patterns != _accept_test_patterns:
+        accept_patterns, accept_func = _compile_match_function(accept_patterns)
+    if ignore_patterns != _ignore_test_patterns:
+        ignore_patterns, ignore_func = _compile_match_function(ignore_patterns)
+
+    # Create a copy since patterns can be mutable and so modified later
+    _accept_test_patterns = tuple(accept_patterns)
+    _ignore_test_patterns = tuple(ignore_patterns)
+
+    if accept_func is not None or ignore_func is not None:
+        def match_function(test_id):
+            accept = True
+            ignore = False
+            if accept_func:
+                accept = accept_func(test_id)
+            if ignore_func:
+                ignore = ignore_func(test_id)
+            return accept and not ignore
+
+        _match_test_func = match_function
+
+
+def _compile_match_function(patterns):
     if not patterns:
         func = None
         # set_match_tests(None) behaves as set_match_tests(())
@@ -2102,10 +2131,7 @@ def set_match_tests(patterns):
 
         func = match_test_regex
 
-    # Create a copy since patterns can be mutable and so modified later
-    _match_test_patterns = tuple(patterns)
-    _match_test_func = func
-
+    return patterns, func
 
 
 def run_unittest(*classes):

Lib/test/test_regrtest.py

@@ -157,6 +157,24 @@ class ParseArgsTestCase(unittest.TestCase):
                 self.assertTrue(ns.single)
                 self.checkError([opt, '-f', 'foo'], "don't go together")
 
+    def test_ignore(self):
+        for opt in '-i', '--ignore':
+            with self.subTest(opt=opt):
+                ns = libregrtest._parse_args([opt, 'pattern'])
+                self.assertEqual(ns.ignore_tests, ['pattern'])
+                self.checkError([opt], 'expected one argument')
+
+        self.addCleanup(support.unlink, support.TESTFN)
+        with open(support.TESTFN, "w") as fp:
+            print('matchfile1', file=fp)
+            print('matchfile2', file=fp)
+
+        filename = os.path.abspath(support.TESTFN)
+        ns = libregrtest._parse_args(['-m', 'match',
+                                      '--ignorefile', filename])
+        self.assertEqual(ns.ignore_tests,
+                         ['matchfile1', 'matchfile2'])
+
     def test_match(self):
         for opt in '-m', '--match':
             with self.subTest(opt=opt):
@@ -960,6 +978,42 @@ class ArgsTestCase(BaseTestCase):
         regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE)
         return [match.group(1) for match in regex.finditer(output)]
 
+    def test_ignorefile(self):
+        code = textwrap.dedent("""
+            import unittest
+
+            class Tests(unittest.TestCase):
+                def test_method1(self):
+                    pass
+                def test_method2(self):
+                    pass
+                def test_method3(self):
+                    pass
+                def test_method4(self):
+                    pass
+        """)
+        all_methods = ['test_method1', 'test_method2',
+                       'test_method3', 'test_method4']
+        testname = self.create_test(code=code)
+
+        # only run a subset
+        filename = support.TESTFN
+        self.addCleanup(support.unlink, filename)
+
+        subset = [
+            # only ignore the method name
+            'test_method1',
+            # ignore the full identifier
+            '%s.Tests.test_method3' % testname]
+        with open(filename, "w") as fp:
+            for name in subset:
+                print(name, file=fp)
+
+        output = self.run_tests("-v", "--ignorefile", filename, testname)
+        methods = self.parse_methods(output)
+        subset = ['test_method2', 'test_method4']
+        self.assertEqual(methods, subset)
+
     def test_matchfile(self):
         code = textwrap.dedent("""
             import unittest

Lib/test/test_support.py

@@ -527,6 +527,7 @@ class TestSupport(unittest.TestCase):
         test_access = Test('test.test_os.FileTests.test_access')
         test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir')
 
+        # Test acceptance
         with support.swap_attr(support, '_match_test_func', None):
             # match all
             support.set_match_tests([])
@@ -534,45 +535,92 @@ class TestSupport(unittest.TestCase):
             self.assertTrue(support.match_test(test_chdir))
 
             # match all using None
-            support.set_match_tests(None)
+            support.set_match_tests(None, None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # match the full test identifier
-            support.set_match_tests([test_access.id()])
+            support.set_match_tests([test_access.id()], None)
             self.assertTrue(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
 
             # match the module name
-            support.set_match_tests(['test_os'])
+            support.set_match_tests(['test_os'], None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # Test '*' pattern
-            support.set_match_tests(['test_*'])
+            support.set_match_tests(['test_*'], None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # Test case sensitivity
-            support.set_match_tests(['filetests'])
+            support.set_match_tests(['filetests'], None)
             self.assertFalse(support.match_test(test_access))
-            support.set_match_tests(['FileTests'])
+            support.set_match_tests(['FileTests'], None)
             self.assertTrue(support.match_test(test_access))
 
             # Test pattern containing '.' and a '*' metacharacter
-            support.set_match_tests(['*test_os.*.test_*'])
+            support.set_match_tests(['*test_os.*.test_*'], None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # Multiple patterns
-            support.set_match_tests([test_access.id(), test_chdir.id()])
+            support.set_match_tests([test_access.id(), test_chdir.id()], None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
-            support.set_match_tests(['test_access', 'DONTMATCH'])
+            support.set_match_tests(['test_access', 'DONTMATCH'], None)
             self.assertTrue(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
 
+        # Test rejection
+        with support.swap_attr(support, '_match_test_func', None):
+            # match all
+            support.set_match_tests(ignore_patterns=[])
+            self.assertTrue(support.match_test(test_access))
+            self.assertTrue(support.match_test(test_chdir))
+
+            # match all using None
+            support.set_match_tests(None, None)
+            self.assertTrue(support.match_test(test_access))
+            self.assertTrue(support.match_test(test_chdir))
+
+            # match the full test identifier
+            support.set_match_tests(None, [test_access.id()])
+            self.assertFalse(support.match_test(test_access))
+            self.assertTrue(support.match_test(test_chdir))
+
+            # match the module name
+            support.set_match_tests(None, ['test_os'])
+            self.assertFalse(support.match_test(test_access))
+            self.assertFalse(support.match_test(test_chdir))
+
+            # Test '*' pattern
+            support.set_match_tests(None, ['test_*'])
+            self.assertFalse(support.match_test(test_access))
+            self.assertFalse(support.match_test(test_chdir))
+
+            # Test case sensitivity
+            support.set_match_tests(None, ['filetests'])
+            self.assertTrue(support.match_test(test_access))
+            support.set_match_tests(None, ['FileTests'])
+            self.assertFalse(support.match_test(test_access))
+
+            # Test pattern containing '.' and a '*' metacharacter
+            support.set_match_tests(None, ['*test_os.*.test_*'])
+            self.assertFalse(support.match_test(test_access))
+            self.assertFalse(support.match_test(test_chdir))
+
+            # Multiple patterns
+            support.set_match_tests(None, [test_access.id(), test_chdir.id()])
+            self.assertFalse(support.match_test(test_access))
+            self.assertFalse(support.match_test(test_chdir))
+
+            support.set_match_tests(None, ['test_access', 'DONTMATCH'])
+            self.assertFalse(support.match_test(test_access))
+            self.assertTrue(support.match_test(test_chdir))
+
     def test_fd_count(self):
         # We cannot test the absolute value of fd_count(): on old Linux
         # kernel or glibc versions, os.urandom() keeps a FD open on

Misc/NEWS.d/next/Tests/2019-10-17-00-49-38.bpo-38502.vUEic7.rst

@@ -0,0 +1,3 @@
+test.regrtest now uses process groups in the multiprocessing mode (-jN command
+line option) if process groups are available: if :func:`os.setsid` and
+:func:`os.killpg` functions are available.

Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst

@@ -0,0 +1,3 @@
+test.regrtest now can receive a list of test patterns to ignore (using the
+-i/--ignore argument) or a file with a list of patterns to ignore (using the
+--ignore-file argument). Patch by Pablo Galindo.