diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 214e430e059..ca63496bf8a 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -577,7 +577,6 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, elif ok == FAILED: bad.append(test) elif ok == ENV_CHANGED: - bad.append(test) environment_changed.append(test) elif ok == SKIPPED: skipped.append(test) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 405fbd52569..7a7805d59f6 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -35,7 +35,7 @@ import multiprocessing.managers import multiprocessing.heap import multiprocessing.pool -from multiprocessing import util +from multiprocessing import util, reduction try: from multiprocessing.sharedctypes import Value, copy @@ -43,6 +43,11 @@ try: except ImportError: HAS_SHAREDCTYPES = False +try: + import msvcrt +except ImportError: + msvcrt = None + # # # @@ -89,6 +94,11 @@ else: timeout = None return handle in _select([handle], [], [], timeout)[0] +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except: + MAXFD = 256 + # # Some tests require ctypes # @@ -1587,6 +1597,76 @@ class _TestConnection(BaseTestCase): self.assertRaises(ValueError, a.send_bytes, msg, 4, -1) + @classmethod + def _is_fd_assigned(cls, fd): + try: + os.fstat(fd) + except OSError as e: + if e.errno == errno.EBADF: + return False + raise + else: + return True + + @classmethod + def _writefd(cls, conn, data, create_dummy_fds=False): + if create_dummy_fds: + for i in range(0, 256): + if not cls._is_fd_assigned(i): + os.dup2(conn.fileno(), i) + fd = reduction.recv_handle(conn) + if msvcrt: + fd = msvcrt.open_osfhandle(fd, os.O_WRONLY) + os.write(fd, data) + os.close(fd) + + def test_fd_transfer(self): + if self.TYPE != 'processes': + self.skipTest("only makes sense with processes") + conn, child_conn = self.Pipe(duplex=True) + + p = self.Process(target=self._writefd, args=(child_conn, b"foo")) + p.start() + with open(test.support.TESTFN, "wb") as f: + fd = f.fileno() + if msvcrt: + fd = msvcrt.get_osfhandle(fd) + reduction.send_handle(conn, fd, p.pid) + p.join() + with open(test.support.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"foo") + + @unittest.skipIf(sys.platform == "win32", + "test semantics don't make sense on Windows") + @unittest.skipIf(MAXFD <= 256, + "largest assignable fd number is too small") + @unittest.skipUnless(hasattr(os, "dup2"), + "test needs os.dup2()") + def test_large_fd_transfer(self): + # With fd > 256 (issue #11657) + if self.TYPE != 'processes': + self.skipTest("only makes sense with processes") + conn, child_conn = self.Pipe(duplex=True) + + p = self.Process(target=self._writefd, args=(child_conn, b"bar", True)) + p.start() + with open(test.support.TESTFN, "wb") as f: + fd = f.fileno() + for newfd in range(256, MAXFD): + if not self._is_fd_assigned(newfd): + break + else: + self.fail("could not find an unassigned large file descriptor") + os.dup2(fd, newfd) + try: + reduction.send_handle(conn, newfd, p.pid) + finally: + os.close(newfd) + p.join() + with open(test.support.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"bar") + + class _TestListenerClient(BaseTestCase): ALLOWED_TYPES = ('processes', 'threads') diff --git a/Misc/NEWS b/Misc/NEWS index b2d9e84b943..4b5a9a6d778 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -268,6 +268,9 @@ Core and Builtins Library ------- +- Issue #11657: Fix sending file descriptors over 255 over a multiprocessing + Pipe. + - Issue #12811: tabnanny.check() now promptly closes checked files. Patch by Anthony Briggs. diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 5d1cf5661aa..1987b951f94 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -111,7 +111,7 @@ multiprocessing_sendfd(PyObject *self, PyObject *args) cmsg->cmsg_type = SCM_RIGHTS; cmsg->cmsg_len = CMSG_LEN(sizeof(int)); msg.msg_controllen = cmsg->cmsg_len; - *CMSG_DATA(cmsg) = fd; + * (int *) CMSG_DATA(cmsg) = fd; Py_BEGIN_ALLOW_THREADS res = sendmsg(conn, &msg, 0); @@ -154,7 +154,7 @@ multiprocessing_recvfd(PyObject *self, PyObject *args) if (res < 0) return PyErr_SetFromErrno(PyExc_OSError); - fd = *CMSG_DATA(cmsg); + fd = * (int *) CMSG_DATA(cmsg); return Py_BuildValue("i", fd); }