* Replaces the internals of the subprocess module from fork through exec on

POSIX systems with a C extension module.  This is required in order for
  the subprocess module to be made thread safe.

  The pure python implementation is retained so that it can continue to be
  used if for some reason the _posixsubprocess extension module is not
  available.

  The unittest executes tests on both code paths to guarantee compatibility.

* Moves PyLong_FromPid and PyLong_AsPid from posixmodule.c into longobject.h.

Code reviewed by jeffrey.yasskin at http://codereview.appspot.com/223077/show

Doc/library/subprocess.rst

@@ -28,7 +28,7 @@ Using the subprocess Module
 This module defines one class called :class:`Popen`:
 
 
-.. class:: Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0)
+.. class:: Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False)
 
    Arguments are:
 
@@ -41,7 +41,8 @@ This module defines one class called :class:`Popen`:
    name for the executing program in utilities such as :program:`ps`.
 
    On Unix, with *shell=False* (default): In this case, the Popen class uses
-   :meth:`os.execvp` to execute the child program. *args* should normally be a
+   :meth:`os.execvp` like behavior to execute the child program.
+   *args* should normally be a
    sequence.  If a string is specified for *args*, it will be used as the name
    or path of the program to execute; this will only work if the program is
    being given no arguments.
@@ -108,7 +109,23 @@ This module defines one class called :class:`Popen`:
    applications should be captured into the same file handle as for stdout.
 
    If *preexec_fn* is set to a callable object, this object will be called in the
-   child process just before the child is executed. (Unix only)
+   child process just before the child is executed.
+   (Unix only)
+
+   .. warning::
+
+      The *preexec_fn* parameter is not safe to use in the presence of threads
+      in your application.  The child process could deadlock before exec is
+      called.
+      If you must use it, keep it trivial!  Minimize the number of libraries
+      you call into.
+
+   .. note::
+
+      If you need to modify the environment for the child use the *env*
+      parameter rather than doing it in a *preexec_fn*.
+      The *start_new_session* parameter can take the place of a previously
+      common use of *preexec_fn* to call os.setsid() in the child.
 
    If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and
    :const:`2` will be closed before the child process is executed. (Unix only).
@@ -124,9 +141,23 @@ This module defines one class called :class:`Popen`:
    searching the executable, so you can't specify the program's path relative to
    *cwd*.
 
+   If *restore_signals* is True (the default) all signals that Python has set to
+   SIG_IGN are restored to SIG_DFL in the child process before the exec.
+   Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals.
+   (Unix only)
+
+   .. versionchanged:: 3.2
+      *restore_signals* was added.
+
+   If *start_new_session* is True the setsid() system call will be made in the
+   child process prior to the execution of the subprocess.  (Unix only)
+
+   .. versionchanged:: 3.2
+      *start_new_session* was added.
+
    If *env* is not ``None``, it must be a mapping that defines the environment
-   variables for the new process; these are used instead of inheriting the current
+   variables for the new process; these are used instead of the default
-   process' environment, which is the default behavior.
+   behavior of inheriting the current process' environment.
 
    .. note::
 

Include/abstract.h

@@ -1232,6 +1232,9 @@ PyAPI_FUNC(int) _PyObject_RealIsInstance(PyObject *inst, PyObject *cls);
 
 PyAPI_FUNC(int) _PyObject_RealIsSubclass(PyObject *derived, PyObject *cls);
 
+PyAPI_FUNC(char *const *) _PySequence_BytesToCharpArray(PyObject* self);
+
+PyAPI_FUNC(void) _Py_FreeCharPArray(char *const array[]);
 
 #ifdef __cplusplus
 }

Include/longobject.h

@@ -41,6 +41,23 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void);
 #define PyLong_AsSocket_t(fd) (SOCKET_T)PyLong_AsLongLong(fd)
 #endif
 
+/* Issue #1983: pid_t can be longer than a C long on some systems */
+#if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT
+#define PARSE_PID "i"
+#define PyLong_FromPid PyLong_FromLong
+#define PyLong_AsPid PyLong_AsLong
+#elif SIZEOF_PID_T == SIZEOF_LONG
+#define PARSE_PID "l"
+#define PyLong_FromPid PyLong_FromLong
+#define PyLong_AsPid PyLong_AsLong
+#elif defined(SIZEOF_LONG_LONG) && SIZEOF_PID_T == SIZEOF_LONG_LONG
+#define PARSE_PID "L"
+#define PyLong_FromPid PyLong_FromLongLong
+#define PyLong_AsPid PyLong_AsLongLong
+#else
+#error "sizeof(pid_t) is neither sizeof(int), sizeof(long) or sizeof(long long)"
+#endif /* SIZEOF_PID_T */
+
 /* For use by intobject.c only */
 PyAPI_DATA(unsigned char) _PyLong_DigitValue[256];
 

Include/pythonrun.h

@@ -81,6 +81,9 @@ PyAPI_FUNC(int) Py_AtExit(void (*func)(void));
 
 PyAPI_FUNC(void) Py_Exit(int);
 
+/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */
+PyAPI_FUNC(void) _Py_RestoreSignals(void);
+
 PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
 
 /* Bootstrap */

Lib/subprocess.py

@@ -29,7 +29,8 @@ class Popen(args, bufsize=0, executable=None,
             stdin=None, stdout=None, stderr=None,
             preexec_fn=None, close_fds=False, shell=False,
             cwd=None, env=None, universal_newlines=False,
-            startupinfo=None, creationflags=0):
+            startupinfo=None, creationflags=0,
+            restore_signals=True, start_new_session=False):
 
 
 Arguments are:
@@ -72,8 +73,11 @@ parent.  Additionally, stderr can be STDOUT, which indicates that the
 stderr data from the applications should be captured into the same
 file handle as for stdout.
 
-If preexec_fn is set to a callable object, this object will be called
+On UNIX, if preexec_fn is set to a callable object, this object will be
-in the child process just before the child is executed.
+called in the child process just before the child is executed.  The use
+of preexec_fn is not thread safe, using it in the presence of threads
+could lead to a deadlock in the child process before the new executable
+is executed.
 
 If close_fds is true, all file descriptors except 0, 1 and 2 will be
 closed before the child process is executed.
@@ -84,6 +88,14 @@ shell.
 If cwd is not None, the current directory will be changed to cwd
 before the child is executed.
 
+On UNIX, if restore_signals is True all signals that Python sets to
+SIG_IGN are restored to SIG_DFL in the child process before the exec.
+Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals.  This
+parameter does nothing on Windows.
+
+On UNIX, if start_new_session is True, the setsid() system call will be made
+in the child process prior to executing the command.
+
 If env is not None, it defines the environment variables for the new
 process.
 
@@ -326,6 +338,7 @@ import os
 import traceback
 import gc
 import signal
+import builtins
 
 # Exception classes used by this module.
 class CalledProcessError(Exception):
@@ -375,6 +388,15 @@ else:
     import fcntl
     import pickle
 
+    try:
+        import _posixsubprocess
+    except ImportError:
+        _posixsubprocess = None
+        import warnings
+        warnings.warn("The _posixsubprocess module is not being used. "
+                      "Child process reliability may suffer if your "
+                      "program uses threads.", RuntimeWarning)
+
     # When select or poll has indicated that the file is writable,
     # we can write up to _PIPE_BUF bytes without risk of blocking.
     # POSIX defines PIPE_BUF as >= 512.
@@ -596,7 +618,8 @@ class Popen(object):
                  stdin=None, stdout=None, stderr=None,
                  preexec_fn=None, close_fds=False, shell=False,
                  cwd=None, env=None, universal_newlines=False,
-                 startupinfo=None, creationflags=0):
+                 startupinfo=None, creationflags=0,
+                 restore_signals=True, start_new_session=False):
         """Create new Popen instance."""
         _cleanup()
 
@@ -642,7 +665,7 @@ class Popen(object):
         # On POSIX, the child objects are file descriptors.  On
         # Windows, these are Windows file handles.  The parent objects
         # are file descriptors on both platforms.  The parent objects
-        # are None when not using PIPEs. The child objects are None
+        # are -1 when not using PIPEs. The child objects are -1
         # when not redirecting.
 
         (p2cread, p2cwrite,
@@ -654,7 +677,8 @@ class Popen(object):
                             startupinfo, creationflags, shell,
                             p2cread, p2cwrite,
                             c2pread, c2pwrite,
-                            errread, errwrite)
+                            errread, errwrite,
+                            restore_signals, start_new_session)
 
         if mswindows:
             if p2cwrite is not None:
@@ -666,15 +690,15 @@ class Popen(object):
 
         if bufsize == 0:
             bufsize = 1  # Nearly unbuffered (XXX for now)
-        if p2cwrite is not None:
+        if p2cwrite != -1:
             self.stdin = io.open(p2cwrite, 'wb', bufsize)
             if self.universal_newlines:
                 self.stdin = io.TextIOWrapper(self.stdin)
-        if c2pread is not None:
+        if c2pread != -1:
             self.stdout = io.open(c2pread, 'rb', bufsize)
             if universal_newlines:
                 self.stdout = io.TextIOWrapper(self.stdout)
-        if errread is not None:
+        if errread != -1:
             self.stderr = io.open(errread, 'rb', bufsize)
             if universal_newlines:
                 self.stderr = io.TextIOWrapper(self.stderr)
@@ -739,11 +763,11 @@ class Popen(object):
             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
             """
             if stdin is None and stdout is None and stderr is None:
-                return (None, None, None, None, None, None)
+                return (-1, -1, -1, -1, -1, -1)
 
-            p2cread, p2cwrite = None, None
+            p2cread, p2cwrite = -1, -1
-            c2pread, c2pwrite = None, None
+            c2pread, c2pwrite = -1, -1
-            errread, errwrite = None, None
+            errread, errwrite = -1, -1
 
             if stdin is None:
                 p2cread = GetStdHandle(STD_INPUT_HANDLE)
@@ -819,7 +843,8 @@ class Popen(object):
                            startupinfo, creationflags, shell,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
-                           errread, errwrite):
+                           errread, errwrite,
+                           unused_restore_signals, unused_start_new_session):
             """Execute program (MS Windows version)"""
 
             if not isinstance(args, str):
@@ -973,9 +998,9 @@ class Popen(object):
             """Construct and return tuple with IO objects:
             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
             """
-            p2cread, p2cwrite = None, None
+            p2cread, p2cwrite = -1, -1
-            c2pread, c2pwrite = None, None
+            c2pread, c2pwrite = -1, -1
-            errread, errwrite = None, None
+            errread, errwrite = -1, -1
 
             if stdin is None:
                 pass
@@ -1034,7 +1059,8 @@ class Popen(object):
                            startupinfo, creationflags, shell,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
-                           errread, errwrite):
+                           errread, errwrite,
+                           restore_signals, start_new_session):
             """Execute program (POSIX version)"""
 
             if isinstance(args, str):
@@ -1048,113 +1074,190 @@ class Popen(object):
             if executable is None:
                 executable = args[0]
 
-            # For transferring possible exec failure from child to parent
+            # For transferring possible exec failure from child to parent.
-            # The first char specifies the exception type: 0 means
+            # Data format: "exception name:hex errno:description"
-            # OSError, 1 means some other error.
+            # Pickle is not used; it is complex and involves memory allocation.
             errpipe_read, errpipe_write = os.pipe()
             try:
                 try:
                     self._set_cloexec_flag(errpipe_write)
 
-                    gc_was_enabled = gc.isenabled()
+                    if _posixsubprocess:
-                    # Disable gc to avoid bug where gc -> file_dealloc ->
+                        fs_encoding = sys.getfilesystemencoding()
-                    # write to stderr -> hang. http://bugs.python.org/issue1336
+                        def fs_encode(s):
-                    gc.disable()
+                            """Encode s for use in the env, fs or cmdline."""
-                    try:
+                            return s.encode(fs_encoding, 'surrogateescape')
-                        self.pid = os.fork()
+
-                    except:
+                        # We must avoid complex work that could involve
+                        # malloc or free in the child process to avoid
+                        # potential deadlocks, thus we do all this here.
+                        # and pass it to fork_exec()
+
+                        if env:
+                            env_list = [fs_encode(k) + b'=' + fs_encode(v)
+                                        for k, v in env.items()]
+                        else:
+                            env_list = None  # Use execv instead of execve.
+                        if os.path.dirname(executable):
+                            executable_list = (fs_encode(executable),)
+                        else:
+                            # This matches the behavior of os._execvpe().
+                            path_list = os.get_exec_path(env)
+                            executable_list = (os.path.join(dir, executable)
+                                               for dir in path_list)
+                            executable_list = tuple(fs_encode(exe)
+                                                    for exe in executable_list)
+                        self.pid = _posixsubprocess.fork_exec(
+                                args, executable_list,
+                                close_fds, cwd, env_list,
+                                p2cread, p2cwrite, c2pread, c2pwrite,
+                                errread, errwrite,
+                                errpipe_read, errpipe_write,
+                                restore_signals, start_new_session, preexec_fn)
+                    else:
+                        # Pure Python implementation: It is not thread safe.
+                        # This implementation may deadlock in the child if your
+                        # parent process has any other threads running.
+
+                        gc_was_enabled = gc.isenabled()
+                        # Disable gc to avoid bug where gc -> file_dealloc ->
+                        # write to stderr -> hang.  See issue1336
+                        gc.disable()
+                        try:
+                            self.pid = os.fork()
+                        except:
+                            if gc_was_enabled:
+                                gc.enable()
+                            raise
+                        self._child_created = True
+                        if self.pid == 0:
+                            # Child
+                            try:
+                                # Close parent's pipe ends
+                                if p2cwrite != -1:
+                                    os.close(p2cwrite)
+                                if c2pread != -1:
+                                    os.close(c2pread)
+                                if errread != -1:
+                                    os.close(errread)
+                                os.close(errpipe_read)
+
+                                # Dup fds for child
+                                if p2cread != -1:
+                                    os.dup2(p2cread, 0)
+                                if c2pwrite != -1:
+                                    os.dup2(c2pwrite, 1)
+                                if errwrite != -1:
+                                    os.dup2(errwrite, 2)
+
+                                # Close pipe fds.  Make sure we don't close the
+                                # same fd more than once, or standard fds.
+                                if p2cread != -1 and p2cread not in (0,):
+                                    os.close(p2cread)
+                                if (c2pwrite != -1 and
+                                    c2pwrite not in (p2cread, 1)):
+                                    os.close(c2pwrite)
+                                if (errwrite != -1 and
+                                    errwrite not in (p2cread, c2pwrite, 2)):
+                                    os.close(errwrite)
+
+                                # Close all other fds, if asked for
+                                if close_fds:
+                                    self._close_fds(but=errpipe_write)
+
+                                if cwd is not None:
+                                    os.chdir(cwd)
+
+                                # This is a copy of Python/pythonrun.c
+                                # _Py_RestoreSignals().  If that were exposed
+                                # as a sys._py_restoresignals func it would be
+                                # better.. but this pure python implementation
+                                # isn't likely to be used much anymore.
+                                if restore_signals:
+                                    signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ')
+                                    for sig in signals:
+                                        if hasattr(signal, sig):
+                                            signal.signal(getattr(signal, sig),
+                                                          signal.SIG_DFL)
+
+                                if start_new_session and hasattr(os, 'setsid'):
+                                    os.setsid()
+
+                                if preexec_fn:
+                                    preexec_fn()
+
+                                if env is None:
+                                    os.execvp(executable, args)
+                                else:
+                                    os.execvpe(executable, args, env)
+
+                            except:
+                                try:
+                                    exc_type, exc_value = sys.exc_info()[:2]
+                                    if isinstance(exc_value, OSError):
+                                        errno = exc_value.errno
+                                    else:
+                                        errno = 0
+                                    message = '%s:%x:%s' % (exc_type.__name__,
+                                                            errno, exc_value)
+                                    os.write(errpipe_write, message.encode())
+                                except:
+                                    # We MUST not allow anything odd happening
+                                    # above to prevent us from exiting below.
+                                    pass
+
+                            # This exitcode won't be reported to applications
+                            # so it really doesn't matter what we return.
+                            os._exit(255)
+
+                        # Parent
                         if gc_was_enabled:
                             gc.enable()
-                        raise
-                    self._child_created = True
-                    if self.pid == 0:
-                        # Child
-                        try:
-                            # Close parent's pipe ends
-                            if p2cwrite is not None:
-                                os.close(p2cwrite)
-                            if c2pread is not None:
-                                os.close(c2pread)
-                            if errread is not None:
-                                os.close(errread)
-                            os.close(errpipe_read)
-
-                            # Dup fds for child
-                            if p2cread is not None:
-                                os.dup2(p2cread, 0)
-                            if c2pwrite is not None:
-                                os.dup2(c2pwrite, 1)
-                            if errwrite is not None:
-                                os.dup2(errwrite, 2)
-
-                            # Close pipe fds.  Make sure we don't close the
-                            # same fd more than once, or standard fds.
-                            if p2cread is not None and p2cread not in (0,):
-                                os.close(p2cread)
-                            if c2pwrite is not None and \
-                                                c2pwrite not in (p2cread, 1):
-                                os.close(c2pwrite)
-                            if (errwrite is not None and
-                                errwrite not in (p2cread, c2pwrite, 2)):
-                                os.close(errwrite)
-
-                            # Close all other fds, if asked for
-                            if close_fds:
-                                self._close_fds(but=errpipe_write)
-
-                            if cwd is not None:
-                                os.chdir(cwd)
-
-                            if preexec_fn:
-                                preexec_fn()
-
-                            if env is None:
-                                os.execvp(executable, args)
-                            else:
-                                os.execvpe(executable, args, env)
-
-                        except:
-                            exc_type, exc_value, tb = sys.exc_info()
-                            # Save the traceback and attach it to the exception
-                            # object
-                            exc_lines = traceback.format_exception(exc_type,
-                                                                   exc_value,
-                                                                   tb)
-                            exc_value.child_traceback = ''.join(exc_lines)
-                            os.write(errpipe_write, pickle.dumps(exc_value))
-
-                        # This exitcode won't be reported to applications, so
-                        # it really doesn't matter what we return.
-                        os._exit(255)
-
-                    # Parent
-                    if gc_was_enabled:
-                        gc.enable()
                 finally:
                     # be sure the FD is closed no matter what
                     os.close(errpipe_write)
 
-                if p2cread is not None and p2cwrite is not None:
+                if p2cread != -1 and p2cwrite != -1:
                     os.close(p2cread)
-                if c2pwrite is not None and c2pread is not None:
+                if c2pwrite != -1 and c2pread != -1:
                     os.close(c2pwrite)
-                if errwrite is not None and errread is not None:
+                if errwrite != -1 and errread != -1:
                     os.close(errwrite)
 
                 # Wait for exec to fail or succeed; possibly raising an
-                # exception (limited to 1 MB)
+                # exception (limited in size)
-                data = _eintr_retry_call(os.read, errpipe_read, 1048576)
+                data = bytearray()
+                while True:
+                    part = _eintr_retry_call(os.read, errpipe_read, 50000)
+                    data += part
+                    if not part or len(data) > 50000:
+                        break
             finally:
                 # be sure the FD is closed no matter what
                 os.close(errpipe_read)
 
             if data:
                 _eintr_retry_call(os.waitpid, self.pid, 0)
-                child_exception = pickle.loads(data)
+                try:
+                    exception_name, hex_errno, err_msg = data.split(b':', 2)
+                except ValueError:
+                    print('Bad exception data:', repr(data))
+                    exception_name = b'RuntimeError'
+                    hex_errno = b'0'
+                    err_msg = b'Unknown'
+                child_exception_type = getattr(
+                        builtins, exception_name.decode('ascii'),
+                        RuntimeError)
                 for fd in (p2cwrite, c2pread, errread):
-                    if fd is not None:
+                    if fd != -1:
                         os.close(fd)
-                raise child_exception
+                err_msg = err_msg.decode()
+                if issubclass(child_exception_type, OSError) and hex_errno:
+                    errno = int(hex_errno, 16)
+                    if errno != 0:
+                        err_msg = os.strerror(errno)
+                    raise child_exception_type(errno, err_msg)
+                raise child_exception_type(err_msg)
 
 
         def _handle_exitstatus(self, sts):

Lib/test/test_subprocess.py

@@ -568,12 +568,53 @@ class POSIXProcessTestCase(unittest.TestCase):
         self.assertFalse(subprocess._active, "subprocess._active not empty")
 
     def test_exceptions(self):
-        # caught & re-raised exceptions
+        nonexistent_dir = "/_this/pa.th/does/not/exist"
-        with self.assertRaises(OSError) as c:
+        try:
+            os.chdir(nonexistent_dir)
+        except OSError as e:
+            # This avoids hard coding the errno value or the OS perror()
+            # string and instead capture the exception that we want to see
+            # below for comparison.
+            desired_exception = e
+        else:
+            self.fail("chdir to nonexistant directory %s succeeded." %
+                      nonexistent_dir)
+
+        # Error in the child re-raised in the parent.
+        try:
             p = subprocess.Popen([sys.executable, "-c", ""],
-                                 cwd="/this/path/does/not/exist")
+                                 cwd=nonexistent_dir)
-        # The attribute child_traceback should contain "os.chdir" somewhere.
+        except OSError as e:
-        self.assertIn("os.chdir", c.exception.child_traceback)
+            # Test that the child process chdir failure actually makes
+            # it up to the parent process as the correct exception.
+            self.assertEqual(desired_exception.errno, e.errno)
+            self.assertEqual(desired_exception.strerror, e.strerror)
+        else:
+            self.fail("Expected OSError: %s" % desired_exception)
+
+    def test_restore_signals(self):
+        # Code coverage for both values of restore_signals to make sure it
+        # at least does not blow up.
+        # A test for behavior would be complex.  Contributions welcome.
+        subprocess.call([sys.executable, "-c", ""], restore_signals=True)
+        subprocess.call([sys.executable, "-c", ""], restore_signals=False)
+
+    def test_start_new_session(self):
+        # For code coverage of calling setsid().  We don't care if we get an
+        # EPERM error from it depending on the test execution environment, that
+        # still indicates that it was called.
+        try:
+            output = subprocess.check_output(
+                    [sys.executable, "-c",
+                     "import os; print(os.getpgid(os.getpid()))"],
+                    start_new_session=True)
+        except OSError as e:
+            if e.errno != errno.EPERM:
+                raise
+        else:
+            parent_pgid = os.getpgid(os.getpid())
+            child_pgid = int(output)
+            self.assertNotEqual(parent_pgid, child_pgid)
 
     def test_run_abort(self):
         # returncode handles signal termination
@@ -584,7 +625,8 @@ class POSIXProcessTestCase(unittest.TestCase):
         self.assertEqual(-p.returncode, signal.SIGABRT)
 
     def test_preexec(self):
-        # preexec function
+        # DISCLAIMER: Setting environment variables is *not* a good use
+        # of a preexec_fn.  This is merely a test.
         p = subprocess.Popen([sys.executable, "-c",
                               'import sys,os;'
                               'sys.stdout.write(os.getenv("FRUIT"))'],
@@ -592,6 +634,22 @@ class POSIXProcessTestCase(unittest.TestCase):
                              preexec_fn=lambda: os.putenv("FRUIT", "apple"))
         self.assertEqual(p.stdout.read(), b"apple")
 
+    def test_preexec_exception(self):
+        def raise_it():
+            raise ValueError("What if two swallows carried a coconut?")
+        try:
+            p = subprocess.Popen([sys.executable, "-c", ""],
+                                 preexec_fn=raise_it)
+        except RuntimeError as e:
+            self.assertTrue(
+                    subprocess._posixsubprocess,
+                    "Expected a ValueError from the preexec_fn")
+        except ValueError as e:
+            self.assertIn("coconut", e.args[0])
+        else:
+            self.fail("Exception raised by preexec_fn did not make it "
+                      "to the parent process.")
+
     def test_args_string(self):
         # args is a string
         fd, fname = mkstemp()
@@ -836,6 +894,20 @@ class ProcessTestCaseNoPoll(ProcessTestCase):
         ProcessTestCase.tearDown(self)
 
 
+@unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False),
+                     "_posixsubprocess extension module not found.")
+class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase):
+    def setUp(self):
+        subprocess._posixsubprocess = None
+        ProcessTestCase.setUp(self)
+        POSIXProcessTestCase.setUp(self)
+
+    def tearDown(self):
+        subprocess._posixsubprocess = sys.modules['_posixsubprocess']
+        POSIXProcessTestCase.tearDown(self)
+        ProcessTestCase.tearDown(self)
+
+
 class HelperFunctionTests(unittest.TestCase):
     @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows")
     def test_eintr_retry_call(self):
@@ -859,6 +931,7 @@ def test_main():
     unit_tests = (ProcessTestCase,
                   POSIXProcessTestCase,
                   Win32ProcessTestCase,
+                  ProcessTestCasePOSIXPurePython,
                   CommandTests,
                   ProcessTestCaseNoPoll,
                   HelperFunctionTests)

Modules/_posixsubprocess.c

@@ -0,0 +1,385 @@
+/* Authors: Gregory P. Smith & Jeffrey Yasskin */
+#include "Python.h"
+#include <unistd.h>
+
+
+#define POSIX_CALL(call)   if ((call) == -1) goto error
+
+
+/* Maximum file descriptor, initialized on module load. */
+static long max_fd;
+
+
+/* Given the gc module call gc.enable() and return 0 on success. */
+static int _enable_gc(PyObject *gc_module)
+{
+    PyObject *result;
+    result = PyObject_CallMethod(gc_module, "enable", NULL);
+    if (result == NULL)
+        return 1;
+    Py_DECREF(result);
+    return 0;
+}
+
+
+/*
+ * This function is code executed in the child process immediately after fork
+ * to set things up and call exec().
+ *
+ * All of the code in this function must only use async-signal-safe functions,
+ * listed at `man 7 signal` or
+ * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
+ *
+ * This restriction is documented at
+ * http://www.opengroup.org/onlinepubs/009695399/functions/fork.html.
+ */
+static void child_exec(char *const exec_array[],
+                       char *const argv[],
+                       char *const envp[],
+                       const char *cwd,
+                       int p2cread, int p2cwrite,
+                       int c2pread, int c2pwrite,
+                       int errread, int errwrite,
+                       int errpipe_read, int errpipe_write,
+                       int close_fds, int restore_signals,
+                       int call_setsid,
+                       PyObject *preexec_fn,
+                       PyObject *preexec_fn_args_tuple)
+{
+    int i, saved_errno, fd_num;
+    PyObject *result;
+    const char* err_msg;
+    /* Buffer large enough to hold a hex integer.  We can't malloc. */
+    char hex_errno[sizeof(saved_errno)*2+1];
+
+    /* Close parent's pipe ends. */
+    if (p2cwrite != -1) {
+        POSIX_CALL(close(p2cwrite));
+    }
+    if (c2pread != -1) {
+        POSIX_CALL(close(c2pread));
+    }
+    if (errread != -1) {
+        POSIX_CALL(close(errread));
+    }
+    POSIX_CALL(close(errpipe_read));
+
+    /* Dup fds for child. */
+    if (p2cread != -1) {
+        POSIX_CALL(dup2(p2cread, 0));  /* stdin */
+    }
+    if (c2pwrite != -1) {
+        POSIX_CALL(dup2(c2pwrite, 1));  /* stdout */
+    }
+    if (errwrite != -1) {
+        POSIX_CALL(dup2(errwrite, 2));  /* stderr */
+    }
+
+    /* Close pipe fds.  Make sure we don't close the same fd more than */
+    /* once, or standard fds. */
+    if (p2cread != -1 && p2cread != 0) {
+        POSIX_CALL(close(p2cread));
+    }
+    if (c2pwrite != -1 && c2pwrite != p2cread && c2pwrite != 1) {
+        POSIX_CALL(close(c2pwrite));
+    }
+    if (errwrite != -1 && errwrite != p2cread &&
+        errwrite != c2pwrite && errwrite != 2) {
+        POSIX_CALL(close(errwrite));
+    }
+
+    /* close() is intentionally not checked for errors here as we are closing */
+    /* a large range of fds, some of which may be invalid. */
+    if (close_fds) {
+        for (fd_num = 3; fd_num < errpipe_write; ++fd_num) {
+            close(fd_num);
+        }
+        for (fd_num = errpipe_write+1; fd_num < max_fd; ++fd_num) {
+            close(fd_num);
+        }
+    }
+
+    if (cwd)
+        POSIX_CALL(chdir(cwd));
+
+    if (restore_signals)
+        _Py_RestoreSignals();
+
+#ifdef HAVE_SETSID
+    if (call_setsid)
+        POSIX_CALL(setsid());
+#endif
+
+    if (preexec_fn != Py_None && preexec_fn_args_tuple) {
+        /* This is where the user has asked us to deadlock their program. */
+        result = PyObject_Call(preexec_fn, preexec_fn_args_tuple, NULL);
+        if (result == NULL) {
+            /* Stringifying the exception or traceback would involve
+             * memory allocation and thus potential for deadlock.
+             * We've already faced potential deadlock by calling back
+             * into Python in the first place, so it probably doesn't
+             * matter but we avoid it to minimize the possibility. */
+            err_msg = "Exception occurred in preexec_fn.";
+            errno = 0;  /* We don't want to report an OSError. */
+            goto error;
+        }
+        /* Py_DECREF(result); - We're about to exec so why bother? */
+    }
+
+    /* This loop matches the Lib/os.py _execvpe()'s PATH search when */
+    /* given the executable_list generated by Lib/subprocess.py.     */
+    saved_errno = 0;
+    for (i = 0; exec_array[i] != NULL; ++i) {
+        const char *executable = exec_array[i];
+        if (envp) {
+            execve(executable, argv, envp);
+        } else {
+            execv(executable, argv);
+        }
+        if (errno != ENOENT && errno != ENOTDIR && saved_errno == 0) {
+            saved_errno = errno;
+        }
+    }
+    /* Report the first exec error, not the last. */
+    if (saved_errno)
+        errno = saved_errno;
+
+error:
+    saved_errno = errno;
+    /* Report the posix error to our parent process. */
+    if (saved_errno) {
+        char *cur;
+        write(errpipe_write, "OSError:", 8);
+        cur = hex_errno + sizeof(hex_errno);
+        while (saved_errno != 0 && cur > hex_errno) {
+            *--cur = "0123456789ABCDEF"[saved_errno % 16];
+            saved_errno /= 16;
+        }
+        write(errpipe_write, cur, hex_errno + sizeof(hex_errno) - cur);
+        write(errpipe_write, ":", 1);
+        /* We can't call strerror(saved_errno).  It is not async signal safe.
+         * The parent process will look the error message up. */
+    } else {
+        write(errpipe_write, "RuntimeError:0:", 15);
+        write(errpipe_write, err_msg, strlen(err_msg));
+    }
+}
+
+
+static PyObject *
+subprocess_fork_exec(PyObject* self, PyObject *args)
+{
+    PyObject *gc_module = NULL;
+    PyObject *executable_list, *py_close_fds;
+    PyObject *env_list, *preexec_fn;
+    PyObject *process_args = NULL, *converted_args = NULL;
+    PyObject *preexec_fn_args_tuple = NULL;
+    int p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite;
+    int errpipe_read, errpipe_write, close_fds, restore_signals;
+    int call_setsid;
+    const char *cwd;
+    pid_t pid;
+    int need_to_reenable_gc = 0;
+    char *const *exec_array, *const *argv = NULL, *const *envp = NULL;
+    Py_ssize_t arg_num;
+
+    if (!PyArg_ParseTuple(
+            args, "OOOzOiiiiiiiiiiO:fork_exec",
+            &process_args, &executable_list, &py_close_fds, &cwd, &env_list,
+            &p2cread, &p2cwrite, &c2pread, &c2pwrite,
+            &errread, &errwrite, &errpipe_read, &errpipe_write,
+            &restore_signals, &call_setsid, &preexec_fn))
+        return NULL;
+
+    close_fds = PyObject_IsTrue(py_close_fds);
+    if (close_fds && errpipe_write < 3) {  /* precondition */
+        PyErr_SetString(PyExc_ValueError, "errpipe_write must be >= 3");
+        return NULL;
+    }
+
+    /* We need to call gc.disable() when we'll be calling preexec_fn */
+    if (preexec_fn != Py_None) {
+        PyObject *result;
+        gc_module = PyImport_ImportModule("gc");
+        if (gc_module == NULL)
+            return NULL;
+        result = PyObject_CallMethod(gc_module, "isenabled", NULL);
+        if (result == NULL)
+            return NULL;
+        need_to_reenable_gc = PyObject_IsTrue(result);
+        Py_DECREF(result);
+        if (need_to_reenable_gc == -1)
+            return NULL;
+        result = PyObject_CallMethod(gc_module, "disable", NULL);
+        if (result == NULL)
+            return NULL;
+        Py_DECREF(result);
+    }
+
+    exec_array = _PySequence_BytesToCharpArray(executable_list);
+    if (!exec_array)
+        return NULL;
+
+    /* Convert args and env into appropriate arguments for exec() */
+    /* These conversions are done in the parent process to avoid allocating
+       or freeing memory in the child process. */
+    if (process_args != Py_None) {
+        /* Equivalent to:  */
+        /*  tuple(PyUnicode_FSConverter(arg) for arg in process_args)  */
+        process_args = PySequence_Fast(process_args, "argv must be a tuple");
+        converted_args = PyTuple_New(PySequence_Size(process_args));
+        if (converted_args == NULL)
+            goto cleanup;
+        for (arg_num = 0; arg_num < PySequence_Size(process_args); ++arg_num) {
+            PyObject *borrowed_arg, *converted_arg;
+            borrowed_arg = PySequence_Fast_GET_ITEM(process_args, arg_num);
+            if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0)
+                goto cleanup;
+            PyTuple_SET_ITEM(converted_args, arg_num, converted_arg);
+        }
+
+        argv = _PySequence_BytesToCharpArray(converted_args);
+        Py_CLEAR(converted_args);
+        Py_CLEAR(process_args);
+        if (!argv)
+            goto cleanup;
+    }
+
+    if (env_list != Py_None) {
+        envp = _PySequence_BytesToCharpArray(env_list);
+        if (!envp)
+            goto cleanup;
+    }
+
+    if (preexec_fn != Py_None) {
+        preexec_fn_args_tuple = PyTuple_New(0);
+        if (!preexec_fn_args_tuple)
+            goto cleanup;
+	_PyImport_AcquireLock();
+    }
+
+    pid = fork();
+    if (pid == 0) {
+        /* Child process */
+        /* 
+         * Code from here to _exit() must only use async-signal-safe functions,
+         * listed at `man 7 signal` or
+         * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
+         */
+
+        if (preexec_fn != Py_None) {
+            /* We'll be calling back into Python later so we need to do this.
+             * This call may not be async-signal-safe but neither is calling
+             * back into Python.  The user asked us to use hope as a strategy
+             * to avoid deadlock... */
+            PyOS_AfterFork();
+        }
+
+        child_exec(exec_array, argv, envp, cwd,
+                   p2cread, p2cwrite, c2pread, c2pwrite,
+                   errread, errwrite, errpipe_read, errpipe_write,
+                   close_fds, restore_signals, call_setsid,
+                   preexec_fn, preexec_fn_args_tuple);
+        _exit(255);
+        return NULL;  /* Dead code to avoid a potential compiler warning. */
+    }
+    if (pid == -1) {
+        /* Capture the errno exception before errno can be clobbered. */
+        PyErr_SetFromErrno(PyExc_OSError);
+    }
+    if (preexec_fn != Py_None &&
+        _PyImport_ReleaseLock() < 0 && !PyErr_Occurred()) {
+        PyErr_SetString(PyExc_RuntimeError,
+                        "not holding the import lock");
+    }
+
+    /* Parent process */
+    if (envp)
+        _Py_FreeCharPArray(envp);
+    if (argv)
+        _Py_FreeCharPArray(argv);
+    _Py_FreeCharPArray(exec_array);
+
+    /* Reenable gc in the parent process (or if fork failed). */
+    if (need_to_reenable_gc && _enable_gc(gc_module)) {
+        Py_XDECREF(gc_module);
+        return NULL;
+    }
+    Py_XDECREF(gc_module);
+
+    if (pid == -1)
+        return NULL;  /* fork() failed.  Exception set earlier. */
+
+    return PyLong_FromPid(pid);
+
+cleanup:
+    if (envp)
+        _Py_FreeCharPArray(envp);
+    if (argv)
+        _Py_FreeCharPArray(argv);
+    _Py_FreeCharPArray(exec_array);
+    Py_XDECREF(converted_args);
+    Py_XDECREF(process_args);
+
+    /* Reenable gc if it was disabled. */
+    if (need_to_reenable_gc)
+        _enable_gc(gc_module);
+    Py_XDECREF(gc_module);
+    return NULL;
+}
+
+
+PyDoc_STRVAR(subprocess_fork_exec_doc,
+"fork_exec(args, executable_list, close_fds, cwd, env,\n\
+          p2cread, p2cwrite, c2pread, c2pwrite,\n\
+          errread, errwrite, errpipe_read, errpipe_write,\n\
+          restore_signals, call_setsid, preexec_fn)\n\
+\n\
+Forks a child process, closes parent file descriptors as appropriate in the\n\
+child and dups the few that are needed before calling exec() in the child\n\
+process.\n\
+\n\
+The preexec_fn, if supplied, will be called immediately before exec.\n\
+WARNING: preexec_fn is NOT SAFE if your application uses threads.\n\
+         It may trigger infrequent, difficult to debug deadlocks.\n\
+\n\
+If an error occurs in the child process before the exec, it is\n\
+serialized and written to the errpipe_write fd per subprocess.py.\n\
+\n\
+Returns: the child process's PID.\n\
+\n\
+Raises: Only on an error in the parent process.\n\
+");
+
+
+/* module level code ********************************************************/
+
+PyDoc_STRVAR(module_doc,
+"A POSIX helper for the subprocess module.");
+
+
+static PyMethodDef module_methods[] = {
+    {"fork_exec", subprocess_fork_exec, METH_VARARGS, subprocess_fork_exec_doc},
+    {NULL, NULL}  /* sentinel */
+};
+
+
+static struct PyModuleDef _posixsubprocessmodule = {
+	PyModuleDef_HEAD_INIT,
+	"_posixsubprocess",
+	module_doc,
+	-1,  /* No memory is needed. */
+	module_methods,
+};
+
+PyMODINIT_FUNC
+PyInit__posixsubprocess(void)
+{
+#ifdef _SC_OPEN_MAX
+    max_fd = sysconf(_SC_OPEN_MAX);
+    if (max_fd == -1)
+#endif
+        max_fd = 256;  /* Matches Lib/subprocess.py */
+
+    return PyModule_Create(&_posixsubprocessmodule);
+}

Modules/posixmodule.c

@@ -303,23 +303,6 @@ extern int lstat(const char *, struct stat *);
 #define WAIT_STATUS_INT(s) (s)
 #endif /* UNION_WAIT */
 
-/* Issue #1983: pid_t can be longer than a C long on some systems */
-#if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT
-#define PARSE_PID "i"
-#define PyLong_FromPid PyLong_FromLong
-#define PyLong_AsPid PyLong_AsLong
-#elif SIZEOF_PID_T == SIZEOF_LONG
-#define PARSE_PID "l"
-#define PyLong_FromPid PyLong_FromLong
-#define PyLong_AsPid PyLong_AsLong
-#elif defined(SIZEOF_LONG_LONG) && SIZEOF_PID_T == SIZEOF_LONG_LONG
-#define PARSE_PID "L"
-#define PyLong_FromPid PyLong_FromLongLong
-#define PyLong_AsPid PyLong_AsLongLong
-#else
-#error "sizeof(pid_t) is neither sizeof(int), sizeof(long) or sizeof(long long)"
-#endif /* SIZEOF_PID_T */
-
 /* Don't use the "_r" form if we don't need it (also, won't have a
    prototype for it, at least on Solaris -- maybe others as well?). */
 #if defined(HAVE_CTERMID_R) && defined(WITH_THREAD)

Objects/abstract.c

@@ -2722,3 +2722,63 @@ PyIter_Next(PyObject *iter)
 		PyErr_Clear();
 	return result;
 }
+
+
+/*
+ * Flatten a sequence of bytes() objects into a C array of
+ * NULL terminated string pointers with a NULL char* terminating the array.
+ * (ie: an argv or env list)
+ *
+ * Memory allocated for the returned list is allocated using malloc() and MUST
+ * be freed by the caller using a free() loop or _Py_FreeCharPArray().
+ */
+char *const *
+_PySequence_BytesToCharpArray(PyObject* self)
+{
+	char **array;
+	Py_ssize_t i, argc;
+
+	argc = PySequence_Size(self);
+	if (argc == -1)
+		return NULL;
+
+	array = malloc((argc + 1) * sizeof(char *));
+	if (array == NULL) {
+		PyErr_NoMemory();
+		return NULL;
+	}
+	for (i = 0; i < argc; ++i) {
+		char *data;
+		PyObject *item = PySequence_GetItem(self, i);
+		data = PyBytes_AsString(item);
+		if (data == NULL) {
+			/* NULL terminate before freeing. */
+			array[i] = NULL;
+			goto fail;
+		}
+		array[i] = strdup(data);
+		if (!array[i]) {
+			PyErr_NoMemory();
+			goto fail;
+		}
+	}
+	array[argc] = NULL;
+
+	return array;
+
+fail:
+	_Py_FreeCharPArray(array);
+	return NULL;
+}
+
+
+/* Free's a NULL terminated char** array of C strings. */
+void
+_Py_FreeCharPArray(char *const array[])
+{
+	Py_ssize_t i;
+	for (i = 0; array[i] != NULL; ++i) {
+		free(array[i]);
+	}
+	free((void*)array);
+}

Python/pythonrun.c

@@ -2130,6 +2130,27 @@ initsigs(void)
 }
 
 
+/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL.
+ *
+ * All of the code in this function must only use async-signal-safe functions,
+ * listed at `man 7 signal` or
+ * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
+ */
+void
+_Py_RestoreSignals(void)
+{
+#ifdef SIGPIPE
+	PyOS_setsig(SIGPIPE, SIG_DFL);
+#endif
+#ifdef SIGXFZ
+	PyOS_setsig(SIGXFZ, SIG_DFL);
+#endif
+#ifdef SIGXFSZ
+	PyOS_setsig(SIGXFSZ, SIG_DFL);
+#endif
+}
+
+
 /*
  * The file descriptor fd is considered ``interactive'' if either
  *   a) isatty(fd) is TRUE, or
@@ -2223,6 +2244,11 @@ PyOS_getsig(int sig)
 #endif
 }
 
+/*
+ * All of the code in this function must only use async-signal-safe functions,
+ * listed at `man 7 signal` or
+ * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
+ */
 PyOS_sighandler_t
 PyOS_setsig(int sig, PyOS_sighandler_t handler)
 {

setup.py

@@ -546,6 +546,9 @@ class PyBuildExt(build_ext):
         # CSV files
         exts.append( Extension('_csv', ['_csv.c']) )
 
+        # POSIX subprocess module helper.
+        exts.append( Extension('_posixsubprocess', ['_posixsubprocess.c']) )
+
         # socket(2)
         exts.append( Extension('_socket', ['socketmodule.c'],
                                depends = ['socketmodule.h']) )