Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

* Replaces the internals of the subprocess module from fork through exec on 

POSIX systems with a C extension module.  This is required in order for
  the subprocess module to be made thread safe.

  The pure python implementation is retained so that it can continue to be
  used if for some reason the _posixsubprocess extension module is not
  available.

  The unittest executes tests on both code paths to guarantee compatibility.

* Moves PyLong_FromPid and PyLong_AsPid from posixmodule.c into longobject.h.

Code reviewed by jeffrey.yasskin at http://codereview.appspot.com/223077/show
This commit is contained in:
Gregory P. Smith 2010-03-14 06:49:55 +00:00
parent dddd5e9098
commit fb94c5f1e5
11 changed files with 816 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
Doc/library/subprocess.rst
View File

@ -28,7 +28,7 @@ Using the subprocess Module
This module defines one class called :class:`Popen`:
.. class:: Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0)
.. class:: Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False)
Arguments are:
@ -41,7 +41,8 @@ This module defines one class called :class:`Popen`:
name for the executing program in utilities such as :program:`ps`.
On Unix, with *shell=False* (default): In this case, the Popen class uses
:meth:`os.execvp` to execute the child program. *args* should normally be a
:meth:`os.execvp` like behavior to execute the child program.
*args* should normally be a
sequence. If a string is specified for *args*, it will be used as the name
or path of the program to execute; this will only work if the program is
being given no arguments.
@ -108,7 +109,23 @@ This module defines one class called :class:`Popen`:
applications should be captured into the same file handle as for stdout.
If *preexec_fn* is set to a callable object, this object will be called in the
child process just before the child is executed. (Unix only)
child process just before the child is executed.
(Unix only)
.. warning::
The *preexec_fn* parameter is not safe to use in the presence of threads
in your application. The child process could deadlock before exec is
called.
If you must use it, keep it trivial! Minimize the number of libraries
you call into.
.. note::
If you need to modify the environment for the child use the *env*
parameter rather than doing it in a *preexec_fn*.
The *start_new_session* parameter can take the place of a previously
common use of *preexec_fn* to call os.setsid() in the child.
If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and
:const:`2` will be closed before the child process is executed. (Unix only).
@ -124,9 +141,23 @@ This module defines one class called :class:`Popen`:
searching the executable, so you can't specify the program's path relative to
*cwd*.
If *restore_signals* is True (the default) all signals that Python has set to
SIG_IGN are restored to SIG_DFL in the child process before the exec.
Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals.
(Unix only)
.. versionchanged:: 3.2
*restore_signals* was added.
If *start_new_session* is True the setsid() system call will be made in the
child process prior to the execution of the subprocess. (Unix only)
.. versionchanged:: 3.2
*start_new_session* was added.
If *env* is not ``None``, it must be a mapping that defines the environment
variables for the new process; these are used instead of inheriting the current
process' environment, which is the default behavior.
variables for the new process; these are used instead of the default
behavior of inheriting the current process' environment.
.. note::

3
Include/abstract.h
View File

@ -1232,6 +1232,9 @@ PyAPI_FUNC(int) _PyObject_RealIsInstance(PyObject *inst, PyObject *cls);
PyAPI_FUNC(int) _PyObject_RealIsSubclass(PyObject *derived, PyObject *cls);
PyAPI_FUNC(char *const *) _PySequence_BytesToCharpArray(PyObject* self);
PyAPI_FUNC(void) _Py_FreeCharPArray(char *const array[]);
#ifdef __cplusplus
}

17
Include/longobject.h
View File

@ -41,6 +41,23 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void);
#define PyLong_AsSocket_t(fd) (SOCKET_T)PyLong_AsLongLong(fd)
#endif
/* Issue #1983: pid_t can be longer than a C long on some systems */
#if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT
#define PARSE_PID "i"
#define PyLong_FromPid PyLong_FromLong
#define PyLong_AsPid PyLong_AsLong
#elif SIZEOF_PID_T == SIZEOF_LONG
#define PARSE_PID "l"
#define PyLong_FromPid PyLong_FromLong
#define PyLong_AsPid PyLong_AsLong
#elif defined(SIZEOF_LONG_LONG) && SIZEOF_PID_T == SIZEOF_LONG_LONG
#define PARSE_PID "L"
#define PyLong_FromPid PyLong_FromLongLong
#define PyLong_AsPid PyLong_AsLongLong
#else
#error "sizeof(pid_t) is neither sizeof(int), sizeof(long) or sizeof(long long)"
#endif /* SIZEOF_PID_T */
/* For use by intobject.c only */
PyAPI_DATA(unsigned char) _PyLong_DigitValue[256];

3
Include/pythonrun.h
View File

@ -81,6 +81,9 @@ PyAPI_FUNC(int) Py_AtExit(void (*func)(void));
PyAPI_FUNC(void) Py_Exit(int);
/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */
PyAPI_FUNC(void) _Py_RestoreSignals(void);
PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
/* Bootstrap */

305
Lib/subprocess.py
View File

@ -29,7 +29,8 @@ class Popen(args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0):
startupinfo=None, creationflags=0,
restore_signals=True, start_new_session=False):
Arguments are:
@ -72,8 +73,11 @@ parent. Additionally, stderr can be STDOUT, which indicates that the
stderr data from the applications should be captured into the same
file handle as for stdout.
If preexec_fn is set to a callable object, this object will be called
in the child process just before the child is executed.
On UNIX, if preexec_fn is set to a callable object, this object will be
called in the child process just before the child is executed. The use
of preexec_fn is not thread safe, using it in the presence of threads
could lead to a deadlock in the child process before the new executable
is executed.
If close_fds is true, all file descriptors except 0, 1 and 2 will be
closed before the child process is executed.
@ -84,6 +88,14 @@ shell.
If cwd is not None, the current directory will be changed to cwd
before the child is executed.
On UNIX, if restore_signals is True all signals that Python sets to
SIG_IGN are restored to SIG_DFL in the child process before the exec.
Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. This
parameter does nothing on Windows.
On UNIX, if start_new_session is True, the setsid() system call will be made
in the child process prior to executing the command.
If env is not None, it defines the environment variables for the new
process.
@ -326,6 +338,7 @@ import os
import traceback
import gc
import signal
import builtins
# Exception classes used by this module.
class CalledProcessError(Exception):
@ -375,6 +388,15 @@ else:
import fcntl
import pickle
try:
import _posixsubprocess
except ImportError:
_posixsubprocess = None
import warnings
warnings.warn("The _posixsubprocess module is not being used. "
"Child process reliability may suffer if your "
"program uses threads.", RuntimeWarning)
# When select or poll has indicated that the file is writable,
# we can write up to _PIPE_BUF bytes without risk of blocking.
# POSIX defines PIPE_BUF as >= 512.
@ -596,7 +618,8 @@ class Popen(object):
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0):
startupinfo=None, creationflags=0,
restore_signals=True, start_new_session=False):
"""Create new Popen instance."""
_cleanup()
@ -642,7 +665,7 @@ class Popen(object):
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# are -1 when not using PIPEs. The child objects are -1
# when not redirecting.
(p2cread, p2cwrite,
@ -654,7 +677,8 @@ class Popen(object):
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
errread, errwrite,
restore_signals, start_new_session)
if mswindows:
if p2cwrite is not None:
@ -666,15 +690,15 @@ class Popen(object):
if bufsize == 0:
bufsize = 1 # Nearly unbuffered (XXX for now)
if p2cwrite is not None:
if p2cwrite != -1:
self.stdin = io.open(p2cwrite, 'wb', bufsize)
if self.universal_newlines:
self.stdin = io.TextIOWrapper(self.stdin)
if c2pread is not None:
if c2pread != -1:
self.stdout = io.open(c2pread, 'rb', bufsize)
if universal_newlines:
self.stdout = io.TextIOWrapper(self.stdout)
if errread is not None:
if errread != -1:
self.stderr = io.open(errread, 'rb', bufsize)
if universal_newlines:
self.stderr = io.TextIOWrapper(self.stderr)
@ -739,11 +763,11 @@ class Popen(object):
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
if stdin is None and stdout is None and stderr is None:
return (None, None, None, None, None, None)
return (-1, -1, -1, -1, -1, -1)
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
p2cread, p2cwrite = -1, -1
c2pread, c2pwrite = -1, -1
errread, errwrite = -1, -1
if stdin is None:
p2cread = GetStdHandle(STD_INPUT_HANDLE)
@ -819,7 +843,8 @@ class Popen(object):
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
errread, errwrite,
unused_restore_signals, unused_start_new_session):
"""Execute program (MS Windows version)"""
if not isinstance(args, str):
@ -973,9 +998,9 @@ class Popen(object):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
p2cread, p2cwrite = -1, -1
c2pread, c2pwrite = -1, -1
errread, errwrite = -1, -1
if stdin is None:
pass
@ -1034,7 +1059,8 @@ class Popen(object):
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if isinstance(args, str):
@ -1048,113 +1074,190 @@ class Popen(object):
if executable is None:
executable = args[0]
# For transferring possible exec failure from child to parent
# The first char specifies the exception type: 0 means
# OSError, 1 means some other error.
# For transferring possible exec failure from child to parent.
# Data format: "exception name:hex errno:description"
# Pickle is not used; it is complex and involves memory allocation.
errpipe_read, errpipe_write = os.pipe()
try:
try:
self._set_cloexec_flag(errpipe_write)
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = os.fork()
except:
if _posixsubprocess:
fs_encoding = sys.getfilesystemencoding()
def fs_encode(s):
"""Encode s for use in the env, fs or cmdline."""
return s.encode(fs_encoding, 'surrogateescape')
# We must avoid complex work that could involve
# malloc or free in the child process to avoid
# potential deadlocks, thus we do all this here.
# and pass it to fork_exec()
if env:
env_list = [fs_encode(k) + b'=' + fs_encode(v)
for k, v in env.items()]
else:
env_list = None # Use execv instead of execve.
if os.path.dirname(executable):
executable_list = (fs_encode(executable),)
else:
# This matches the behavior of os._execvpe().
path_list = os.get_exec_path(env)
executable_list = (os.path.join(dir, executable)
for dir in path_list)
executable_list = tuple(fs_encode(exe)
for exe in executable_list)
self.pid = _posixsubprocess.fork_exec(
args, executable_list,
close_fds, cwd, env_list,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session, preexec_fn)
else:
# Pure Python implementation: It is not thread safe.
# This implementation may deadlock in the child if your
# parent process has any other threads running.
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. See issue1336
gc.disable()
try:
self.pid = os.fork()
except:
if gc_was_enabled:
gc.enable()
raise
self._child_created = True
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite != -1:
os.close(p2cwrite)
if c2pread != -1:
os.close(c2pread)
if errread != -1:
os.close(errread)
os.close(errpipe_read)
# Dup fds for child
if p2cread != -1:
os.dup2(p2cread, 0)
if c2pwrite != -1:
os.dup2(c2pwrite, 1)
if errwrite != -1:
os.dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the
# same fd more than once, or standard fds.
if p2cread != -1 and p2cread not in (0,):
os.close(p2cread)
if (c2pwrite != -1 and
c2pwrite not in (p2cread, 1)):
os.close(c2pwrite)
if (errwrite != -1 and
errwrite not in (p2cread, c2pwrite, 2)):
os.close(errwrite)
# Close all other fds, if asked for
if close_fds:
self._close_fds(but=errpipe_write)
if cwd is not None:
os.chdir(cwd)
# This is a copy of Python/pythonrun.c
# _Py_RestoreSignals(). If that were exposed
# as a sys._py_restoresignals func it would be
# better.. but this pure python implementation
# isn't likely to be used much anymore.
if restore_signals:
signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ')
for sig in signals:
if hasattr(signal, sig):
signal.signal(getattr(signal, sig),
signal.SIG_DFL)
if start_new_session and hasattr(os, 'setsid'):
os.setsid()
if preexec_fn:
preexec_fn()
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
try:
exc_type, exc_value = sys.exc_info()[:2]
if isinstance(exc_value, OSError):
errno = exc_value.errno
else:
errno = 0
message = '%s:%x:%s' % (exc_type.__name__,
errno, exc_value)
os.write(errpipe_write, message.encode())
except:
# We MUST not allow anything odd happening
# above to prevent us from exiting below.
pass
# This exitcode won't be reported to applications
# so it really doesn't matter what we return.
os._exit(255)
# Parent
if gc_was_enabled:
gc.enable()
raise
self._child_created = True
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
# Dup fds for child
if p2cread is not None:
os.dup2(p2cread, 0)
if c2pwrite is not None:
os.dup2(c2pwrite, 1)
if errwrite is not None:
os.dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the
# same fd more than once, or standard fds.
if p2cread is not None and p2cread not in (0,):
os.close(p2cread)
if c2pwrite is not None and \
c2pwrite not in (p2cread, 1):
os.close(c2pwrite)
if (errwrite is not None and
errwrite not in (p2cread, c2pwrite, 2)):
os.close(errwrite)
# Close all other fds, if asked for
if close_fds:
self._close_fds(but=errpipe_write)
if cwd is not None:
os.chdir(cwd)
if preexec_fn:
preexec_fn()
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
exc_type, exc_value, tb = sys.exc_info()
# Save the traceback and attach it to the exception
# object
exc_lines = traceback.format_exception(exc_type,
exc_value,
tb)
exc_value.child_traceback = ''.join(exc_lines)
os.write(errpipe_write, pickle.dumps(exc_value))
# This exitcode won't be reported to applications, so
# it really doesn't matter what we return.
os._exit(255)
# Parent
if gc_was_enabled:
gc.enable()
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
if p2cread is not None and p2cwrite is not None:
if p2cread != -1 and p2cwrite != -1:
os.close(p2cread)
if c2pwrite is not None and c2pread is not None:
if c2pwrite != -1 and c2pread != -1:
os.close(c2pwrite)
if errwrite is not None and errread is not None:
if errwrite != -1 and errread != -1:
os.close(errwrite)
# Wait for exec to fail or succeed; possibly raising an
# exception (limited to 1 MB)
data = _eintr_retry_call(os.read, errpipe_read, 1048576)
# exception (limited in size)
data = bytearray()
while True:
part = _eintr_retry_call(os.read, errpipe_read, 50000)
data += part
if not part or len(data) > 50000:
break
finally:
# be sure the FD is closed no matter what
os.close(errpipe_read)
if data:
_eintr_retry_call(os.waitpid, self.pid, 0)
child_exception = pickle.loads(data)
try:
exception_name, hex_errno, err_msg = data.split(b':', 2)
except ValueError:
print('Bad exception data:', repr(data))
exception_name = b'RuntimeError'
hex_errno = b'0'
err_msg = b'Unknown'
child_exception_type = getattr(
builtins, exception_name.decode('ascii'),
RuntimeError)
for fd in (p2cwrite, c2pread, errread):
if fd is not None:
if fd != -1:
os.close(fd)
raise child_exception
err_msg = err_msg.decode()
if issubclass(child_exception_type, OSError) and hex_errno:
errno = int(hex_errno, 16)
if errno != 0:
err_msg = os.strerror(errno)
raise child_exception_type(errno, err_msg)
raise child_exception_type(err_msg)
def _handle_exitstatus(self, sts):

85
Lib/test/test_subprocess.py
View File

@ -568,12 +568,53 @@ class POSIXProcessTestCase(unittest.TestCase):
self.assertFalse(subprocess._active, "subprocess._active not empty")
def test_exceptions(self):
# caught & re-raised exceptions
with self.assertRaises(OSError) as c:
nonexistent_dir = "/_this/pa.th/does/not/exist"
try:
os.chdir(nonexistent_dir)
except OSError as e:
# This avoids hard coding the errno value or the OS perror()
# string and instead capture the exception that we want to see
# below for comparison.
desired_exception = e
else:
self.fail("chdir to nonexistant directory %s succeeded." %
nonexistent_dir)
# Error in the child re-raised in the parent.
try:
p = subprocess.Popen([sys.executable, "-c", ""],
cwd="/this/path/does/not/exist")
# The attribute child_traceback should contain "os.chdir" somewhere.
self.assertIn("os.chdir", c.exception.child_traceback)
cwd=nonexistent_dir)
except OSError as e:
# Test that the child process chdir failure actually makes
# it up to the parent process as the correct exception.
self.assertEqual(desired_exception.errno, e.errno)
self.assertEqual(desired_exception.strerror, e.strerror)
else:
self.fail("Expected OSError: %s" % desired_exception)
def test_restore_signals(self):
# Code coverage for both values of restore_signals to make sure it
# at least does not blow up.
# A test for behavior would be complex. Contributions welcome.
subprocess.call([sys.executable, "-c", ""], restore_signals=True)
subprocess.call([sys.executable, "-c", ""], restore_signals=False)
def test_start_new_session(self):
# For code coverage of calling setsid(). We don't care if we get an
# EPERM error from it depending on the test execution environment, that
# still indicates that it was called.
try:
output = subprocess.check_output(
[sys.executable, "-c",
"import os; print(os.getpgid(os.getpid()))"],
start_new_session=True)
except OSError as e:
if e.errno != errno.EPERM:
raise
else:
parent_pgid = os.getpgid(os.getpid())
child_pgid = int(output)
self.assertNotEqual(parent_pgid, child_pgid)
def test_run_abort(self):
# returncode handles signal termination
@ -584,7 +625,8 @@ class POSIXProcessTestCase(unittest.TestCase):
self.assertEqual(-p.returncode, signal.SIGABRT)
def test_preexec(self):
# preexec function
# DISCLAIMER: Setting environment variables is *not* a good use
# of a preexec_fn. This is merely a test.
p = subprocess.Popen([sys.executable, "-c",
'import sys,os;'
'sys.stdout.write(os.getenv("FRUIT"))'],
@ -592,6 +634,22 @@ class POSIXProcessTestCase(unittest.TestCase):
preexec_fn=lambda: os.putenv("FRUIT", "apple"))
self.assertEqual(p.stdout.read(), b"apple")
def test_preexec_exception(self):
def raise_it():
raise ValueError("What if two swallows carried a coconut?")
try:
p = subprocess.Popen([sys.executable, "-c", ""],
preexec_fn=raise_it)
except RuntimeError as e:
self.assertTrue(
subprocess._posixsubprocess,
"Expected a ValueError from the preexec_fn")
except ValueError as e:
self.assertIn("coconut", e.args[0])
else:
self.fail("Exception raised by preexec_fn did not make it "
"to the parent process.")
def test_args_string(self):
# args is a string
fd, fname = mkstemp()
@ -836,6 +894,20 @@ class ProcessTestCaseNoPoll(ProcessTestCase):
ProcessTestCase.tearDown(self)
@unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False),
"_posixsubprocess extension module not found.")
class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase):
def setUp(self):
subprocess._posixsubprocess = None
ProcessTestCase.setUp(self)
POSIXProcessTestCase.setUp(self)
def tearDown(self):
subprocess._posixsubprocess = sys.modules['_posixsubprocess']
POSIXProcessTestCase.tearDown(self)
ProcessTestCase.tearDown(self)
class HelperFunctionTests(unittest.TestCase):
@unittest.skipIf(mswindows, "errno and EINTR make no sense on windows")
def test_eintr_retry_call(self):
@ -859,6 +931,7 @@ def test_main():
unit_tests = (ProcessTestCase,
POSIXProcessTestCase,
Win32ProcessTestCase,
ProcessTestCasePOSIXPurePython,
CommandTests,
ProcessTestCaseNoPoll,
HelperFunctionTests)

385
Modules/_posixsubprocess.c Normal file
View File

@ -0,0 +1,385 @@
/* Authors: Gregory P. Smith & Jeffrey Yasskin */
#include "Python.h"
#include <unistd.h>
#define POSIX_CALL(call) if ((call) == -1) goto error
/* Maximum file descriptor, initialized on module load. */
static long max_fd;
/* Given the gc module call gc.enable() and return 0 on success. */
static int _enable_gc(PyObject *gc_module)
{
PyObject *result;
result = PyObject_CallMethod(gc_module, "enable", NULL);
if (result == NULL)
return 1;
Py_DECREF(result);
return 0;
}
/*
* This function is code executed in the child process immediately after fork
* to set things up and call exec().
*
* All of the code in this function must only use async-signal-safe functions,
* listed at `man 7 signal` or
* http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
*
* This restriction is documented at
* http://www.opengroup.org/onlinepubs/009695399/functions/fork.html.
*/
static void child_exec(char *const exec_array[],
char *const argv[],
char *const envp[],
const char *cwd,
int p2cread, int p2cwrite,
int c2pread, int c2pwrite,
int errread, int errwrite,
int errpipe_read, int errpipe_write,
int close_fds, int restore_signals,
int call_setsid,
PyObject *preexec_fn,
PyObject *preexec_fn_args_tuple)
{
int i, saved_errno, fd_num;
PyObject *result;
const char* err_msg;
/* Buffer large enough to hold a hex integer. We can't malloc. */
char hex_errno[sizeof(saved_errno)*2+1];
/* Close parent's pipe ends. */
if (p2cwrite != -1) {
POSIX_CALL(close(p2cwrite));
}
if (c2pread != -1) {
POSIX_CALL(close(c2pread));
}
if (errread != -1) {
POSIX_CALL(close(errread));
}
POSIX_CALL(close(errpipe_read));
/* Dup fds for child. */
if (p2cread != -1) {
POSIX_CALL(dup2(p2cread, 0)); /* stdin */
}
if (c2pwrite != -1) {
POSIX_CALL(dup2(c2pwrite, 1)); /* stdout */
}
if (errwrite != -1) {
POSIX_CALL(dup2(errwrite, 2)); /* stderr */
}
/* Close pipe fds. Make sure we don't close the same fd more than */
/* once, or standard fds. */
if (p2cread != -1 && p2cread != 0) {
POSIX_CALL(close(p2cread));
}
if (c2pwrite != -1 && c2pwrite != p2cread && c2pwrite != 1) {
POSIX_CALL(close(c2pwrite));
}
if (errwrite != -1 && errwrite != p2cread &&
errwrite != c2pwrite && errwrite != 2) {
POSIX_CALL(close(errwrite));
}
/* close() is intentionally not checked for errors here as we are closing */
/* a large range of fds, some of which may be invalid. */
if (close_fds) {
for (fd_num = 3; fd_num < errpipe_write; ++fd_num) {
close(fd_num);
}
for (fd_num = errpipe_write+1; fd_num < max_fd; ++fd_num) {
close(fd_num);
}
}
if (cwd)
POSIX_CALL(chdir(cwd));
if (restore_signals)
_Py_RestoreSignals();
#ifdef HAVE_SETSID
if (call_setsid)
POSIX_CALL(setsid());
#endif
if (preexec_fn != Py_None && preexec_fn_args_tuple) {
/* This is where the user has asked us to deadlock their program. */
result = PyObject_Call(preexec_fn, preexec_fn_args_tuple, NULL);
if (result == NULL) {
/* Stringifying the exception or traceback would involve
* memory allocation and thus potential for deadlock.
* We've already faced potential deadlock by calling back
* into Python in the first place, so it probably doesn't
* matter but we avoid it to minimize the possibility. */
err_msg = "Exception occurred in preexec_fn.";
errno = 0; /* We don't want to report an OSError. */
goto error;
}
/* Py_DECREF(result); - We're about to exec so why bother? */
}
/* This loop matches the Lib/os.py _execvpe()'s PATH search when */
/* given the executable_list generated by Lib/subprocess.py. */
saved_errno = 0;
for (i = 0; exec_array[i] != NULL; ++i) {
const char *executable = exec_array[i];
if (envp) {
execve(executable, argv, envp);
} else {
execv(executable, argv);
}
if (errno != ENOENT && errno != ENOTDIR && saved_errno == 0) {
saved_errno = errno;
}
}
/* Report the first exec error, not the last. */
if (saved_errno)
errno = saved_errno;
error:
saved_errno = errno;
/* Report the posix error to our parent process. */
if (saved_errno) {
char *cur;
write(errpipe_write, "OSError:", 8);
cur = hex_errno + sizeof(hex_errno);
while (saved_errno != 0 && cur > hex_errno) {
*--cur = "0123456789ABCDEF"[saved_errno % 16];
saved_errno /= 16;
}
write(errpipe_write, cur, hex_errno + sizeof(hex_errno) - cur);
write(errpipe_write, ":", 1);
/* We can't call strerror(saved_errno). It is not async signal safe.
* The parent process will look the error message up. */
} else {
write(errpipe_write, "RuntimeError:0:", 15);
write(errpipe_write, err_msg, strlen(err_msg));
}
}
static PyObject *
subprocess_fork_exec(PyObject* self, PyObject *args)
{
PyObject *gc_module = NULL;
PyObject *executable_list, *py_close_fds;
PyObject *env_list, *preexec_fn;
PyObject *process_args = NULL, *converted_args = NULL;
PyObject *preexec_fn_args_tuple = NULL;
int p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite;
int errpipe_read, errpipe_write, close_fds, restore_signals;
int call_setsid;
const char *cwd;
pid_t pid;
int need_to_reenable_gc = 0;
char *const *exec_array, *const *argv = NULL, *const *envp = NULL;
Py_ssize_t arg_num;
if (!PyArg_ParseTuple(
args, "OOOzOiiiiiiiiiiO:fork_exec",
&process_args, &executable_list, &py_close_fds, &cwd, &env_list,
&p2cread, &p2cwrite, &c2pread, &c2pwrite,
&errread, &errwrite, &errpipe_read, &errpipe_write,
&restore_signals, &call_setsid, &preexec_fn))
return NULL;
close_fds = PyObject_IsTrue(py_close_fds);
if (close_fds && errpipe_write < 3) { /* precondition */
PyErr_SetString(PyExc_ValueError, "errpipe_write must be >= 3");
return NULL;
}
/* We need to call gc.disable() when we'll be calling preexec_fn */
if (preexec_fn != Py_None) {
PyObject *result;
gc_module = PyImport_ImportModule("gc");
if (gc_module == NULL)
return NULL;
result = PyObject_CallMethod(gc_module, "isenabled", NULL);
if (result == NULL)
return NULL;
need_to_reenable_gc = PyObject_IsTrue(result);
Py_DECREF(result);
if (need_to_reenable_gc == -1)
return NULL;
result = PyObject_CallMethod(gc_module, "disable", NULL);
if (result == NULL)
return NULL;
Py_DECREF(result);
}
exec_array = _PySequence_BytesToCharpArray(executable_list);
if (!exec_array)
return NULL;
/* Convert args and env into appropriate arguments for exec() */
/* These conversions are done in the parent process to avoid allocating
or freeing memory in the child process. */
if (process_args != Py_None) {
/* Equivalent to: */
/* tuple(PyUnicode_FSConverter(arg) for arg in process_args) */
process_args = PySequence_Fast(process_args, "argv must be a tuple");
converted_args = PyTuple_New(PySequence_Size(process_args));
if (converted_args == NULL)
goto cleanup;
for (arg_num = 0; arg_num < PySequence_Size(process_args); ++arg_num) {
PyObject *borrowed_arg, *converted_arg;
borrowed_arg = PySequence_Fast_GET_ITEM(process_args, arg_num);
if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0)
goto cleanup;
PyTuple_SET_ITEM(converted_args, arg_num, converted_arg);
}
argv = _PySequence_BytesToCharpArray(converted_args);
Py_CLEAR(converted_args);
Py_CLEAR(process_args);
if (!argv)
goto cleanup;
}
if (env_list != Py_None) {
envp = _PySequence_BytesToCharpArray(env_list);
if (!envp)
goto cleanup;
}
if (preexec_fn != Py_None) {
preexec_fn_args_tuple = PyTuple_New(0);
if (!preexec_fn_args_tuple)
goto cleanup;
_PyImport_AcquireLock();
}
pid = fork();
if (pid == 0) {
/* Child process */
/*
* Code from here to _exit() must only use async-signal-safe functions,
* listed at `man 7 signal` or
* http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
*/
if (preexec_fn != Py_None) {
/* We'll be calling back into Python later so we need to do this.
* This call may not be async-signal-safe but neither is calling
* back into Python. The user asked us to use hope as a strategy
* to avoid deadlock... */
PyOS_AfterFork();
}
child_exec(exec_array, argv, envp, cwd,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite, errpipe_read, errpipe_write,
close_fds, restore_signals, call_setsid,
preexec_fn, preexec_fn_args_tuple);
_exit(255);
return NULL; /* Dead code to avoid a potential compiler warning. */
}
if (pid == -1) {
/* Capture the errno exception before errno can be clobbered. */
PyErr_SetFromErrno(PyExc_OSError);
}
if (preexec_fn != Py_None &&
_PyImport_ReleaseLock() < 0 && !PyErr_Occurred()) {
PyErr_SetString(PyExc_RuntimeError,
"not holding the import lock");
}
/* Parent process */
if (envp)
_Py_FreeCharPArray(envp);
if (argv)
_Py_FreeCharPArray(argv);
_Py_FreeCharPArray(exec_array);
/* Reenable gc in the parent process (or if fork failed). */
if (need_to_reenable_gc && _enable_gc(gc_module)) {
Py_XDECREF(gc_module);
return NULL;
}
Py_XDECREF(gc_module);
if (pid == -1)
return NULL; /* fork() failed. Exception set earlier. */
return PyLong_FromPid(pid);
cleanup:
if (envp)
_Py_FreeCharPArray(envp);
if (argv)
_Py_FreeCharPArray(argv);
_Py_FreeCharPArray(exec_array);
Py_XDECREF(converted_args);
Py_XDECREF(process_args);
/* Reenable gc if it was disabled. */
if (need_to_reenable_gc)
_enable_gc(gc_module);
Py_XDECREF(gc_module);
return NULL;
}
PyDoc_STRVAR(subprocess_fork_exec_doc,
"fork_exec(args, executable_list, close_fds, cwd, env,\n\
p2cread, p2cwrite, c2pread, c2pwrite,\n\
errread, errwrite, errpipe_read, errpipe_write,\n\
restore_signals, call_setsid, preexec_fn)\n\
\n\
Forks a child process, closes parent file descriptors as appropriate in the\n\
child and dups the few that are needed before calling exec() in the child\n\
process.\n\
\n\
The preexec_fn, if supplied, will be called immediately before exec.\n\
WARNING: preexec_fn is NOT SAFE if your application uses threads.\n\
It may trigger infrequent, difficult to debug deadlocks.\n\
\n\
If an error occurs in the child process before the exec, it is\n\
serialized and written to the errpipe_write fd per subprocess.py.\n\
\n\
Returns: the child process's PID.\n\
\n\
Raises: Only on an error in the parent process.\n\
");
/* module level code ********************************************************/
PyDoc_STRVAR(module_doc,
"A POSIX helper for the subprocess module.");
static PyMethodDef module_methods[] = {
{"fork_exec", subprocess_fork_exec, METH_VARARGS, subprocess_fork_exec_doc},
{NULL, NULL} /* sentinel */
};
static struct PyModuleDef _posixsubprocessmodule = {
PyModuleDef_HEAD_INIT,
"_posixsubprocess",
module_doc,
-1, /* No memory is needed. */
module_methods,
};
PyMODINIT_FUNC
PyInit__posixsubprocess(void)
{
#ifdef _SC_OPEN_MAX
max_fd = sysconf(_SC_OPEN_MAX);
if (max_fd == -1)
#endif
max_fd = 256; /* Matches Lib/subprocess.py */
return PyModule_Create(&_posixsubprocessmodule);
}

17
Modules/posixmodule.c
View File

@ -303,23 +303,6 @@ extern int lstat(const char *, struct stat *);
#define WAIT_STATUS_INT(s) (s)
#endif /* UNION_WAIT */
/* Issue #1983: pid_t can be longer than a C long on some systems */
#if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT
#define PARSE_PID "i"
#define PyLong_FromPid PyLong_FromLong
#define PyLong_AsPid PyLong_AsLong
#elif SIZEOF_PID_T == SIZEOF_LONG
#define PARSE_PID "l"
#define PyLong_FromPid PyLong_FromLong
#define PyLong_AsPid PyLong_AsLong
#elif defined(SIZEOF_LONG_LONG) && SIZEOF_PID_T == SIZEOF_LONG_LONG
#define PARSE_PID "L"
#define PyLong_FromPid PyLong_FromLongLong
#define PyLong_AsPid PyLong_AsLongLong
#else
#error "sizeof(pid_t) is neither sizeof(int), sizeof(long) or sizeof(long long)"
#endif /* SIZEOF_PID_T */
/* Don't use the "_r" form if we don't need it (also, won't have a
prototype for it, at least on Solaris -- maybe others as well?). */
#if defined(HAVE_CTERMID_R) && defined(WITH_THREAD)

60
Objects/abstract.c
View File

@ -2722,3 +2722,63 @@ PyIter_Next(PyObject *iter)
PyErr_Clear();
return result;
}
/*
* Flatten a sequence of bytes() objects into a C array of
* NULL terminated string pointers with a NULL char* terminating the array.
* (ie: an argv or env list)
*
* Memory allocated for the returned list is allocated using malloc() and MUST
* be freed by the caller using a free() loop or _Py_FreeCharPArray().
*/
char *const *
_PySequence_BytesToCharpArray(PyObject* self)
{
char **array;
Py_ssize_t i, argc;
argc = PySequence_Size(self);
if (argc == -1)
return NULL;
array = malloc((argc + 1) * sizeof(char *));
if (array == NULL) {
PyErr_NoMemory();
return NULL;
}
for (i = 0; i < argc; ++i) {
char *data;
PyObject *item = PySequence_GetItem(self, i);
data = PyBytes_AsString(item);
if (data == NULL) {
/* NULL terminate before freeing. */
array[i] = NULL;
goto fail;
}
array[i] = strdup(data);
if (!array[i]) {
PyErr_NoMemory();
goto fail;
}
}
array[argc] = NULL;
return array;
fail:
_Py_FreeCharPArray(array);
return NULL;
}
/* Free's a NULL terminated char** array of C strings. */
void
_Py_FreeCharPArray(char *const array[])
{
Py_ssize_t i;
for (i = 0; array[i] != NULL; ++i) {
free(array[i]);
}
free((void*)array);
}

26
Python/pythonrun.c
View File

@ -2130,6 +2130,27 @@ initsigs(void)
}
/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL.
*
* All of the code in this function must only use async-signal-safe functions,
* listed at `man 7 signal` or
* http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
*/
void
_Py_RestoreSignals(void)
{
#ifdef SIGPIPE
PyOS_setsig(SIGPIPE, SIG_DFL);
#endif
#ifdef SIGXFZ
PyOS_setsig(SIGXFZ, SIG_DFL);
#endif
#ifdef SIGXFSZ
PyOS_setsig(SIGXFSZ, SIG_DFL);
#endif
}
/*
* The file descriptor fd is considered ``interactive'' if either
* a) isatty(fd) is TRUE, or
@ -2223,6 +2244,11 @@ PyOS_getsig(int sig)
#endif
}
/*
* All of the code in this function must only use async-signal-safe functions,
* listed at `man 7 signal` or
* http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
*/
PyOS_sighandler_t
PyOS_setsig(int sig, PyOS_sighandler_t handler)
{

3
setup.py
View File

@ -546,6 +546,9 @@ class PyBuildExt(build_ext):
# CSV files
exts.append( Extension('_csv', ['_csv.c']) )
# POSIX subprocess module helper.
exts.append( Extension('_posixsubprocess', ['_posixsubprocess.c']) )
# socket(2)
exts.append( Extension('_socket', ['socketmodule.c'],
depends = ['socketmodule.h']) )