bpo-32604: Expose the subinterpreters C-API in a "private" stdlib module. (gh-1748)

The module is primarily intended for internal use in the test suite.  Building the module under Windows will come in a follow-up PR.
This commit is contained in:
Eric Snow 2018-01-29 18:23:44 -07:00 committed by GitHub
parent 332cd5ee4f
commit 7f8bfc9b9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 3539 additions and 1 deletions

View File

@ -65,6 +65,79 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate(
PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config);
/* interpreter state */
PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(PY_INT64_T);
/* cross-interpreter data */
struct _xid;
// _PyCrossInterpreterData is similar to Py_buffer as an effectively
// opaque struct that holds data outside the object machinery. This
// is necessary to pass between interpreters in the same process.
typedef struct _xid {
// data is the cross-interpreter-safe derivation of a Python object
// (see _PyObject_GetCrossInterpreterData). It will be NULL if the
// new_object func (below) encodes the data.
void *data;
// obj is the Python object from which the data was derived. This
// is non-NULL only if the data remains bound to the object in some
// way, such that the object must be "released" (via a decref) when
// the data is released. In that case it is automatically
// incref'ed (to match the automatic decref when releaed).
PyObject *obj;
// interp is the ID of the owning interpreter of the original
// object. It corresponds to the active interpreter when
// _PyObject_GetCrossInterpreterData() was called. This should only
// be set by the cross-interpreter machinery.
//
// We use the ID rather than the PyInterpreterState to avoid issues
// with deleted interpreters.
int64_t interp;
// new_object is a function that returns a new object in the current
// interpreter given the data. The resulting object (a new
// reference) will be equivalent to the original object. This field
// is required.
PyObject *(*new_object)(struct _xid *);
// free is called when the data is released. If it is NULL then
// nothing will be done to free the data. For some types this is
// okay (e.g. bytes) and for those types this field should be set
// to NULL. However, for most the data was allocated just for
// cross-interpreter use, so it must be freed when
// _PyCrossInterpreterData_Release is called or the memory will
// leak. In that case, at the very least this field should be set
// to PyMem_RawFree (the default if not explicitly set to NULL).
// The call will happen with the original interpreter activated.
void (*free)(void *);
} _PyCrossInterpreterData;
typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *);
PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *);
PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *);
PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *);
PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *);
/* cross-interpreter data registry */
/* For now we use a global registry of shareable classes. An
alternative would be to add a tp_* slot for a class's
crossinterpdatafunc. It would be simpler and more efficient. */
PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc);
PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *);
struct _xidregitem;
struct _xidregitem {
PyTypeObject *cls;
crossinterpdatafunc getdata;
struct _xidregitem *next;
};
/* Full Python runtime state */
typedef struct pyruntimestate {
@ -86,6 +159,11 @@ typedef struct pyruntimestate {
using a Python int. */
int64_t next_id;
} interpreters;
// XXX Remove this field once we have a tp_* slot.
struct _xidregistry {
PyThread_type_lock mutex;
struct _xidregitem *head;
} xidregistry;
#define NEXITFUNCS 32
void (*exitfuncs[NEXITFUNCS])(void);

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,4 @@
Add a new "_xxsubinterpreters" extension module that exposes the existing
subinterpreter C-API and a new cross-interpreter data sharing mechanism. The
module is primarily intended for more thorough testing of the existing
subinterpreter support.

File diff suppressed because it is too large Load Diff

View File

@ -54,8 +54,13 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
if (runtime->interpreters.mutex == NULL) {
return _Py_INIT_ERR("Can't initialize threads for interpreter");
}
runtime->interpreters.next_id = -1;
runtime->xidregistry.mutex = PyThread_allocate_lock();
if (runtime->xidregistry.mutex == NULL) {
return _Py_INIT_ERR("Can't initialize threads for cross-interpreter data registry");
}
return _Py_INIT_OK();
}
@ -166,6 +171,7 @@ PyInterpreterState_New(void)
/* overflow or Py_Initialize() not called! */
PyErr_SetString(PyExc_RuntimeError,
"failed to get an interpreter ID");
/* XXX deallocate! */
interp = NULL;
} else {
interp->id = _PyRuntime.interpreters.next_id;
@ -256,6 +262,28 @@ PyInterpreterState_GetID(PyInterpreterState *interp)
}
PyInterpreterState *
_PyInterpreterState_LookUpID(PY_INT64_T requested_id)
{
if (requested_id < 0)
goto error;
PyInterpreterState *interp = PyInterpreterState_Head();
while (interp != NULL) {
PY_INT64_T id = PyInterpreterState_GetID(interp);
if (id < 0)
return NULL;
if (requested_id == id)
return interp;
interp = PyInterpreterState_Next(interp);
}
error:
PyErr_Format(PyExc_RuntimeError,
"unrecognized interpreter ID %lld", requested_id);
return NULL;
}
/* Default implementation for _PyThreadState_GetFrame */
static struct _frame *
threadstate_getframe(PyThreadState *self)
@ -1024,6 +1052,251 @@ PyGILState_Release(PyGILState_STATE oldstate)
}
/**************************/
/* cross-interpreter data */
/**************************/
/* cross-interpreter data */
crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *);
/* This is a separate func from _PyCrossInterpreterData_Lookup in order
to keep the registry code separate. */
static crossinterpdatafunc
_lookup_getdata(PyObject *obj)
{
crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj);
if (getdata == NULL && PyErr_Occurred() == 0)
PyErr_Format(PyExc_ValueError,
"%S does not support cross-interpreter data", obj);
return getdata;
}
int
_PyObject_CheckCrossInterpreterData(PyObject *obj)
{
crossinterpdatafunc getdata = _lookup_getdata(obj);
if (getdata == NULL) {
return -1;
}
return 0;
}
static int
_check_xidata(_PyCrossInterpreterData *data)
{
// data->data can be anything, including NULL, so we don't check it.
// data->obj may be NULL, so we don't check it.
if (data->interp < 0) {
PyErr_SetString(PyExc_SystemError, "missing interp");
return -1;
}
if (data->new_object == NULL) {
PyErr_SetString(PyExc_SystemError, "missing new_object func");
return -1;
}
// data->free may be NULL, so we don't check it.
return 0;
}
int
_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
{
PyThreadState *tstate = PyThreadState_Get();
// PyThreadState_Get() aborts if lookup fails, so we don't need
// to check the result for NULL.
PyInterpreterState *interp = tstate->interp;
// Reset data before re-populating.
*data = (_PyCrossInterpreterData){0};
data->free = PyMem_RawFree; // Set a default that may be overridden.
// Call the "getdata" func for the object.
Py_INCREF(obj);
crossinterpdatafunc getdata = _lookup_getdata(obj);
if (getdata == NULL) {
Py_DECREF(obj);
return -1;
}
int res = getdata(obj, data);
Py_DECREF(obj);
if (res != 0) {
return -1;
}
// Fill in the blanks and validate the result.
Py_XINCREF(data->obj);
data->interp = interp->id;
if (_check_xidata(data) != 0) {
_PyCrossInterpreterData_Release(data);
return -1;
}
return 0;
}
void
_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data)
{
if (data->data == NULL && data->obj == NULL) {
// Nothing to release!
return;
}
// Switch to the original interpreter.
PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp);
if (interp == NULL) {
// The intepreter was already destroyed.
if (data->free != NULL) {
// XXX Someone leaked some memory...
}
return;
}
PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
PyThreadState *save_tstate = PyThreadState_Swap(tstate);
// "Release" the data and/or the object.
if (data->free != NULL) {
data->free(data->data);
}
Py_XDECREF(data->obj);
// Switch back.
if (save_tstate != NULL)
PyThreadState_Swap(save_tstate);
}
PyObject *
_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data)
{
return data->new_object(data);
}
/* registry of {type -> crossinterpdatafunc} */
/* For now we use a global registry of shareable classes. An
alternative would be to add a tp_* slot for a class's
crossinterpdatafunc. It would be simpler and more efficient. */
static int
_register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata)
{
// Note that we effectively replace already registered classes
// rather than failing.
struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem));
if (newhead == NULL)
return -1;
newhead->cls = cls;
newhead->getdata = getdata;
newhead->next = _PyRuntime.xidregistry.head;
_PyRuntime.xidregistry.head = newhead;
return 0;
}
static void _register_builtins_for_crossinterpreter_data(void);
int
_PyCrossInterpreterData_Register_Class(PyTypeObject *cls,
crossinterpdatafunc getdata)
{
if (!PyType_Check(cls)) {
PyErr_Format(PyExc_ValueError, "only classes may be registered");
return -1;
}
if (getdata == NULL) {
PyErr_Format(PyExc_ValueError, "missing 'getdata' func");
return -1;
}
// Make sure the class isn't ever deallocated.
Py_INCREF((PyObject *)cls);
PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK);
if (_PyRuntime.xidregistry.head == NULL) {
_register_builtins_for_crossinterpreter_data();
}
int res = _register_xidata(cls, getdata);
PyThread_release_lock(_PyRuntime.xidregistry.mutex);
return res;
}
crossinterpdatafunc
_PyCrossInterpreterData_Lookup(PyObject *obj)
{
PyObject *cls = PyObject_Type(obj);
crossinterpdatafunc getdata = NULL;
PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK);
struct _xidregitem *cur = _PyRuntime.xidregistry.head;
if (cur == NULL) {
_register_builtins_for_crossinterpreter_data();
cur = _PyRuntime.xidregistry.head;
}
for(; cur != NULL; cur = cur->next) {
if (cur->cls == (PyTypeObject *)cls) {
getdata = cur->getdata;
break;
}
}
PyThread_release_lock(_PyRuntime.xidregistry.mutex);
return getdata;
}
/* cross-interpreter data for builtin types */
static PyObject *
_new_bytes_object(_PyCrossInterpreterData *data)
{
return PyBytes_FromString((char *)(data->data));
}
static int
_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data)
{
data->data = (void *)(PyBytes_AS_STRING(obj));
data->obj = obj; // Will be "released" (decref'ed) when data released.
data->new_object = _new_bytes_object;
data->free = NULL; // Do not free the data (it belongs to the object).
return 0;
}
static PyObject *
_new_none_object(_PyCrossInterpreterData *data)
{
// XXX Singleton refcounts are problematic across interpreters...
Py_INCREF(Py_None);
return Py_None;
}
static int
_none_shared(PyObject *obj, _PyCrossInterpreterData *data)
{
data->data = NULL;
// data->obj remains NULL
data->new_object = _new_none_object;
data->free = NULL; // There is nothing to free.
return 0;
}
static void
_register_builtins_for_crossinterpreter_data(void)
{
// None
if (_register_xidata((PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) {
Py_FatalError("could not register None for cross-interpreter sharing");
}
// bytes
if (_register_xidata(&PyBytes_Type, _bytes_shared) != 0) {
Py_FatalError("could not register bytes for cross-interpreter sharing");
}
}
#ifdef __cplusplus
}
#endif

View File

@ -755,6 +755,10 @@ class PyBuildExt(build_ext):
['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c'])
)
# Python interface to subinterpreter C-API.
exts.append(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'],
define_macros=[('Py_BUILD_CORE', '')]))
#
# Here ends the simple stuff. From here on, modules need certain
# libraries, are platform-specific, or present other surprises.