bpo-32604: Expose the subinterpreters C-API in a "private" stdlib module. (gh-1748)
The module is primarily intended for internal use in the test suite. Building the module under Windows will come in a follow-up PR.
This commit is contained in:
parent
332cd5ee4f
commit
7f8bfc9b9a
|
@ -65,6 +65,79 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate(
|
|||
PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config);
|
||||
|
||||
|
||||
/* interpreter state */
|
||||
|
||||
PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(PY_INT64_T);
|
||||
|
||||
|
||||
/* cross-interpreter data */
|
||||
|
||||
struct _xid;
|
||||
|
||||
// _PyCrossInterpreterData is similar to Py_buffer as an effectively
|
||||
// opaque struct that holds data outside the object machinery. This
|
||||
// is necessary to pass between interpreters in the same process.
|
||||
typedef struct _xid {
|
||||
// data is the cross-interpreter-safe derivation of a Python object
|
||||
// (see _PyObject_GetCrossInterpreterData). It will be NULL if the
|
||||
// new_object func (below) encodes the data.
|
||||
void *data;
|
||||
// obj is the Python object from which the data was derived. This
|
||||
// is non-NULL only if the data remains bound to the object in some
|
||||
// way, such that the object must be "released" (via a decref) when
|
||||
// the data is released. In that case it is automatically
|
||||
// incref'ed (to match the automatic decref when releaed).
|
||||
PyObject *obj;
|
||||
// interp is the ID of the owning interpreter of the original
|
||||
// object. It corresponds to the active interpreter when
|
||||
// _PyObject_GetCrossInterpreterData() was called. This should only
|
||||
// be set by the cross-interpreter machinery.
|
||||
//
|
||||
// We use the ID rather than the PyInterpreterState to avoid issues
|
||||
// with deleted interpreters.
|
||||
int64_t interp;
|
||||
// new_object is a function that returns a new object in the current
|
||||
// interpreter given the data. The resulting object (a new
|
||||
// reference) will be equivalent to the original object. This field
|
||||
// is required.
|
||||
PyObject *(*new_object)(struct _xid *);
|
||||
// free is called when the data is released. If it is NULL then
|
||||
// nothing will be done to free the data. For some types this is
|
||||
// okay (e.g. bytes) and for those types this field should be set
|
||||
// to NULL. However, for most the data was allocated just for
|
||||
// cross-interpreter use, so it must be freed when
|
||||
// _PyCrossInterpreterData_Release is called or the memory will
|
||||
// leak. In that case, at the very least this field should be set
|
||||
// to PyMem_RawFree (the default if not explicitly set to NULL).
|
||||
// The call will happen with the original interpreter activated.
|
||||
void (*free)(void *);
|
||||
} _PyCrossInterpreterData;
|
||||
|
||||
typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *);
|
||||
PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *);
|
||||
|
||||
PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *);
|
||||
PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *);
|
||||
PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *);
|
||||
|
||||
/* cross-interpreter data registry */
|
||||
|
||||
/* For now we use a global registry of shareable classes. An
|
||||
alternative would be to add a tp_* slot for a class's
|
||||
crossinterpdatafunc. It would be simpler and more efficient. */
|
||||
|
||||
PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc);
|
||||
PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *);
|
||||
|
||||
struct _xidregitem;
|
||||
|
||||
struct _xidregitem {
|
||||
PyTypeObject *cls;
|
||||
crossinterpdatafunc getdata;
|
||||
struct _xidregitem *next;
|
||||
};
|
||||
|
||||
|
||||
/* Full Python runtime state */
|
||||
|
||||
typedef struct pyruntimestate {
|
||||
|
@ -86,6 +159,11 @@ typedef struct pyruntimestate {
|
|||
using a Python int. */
|
||||
int64_t next_id;
|
||||
} interpreters;
|
||||
// XXX Remove this field once we have a tp_* slot.
|
||||
struct _xidregistry {
|
||||
PyThread_type_lock mutex;
|
||||
struct _xidregitem *head;
|
||||
} xidregistry;
|
||||
|
||||
#define NEXITFUNCS 32
|
||||
void (*exitfuncs[NEXITFUNCS])(void);
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,4 @@
|
|||
Add a new "_xxsubinterpreters" extension module that exposes the existing
|
||||
subinterpreter C-API and a new cross-interpreter data sharing mechanism. The
|
||||
module is primarily intended for more thorough testing of the existing
|
||||
subinterpreter support.
|
File diff suppressed because it is too large
Load Diff
275
Python/pystate.c
275
Python/pystate.c
|
@ -54,8 +54,13 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
|
|||
if (runtime->interpreters.mutex == NULL) {
|
||||
return _Py_INIT_ERR("Can't initialize threads for interpreter");
|
||||
}
|
||||
|
||||
runtime->interpreters.next_id = -1;
|
||||
|
||||
runtime->xidregistry.mutex = PyThread_allocate_lock();
|
||||
if (runtime->xidregistry.mutex == NULL) {
|
||||
return _Py_INIT_ERR("Can't initialize threads for cross-interpreter data registry");
|
||||
}
|
||||
|
||||
return _Py_INIT_OK();
|
||||
}
|
||||
|
||||
|
@ -166,6 +171,7 @@ PyInterpreterState_New(void)
|
|||
/* overflow or Py_Initialize() not called! */
|
||||
PyErr_SetString(PyExc_RuntimeError,
|
||||
"failed to get an interpreter ID");
|
||||
/* XXX deallocate! */
|
||||
interp = NULL;
|
||||
} else {
|
||||
interp->id = _PyRuntime.interpreters.next_id;
|
||||
|
@ -256,6 +262,28 @@ PyInterpreterState_GetID(PyInterpreterState *interp)
|
|||
}
|
||||
|
||||
|
||||
PyInterpreterState *
|
||||
_PyInterpreterState_LookUpID(PY_INT64_T requested_id)
|
||||
{
|
||||
if (requested_id < 0)
|
||||
goto error;
|
||||
|
||||
PyInterpreterState *interp = PyInterpreterState_Head();
|
||||
while (interp != NULL) {
|
||||
PY_INT64_T id = PyInterpreterState_GetID(interp);
|
||||
if (id < 0)
|
||||
return NULL;
|
||||
if (requested_id == id)
|
||||
return interp;
|
||||
interp = PyInterpreterState_Next(interp);
|
||||
}
|
||||
|
||||
error:
|
||||
PyErr_Format(PyExc_RuntimeError,
|
||||
"unrecognized interpreter ID %lld", requested_id);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Default implementation for _PyThreadState_GetFrame */
|
||||
static struct _frame *
|
||||
threadstate_getframe(PyThreadState *self)
|
||||
|
@ -1024,6 +1052,251 @@ PyGILState_Release(PyGILState_STATE oldstate)
|
|||
}
|
||||
|
||||
|
||||
/**************************/
|
||||
/* cross-interpreter data */
|
||||
/**************************/
|
||||
|
||||
/* cross-interpreter data */
|
||||
|
||||
crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *);
|
||||
|
||||
/* This is a separate func from _PyCrossInterpreterData_Lookup in order
|
||||
to keep the registry code separate. */
|
||||
static crossinterpdatafunc
|
||||
_lookup_getdata(PyObject *obj)
|
||||
{
|
||||
crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj);
|
||||
if (getdata == NULL && PyErr_Occurred() == 0)
|
||||
PyErr_Format(PyExc_ValueError,
|
||||
"%S does not support cross-interpreter data", obj);
|
||||
return getdata;
|
||||
}
|
||||
|
||||
int
|
||||
_PyObject_CheckCrossInterpreterData(PyObject *obj)
|
||||
{
|
||||
crossinterpdatafunc getdata = _lookup_getdata(obj);
|
||||
if (getdata == NULL) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
_check_xidata(_PyCrossInterpreterData *data)
|
||||
{
|
||||
// data->data can be anything, including NULL, so we don't check it.
|
||||
|
||||
// data->obj may be NULL, so we don't check it.
|
||||
|
||||
if (data->interp < 0) {
|
||||
PyErr_SetString(PyExc_SystemError, "missing interp");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (data->new_object == NULL) {
|
||||
PyErr_SetString(PyExc_SystemError, "missing new_object func");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// data->free may be NULL, so we don't check it.
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
|
||||
{
|
||||
PyThreadState *tstate = PyThreadState_Get();
|
||||
// PyThreadState_Get() aborts if lookup fails, so we don't need
|
||||
// to check the result for NULL.
|
||||
PyInterpreterState *interp = tstate->interp;
|
||||
|
||||
// Reset data before re-populating.
|
||||
*data = (_PyCrossInterpreterData){0};
|
||||
data->free = PyMem_RawFree; // Set a default that may be overridden.
|
||||
|
||||
// Call the "getdata" func for the object.
|
||||
Py_INCREF(obj);
|
||||
crossinterpdatafunc getdata = _lookup_getdata(obj);
|
||||
if (getdata == NULL) {
|
||||
Py_DECREF(obj);
|
||||
return -1;
|
||||
}
|
||||
int res = getdata(obj, data);
|
||||
Py_DECREF(obj);
|
||||
if (res != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Fill in the blanks and validate the result.
|
||||
Py_XINCREF(data->obj);
|
||||
data->interp = interp->id;
|
||||
if (_check_xidata(data) != 0) {
|
||||
_PyCrossInterpreterData_Release(data);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void
|
||||
_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data)
|
||||
{
|
||||
if (data->data == NULL && data->obj == NULL) {
|
||||
// Nothing to release!
|
||||
return;
|
||||
}
|
||||
|
||||
// Switch to the original interpreter.
|
||||
PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp);
|
||||
if (interp == NULL) {
|
||||
// The intepreter was already destroyed.
|
||||
if (data->free != NULL) {
|
||||
// XXX Someone leaked some memory...
|
||||
}
|
||||
return;
|
||||
}
|
||||
PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
|
||||
PyThreadState *save_tstate = PyThreadState_Swap(tstate);
|
||||
|
||||
// "Release" the data and/or the object.
|
||||
if (data->free != NULL) {
|
||||
data->free(data->data);
|
||||
}
|
||||
Py_XDECREF(data->obj);
|
||||
|
||||
// Switch back.
|
||||
if (save_tstate != NULL)
|
||||
PyThreadState_Swap(save_tstate);
|
||||
}
|
||||
|
||||
PyObject *
|
||||
_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data)
|
||||
{
|
||||
return data->new_object(data);
|
||||
}
|
||||
|
||||
/* registry of {type -> crossinterpdatafunc} */
|
||||
|
||||
/* For now we use a global registry of shareable classes. An
|
||||
alternative would be to add a tp_* slot for a class's
|
||||
crossinterpdatafunc. It would be simpler and more efficient. */
|
||||
|
||||
static int
|
||||
_register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata)
|
||||
{
|
||||
// Note that we effectively replace already registered classes
|
||||
// rather than failing.
|
||||
struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem));
|
||||
if (newhead == NULL)
|
||||
return -1;
|
||||
newhead->cls = cls;
|
||||
newhead->getdata = getdata;
|
||||
newhead->next = _PyRuntime.xidregistry.head;
|
||||
_PyRuntime.xidregistry.head = newhead;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void _register_builtins_for_crossinterpreter_data(void);
|
||||
|
||||
int
|
||||
_PyCrossInterpreterData_Register_Class(PyTypeObject *cls,
|
||||
crossinterpdatafunc getdata)
|
||||
{
|
||||
if (!PyType_Check(cls)) {
|
||||
PyErr_Format(PyExc_ValueError, "only classes may be registered");
|
||||
return -1;
|
||||
}
|
||||
if (getdata == NULL) {
|
||||
PyErr_Format(PyExc_ValueError, "missing 'getdata' func");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Make sure the class isn't ever deallocated.
|
||||
Py_INCREF((PyObject *)cls);
|
||||
|
||||
PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK);
|
||||
if (_PyRuntime.xidregistry.head == NULL) {
|
||||
_register_builtins_for_crossinterpreter_data();
|
||||
}
|
||||
int res = _register_xidata(cls, getdata);
|
||||
PyThread_release_lock(_PyRuntime.xidregistry.mutex);
|
||||
return res;
|
||||
}
|
||||
|
||||
crossinterpdatafunc
|
||||
_PyCrossInterpreterData_Lookup(PyObject *obj)
|
||||
{
|
||||
PyObject *cls = PyObject_Type(obj);
|
||||
crossinterpdatafunc getdata = NULL;
|
||||
PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK);
|
||||
struct _xidregitem *cur = _PyRuntime.xidregistry.head;
|
||||
if (cur == NULL) {
|
||||
_register_builtins_for_crossinterpreter_data();
|
||||
cur = _PyRuntime.xidregistry.head;
|
||||
}
|
||||
for(; cur != NULL; cur = cur->next) {
|
||||
if (cur->cls == (PyTypeObject *)cls) {
|
||||
getdata = cur->getdata;
|
||||
break;
|
||||
}
|
||||
}
|
||||
PyThread_release_lock(_PyRuntime.xidregistry.mutex);
|
||||
return getdata;
|
||||
}
|
||||
|
||||
/* cross-interpreter data for builtin types */
|
||||
|
||||
static PyObject *
|
||||
_new_bytes_object(_PyCrossInterpreterData *data)
|
||||
{
|
||||
return PyBytes_FromString((char *)(data->data));
|
||||
}
|
||||
|
||||
static int
|
||||
_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data)
|
||||
{
|
||||
data->data = (void *)(PyBytes_AS_STRING(obj));
|
||||
data->obj = obj; // Will be "released" (decref'ed) when data released.
|
||||
data->new_object = _new_bytes_object;
|
||||
data->free = NULL; // Do not free the data (it belongs to the object).
|
||||
return 0;
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
_new_none_object(_PyCrossInterpreterData *data)
|
||||
{
|
||||
// XXX Singleton refcounts are problematic across interpreters...
|
||||
Py_INCREF(Py_None);
|
||||
return Py_None;
|
||||
}
|
||||
|
||||
static int
|
||||
_none_shared(PyObject *obj, _PyCrossInterpreterData *data)
|
||||
{
|
||||
data->data = NULL;
|
||||
// data->obj remains NULL
|
||||
data->new_object = _new_none_object;
|
||||
data->free = NULL; // There is nothing to free.
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
_register_builtins_for_crossinterpreter_data(void)
|
||||
{
|
||||
// None
|
||||
if (_register_xidata((PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) {
|
||||
Py_FatalError("could not register None for cross-interpreter sharing");
|
||||
}
|
||||
|
||||
// bytes
|
||||
if (_register_xidata(&PyBytes_Type, _bytes_shared) != 0) {
|
||||
Py_FatalError("could not register bytes for cross-interpreter sharing");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
4
setup.py
4
setup.py
|
@ -755,6 +755,10 @@ class PyBuildExt(build_ext):
|
|||
['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c'])
|
||||
)
|
||||
|
||||
# Python interface to subinterpreter C-API.
|
||||
exts.append(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'],
|
||||
define_macros=[('Py_BUILD_CORE', '')]))
|
||||
|
||||
#
|
||||
# Here ends the simple stuff. From here on, modules need certain
|
||||
# libraries, are platform-specific, or present other surprises.
|
||||
|
|
Loading…
Reference in New Issue