Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-32604: Improve subinterpreter tests. (#6914) 

Add more tests for subinterpreters. This patch also fixes a few small defects in the channel implementation.
This commit is contained in:
Eric Snow 2018-05-16 15:04:57 -04:00 committed by GitHub
parent 55e53c3093
commit 6d2cd9036c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 1300 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1296
Lib/test/test__xxsubinterpreters.py
View File

File diff suppressed because it is too large Load Diff

250
Modules/_xxsubinterpretersmodule.c
View File

@ -1250,7 +1250,9 @@ _channel_recv(_channels *channels, int64_t id)
_PyCrossInterpreterData *data = _channel_next(chan, interp->id);
PyThread_release_lock(mutex);
if (data == NULL) {
PyErr_Format(ChannelEmptyError, "channel %d is empty", id);
if (!PyErr_Occurred()) {
PyErr_Format(ChannelEmptyError, "channel %d is empty", id);
}
return NULL;
}
@ -1304,12 +1306,13 @@ typedef struct channelid {
PyObject_HEAD
int64_t id;
int end;
int resolve;
_channels *channels;
} channelid;
static channelid *
newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels,
int force)
int force, int resolve)
{
channelid *self = PyObject_New(channelid, cls);
if (self == NULL) {
@ -1317,6 +1320,7 @@ newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels,
}
self->id = cid;
self->end = end;
self->resolve = resolve;
self->channels = channels;
if (_channels_add_id_object(channels, cid) != 0) {
@ -1337,14 +1341,15 @@ static _channels * _global_channels(void);
static PyObject *
channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", "send", "recv", "force", NULL};
static char *kwlist[] = {"id", "send", "recv", "force", "_resolve", NULL};
PyObject *id;
int send = -1;
int recv = -1;
int force = 0;
int resolve = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O|$ppp:ChannelID.__init__", kwlist,
&id, &send, &recv, &force))
"O|$pppp:ChannelID.__new__", kwlist,
&id, &send, &recv, &force, &resolve))
return NULL;
// Coerce and check the ID.
@ -1376,7 +1381,8 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
end = CHANNEL_RECV;
}
return (PyObject *)newchannelid(cls, cid, end, _global_channels(), force);
return (PyObject *)newchannelid(cls, cid, end, _global_channels(),
force, resolve);
}
static void
@ -1409,6 +1415,13 @@ channelid_repr(PyObject *self)
return PyUnicode_FromFormat(fmt, name, cid->id);
}
static PyObject *
channelid_str(PyObject *self)
{
channelid *cid = (channelid *)self;
return PyUnicode_FromFormat("%d", cid->id);
}
PyObject *
channelid_int(PyObject *self)
{
@ -1519,14 +1532,49 @@ channelid_richcompare(PyObject *self, PyObject *other, int op)
struct _channelid_xid {
int64_t id;
int end;
int resolve;
};
static PyObject *
_channelid_from_xid(_PyCrossInterpreterData *data)
{
struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
return (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end,
_global_channels(), 0);
// Note that we do not preserve the "resolve" flag.
PyObject *cid = (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end,
_global_channels(), 0, 0);
if (xid->end == 0) {
return cid;
}
if (!xid->resolve) {
return cid;
}
/* Try returning a high-level channel end but fall back to the ID. */
PyObject *highlevel = PyImport_ImportModule("interpreters");
if (highlevel == NULL) {
PyErr_Clear();
highlevel = PyImport_ImportModule("test.support.interpreters");
if (highlevel == NULL) {
goto error;
}
}
const char *clsname = (xid->end == CHANNEL_RECV) ? "RecvChannel" :
"SendChannel";
PyObject *cls = PyObject_GetAttrString(highlevel, clsname);
Py_DECREF(highlevel);
if (cls == NULL) {
goto error;
}
PyObject *chan = PyObject_CallFunctionObjArgs(cls, cid, NULL);
if (chan == NULL) {
goto error;
}
Py_DECREF(cid);
return chan;
error:
PyErr_Clear();
return cid;
}
static int
@ -1538,6 +1586,7 @@ _channelid_shared(PyObject *obj, _PyCrossInterpreterData *data)
}
xid->id = ((channelid *)obj)->id;
xid->end = ((channelid *)obj)->end;
xid->resolve = ((channelid *)obj)->resolve;
data->data = xid;
data->obj = obj;
@ -1553,7 +1602,7 @@ channelid_end(PyObject *self, void *end)
channelid *cid = (channelid *)self;
if (end != NULL) {
return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end,
cid->channels, force);
cid->channels, force, cid->resolve);
}
if (cid->end == CHANNEL_SEND) {
@ -1597,7 +1646,7 @@ static PyTypeObject ChannelIDtype = {
0, /* tp_as_mapping */
channelid_hash, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
(reprfunc)channelid_str, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
@ -1878,6 +1927,13 @@ interpid_repr(PyObject *self)
return PyUnicode_FromFormat("%s(%d)", name, id->id);
}
static PyObject *
interpid_str(PyObject *self)
{
interpid *id = (interpid *)self;
return PyUnicode_FromFormat("%d", id->id);
}
PyObject *
interpid_int(PyObject *self)
{
@ -1999,7 +2055,7 @@ static PyTypeObject InterpreterIDtype = {
0, /* tp_as_mapping */
interpid_hash, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
(reprfunc)interpid_str, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
@ -2115,10 +2171,13 @@ Create a new interpreter and return a unique generated ID.");
static PyObject *
interp_destroy(PyObject *self, PyObject *args)
interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", NULL};
PyObject *id;
if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id)) {
// XXX Use "L" for id?
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:destroy", kwlist, &id)) {
return NULL;
}
if (!PyLong_Check(id)) {
@ -2162,7 +2221,7 @@ interp_destroy(PyObject *self, PyObject *args)
}
PyDoc_STRVAR(destroy_doc,
"destroy(ID)\n\
"destroy(id)\n\
\n\
Destroy the identified interpreter.\n\
\n\
@ -2228,7 +2287,8 @@ static PyObject *
interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored))
{
// Currently, 0 is always the main interpreter.
return PyLong_FromLongLong(0);
PY_INT64_T id = 0;
return (PyObject *)newinterpid(&InterpreterIDtype, id, 0);
}
PyDoc_STRVAR(get_main_doc,
@ -2238,22 +2298,20 @@ Return the ID of main interpreter.");
static PyObject *
interp_run_string(PyObject *self, PyObject *args)
interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", "script", "shared", NULL};
PyObject *id, *code;
PyObject *shared = NULL;
if (!PyArg_UnpackTuple(args, "run_string", 2, 3, &id, &code, &shared)) {
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"OU|O:run_string", kwlist,
&id, &code, &shared)) {
return NULL;
}
if (!PyLong_Check(id)) {
PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int");
return NULL;
}
if (!PyUnicode_Check(code)) {
PyErr_SetString(PyExc_TypeError,
"second arg (code) must be a string");
return NULL;
}
// Look up the interpreter.
PyInterpreterState *interp = _look_up(id);
@ -2281,7 +2339,7 @@ interp_run_string(PyObject *self, PyObject *args)
}
PyDoc_STRVAR(run_string_doc,
"run_string(ID, sourcetext)\n\
"run_string(id, script, shared)\n\
\n\
Execute the provided string in the identified interpreter.\n\
\n\
@ -2289,12 +2347,15 @@ See PyRun_SimpleStrings.");
static PyObject *
object_is_shareable(PyObject *self, PyObject *args)
object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"obj", NULL};
PyObject *obj;
if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) {
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:is_shareable", kwlist, &obj)) {
return NULL;
}
if (_PyObject_CheckCrossInterpreterData(obj) == 0) {
Py_RETURN_TRUE;
}
@ -2310,10 +2371,12 @@ False otherwise.");
static PyObject *
interp_is_running(PyObject *self, PyObject *args)
interp_is_running(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", NULL};
PyObject *id;
if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id)) {
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:is_running", kwlist, &id)) {
return NULL;
}
if (!PyLong_Check(id)) {
@ -2348,7 +2411,7 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored))
return NULL;
}
PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0,
&_globals.channels, 0);
&_globals.channels, 0, 0);
if (id == NULL) {
if (_channel_destroy(&_globals.channels, cid) != 0) {
// XXX issue a warning?
@ -2360,15 +2423,17 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored))
}
PyDoc_STRVAR(channel_create_doc,
"channel_create() -> ID\n\
"channel_create() -> cid\n\
\n\
Create a new cross-interpreter channel and return a unique generated ID.");
static PyObject *
channel_destroy(PyObject *self, PyObject *args)
channel_destroy(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"cid", NULL};
PyObject *id;
if (!PyArg_UnpackTuple(args, "channel_destroy", 1, 1, &id)) {
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:channel_destroy", kwlist, &id)) {
return NULL;
}
int64_t cid = _coerce_id(id);
@ -2383,7 +2448,7 @@ channel_destroy(PyObject *self, PyObject *args)
}
PyDoc_STRVAR(channel_destroy_doc,
"channel_destroy(ID)\n\
"channel_destroy(cid)\n\
\n\
Close and finalize the channel. Afterward attempts to use the channel\n\
will behave as though it never existed.");
@ -2406,7 +2471,7 @@ channel_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
int64_t *cur = cids;
for (int64_t i=0; i < count; cur++, i++) {
PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cur, 0,
&_globals.channels, 0);
&_globals.channels, 0, 0);
if (id == NULL) {
Py_DECREF(ids);
ids = NULL;
@ -2421,16 +2486,18 @@ finally:
}
PyDoc_STRVAR(channel_list_all_doc,
"channel_list_all() -> [ID]\n\
"channel_list_all() -> [cid]\n\
\n\
Return the list of all IDs for active channels.");
static PyObject *
channel_send(PyObject *self, PyObject *args)
channel_send(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"cid", "obj", NULL};
PyObject *id;
PyObject *obj;
if (!PyArg_UnpackTuple(args, "channel_send", 2, 2, &id, &obj)) {
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"OO:channel_send", kwlist, &id, &obj)) {
return NULL;
}
int64_t cid = _coerce_id(id);
@ -2445,15 +2512,17 @@ channel_send(PyObject *self, PyObject *args)
}
PyDoc_STRVAR(channel_send_doc,
"channel_send(ID, obj)\n\
"channel_send(cid, obj)\n\
\n\
Add the object's data to the channel's queue.");
static PyObject *
channel_recv(PyObject *self, PyObject *args)
channel_recv(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"cid", NULL};
PyObject *id;
if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) {
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:channel_recv", kwlist, &id)) {
return NULL;
}
int64_t cid = _coerce_id(id);
@ -2465,17 +2534,34 @@ channel_recv(PyObject *self, PyObject *args)
}
PyDoc_STRVAR(channel_recv_doc,
"channel_recv(ID) -> obj\n\
"channel_recv(cid) -> obj\n\
\n\
Return a new object from the data at the from of the channel's queue.");
static PyObject *
channel_close(PyObject *self, PyObject *id)
channel_close(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"cid", "send", "recv", "force", NULL};
PyObject *id;
int send = 0;
int recv = 0;
int force = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O|$ppp:channel_close", kwlist,
&id, &send, &recv, &force)) {
return NULL;
}
int64_t cid = _coerce_id(id);
if (cid < 0) {
return NULL;
}
if (send == 0 && recv == 0) {
send = 1;
recv = 1;
}
// XXX Handle the ends.
// XXX Handle force is True.
if (_channel_close(&_globals.channels, cid) != 0) {
return NULL;
@ -2484,48 +2570,66 @@ channel_close(PyObject *self, PyObject *id)
}
PyDoc_STRVAR(channel_close_doc,
"channel_close(ID)\n\
"channel_close(cid, *, send=None, recv=None, force=False)\n\
\n\
Close the channel for all interpreters. Once the channel's ID has\n\
no more ref counts the channel will be destroyed.");
Close the channel for all interpreters.\n\
\n\
If the channel is empty then the keyword args are ignored and both\n\
ends are immediately closed. Otherwise, if 'force' is True then\n\
all queued items are released and both ends are immediately\n\
closed.\n\
\n\
If the channel is not empty *and* 'force' is False then following\n\
happens:\n\
\n\
* recv is True (regardless of send):\n\
- raise ChannelNotEmptyError\n\
* recv is None and send is None:\n\
- raise ChannelNotEmptyError\n\
* send is True and recv is not True:\n\
- fully close the 'send' end\n\
- close the 'recv' end to interpreters not already receiving\n\
- fully close it once empty\n\
\n\
Closing an already closed channel results in a ChannelClosedError.\n\
\n\
Once the channel's ID has no more ref counts in any interpreter\n\
the channel will be destroyed.");
static PyObject *
channel_drop_interpreter(PyObject *self, PyObject *args, PyObject *kwds)
channel_release(PyObject *self, PyObject *args, PyObject *kwds)
{
// Note that only the current interpreter is affected.
static char *kwlist[] = {"id", "send", "recv", NULL};
static char *kwlist[] = {"cid", "send", "recv", "force", NULL};
PyObject *id;
int send = -1;
int recv = -1;
int send = 0;
int recv = 0;
int force = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O|$pp:channel_drop_interpreter", kwlist,
&id, &send, &recv))
"O|$ppp:channel_release", kwlist,
&id, &send, &recv, &force)) {
return NULL;
}
int64_t cid = _coerce_id(id);
if (cid < 0) {
return NULL;
}
if (send < 0 && recv < 0) {
if (send == 0 && recv == 0) {
send = 1;
recv = 1;
}
else {
if (send < 0) {
send = 0;
}
if (recv < 0) {
recv = 0;
}
}
// XXX Handle force is True.
// XXX Fix implicit release.
if (_channel_drop(&_globals.channels, cid, send, recv) != 0) {
return NULL;
}
Py_RETURN_NONE;
}
PyDoc_STRVAR(channel_drop_interpreter_doc,
"channel_drop_interpreter(ID, *, send=None, recv=None)\n\
PyDoc_STRVAR(channel_release_doc,
"channel_release(cid, *, send=None, recv=None, force=True)\n\
\n\
Close the channel for the current interpreter. 'send' and 'recv'\n\
(bool) may be used to indicate the ends to close. By default both\n\
@ -2541,7 +2645,7 @@ static PyMethodDef module_functions[] = {
{"create", (PyCFunction)interp_create,
METH_VARARGS, create_doc},
{"destroy", (PyCFunction)interp_destroy,
METH_VARARGS, destroy_doc},
METH_VARARGS | METH_KEYWORDS, destroy_doc},
{"list_all", interp_list_all,
METH_NOARGS, list_all_doc},
{"get_current", interp_get_current,
@ -2549,27 +2653,27 @@ static PyMethodDef module_functions[] = {
{"get_main", interp_get_main,
METH_NOARGS, get_main_doc},
{"is_running", (PyCFunction)interp_is_running,
METH_VARARGS, is_running_doc},
METH_VARARGS | METH_KEYWORDS, is_running_doc},
{"run_string", (PyCFunction)interp_run_string,
METH_VARARGS, run_string_doc},
METH_VARARGS | METH_KEYWORDS, run_string_doc},
{"is_shareable", (PyCFunction)object_is_shareable,
METH_VARARGS, is_shareable_doc},
METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
{"channel_create", channel_create,
METH_NOARGS, channel_create_doc},
{"channel_destroy", (PyCFunction)channel_destroy,
METH_VARARGS, channel_destroy_doc},
METH_VARARGS | METH_KEYWORDS, channel_destroy_doc},
{"channel_list_all", channel_list_all,
METH_NOARGS, channel_list_all_doc},
{"channel_send", (PyCFunction)channel_send,
METH_VARARGS, channel_send_doc},
METH_VARARGS | METH_KEYWORDS, channel_send_doc},
{"channel_recv", (PyCFunction)channel_recv,
METH_VARARGS, channel_recv_doc},
{"channel_close", channel_close,
METH_O, channel_close_doc},
{"channel_drop_interpreter", (PyCFunction)channel_drop_interpreter,
METH_VARARGS | METH_KEYWORDS, channel_drop_interpreter_doc},
METH_VARARGS | METH_KEYWORDS, channel_recv_doc},
{"channel_close", (PyCFunction)channel_close,
METH_VARARGS | METH_KEYWORDS, channel_close_doc},
{"channel_release", (PyCFunction)channel_release,
METH_VARARGS | METH_KEYWORDS, channel_release_doc},
{"_channel_id", (PyCFunction)channel__channel_id,
METH_VARARGS | METH_KEYWORDS, NULL},

80
Python/pystate.c
View File

@ -1308,6 +1308,10 @@ _PyCrossInterpreterData_Register_Class(PyTypeObject *cls,
return res;
}
/* Cross-interpreter objects are looked up by exact match on the class.
We can reassess this policy when we move from a global registry to a
tp_* slot. */
crossinterpdatafunc
_PyCrossInterpreterData_Lookup(PyObject *obj)
{
@ -1332,19 +1336,79 @@ _PyCrossInterpreterData_Lookup(PyObject *obj)
/* cross-interpreter data for builtin types */
struct _shared_bytes_data {
char *bytes;
Py_ssize_t len;
};
static PyObject *
_new_bytes_object(_PyCrossInterpreterData *data)
{
return PyBytes_FromString((char *)(data->data));
struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data);
return PyBytes_FromStringAndSize(shared->bytes, shared->len);
}
static int
_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data)
{
data->data = (void *)(PyBytes_AS_STRING(obj));
struct _shared_bytes_data *shared = PyMem_NEW(struct _shared_bytes_data, 1);
if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) {
return -1;
}
data->data = (void *)shared;
data->obj = obj; // Will be "released" (decref'ed) when data released.
data->new_object = _new_bytes_object;
data->free = NULL; // Do not free the data (it belongs to the object).
data->free = PyMem_Free;
return 0;
}
struct _shared_str_data {
int kind;
const void *buffer;
Py_ssize_t len;
};
static PyObject *
_new_str_object(_PyCrossInterpreterData *data)
{
struct _shared_str_data *shared = (struct _shared_str_data *)(data->data);
return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len);
}
static int
_str_shared(PyObject *obj, _PyCrossInterpreterData *data)
{
struct _shared_str_data *shared = PyMem_NEW(struct _shared_str_data, 1);
shared->kind = PyUnicode_KIND(obj);
shared->buffer = PyUnicode_DATA(obj);
shared->len = PyUnicode_GET_LENGTH(obj) - 1;
data->data = (void *)shared;
data->obj = obj; // Will be "released" (decref'ed) when data released.
data->new_object = _new_str_object;
data->free = PyMem_Free;
return 0;
}
static PyObject *
_new_long_object(_PyCrossInterpreterData *data)
{
return PyLong_FromLongLong((int64_t)(data->data));
}
static int
_long_shared(PyObject *obj, _PyCrossInterpreterData *data)
{
int64_t value = PyLong_AsLongLong(obj);
if (value == -1 && PyErr_Occurred()) {
if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
PyErr_SetString(PyExc_OverflowError, "try sending as bytes");
}
return -1;
}
data->data = (void *)value;
data->obj = NULL;
data->new_object = _new_long_object;
data->free = NULL;
return 0;
}
@ -1374,10 +1438,20 @@ _register_builtins_for_crossinterpreter_data(void)
Py_FatalError("could not register None for cross-interpreter sharing");
}
// int
if (_register_xidata(&PyLong_Type, _long_shared) != 0) {
Py_FatalError("could not register int for cross-interpreter sharing");
}
// bytes
if (_register_xidata(&PyBytes_Type, _bytes_shared) != 0) {
Py_FatalError("could not register bytes for cross-interpreter sharing");
}
// str
if (_register_xidata(&PyUnicode_Type, _str_shared) != 0) {
Py_FatalError("could not register str for cross-interpreter sharing");
}
}