Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-32604: Clean up created subinterpreters before runtime finalization. (gh-5709)

This commit is contained in:
Eric Snow 2018-02-16 18:53:40 -07:00 committed by GitHub
parent bd093355a6
commit 4c6955e2b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 410 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Include/internal/pystate.h
View File

@ -69,6 +69,10 @@ PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config);
PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(PY_INT64_T);
PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *);
PyAPI_FUNC(void) _PyInterpreterState_IDIncref(PyInterpreterState *);
PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *);
/* cross-interpreter data */

4
Include/pystate.h
View File

@ -8,6 +8,8 @@
extern "C" {
#endif
#include "pythread.h"
/* This limitation is for performance and simplicity. If needed it can be
removed (with effort). */
#define MAX_CO_EXTRA_USERS 255
@ -111,6 +113,8 @@ typedef struct _is {
struct _ts *tstate_head;
int64_t id;
int64_t id_refcount;
PyThread_type_lock id_mutex;
PyObject *modules;
PyObject *modules_by_index;

78
Lib/test/test__xxsubinterpreters.py
View File

@ -152,7 +152,7 @@ class GetCurrentTests(TestBase):
interp = interpreters.create()
out = _run_output(interp, dedent("""
import _xxsubinterpreters as _interpreters
print(_interpreters.get_current())
print(int(_interpreters.get_current()))
"""))
cur = int(out.strip())
_, expected = interpreters.list_all()
@ -172,7 +172,7 @@ class GetMainTests(TestBase):
interp = interpreters.create()
out = _run_output(interp, dedent("""
import _xxsubinterpreters as _interpreters
print(_interpreters.get_main())
print(int(_interpreters.get_main()))
"""))
main = int(out.strip())
self.assertEqual(main, expected)
@ -196,7 +196,7 @@ class IsRunningTests(TestBase):
interp = interpreters.create()
out = _run_output(interp, dedent(f"""
import _xxsubinterpreters as _interpreters
if _interpreters.is_running({interp}):
if _interpreters.is_running({int(interp)}):
print(True)
else:
print(False)
@ -218,6 +218,63 @@ class IsRunningTests(TestBase):
interpreters.is_running(-1)
class InterpreterIDTests(TestBase):
def test_with_int(self):
id = interpreters.InterpreterID(10, force=True)
self.assertEqual(int(id), 10)
def test_coerce_id(self):
id = interpreters.InterpreterID('10', force=True)
self.assertEqual(int(id), 10)
id = interpreters.InterpreterID(10.0, force=True)
self.assertEqual(int(id), 10)
class Int(str):
def __init__(self, value):
self._value = value
def __int__(self):
return self._value
id = interpreters.InterpreterID(Int(10), force=True)
self.assertEqual(int(id), 10)
def test_bad_id(self):
for id in [-1, 'spam']:
with self.subTest(id):
with self.assertRaises(ValueError):
interpreters.InterpreterID(id)
with self.assertRaises(OverflowError):
interpreters.InterpreterID(2**64)
with self.assertRaises(TypeError):
interpreters.InterpreterID(object())
def test_does_not_exist(self):
id = interpreters.channel_create()
with self.assertRaises(RuntimeError):
interpreters.InterpreterID(int(id) + 1) # unforced
def test_repr(self):
id = interpreters.InterpreterID(10, force=True)
self.assertEqual(repr(id), 'InterpreterID(10)')
def test_equality(self):
id1 = interpreters.create()
id2 = interpreters.InterpreterID(int(id1))
id3 = interpreters.create()
self.assertTrue(id1 == id1)
self.assertTrue(id1 == id2)
self.assertTrue(id1 == int(id1))
self.assertFalse(id1 == id3)
self.assertFalse(id1 != id1)
self.assertFalse(id1 != id2)
self.assertTrue(id1 != id3)
class CreateTests(TestBase):
def test_in_main(self):
@ -256,7 +313,7 @@ class CreateTests(TestBase):
out = _run_output(id1, dedent("""
import _xxsubinterpreters as _interpreters
id = _interpreters.create()
print(id)
print(int(id))
"""))
id2 = int(out.strip())
@ -271,7 +328,7 @@ class CreateTests(TestBase):
out = _run_output(id1, dedent("""
import _xxsubinterpreters as _interpreters
id = _interpreters.create()
print(id)
print(int(id))
"""))
id2 = int(out.strip())
@ -365,7 +422,7 @@ class DestroyTests(TestBase):
script = dedent(f"""
import _xxsubinterpreters as _interpreters
try:
_interpreters.destroy({id})
_interpreters.destroy({int(id)})
except RuntimeError:
pass
""")
@ -377,10 +434,10 @@ class DestroyTests(TestBase):
main, = interpreters.list_all()
id1 = interpreters.create()
id2 = interpreters.create()
script = dedent("""
script = dedent(f"""
import _xxsubinterpreters as _interpreters
_interpreters.destroy({})
""").format(id2)
_interpreters.destroy({int(id2)})
""")
interpreters.run_string(id1, script)
self.assertEqual(set(interpreters.list_all()), {main, id1})
@ -699,11 +756,14 @@ class RunStringTests(TestBase):
'spam': 42,
})
# XXX Fix this test!
@unittest.skip('blocking forever')
def test_still_running_at_exit(self):
script = dedent(f"""
from textwrap import dedent
import threading
import _xxsubinterpreters as _interpreters
id = _interpreters.create()
def f():
_interpreters.run_string(id, dedent('''
import time

297
Modules/_xxsubinterpretersmodule.c
View File

@ -1653,27 +1653,6 @@ interp_exceptions_init(PyObject *ns)
return 0;
}
static PyInterpreterState *
_look_up(PyObject *requested_id)
{
long long id = PyLong_AsLongLong(requested_id);
if (id == -1 && PyErr_Occurred() != NULL) {
return NULL;
}
assert(id <= INT64_MAX);
return _PyInterpreterState_LookUpID(id);
}
static PyObject *
_get_id(PyInterpreterState *interp)
{
PY_INT64_T id = PyInterpreterState_GetID(interp);
if (id < 0) {
return NULL;
}
return PyLong_FromLongLong(id);
}
static int
_is_running(PyInterpreterState *interp)
{
@ -1809,6 +1788,265 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr,
return result;
}
/* InterpreterID class */
static PyTypeObject InterpreterIDtype;
typedef struct interpid {
PyObject_HEAD
int64_t id;
} interpid;
static interpid *
newinterpid(PyTypeObject *cls, int64_t id, int force)
{
PyInterpreterState *interp = _PyInterpreterState_LookUpID(id);
if (interp == NULL) {
if (force) {
PyErr_Clear();
}
else {
return NULL;
}
}
interpid *self = PyObject_New(interpid, cls);
if (self == NULL) {
return NULL;
}
self->id = id;
if (interp != NULL) {
_PyInterpreterState_IDIncref(interp);
}
return self;
}
static PyObject *
interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", "force", NULL};
PyObject *idobj;
int force = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O|$p:InterpreterID.__init__", kwlist,
&idobj, &force)) {
return NULL;
}
// Coerce and check the ID.
int64_t id;
if (PyObject_TypeCheck(idobj, &InterpreterIDtype)) {
id = ((interpid *)idobj)->id;
}
else {
id = _coerce_id(idobj);
if (id < 0) {
return NULL;
}
}
return (PyObject *)newinterpid(cls, id, force);
}
static void
interpid_dealloc(PyObject *v)
{
int64_t id = ((interpid *)v)->id;
PyInterpreterState *interp = _PyInterpreterState_LookUpID(id);
if (interp != NULL) {
_PyInterpreterState_IDDecref(interp);
}
else {
// already deleted
PyErr_Clear();
}
Py_TYPE(v)->tp_free(v);
}
static PyObject *
interpid_repr(PyObject *self)
{
PyTypeObject *type = Py_TYPE(self);
const char *name = _PyType_Name(type);
interpid *id = (interpid *)self;
return PyUnicode_FromFormat("%s(%d)", name, id->id);
}
PyObject *
interpid_int(PyObject *self)
{
interpid *id = (interpid *)self;
return PyLong_FromLongLong(id->id);
}
static PyNumberMethods interpid_as_number = {
0, /* nb_add */
0, /* nb_subtract */
0, /* nb_multiply */
0, /* nb_remainder */
0, /* nb_divmod */
0, /* nb_power */
0, /* nb_negative */
0, /* nb_positive */
0, /* nb_absolute */
0, /* nb_bool */
0, /* nb_invert */
0, /* nb_lshift */
0, /* nb_rshift */
0, /* nb_and */
0, /* nb_xor */
0, /* nb_or */
(unaryfunc)interpid_int, /* nb_int */
0, /* nb_reserved */
0, /* nb_float */
0, /* nb_inplace_add */
0, /* nb_inplace_subtract */
0, /* nb_inplace_multiply */
0, /* nb_inplace_remainder */
0, /* nb_inplace_power */
0, /* nb_inplace_lshift */
0, /* nb_inplace_rshift */
0, /* nb_inplace_and */
0, /* nb_inplace_xor */
0, /* nb_inplace_or */
0, /* nb_floor_divide */
0, /* nb_true_divide */
0, /* nb_inplace_floor_divide */
0, /* nb_inplace_true_divide */
(unaryfunc)interpid_int, /* nb_index */
};
static Py_hash_t
interpid_hash(PyObject *self)
{
interpid *id = (interpid *)self;
PyObject *obj = PyLong_FromLongLong(id->id);
if (obj == NULL) {
return -1;
}
Py_hash_t hash = PyObject_Hash(obj);
Py_DECREF(obj);
return hash;
}
static PyObject *
interpid_richcompare(PyObject *self, PyObject *other, int op)
{
if (op != Py_EQ && op != Py_NE) {
Py_RETURN_NOTIMPLEMENTED;
}
if (!PyObject_TypeCheck(self, &InterpreterIDtype)) {
Py_RETURN_NOTIMPLEMENTED;
}
interpid *id = (interpid *)self;
int equal;
if (PyObject_TypeCheck(other, &InterpreterIDtype)) {
interpid *otherid = (interpid *)other;
equal = (id->id == otherid->id);
}
else {
other = PyNumber_Long(other);
if (other == NULL) {
PyErr_Clear();
Py_RETURN_NOTIMPLEMENTED;
}
int64_t otherid = PyLong_AsLongLong(other);
Py_DECREF(other);
if (otherid == -1 && PyErr_Occurred() != NULL) {
return NULL;
}
if (otherid < 0) {
equal = 0;
}
else {
equal = (id->id == otherid);
}
}
if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) {
Py_RETURN_TRUE;
}
Py_RETURN_FALSE;
}
PyDoc_STRVAR(interpid_doc,
"A interpreter ID identifies a interpreter and may be used as an int.");
static PyTypeObject InterpreterIDtype = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"interpreters.InterpreterID", /* tp_name */
sizeof(interpid), /* tp_size */
0, /* tp_itemsize */
(destructor)interpid_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
(reprfunc)interpid_repr, /* tp_repr */
&interpid_as_number, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
interpid_hash, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_LONG_SUBCLASS, /* tp_flags */
interpid_doc, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
interpid_richcompare, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
interpid_new, /* tp_new */
};
static PyObject *
_get_id(PyInterpreterState *interp)
{
PY_INT64_T id = PyInterpreterState_GetID(interp);
if (id < 0) {
return NULL;
}
return (PyObject *)newinterpid(&InterpreterIDtype, id, 0);
}
static PyInterpreterState *
_look_up(PyObject *requested_id)
{
int64_t id;
if (PyObject_TypeCheck(requested_id, &InterpreterIDtype)) {
id = ((interpid *)requested_id)->id;
}
else {
id = PyLong_AsLongLong(requested_id);
if (id == -1 && PyErr_Occurred() != NULL) {
return NULL;
}
assert(id <= INT64_MAX);
}
return _PyInterpreterState_LookUpID(id);
}
/* module level code ********************************************************/
@ -1852,7 +2090,16 @@ interp_create(PyObject *self, PyObject *args)
PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed");
return NULL;
}
if (_PyInterpreterState_IDInitref(tstate->interp) != 0) {
goto error;
};
return _get_id(tstate->interp);
error:
save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
PyThreadState_Swap(save_tstate);
return NULL;
}
PyDoc_STRVAR(create_doc,
@ -2359,6 +2606,10 @@ PyInit__xxsubinterpreters(void)
if (PyType_Ready(&ChannelIDtype) != 0) {
return NULL;
}
InterpreterIDtype.tp_base = &PyLong_Type;
if (PyType_Ready(&InterpreterIDtype) != 0) {
return NULL;
}
/* Create the module */
PyObject *module = PyModule_Create(&interpretersmodule);
@ -2380,6 +2631,10 @@ PyInit__xxsubinterpreters(void)
if (PyDict_SetItemString(ns, "ChannelID", (PyObject *)&ChannelIDtype) != 0) {
return NULL;
}
Py_INCREF(&InterpreterIDtype);
if (PyDict_SetItemString(ns, "InterpreterID", (PyObject *)&InterpreterIDtype) != 0) {
return NULL;
}
if (_PyCrossInterpreterData_Register_Class(&ChannelIDtype, _channelid_shared)) {
return NULL;

58
Python/pystate.c
View File

@ -125,7 +125,8 @@ PyInterpreterState_New(void)
return NULL;
}
interp->id_refcount = -1;
interp->id_mutex = NULL;
interp->modules = NULL;
interp->modules_by_index = NULL;
interp->sysdict = NULL;
@ -247,6 +248,9 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
Py_FatalError("PyInterpreterState_Delete: remaining subinterpreters");
}
HEAD_UNLOCK();
if (interp->id_mutex != NULL) {
PyThread_free_lock(interp->id_mutex);
}
PyMem_RawFree(interp);
}
@ -284,6 +288,58 @@ error:
return NULL;
}
int
_PyInterpreterState_IDInitref(PyInterpreterState *interp)
{
if (interp->id_mutex != NULL) {
return 0;
}
interp->id_mutex = PyThread_allocate_lock();
if (interp->id_mutex == NULL) {
PyErr_SetString(PyExc_RuntimeError,
"failed to create init interpreter ID mutex");
return -1;
}
interp->id_refcount = 0;
return 0;
}
void
_PyInterpreterState_IDIncref(PyInterpreterState *interp)
{
if (interp->id_mutex == NULL) {
return;
}
PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK);
interp->id_refcount += 1;
PyThread_release_lock(interp->id_mutex);
}
void
_PyInterpreterState_IDDecref(PyInterpreterState *interp)
{
if (interp->id_mutex == NULL) {
return;
}
PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK);
assert(interp->id_refcount != 0);
interp->id_refcount -= 1;
int64_t refcount = interp->id_refcount;
PyThread_release_lock(interp->id_mutex);
if (refcount == 0) {
PyThreadState *tstate, *save_tstate;
tstate = PyInterpreterState_ThreadHead(interp);
save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
PyThreadState_Swap(save_tstate);
}
}
/* Default implementation for _PyThreadState_GetFrame */
static struct _frame *
threadstate_getframe(PyThreadState *self)