gh-76785: More Fixes for test.support.interpreters (gh-113012)

This brings the module (along with the associated extension modules) mostly in sync with PEP 734.  There are only a few small things to wrap up.
This commit is contained in:
Eric Snow 2023-12-12 10:43:30 -07:00 committed by GitHub
parent cde1417175
commit a49b427b02
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1899 additions and 88 deletions

View File

@ -3,13 +3,11 @@
import queue
import time
import weakref
import _xxinterpchannels as _channels
import _xxinterpchannels as _queues
import _xxinterpqueues as _queues
# aliases:
from _xxinterpchannels import (
ChannelError as QueueError,
ChannelNotFoundError as QueueNotFoundError,
from _xxinterpqueues import (
QueueError, QueueNotFoundError,
)
__all__ = [
@ -19,14 +17,27 @@ __all__ = [
]
class QueueEmpty(_queues.QueueEmpty, queue.Empty):
"""Raised from get_nowait() when the queue is empty.
It is also raised from get() if it times out.
"""
class QueueFull(_queues.QueueFull, queue.Full):
"""Raised from put_nowait() when the queue is full.
It is also raised from put() if it times out.
"""
def create(maxsize=0):
"""Return a new cross-interpreter queue.
The queue may be used to pass data safely between interpreters.
"""
# XXX honor maxsize
qid = _queues.create()
return Queue._with_maxsize(qid, maxsize)
qid = _queues.create(maxsize)
return Queue(qid)
def list_all():
@ -35,53 +46,37 @@ def list_all():
for qid in _queues.list_all()]
class QueueEmpty(queue.Empty):
"""Raised from get_nowait() when the queue is empty.
It is also raised from get() if it times out.
"""
class QueueFull(queue.Full):
"""Raised from put_nowait() when the queue is full.
It is also raised from put() if it times out.
"""
_known_queues = weakref.WeakValueDictionary()
class Queue:
"""A cross-interpreter queue."""
@classmethod
def _with_maxsize(cls, id, maxsize):
if not isinstance(maxsize, int):
raise TypeError(f'maxsize must be an int, got {maxsize!r}')
elif maxsize < 0:
maxsize = 0
else:
maxsize = int(maxsize)
self = cls(id)
self._maxsize = maxsize
return self
def __new__(cls, id, /):
# There is only one instance for any given ID.
if isinstance(id, int):
id = _channels._channel_id(id, force=False)
elif not isinstance(id, _channels.ChannelID):
id = int(id)
else:
raise TypeError(f'id must be an int, got {id!r}')
key = int(id)
try:
self = _known_queues[key]
self = _known_queues[id]
except KeyError:
self = super().__new__(cls)
self._id = id
self._maxsize = 0
_known_queues[key] = self
_known_queues[id] = self
_queues.bind(id)
return self
def __del__(self):
try:
_queues.release(self._id)
except QueueNotFoundError:
pass
try:
del _known_queues[self._id]
except KeyError:
pass
def __repr__(self):
return f'{type(self).__name__}({self.id})'
@ -90,39 +85,58 @@ class Queue:
@property
def id(self):
return int(self._id)
return self._id
@property
def maxsize(self):
return self._maxsize
@property
def _info(self):
return _channels.get_info(self._id)
try:
return self._maxsize
except AttributeError:
self._maxsize = _queues.get_maxsize(self._id)
return self._maxsize
def empty(self):
return self._info.count == 0
return self.qsize() == 0
def full(self):
if self._maxsize <= 0:
return False
return self._info.count >= self._maxsize
return _queues.is_full(self._id)
def qsize(self):
return self._info.count
return _queues.get_count(self._id)
def put(self, obj, timeout=None):
# XXX block if full
_channels.send(self._id, obj, blocking=False)
def put(self, obj, timeout=None, *,
_delay=10 / 1000, # 10 milliseconds
):
"""Add the object to the queue.
This blocks while the queue is full.
"""
if timeout is not None:
timeout = int(timeout)
if timeout < 0:
raise ValueError(f'timeout value must be non-negative')
end = time.time() + timeout
while True:
try:
_queues.put(self._id, obj)
except _queues.QueueFull as exc:
if timeout is not None and time.time() >= end:
exc.__class__ = QueueFull
raise # re-raise
time.sleep(_delay)
else:
break
def put_nowait(self, obj):
# XXX raise QueueFull if full
return _channels.send(self._id, obj, blocking=False)
try:
return _queues.put(self._id, obj)
except _queues.QueueFull as exc:
exc.__class__ = QueueFull
raise # re-raise
def get(self, timeout=None, *,
_sentinel=object(),
_delay=10 / 1000, # 10 milliseconds
):
_delay=10 / 1000, # 10 milliseconds
):
"""Return the next object from the queue.
This blocks while the queue is empty.
@ -132,25 +146,27 @@ class Queue:
if timeout < 0:
raise ValueError(f'timeout value must be non-negative')
end = time.time() + timeout
obj = _channels.recv(self._id, _sentinel)
while obj is _sentinel:
time.sleep(_delay)
if timeout is not None and time.time() >= end:
raise QueueEmpty
obj = _channels.recv(self._id, _sentinel)
while True:
try:
return _queues.get(self._id)
except _queues.QueueEmpty as exc:
if timeout is not None and time.time() >= end:
exc.__class__ = QueueEmpty
raise # re-raise
time.sleep(_delay)
return obj
def get_nowait(self, *, _sentinel=object()):
def get_nowait(self):
"""Return the next object from the channel.
If the queue is empty then raise QueueEmpty. Otherwise this
is the same as get().
"""
obj = _channels.recv(self._id, _sentinel)
if obj is _sentinel:
raise QueueEmpty
return obj
try:
return _queues.get(self._id)
except _queues.QueueEmpty as exc:
exc.__class__ = QueueEmpty
raise # re-raise
# XXX add this:
#_channels._register_queue_type(Queue)
_queues._register_queue_type(Queue)

View File

@ -5,13 +5,21 @@ import time
from test.support import import_helper
# Raise SkipTest if subinterpreters not supported.
import_helper.import_module('_xxinterpchannels')
#import_helper.import_module('_xxinterpqueues')
_queues = import_helper.import_module('_xxinterpqueues')
from test.support import interpreters
from test.support.interpreters import queues
from .utils import _run_output, TestBase
class TestBase(TestBase):
def tearDown(self):
for qid in _queues.list_all():
try:
_queues.destroy(qid)
except Exception:
pass
class QueueTests(TestBase):
def test_create(self):
@ -32,20 +40,47 @@ class QueueTests(TestBase):
self.assertEqual(queue.maxsize, 0)
with self.subTest('negative maxsize'):
queue = queues.create(-1)
self.assertEqual(queue.maxsize, 0)
queue = queues.create(-10)
self.assertEqual(queue.maxsize, -10)
with self.subTest('bad maxsize'):
with self.assertRaises(TypeError):
queues.create('1')
@unittest.expectedFailure
def test_shareable(self):
queue1 = queues.create()
queue2 = queues.create()
queue1.put(queue2)
queue3 = queue1.get()
self.assertIs(queue3, queue1)
interp = interpreters.create()
interp.exec_sync(dedent(f"""
from test.support.interpreters import queues
queue1 = queues.Queue({queue1.id})
"""));
with self.subTest('same interpreter'):
queue2 = queues.create()
queue1.put(queue2)
queue3 = queue1.get()
self.assertIs(queue3, queue2)
with self.subTest('from current interpreter'):
queue4 = queues.create()
queue1.put(queue4)
out = _run_output(interp, dedent("""
queue4 = queue1.get()
print(queue4.id)
"""))
qid = int(out)
self.assertEqual(qid, queue4.id)
with self.subTest('from subinterpreter'):
out = _run_output(interp, dedent("""
queue5 = queues.create()
queue1.put(queue5)
print(queue5.id)
"""))
qid = int(out)
queue5 = queue1.get()
self.assertEqual(queue5.id, qid)
def test_id_type(self):
queue = queues.create()
@ -137,7 +172,6 @@ class TestQueueOps(TestBase):
self.assertEqual(actual, expected)
@unittest.expectedFailure
def test_put_timeout(self):
queue = queues.create(2)
queue.put(None)
@ -147,7 +181,6 @@ class TestQueueOps(TestBase):
queue.get()
queue.put(None)
@unittest.expectedFailure
def test_put_nowait(self):
queue = queues.create(2)
queue.put_nowait(None)
@ -179,31 +212,64 @@ class TestQueueOps(TestBase):
assert obj is not orig, 'expected: obj is not orig'
"""))
@unittest.expectedFailure
def test_put_get_different_interpreters(self):
interp = interpreters.create()
queue1 = queues.create()
queue2 = queues.create()
self.assertEqual(len(queues.list_all()), 2)
obj1 = b'spam'
queue1.put(obj1)
out = _run_output(
interpreters.create(),
interp,
dedent(f"""
import test.support.interpreters.queue as queues
from test.support.interpreters import queues
queue1 = queues.Queue({queue1.id})
queue2 = queues.Queue({queue2.id})
assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1'
obj = queue1.get()
assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0'
assert obj == b'spam', 'expected: obj == obj1'
# When going to another interpreter we get a copy.
assert id(obj) != {id(obj1)}, 'expected: obj is not obj1'
obj2 = b'eggs'
print(id(obj2))
assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0'
queue2.put(obj2)
assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1'
"""))
obj2 = queue2.get()
self.assertEqual(len(queues.list_all()), 2)
self.assertEqual(queue1.qsize(), 0)
self.assertEqual(queue2.qsize(), 1)
obj2 = queue2.get()
self.assertEqual(obj2, b'eggs')
self.assertNotEqual(id(obj2), int(out))
def test_put_cleared_with_subinterpreter(self):
interp = interpreters.create()
queue = queues.create()
out = _run_output(
interp,
dedent(f"""
from test.support.interpreters import queues
queue = queues.Queue({queue.id})
obj1 = b'spam'
obj2 = b'eggs'
queue.put(obj1)
queue.put(obj2)
"""))
self.assertEqual(queue.qsize(), 2)
obj1 = queue.get()
self.assertEqual(obj1, b'spam')
self.assertEqual(queue.qsize(), 1)
del interp
self.assertEqual(queue.qsize(), 0)
def test_put_get_different_threads(self):
queue1 = queues.create()
queue2 = queues.create()

View File

@ -273,6 +273,7 @@ PYTHONPATH=$(COREPYTHONPATH)
#_xxsubinterpreters _xxsubinterpretersmodule.c
#_xxinterpchannels _xxinterpchannelsmodule.c
#_xxinterpqueues _xxinterpqueuesmodule.c
#_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
#_testbuffer _testbuffer.c
#_testinternalcapi _testinternalcapi.c

View File

@ -41,8 +41,11 @@
@MODULE__QUEUE_TRUE@_queue _queuemodule.c
@MODULE__RANDOM_TRUE@_random _randommodule.c
@MODULE__STRUCT_TRUE@_struct _struct.c
# build supports subinterpreters
@MODULE__XXSUBINTERPRETERS_TRUE@_xxsubinterpreters _xxsubinterpretersmodule.c
@MODULE__XXINTERPCHANNELS_TRUE@_xxinterpchannels _xxinterpchannelsmodule.c
@MODULE__XXINTERPQUEUES_TRUE@_xxinterpqueues _xxinterpqueuesmodule.c
@MODULE__ZONEINFO_TRUE@_zoneinfo _zoneinfo.c
# needs libm

View File

@ -2629,10 +2629,11 @@ _get_current_channelend_type(int end)
cls = state->recv_channel_type;
}
if (cls == NULL) {
PyObject *highlevel = PyImport_ImportModule("interpreters");
// Force the module to be loaded, to register the type.
PyObject *highlevel = PyImport_ImportModule("interpreters.channel");
if (highlevel == NULL) {
PyErr_Clear();
highlevel = PyImport_ImportModule("test.support.interpreters");
highlevel = PyImport_ImportModule("test.support.interpreters.channel");
if (highlevel == NULL) {
return NULL;
}

File diff suppressed because it is too large Load Diff

View File

@ -37,6 +37,7 @@ extern PyObject* PyInit__weakref(void);
extern PyObject* PyInit_xxsubtype(void);
extern PyObject* PyInit__xxsubinterpreters(void);
extern PyObject* PyInit__xxinterpchannels(void);
extern PyObject* PyInit__xxinterpqueues(void);
extern PyObject* PyInit__random(void);
extern PyObject* PyInit_itertools(void);
extern PyObject* PyInit__collections(void);
@ -142,6 +143,7 @@ struct _inittab _PyImport_Inittab[] = {
{"xxsubtype", PyInit_xxsubtype},
{"_xxsubinterpreters", PyInit__xxsubinterpreters},
{"_xxinterpchannels", PyInit__xxinterpchannels},
{"_xxinterpqueues", PyInit__xxinterpqueues},
#ifdef _Py_HAVE_ZLIB
{"zlib", PyInit_zlib},
#endif

View File

@ -458,6 +458,7 @@
<ClCompile Include="..\Modules\xxsubtype.c" />
<ClCompile Include="..\Modules\_xxsubinterpretersmodule.c" />
<ClCompile Include="..\Modules\_xxinterpchannelsmodule.c" />
<ClCompile Include="..\Modules\_xxinterpqueuesmodule.c" />
<ClCompile Include="..\Modules\_io\fileio.c" />
<ClCompile Include="..\Modules\_io\bytesio.c" />
<ClCompile Include="..\Modules\_io\stringio.c" />

View File

@ -1505,6 +1505,9 @@
<ClCompile Include="..\Modules\_xxinterpchannelsmodule.c">
<Filter>Modules</Filter>
</ClCompile>
<ClCompile Include="..\Modules\_xxinterpqueuesmodule.c">
<Filter>Modules</Filter>
</ClCompile>
<ClCompile Include="..\Parser\string_parser.c">
<Filter>Parser</Filter>
</ClCompile>

View File

@ -36,6 +36,7 @@ IGNORE = {
'_testsinglephase',
'_xxsubinterpreters',
'_xxinterpchannels',
'_xxinterpqueues',
'_xxtestfuzz',
'idlelib.idle_test',
'test',

View File

@ -165,6 +165,7 @@ Python/pylifecycle.c fatal_error reentrant -
# explicitly protected, internal-only
Modules/_xxinterpchannelsmodule.c - _globals -
Modules/_xxinterpqueuesmodule.c - _globals -
# set once during module init
Modules/_decimal/_decimal.c - minalloc_is_set -

Can't render this file because it has a wrong number of fields in line 4.

29
configure generated vendored
View File

@ -769,6 +769,8 @@ MODULE__MULTIPROCESSING_FALSE
MODULE__MULTIPROCESSING_TRUE
MODULE__ZONEINFO_FALSE
MODULE__ZONEINFO_TRUE
MODULE__XXINTERPQUEUES_FALSE
MODULE__XXINTERPQUEUES_TRUE
MODULE__XXINTERPCHANNELS_FALSE
MODULE__XXINTERPCHANNELS_TRUE
MODULE__XXSUBINTERPRETERS_FALSE
@ -28025,6 +28027,7 @@ case $ac_sys_system in #(
py_cv_module__tkinter=n/a
py_cv_module__xxsubinterpreters=n/a
py_cv_module__xxinterpchannels=n/a
py_cv_module__xxinterpqueues=n/a
py_cv_module_grp=n/a
py_cv_module_pwd=n/a
py_cv_module_resource=n/a
@ -28524,6 +28527,28 @@ then :
fi
if test "$py_cv_module__xxinterpqueues" != "n/a"
then :
py_cv_module__xxinterpqueues=yes
fi
if test "$py_cv_module__xxinterpqueues" = yes; then
MODULE__XXINTERPQUEUES_TRUE=
MODULE__XXINTERPQUEUES_FALSE='#'
else
MODULE__XXINTERPQUEUES_TRUE='#'
MODULE__XXINTERPQUEUES_FALSE=
fi
as_fn_append MODULE_BLOCK "MODULE__XXINTERPQUEUES_STATE=$py_cv_module__xxinterpqueues$as_nl"
if test "x$py_cv_module__xxinterpqueues" = xyes
then :
fi
@ -30760,6 +30785,10 @@ if test -z "${MODULE__XXINTERPCHANNELS_TRUE}" && test -z "${MODULE__XXINTERPCHAN
as_fn_error $? "conditional \"MODULE__XXINTERPCHANNELS\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
fi
if test -z "${MODULE__XXINTERPQUEUES_TRUE}" && test -z "${MODULE__XXINTERPQUEUES_FALSE}"; then
as_fn_error $? "conditional \"MODULE__XXINTERPQUEUES\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
fi
if test -z "${MODULE__ZONEINFO_TRUE}" && test -z "${MODULE__ZONEINFO_FALSE}"; then
as_fn_error $? "conditional \"MODULE__ZONEINFO\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5

View File

@ -7120,6 +7120,7 @@ AS_CASE([$ac_sys_system],
[_tkinter],
[_xxsubinterpreters],
[_xxinterpchannels],
[_xxinterpqueues],
[grp],
[pwd],
[resource],
@ -7236,6 +7237,7 @@ PY_STDLIB_MOD_SIMPLE([_struct])
PY_STDLIB_MOD_SIMPLE([_typing])
PY_STDLIB_MOD_SIMPLE([_xxsubinterpreters])
PY_STDLIB_MOD_SIMPLE([_xxinterpchannels])
PY_STDLIB_MOD_SIMPLE([_xxinterpqueues])
PY_STDLIB_MOD_SIMPLE([_zoneinfo])
dnl multiprocessing modules