cpython/Lib/multiprocessing/queues.py

348 lines
11 KiB
Python
Raw Normal View History

#
# Module implementing queues
#
# multiprocessing/queues.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
2009-03-31 00:37:07 -03:00
__all__ = ['Queue', 'SimpleQueue', 'JoinableQueue']
import sys
import os
import threading
import collections
import time
import weakref
import errno
from queue import Empty, Full
import _multiprocessing
from . import connection
from . import context
_ForkingPickler = context.reduction.ForkingPickler
from .util import debug, info, Finalize, register_after_fork, is_exiting
#
# Queue type using a pipe, buffer and thread
#
class Queue(object):
def __init__(self, maxsize=0, *, ctx):
if maxsize <= 0:
# Can raise ImportError (see issues #3770 and #23400)
from .synchronize import SEM_VALUE_MAX as maxsize
self._maxsize = maxsize
self._reader, self._writer = connection.Pipe(duplex=False)
self._rlock = ctx.Lock()
self._opid = os.getpid()
if sys.platform == 'win32':
self._wlock = None
else:
self._wlock = ctx.Lock()
self._sem = ctx.BoundedSemaphore(maxsize)
# For use by concurrent.futures
self._ignore_epipe = False
self._after_fork()
if sys.platform != 'win32':
register_after_fork(self, Queue._after_fork)
def __getstate__(self):
context.assert_spawning(self)
return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
self._rlock, self._wlock, self._sem, self._opid)
def __setstate__(self, state):
(self._ignore_epipe, self._maxsize, self._reader, self._writer,
self._rlock, self._wlock, self._sem, self._opid) = state
self._after_fork()
def _after_fork(self):
debug('Queue._after_fork()')
self._notempty = threading.Condition(threading.Lock())
self._buffer = collections.deque()
self._thread = None
self._jointhread = None
self._joincancelled = False
self._closed = False
self._close = None
self._send_bytes = self._writer.send_bytes
self._recv_bytes = self._reader.recv_bytes
self._poll = self._reader.poll
def put(self, obj, block=True, timeout=None):
assert not self._closed, "Queue {0!r} has been closed".format(self)
if not self._sem.acquire(block, timeout):
raise Full
with self._notempty:
if self._thread is None:
self._start_thread()
self._buffer.append(obj)
self._notempty.notify()
def get(self, block=True, timeout=None):
if block and timeout is None:
with self._rlock:
res = self._recv_bytes()
self._sem.release()
else:
if block:
deadline = time.time() + timeout
if not self._rlock.acquire(block, timeout):
raise Empty
try:
if block:
timeout = deadline - time.time()
if not self._poll(timeout):
raise Empty
elif not self._poll():
raise Empty
res = self._recv_bytes()
self._sem.release()
finally:
self._rlock.release()
# unserialize the data after having released the lock
return _ForkingPickler.loads(res)
def qsize(self):
Merged revisions 73196,73278-73280,73299,73308,73312-73313,73317-73318,73321,73324,73331,73335,73340,73363 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r73196 | benjamin.peterson | 2009-06-03 20:40:29 -0500 (Wed, 03 Jun 2009) | 1 line use the offical api ........ r73278 | benjamin.peterson | 2009-06-07 17:33:11 -0500 (Sun, 07 Jun 2009) | 1 line inherit from object ........ r73279 | benjamin.peterson | 2009-06-07 17:35:00 -0500 (Sun, 07 Jun 2009) | 1 line always inherit from an appropiate base class ........ r73280 | benjamin.peterson | 2009-06-07 17:54:35 -0500 (Sun, 07 Jun 2009) | 1 line use booleans for flags ........ r73299 | georg.brandl | 2009-06-08 13:41:36 -0500 (Mon, 08 Jun 2009) | 1 line Typo fix. ........ r73308 | benjamin.peterson | 2009-06-08 17:18:32 -0500 (Mon, 08 Jun 2009) | 1 line remove useless assertion ........ r73312 | benjamin.peterson | 2009-06-08 18:44:13 -0500 (Mon, 08 Jun 2009) | 1 line remove error checks already done in set_context() ........ r73313 | r.david.murray | 2009-06-08 19:44:22 -0500 (Mon, 08 Jun 2009) | 4 lines Issue 2947: document how return code handling translates from os.popen to subprocess. Also fixes reference link in the os.spawn documentation. ........ r73317 | benjamin.peterson | 2009-06-09 12:24:26 -0500 (Tue, 09 Jun 2009) | 1 line make ast.c depend on the grammar ........ r73318 | benjamin.peterson | 2009-06-09 12:29:51 -0500 (Tue, 09 Jun 2009) | 1 line explain why keyword names are not just NAME ........ r73321 | benjamin.peterson | 2009-06-09 16:13:43 -0500 (Tue, 09 Jun 2009) | 1 line update symbol.py from with statement changes ........ r73324 | amaury.forgeotdarc | 2009-06-09 17:53:16 -0500 (Tue, 09 Jun 2009) | 2 lines Avoid invoking the parser/compiler just to test the presence of a function. ........ r73331 | benjamin.peterson | 2009-06-10 08:45:31 -0500 (Wed, 10 Jun 2009) | 1 line fix spelling ........ r73335 | raymond.hettinger | 2009-06-10 11:15:40 -0500 (Wed, 10 Jun 2009) | 1 line Fix signed/unsigned compiler warning. ........ r73340 | amaury.forgeotdarc | 2009-06-10 15:30:19 -0500 (Wed, 10 Jun 2009) | 2 lines Fix a typo spotted by Nick Coghlan. ........ r73363 | benjamin.peterson | 2009-06-11 12:51:17 -0500 (Thu, 11 Jun 2009) | 1 line use multi-with syntax ........
2009-06-11 19:54:11 -03:00
# Raises NotImplementedError on Mac OSX because of broken sem_getvalue()
return self._maxsize - self._sem._semlock._get_value()
def empty(self):
return not self._poll()
def full(self):
return self._sem._semlock._is_zero()
def get_nowait(self):
return self.get(False)
def put_nowait(self, obj):
return self.put(obj, False)
def close(self):
self._closed = True
try:
self._reader.close()
finally:
close = self._close
if close:
self._close = None
close()
def join_thread(self):
debug('Queue.join_thread()')
assert self._closed, "Queue {0!r} not closed".format(self)
if self._jointhread:
self._jointhread()
def cancel_join_thread(self):
debug('Queue.cancel_join_thread()')
self._joincancelled = True
try:
self._jointhread.cancel()
except AttributeError:
pass
def _start_thread(self):
debug('Queue._start_thread()')
# Start thread which transfers data from buffer to pipe
self._buffer.clear()
self._thread = threading.Thread(
target=Queue._feed,
args=(self._buffer, self._notempty, self._send_bytes,
self._wlock, self._writer.close, self._ignore_epipe),
name='QueueFeederThread'
)
self._thread.daemon = True
debug('doing self._thread.start()')
self._thread.start()
debug('... done self._thread.start()')
if not self._joincancelled:
self._jointhread = Finalize(
self._thread, Queue._finalize_join,
[weakref.ref(self._thread)],
exitpriority=-5
)
# Send sentinel to the thread queue object when garbage collected
self._close = Finalize(
self, Queue._finalize_close,
[self._buffer, self._notempty],
exitpriority=10
)
@staticmethod
def _finalize_join(twr):
debug('joining queue thread')
thread = twr()
if thread is not None:
thread.join()
debug('... queue thread joined')
else:
debug('... queue thread already dead')
@staticmethod
def _finalize_close(buffer, notempty):
debug('telling queue thread to quit')
with notempty:
buffer.append(_sentinel)
notempty.notify()
@staticmethod
def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe):
debug('starting thread to feed data to pipe')
nacquire = notempty.acquire
nrelease = notempty.release
nwait = notempty.wait
bpopleft = buffer.popleft
sentinel = _sentinel
if sys.platform != 'win32':
wacquire = writelock.acquire
wrelease = writelock.release
else:
wacquire = None
while 1:
try:
nacquire()
try:
if not buffer:
nwait()
finally:
nrelease()
try:
while 1:
obj = bpopleft()
if obj is sentinel:
debug('feeder thread got sentinel -- exiting')
close()
return
# serialize the data before acquiring the lock
obj = _ForkingPickler.dumps(obj)
if wacquire is None:
send_bytes(obj)
else:
wacquire()
try:
send_bytes(obj)
finally:
wrelease()
except IndexError:
pass
except Exception as e:
if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE:
return
# Since this runs in a daemon thread the resources it uses
# may be become unusable while the process is cleaning up.
# We ignore errors which happen after the process has
# started to cleanup.
if is_exiting():
info('error in queue thread: %s', e)
return
else:
import traceback
traceback.print_exc()
_sentinel = object()
#
# A queue type which also supports join() and task_done() methods
#
# Note that if you do not call task_done() for each finished task then
# eventually the counter's semaphore may overflow causing Bad Things
# to happen.
#
class JoinableQueue(Queue):
def __init__(self, maxsize=0, *, ctx):
Queue.__init__(self, maxsize, ctx=ctx)
self._unfinished_tasks = ctx.Semaphore(0)
self._cond = ctx.Condition()
def __getstate__(self):
return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
def __setstate__(self, state):
Queue.__setstate__(self, state[:-2])
self._cond, self._unfinished_tasks = state[-2:]
Merged revisions 74277,74321,74323,74326,74355,74465,74467,74488,74492,74513,74531,74549,74553,74625,74632,74643-74644,74647,74652,74666,74671,74727,74739 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r74277 | sean.reifschneider | 2009-08-01 18:54:55 -0500 (Sat, 01 Aug 2009) | 3 lines - Issue #6624: yArg_ParseTuple with "s" format when parsing argument with NUL: Bogus TypeError detail string. ........ r74321 | guilherme.polo | 2009-08-05 11:51:41 -0500 (Wed, 05 Aug 2009) | 1 line Easier reference to find (at least while svn continues being used). ........ r74323 | guilherme.polo | 2009-08-05 18:48:26 -0500 (Wed, 05 Aug 2009) | 1 line Typo. ........ r74326 | jesse.noller | 2009-08-05 21:05:56 -0500 (Wed, 05 Aug 2009) | 1 line Fix issue 4660: spurious task_done errors in multiprocessing, remove doc note for from_address ........ r74355 | gregory.p.smith | 2009-08-12 12:02:37 -0500 (Wed, 12 Aug 2009) | 2 lines comment typo fix ........ r74465 | vinay.sajip | 2009-08-15 18:23:12 -0500 (Sat, 15 Aug 2009) | 1 line Added section on logging to one file from multiple processes. ........ r74467 | vinay.sajip | 2009-08-15 18:34:47 -0500 (Sat, 15 Aug 2009) | 1 line Refined section on logging to one file from multiple processes. ........ r74488 | vinay.sajip | 2009-08-17 08:14:37 -0500 (Mon, 17 Aug 2009) | 1 line Further refined section on logging to one file from multiple processes. ........ r74492 | r.david.murray | 2009-08-17 14:26:49 -0500 (Mon, 17 Aug 2009) | 2 lines Issue 6685: 'toupper' -> 'upper' in cgi doc example explanation. ........ r74513 | skip.montanaro | 2009-08-18 09:37:52 -0500 (Tue, 18 Aug 2009) | 1 line missing module ref (issue6723) ........ r74531 | vinay.sajip | 2009-08-20 17:04:32 -0500 (Thu, 20 Aug 2009) | 1 line Added section on exceptions raised during logging. ........ r74549 | benjamin.peterson | 2009-08-24 12:42:36 -0500 (Mon, 24 Aug 2009) | 1 line fix pdf building by teaching latex the right encoding package ........ r74553 | r.david.murray | 2009-08-26 20:04:59 -0500 (Wed, 26 Aug 2009) | 2 lines Remove leftover text from end of sentence. ........ r74625 | benjamin.peterson | 2009-09-01 17:27:57 -0500 (Tue, 01 Sep 2009) | 1 line remove the check that classmethod's argument is a callable ........ r74632 | georg.brandl | 2009-09-03 02:27:26 -0500 (Thu, 03 Sep 2009) | 1 line #6828: fix wrongly highlighted blocks. ........ r74643 | georg.brandl | 2009-09-04 01:59:20 -0500 (Fri, 04 Sep 2009) | 2 lines Issue #2666: Handle BROWSER environment variable properly for unknown browser names in the webbrowser module. ........ r74644 | georg.brandl | 2009-09-04 02:55:14 -0500 (Fri, 04 Sep 2009) | 1 line #5047: remove Monterey support from configure. ........ r74647 | georg.brandl | 2009-09-04 03:17:04 -0500 (Fri, 04 Sep 2009) | 2 lines Issue #5275: In Cookie's Cookie.load(), properly handle non-string arguments as documented. ........ r74652 | georg.brandl | 2009-09-04 06:25:37 -0500 (Fri, 04 Sep 2009) | 1 line #6756: add some info about the "acct" parameter. ........ r74666 | georg.brandl | 2009-09-05 04:04:09 -0500 (Sat, 05 Sep 2009) | 1 line #6841: remove duplicated word. ........ r74671 | georg.brandl | 2009-09-05 11:47:17 -0500 (Sat, 05 Sep 2009) | 1 line #6843: add link from filterwarnings to where the meaning of the arguments is covered. ........ r74727 | benjamin.peterson | 2009-09-08 18:04:22 -0500 (Tue, 08 Sep 2009) | 1 line #6865 fix ref counting in initialization of pwd module ........ r74739 | georg.brandl | 2009-09-11 02:55:20 -0500 (Fri, 11 Sep 2009) | 1 line Move function back to its section. ........
2009-09-11 19:24:02 -03:00
def put(self, obj, block=True, timeout=None):
assert not self._closed, "Queue {0!r} is closed".format(self)
Merged revisions 74277,74321,74323,74326,74355,74465,74467,74488,74492,74513,74531,74549,74553,74625,74632,74643-74644,74647,74652,74666,74671,74727,74739 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r74277 | sean.reifschneider | 2009-08-01 18:54:55 -0500 (Sat, 01 Aug 2009) | 3 lines - Issue #6624: yArg_ParseTuple with "s" format when parsing argument with NUL: Bogus TypeError detail string. ........ r74321 | guilherme.polo | 2009-08-05 11:51:41 -0500 (Wed, 05 Aug 2009) | 1 line Easier reference to find (at least while svn continues being used). ........ r74323 | guilherme.polo | 2009-08-05 18:48:26 -0500 (Wed, 05 Aug 2009) | 1 line Typo. ........ r74326 | jesse.noller | 2009-08-05 21:05:56 -0500 (Wed, 05 Aug 2009) | 1 line Fix issue 4660: spurious task_done errors in multiprocessing, remove doc note for from_address ........ r74355 | gregory.p.smith | 2009-08-12 12:02:37 -0500 (Wed, 12 Aug 2009) | 2 lines comment typo fix ........ r74465 | vinay.sajip | 2009-08-15 18:23:12 -0500 (Sat, 15 Aug 2009) | 1 line Added section on logging to one file from multiple processes. ........ r74467 | vinay.sajip | 2009-08-15 18:34:47 -0500 (Sat, 15 Aug 2009) | 1 line Refined section on logging to one file from multiple processes. ........ r74488 | vinay.sajip | 2009-08-17 08:14:37 -0500 (Mon, 17 Aug 2009) | 1 line Further refined section on logging to one file from multiple processes. ........ r74492 | r.david.murray | 2009-08-17 14:26:49 -0500 (Mon, 17 Aug 2009) | 2 lines Issue 6685: 'toupper' -> 'upper' in cgi doc example explanation. ........ r74513 | skip.montanaro | 2009-08-18 09:37:52 -0500 (Tue, 18 Aug 2009) | 1 line missing module ref (issue6723) ........ r74531 | vinay.sajip | 2009-08-20 17:04:32 -0500 (Thu, 20 Aug 2009) | 1 line Added section on exceptions raised during logging. ........ r74549 | benjamin.peterson | 2009-08-24 12:42:36 -0500 (Mon, 24 Aug 2009) | 1 line fix pdf building by teaching latex the right encoding package ........ r74553 | r.david.murray | 2009-08-26 20:04:59 -0500 (Wed, 26 Aug 2009) | 2 lines Remove leftover text from end of sentence. ........ r74625 | benjamin.peterson | 2009-09-01 17:27:57 -0500 (Tue, 01 Sep 2009) | 1 line remove the check that classmethod's argument is a callable ........ r74632 | georg.brandl | 2009-09-03 02:27:26 -0500 (Thu, 03 Sep 2009) | 1 line #6828: fix wrongly highlighted blocks. ........ r74643 | georg.brandl | 2009-09-04 01:59:20 -0500 (Fri, 04 Sep 2009) | 2 lines Issue #2666: Handle BROWSER environment variable properly for unknown browser names in the webbrowser module. ........ r74644 | georg.brandl | 2009-09-04 02:55:14 -0500 (Fri, 04 Sep 2009) | 1 line #5047: remove Monterey support from configure. ........ r74647 | georg.brandl | 2009-09-04 03:17:04 -0500 (Fri, 04 Sep 2009) | 2 lines Issue #5275: In Cookie's Cookie.load(), properly handle non-string arguments as documented. ........ r74652 | georg.brandl | 2009-09-04 06:25:37 -0500 (Fri, 04 Sep 2009) | 1 line #6756: add some info about the "acct" parameter. ........ r74666 | georg.brandl | 2009-09-05 04:04:09 -0500 (Sat, 05 Sep 2009) | 1 line #6841: remove duplicated word. ........ r74671 | georg.brandl | 2009-09-05 11:47:17 -0500 (Sat, 05 Sep 2009) | 1 line #6843: add link from filterwarnings to where the meaning of the arguments is covered. ........ r74727 | benjamin.peterson | 2009-09-08 18:04:22 -0500 (Tue, 08 Sep 2009) | 1 line #6865 fix ref counting in initialization of pwd module ........ r74739 | georg.brandl | 2009-09-11 02:55:20 -0500 (Fri, 11 Sep 2009) | 1 line Move function back to its section. ........
2009-09-11 19:24:02 -03:00
if not self._sem.acquire(block, timeout):
raise Full
with self._notempty, self._cond:
Merged revisions 74277,74321,74323,74326,74355,74465,74467,74488,74492,74513,74531,74549,74553,74625,74632,74643-74644,74647,74652,74666,74671,74727,74739 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r74277 | sean.reifschneider | 2009-08-01 18:54:55 -0500 (Sat, 01 Aug 2009) | 3 lines - Issue #6624: yArg_ParseTuple with "s" format when parsing argument with NUL: Bogus TypeError detail string. ........ r74321 | guilherme.polo | 2009-08-05 11:51:41 -0500 (Wed, 05 Aug 2009) | 1 line Easier reference to find (at least while svn continues being used). ........ r74323 | guilherme.polo | 2009-08-05 18:48:26 -0500 (Wed, 05 Aug 2009) | 1 line Typo. ........ r74326 | jesse.noller | 2009-08-05 21:05:56 -0500 (Wed, 05 Aug 2009) | 1 line Fix issue 4660: spurious task_done errors in multiprocessing, remove doc note for from_address ........ r74355 | gregory.p.smith | 2009-08-12 12:02:37 -0500 (Wed, 12 Aug 2009) | 2 lines comment typo fix ........ r74465 | vinay.sajip | 2009-08-15 18:23:12 -0500 (Sat, 15 Aug 2009) | 1 line Added section on logging to one file from multiple processes. ........ r74467 | vinay.sajip | 2009-08-15 18:34:47 -0500 (Sat, 15 Aug 2009) | 1 line Refined section on logging to one file from multiple processes. ........ r74488 | vinay.sajip | 2009-08-17 08:14:37 -0500 (Mon, 17 Aug 2009) | 1 line Further refined section on logging to one file from multiple processes. ........ r74492 | r.david.murray | 2009-08-17 14:26:49 -0500 (Mon, 17 Aug 2009) | 2 lines Issue 6685: 'toupper' -> 'upper' in cgi doc example explanation. ........ r74513 | skip.montanaro | 2009-08-18 09:37:52 -0500 (Tue, 18 Aug 2009) | 1 line missing module ref (issue6723) ........ r74531 | vinay.sajip | 2009-08-20 17:04:32 -0500 (Thu, 20 Aug 2009) | 1 line Added section on exceptions raised during logging. ........ r74549 | benjamin.peterson | 2009-08-24 12:42:36 -0500 (Mon, 24 Aug 2009) | 1 line fix pdf building by teaching latex the right encoding package ........ r74553 | r.david.murray | 2009-08-26 20:04:59 -0500 (Wed, 26 Aug 2009) | 2 lines Remove leftover text from end of sentence. ........ r74625 | benjamin.peterson | 2009-09-01 17:27:57 -0500 (Tue, 01 Sep 2009) | 1 line remove the check that classmethod's argument is a callable ........ r74632 | georg.brandl | 2009-09-03 02:27:26 -0500 (Thu, 03 Sep 2009) | 1 line #6828: fix wrongly highlighted blocks. ........ r74643 | georg.brandl | 2009-09-04 01:59:20 -0500 (Fri, 04 Sep 2009) | 2 lines Issue #2666: Handle BROWSER environment variable properly for unknown browser names in the webbrowser module. ........ r74644 | georg.brandl | 2009-09-04 02:55:14 -0500 (Fri, 04 Sep 2009) | 1 line #5047: remove Monterey support from configure. ........ r74647 | georg.brandl | 2009-09-04 03:17:04 -0500 (Fri, 04 Sep 2009) | 2 lines Issue #5275: In Cookie's Cookie.load(), properly handle non-string arguments as documented. ........ r74652 | georg.brandl | 2009-09-04 06:25:37 -0500 (Fri, 04 Sep 2009) | 1 line #6756: add some info about the "acct" parameter. ........ r74666 | georg.brandl | 2009-09-05 04:04:09 -0500 (Sat, 05 Sep 2009) | 1 line #6841: remove duplicated word. ........ r74671 | georg.brandl | 2009-09-05 11:47:17 -0500 (Sat, 05 Sep 2009) | 1 line #6843: add link from filterwarnings to where the meaning of the arguments is covered. ........ r74727 | benjamin.peterson | 2009-09-08 18:04:22 -0500 (Tue, 08 Sep 2009) | 1 line #6865 fix ref counting in initialization of pwd module ........ r74739 | georg.brandl | 2009-09-11 02:55:20 -0500 (Fri, 11 Sep 2009) | 1 line Move function back to its section. ........
2009-09-11 19:24:02 -03:00
if self._thread is None:
self._start_thread()
self._buffer.append(obj)
self._unfinished_tasks.release()
self._notempty.notify()
def task_done(self):
with self._cond:
if not self._unfinished_tasks.acquire(False):
raise ValueError('task_done() called too many times')
if self._unfinished_tasks._semlock._is_zero():
self._cond.notify_all()
def join(self):
with self._cond:
if not self._unfinished_tasks._semlock._is_zero():
self._cond.wait()
#
# Simplified Queue type -- really just a locked pipe
#
class SimpleQueue(object):
def __init__(self, *, ctx):
self._reader, self._writer = connection.Pipe(duplex=False)
self._rlock = ctx.Lock()
self._poll = self._reader.poll
if sys.platform == 'win32':
self._wlock = None
else:
self._wlock = ctx.Lock()
def empty(self):
return not self._poll()
def __getstate__(self):
context.assert_spawning(self)
return (self._reader, self._writer, self._rlock, self._wlock)
def __setstate__(self, state):
(self._reader, self._writer, self._rlock, self._wlock) = state
self._poll = self._reader.poll
def get(self):
with self._rlock:
res = self._reader.recv_bytes()
# unserialize the data after having released the lock
return _ForkingPickler.loads(res)
def put(self, obj):
# serialize the data before acquiring the lock
obj = _ForkingPickler.dumps(obj)
if self._wlock is None:
# writes to a message oriented win32 pipe are atomic
self._writer.send_bytes(obj)
else:
with self._wlock:
self._writer.send_bytes(obj)