cpython/Lib/multiprocessing/connection.py

419 lines
12 KiB
Python
Raw Normal View History

#
# A higher level module for using sockets (or Windows named pipes)
#
# multiprocessing/connection.py
#
# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
#
__all__ = [ 'Client', 'Listener', 'Pipe' ]
import os
import sys
import socket
import time
import tempfile
import itertools
import _multiprocessing
from multiprocessing import current_process
from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug
from multiprocessing.forking import duplicate, close
#
#
#
BUFSIZE = 8192
_mmap_counter = itertools.count()
default_family = 'AF_INET'
families = ['AF_INET']
if hasattr(socket, 'AF_UNIX'):
default_family = 'AF_UNIX'
families += ['AF_UNIX']
if sys.platform == 'win32':
default_family = 'AF_PIPE'
families += ['AF_PIPE']
#
#
#
def arbitrary_address(family):
'''
Return an arbitrary free address for the given family
'''
if family == 'AF_INET':
return ('localhost', 0)
elif family == 'AF_UNIX':
return tempfile.mktemp(prefix='listener-', dir=get_temp_dir())
elif family == 'AF_PIPE':
return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
(os.getpid(), next(_mmap_counter)))
else:
raise ValueError('unrecognized family')
def address_type(address):
'''
Return the types of the address
This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
'''
if type(address) == tuple:
return 'AF_INET'
elif type(address) is str and address.startswith('\\\\'):
return 'AF_PIPE'
elif type(address) is str:
return 'AF_UNIX'
else:
raise ValueError('address type of %r unrecognized' % address)
#
# Public functions
#
class Listener(object):
'''
Returns a listener object.
This is a wrapper for a bound socket which is 'listening' for
connections, or for a Windows named pipe.
'''
def __init__(self, address=None, family=None, backlog=1, authkey=None):
family = family or (address and address_type(address)) \
or default_family
address = address or arbitrary_address(family)
if family == 'AF_PIPE':
self._listener = PipeListener(address, backlog)
else:
self._listener = SocketListener(address, family, backlog)
if authkey is not None and not isinstance(authkey, bytes):
raise TypeError('authkey should be a byte string')
self._authkey = authkey
def accept(self):
'''
Accept a connection on the bound socket or named pipe of `self`.
Returns a `Connection` object.
'''
c = self._listener.accept()
if self._authkey:
deliver_challenge(c, self._authkey)
answer_challenge(c, self._authkey)
return c
def close(self):
'''
Close the bound socket or named pipe of `self`.
'''
return self._listener.close()
address = property(lambda self: self._listener._address)
last_accepted = property(lambda self: self._listener._last_accepted)
def Client(address, family=None, authkey=None):
'''
Returns a connection to the address of a `Listener`
'''
family = family or address_type(address)
if family == 'AF_PIPE':
c = PipeClient(address)
else:
c = SocketClient(address)
if authkey is not None and not isinstance(authkey, bytes):
raise TypeError('authkey should be a byte string')
if authkey is not None:
answer_challenge(c, authkey)
deliver_challenge(c, authkey)
return c
if sys.platform != 'win32':
def Pipe(duplex=True):
'''
Returns pair of connection objects at either end of a pipe
'''
if duplex:
s1, s2 = socket.socketpair()
c1 = _multiprocessing.Connection(os.dup(s1.fileno()))
c2 = _multiprocessing.Connection(os.dup(s2.fileno()))
s1.close()
s2.close()
else:
fd1, fd2 = os.pipe()
c1 = _multiprocessing.Connection(fd1, writable=False)
c2 = _multiprocessing.Connection(fd2, readable=False)
return c1, c2
else:
from ._multiprocessing import win32
def Pipe(duplex=True):
'''
Returns pair of connection objects at either end of a pipe
'''
address = arbitrary_address('AF_PIPE')
if duplex:
openmode = win32.PIPE_ACCESS_DUPLEX
access = win32.GENERIC_READ | win32.GENERIC_WRITE
obsize, ibsize = BUFSIZE, BUFSIZE
else:
openmode = win32.PIPE_ACCESS_INBOUND
access = win32.GENERIC_WRITE
obsize, ibsize = 0, BUFSIZE
h1 = win32.CreateNamedPipe(
address, openmode,
win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
win32.PIPE_WAIT,
1, obsize, ibsize, win32.NMPWAIT_WAIT_FOREVER, win32.NULL
)
h2 = win32.CreateFile(
address, access, 0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL
)
win32.SetNamedPipeHandleState(
h2, win32.PIPE_READMODE_MESSAGE, None, None
)
try:
win32.ConnectNamedPipe(h1, win32.NULL)
except WindowsError as e:
if e.args[0] != win32.ERROR_PIPE_CONNECTED:
raise
c1 = _multiprocessing.PipeConnection(h1, writable=duplex)
c2 = _multiprocessing.PipeConnection(h2, readable=duplex)
return c1, c2
#
# Definitions for connections based on sockets
#
class SocketListener(object):
'''
Merged revisions 65437,65469,65476,65480,65502,65528,65539,65543,65558,65561-65562,65565,65591,65601,65608,65610,65639 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r65437 | georg.brandl | 2008-08-03 22:28:55 +0000 (Sun, 03 Aug 2008) | 2 lines Note the removal of several committers. ........ r65469 | gregory.p.smith | 2008-08-04 01:03:50 +0000 (Mon, 04 Aug 2008) | 3 lines issue1606: Add warnings to the subprocess documentation about common pitfalls of using pipes that cause deadlocks. ........ r65476 | georg.brandl | 2008-08-04 06:29:36 +0000 (Mon, 04 Aug 2008) | 2 lines Fix markup. ........ r65480 | georg.brandl | 2008-08-04 07:31:50 +0000 (Mon, 04 Aug 2008) | 3 lines Clarify the meaning of the select() parameters and sync names with docstring. ........ r65502 | gregory.p.smith | 2008-08-04 18:34:07 +0000 (Mon, 04 Aug 2008) | 2 lines more cleanup ups of the recently added warnings in the subprocess docs. ........ r65528 | brett.cannon | 2008-08-04 21:52:25 +0000 (Mon, 04 Aug 2008) | 4 lines Add a note about all the modules/packages changed to silence -3 warnings. More changes are needed once some decisions are made, but this is the work up to this point. ........ r65539 | andrew.kuchling | 2008-08-05 01:38:08 +0000 (Tue, 05 Aug 2008) | 6 lines #3367 from Kristjan Valur Jonsson: If a PyTokenizer_FromString() is called with an empty string, the tokenizer's line_start member never gets initialized. Later, it is compared with the token pointer 'a' in parsetok.c:193 and that behavior can result in undefined behavior. ........ r65543 | andrew.kuchling | 2008-08-05 02:05:23 +0000 (Tue, 05 Aug 2008) | 1 line #3367: revert rev. 65539: this change causes test_parser to fail ........ r65558 | georg.brandl | 2008-08-06 17:20:41 +0000 (Wed, 06 Aug 2008) | 2 lines Fix longstringitem definition. #3505. ........ r65561 | mark.dickinson | 2008-08-06 20:12:30 +0000 (Wed, 06 Aug 2008) | 2 lines Docstring typo ........ r65562 | mark.dickinson | 2008-08-06 21:36:57 +0000 (Wed, 06 Aug 2008) | 2 lines Remove duplicate import ........ r65565 | andrew.kuchling | 2008-08-07 01:47:34 +0000 (Thu, 07 Aug 2008) | 1 line Add some items ........ r65591 | georg.brandl | 2008-08-08 06:42:20 +0000 (Fri, 08 Aug 2008) | 2 lines #3519: callee is an expression too. ........ r65601 | georg.brandl | 2008-08-08 15:34:34 +0000 (Fri, 08 Aug 2008) | 2 lines Remove mention of backquotes in the tutorial. ........ r65608 | guido.van.rossum | 2008-08-09 14:55:34 +0000 (Sat, 09 Aug 2008) | 2 lines Add news item about _sre.compile() re-bytecode validator. ........ r65610 | antoine.pitrou | 2008-08-09 17:27:23 +0000 (Sat, 09 Aug 2008) | 3 lines move NEWS entry to the appropriate section (oops!) ........ r65639 | georg.brandl | 2008-08-11 10:27:31 +0000 (Mon, 11 Aug 2008) | 2 lines #3540: fix exception name. ........
2008-08-12 05:18:18 -03:00
Representation of a socket which is bound to an address and listening
'''
def __init__(self, address, family, backlog=1):
self._socket = socket.socket(getattr(socket, family))
self._socket.bind(address)
self._socket.listen(backlog)
address = self._socket.getsockname()
if type(address) is tuple:
address = (socket.getfqdn(address[0]),) + address[1:]
self._address = address
self._family = family
self._last_accepted = None
if family == 'AF_UNIX':
self._unlink = Finalize(
Merged revisions 64722,64729,64753,64845-64846,64849,64871,64880-64882,64885,64888,64897,64900-64901,64915,64926-64929,64938-64941,64944,64961,64966,64973 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r64722 | georg.brandl | 2008-07-05 12:13:36 +0200 (Sat, 05 Jul 2008) | 4 lines #2663: support an *ignore* argument to shutil.copytree(). Patch by Tarek Ziade. This is a new feature, but Barry authorized adding it in the beta period. ........ r64729 | mark.dickinson | 2008-07-05 13:33:52 +0200 (Sat, 05 Jul 2008) | 5 lines Issue 3188: accept float('infinity') as well as float('inf'). This makes the float constructor behave in the same way as specified by various other language standards, including C99, IEEE 754r, and the IBM Decimal standard. ........ r64753 | gregory.p.smith | 2008-07-06 05:35:58 +0200 (Sun, 06 Jul 2008) | 4 lines - Issue #2862: Make int and float freelist management consistent with other freelists. Changes their CompactFreeList apis into ClearFreeList apis and calls them via gc.collect(). ........ r64845 | raymond.hettinger | 2008-07-10 16:03:19 +0200 (Thu, 10 Jul 2008) | 1 line Issue 3301: Bisect functions behaved badly when lo was negative. ........ r64846 | raymond.hettinger | 2008-07-10 16:34:57 +0200 (Thu, 10 Jul 2008) | 1 line Issue 3285: Fractions from_float() and from_decimal() accept Integral arguments. ........ r64849 | andrew.kuchling | 2008-07-10 16:43:31 +0200 (Thu, 10 Jul 2008) | 1 line Wording changes ........ r64871 | raymond.hettinger | 2008-07-11 14:00:21 +0200 (Fri, 11 Jul 2008) | 1 line Add cautionary note on the use of PySequence_Fast_ITEMS. ........ r64880 | amaury.forgeotdarc | 2008-07-11 23:28:25 +0200 (Fri, 11 Jul 2008) | 5 lines #3317 in zipfile module, restore the previous names of global variables: some applications relied on them. Also remove duplicated lines. ........ r64881 | amaury.forgeotdarc | 2008-07-11 23:45:06 +0200 (Fri, 11 Jul 2008) | 3 lines #3342: In tracebacks, printed source lines were not indented since r62555. #3343: Py_DisplaySourceLine should be a private function. Rename it to _Py_DisplaySourceLine. ........ r64882 | josiah.carlson | 2008-07-12 00:17:14 +0200 (Sat, 12 Jul 2008) | 2 lines Fix for the AttributeError in test_asynchat. ........ r64885 | josiah.carlson | 2008-07-12 01:26:59 +0200 (Sat, 12 Jul 2008) | 2 lines Fixed test for asyncore. ........ r64888 | matthias.klose | 2008-07-12 09:51:48 +0200 (Sat, 12 Jul 2008) | 2 lines - Fix bashisms in Tools/faqwiz/move-faqwiz.sh ........ r64897 | benjamin.peterson | 2008-07-12 22:16:19 +0200 (Sat, 12 Jul 2008) | 1 line fix various doc typos #3320 ........ r64900 | alexandre.vassalotti | 2008-07-13 00:06:53 +0200 (Sun, 13 Jul 2008) | 2 lines Fixed typo. ........ r64901 | benjamin.peterson | 2008-07-13 01:41:19 +0200 (Sun, 13 Jul 2008) | 1 line #1778443 robotparser fixes from Aristotelis Mikropoulos ........ r64915 | nick.coghlan | 2008-07-13 16:52:36 +0200 (Sun, 13 Jul 2008) | 1 line Fix issue 3221 by emitting a RuntimeWarning instead of raising SystemError when the parent module can't be found during an absolute import (likely due to non-PEP 361 aware code which sets a module level __package__ attribute) ........ r64926 | martin.v.loewis | 2008-07-13 22:31:49 +0200 (Sun, 13 Jul 2008) | 2 lines Add turtle into the module index. ........ r64927 | alexandre.vassalotti | 2008-07-13 22:42:44 +0200 (Sun, 13 Jul 2008) | 3 lines Issue #3274: Use a less common identifier for the temporary variable in Py_CLEAR(). ........ r64928 | andrew.kuchling | 2008-07-13 23:43:25 +0200 (Sun, 13 Jul 2008) | 1 line Re-word ........ r64929 | andrew.kuchling | 2008-07-13 23:43:52 +0200 (Sun, 13 Jul 2008) | 1 line Add various items; move ctypes items into a subsection of their own ........ r64938 | andrew.kuchling | 2008-07-14 02:35:32 +0200 (Mon, 14 Jul 2008) | 1 line Typo fixes ........ r64939 | andrew.kuchling | 2008-07-14 02:40:55 +0200 (Mon, 14 Jul 2008) | 1 line Typo fix ........ r64940 | andrew.kuchling | 2008-07-14 03:18:16 +0200 (Mon, 14 Jul 2008) | 1 line Typo fix ........ r64941 | andrew.kuchling | 2008-07-14 03:18:31 +0200 (Mon, 14 Jul 2008) | 1 line Expand the multiprocessing section ........ r64944 | gregory.p.smith | 2008-07-14 08:06:48 +0200 (Mon, 14 Jul 2008) | 7 lines Fix posix.fork1() / os.fork1() to only call PyOS_AfterFork() in the child process rather than both parent and child. Does anyone actually use fork1()? It appears to be a Solaris thing but if Python is built with pthreads on Solaris, fork1() and fork() should be the same. ........ r64961 | jesse.noller | 2008-07-15 15:47:33 +0200 (Tue, 15 Jul 2008) | 1 line multiprocessing/connection.py patch to remove fqdn oddness for issue 3270 ........ r64966 | nick.coghlan | 2008-07-15 17:40:22 +0200 (Tue, 15 Jul 2008) | 1 line Add missing NEWS entry for r64962 ........ r64973 | jesse.noller | 2008-07-15 20:29:18 +0200 (Tue, 15 Jul 2008) | 1 line Revert 3270 patch: self._address is in pretty widespread use, need to revisit ........
2008-07-16 09:55:28 -03:00
self, os.unlink, args=(address,), exitpriority=0
)
else:
self._unlink = None
def accept(self):
s, self._last_accepted = self._socket.accept()
fd = duplicate(s.fileno())
conn = _multiprocessing.Connection(fd)
s.close()
return conn
def close(self):
self._socket.close()
if self._unlink is not None:
self._unlink()
def SocketClient(address):
'''
Return a connection object connected to the socket given by `address`
'''
family = address_type(address)
s = socket.socket( getattr(socket, family) )
while 1:
try:
s.connect(address)
except socket.error as e:
if e.args[0] != 10061: # 10061 => connection refused
debug('failed to connect to address %s', address)
raise
time.sleep(0.01)
else:
break
else:
raise
fd = duplicate(s.fileno())
conn = _multiprocessing.Connection(fd)
s.close()
return conn
#
# Definitions for connections based on named pipes
#
if sys.platform == 'win32':
class PipeListener(object):
'''
Representation of a named pipe
'''
def __init__(self, address, backlog=None):
self._address = address
handle = win32.CreateNamedPipe(
address, win32.PIPE_ACCESS_DUPLEX,
win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
win32.PIPE_WAIT,
win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
win32.NMPWAIT_WAIT_FOREVER, win32.NULL
)
self._handle_queue = [handle]
self._last_accepted = None
sub_debug('listener created with address=%r', self._address)
self.close = Finalize(
self, PipeListener._finalize_pipe_listener,
args=(self._handle_queue, self._address), exitpriority=0
)
def accept(self):
newhandle = win32.CreateNamedPipe(
self._address, win32.PIPE_ACCESS_DUPLEX,
win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
win32.PIPE_WAIT,
win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
win32.NMPWAIT_WAIT_FOREVER, win32.NULL
)
self._handle_queue.append(newhandle)
handle = self._handle_queue.pop(0)
try:
win32.ConnectNamedPipe(handle, win32.NULL)
except WindowsError as e:
if e.args[0] != win32.ERROR_PIPE_CONNECTED:
raise
return _multiprocessing.PipeConnection(handle)
@staticmethod
def _finalize_pipe_listener(queue, address):
sub_debug('closing listener with address=%r', address)
for handle in queue:
close(handle)
def PipeClient(address):
'''
Return a connection object connected to the pipe given by `address`
'''
while 1:
try:
win32.WaitNamedPipe(address, 1000)
h = win32.CreateFile(
address, win32.GENERIC_READ | win32.GENERIC_WRITE,
0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL
)
except WindowsError as e:
if e.args[0] not in (win32.ERROR_SEM_TIMEOUT,
win32.ERROR_PIPE_BUSY):
raise
else:
break
else:
raise
win32.SetNamedPipeHandleState(
h, win32.PIPE_READMODE_MESSAGE, None, None
)
return _multiprocessing.PipeConnection(h)
#
# Authentication stuff
#
MESSAGE_LENGTH = 20
CHALLENGE = b'#CHALLENGE#'
WELCOME = b'#WELCOME#'
FAILURE = b'#FAILURE#'
def deliver_challenge(connection, authkey):
import hmac
assert isinstance(authkey, bytes)
message = os.urandom(MESSAGE_LENGTH)
connection.send_bytes(CHALLENGE + message)
digest = hmac.new(authkey, message).digest()
response = connection.recv_bytes(256) # reject large message
if response == digest:
connection.send_bytes(WELCOME)
else:
connection.send_bytes(FAILURE)
raise AuthenticationError('digest received was wrong')
def answer_challenge(connection, authkey):
import hmac
assert isinstance(authkey, bytes)
message = connection.recv_bytes(256) # reject large message
assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
message = message[len(CHALLENGE):]
digest = hmac.new(authkey, message).digest()
connection.send_bytes(digest)
response = connection.recv_bytes(256) # reject large message
if response != WELCOME:
raise AuthenticationError('digest sent was rejected')
#
# Support for using xmlrpclib for serialization
#
class ConnectionWrapper(object):
def __init__(self, conn, dumps, loads):
self._conn = conn
self._dumps = dumps
self._loads = loads
for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
obj = getattr(conn, attr)
setattr(self, attr, obj)
def send(self, obj):
s = self._dumps(obj)
self._conn.send_bytes(s)
def recv(self):
s = self._conn.recv_bytes()
return self._loads(s)
def _xml_dumps(obj):
return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf8')
def _xml_loads(s):
(obj,), method = xmlrpclib.loads(s.decode('utf8'))
return obj
class XmlListener(Listener):
def accept(self):
global xmlrpclib
import xmlrpc.client as xmlrpclib
obj = Listener.accept(self)
return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
def XmlClient(*args, **kwds):
global xmlrpclib
import xmlrpc.client as xmlrpclib
return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)