Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-121313: multiprocessing: simplify by increasing the connection buffer size to 64KiB (GH-123559) 

Increases the multiprocessing connection buffer size from 8k to 64k for efficiency, without overallocating.

Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
Co-authored-by: Victor Stinner <vstinner@python.org>
This commit is contained in:
Inada Naoki 2024-09-03 12:32:38 +09:00 committed by GitHub
parent 1f4a49ea53
commit 13f61bf7f1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 5 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Lib/multiprocessing/connection.py
View File

@ -13,7 +13,6 @@ import errno
import io
import itertools
import os
import stat
import sys
import socket
import struct
@ -40,7 +39,9 @@ except ImportError:
#
#
BUFSIZE = 8192
# 64 KiB is the default PIPE buffer size of most POSIX platforms.
BUFSIZE = 64 * 1024
# A very generous timeout when it comes to local connections...
CONNECTION_TIMEOUT = 20.
@ -361,11 +362,6 @@ if _winapi:
f.write(ov.getbuffer())
return f
"""
The default size of a pipe on Linux systems is 16 times the base page size:
https://man7.org/linux/man-pages/man7/pipe.7.html
"""
PAGES_PER_PIPE = 16
class Connection(_ConnectionBase):
"""
@ -378,14 +374,11 @@ class Connection(_ConnectionBase):
_close(self._handle)
_write = _multiprocessing.send
_read = _multiprocessing.recv
_default_pipe_size = 0
else:
def _close(self, _close=os.close):
_close(self._handle)
_write = os.write
_read = os.read
_base_page_size = os.sysconf(os.sysconf_names['SC_PAGESIZE'])
_default_pipe_size = _base_page_size * PAGES_PER_PIPE
def _send(self, buf, write=_write):
remaining = len(buf)
@ -400,13 +393,8 @@ class Connection(_ConnectionBase):
buf = io.BytesIO()
handle = self._handle
remaining = size
is_pipe = False
if size > self._default_pipe_size > 0:
mode = os.fstat(handle).st_mode
is_pipe = stat.S_ISFIFO(mode)
limit = self._default_pipe_size if is_pipe else remaining
while remaining > 0:
to_read = min(limit, remaining)
to_read = min(BUFSIZE, remaining)
chunk = read(handle, to_read)
n = len(chunk)
if n == 0:

1
Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst
View File

@ -1 +0,0 @@
Limit reading size in multiprocessing connection._recv for pipes to default pipe size of 16 times base page size, in order to avoid memory overallocation and unnecessary memory management system calls.

1
Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst Normal file
View File

@ -0,0 +1 @@
Limit the reading size in the :class:`multiprocessing.connection.Connection` class to 64 KiB to prevent memory overallocation and unnecessary memory management system calls.