bpo-30919: shared memory allocation performance regression in multiprocessing (#2708)
* Fix #30919: shared memory allocation performance regression in multiprocessing * Change strategy for Arena directory choice * Add blurb
This commit is contained in:
parent
2b1e6e9696
commit
3051f0b78e
|
@ -60,26 +60,32 @@ if sys.platform == 'win32':
|
||||||
else:
|
else:
|
||||||
|
|
||||||
class Arena(object):
|
class Arena(object):
|
||||||
|
if sys.platform == 'linux':
|
||||||
|
_dir_candidates = ['/dev/shm']
|
||||||
|
else:
|
||||||
|
_dir_candidates = []
|
||||||
|
|
||||||
def __init__(self, size, fd=-1):
|
def __init__(self, size, fd=-1):
|
||||||
self.size = size
|
self.size = size
|
||||||
self.fd = fd
|
self.fd = fd
|
||||||
if fd == -1:
|
if fd == -1:
|
||||||
self.fd, name = tempfile.mkstemp(
|
self.fd, name = tempfile.mkstemp(
|
||||||
prefix='pym-%d-'%os.getpid(), dir=util.get_temp_dir())
|
prefix='pym-%d-'%os.getpid(),
|
||||||
|
dir=self._choose_dir(size))
|
||||||
os.unlink(name)
|
os.unlink(name)
|
||||||
util.Finalize(self, os.close, (self.fd,))
|
util.Finalize(self, os.close, (self.fd,))
|
||||||
with open(self.fd, 'wb', closefd=False) as f:
|
os.ftruncate(self.fd, size)
|
||||||
bs = 1024 * 1024
|
|
||||||
if size >= bs:
|
|
||||||
zeros = b'\0' * bs
|
|
||||||
for _ in range(size // bs):
|
|
||||||
f.write(zeros)
|
|
||||||
del zeros
|
|
||||||
f.write(b'\0' * (size % bs))
|
|
||||||
assert f.tell() == size
|
|
||||||
self.buffer = mmap.mmap(self.fd, self.size)
|
self.buffer = mmap.mmap(self.fd, self.size)
|
||||||
|
|
||||||
|
def _choose_dir(self, size):
|
||||||
|
# Choose a non-storage backed directory if possible,
|
||||||
|
# to improve performance
|
||||||
|
for d in self._dir_candidates:
|
||||||
|
st = os.statvfs(d)
|
||||||
|
if st.f_bavail * st.f_frsize >= size: # enough free space?
|
||||||
|
return d
|
||||||
|
return util.get_temp_dir()
|
||||||
|
|
||||||
def reduce_arena(a):
|
def reduce_arena(a):
|
||||||
if a.fd == -1:
|
if a.fd == -1:
|
||||||
raise ValueError('Arena is unpicklable because '
|
raise ValueError('Arena is unpicklable because '
|
||||||
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
Fix shared memory performance regression in multiprocessing in 3.x.
|
||||||
|
|
||||||
|
Shared memory used anonymous memory mappings in 2.x, while 3.x mmaps actual
|
||||||
|
files. Try to be careful to do as little disk I/O as possible.
|
Loading…
Reference in New Issue