Revert "bpo-34172: multiprocessing.Pool leaks resources after being deleted (GH-8450)" (GH-10971)
This reverts commit 97bfe8d3eb
.
This commit is contained in:
parent
8752dfbd1f
commit
9dfc754d61
|
@ -149,9 +149,8 @@ class Pool(object):
|
||||||
'''
|
'''
|
||||||
_wrap_exception = True
|
_wrap_exception = True
|
||||||
|
|
||||||
@staticmethod
|
def Process(self, *args, **kwds):
|
||||||
def Process(ctx, *args, **kwds):
|
return self._ctx.Process(*args, **kwds)
|
||||||
return ctx.Process(*args, **kwds)
|
|
||||||
|
|
||||||
def __init__(self, processes=None, initializer=None, initargs=(),
|
def __init__(self, processes=None, initializer=None, initargs=(),
|
||||||
maxtasksperchild=None, context=None):
|
maxtasksperchild=None, context=None):
|
||||||
|
@ -186,15 +185,13 @@ class Pool(object):
|
||||||
|
|
||||||
self._worker_handler = threading.Thread(
|
self._worker_handler = threading.Thread(
|
||||||
target=Pool._handle_workers,
|
target=Pool._handle_workers,
|
||||||
args=(self._cache, self._taskqueue, self._ctx, self.Process,
|
args=(self, )
|
||||||
self._processes, self._pool, self._inqueue, self._outqueue,
|
|
||||||
self._initializer, self._initargs, self._maxtasksperchild,
|
|
||||||
self._wrap_exception)
|
|
||||||
)
|
)
|
||||||
self._worker_handler.daemon = True
|
self._worker_handler.daemon = True
|
||||||
self._worker_handler._state = RUN
|
self._worker_handler._state = RUN
|
||||||
self._worker_handler.start()
|
self._worker_handler.start()
|
||||||
|
|
||||||
|
|
||||||
self._task_handler = threading.Thread(
|
self._task_handler = threading.Thread(
|
||||||
target=Pool._handle_tasks,
|
target=Pool._handle_tasks,
|
||||||
args=(self._taskqueue, self._quick_put, self._outqueue,
|
args=(self._taskqueue, self._quick_put, self._outqueue,
|
||||||
|
@ -220,62 +217,43 @@ class Pool(object):
|
||||||
exitpriority=15
|
exitpriority=15
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
def _join_exited_workers(self):
|
||||||
def _join_exited_workers(pool):
|
|
||||||
"""Cleanup after any worker processes which have exited due to reaching
|
"""Cleanup after any worker processes which have exited due to reaching
|
||||||
their specified lifetime. Returns True if any workers were cleaned up.
|
their specified lifetime. Returns True if any workers were cleaned up.
|
||||||
"""
|
"""
|
||||||
cleaned = False
|
cleaned = False
|
||||||
for i in reversed(range(len(pool))):
|
for i in reversed(range(len(self._pool))):
|
||||||
worker = pool[i]
|
worker = self._pool[i]
|
||||||
if worker.exitcode is not None:
|
if worker.exitcode is not None:
|
||||||
# worker exited
|
# worker exited
|
||||||
util.debug('cleaning up worker %d' % i)
|
util.debug('cleaning up worker %d' % i)
|
||||||
worker.join()
|
worker.join()
|
||||||
cleaned = True
|
cleaned = True
|
||||||
del pool[i]
|
del self._pool[i]
|
||||||
return cleaned
|
return cleaned
|
||||||
|
|
||||||
def _repopulate_pool(self):
|
def _repopulate_pool(self):
|
||||||
return self._repopulate_pool_static(self._ctx, self.Process,
|
|
||||||
self._processes,
|
|
||||||
self._pool, self._inqueue,
|
|
||||||
self._outqueue, self._initializer,
|
|
||||||
self._initargs,
|
|
||||||
self._maxtasksperchild,
|
|
||||||
self._wrap_exception)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _repopulate_pool_static(ctx, Process, processes, pool, inqueue,
|
|
||||||
outqueue, initializer, initargs,
|
|
||||||
maxtasksperchild, wrap_exception):
|
|
||||||
"""Bring the number of pool processes up to the specified number,
|
"""Bring the number of pool processes up to the specified number,
|
||||||
for use after reaping workers which have exited.
|
for use after reaping workers which have exited.
|
||||||
"""
|
"""
|
||||||
for i in range(processes - len(pool)):
|
for i in range(self._processes - len(self._pool)):
|
||||||
w = Process(ctx, target=worker,
|
w = self.Process(target=worker,
|
||||||
args=(inqueue, outqueue,
|
args=(self._inqueue, self._outqueue,
|
||||||
initializer,
|
self._initializer,
|
||||||
initargs, maxtasksperchild,
|
self._initargs, self._maxtasksperchild,
|
||||||
wrap_exception)
|
self._wrap_exception)
|
||||||
)
|
)
|
||||||
w.name = w.name.replace('Process', 'PoolWorker')
|
w.name = w.name.replace('Process', 'PoolWorker')
|
||||||
w.daemon = True
|
w.daemon = True
|
||||||
w.start()
|
w.start()
|
||||||
pool.append(w)
|
self._pool.append(w)
|
||||||
util.debug('added worker')
|
util.debug('added worker')
|
||||||
|
|
||||||
@staticmethod
|
def _maintain_pool(self):
|
||||||
def _maintain_pool(ctx, Process, processes, pool, inqueue, outqueue,
|
|
||||||
initializer, initargs, maxtasksperchild,
|
|
||||||
wrap_exception):
|
|
||||||
"""Clean up any exited workers and start replacements for them.
|
"""Clean up any exited workers and start replacements for them.
|
||||||
"""
|
"""
|
||||||
if Pool._join_exited_workers(pool):
|
if self._join_exited_workers():
|
||||||
Pool._repopulate_pool_static(ctx, Process, processes, pool,
|
self._repopulate_pool()
|
||||||
inqueue, outqueue, initializer,
|
|
||||||
initargs, maxtasksperchild,
|
|
||||||
wrap_exception)
|
|
||||||
|
|
||||||
def _setup_queues(self):
|
def _setup_queues(self):
|
||||||
self._inqueue = self._ctx.SimpleQueue()
|
self._inqueue = self._ctx.SimpleQueue()
|
||||||
|
@ -433,20 +411,16 @@ class Pool(object):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _handle_workers(cache, taskqueue, ctx, Process, processes, pool,
|
def _handle_workers(pool):
|
||||||
inqueue, outqueue, initializer, initargs,
|
|
||||||
maxtasksperchild, wrap_exception):
|
|
||||||
thread = threading.current_thread()
|
thread = threading.current_thread()
|
||||||
|
|
||||||
# Keep maintaining workers until the cache gets drained, unless the pool
|
# Keep maintaining workers until the cache gets drained, unless the pool
|
||||||
# is terminated.
|
# is terminated.
|
||||||
while thread._state == RUN or (cache and thread._state != TERMINATE):
|
while thread._state == RUN or (pool._cache and thread._state != TERMINATE):
|
||||||
Pool._maintain_pool(ctx, Process, processes, pool, inqueue,
|
pool._maintain_pool()
|
||||||
outqueue, initializer, initargs,
|
|
||||||
maxtasksperchild, wrap_exception)
|
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
# send sentinel to stop workers
|
# send sentinel to stop workers
|
||||||
taskqueue.put(None)
|
pool._taskqueue.put(None)
|
||||||
util.debug('worker handler exiting')
|
util.debug('worker handler exiting')
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -828,7 +802,7 @@ class ThreadPool(Pool):
|
||||||
_wrap_exception = False
|
_wrap_exception = False
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def Process(ctx, *args, **kwds):
|
def Process(*args, **kwds):
|
||||||
from .dummy import Process
|
from .dummy import Process
|
||||||
return Process(*args, **kwds)
|
return Process(*args, **kwds)
|
||||||
|
|
||||||
|
|
|
@ -2558,13 +2558,6 @@ class _TestPool(BaseTestCase):
|
||||||
# they were released too.
|
# they were released too.
|
||||||
self.assertEqual(CountedObject.n_instances, 0)
|
self.assertEqual(CountedObject.n_instances, 0)
|
||||||
|
|
||||||
@support.reap_threads
|
|
||||||
def test_del_pool(self):
|
|
||||||
p = self.Pool(1)
|
|
||||||
wr = weakref.ref(p)
|
|
||||||
del p
|
|
||||||
gc.collect()
|
|
||||||
self.assertIsNone(wr())
|
|
||||||
|
|
||||||
def raising():
|
def raising():
|
||||||
raise KeyError("key")
|
raise KeyError("key")
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
Fix a reference issue inside multiprocessing.Pool that caused the pool to remain alive if it was deleted without being closed or terminated explicitly.
|
|
Loading…
Reference in New Issue