From 81dee6b4d4acc061e568e10f976eb77b084ddd26 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 11 Apr 2011 00:18:59 +0200 Subject: [PATCH] =?UTF-8?q?Issue=20#8428:=20Fix=20a=20race=20condition=20i?= =?UTF-8?q?n=20multiprocessing.Pool=20when=20terminating=20worker=20proces?= =?UTF-8?q?ses:=20new=20processes=20would=20be=20spawned=20while=20the=20p?= =?UTF-8?q?ool=20is=20being=20shut=20down.=20=20Patch=20by=20Charles-Fran?= =?UTF-8?q?=C3=A7ois=20Natali.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/multiprocessing/pool.py | 9 +++++++-- Misc/NEWS | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index ff7c29c85b6..efa0056732b 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -322,6 +322,8 @@ class Pool(object): while pool._worker_handler._state == RUN and pool._state == RUN: pool._maintain_pool() time.sleep(0.1) + # send sentinel to stop workers + pool._taskqueue.put(None) debug('worker handler exiting') @staticmethod @@ -440,7 +442,6 @@ class Pool(object): if self._state == RUN: self._state = CLOSE self._worker_handler._state = CLOSE - self._taskqueue.put(None) def terminate(self): debug('terminating pool') @@ -474,7 +475,6 @@ class Pool(object): worker_handler._state = TERMINATE task_handler._state = TERMINATE - taskqueue.put(None) # sentinel debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) @@ -484,6 +484,11 @@ class Pool(object): result_handler._state = TERMINATE outqueue.put(None) # sentinel + # We must wait for the worker handler to exit before terminating + # workers because we don't want workers to be restarted behind our back. + debug('joining worker handler') + worker_handler.join() + # Terminate workers which haven't already finished. if pool and hasattr(pool[0], 'terminate'): debug('terminating workers') diff --git a/Misc/NEWS b/Misc/NEWS index b5dc98b2388..413f387c47d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -53,6 +53,10 @@ Core and Builtins Library ------- +- Issue #8428: Fix a race condition in multiprocessing.Pool when terminating + worker processes: new processes would be spawned while the pool is being + shut down. Patch by Charles-François Natali. + - Issue #7311: fix html.parser to accept non-ASCII attribute values. - Issue #11605: email.parser.BytesFeedParser was incorrectly converting multipart