From f8859e1808eff603b727d1adbeb38f745c9fedb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Mon, 24 Oct 2011 18:45:29 +0200 Subject: [PATCH 1/2] Issue #10332: multiprocessing: fix a race condition when a Pool is closed before all tasks have completed. --- Lib/multiprocessing/pool.py | 6 +++++- Lib/test/test_multiprocessing.py | 14 ++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index e450319d4f4..04e7c447d17 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -321,7 +321,11 @@ class Pool(object): @staticmethod def _handle_workers(pool): - while pool._worker_handler._state == RUN and pool._state == RUN: + thread = threading.current_thread() + + # Keep maintaining workers until the cache gets drained, unless the pool + # is terminated. + while thread._state == RUN or (pool._cache and thread._state != TERMINATE): pool._maintain_pool() time.sleep(0.1) # send sentinel to stop workers diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 6940d0e2f7e..45bf4549fee 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -1217,6 +1217,20 @@ class _TestPoolWorkerLifetime(BaseTestCase): p.close() p.join() + def test_pool_worker_lifetime_early_close(self): + # Issue #10332: closing a pool whose workers have limited lifetimes + # before all the tasks completed would make join() hang. + p = multiprocessing.Pool(3, maxtasksperchild=1) + results = [] + for i in range(6): + results.append(p.apply_async(sqr, (i, 0.3))) + p.close() + p.join() + # check the results + for (j, res) in enumerate(results): + self.assertEqual(res.get(), sqr(j)) + + # # Test that manager has expected number of shared objects left # diff --git a/Misc/NEWS b/Misc/NEWS index 3909420e12e..f7a741a32aa 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -58,6 +58,9 @@ Core and Builtins Library ------- +- Issue #10332: multiprocessing: fix a race condition when a Pool is closed + before all tasks have completed. + - Issue #13255: wrong docstrings in array module. - Issue #9168: now smtpd is able to bind privileged port. From a94200e6ce17d794c121cbeff7d9b61b206a2429 Mon Sep 17 00:00:00 2001 From: Petri Lehtinen Date: Mon, 24 Oct 2011 21:12:58 +0300 Subject: [PATCH 2/2] Issue #13018: Fix reference leaks in error paths in dictobject.c. Patch by Suman Saha. --- Misc/NEWS | 3 +++ Objects/dictobject.c | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index c078623d3b4..115ff696a6b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.2.3? Core and Builtins ----------------- +- Issue #13018: Fix reference leaks in error paths in dictobject.c. + Patch by Suman Saha. + - Issue #1294232: In a few cases involving metaclass inheritance, the interpreter would sometimes invoke the wrong metaclass when building a new class object. These cases now behave correctly. Patch by Daniel Urban. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 3fa5cc4b95b..79894e56049 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1314,14 +1314,18 @@ dict_fromkeys(PyObject *cls, PyObject *args) PyObject *key; Py_hash_t hash; - if (dictresize(mp, Py_SIZE(seq))) + if (dictresize(mp, Py_SIZE(seq))) { + Py_DECREF(d); return NULL; + } while (_PyDict_Next(seq, &pos, &key, &oldvalue, &hash)) { Py_INCREF(key); Py_INCREF(value); - if (insertdict(mp, key, hash, value)) + if (insertdict(mp, key, hash, value)) { + Py_DECREF(d); return NULL; + } } return d; } @@ -1332,14 +1336,18 @@ dict_fromkeys(PyObject *cls, PyObject *args) PyObject *key; Py_hash_t hash; - if (dictresize(mp, PySet_GET_SIZE(seq))) + if (dictresize(mp, PySet_GET_SIZE(seq))) { + Py_DECREF(d); return NULL; + } while (_PySet_NextEntry(seq, &pos, &key, &hash)) { Py_INCREF(key); Py_INCREF(value); - if (insertdict(mp, key, hash, value)) + if (insertdict(mp, key, hash, value)) { + Py_DECREF(d); return NULL; + } } return d; }