From 59fb38b5820e9f61f6e64e506c2dad98f7f32971 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 9 Sep 2012 13:16:15 -0400 Subject: [PATCH 1/4] Issue #15881: Fixed atexit hook in multiprocessing. --- Doc/ACKS.txt | 1 + Lib/multiprocessing/util.py | 50 ++++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/Doc/ACKS.txt b/Doc/ACKS.txt index 26f7ea907f6..69735ca06c1 100644 --- a/Doc/ACKS.txt +++ b/Doc/ACKS.txt @@ -236,3 +236,4 @@ docs@python.org), and we'll be glad to correct the problem. * Moshe Zadka * Milan Zamazal * Cheng Zhang + * Chris McDonough diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 30b7a85fa26..68d6c1f9c4a 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -247,6 +247,12 @@ def _run_finalizers(minpriority=None): Finalizers with highest priority are called first; finalizers with the same priority will be called in reverse order of creation. ''' + if _finalizer_registry is None: + # This function may be called after this module's globals are + # destroyed. See the _exit_function function in this module for more + # notes. + return + if minpriority is None: f = lambda p : p[0][0] is not None else: @@ -278,24 +284,40 @@ def is_exiting(): _exiting = False -def _exit_function(): +def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, + active_children=active_children, + current_process=current_process): + # We hold on to references to functions in the arglist due to the + # situation described below, where this function is called after this + # module's globals are destroyed. + global _exiting - info('process shutting down') - debug('running all "atexit" finalizers with priority >= 0') - _run_finalizers(0) + if not _exiting: + info('process shutting down') + debug('running all "atexit" finalizers with priority >= 0') + _run_finalizers(0) + if current_process() is not None: + # We check if the current process is None here because if + # it's None, any call to ``active_children()`` will throw an + # AttributeError (active_children winds up trying to get + # attributes from util._current_process). This happens in a + # variety of shutdown circumstances that are not well-understood + # because module-scope variables are not apparently supposed to + # be destroyed until after this function is called. However, + # they are indeed destroyed before this function is called. See + # issues #9775 and #15881. Also related: #4106, #9205, and #9207. + for p in active_children(): + if p._daemonic: + info('calling terminate() for daemon %s', p.name) + p._popen.terminate() - for p in active_children(): - if p._daemonic: - info('calling terminate() for daemon %s', p.name) - p._popen.terminate() + for p in active_children(): + info('calling join() for process %s', p.name) + p.join() - for p in active_children(): - info('calling join() for process %s', p.name) - p.join() - - debug('running the remaining "atexit" finalizers') - _run_finalizers() + debug('running the remaining "atexit" finalizers') + _run_finalizers() atexit.register(_exit_function) From 36351564a252d396fcaeb63b785b21226bfacd67 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 9 Sep 2012 13:22:45 -0400 Subject: [PATCH 2/4] Fixed whitespace --- Lib/multiprocessing/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 68d6c1f9c4a..5e501bda625 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -252,7 +252,7 @@ def _run_finalizers(minpriority=None): # destroyed. See the _exit_function function in this module for more # notes. return - + if minpriority is None: f = lambda p : p[0][0] is not None else: From 5bedef3e64825c65e7bae51d2aea73f8bea34dd8 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 9 Sep 2012 13:31:08 -0400 Subject: [PATCH 3/4] Issue #15881: Fixed 3.2 backport. --- Lib/multiprocessing/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 5e501bda625..20bba3764c2 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -294,6 +294,8 @@ def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, global _exiting if not _exiting: + _exiting = True + info('process shutting down') debug('running all "atexit" finalizers with priority >= 0') _run_finalizers(0) From 7330da42978ffcde525295ffc928ee3e0eedfb1b Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 9 Sep 2012 14:11:45 -0400 Subject: [PATCH 4/4] Issue #15881: Added NEWS entry and proper credit. --- Doc/ACKS.txt | 1 - Misc/NEWS | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/ACKS.txt b/Doc/ACKS.txt index 69735ca06c1..26f7ea907f6 100644 --- a/Doc/ACKS.txt +++ b/Doc/ACKS.txt @@ -236,4 +236,3 @@ docs@python.org), and we'll be glad to correct the problem. * Moshe Zadka * Milan Zamazal * Cheng Zhang - * Chris McDonough diff --git a/Misc/NEWS b/Misc/NEWS index de76a78f5e5..8672b0a468b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -120,6 +120,9 @@ Core and Builtins Library ------- +- Issue #15881: Fixed atexit hook in multiprocessing. Original patch + by Chris McDonough. + - Issue #15340: Fix importing the random module when /dev/urandom cannot be opened. This was a regression caused by the hash randomization patch.