From e2202365eddfa7a2af88920d272f9a4a3f77f9ad Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 7 Sep 2013 15:19:30 +0300 Subject: [PATCH 01/34] #18894: remove mention of deprecated fail* methods. --- Doc/library/unittest.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 0c00a261c44..bdecb3ada53 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1577,8 +1577,7 @@ Loading and running tests A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.fail\*` or - :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :meth:`TestCase.assert\*` methods. .. attribute:: skipped From e64a91a890eb8c114c7626ad1dbc20289cf52a7e Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 7 Sep 2013 15:23:36 +0300 Subject: [PATCH 02/34] #18895: split a sentence in unittest docs. --- Doc/library/unittest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index bdecb3ada53..4c39b826001 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1674,7 +1674,7 @@ Loading and running tests .. method:: addError(test, err) - Called when the test case *test* raises an unexpected exception *err* is a + Called when the test case *test* raises an unexpected exception. *err* is a tuple of the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. From b988ee0632a71dbc5e1c599ef46bff24b915c1c6 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Sat, 7 Sep 2013 17:40:45 +0100 Subject: [PATCH 03/34] Fix conversion from Py_ssize_t to int. --- Modules/_multiprocessing/multiprocessing.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 30cb5eb4511..1aaf3605714 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -99,13 +99,15 @@ multiprocessing_send(PyObject *self, PyObject *args) { HANDLE handle; Py_buffer buf; - int ret; + int ret, length; if (!PyArg_ParseTuple(args, F_HANDLE "y*:send" , &handle, &buf)) return NULL; + length = (int)Py_MIN(buf.len, INT_MAX); + Py_BEGIN_ALLOW_THREADS - ret = send((SOCKET) handle, buf.buf, buf.len, 0); + ret = send((SOCKET) handle, buf.buf, length, 0); Py_END_ALLOW_THREADS PyBuffer_Release(&buf); From 1b90227c0dbf69048653ef4fc70570bc6399781d Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 7 Sep 2013 11:28:58 -0700 Subject: [PATCH 04/34] Removing the mention of os.isatty mention as Unix only Correct the wrong documentation. --- Doc/library/os.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 8bbb2ec6061..074f12cea27 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -744,8 +744,6 @@ as internal buffering of data. As of Python 3.3, this is equivalent to ``os.pathconf(fd, name)``. - Availability: Unix. - .. function:: fstat(fd) From 8ef519b24f61d6e4307a4886920482c30f93aa7a Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 7 Sep 2013 13:59:17 -0700 Subject: [PATCH 05/34] Fix License URL display and add test to check for license url presence. Fixes issue #18206 Patch contributed by Berker Peksag and py.user --- Lib/site.py | 3 ++- Lib/test/test_site.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 13e73364ae4..7e097014155 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -451,7 +451,8 @@ def setcopyright(): for supporting Python development. See www.python.org for more information.""") here = os.path.dirname(os.__file__) builtins.license = _Printer( - "license", "See http://www.python.org/%.3s/license.html" % sys.version, + "license", + "See http://www.python.org/download/releases/%.5s/license/" % sys.version, ["LICENSE.txt", "LICENSE"], [os.path.join(here, os.pardir), here, os.curdir]) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 29286c710b5..2392d437359 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -5,6 +5,7 @@ executing have not been removed. """ import unittest +import test.support from test.support import run_unittest, TESTFN, EnvironmentVarGuard from test.support import captured_stderr import builtins @@ -377,9 +378,10 @@ class ImportSideEffectTests(unittest.TestCase): self.assertTrue(hasattr(builtins, "exit")) def test_setting_copyright(self): - # 'copyright' and 'credits' should be in builtins + # 'copyright', 'credits', and 'license' should be in builtins self.assertTrue(hasattr(builtins, "copyright")) self.assertTrue(hasattr(builtins, "credits")) + self.assertTrue(hasattr(builtins, "license")) def test_setting_help(self): # 'help' should be set in builtins @@ -405,8 +407,30 @@ class ImportSideEffectTests(unittest.TestCase): else: self.fail("sitecustomize not imported automatically") + +class LicenseURL(unittest.TestCase): + """Test accessibility of the license.""" + + @unittest.skipUnless(str(license).startswith('See http://'), + 'license is available as a file') + def test_license_page(self): + """urlopen should return the license page""" + pat = r'^See (http://www\.python\.org/download/releases/[^/]+/license/)$' + mo = re.search(pat, str(license)) + self.assertIsNotNone(mo, msg='can\'t find appropriate url in license') + if mo is not None: + url = mo.group(1) + with test.support.transient_internet(url): + import urllib.request, urllib.error + try: + with urllib.request.urlopen(url) as data: + code = data.getcode() + except urllib.error.HTTPError as e: + code = e.code + self.assertEqual(code, 200, msg=url) + def test_main(): - run_unittest(HelperFunctionsTests, ImportSideEffectTests) + run_unittest(HelperFunctionsTests, ImportSideEffectTests, LicenseURL) if __name__ == "__main__": test_main() From eda7c641515f8af1f8700634b93eba859967379c Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 7 Sep 2013 14:12:55 -0700 Subject: [PATCH 06/34] Fix the merge conflict --- Lib/site.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 0bb2ea0081e..c4ea6f6adb9 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -361,14 +361,8 @@ def setcopyright(): Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands for supporting Python development. See www.python.org for more information.""") here = os.path.dirname(os.__file__) -<<<<<<< local builtins.license = _sitebuiltins._Printer( "license", "See http://www.python.org/%.3s/license.html" % sys.version, -======= - builtins.license = _Printer( - "license", - "See http://www.python.org/download/releases/%.5s/license/" % sys.version, ->>>>>>> other ["LICENSE.txt", "LICENSE"], [os.path.join(here, os.pardir), here, os.curdir]) From 7b4769937fb612d576b6829c3b834f3dd31752f1 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 7 Sep 2013 23:38:37 +0200 Subject: [PATCH 07/34] Issue #18808: Thread.join() now waits for the underlying thread state to be destroyed before returning. This prevents unpredictable aborts in Py_EndInterpreter() when some non-daemon threads are still running. --- Include/pystate.h | 26 ++++++++++++ Lib/_dummy_thread.py | 4 ++ Lib/test/test_threading.py | 70 ++++++++++++++++++++++++++++++- Lib/threading.py | 85 ++++++++++++++++++++++++-------------- Misc/NEWS | 4 ++ Modules/_threadmodule.c | 62 +++++++++++++++++++++++++++ Python/pystate.c | 5 +++ 7 files changed, 223 insertions(+), 33 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index e41fe4c0d5f..ddc68922cb1 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -118,6 +118,32 @@ typedef struct _ts { int trash_delete_nesting; PyObject *trash_delete_later; + /* Called when a thread state is deleted normally, but not when it + * is destroyed after fork(). + * Pain: to prevent rare but fatal shutdown errors (issue 18808), + * Thread.join() must wait for the join'ed thread's tstate to be unlinked + * from the tstate chain. That happens at the end of a thread's life, + * in pystate.c. + * The obvious way doesn't quite work: create a lock which the tstate + * unlinking code releases, and have Thread.join() wait to acquire that + * lock. The problem is that we _are_ at the end of the thread's life: + * if the thread holds the last reference to the lock, decref'ing the + * lock will delete the lock, and that may trigger arbitrary Python code + * if there's a weakref, with a callback, to the lock. But by this time + * _PyThreadState_Current is already NULL, so only the simplest of C code + * can be allowed to run (in particular it must not be possible to + * release the GIL). + * So instead of holding the lock directly, the tstate holds a weakref to + * the lock: that's the value of on_delete_data below. Decref'ing a + * weakref is harmless. + * on_delete points to _threadmodule.c's static release_sentinel() function. + * After the tstate is unlinked, release_sentinel is called with the + * weakref-to-lock (on_delete_data) argument, and release_sentinel releases + * the indirectly held lock. + */ + void (*on_delete)(void *); + void *on_delete_data; + /* XXX signal handlers should also be here */ } PyThreadState; diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py index 13b1f26965a..b67cfb95be8 100644 --- a/Lib/_dummy_thread.py +++ b/Lib/_dummy_thread.py @@ -81,6 +81,10 @@ def stack_size(size=None): raise error("setting thread stack size not supported") return 0 +def _set_sentinel(): + """Dummy implementation of _thread._set_sentinel().""" + return LockType() + class LockType(object): """Class implementing dummy implementation of _thread.LockType. diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 971a63556d2..79b10ed3faf 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -539,6 +539,40 @@ class ThreadTests(BaseTestCase): self.assertEqual(err, b"") self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + def test_tstate_lock(self): + # Test an implementation detail of Thread objects. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + time.sleep(0.01) + # The tstate lock is None until the thread is started + t = threading.Thread(target=f) + self.assertIs(t._tstate_lock, None) + t.start() + started.acquire() + self.assertTrue(t.is_alive()) + # The tstate lock can't be acquired when the thread is running + # (or suspended). + tstate_lock = t._tstate_lock + self.assertFalse(tstate_lock.acquire(timeout=0), False) + finish.release() + # When the thread ends, the state_lock can be successfully + # acquired. + self.assertTrue(tstate_lock.acquire(timeout=5), False) + # But is_alive() is still True: we hold _tstate_lock now, which + # prevents is_alive() from knowing the thread's end-of-life C code + # is done. + self.assertTrue(t.is_alive()) + # Let is_alive() find out the C code is done. + tstate_lock.release() + self.assertFalse(t.is_alive()) + # And verify the thread disposed of _tstate_lock. + self.assertTrue(t._tstate_lock is None) + class ThreadJoinOnShutdown(BaseTestCase): @@ -669,7 +703,7 @@ class ThreadJoinOnShutdown(BaseTestCase): # someone else tries to fix this test case by acquiring this lock # before forking instead of resetting it, the test case will # deadlock when it shouldn't. - condition = w._block + condition = w._stopped._cond orig_acquire = condition.acquire call_count_lock = threading.Lock() call_count = 0 @@ -733,7 +767,7 @@ class ThreadJoinOnShutdown(BaseTestCase): # causes the worker to fork. At this point, the problematic waiter # lock has been acquired once by the waiter and has been put onto # the waiters list. - condition = w._block + condition = w._stopped._cond orig_release_save = condition._release_save def my_release_save(): global start_fork @@ -867,6 +901,38 @@ class SubinterpThreadingTests(BaseTestCase): # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") + def test_threads_join_2(self): + # Same as above, but a delay gets introduced after the thread's + # Python code returned but before the thread state is deleted. + # To achieve this, we register a thread-local object which sleeps + # a bit when deallocated. + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + class Sleeper: + def __del__(self): + time.sleep(0.05) + + tls = threading.local() + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + tls.x = Sleeper() + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = _testcapi.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + def test_daemon_threads_fatal_error(self): subinterp_code = r"""if 1: import os diff --git a/Lib/threading.py b/Lib/threading.py index b6d19d58483..d49be0be553 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -33,6 +33,7 @@ __all__ = ['active_count', 'Condition', 'current_thread', 'enumerate', 'Event', # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread _allocate_lock = _thread.allocate_lock +_set_sentinel = _thread._set_sentinel get_ident = _thread.get_ident ThreadError = _thread.error try: @@ -548,28 +549,33 @@ class Thread: else: self._daemonic = current_thread().daemon self._ident = None + self._tstate_lock = None self._started = Event() - self._stopped = False - self._block = Condition(Lock()) + self._stopped = Event() self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances self._stderr = _sys.stderr _dangling.add(self) - def _reset_internal_locks(self): + def _reset_internal_locks(self, is_alive): # private! Called by _after_fork() to reset our internal locks as # they may be in an invalid state leading to a deadlock or crash. - if hasattr(self, '_block'): # DummyThread deletes _block - self._block.__init__() self._started._reset_internal_locks() + self._stopped._reset_internal_locks() + if is_alive: + self._set_tstate_lock() + else: + # The thread isn't alive after fork: it doesn't have a tstate + # anymore. + self._tstate_lock = None def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" - if self._stopped: + if self._stopped.is_set(): status = "stopped" if self._daemonic: status += " daemon" @@ -625,9 +631,18 @@ class Thread: def _set_ident(self): self._ident = get_ident() + def _set_tstate_lock(self): + """ + Set a lock object which will be released by the interpreter when + the underlying thread state (see pystate.h) gets deleted. + """ + self._tstate_lock = _set_sentinel() + self._tstate_lock.acquire() + def _bootstrap_inner(self): try: self._set_ident() + self._set_tstate_lock() self._started.set() with _active_limbo_lock: _active[self._ident] = self @@ -691,10 +706,7 @@ class Thread: pass def _stop(self): - self._block.acquire() - self._stopped = True - self._block.notify_all() - self._block.release() + self._stopped.set() def _delete(self): "Remove current thread from the dict of currently running threads." @@ -738,21 +750,29 @@ class Thread: raise RuntimeError("cannot join thread before it is started") if self is current_thread(): raise RuntimeError("cannot join current thread") + if not self.is_alive(): + return + self._stopped.wait(timeout) + if self._stopped.is_set(): + self._wait_for_tstate_lock(timeout is None) - self._block.acquire() - try: - if timeout is None: - while not self._stopped: - self._block.wait() - else: - deadline = _time() + timeout - while not self._stopped: - delay = deadline - _time() - if delay <= 0: - break - self._block.wait(delay) - finally: - self._block.release() + def _wait_for_tstate_lock(self, block): + # Issue #18808: wait for the thread state to be gone. + # When self._stopped is set, the Python part of the thread is done, + # but the thread's tstate has not yet been destroyed. The C code + # releases self._tstate_lock when the C part of the thread is done + # (the code at the end of the thread's life to remove all knowledge + # of the thread from the C data structures). + # This method waits to acquire _tstate_lock if `block` is True, or + # sees whether it can be acquired immediately if `block` is False. + # If it does acquire the lock, the C code is done, and _tstate_lock + # is set to None. + lock = self._tstate_lock + if lock is None: + return # already determined that the C code is done + if lock.acquire(block): + lock.release() + self._tstate_lock = None @property def name(self): @@ -771,7 +791,14 @@ class Thread: def is_alive(self): assert self._initialized, "Thread.__init__() not called" - return self._started.is_set() and not self._stopped + if not self._started.is_set(): + return False + if not self._stopped.is_set(): + return True + # The Python part of the thread is done, but the C part may still be + # waiting to run. + self._wait_for_tstate_lock(False) + return self._tstate_lock is not None isAlive = is_alive @@ -854,11 +881,6 @@ class _DummyThread(Thread): def __init__(self): Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True) - # Thread._block consumes an OS-level locking primitive, which - # can never be used by a _DummyThread. Since a _DummyThread - # instance is immortal, that's bad, so release this resource. - del self._block - self._started.set() self._set_ident() with _active_limbo_lock: @@ -952,15 +974,16 @@ def _after_fork(): for thread in _enumerate(): # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. - thread._reset_internal_locks() if thread is current: # There is only one active thread. We reset the ident to # its new value since it can have changed. + thread._reset_internal_locks(True) ident = get_ident() thread._ident = ident new_active[ident] = thread else: # All the others are already stopped. + thread._reset_internal_locks(False) thread._stop() _limbo.clear() diff --git a/Misc/NEWS b/Misc/NEWS index 757ac89562c..d375a3a6304 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,10 @@ Core and Builtins Library ------- +- Issue #18808: Thread.join() now waits for the underlying thread state to + be destroyed before returning. This prevents unpredictable aborts in + Py_EndInterpreter() when some non-daemon threads are still running. + - Issue #18458: Prevent crashes with newer versions of libedit. Its readline emulation has changed from 0-based indexing to 1-based like gnu readline. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index cbb29014b5d..d83d1176364 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1172,6 +1172,66 @@ yet finished.\n\ This function is meant for internal and specialized purposes only.\n\ In most applications `threading.enumerate()` should be used instead."); +static void +release_sentinel(void *wr) +{ + /* Tricky: this function is called when the current thread state + is being deleted. Therefore, only simple C code can safely + execute here. */ + PyObject *obj = PyWeakref_GET_OBJECT(wr); + lockobject *lock; + if (obj != Py_None) { + assert(Py_TYPE(obj) == &Locktype); + lock = (lockobject *) obj; + if (lock->locked) { + PyThread_release_lock(lock->lock_lock); + lock->locked = 0; + } + } + /* Deallocating a weakref with a NULL callback only calls + PyObject_GC_Del(), which can't call any Python code. */ + Py_DECREF(wr); +} + +static PyObject * +thread__set_sentinel(PyObject *self) +{ + PyObject *wr; + PyThreadState *tstate = PyThreadState_Get(); + lockobject *lock; + + if (tstate->on_delete_data != NULL) { + /* We must support the re-creation of the lock from a + fork()ed child. */ + assert(tstate->on_delete == &release_sentinel); + wr = (PyObject *) tstate->on_delete_data; + tstate->on_delete = NULL; + tstate->on_delete_data = NULL; + Py_DECREF(wr); + } + lock = newlockobject(); + if (lock == NULL) + return NULL; + /* The lock is owned by whoever called _set_sentinel(), but the weakref + hangs to the thread state. */ + wr = PyWeakref_NewRef((PyObject *) lock, NULL); + if (wr == NULL) { + Py_DECREF(lock); + return NULL; + } + tstate->on_delete_data = (void *) wr; + tstate->on_delete = &release_sentinel; + return (PyObject *) lock; +} + +PyDoc_STRVAR(_set_sentinel_doc, +"_set_sentinel() -> lock\n\ +\n\ +Set a sentinel lock that will be released when the current thread\n\ +state is finalized (after it is untied from the interpreter).\n\ +\n\ +This is a private API for the threading module."); + static PyObject * thread_stack_size(PyObject *self, PyObject *args) { @@ -1247,6 +1307,8 @@ static PyMethodDef thread_methods[] = { METH_NOARGS, _count_doc}, {"stack_size", (PyCFunction)thread_stack_size, METH_VARARGS, stack_size_doc}, + {"_set_sentinel", (PyCFunction)thread__set_sentinel, + METH_NOARGS, _set_sentinel_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/pystate.c b/Python/pystate.c index 924b6a29090..ecd00ce5b42 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -208,6 +208,8 @@ new_threadstate(PyInterpreterState *interp, int init) tstate->trash_delete_nesting = 0; tstate->trash_delete_later = NULL; + tstate->on_delete = NULL; + tstate->on_delete_data = NULL; if (init) _PyThreadState_Init(tstate); @@ -390,6 +392,9 @@ tstate_delete_common(PyThreadState *tstate) if (tstate->next) tstate->next->prev = tstate->prev; HEAD_UNLOCK(); + if (tstate->on_delete != NULL) { + tstate->on_delete(tstate->on_delete_data); + } PyMem_RawFree(tstate); } From ae7b00e2d3b92991ef1d7685f2bd4b15e5263b84 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 7 Sep 2013 15:05:00 -0700 Subject: [PATCH 08/34] Move the overview comment to the top of the file. --- Objects/setobject.c | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 0aec100294c..b63a96684e8 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1,10 +1,30 @@ /* set object implementation + Written and maintained by Raymond D. Hettinger Derived from Lib/sets.py and Objects/dictobject.c. Copyright (c) 2003-2013 Python Software Foundation. All rights reserved. + + The basic lookup function used by all operations. + This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. + + The initial probe index is computed as hash mod the table size. + Subsequent probe indices are computed as explained in Objects/dictobject.c. + + To improve cache locality, each probe inspects a series of consecutive + nearby entries before moving on to probes elsewhere in memory. This leaves + us with a hybrid of linear probing and open addressing. The linear probing + reduces the cost of hash collisions because consecutive memory accesses + tend to be much cheaper than scattered probes. After LINEAR_PROBES steps, + we then use open addressing with the upper bits from the hash value. This + helps break-up long chains of collisions. + + All arithmetic on hash should ignore overflow. + + Unlike the dictionary implementation, the lookkey functions can return + NULL if the rich comparison returns an error. */ #include "Python.h" @@ -44,28 +64,6 @@ PyObject *_PySet_Dummy = dummy; static PySetObject *free_list[PySet_MAXFREELIST]; static int numfree = 0; - -/* -The basic lookup function used by all operations. -This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. - -The initial probe index is computed as hash mod the table size. -Subsequent probe indices are computed as explained in Objects/dictobject.c. - -To improve cache locality, each probe inspects a series of consecutive -nearby entries before moving on to probes elsewhere in memory. This leaves -us with a hybrid of linear probing and open addressing. The linear probing -reduces the cost of hash collisions because consecutive memory accesses -tend to be much cheaper than scattered probes. After LINEAR_PROBES steps, -we then use open addressing with the upper bits from the hash value. This -helps break-up long chains of collisions. - -All arithmetic on hash should ignore overflow. - -Unlike the dictionary implementation, the lookkey functions can return -NULL if the rich comparison returns an error. -*/ - static setentry * set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) { From 04fd9dd52bc81041126cb8df86a5e7629113a193 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 7 Sep 2013 17:41:01 -0700 Subject: [PATCH 09/34] Small rearrangement to bring together the three functions for probing the hash table. --- Objects/setobject.c | 71 +++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index b63a96684e8..362273ab49f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -64,6 +64,9 @@ PyObject *_PySet_Dummy = dummy; static PySetObject *free_list[PySet_MAXFREELIST]; static int numfree = 0; +/* ======================================================================== */ +/* ======= Begin logic for probing the hash table ========================= */ + static setentry * set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) { @@ -197,38 +200,6 @@ set_lookkey_unicode(PySetObject *so, PyObject *key, Py_hash_t hash) return freeslot == NULL ? entry : freeslot; } -/* -Internal routine to insert a new key into the table. -Used by the public insert routine. -Eats a reference to key. -*/ -static int -set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) -{ - setentry *entry; - - assert(so->lookup != NULL); - entry = so->lookup(so, key, hash); - if (entry == NULL) - return -1; - if (entry->key == NULL) { - /* UNUSED */ - so->fill++; - entry->key = key; - entry->hash = hash; - so->used++; - } else if (entry->key == dummy) { - /* DUMMY */ - entry->key = key; - entry->hash = hash; - so->used++; - } else { - /* ACTIVE */ - Py_DECREF(key); - } - return 0; -} - /* Internal routine used by set_table_resize() to insert an item which is known to be absent from the set. This routine also assumes that @@ -266,6 +237,42 @@ set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) so->used++; } +/* ======== End logic for probing the hash table ========================== */ +/* ======================================================================== */ + + +/* +Internal routine to insert a new key into the table. +Used by the public insert routine. +Eats a reference to key. +*/ +static int +set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) +{ + setentry *entry; + + assert(so->lookup != NULL); + entry = so->lookup(so, key, hash); + if (entry == NULL) + return -1; + if (entry->key == NULL) { + /* UNUSED */ + so->fill++; + entry->key = key; + entry->hash = hash; + so->used++; + } else if (entry->key == dummy) { + /* DUMMY */ + entry->key = key; + entry->hash = hash; + so->used++; + } else { + /* ACTIVE */ + Py_DECREF(key); + } + return 0; +} + /* Restructure the table by allocating a new table and reinserting all keys again. When entries have been deleted, the new table may From 21101f7038331984eee88fb6f876a3c6e2c3dfbb Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 7 Sep 2013 17:51:58 -0700 Subject: [PATCH 10/34] Correct Profile class usage example. Addresses issue #18033. Patch contributed by Olivier Hervieu and Dmi Baranov. --- Doc/library/profile.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 3f2a02dca42..aefc024d40e 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -247,11 +247,13 @@ functions: import cProfile, pstats, io pr = cProfile.Profile() pr.enable() - ... do something ... + # ... do something ... pr.disable() s = io.StringIO() - ps = pstats.Stats(pr, stream=s) - ps.print_results() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print(s.getvalue()) .. method:: enable() From 0494c2ae7f6a347590d66bb4fe7980dc0dc9ccda Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 8 Sep 2013 11:40:34 +1000 Subject: [PATCH 11/34] Close #18952: correctly download test support data When test.support was converted to a package, it started silently skipping the tests which needed to download support data to run. This change refactors the affected code, and also tidies up test.support.findfile to remove the unused *here* parameter, document the *subdir* parameter and rename the *filename* parameter to avoid shadowing the file builtin and be consistent with the documentation. The unexpected skips were noticed and reported by Zachary Ware --- Doc/library/test.rst | 5 ++++- Lib/test/support/__init__.py | 31 +++++++++++++++++++------------ Misc/NEWS | 4 ++++ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index bce0f647f39..c1270f4320b 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -263,12 +263,15 @@ The :mod:`test.support` module defines the following functions: Used when tests are executed by :mod:`test.regrtest`. -.. function:: findfile(filename) +.. function:: findfile(filename, subdir=None) Return the path to the file named *filename*. If no match is found *filename* is returned. This does not equal a failure since it could be the path to the file. + Setting *subdir* indicates a relative path to use to find the file + rather than looking directly in the path directories. + .. function:: run_unittest(\*classes) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 956fe988ccb..dbd78467373 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -853,24 +853,31 @@ if hasattr(os, "umask"): finally: os.umask(oldmask) -# TEST_HOME refers to the top level directory of the "test" package +# TEST_HOME_DIR refers to the top level directory of the "test" package # that contains Python's regression test suite -TEST_HOME = os.path.dirname(os.path.abspath(__file__)) +TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR) -def findfile(file, here=TEST_HOME, subdir=None): +# TEST_DATA_DIR is used as a target download location for remote resources +TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data") + +def findfile(filename, subdir=None): """Try to find a file on sys.path or in the test directory. If it is not found the argument passed to the function is returned (this does not - necessarily signal failure; could still be the legitimate path).""" - if os.path.isabs(file): - return file + necessarily signal failure; could still be the legitimate path). + + Setting *subdir* indicates a relative path to use to find the file + rather than looking directly in the path directories. + """ + if os.path.isabs(filename): + return filename if subdir is not None: - file = os.path.join(subdir, file) - path = sys.path - path = [os.path.dirname(here)] + path + filename = os.path.join(subdir, filename) + path = [TEST_HOME_DIR] + sys.path for dn in path: - fn = os.path.join(dn, file) + fn = os.path.join(dn, filename) if os.path.exists(fn): return fn - return file + return filename def create_empty_file(filename): """Create an empty file. If the file already exists, truncate it.""" @@ -907,7 +914,7 @@ def open_urlresource(url, *args, **kw): filename = urllib.parse.urlparse(url)[2].split('/')[-1] # '/': it's URL! - fn = os.path.join(os.path.dirname(__file__), "data", filename) + fn = os.path.join(TEST_DATA_DIR, filename) def check_valid_file(fn): f = open(fn, *args, **kw) diff --git a/Misc/NEWS b/Misc/NEWS index 1c0f3b23b6a..ed5f6cd1c3d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -332,6 +332,10 @@ IDLE Tests ----- +- Issue #18952: Fix regression in support data downloads introduced when + test.support was converted to a package. Regression noticed by Zachary + Ware. + - Issue #12037: Fix test_email for desktop Windows. - Issue #15507: test_subprocess's test_send_signal could fail if the test From 68d7f78703d8b1dbae79d51d8910b21bfefa1a32 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 7 Sep 2013 21:23:03 -0500 Subject: [PATCH 12/34] Issue 18808: blind attempt to repair some buildbot failures. test_is_alive_after_fork is failing on some old Linux kernels, but passing on all newer ones. Since virtually anything can go wrong with locks when mixing threads with fork, replace the most likely cause with a redundant simple data member. --- Lib/threading.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/threading.py b/Lib/threading.py index d49be0be553..4d3c6f6cf7d 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -552,6 +552,10 @@ class Thread: self._tstate_lock = None self._started = Event() self._stopped = Event() + # _is_stopped should be the same as _stopped.is_set(). The bizarre + # duplication is to allow test_is_alive_after_fork to pass on old + # Linux kernels. See issue 18808. + self._is_stopped = False self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances @@ -707,6 +711,7 @@ class Thread: def _stop(self): self._stopped.set() + self._is_stopped = True def _delete(self): "Remove current thread from the dict of currently running threads." @@ -793,7 +798,7 @@ class Thread: assert self._initialized, "Thread.__init__() not called" if not self._started.is_set(): return False - if not self._stopped.is_set(): + if not self._is_stopped: return True # The Python part of the thread is done, but the C part may still be # waiting to run. From 8f8839e10adc4d2b89ca26642c89c22382b1ce2f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 7 Sep 2013 20:26:50 -0700 Subject: [PATCH 13/34] Remove the freelist scheme for setobjects. The setobject freelist was consuming memory but not providing much value. Even when a freelisted setobject was available, most of the setobject fields still needed to be initialized and the small table still required a memset(). This meant that the custom freelisting scheme for sets was providing almost no incremental benefit over the default Python freelist scheme used by _PyObject_Malloc() in Objects/obmalloc.c. --- Include/setobject.h | 1 - Objects/object.c | 1 - Objects/setobject.c | 55 +++++++-------------------------------------- 3 files changed, 8 insertions(+), 49 deletions(-) diff --git a/Include/setobject.h b/Include/setobject.h index f377a738a8d..ae3f556365d 100644 --- a/Include/setobject.h +++ b/Include/setobject.h @@ -105,7 +105,6 @@ PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set); PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); PyAPI_FUNC(int) PySet_ClearFreeList(void); -PyAPI_FUNC(void) _PySet_DebugMallocStats(FILE *out); #endif #ifdef __cplusplus diff --git a/Objects/object.c b/Objects/object.c index 693d8c73b1c..8018c6a58c4 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1955,7 +1955,6 @@ _PyObject_DebugTypeStats(FILE *out) _PyFrame_DebugMallocStats(out); _PyList_DebugMallocStats(out); _PyMethod_DebugMallocStats(out); - _PySet_DebugMallocStats(out); _PyTuple_DebugMallocStats(out); } diff --git a/Objects/setobject.c b/Objects/setobject.c index 362273ab49f..e2cb666b675 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -57,13 +57,6 @@ PyObject *_PySet_Dummy = dummy; INIT_NONZERO_SET_SLOTS(so); \ } while(0) -/* Reuse scheme to save calls to malloc, free, and memset */ -#ifndef PySet_MAXFREELIST -#define PySet_MAXFREELIST 80 -#endif -static PySetObject *free_list[PySet_MAXFREELIST]; -static int numfree = 0; - /* ======================================================================== */ /* ======= Begin logic for probing the hash table ========================= */ @@ -565,10 +558,7 @@ set_dealloc(PySetObject *so) } if (so->table != so->smalltable) PyMem_DEL(so->table); - if (numfree < PySet_MAXFREELIST && PyAnySet_CheckExact(so)) - free_list[numfree++] = so; - else - Py_TYPE(so)->tp_free(so); + Py_TYPE(so)->tp_free(so); Py_TRASHCAN_SAFE_END(so) } @@ -1023,22 +1013,12 @@ make_new_set(PyTypeObject *type, PyObject *iterable) PySetObject *so = NULL; /* create PySetObject structure */ - if (numfree && - (type == &PySet_Type || type == &PyFrozenSet_Type)) { - so = free_list[--numfree]; - assert (so != NULL && PyAnySet_CheckExact(so)); - Py_TYPE(so) = type; - _Py_NewReference((PyObject *)so); - EMPTY_TO_MINSIZE(so); - PyObject_GC_Track(so); - } else { - so = (PySetObject *)type->tp_alloc(type, 0); - if (so == NULL) - return NULL; - /* tp_alloc has already zeroed the structure */ - assert(so->table == NULL && so->fill == 0 && so->used == 0); - INIT_NONZERO_SET_SLOTS(so); - } + so = (PySetObject *)type->tp_alloc(type, 0); + if (so == NULL) + return NULL; + /* tp_alloc has already zeroed the structure */ + assert(so->table == NULL && so->fill == 0 && so->used == 0); + INIT_NONZERO_SET_SLOTS(so); so->lookup = set_lookkey_unicode; so->weakreflist = NULL; @@ -1103,34 +1083,15 @@ frozenset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) int PySet_ClearFreeList(void) { - int freelist_size = numfree; - PySetObject *so; - - while (numfree) { - numfree--; - so = free_list[numfree]; - PyObject_GC_Del(so); - } - return freelist_size; + return 0; } void PySet_Fini(void) { - PySet_ClearFreeList(); Py_CLEAR(emptyfrozenset); } -/* Print summary info about the state of the optimized allocator */ -void -_PySet_DebugMallocStats(FILE *out) -{ - _PyDebugAllocatorStats(out, - "free PySetObject", - numfree, sizeof(PySetObject)); -} - - static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { From 4ea9080da982d05c60484ab4e9bda3a946c1190b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 7 Sep 2013 21:01:29 -0700 Subject: [PATCH 14/34] Improve code clarity by removing two unattractive macros. --- Objects/setobject.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index e2cb666b675..7364fca7373 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -45,17 +45,6 @@ static PyObject _dummy_struct; /* Exported for the gdb plugin's benefit. */ PyObject *_PySet_Dummy = dummy; -#define INIT_NONZERO_SET_SLOTS(so) do { \ - (so)->table = (so)->smalltable; \ - (so)->mask = PySet_MINSIZE - 1; \ - (so)->hash = -1; \ - } while(0) - -#define EMPTY_TO_MINSIZE(so) do { \ - memset((so)->smalltable, 0, sizeof((so)->smalltable)); \ - (so)->used = (so)->fill = 0; \ - INIT_NONZERO_SET_SLOTS(so); \ - } while(0) /* ======================================================================== */ /* ======= Begin logic for probing the hash table ========================= */ @@ -439,6 +428,17 @@ set_discard_key(PySetObject *so, PyObject *key) return DISCARD_FOUND; } +static void +set_empty_to_minsize(PySetObject *so) +{ + memset(so->smalltable, 0, sizeof(so->smalltable)); + so->fill = 0; + so->used = 0; + so->mask = PySet_MINSIZE - 1; + so->table = so->smalltable; + so->hash = -1; +} + static int set_clear_internal(PySetObject *so) { @@ -466,7 +466,7 @@ set_clear_internal(PySetObject *so) */ fill = so->fill; if (table_is_malloced) - EMPTY_TO_MINSIZE(so); + set_empty_to_minsize(so); else if (fill > 0) { /* It's a small table with something that needs to be cleared. @@ -475,7 +475,7 @@ set_clear_internal(PySetObject *so) */ memcpy(small_copy, table, sizeof(small_copy)); table = small_copy; - EMPTY_TO_MINSIZE(so); + set_empty_to_minsize(so); } /* else it's a small table that's already empty */ @@ -1016,11 +1016,13 @@ make_new_set(PyTypeObject *type, PyObject *iterable) so = (PySetObject *)type->tp_alloc(type, 0); if (so == NULL) return NULL; - /* tp_alloc has already zeroed the structure */ - assert(so->table == NULL && so->fill == 0 && so->used == 0); - INIT_NONZERO_SET_SLOTS(so); + so->fill = 0; + so->used = 0; + so->mask = PySet_MINSIZE - 1; + so->table = so->smalltable; so->lookup = set_lookkey_unicode; + so->hash = -1; so->weakreflist = NULL; if (iterable != NULL) { From 583cd03fd120d14ea65159e17a6407e9447a125e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 7 Sep 2013 22:06:35 -0700 Subject: [PATCH 15/34] Minor code beautification. --- Objects/setobject.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 7364fca7373..9df2fd322fb 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -446,14 +446,13 @@ set_clear_internal(PySetObject *so) int table_is_malloced; Py_ssize_t fill; setentry small_copy[PySet_MINSIZE]; -#ifdef Py_DEBUG - Py_ssize_t i, n; - assert (PyAnySet_Check(so)); - n = so->mask + 1; - i = 0; +#ifdef Py_DEBUG + Py_ssize_t i = 0; + Py_ssize_t n = so->mask + 1; #endif + assert (PyAnySet_Check(so)); table = so->table; assert(table != NULL); table_is_malloced = table != so->smalltable; @@ -2366,7 +2365,7 @@ test_c_api(PySetObject *so) Py_ssize_t count; char *s; Py_ssize_t i; - PyObject *elem=NULL, *dup=NULL, *t, *f, *dup2, *x; + PyObject *elem=NULL, *dup=NULL, *t, *f, *dup2, *x=NULL; PyObject *ob = (PyObject *)so; Py_hash_t hash; PyObject *str; From 4fec47e5edcd2d24b219e74fd73c648c065f0aa7 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 7 Sep 2013 23:19:29 -0700 Subject: [PATCH 16/34] Fix the dead link of IEEE_854-1987 standard with the Wikipedia entry. Addresses issue #18438 --- Lib/decimal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/decimal.py b/Lib/decimal.py index 1826deba801..96d9df4a8c8 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -21,7 +21,7 @@ the General Decimal Arithmetic Specification: and IEEE standard 854-1987: - www.cs.berkeley.edu/~ejr/projects/754/private/drafts/854-1987/dir.html + http://en.wikipedia.org/wiki/IEEE_854-1987 Decimal floating point has finite precision with arbitrarily large bounds. From 742d8716ff08358a585406cde5b03e71d46968c0 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 8 Sep 2013 00:25:57 -0700 Subject: [PATCH 17/34] Put the defines in the logical section and fix indentation. --- Objects/setobject.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 9df2fd322fb..23d624f9158 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -31,12 +31,6 @@ #include "structmember.h" #include "stringlib/eq.h" -/* This must be >= 1 */ -#define PERTURB_SHIFT 5 - -/* This should be >= PySet_MINSIZE - 1 */ -#define LINEAR_PROBES 9 - /* Object used as dummy key to fill deleted entries */ static PyObject _dummy_struct; @@ -49,6 +43,12 @@ PyObject *_PySet_Dummy = dummy; /* ======================================================================== */ /* ======= Begin logic for probing the hash table ========================= */ +/* This should be >= PySet_MINSIZE - 1 */ +#define LINEAR_PROBES 9 + +/* This must be >= 1 */ +#define PERTURB_SHIFT 5 + static setentry * set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) { @@ -151,8 +151,8 @@ set_lookkey_unicode(PySetObject *so, PyObject *key, Py_hash_t hash) while (1) { if (entry->key == key || (entry->hash == hash - && entry->key != dummy - && unicode_eq(entry->key, key))) + && entry->key != dummy + && unicode_eq(entry->key, key))) return entry; if (entry->key == dummy && freeslot == NULL) freeslot = entry; From 45e255167eb8cc34c85439dbdc8074a9b25bec2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Sun, 8 Sep 2013 11:30:53 +0200 Subject: [PATCH 18/34] Issue #18934: Use poll/select-based selectors for multiprocessing.Connection, to avoid one extra FD per Connection. --- Lib/multiprocessing/connection.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 59fb6640d5b..27fda9f22c6 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -878,13 +878,21 @@ else: import selectors + # poll/select have the advantage of not requiring any extra file + # descriptor, contrarily to epoll/kqueue (also, they require a single + # syscall). + if hasattr(selectors, 'PollSelector'): + _WaitSelector = selectors.PollSelector + else: + _WaitSelector = selectors.SelectSelector + def wait(object_list, timeout=None): ''' Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. ''' - with selectors.DefaultSelector() as selector: + with _WaitSelector() as selector: for obj in object_list: selector.register(obj, selectors.EVENT_READ) From 9437d7a7fe81a5f80122d8c86ac469d20259eeea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Sun, 8 Sep 2013 11:34:42 +0200 Subject: [PATCH 19/34] Issue #18963: Fix test_selectors.test_above_fd_setsize on OS X, where the default RLIMIT_NOFILE hard limit can be RLIMIT_INFINITY. --- Lib/test/test_selectors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index 2657a505b4e..8041187cbfb 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -301,7 +301,6 @@ class BaseSelectorTestCase(unittest.TestCase): class ScalableSelectorMixIn: - @support.requires_mac_ver(10, 5) @unittest.skipUnless(resource, "Test needs resource module") def test_above_fd_setsize(self): # A scalable implementation should have no problem with more than @@ -313,7 +312,7 @@ class ScalableSelectorMixIn: self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, (soft, hard)) NUM_FDS = hard - except OSError: + except (OSError, ValueError): NUM_FDS = soft # guard for already allocated FDs (stdin, stdout...) From 88983500767e3bd042170552c12f9f5280dd4b3a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 8 Sep 2013 11:36:23 +0200 Subject: [PATCH 20/34] Close #18957: The PYTHONFAULTHANDLER environment variable now only enables the faulthandler module if the variable is non-empty. Same behaviour than other variables like PYTHONDONTWRITEBYTECODE. --- Doc/using/cmdline.rst | 16 ++++++++-------- Lib/test/test_faulthandler.py | 30 +++++++++++++++++++++++------- Misc/NEWS | 7 +++++++ Modules/faulthandler.c | 5 ++++- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index c14f6c76e00..908a17c5aac 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -511,9 +511,9 @@ conflict. .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the - import of source modules. This is equivalent to specifying the :option:`-B` - option. + If this is set to a non-empty string, Python won't try to write ``.pyc`` or + ``.pyo`` files on the import of source modules. This is equivalent to + specifying the :option:`-B` option. .. envvar:: PYTHONHASHSEED @@ -582,11 +582,11 @@ conflict. .. envvar:: PYTHONFAULTHANDLER - If this environment variable is set, :func:`faulthandler.enable` is called - at startup: install a handler for :const:`SIGSEGV`, :const:`SIGFPE`, - :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` signals to dump the - Python traceback. This is equivalent to :option:`-X` ``faulthandler`` - option. + If this environment variable is set to a non-empty string, + :func:`faulthandler.enable` is called at startup: install a handler for + :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and + :const:`SIGILL` signals to dump the Python traceback. This is equivalent to + :option:`-X` ``faulthandler`` option. .. versionadded:: 3.3 diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 4a8becfe38f..d78bcb0a95f 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -265,17 +265,33 @@ faulthandler._sigsegv() # By default, the module should be disabled code = "import faulthandler; print(faulthandler.is_enabled())" args = (sys.executable, '-E', '-c', code) - # use subprocess module directly because test.script_helper adds - # "-X faulthandler" to the command line - stdout = subprocess.check_output(args) - self.assertEqual(stdout.rstrip(), b"False") + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"False") def test_sys_xoptions(self): # Test python -X faulthandler code = "import faulthandler; print(faulthandler.is_enabled())" - rc, stdout, stderr = assert_python_ok("-X", "faulthandler", "-c", code) - stdout = (stdout + stderr).strip() - self.assertEqual(stdout, b"True") + args = (sys.executable, "-E", "-X", "faulthandler", "-c", code) + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"True") + + def test_env_var(self): + # empty env var + code = "import faulthandler; print(faulthandler.is_enabled())" + args = (sys.executable, "-c", code) + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '' + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"False") + + # non-empty env var + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '1' + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"True") def check_dump_traceback(self, filename): """ diff --git a/Misc/NEWS b/Misc/NEWS index f97b504e7f2..128e740b746 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -7,6 +7,13 @@ What's New in Python 3.4.0 Alpha 3? Projected Release date: 2013-10-XX +Library +------- + +- The :envvar:`PYTHONFAULTHANDLER` environment variable now only enables the + faulthandler module if the variable is non-empty. Same behaviour than other + variables like :envvar:`PYTHONDONTWRITEBYTECODE`. + Tests ----- diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 172945d1a40..47bc9e8d3b3 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1048,8 +1048,11 @@ faulthandler_env_options(void) { PyObject *xoptions, *key, *module, *res; _Py_IDENTIFIER(enable); + char *p; - if (!Py_GETENV("PYTHONFAULTHANDLER")) { + if (!((p = Py_GETENV("PYTHONFAULTHANDLER")) && *p != '\0')) { + /* PYTHONFAULTHANDLER environment variable is missing + or an empty string */ int has_key; xoptions = PySys_GetXOptions(); From 7ba6b0f943ed9fc1782ec5813a07440d98d64554 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 8 Sep 2013 11:47:54 +0200 Subject: [PATCH 21/34] Issue #18904: Improve os.get/set_inheritable() tests --- Lib/test/test_os.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d0dd364f89a..aa48045a303 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -34,6 +34,10 @@ try: import resource except ImportError: resource = None +try: + import fcntl +except ImportError: + fcntl = None from test.script_helper import assert_python_ok @@ -2300,19 +2304,37 @@ class CPUCountTests(unittest.TestCase): class FDInheritanceTests(unittest.TestCase): - def test_get_inheritable(self): + def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) - for inheritable in (False, True): - os.set_inheritable(fd, inheritable) - self.assertEqual(os.get_inheritable(fd), inheritable) + self.assertEqual(os.get_inheritable(fd), False) - def test_set_inheritable(self): - fd = os.open(__file__, os.O_RDONLY) - self.addCleanup(os.close, fd) os.set_inheritable(fd, True) self.assertEqual(os.get_inheritable(fd), True) + if fcntl: + def test_get_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(os.get_inheritable(fd), True) + + def test_set_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + os.set_inheritable(fd, True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) From a3c18d0f1423b192f82bd785cea09346942cf18e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 8 Sep 2013 11:53:09 +0200 Subject: [PATCH 22/34] Issue #18904: test_socket: add inheritance tests using fcntl and FD_CLOEXEC --- Lib/test/test_socket.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 620576812a8..fc478b6a873 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -26,6 +26,10 @@ try: import multiprocessing except ImportError: multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None HOST = support.HOST MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return @@ -4804,6 +4808,32 @@ class InheritanceTest(unittest.TestCase): sock.set_inheritable(False) self.assertEqual(sock.get_inheritable(), False) + if fcntl: + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + @unittest.skipUnless(hasattr(socket, "socketpair"), "need socket.socketpair()") def test_socketpair(self): From 833bf1fcb256ea28d0ecf16ab097ff9de2396cb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Sun, 8 Sep 2013 12:27:33 +0200 Subject: [PATCH 23/34] Issue #18935: Fix test_regrtest.test_timeout when built --without-threads (the '--timeout' option requires faulthandler.dump_traceback_later). --- Lib/test/test_regrtest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 289fb229f2e..353874b5a3e 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -3,6 +3,7 @@ Tests of regrtest.py. """ import argparse +import faulthandler import getopt import os.path import unittest @@ -25,6 +26,8 @@ class ParseArgsTestCase(unittest.TestCase): regrtest._parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) + @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), + "faulthandler.dump_traceback_later() required") def test_timeout(self): ns = regrtest._parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) From 807ba8552a5e6dffa4d15f8f9a1e2679507efcfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Sun, 8 Sep 2013 12:31:32 +0200 Subject: [PATCH 24/34] Issue #18963: skip test_selectors.test_above_fd_setsize on older OS X versions. --- Lib/test/test_selectors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index 8041187cbfb..6ce4d8a2169 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -301,6 +301,8 @@ class BaseSelectorTestCase(unittest.TestCase): class ScalableSelectorMixIn: + # see issue #18963 for why it's skipped on older OS X versions + @support.requires_mac_ver(10, 5) @unittest.skipUnless(resource, "Test needs resource module") def test_above_fd_setsize(self): # A scalable implementation should have no problem with more than From 5da7e7959e6d3032a564eea68746e03764ed2f68 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 8 Sep 2013 13:19:06 +0200 Subject: [PATCH 25/34] Issue #18808 again: fix the after-fork logic for not-yet-started or already-stopped threads. (AFAICT, in theory, we must reset all the locks, not just those in use) --- Lib/test/test_threading.py | 2 +- Lib/threading.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 79b10ed3faf..58b0b4e945b 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -109,7 +109,7 @@ class ThreadTests(BaseTestCase): if verbose: print('waiting for all tasks to complete') for t in threads: - t.join(NUMTASKS) + t.join() self.assertTrue(not t.is_alive()) self.assertNotEqual(t.ident, 0) self.assertFalse(t.ident is None) diff --git a/Lib/threading.py b/Lib/threading.py index 4d3c6f6cf7d..b4b73a8a9c3 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -517,8 +517,6 @@ def _newname(template="Thread-%d"): _active_limbo_lock = _allocate_lock() _active = {} # maps thread id to Thread object _limbo = {} - -# For debug and leak testing _dangling = WeakSet() # Main class for threads @@ -552,14 +550,11 @@ class Thread: self._tstate_lock = None self._started = Event() self._stopped = Event() - # _is_stopped should be the same as _stopped.is_set(). The bizarre - # duplication is to allow test_is_alive_after_fork to pass on old - # Linux kernels. See issue 18808. - self._is_stopped = False self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances self._stderr = _sys.stderr + # For debugging and _after_fork() _dangling.add(self) def _reset_internal_locks(self, is_alive): @@ -711,7 +706,6 @@ class Thread: def _stop(self): self._stopped.set() - self._is_stopped = True def _delete(self): "Remove current thread from the dict of currently running threads." @@ -798,7 +792,7 @@ class Thread: assert self._initialized, "Thread.__init__() not called" if not self._started.is_set(): return False - if not self._is_stopped: + if not self._stopped.is_set(): return True # The Python part of the thread is done, but the C part may still be # waiting to run. @@ -976,7 +970,11 @@ def _after_fork(): current = current_thread() _main_thread = current with _active_limbo_lock: - for thread in _enumerate(): + # Dangling thread instances must still have their locks reset, + # because someone may join() them. + threads = set(_enumerate()) + threads.update(_dangling) + for thread in threads: # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. if thread is current: From 4f7a36f84f8180955d37aea4e8b44c4d4950d694 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 8 Sep 2013 14:14:38 +0200 Subject: [PATCH 26/34] Issue #18904: test_os and test_socket use unittest.skipIf() to check if fcntl module is present (to record skipped tests) --- Lib/test/test_os.py | 37 +++++++++++++++++++------------------ Lib/test/test_socket.py | 41 +++++++++++++++++++++-------------------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index aa48045a303..39b0e80125b 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2312,28 +2312,29 @@ class FDInheritanceTests(unittest.TestCase): os.set_inheritable(fd, True) self.assertEqual(os.get_inheritable(fd), True) - if fcntl: - def test_get_inheritable_cloexec(self): - fd = os.open(__file__, os.O_RDONLY) - self.addCleanup(os.close, fd) - self.assertEqual(os.get_inheritable(fd), False) + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) - # clear FD_CLOEXEC flag - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - flags &= ~fcntl.FD_CLOEXEC - fcntl.fcntl(fd, fcntl.F_SETFD, flags) + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) - self.assertEqual(os.get_inheritable(fd), True) + self.assertEqual(os.get_inheritable(fd), True) - def test_set_inheritable_cloexec(self): - fd = os.open(__file__, os.O_RDONLY) - self.addCleanup(os.close, fd) - self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, - fcntl.FD_CLOEXEC) + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) - os.set_inheritable(fd, True) - self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, - 0) + os.set_inheritable(fd, True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) def test_open(self): fd = os.open(__file__, os.O_RDONLY) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index fc478b6a873..490f776d21e 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -4808,30 +4808,31 @@ class InheritanceTest(unittest.TestCase): sock.set_inheritable(False) self.assertEqual(sock.get_inheritable(), False) - if fcntl: - def test_get_inheritable_cloexec(self): - sock = socket.socket() - with sock: - fd = sock.fileno() - self.assertEqual(sock.get_inheritable(), False) + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) - # clear FD_CLOEXEC flag - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - flags &= ~fcntl.FD_CLOEXEC - fcntl.fcntl(fd, fcntl.F_SETFD, flags) + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) - self.assertEqual(sock.get_inheritable(), True) + self.assertEqual(sock.get_inheritable(), True) - def test_set_inheritable_cloexec(self): - sock = socket.socket() - with sock: - fd = sock.fileno() - self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, - fcntl.FD_CLOEXEC) + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) - sock.set_inheritable(True) - self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, - 0) + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) @unittest.skipUnless(hasattr(socket, "socketpair"), From dfa689bfddad0d262fc788445f91851307897327 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 8 Sep 2013 20:29:37 +0300 Subject: [PATCH 27/34] Fixed tests with Tcl/Tk <8.5 (closes #18964). --- Lib/test/test_tcl.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 4e52fd4882e..cf717d8ce51 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -15,6 +15,14 @@ support.import_fresh_module('tkinter') from tkinter import Tcl from _tkinter import TclError +tcl_version = _tkinter.TCL_VERSION.split('.') +try: + for i in range(len(tcl_version)): + tcl_version[i] = int(tcl_version[i]) +except ValueError: + pass +tcl_version = tuple(tcl_version) + class TkinterTest(unittest.TestCase): @@ -200,9 +208,12 @@ class TclTest(unittest.TestCase): (('a', 3.4), ('a', 3.4)), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (1, '\u20ac', '\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)), + (1, '\u20ac', '\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(splitlist(arg), res, msg=arg) self.assertRaises(TclError, splitlist, '{') @@ -234,9 +245,12 @@ class TclTest(unittest.TestCase): (('a', (2, 3.4)), ('a', (2, 3.4))), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (12, '\u20ac', '\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), + (12, '\u20ac', '\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(split(arg), res, msg=arg) From c70018044304e4c68d5f2f036541cd8f6fe54555 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 8 Sep 2013 20:42:13 +0300 Subject: [PATCH 28/34] Fix a typo. (closes #18953) --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index ed5f6cd1c3d..51e055786e3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -69,7 +69,7 @@ Library ------- - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in - the _sre moduel. + the _sre module. - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. From c363a23eff25d8ee74de145be4f447602a5c1a29 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sun, 8 Sep 2013 18:44:40 -0500 Subject: [PATCH 29/34] Issue 18984: Remove ._stopped Event from Thread internals. The fix for issue 18808 left us checking two things to be sure a Thread was done: an Event (._stopped) and a mutex (._tstate_lock). Clumsy & brittle. This patch removes the Event, leaving just a happy lock :-) The bulk of the patch removes two excruciating tests, which were verifying sanity of the internals of the ._stopped Event after a fork. Thanks to Antoine Pitrou for verifying that's the only real value these tests had. One consequence of moving from an Event to a mutex: waiters (threads calling Thread.join()) used to block each on their own unique mutex (internal to the ._stopped event), but now all contend on the same mutex (._tstate_lock). These approaches have different performance characteristics on different platforms. I don't think it matters in this context. --- Lib/test/test_threading.py | 138 +------------------------------------ Lib/threading.py | 59 ++++++++-------- 2 files changed, 30 insertions(+), 167 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 58b0b4e945b..75ae247de7b 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -647,144 +647,8 @@ class ThreadJoinOnShutdown(BaseTestCase): """ self._run_and_join(script) - def assertScriptHasOutput(self, script, expected_output): - rc, out, err = assert_python_ok("-c", script) - data = out.decode().replace('\r', '') - self.assertEqual(data, expected_output) - - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_4_joining_across_fork_in_worker_thread(self): - # There used to be a possible deadlock when forking from a child - # thread. See http://bugs.python.org/issue6643. - - # The script takes the following steps: - # - The main thread in the parent process starts a new thread and then - # tries to join it. - # - The join operation acquires the Lock inside the thread's _block - # Condition. (See threading.py:Thread.join().) - # - We stub out the acquire method on the condition to force it to wait - # until the child thread forks. (See LOCK ACQUIRED HERE) - # - The child thread forks. (See LOCK HELD and WORKER THREAD FORKS - # HERE) - # - The main thread of the parent process enters Condition.wait(), - # which releases the lock on the child thread. - # - The child process returns. Without the necessary fix, when the - # main thread of the child process (which used to be the child thread - # in the parent process) attempts to exit, it will try to acquire the - # lock in the Thread._block Condition object and hang, because the - # lock was held across the fork. - - script = """if 1: - import os, time, threading - - finish_join = False - start_fork = False - - def worker(): - # Wait until this thread's lock is acquired before forking to - # create the deadlock. - global finish_join - while not start_fork: - time.sleep(0.01) - # LOCK HELD: Main thread holds lock across this call. - childpid = os.fork() - finish_join = True - if childpid != 0: - # Parent process just waits for child. - os.waitpid(childpid, 0) - # Child process should just return. - - w = threading.Thread(target=worker) - - # Stub out the private condition variable's lock acquire method. - # This acquires the lock and then waits until the child has forked - # before returning, which will release the lock soon after. If - # someone else tries to fix this test case by acquiring this lock - # before forking instead of resetting it, the test case will - # deadlock when it shouldn't. - condition = w._stopped._cond - orig_acquire = condition.acquire - call_count_lock = threading.Lock() - call_count = 0 - def my_acquire(): - global call_count - global start_fork - orig_acquire() # LOCK ACQUIRED HERE - start_fork = True - if call_count == 0: - while not finish_join: - time.sleep(0.01) # WORKER THREAD FORKS HERE - with call_count_lock: - call_count += 1 - condition.acquire = my_acquire - - w.start() - w.join() - print('end of main') - """ - self.assertScriptHasOutput(script, "end of main\n") - - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_5_clear_waiter_locks_to_avoid_crash(self): - # Check that a spawned thread that forks doesn't segfault on certain - # platforms, namely OS X. This used to happen if there was a waiter - # lock in the thread's condition variable's waiters list. Even though - # we know the lock will be held across the fork, it is not safe to - # release locks held across forks on all platforms, so releasing the - # waiter lock caused a segfault on OS X. Furthermore, since locks on - # OS X are (as of this writing) implemented with a mutex + condition - # variable instead of a semaphore, while we know that the Python-level - # lock will be acquired, we can't know if the internal mutex will be - # acquired at the time of the fork. - - script = """if True: - import os, time, threading - - start_fork = False - - def worker(): - # Wait until the main thread has attempted to join this thread - # before continuing. - while not start_fork: - time.sleep(0.01) - childpid = os.fork() - if childpid != 0: - # Parent process just waits for child. - (cpid, rc) = os.waitpid(childpid, 0) - assert cpid == childpid - assert rc == 0 - print('end of worker thread') - else: - # Child process should just return. - pass - - w = threading.Thread(target=worker) - - # Stub out the private condition variable's _release_save method. - # This releases the condition's lock and flips the global that - # causes the worker to fork. At this point, the problematic waiter - # lock has been acquired once by the waiter and has been put onto - # the waiters list. - condition = w._stopped._cond - orig_release_save = condition._release_save - def my_release_save(): - global start_fork - orig_release_save() - # Waiter lock held here, condition lock released. - start_fork = True - condition._release_save = my_release_save - - w.start() - w.join() - print('end of main thread') - """ - output = "end of worker thread\nend of main thread\n" - self.assertScriptHasOutput(script, output) - - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_6_daemon_threads(self): + def test_4_daemon_threads(self): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. diff --git a/Lib/threading.py b/Lib/threading.py index b4b73a8a9c3..1921ee34cd5 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -549,7 +549,7 @@ class Thread: self._ident = None self._tstate_lock = None self._started = Event() - self._stopped = Event() + self._is_stopped = False self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances @@ -561,7 +561,6 @@ class Thread: # private! Called by _after_fork() to reset our internal locks as # they may be in an invalid state leading to a deadlock or crash. self._started._reset_internal_locks() - self._stopped._reset_internal_locks() if is_alive: self._set_tstate_lock() else: @@ -574,7 +573,7 @@ class Thread: status = "initial" if self._started.is_set(): status = "started" - if self._stopped.is_set(): + if self._is_stopped: status = "stopped" if self._daemonic: status += " daemon" @@ -696,7 +695,6 @@ class Thread: pass finally: with _active_limbo_lock: - self._stop() try: # We don't call self._delete() because it also # grabs _active_limbo_lock. @@ -705,7 +703,8 @@ class Thread: pass def _stop(self): - self._stopped.set() + self._is_stopped = True + self._tstate_lock = None def _delete(self): "Remove current thread from the dict of currently running threads." @@ -749,29 +748,24 @@ class Thread: raise RuntimeError("cannot join thread before it is started") if self is current_thread(): raise RuntimeError("cannot join current thread") - if not self.is_alive(): - return - self._stopped.wait(timeout) - if self._stopped.is_set(): - self._wait_for_tstate_lock(timeout is None) + if timeout is None: + self._wait_for_tstate_lock() + else: + self._wait_for_tstate_lock(timeout=timeout) - def _wait_for_tstate_lock(self, block): + def _wait_for_tstate_lock(self, block=True, timeout=-1): # Issue #18808: wait for the thread state to be gone. - # When self._stopped is set, the Python part of the thread is done, - # but the thread's tstate has not yet been destroyed. The C code - # releases self._tstate_lock when the C part of the thread is done - # (the code at the end of the thread's life to remove all knowledge - # of the thread from the C data structures). - # This method waits to acquire _tstate_lock if `block` is True, or - # sees whether it can be acquired immediately if `block` is False. - # If it does acquire the lock, the C code is done, and _tstate_lock - # is set to None. + # At the end of the thread's life, after all knowledge of the thread + # is removed from C data structures, C code releases our _tstate_lock. + # This method passes its arguments to _tstate_lock.aquire(). + # If the lock is acquired, the C code is done, and self._stop() is + # called. That sets ._is_stopped to True, and ._tstate_lock to None. lock = self._tstate_lock - if lock is None: - return # already determined that the C code is done - if lock.acquire(block): + if lock is None: # already determined that the C code is done + assert self._is_stopped + elif lock.acquire(block, timeout): lock.release() - self._tstate_lock = None + self._stop() @property def name(self): @@ -790,14 +784,10 @@ class Thread: def is_alive(self): assert self._initialized, "Thread.__init__() not called" - if not self._started.is_set(): + if self._is_stopped or not self._started.is_set(): return False - if not self._stopped.is_set(): - return True - # The Python part of the thread is done, but the C part may still be - # waiting to run. self._wait_for_tstate_lock(False) - return self._tstate_lock is not None + return not self._is_stopped isAlive = is_alive @@ -861,6 +851,7 @@ class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) + self._set_tstate_lock() self._started.set() self._set_ident() with _active_limbo_lock: @@ -925,6 +916,14 @@ from _thread import stack_size _main_thread = _MainThread() def _shutdown(): + # Obscure: other threads may be waiting to join _main_thread. That's + # dubious, but some code does it. We can't wait for C code to release + # the main thread's tstate_lock - that won't happen until the interpreter + # is nearly dead. So we release it here. Note that just calling _stop() + # isn't enough: other threads may already be waiting on _tstate_lock. + assert _main_thread._tstate_lock is not None + assert _main_thread._tstate_lock.locked() + _main_thread._tstate_lock.release() _main_thread._stop() t = _pickSomeNonDaemonThread() while t: From 8df58f7ae87a2debeae47813537d5c105bea6f38 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 9 Sep 2013 01:29:40 -0500 Subject: [PATCH 30/34] Issue 18752: Make chain.from_iterable() more visible in the documentation. --- Doc/library/itertools.rst | 1 + Modules/itertoolsmodule.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 0ee93edc9a1..aa793e26c50 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -48,6 +48,7 @@ Iterator Arguments Results ==================== ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` +chain.from_iterable iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` :func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` :func:`filterfalse` pred, seq elements of seq where pred(elem) is False ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 4bc9192abca..01231816123 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4,7 +4,7 @@ /* Itertools module written and maintained by Raymond D. Hettinger - Copyright (c) 2003 Python Software Foundation. + Copyright (c) 2003-2013 Python Software Foundation. All rights reserved. */ @@ -4456,6 +4456,7 @@ repeat(elem [,n]) --> elem, elem, elem, ... endlessly or up to n times\n\ Iterators terminating on the shortest input sequence:\n\ accumulate(p[, func]) --> p0, p0+p1, p0+p1+p2\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ... \n\ +chain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ... \n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\ From 455bfdea2fd24e5b5f966860b9987fd6321dcde0 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Sun, 8 Sep 2013 23:48:34 -0700 Subject: [PATCH 31/34] Close #18980: Enum doc fixes. Patch from Elazar Gershuni. --- Doc/library/enum.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 3661469935c..236275dc34b 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -38,7 +38,8 @@ follows:: ... blue = 3 ... -..note: Nomenclature +.. note:: Nomenclature + - The class :class:`Color` is an *enumeration* (or *enum*) - The attributes :attr:`Color.red`, :attr:`Color.green`, etc., are *enumeration members* (or *enum members*). @@ -474,7 +475,7 @@ Some rules: 4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. -5. :class:`str`.:meth:`__format__` (or :func:`format`) will use the mixed-in +5. :meth:`str.__format__` (or :func:`format`) will use the mixed-in type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or :func:`repr` is desired use the `!s` or `!r` :class:`str` format codes. From 1e21ebcc2ac1c74fbea6bbff0460227a90b8f594 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 9 Sep 2013 01:54:27 -0500 Subject: [PATCH 32/34] Issue 18301: The classmethod decorator didn't fit well with the rough-equivalent example code. --- Doc/library/itertools.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 0ee93edc9a1..9b64c7d2d6e 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -156,9 +156,8 @@ loops that truncate the stream. .. classmethod:: chain.from_iterable(iterable) Alternate constructor for :func:`chain`. Gets chained inputs from a - single iterable argument that is evaluated lazily. Equivalent to:: + single iterable argument that is evaluated lazily. Roughly equivalent to:: - @classmethod def from_iterable(iterables): # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F for it in iterables: From fb92f393b0a79bd6927c70fe299dcd7eac114703 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 9 Sep 2013 02:01:35 -0500 Subject: [PATCH 33/34] Issue 18752: Make chain.from_iterable() more visible in the documentation. --- Doc/library/itertools.rst | 1 + Modules/itertoolsmodule.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 9b64c7d2d6e..25f34bff160 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -48,6 +48,7 @@ Iterator Arguments Results ==================== ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` +chain.from_iterable iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` :func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` :func:`filterfalse` pred, seq elements of seq where pred(elem) is False ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 4bc9192abca..01231816123 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4,7 +4,7 @@ /* Itertools module written and maintained by Raymond D. Hettinger - Copyright (c) 2003 Python Software Foundation. + Copyright (c) 2003-2013 Python Software Foundation. All rights reserved. */ @@ -4456,6 +4456,7 @@ repeat(elem [,n]) --> elem, elem, elem, ... endlessly or up to n times\n\ Iterators terminating on the shortest input sequence:\n\ accumulate(p[, func]) --> p0, p0+p1, p0+p1+p2\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ... \n\ +chain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ... \n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\ From 51ef074d57de41b1b2246f6ba2a4ed26392d4d80 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Mon, 9 Sep 2013 10:34:24 +0100 Subject: [PATCH 34/34] Docstring typo fix: Arithmentic -> Arithmetic. --- Lib/unittest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index 201a3f0d2da..a5d50af78f2 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -11,7 +11,7 @@ Simple usage: import unittest - class IntegerArithmenticTestCase(unittest.TestCase): + class IntegerArithmeticTestCase(unittest.TestCase): def testAdd(self): ## test method names begin 'test*' self.assertEqual((1 + 2), 3) self.assertEqual(0 + 1, 1)