From 2914bb32e2adf8dff77c0ca58b33201bc94e398c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 29 Jan 2018 11:57:45 +0100 Subject: [PATCH] bpo-20891: Py_Initialize() now creates the GIL (#4700) The GIL is no longer created "on demand" to fix a race condition when PyGILState_Ensure() is called in a non-Python thread. --- Doc/c-api/init.rst | 63 +++++++------------ .../2017-12-04-18-34-11.bpo-20891.C2TsfR.rst | 3 + Python/ceval.c | 25 ++++---- Python/pylifecycle.c | 4 ++ 4 files changed, 43 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-12-04-18-34-11.bpo-20891.C2TsfR.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 02189a9e359..bae49d5ba81 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -687,15 +687,14 @@ This is so common that a pair of macros exists to simplify it:: The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. These two macros are still available when Python is compiled without -thread support (they simply have an empty expansion). +block. -When thread support is enabled, the block above expands to the following code:: +The block above expands to the following code:: PyThreadState *_save; _save = PyEval_SaveThread(); - ...Do some blocking I/O operation... + ... Do some blocking I/O operation ... PyEval_RestoreThread(_save); .. index:: @@ -818,36 +817,24 @@ code, or when embedding the Python interpreter: This is a no-op when called for a second time. + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + .. versionchanged:: 3.2 This function cannot be called before :c:func:`Py_Initialize()` anymore. .. index:: module: _thread - .. note:: - - When only the main thread exists, no GIL operations are needed. This is a - common situation (most Python programs do not use threads), and the lock - operations slow the interpreter down a bit. Therefore, the lock is not - created initially. This situation is equivalent to having acquired the lock: - when there is only a single thread, all object accesses are safe. Therefore, - when this function initializes the global interpreter lock, it also acquires - it. Before the Python :mod:`_thread` module creates a new thread, knowing - that either it has the lock or the lock hasn't been created yet, it calls - :c:func:`PyEval_InitThreads`. When this call returns, it is guaranteed that - the lock has been created and that the calling thread has acquired it. - - It is **not** safe to call this function when it is unknown which thread (if - any) currently has the global interpreter lock. - - This function is not available when thread support is disabled at compile time. - .. c:function:: int PyEval_ThreadsInitialized() Returns a non-zero value if :c:func:`PyEval_InitThreads` has been called. This function can be called without holding the GIL, and therefore can be used to - avoid calls to the locking API when running single-threaded. This function is - not available when thread support is disabled at compile time. + avoid calls to the locking API when running single-threaded. + + .. versionchanged:: 3.7 + The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`. .. c:function:: PyThreadState* PyEval_SaveThread() @@ -855,8 +842,7 @@ code, or when embedding the Python interpreter: Release the global interpreter lock (if it has been created and thread support is enabled) and reset the thread state to *NULL*, returning the previous thread state (which is not *NULL*). If the lock has been created, - the current thread must have acquired it. (This function is available even - when thread support is disabled at compile time.) + the current thread must have acquired it. .. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) @@ -864,8 +850,7 @@ code, or when embedding the Python interpreter: Acquire the global interpreter lock (if it has been created and thread support is enabled) and set the thread state to *tstate*, which must not be *NULL*. If the lock has been created, the current thread must not have - acquired it, otherwise deadlock ensues. (This function is available even - when thread support is disabled at compile time.) + acquired it, otherwise deadlock ensues. .. c:function:: PyThreadState* PyThreadState_Get() @@ -957,7 +942,7 @@ example usage in the Python source distribution. This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. Note that it contains an opening brace; it must be matched with a following :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. It is a no-op when thread support is disabled at compile time. + macro. .. c:macro:: Py_END_ALLOW_THREADS @@ -965,29 +950,29 @@ example usage in the Python source distribution. This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains a closing brace; it must be matched with an earlier :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. It is a no-op when thread support is disabled at compile time. + this macro. .. c:macro:: Py_BLOCK_THREADS This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. It is a no-op when - thread support is disabled at compile time. + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. .. c:macro:: Py_UNBLOCK_THREADS This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. It is a no-op when thread support is disabled at compile time. + declaration. Low-level API ------------- -All of the following functions are only available when thread support is enabled -at compile time, and must be called only when the global interpreter lock has -been created. +All of the following functions must be called after :c:func:`Py_Initialize`. + +.. versionchanged:: 3.7 + :c:func:`Py_Initialize()` now initializes the :term:`GIL`. .. c:function:: PyInterpreterState* PyInterpreterState_New() @@ -1068,8 +1053,7 @@ been created. If this thread already has the lock, deadlock ensues. :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when thread support isn't enabled or when threads have - not been initialized). + available (even when threads have not been initialized). .. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) @@ -1081,8 +1065,7 @@ been created. reported. :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when thread support isn't enabled or when threads have - not been initialized). + available (even when threads have not been initialized). .. c:function:: void PyEval_AcquireLock() diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-04-18-34-11.bpo-20891.C2TsfR.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-04-18-34-11.bpo-20891.C2TsfR.rst new file mode 100644 index 00000000000..abf9c3c2e39 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-04-18-34-11.bpo-20891.C2TsfR.rst @@ -0,0 +1,3 @@ +Py_Initialize() now creates the GIL. The GIL is no longer created "on demand" +to fix a race condition when PyGILState_Ensure() is called in a non-Python +thread. diff --git a/Python/ceval.c b/Python/ceval.c index 128ec2c0045..52a42b00724 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -254,8 +254,8 @@ PyEval_SaveThread(void) PyThreadState *tstate = PyThreadState_Swap(NULL); if (tstate == NULL) Py_FatalError("PyEval_SaveThread: NULL tstate"); - if (gil_created()) - drop_gil(tstate); + assert(gil_created()); + drop_gil(tstate); return tstate; } @@ -264,17 +264,18 @@ PyEval_RestoreThread(PyThreadState *tstate) { if (tstate == NULL) Py_FatalError("PyEval_RestoreThread: NULL tstate"); - if (gil_created()) { - int err = errno; - take_gil(tstate); - /* _Py_Finalizing is protected by the GIL */ - if (_Py_IsFinalizing() && !_Py_CURRENTLY_FINALIZING(tstate)) { - drop_gil(tstate); - PyThread_exit_thread(); - Py_UNREACHABLE(); - } - errno = err; + assert(gil_created()); + + int err = errno; + take_gil(tstate); + /* _Py_Finalizing is protected by the GIL */ + if (_Py_IsFinalizing() && !_Py_CURRENTLY_FINALIZING(tstate)) { + drop_gil(tstate); + PyThread_exit_thread(); + Py_UNREACHABLE(); } + errno = err; + PyThreadState_Swap(tstate); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d46784a2f6b..82ab9154256 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -681,9 +681,13 @@ _Py_InitializeCore(const _PyCoreConfig *core_config) Instead we destroy the previously created GIL here, which ensures that we can call Py_Initialize / Py_FinalizeEx multiple times. */ _PyEval_FiniThreads(); + /* Auto-thread-state API */ _PyGILState_Init(interp, tstate); + /* Create the GIL */ + PyEval_InitThreads(); + _Py_ReadyTypes(); if (!_PyFrame_Init())