bpo-25658: Implement PEP 539 for Thread Specific Storage (TSS) API (GH-1362)

See PEP 539 for details.

Highlights of changes:

- Add Thread Specific Storage (TSS) API
- Document the Thread Local Storage (TLS) API as deprecated
- Update code that used TLS API to use TSS API

Doc/c-api/init.rst

@@ -1192,3 +1192,160 @@ These functions are only intended to be used by advanced debugging tools.
    Return the next thread state object after *tstate* from the list of all such
    objects belonging to the same :c:type:`PyInterpreterState` object.
 
+
+.. _thread-local-storage:
+
+Thread Local Storage Support
+============================
+
+.. sectionauthor:: Masayuki Yamamoto <ma3yuki.8mamo10@gmail.com>
+
+The Python interpreter provides low-level support for thread-local storage
+(TLS) which wraps the underlying native TLS implementation to support the
+Python-level thread local storage API (:class:`threading.local`).  The
+CPython C level APIs are similar to those offered by pthreads and Windows:
+use a thread key and functions to associate a :c:type:`void\*` value per
+thread.
+
+The GIL does *not* need to be held when calling these functions; they supply
+their own locking.
+
+Note that :file:`Python.h` does not include the declaration of the TLS APIs,
+you need to include :file:`pythread.h` to use thread-local storage.
+
+.. note::
+   None of these API functions handle memory management on behalf of the
+   :c:type:`void\*` values.  You need to allocate and deallocate them yourself.
+   If the :c:type:`void\*` values happen to be :c:type:`PyObject\*`, these
+   functions don't do refcount operations on them either.
+
+.. _thread-specific-storage-api:
+
+Thread Specific Storage (TSS) API
+---------------------------------
+
+TSS API is introduced to supersede the use of the existing TLS API within the
+CPython interpreter.  This API uses a new type :c:type:`Py_tss_t` instead of
+:c:type:`int` to represent thread keys.
+
+.. versionadded:: 3.7
+
+.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`)
+
+
+.. c:type:: Py_tss_t
+
+   This data structure represents the state of a thread key, the definition of
+   which may depend on the underlying TLS implementation, and it has an
+   internal field representing the key's initialization state.  There are no
+   public members in this structure.
+
+   When :ref:`Py_LIMITED_API <stable>` is not defined, static allocation of
+   this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed.
+
+
+.. c:macro:: Py_tss_NEEDS_INIT
+
+   This macro expands to the default value for :c:type:`Py_tss_t` variables.
+   Note that this macro won't be defined with :ref:`Py_LIMITED_API <stable>`.
+
+
+Dynamic Allocation
+~~~~~~~~~~~~~~~~~~
+
+Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules
+built with :ref:`Py_LIMITED_API <stable>`, where static allocation of this type
+is not possible due to its implementation being opaque at build time.
+
+
+.. c:function:: Py_tss_t* PyThread_tss_alloc()
+
+   Return a value which is the same state as a value initialized with
+   :c:macro:`Py_tss_NEEDS_INIT`, or *NULL* in the case of dynamic allocation
+   failure.
+
+
+.. c:function:: void PyThread_tss_free(Py_tss_t *key)
+
+   Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after
+   first calling :c:func:`PyThread_tss_delete` to ensure any associated
+   thread locals have been unassigned. This is a no-op if the *key*
+   argument is `NULL`.
+
+   .. note::
+      A freed key becomes a dangling pointer, you should reset the key to
+      `NULL`.
+
+
+Methods
+~~~~~~~
+
+The parameter *key* of these functions must not be *NULL*.  Moreover, the
+behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are
+undefined if the given :c:type:`Py_tss_t` has not been initialized by
+:c:func:`PyThread_tss_create`.
+
+
+.. c:function:: int PyThread_tss_is_created(Py_tss_t *key)
+
+   Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized
+   by :c:func:`PyThread_tss_create`.
+
+
+.. c:function:: int PyThread_tss_create(Py_tss_t *key)
+
+   Return a zero value on successful initialization of a TSS key.  The behavior
+   is undefined if the value pointed to by the *key* argument is not
+   initialized by :c:macro:`Py_tss_NEEDS_INIT`.  This function can be called
+   repeatedly on the same key -- calling it on an already initialized key is a
+   no-op and immediately returns success.
+
+
+.. c:function:: void PyThread_tss_delete(Py_tss_t *key)
+
+   Destroy a TSS key to forget the values associated with the key across all
+   threads, and change the key's initialization state to uninitialized.  A
+   destroyed key is able to be initialized again by
+   :c:func:`PyThread_tss_create`. This function can be called repeatedly on
+   the same key -- calling it on an already destroyed key is a no-op.
+
+
+.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value)
+
+   Return a zero value to indicate successfully associating a :c:type:`void\*`
+   value with a TSS key in the current thread.  Each thread has a distinct
+   mapping of the key to a :c:type:`void\*` value.
+
+
+.. c:function:: void* PyThread_tss_get(Py_tss_t *key)
+
+   Return the :c:type:`void\*` value associated with a TSS key in the current
+   thread.  This returns *NULL* if no value is associated with the key in the
+   current thread.
+
+
+.. _thread-local-storage-api:
+
+Thread Local Storage (TLS) API
+------------------------------
+
+.. deprecated:: 3.7
+   This API is superseded by
+   :ref:`Thread Specific Storage (TSS) API <thread-specific-storage-api>`.
+
+.. note::
+   This version of the API does not support platforms where the native TLS key
+   is defined in a way that cannot be safely cast to ``int``.  On such platforms,
+   :c:func:`PyThread_create_key` will return immediately with a failure status,
+   and the other TLS functions will all be no-ops on such platforms.
+
+Due to the compatibility problem noted above, this version of the API should not
+be used in new code.
+
+.. c:function:: int PyThread_create_key()
+.. c:function:: void PyThread_delete_key(int key)
+.. c:function:: int PyThread_set_key_value(int key, void *value)
+.. c:function:: void* PyThread_get_key_value(int key)
+.. c:function:: void PyThread_delete_key_value(int key)
+.. c:function:: void PyThread_ReInitTLS()
+

Doc/whatsnew/3.7.rst

@@ -127,6 +127,38 @@ built-in ``breakpoint()``.
       PEP written and implemented by Barry Warsaw
 
 
+.. _whatsnew37-pep539:
+
+PEP 539: A New C-API for Thread-Local Storage in CPython
+--------------------------------------------------------
+
+While Python provides a C API for thread-local storage support; the existing
+:ref:`Thread Local Storage (TLS) API <thread-local-storage-api>` has used
+:c:type:`int` to represent TLS keys across all platforms.  This has not
+generally been a problem for officially-support platforms, but that is neither
+POSIX-compliant, nor portable in any practical sense.
+
+:pep:`539` changes this by providing a new :ref:`Thread Specific Storage (TSS)
+API <thread-specific-storage-api>` to CPython which supersedes use of the
+existing TLS API within the CPython interpreter, while deprecating the existing
+API.  The TSS API uses a new type :c:type:`Py_tss_t` instead of :c:type:`int`
+to represent TSS keys--an opaque type the definition of which may depend on
+the underlying TLS implementation.  Therefore, this will allow to build CPython
+on platforms where the native TLS key is defined in a way that cannot be safely
+cast to :c:type:`int`.
+
+Note that on platforms where the native TLS key is defined in a way that cannot
+be safely cast to :c:type:`int`, all functions of the existing TLS API will be
+no-op and immediately return failure. This indicates clearly that the old API
+is not supported on platforms where it cannot be used reliably, and that no
+effort will be made to add such support.
+
+.. seealso::
+
+    :pep:`539` -- A New C-API for Thread-Local Storage in CPython
+       PEP written by Erik M. Bray; implementation by Masayuki Yamamoto.
+
+
 Other Language Changes
 ======================
 

Include/internal/pystate.h

@@ -26,7 +26,7 @@ struct _gilstate_runtime_state {
     */
     /* TODO: Given interp_main, it may be possible to kill this ref */
     PyInterpreterState *autoInterpreterState;
-    int autoTLSkey;
+    Py_tss_t autoTSSkey;
 };
 
 /* hook for PyEval_GetFrame(), requested for Psyco */

Include/pythread.h

@@ -77,15 +77,69 @@ PyAPI_FUNC(int) PyThread_set_stacksize(size_t);
 PyAPI_FUNC(PyObject*) PyThread_GetInfo(void);
 #endif
 
-/* Thread Local Storage (TLS) API */
+
-PyAPI_FUNC(int) PyThread_create_key(void);
+/* Thread Local Storage (TLS) API
-PyAPI_FUNC(void) PyThread_delete_key(int);
+   TLS API is DEPRECATED.  Use Thread Specific Storage (TSS) API.
-PyAPI_FUNC(int) PyThread_set_key_value(int, void *);
+
-PyAPI_FUNC(void *) PyThread_get_key_value(int);
+   The existing TLS API has used int to represent TLS keys across all
-PyAPI_FUNC(void) PyThread_delete_key_value(int key);
+   platforms, but it is not POSIX-compliant.  Therefore, the new TSS API uses
+   opaque data type to represent TSS keys to be compatible (see PEP 539).
+*/
+PyAPI_FUNC(int) PyThread_create_key(void) Py_DEPRECATED(3.7);
+PyAPI_FUNC(void) PyThread_delete_key(int key) Py_DEPRECATED(3.7);
+PyAPI_FUNC(int) PyThread_set_key_value(int key, void *value) Py_DEPRECATED(3.7);
+PyAPI_FUNC(void *) PyThread_get_key_value(int key) Py_DEPRECATED(3.7);
+PyAPI_FUNC(void) PyThread_delete_key_value(int key) Py_DEPRECATED(3.7);
 
 /* Cleanup after a fork */
-PyAPI_FUNC(void) PyThread_ReInitTLS(void);
+PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7);
+
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000
+/* New in 3.7 */
+/* Thread Specific Storage (TSS) API */
+
+typedef struct _Py_tss_t Py_tss_t;  /* opaque */
+
+#ifndef Py_LIMITED_API
+#if defined(_POSIX_THREADS)
+    /* Darwin needs pthread.h to know type name the pthread_key_t. */
+#   include <pthread.h>
+#   define NATIVE_TSS_KEY_T     pthread_key_t
+#elif defined(NT_THREADS)
+    /* In Windows, native TSS key type is DWORD,
+       but hardcode the unsigned long to avoid errors for include directive.
+    */
+#   define NATIVE_TSS_KEY_T     unsigned long
+#else
+#   error "Require native threads. See https://bugs.python.org/issue31370"
+#endif
+
+/* When Py_LIMITED_API is not defined, the type layout of Py_tss_t is
+   exposed to allow static allocation in the API clients.  Even in this case,
+   you must handle TSS keys through API functions due to compatibility.
+*/
+struct _Py_tss_t {
+    int _is_initialized;
+    NATIVE_TSS_KEY_T _key;
+};
+
+#undef NATIVE_TSS_KEY_T
+
+/* When static allocation, you must initialize with Py_tss_NEEDS_INIT. */
+#define Py_tss_NEEDS_INIT   {0}
+#endif  /* !Py_LIMITED_API */
+
+PyAPI_FUNC(Py_tss_t *) PyThread_tss_alloc(void);
+PyAPI_FUNC(void) PyThread_tss_free(Py_tss_t *key);
+
+/* The parameter key must not be NULL. */
+PyAPI_FUNC(int) PyThread_tss_is_created(Py_tss_t *key);
+PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key);
+PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key);
+PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t *key, void *value);
+PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key);
+#endif  /* New in 3.7 */
 
 #ifdef __cplusplus
 }

Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst

@@ -0,0 +1,4 @@
+Implement PEP 539 for Thread Specific Stroage (TSS) API: it is a new Thread
+Local Storage (TLS) API to CPython which would supersede use of the existing
+TLS API within the CPython interpreter, while deprecating the existing API.
+PEP written by Erik M. Bray, patch by Masayuki Yamamoto.

Modules/_testcapimodule.c

@@ -4306,6 +4306,61 @@ py_w_stopcode(PyObject *self, PyObject *args)
 #endif
 
 
+static PyObject *
+test_pythread_tss_key_state(PyObject *self, PyObject *args)
+{
+    Py_tss_t tss_key = Py_tss_NEEDS_INIT;
+    if (PyThread_tss_is_created(&tss_key)) {
+        return raiseTestError("test_pythread_tss_key_state",
+                              "TSS key not in an uninitialized state at "
+                              "creation time");
+    }
+    if (PyThread_tss_create(&tss_key) != 0) {
+        PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_create failed");
+        return NULL;
+    }
+    if (!PyThread_tss_is_created(&tss_key)) {
+        return raiseTestError("test_pythread_tss_key_state",
+                              "PyThread_tss_create succeeded, "
+                              "but with TSS key in an uninitialized state");
+    }
+    if (PyThread_tss_create(&tss_key) != 0) {
+        return raiseTestError("test_pythread_tss_key_state",
+                              "PyThread_tss_create unsuccessful with "
+                              "an already initialized key");
+    }
+#define CHECK_TSS_API(expr) \
+        (void)(expr); \
+        if (!PyThread_tss_is_created(&tss_key)) { \
+            return raiseTestError("test_pythread_tss_key_state", \
+                                  "TSS key initialization state was not " \
+                                  "preserved after calling " #expr); }
+    CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL));
+    CHECK_TSS_API(PyThread_tss_get(&tss_key));
+#undef CHECK_TSS_API
+    PyThread_tss_delete(&tss_key);
+    if (PyThread_tss_is_created(&tss_key)) {
+        return raiseTestError("test_pythread_tss_key_state",
+                              "PyThread_tss_delete called, but did not "
+                              "set the key state to uninitialized");
+    }
+
+    Py_tss_t *ptr_key = PyThread_tss_alloc();
+    if (ptr_key == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_alloc failed");
+        return NULL;
+    }
+    if (PyThread_tss_is_created(ptr_key)) {
+        return raiseTestError("test_pythread_tss_key_state",
+                              "TSS key not in an uninitialized state at "
+                              "allocation time");
+    }
+    PyThread_tss_free(ptr_key);
+    ptr_key = NULL;
+    Py_RETURN_NONE;
+}
+
+
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",   (PyCFunction)raise_memoryerror,  METH_NOARGS},
@@ -4518,6 +4573,7 @@ static PyMethodDef TestMethods[] = {
 #ifdef W_STOPCODE
     {"W_STOPCODE", py_w_stopcode, METH_VARARGS},
 #endif
+    {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS},
     {NULL, NULL} /* sentinel */
 };
 

Modules/_tracemalloc.c

@@ -167,14 +167,7 @@ tracemalloc_error(const char *format, ...)
 #if defined(TRACE_RAW_MALLOC)
 #define REENTRANT_THREADLOCAL
 
-/* If your OS does not provide native thread local storage, you can implement
+static Py_tss_t tracemalloc_reentrant_key = Py_tss_NEEDS_INIT;
-   it manually using a lock. Functions of thread.c cannot be used because
-   they use PyMem_RawMalloc() which leads to a reentrant call. */
-#if !(defined(_POSIX_THREADS) || defined(NT_THREADS))
-#  error "need native thread local storage (TLS)"
-#endif
-
-static int tracemalloc_reentrant_key = -1;
 
 /* Any non-NULL pointer can be used */
 #define REENTRANT Py_True
@@ -184,8 +177,8 @@ get_reentrant(void)
 {
     void *ptr;
 
-    assert(tracemalloc_reentrant_key != -1);
+    assert(PyThread_tss_is_created(&tracemalloc_reentrant_key));
-    ptr = PyThread_get_key_value(tracemalloc_reentrant_key);
+    ptr = PyThread_tss_get(&tracemalloc_reentrant_key);
     if (ptr != NULL) {
         assert(ptr == REENTRANT);
         return 1;
@@ -198,15 +191,15 @@ static void
 set_reentrant(int reentrant)
 {
     assert(reentrant == 0 || reentrant == 1);
-    assert(tracemalloc_reentrant_key != -1);
+    assert(PyThread_tss_is_created(&tracemalloc_reentrant_key));
 
     if (reentrant) {
         assert(!get_reentrant());
-        PyThread_set_key_value(tracemalloc_reentrant_key, REENTRANT);
+        PyThread_tss_set(&tracemalloc_reentrant_key, REENTRANT);
     }
     else {
         assert(get_reentrant());
-        PyThread_set_key_value(tracemalloc_reentrant_key, NULL);
+        PyThread_tss_set(&tracemalloc_reentrant_key, NULL);
     }
 }
 
@@ -975,8 +968,7 @@ tracemalloc_init(void)
     PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw);
 
 #ifdef REENTRANT_THREADLOCAL
-    tracemalloc_reentrant_key = PyThread_create_key();
+    if (PyThread_tss_create(&tracemalloc_reentrant_key) != 0) {
-    if (tracemalloc_reentrant_key == -1) {
 #ifdef MS_WINDOWS
         PyErr_SetFromWindowsErr(0);
 #else
@@ -1061,8 +1053,7 @@ tracemalloc_deinit(void)
 #endif
 
 #ifdef REENTRANT_THREADLOCAL
-    PyThread_delete_key(tracemalloc_reentrant_key);
+    PyThread_tss_delete(&tracemalloc_reentrant_key);
-    tracemalloc_reentrant_key = -1;
 #endif
 
     Py_XDECREF(unknown_filename);

Modules/faulthandler.c

@@ -232,7 +232,7 @@ faulthandler_dump_traceback(int fd, int all_threads,
 
        PyThreadState_Get() doesn't give the state of the thread that caused the
        fault if the thread released the GIL, and so this function cannot be
-       used. Read the thread local storage (TLS) instead: call
+       used. Read the thread specific storage (TSS) instead: call
        PyGILState_GetThisThreadState(). */
     tstate = PyGILState_GetThisThreadState();
 

Modules/posixmodule.c

@@ -454,9 +454,6 @@ PyOS_AfterFork_Parent(void)
 void
 PyOS_AfterFork_Child(void)
 {
-    /* PyThread_ReInitTLS() must be called early, to make sure that the TLS API
-     * can be called safely. */
-    PyThread_ReInitTLS();
     _PyGILState_Reinit();
     PyEval_ReInitThreads();
     _PyImport_ReInitLock();

PC/python3.def

@@ -570,6 +570,13 @@ EXPORTS
   PyThreadState_New=python37.PyThreadState_New
   PyThreadState_SetAsyncExc=python37.PyThreadState_SetAsyncExc
   PyThreadState_Swap=python37.PyThreadState_Swap
+  PyThread_tss_alloc=python37.PyThread_tss_alloc
+  PyThread_tss_create=python37.PyThread_tss_create
+  PyThread_tss_delete=python37.PyThread_tss_delete
+  PyThread_tss_free=python37.PyThread_tss_free
+  PyThread_tss_get=python37.PyThread_tss_get
+  PyThread_tss_is_created=python37.PyThread_tss_is_created
+  PyThread_tss_set=python37.PyThread_tss_set
   PyTraceBack_Here=python37.PyTraceBack_Here
   PyTraceBack_Print=python37.PyTraceBack_Print
   PyTraceBack_Type=python37.PyTraceBack_Type DATA

Python/pystate.c

@@ -46,7 +46,12 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime)
     _PyEval_Initialize(&runtime->ceval);
 
     runtime->gilstate.check_enabled = 1;
-    runtime->gilstate.autoTLSkey = -1;
+    /* A TSS key must be initialized with Py_tss_NEEDS_INIT
+       in accordance with the specification. */
+    {
+        Py_tss_t initial = Py_tss_NEEDS_INIT;
+        runtime->gilstate.autoTSSkey = initial;
+    }
 
     runtime->interpreters.mutex = PyThread_allocate_lock();
     if (runtime->interpreters.mutex == NULL)
@@ -485,9 +490,9 @@ PyThreadState_Delete(PyThreadState *tstate)
     if (tstate == GET_TSTATE())
         Py_FatalError("PyThreadState_Delete: tstate is still current");
     if (_PyRuntime.gilstate.autoInterpreterState &&
-        PyThread_get_key_value(_PyRuntime.gilstate.autoTLSkey) == tstate)
+        PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey) == tstate)
     {
-        PyThread_delete_key_value(_PyRuntime.gilstate.autoTLSkey);
+        PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, NULL);
     }
     tstate_delete_common(tstate);
 }
@@ -502,9 +507,9 @@ PyThreadState_DeleteCurrent()
             "PyThreadState_DeleteCurrent: no current tstate");
     tstate_delete_common(tstate);
     if (_PyRuntime.gilstate.autoInterpreterState &&
-        PyThread_get_key_value(_PyRuntime.gilstate.autoTLSkey) == tstate)
+        PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey) == tstate)
     {
-        PyThread_delete_key_value(_PyRuntime.gilstate.autoTLSkey);
+        PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, NULL);
     }
     SET_TSTATE(NULL);
     PyEval_ReleaseLock();
@@ -761,11 +766,11 @@ void
 _PyGILState_Init(PyInterpreterState *i, PyThreadState *t)
 {
     assert(i && t); /* must init with valid states */
-    _PyRuntime.gilstate.autoTLSkey = PyThread_create_key();
+    if (PyThread_tss_create(&_PyRuntime.gilstate.autoTSSkey) != 0) {
-    if (_PyRuntime.gilstate.autoTLSkey == -1)
+        Py_FatalError("Could not allocate TSS entry");
-        Py_FatalError("Could not allocate TLS entry");
+    }
     _PyRuntime.gilstate.autoInterpreterState = i;
-    assert(PyThread_get_key_value(_PyRuntime.gilstate.autoTLSkey) == NULL);
+    assert(PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey) == NULL);
     assert(t->gilstate_counter == 0);
 
     _PyGILState_NoteThreadState(t);
@@ -780,14 +785,13 @@ _PyGILState_GetInterpreterStateUnsafe(void)
 void
 _PyGILState_Fini(void)
 {
-    PyThread_delete_key(_PyRuntime.gilstate.autoTLSkey);
+    PyThread_tss_delete(&_PyRuntime.gilstate.autoTSSkey);
-    _PyRuntime.gilstate.autoTLSkey = -1;
     _PyRuntime.gilstate.autoInterpreterState = NULL;
 }
 
-/* Reset the TLS key - called by PyOS_AfterFork_Child().
+/* Reset the TSS key - called by PyOS_AfterFork_Child().
  * This should not be necessary, but some - buggy - pthread implementations
- * don't reset TLS upon fork(), see issue #10517.
+ * don't reset TSS upon fork(), see issue #10517.
  */
 void
 _PyGILState_Reinit(void)
@@ -796,15 +800,18 @@ _PyGILState_Reinit(void)
     if (_PyRuntime.interpreters.mutex == NULL)
         Py_FatalError("Can't initialize threads for interpreter");
     PyThreadState *tstate = PyGILState_GetThisThreadState();
-    PyThread_delete_key(_PyRuntime.gilstate.autoTLSkey);
+    PyThread_tss_delete(&_PyRuntime.gilstate.autoTSSkey);
-    if ((_PyRuntime.gilstate.autoTLSkey = PyThread_create_key()) == -1)
+    if (PyThread_tss_create(&_PyRuntime.gilstate.autoTSSkey) != 0) {
-        Py_FatalError("Could not allocate TLS entry");
+        Py_FatalError("Could not allocate TSS entry");
+    }
 
     /* If the thread had an associated auto thread state, reassociate it with
      * the new key. */
-    if (tstate && PyThread_set_key_value(_PyRuntime.gilstate.autoTLSkey,
+    if (tstate &&
-                                         (void *)tstate) < 0)
+        PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, (void *)tstate) != 0)
-        Py_FatalError("Couldn't create autoTLSkey mapping");
+    {
+        Py_FatalError("Couldn't create autoTSSkey mapping");
+    }
 }
 
 /* When a thread state is created for a thread by some mechanism other than
@@ -815,13 +822,13 @@ _PyGILState_Reinit(void)
 static void
 _PyGILState_NoteThreadState(PyThreadState* tstate)
 {
-    /* If autoTLSkey isn't initialized, this must be the very first
+    /* If autoTSSkey isn't initialized, this must be the very first
        threadstate created in Py_Initialize().  Don't do anything for now
        (we'll be back here when _PyGILState_Init is called). */
     if (!_PyRuntime.gilstate.autoInterpreterState)
         return;
 
-    /* Stick the thread state for this thread in thread local storage.
+    /* Stick the thread state for this thread in thread specific storage.
 
        The only situation where you can legitimately have more than one
        thread state for an OS level thread is when there are multiple
@@ -833,12 +840,11 @@ _PyGILState_NoteThreadState(PyThreadState* tstate)
        The first thread state created for that given OS level thread will
        "win", which seems reasonable behaviour.
     */
-    if (PyThread_get_key_value(_PyRuntime.gilstate.autoTLSkey) == NULL) {
+    if (PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey) == NULL) {
-        if ((PyThread_set_key_value(_PyRuntime.gilstate.autoTLSkey,
+        if ((PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, (void *)tstate)
-                                    (void *)tstate)
+             ) != 0)
-             ) < 0)
         {
-            Py_FatalError("Couldn't create autoTLSkey mapping");
+            Py_FatalError("Couldn't create autoTSSkey mapping");
         }
     }
 
@@ -852,8 +858,7 @@ PyGILState_GetThisThreadState(void)
 {
     if (_PyRuntime.gilstate.autoInterpreterState == NULL)
         return NULL;
-    return (PyThreadState *)PyThread_get_key_value(
+    return (PyThreadState *)PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey);
-                _PyRuntime.gilstate.autoTLSkey);
 }
 
 int
@@ -864,8 +869,9 @@ PyGILState_Check(void)
     if (!_PyGILState_check_enabled)
         return 1;
 
-    if (_PyRuntime.gilstate.autoTLSkey == -1)
+    if (!PyThread_tss_is_created(&_PyRuntime.gilstate.autoTSSkey)) {
         return 1;
+    }
 
     tstate = GET_TSTATE();
     if (tstate == NULL)
@@ -886,8 +892,7 @@ PyGILState_Ensure(void)
     */
     /* Py_Initialize() hasn't been called! */
     assert(_PyRuntime.gilstate.autoInterpreterState);
-    tcur = (PyThreadState *)PyThread_get_key_value(
+    tcur = (PyThreadState *)PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey);
-                _PyRuntime.gilstate.autoTLSkey);
     if (tcur == NULL) {
         /* At startup, Python has no concrete GIL. If PyGILState_Ensure() is
            called from a new thread for the first time, we need the create the
@@ -919,8 +924,8 @@ PyGILState_Ensure(void)
 void
 PyGILState_Release(PyGILState_STATE oldstate)
 {
-    PyThreadState *tcur = (PyThreadState *)PyThread_get_key_value(
+    PyThreadState *tcur = (PyThreadState *)PyThread_tss_get(
-                                _PyRuntime.gilstate.autoTLSkey);
+                                &_PyRuntime.gilstate.autoTSSkey);
     if (tcur == NULL)
         Py_FatalError("auto-releasing thread-state, "
                       "but no thread-state for this thread");

Python/thread.c

@@ -84,7 +84,7 @@ PyThread_init_thread(void)
 #   define PYTHREAD_NAME "nt"
 #   include "thread_nt.h"
 #else
-#   error "Require native thread feature. See https://bugs.python.org/issue30832"
+#   error "Require native threads. See https://bugs.python.org/issue31370"
 #endif
 
 
@@ -111,41 +111,37 @@ PyThread_set_stacksize(size_t size)
 }
 
 
-/* ------------------------------------------------------------------------
+/* Thread Specific Storage (TSS) API
-Per-thread data ("key") support.
 
-Use PyThread_create_key() to create a new key.  This is typically shared
+   Cross-platform components of TSS API implementation.
-across threads.
+*/
 
-Use PyThread_set_key_value(thekey, value) to associate void* value with
+Py_tss_t *
-thekey in the current thread.  Each thread has a distinct mapping of thekey
+PyThread_tss_alloc(void)
-to a void* value.  Caution:  if the current thread already has a mapping
+{
-for thekey, value is ignored.
+    Py_tss_t *new_key = (Py_tss_t *)PyMem_RawMalloc(sizeof(Py_tss_t));
+    if (new_key == NULL) {
+        return NULL;
+    }
+    new_key->_is_initialized = 0;
+    return new_key;
+}
 
-Use PyThread_get_key_value(thekey) to retrieve the void* value associated
+void
-with thekey in the current thread.  This returns NULL if no value is
+PyThread_tss_free(Py_tss_t *key)
-associated with thekey in the current thread.
+{
+    if (key != NULL) {
+        PyThread_tss_delete(key);
+        PyMem_RawFree((void *)key);
+    }
+}
 
-Use PyThread_delete_key_value(thekey) to forget the current thread's associated
+int
-value for thekey.  PyThread_delete_key(thekey) forgets the values associated
+PyThread_tss_is_created(Py_tss_t *key)
-with thekey across *all* threads.
+{
-
+    assert(key != NULL);
-While some of these functions have error-return values, none set any
+    return key->_is_initialized;
-Python exception.
+}
-
-None of the functions does memory management on behalf of the void* values.
-You need to allocate and deallocate them yourself.  If the void* values
-happen to be PyObject*, these functions don't do refcount operations on
-them either.
-
-The GIL does not need to be held when calling these functions; they supply
-their own locking.  This isn't true of PyThread_create_key(), though (see
-next paragraph).
-
-There's a hidden assumption that PyThread_create_key() will be called before
-any of the other functions are called.  There's also a hidden assumption
-that calls to PyThread_create_key() are serialized externally.
------------------------------------------------------------------------- */
 
 
 PyDoc_STRVAR(threadinfo__doc__,

Python/thread_nt.h

@@ -349,10 +349,15 @@ _pythread_nt_set_stacksize(size_t size)
 #define THREAD_SET_STACKSIZE(x) _pythread_nt_set_stacksize(x)
 
 
+/* Thread Local Storage (TLS) API
+
+   This API is DEPRECATED since Python 3.7.  See PEP 539 for details.
+*/
+
 int
 PyThread_create_key(void)
 {
-    DWORD result= TlsAlloc();
+    DWORD result = TlsAlloc();
     if (result == TLS_OUT_OF_INDEXES)
         return -1;
     return (int)result;
@@ -367,12 +372,8 @@ PyThread_delete_key(int key)
 int
 PyThread_set_key_value(int key, void *value)
 {
-    BOOL ok;
+    BOOL ok = TlsSetValue(key, value);
-
+    return ok ? 0 : -1;
-    ok = TlsSetValue(key, value);
-    if (!ok)
-        return -1;
-    return 0;
 }
 
 void *
@@ -399,9 +400,74 @@ PyThread_delete_key_value(int key)
     TlsSetValue(key, NULL);
 }
 
+
 /* reinitialization of TLS is not necessary after fork when using
  * the native TLS functions.  And forking isn't supported on Windows either.
  */
 void
 PyThread_ReInitTLS(void)
-{}
+{
+}
+
+
+/* Thread Specific Storage (TSS) API
+
+   Platform-specific components of TSS API implementation.
+*/
+
+int
+PyThread_tss_create(Py_tss_t *key)
+{
+    assert(key != NULL);
+    /* If the key has been created, function is silently skipped. */
+    if (key->_is_initialized) {
+        return 0;
+    }
+
+    DWORD result = TlsAlloc();
+    if (result == TLS_OUT_OF_INDEXES) {
+        return -1;
+    }
+    /* In Windows, platform-specific key type is DWORD. */
+    key->_key = result;
+    key->_is_initialized = 1;
+    return 0;
+}
+
+void
+PyThread_tss_delete(Py_tss_t *key)
+{
+    assert(key != NULL);
+    /* If the key has not been created, function is silently skipped. */
+    if (!key->_is_initialized) {
+        return;
+    }
+
+    TlsFree(key->_key);
+    key->_key = TLS_OUT_OF_INDEXES;
+    key->_is_initialized = 0;
+}
+
+int
+PyThread_tss_set(Py_tss_t *key, void *value)
+{
+    assert(key != NULL);
+    BOOL ok = TlsSetValue(key->_key, value);
+    return ok ? 0 : -1;
+}
+
+void *
+PyThread_tss_get(Py_tss_t *key)
+{
+    assert(key != NULL);
+    /* because TSS is used in the Py_END_ALLOW_THREAD macro,
+     * it is necessary to preserve the windows error state, because
+     * it is assumed to be preserved across the call to the macro.
+     * Ideally, the macro should be fixed, but it is simpler to
+     * do it here.
+     */
+    DWORD error = GetLastError();
+    void *result = TlsGetValue(key->_key);
+    SetLastError(error);
+    return result;
+}

Python/thread_pthread.h

@@ -589,9 +589,25 @@ _pythread_pthread_set_stacksize(size_t size)
 #define THREAD_SET_STACKSIZE(x) _pythread_pthread_set_stacksize(x)
 
 
+/* Thread Local Storage (TLS) API
+
+   This API is DEPRECATED since Python 3.7.  See PEP 539 for details.
+*/
+
+/* Issue #25658: On platforms where native TLS key is defined in a way that
+   cannot be safely cast to int, PyThread_create_key returns immediately a
+   failure status and other TLS functions all are no-ops.  This indicates
+   clearly that the old API is not supported on platforms where it cannot be
+   used reliably, and that no effort will be made to add such support.
+
+   Note: PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT will be unnecessary after
+   removing this API.
+*/
+
 int
 PyThread_create_key(void)
 {
+#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT
     pthread_key_t key;
     int fail = pthread_key_create(&key, NULL);
     if (fail)
@@ -603,34 +619,102 @@ PyThread_create_key(void)
         return -1;
     }
     return (int)key;
+#else
+    return -1;  /* never return valid key value. */
+#endif
 }
 
 void
 PyThread_delete_key(int key)
 {
+#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT
     pthread_key_delete(key);
+#endif
 }
 
 void
 PyThread_delete_key_value(int key)
 {
+#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT
     pthread_setspecific(key, NULL);
+#endif
 }
 
 int
 PyThread_set_key_value(int key, void *value)
 {
-    int fail;
+#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT
-    fail = pthread_setspecific(key, value);
+    int fail = pthread_setspecific(key, value);
     return fail ? -1 : 0;
+#else
+    return -1;
+#endif
 }
 
 void *
 PyThread_get_key_value(int key)
 {
+#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT
     return pthread_getspecific(key);
+#else
+    return NULL;
+#endif
 }
 
+
 void
 PyThread_ReInitTLS(void)
-{}
+{
+}
+
+
+/* Thread Specific Storage (TSS) API
+
+   Platform-specific components of TSS API implementation.
+*/
+
+int
+PyThread_tss_create(Py_tss_t *key)
+{
+    assert(key != NULL);
+    /* If the key has been created, function is silently skipped. */
+    if (key->_is_initialized) {
+        return 0;
+    }
+
+    int fail = pthread_key_create(&(key->_key), NULL);
+    if (fail) {
+        return -1;
+    }
+    key->_is_initialized = 1;
+    return 0;
+}
+
+void
+PyThread_tss_delete(Py_tss_t *key)
+{
+    assert(key != NULL);
+    /* If the key has not been created, function is silently skipped. */
+    if (!key->_is_initialized) {
+        return;
+    }
+
+    pthread_key_delete(key->_key);
+    /* pthread has not provided the defined invalid value for the key. */
+    key->_is_initialized = 0;
+}
+
+int
+PyThread_tss_set(Py_tss_t *key, void *value)
+{
+    assert(key != NULL);
+    int fail = pthread_setspecific(key->_key, value);
+    return fail ? -1 : 0;
+}
+
+void *
+PyThread_tss_get(Py_tss_t *key)
+{
+    assert(key != NULL);
+    return pthread_getspecific(key->_key);
+}

Python/traceback.c

@@ -760,7 +760,7 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp,
 
            PyThreadState_Get() doesn't give the state of the thread that caused
            the fault if the thread released the GIL, and so this function
-           cannot be used. Read the thread local storage (TLS) instead: call
+           cannot be used. Read the thread specific storage (TSS) instead: call
            PyGILState_GetThisThreadState(). */
         current_tstate = PyGILState_GetThisThreadState();
     }

configure

@@ -8883,6 +8883,75 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+fi
+
+# Issue #25658: POSIX hasn't defined that pthread_key_t is compatible with int.
+# This checking will be unnecessary after removing deprecated TLS API.
+# The cast to long int works around a bug in the HP C Compiler
+# version HP92453-01 B.11.11.23709.GP, which incorrectly rejects
+# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'.
+# This bug is HP SR number 8606223364.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking size of pthread_key_t" >&5
+$as_echo_n "checking size of pthread_key_t... " >&6; }
+if ${ac_cv_sizeof_pthread_key_t+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_key_t))" "ac_cv_sizeof_pthread_key_t"        "#include <pthread.h>
+"; then :
+
+else
+  if test "$ac_cv_type_pthread_key_t" = yes; then
+     { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error 77 "cannot compute sizeof (pthread_key_t)
+See \`config.log' for more details" "$LINENO" 5; }
+   else
+     ac_cv_sizeof_pthread_key_t=0
+   fi
+fi
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_pthread_key_t" >&5
+$as_echo "$ac_cv_sizeof_pthread_key_t" >&6; }
+
+
+
+cat >>confdefs.h <<_ACEOF
+#define SIZEOF_PTHREAD_KEY_T $ac_cv_sizeof_pthread_key_t
+_ACEOF
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether pthread_key_t is compatible with int" >&5
+$as_echo_n "checking whether pthread_key_t is compatible with int... " >&6; }
+if test "$ac_cv_sizeof_pthread_key_t" -eq "$ac_cv_sizeof_int" ; then
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <pthread.h>
+int
+main ()
+{
+pthread_key_t k; k * 1;
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  ac_pthread_key_t_is_arithmetic_type=yes
+else
+  ac_pthread_key_t_is_arithmetic_type=no
+
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pthread_key_t_is_arithmetic_type" >&5
+$as_echo "$ac_pthread_key_t_is_arithmetic_type" >&6; }
+  if test "$ac_pthread_key_t_is_arithmetic_type" = yes ; then
+
+$as_echo "#define PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT 1" >>confdefs.h
+
+  fi
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
 fi
 CC="$ac_save_cc"
 

configure.ac

@@ -2263,6 +2263,25 @@ if test "$have_pthread_t" = yes ; then
 #endif
   ])
 fi
+
+# Issue #25658: POSIX hasn't defined that pthread_key_t is compatible with int.
+# This checking will be unnecessary after removing deprecated TLS API.
+AC_CHECK_SIZEOF(pthread_key_t, [], [[#include <pthread.h>]])
+AC_MSG_CHECKING(whether pthread_key_t is compatible with int)
+if test "$ac_cv_sizeof_pthread_key_t" -eq "$ac_cv_sizeof_int" ; then
+  AC_COMPILE_IFELSE(
+    [AC_LANG_PROGRAM([[#include <pthread.h>]], [[pthread_key_t k; k * 1;]])],
+    [ac_pthread_key_t_is_arithmetic_type=yes],
+    [ac_pthread_key_t_is_arithmetic_type=no]
+  )
+  AC_MSG_RESULT($ac_pthread_key_t_is_arithmetic_type)
+  if test "$ac_pthread_key_t_is_arithmetic_type" = yes ; then
+    AC_DEFINE(PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT, 1,
+              [Define if pthread_key_t is compatible with int.])
+  fi
+else
+  AC_MSG_RESULT(no)
+fi
 CC="$ac_save_cc"
 
 AC_SUBST(OTHER_LIBTOOL_OPT)

pyconfig.h.in

@@ -1241,6 +1241,9 @@
 /* Define if POSIX semaphores aren't enabled on your system */
 #undef POSIX_SEMAPHORES_NOT_ENABLED
 
+/* Define if pthread_key_t is compatible with int. */
+#undef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT
+
 /* Defined if PTHREAD_SCOPE_SYSTEM supported. */
 #undef PTHREAD_SYSTEM_SCHED_SUPPORTED
 
@@ -1302,6 +1305,9 @@
 /* The size of `pid_t', as computed by sizeof. */
 #undef SIZEOF_PID_T
 
+/* The size of `pthread_key_t', as computed by sizeof. */
+#undef SIZEOF_PTHREAD_KEY_T
+
 /* The size of `pthread_t', as computed by sizeof. */
 #undef SIZEOF_PTHREAD_T