gh-110014: Fix _POSIX_THREADS and _POSIX_SEMAPHORES usage (#110139)

* pycore_pythread.h is now the central place to make sure that
  _POSIX_THREADS and _POSIX_SEMAPHORES macros are defined if
  available.
* Make sure that pycore_pythread.h is included when _POSIX_THREADS
  and _POSIX_SEMAPHORES macros are tested.
* PY_TIMEOUT_MAX is now defined as a constant, since its value
  depends on _POSIX_THREADS, instead of being defined as a macro.
* Prevent integer overflow in the preprocessor when computing
  PY_TIMEOUT_MAX_VALUE on Windows:
  replace "0xFFFFFFFELL * 1000 < LLONG_MAX"
  with "0xFFFFFFFELL < LLONG_MAX / 1000".
* Document the change and give hints how to fix affected code.
* Add an exception for PY_TIMEOUT_MAX  name to smelly.py
* Add PY_TIMEOUT_MAX to the stable ABI

Doc/data/stable_abi.dat

@@ -1,4 +1,5 @@
 role,name,added,ifdef_note,struct_abi_kind
+var,PY_TIMEOUT_MAX,3.2,,
 macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,,
 function,PyAIter_Check,3.10,,
 function,PyArg_Parse,3.2,,

Doc/whatsnew/3.13.rst

@@ -989,6 +989,10 @@ Porting to Python 3.13
 * ``Python.h`` no longer includes the ``<unistd.h>`` standard header file. If
   needed, it should now be included explicitly. For example, it provides the
   functions: ``close()``, ``getpagesize()``, ``getpid()`` and ``sysconf()``.
+  As a consequence, ``_POSIX_SEMAPHORES`` and ``_POSIX_THREADS`` macros are no
+  longer defined by ``Python.h``. The ``HAVE_UNISTD_H`` and ``HAVE_PTHREAD_H``
+  macros defined by ``Python.h`` can be used to decide if ``<unistd.h>`` and
+  ``<pthread.h>`` header files can be included.
   (Contributed by Victor Stinner in :gh:`108765`.)
 
 * ``Python.h`` no longer includes these standard header files: ``<time.h>``,

Include/internal/pycore_condvar.h

@@ -5,18 +5,8 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#ifndef MS_WINDOWS
-#  include <unistd.h>             // _POSIX_THREADS
-#endif
+#include "pycore_pythread.h"      // _POSIX_THREADS
 
-#ifndef _POSIX_THREADS
-/* This means pthreads are not implemented in libc headers, hence the macro
-   not present in unistd.h. But they still can be implemented as an external
-   library (e.g. gnu pth in pthread emulation) */
-# ifdef HAVE_PTHREAD_H
-#  include <pthread.h>            // _POSIX_THREADS
-# endif
-#endif
 
 #ifdef _POSIX_THREADS
 /*

Include/internal/pycore_pythread.h

@@ -9,29 +9,29 @@ extern "C" {
 #endif
 
 
-#ifndef _POSIX_THREADS
-/* This means pthreads are not implemented in libc headers, hence the macro
-   not present in unistd.h. But they still can be implemented as an external
-   library (e.g. gnu pth in pthread emulation) */
-#  ifdef HAVE_PTHREAD_H
-#    include <pthread.h>            // _POSIX_THREADS
-#  endif
-# ifndef _POSIX_THREADS
-/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then
-   enough of the Posix threads package is implemented to support python
-   threads.
+// Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available
+#if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \
+                            && !defined(_POSIX_SEMAPHORES))
+#  include <unistd.h>             // _POSIX_THREADS, _POSIX_SEMAPHORES
+#endif
+#if (defined(HAVE_PTHREAD_H) && !defined(_POSIX_THREADS) \
+                             && !defined(_POSIX_SEMAPHORES))
+   // This means pthreads are not implemented in libc headers, hence the macro
+   // not present in <unistd.h>. But they still can be implemented as an
+   // external library (e.g. gnu pth in pthread emulation)
+#  include <pthread.h>            // _POSIX_THREADS, _POSIX_SEMAPHORES
+#endif
+#if !defined(_POSIX_THREADS) && defined(__hpux) && defined(_SC_THREADS)
+   // Check if we're running on HP-UX and _SC_THREADS is defined. If so, then
+   // enough of the POSIX threads package is implemented to support Python
+   // threads.
+   //
+   // This is valid for HP-UX 11.23 running on an ia64 system. If needed, add
+   // a check of __ia64 to verify that we're running on an ia64 system instead
+   // of a pa-risc system.
+#  define _POSIX_THREADS
+#endif
 
-   This is valid for HP-UX 11.23 running on an ia64 system. If needed, add
-   a check of __ia64 to verify that we're running on an ia64 system instead
-   of a pa-risc system.
-*/
-#  ifdef __hpux
-#   ifdef _SC_THREADS
-#    define _POSIX_THREADS
-#   endif
-#  endif
-# endif /* _POSIX_THREADS */
-#endif /* _POSIX_THREADS */
 
 #if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS)
 #  define _USE_PTHREADS

Include/internal/pycore_semaphore.h

@@ -7,7 +7,8 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_time.h"            // _PyTime_t
+#include "pycore_pythread.h"      // _POSIX_SEMAPHORES
+#include "pycore_time.h"          // _PyTime_t
 
 #ifdef MS_WINDOWS
 #   define WIN32_LEAN_AND_MEAN
@@ -26,6 +27,7 @@
 #   include <semaphore.h>
 #endif
 
+
 #ifdef __cplusplus
 extern "C" {
 #endif

Include/pythread.h

@@ -44,22 +44,7 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int);
 */
 #define PY_TIMEOUT_T long long
 
-#if defined(_POSIX_THREADS)
-   /* PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000),
-      convert microseconds to nanoseconds. */
-#  define PY_TIMEOUT_MAX (LLONG_MAX / 1000)
-#elif defined (NT_THREADS)
-   // WaitForSingleObject() accepts timeout in milliseconds in the range
-   // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no
-   // timeout. 0xFFFFFFFE milliseconds is around 49.7 days.
-#  if 0xFFFFFFFELL * 1000 < LLONG_MAX
-#    define PY_TIMEOUT_MAX (0xFFFFFFFELL * 1000)
-#  else
-#    define PY_TIMEOUT_MAX LLONG_MAX
-#  endif
-#else
-#  define PY_TIMEOUT_MAX LLONG_MAX
-#endif
+PyAPI_DATA(const long long) PY_TIMEOUT_MAX;
 
 
 /* If microseconds == 0, the call is non-blocking: it returns immediately

Lib/test/test_stable_abi_ctypes.py

@@ -35,6 +35,7 @@ class TestStableABIAvailability(unittest.TestCase):
 
 SYMBOL_NAMES = (
 
+    "PY_TIMEOUT_MAX",
     "PyAIter_Check",
     "PyArg_Parse",
     "PyArg_ParseTuple",

Misc/stable_abi.toml

@@ -1843,6 +1843,10 @@
 [function.PyThread_start_new_thread]
     added = '3.2'
 
+# Not mentioned in PEP 384, was implemented as a macro in Python <= 3.12
+[data.PY_TIMEOUT_MAX]
+    added = '3.2'
+
 # The following were added in PC/python3.def in Python 3.3:
 # 7800f75827b1be557be16f3b18f5170fbf9fae08
 # 9c56409d3353b8cd4cfc19e0467bbe23fd34fc92

PC/python3dll.c

@@ -768,6 +768,7 @@ EXPORT_DATA(Py_FileSystemDefaultEncodeErrors)
 EXPORT_DATA(Py_FileSystemDefaultEncoding)
 EXPORT_DATA(Py_GenericAliasType)
 EXPORT_DATA(Py_HasFileSystemDefaultEncoding)
+EXPORT_DATA(PY_TIMEOUT_MAX)
 EXPORT_DATA(Py_UTF8Mode)
 EXPORT_DATA(Py_Version)
 EXPORT_DATA(PyBaseObject_Type)

Python/condvar.h

@@ -41,7 +41,8 @@
 #define _CONDVAR_IMPL_H_
 
 #include "Python.h"
-#include "pycore_condvar.h"
+#include "pycore_pythread.h"      // _POSIX_THREADS
+
 
 #ifdef _POSIX_THREADS
 /*

Python/thread.c

@@ -8,7 +8,7 @@
 #include "Python.h"
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_structseq.h"     // _PyStructSequence_FiniBuiltin()
-#include "pycore_pythread.h"
+#include "pycore_pythread.h"      // _POSIX_THREADS
 
 #ifndef DONT_HAVE_STDIO_H
 #  include <stdio.h>
@@ -17,6 +17,26 @@
 #include <stdlib.h>
 
 
+// Define PY_TIMEOUT_MAX constant.
+#ifdef _POSIX_THREADS
+   // PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000),
+   // convert microseconds to nanoseconds.
+#  define PY_TIMEOUT_MAX_VALUE (LLONG_MAX / 1000)
+#elif defined (NT_THREADS)
+   // WaitForSingleObject() accepts timeout in milliseconds in the range
+   // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no
+   // timeout. 0xFFFFFFFE milliseconds is around 49.7 days.
+#  if 0xFFFFFFFELL < LLONG_MAX / 1000
+#    define PY_TIMEOUT_MAX_VALUE (0xFFFFFFFELL * 1000)
+#  else
+#    define PY_TIMEOUT_MAX_VALUE LLONG_MAX
+#  endif
+#else
+#  define PY_TIMEOUT_MAX_VALUE LLONG_MAX
+#endif
+const long long PY_TIMEOUT_MAX = PY_TIMEOUT_MAX_VALUE;
+
+
 static void PyThread__init_thread(void); /* Forward */
 
 #define initialized _PyRuntime.threads.initialized

Python/thread_pthread.h

@@ -1,4 +1,5 @@
-#include "pycore_interp.h"    // _PyInterpreterState.threads.stacksize
+#include "pycore_interp.h"        // _PyInterpreterState.threads.stacksize
+#include "pycore_pythread.h"      // _POSIX_SEMAPHORES
 
 /* Posix threads interface */
 
@@ -84,10 +85,10 @@
 /* On FreeBSD 4.x, _POSIX_SEMAPHORES is defined empty, so
    we need to add 0 to make it work there as well. */
 #if (_POSIX_SEMAPHORES+0) == -1
-#define HAVE_BROKEN_POSIX_SEMAPHORES
+#  define HAVE_BROKEN_POSIX_SEMAPHORES
 #else
-#include <semaphore.h>
-#include <errno.h>
+#  include <semaphore.h>
+#  include <errno.h>
 #endif
 #endif
 

Tools/build/smelly.py

@@ -11,6 +11,11 @@ ALLOWED_PREFIXES = ('Py', '_Py')
 if sys.platform == 'darwin':
     ALLOWED_PREFIXES += ('__Py',)
 
+# "Legacy": some old symbols are prefixed by "PY_".
+EXCEPTIONS = frozenset({
+    'PY_TIMEOUT_MAX',
+})
+
 IGNORED_EXTENSION = "_ctypes_test"
 # Ignore constructor and destructor functions
 IGNORED_SYMBOLS = {'_init', '_fini'}
@@ -72,7 +77,7 @@ def get_smelly_symbols(stdout):
         symbol = parts[-1]
         result = '%s (type: %s)' % (symbol, symtype)
 
-        if symbol.startswith(ALLOWED_PREFIXES):
+        if symbol.startswith(ALLOWED_PREFIXES) or symbol in EXCEPTIONS:
             python_symbols.append(result)
             continue