gh-95174: Add pthread stubs for WASI (GH-95234)

Co-authored-by: Brett Cannon <brett@python.org>

Doc/library/sys.rst

@@ -1658,6 +1658,8 @@ always available.
    |                  |                                                         |
    |                  |  * ``'nt'``: Windows threads                            |
    |                  |  * ``'pthread'``: POSIX threads                         |
+   |                  |  * ``'pthread-stubs'``: stub POSIX threads              |
+   |                  |    (on WebAssembly platforms without threading support) |
    |                  |  * ``'solaris'``: Solaris threads                       |
    +------------------+---------------------------------------------------------+
    | :const:`lock`    | Name of the lock implementation:                        |

Include/cpython/pthread_stubs.h

@@ -0,0 +1,88 @@
+#ifndef Py_CPYTHON_PTRHEAD_STUBS_H
+#define Py_CPYTHON_PTRHEAD_STUBS_H
+
+#if !defined(HAVE_PTHREAD_STUBS)
+#  error "this header file requires stubbed pthreads."
+#endif
+
+#ifndef _POSIX_THREADS
+#  define _POSIX_THREADS 1
+#endif
+
+/* Minimal pthread stubs for CPython.
+ *
+ * The stubs implement the minimum pthread API for CPython.
+ * - pthread_create() fails.
+ * - pthread_exit() calls exit(0).
+ * - pthread_key_*() functions implement minimal TSS without destructor.
+ * - all other functions do nothing and return 0.
+ */
+
+#ifdef __wasi__
+// WASI's bits/alltypes.h provides type definitions when __NEED_ is set.
+// The header file can be included multiple times.
+#  define __NEED_pthread_cond_t 1
+#  define __NEED_pthread_condattr_t 1
+#  define __NEED_pthread_mutex_t 1
+#  define __NEED_pthread_mutexattr_t 1
+#  define __NEED_pthread_key_t 1
+#  define __NEED_pthread_t 1
+#  define __NEED_pthread_attr_t 1
+#  include <bits/alltypes.h>
+#else
+typedef struct { void *__x; } pthread_cond_t;
+typedef struct { unsigned __attr; } pthread_condattr_t;
+typedef struct { void *__x; } pthread_mutex_t;
+typedef struct { unsigned __attr; } pthread_mutexattr_t;
+typedef unsigned pthread_key_t;
+typedef unsigned pthread_t;
+typedef struct { unsigned __attr; } pthread_attr_t;
+#endif
+
+// mutex
+PyAPI_FUNC(int) pthread_mutex_init(pthread_mutex_t *restrict mutex,
+                                   const pthread_mutexattr_t *restrict attr);
+PyAPI_FUNC(int) pthread_mutex_destroy(pthread_mutex_t *mutex);
+PyAPI_FUNC(int) pthread_mutex_trylock(pthread_mutex_t *mutex);
+PyAPI_FUNC(int) pthread_mutex_lock(pthread_mutex_t *mutex);
+PyAPI_FUNC(int) pthread_mutex_unlock(pthread_mutex_t *mutex);
+
+// condition
+PyAPI_FUNC(int) pthread_cond_init(pthread_cond_t *restrict cond,
+                                  const pthread_condattr_t *restrict attr);
+PyAPI_FUNC(int) pthread_cond_destroy(pthread_cond_t *cond);
+PyAPI_FUNC(int) pthread_cond_wait(pthread_cond_t *restrict cond,
+                                  pthread_mutex_t *restrict mutex);
+PyAPI_FUNC(int) pthread_cond_timedwait(pthread_cond_t *restrict cond,
+                                       pthread_mutex_t *restrict mutex,
+                                       const struct timespec *restrict abstime);
+PyAPI_FUNC(int) pthread_cond_signal(pthread_cond_t *cond);
+PyAPI_FUNC(int) pthread_condattr_init(pthread_condattr_t *attr);
+PyAPI_FUNC(int) pthread_condattr_setclock(
+    pthread_condattr_t *attr, clockid_t clock_id);
+
+// pthread
+PyAPI_FUNC(int) pthread_create(pthread_t *restrict thread,
+                               const pthread_attr_t *restrict attr,
+                               void *(*start_routine)(void *),
+                               void *restrict arg);
+PyAPI_FUNC(int) pthread_detach(pthread_t thread);
+PyAPI_FUNC(pthread_t) pthread_self(void);
+PyAPI_FUNC(int) pthread_exit(void *retval) __attribute__ ((__noreturn__));
+PyAPI_FUNC(int) pthread_attr_init(pthread_attr_t *attr);
+PyAPI_FUNC(int) pthread_attr_setstacksize(pthread_attr_t *attr, size_t stacksize);
+PyAPI_FUNC(int) pthread_attr_destroy(pthread_attr_t *attr);
+
+
+// pthread_key
+#ifndef PTHREAD_KEYS_MAX
+#  define PTHREAD_KEYS_MAX 128
+#endif
+
+PyAPI_FUNC(int) pthread_key_create(pthread_key_t *key,
+                                   void (*destr_function)(void *));
+PyAPI_FUNC(int) pthread_key_delete(pthread_key_t key);
+PyAPI_FUNC(void *) pthread_getspecific(pthread_key_t key);
+PyAPI_FUNC(int) pthread_setspecific(pthread_key_t key, const void *value);
+
+#endif // Py_CPYTHON_PTRHEAD_STUBS_H

Include/cpython/pythread.h

@@ -20,6 +20,9 @@ PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock);
        but hardcode the unsigned long to avoid errors for include directive.
     */
 #   define NATIVE_TSS_KEY_T     unsigned long
+#elif defined(HAVE_PTHREAD_STUBS)
+#   include "cpython/pthread_stubs.h"
+#   define NATIVE_TSS_KEY_T     pthread_key_t
 #else
 #   error "Require native threads. See https://bugs.python.org/issue31370"
 #endif

Lib/test/pythoninfo.py

@@ -588,8 +588,8 @@ def collect_socket(info_add):
 
     try:
         hostname = socket.gethostname()
-    except OSError:
-        # WASI SDK 15.0 does not have gethostname(2).
+    except (OSError, AttributeError):
+        # WASI SDK 16.0 does not have gethostname(2).
         if sys.platform != "wasi":
             raise
     else:

Lib/test/test_sys.py

@@ -626,7 +626,7 @@ class SysModuleTest(unittest.TestCase):
     def test_thread_info(self):
         info = sys.thread_info
         self.assertEqual(len(info), 3)
-        self.assertIn(info.name, ('nt', 'pthread', 'solaris', None))
+        self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None))
         self.assertIn(info.lock, ('semaphore', 'mutex+cond', None))
 
     @unittest.skipUnless(support.is_emscripten, "only available on Emscripten")

Lib/test/test_threadsignals.py

@@ -36,7 +36,9 @@ def send_signals():
     os.kill(process_pid, signal.SIGUSR2)
     signalled_all.release()
 
+
 @threading_helper.requires_working_threading()
+@unittest.skipUnless(hasattr(signal, "alarm"), "test requires signal.alarm")
 class ThreadSignals(unittest.TestCase):
 
     def test_signals(self):

Makefile.pre.in

@@ -1544,6 +1544,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/cpython/objimpl.h \
 		$(srcdir)/Include/cpython/odictobject.h \
 		$(srcdir)/Include/cpython/picklebufobject.h \
+		$(srcdir)/Include/cpython/pthread_stubs.h \
 		$(srcdir)/Include/cpython/pyctype.h \
 		$(srcdir)/Include/cpython/pydebug.h \
 		$(srcdir)/Include/cpython/pyerrors.h \

Misc/NEWS.d/next/Build/2022-07-25-09-48-43.gh-issue-95145.ZNS3dj.rst

@@ -0,0 +1,2 @@
+wasm32-wasi builds no longer depend on WASIX's pthread stubs. Python now has
+its own stubbed pthread API.

Python/thread.c

@@ -55,8 +55,15 @@ PyThread_init_thread(void)
     PyThread__init_thread();
 }
 
-#if defined(_POSIX_THREADS)
+#if defined(HAVE_PTHREAD_STUBS)
+#   define PYTHREAD_NAME "pthread-stubs"
+#   include "thread_pthread_stubs.h"
+#elif defined(_POSIX_THREADS)
+#   if defined(__EMSCRIPTEN__) || !defined(__EMSCRIPTEN_PTHREADS__)
+#     define PYTHREAD_NAME "pthread-stubs"
+#   else
 #     define PYTHREAD_NAME "pthread"
+#   endif
 #   include "thread_pthread.h"
 #elif defined(NT_THREADS)
 #   define PYTHREAD_NAME "nt"
@@ -171,7 +178,9 @@ PyThread_GetInfo(void)
     }
     PyStructSequence_SET_ITEM(threadinfo, pos++, value);
 
-#ifdef _POSIX_THREADS
+#ifdef HAVE_PTHREAD_STUBS
+    value = Py_NewRef(Py_None);
+#elif defined(_POSIX_THREADS)
 #ifdef USE_SEMAPHORES
     value = PyUnicode_FromString("semaphore");
 #else
@@ -182,8 +191,7 @@ PyThread_GetInfo(void)
         return NULL;
     }
 #else
-    Py_INCREF(Py_None);
-    value = Py_None;
+    value = Py_NewRef(Py_None);
 #endif
     PyStructSequence_SET_ITEM(threadinfo, pos++, value);
 

Python/thread_pthread.h

@@ -7,7 +7,9 @@
 #if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR)
 #define destructor xxdestructor
 #endif
-#include <pthread.h>
+#ifndef HAVE_PTHREAD_STUBS
+#  include <pthread.h>
+#endif
 #if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR)
 #undef destructor
 #endif

Python/thread_pthread_stubs.h

@@ -0,0 +1,185 @@
+#include "cpython/pthread_stubs.h"
+
+// mutex
+int
+pthread_mutex_init(pthread_mutex_t *restrict mutex,
+                   const pthread_mutexattr_t *restrict attr)
+{
+    return 0;
+}
+
+int
+pthread_mutex_destroy(pthread_mutex_t *mutex)
+{
+    return 0;
+}
+
+int
+pthread_mutex_trylock(pthread_mutex_t *mutex)
+{
+    return 0;
+}
+
+int
+pthread_mutex_lock(pthread_mutex_t *mutex)
+{
+    return 0;
+}
+
+int
+pthread_mutex_unlock(pthread_mutex_t *mutex)
+{
+    return 0;
+}
+
+// condition
+int
+pthread_cond_init(pthread_cond_t *restrict cond,
+                  const pthread_condattr_t *restrict attr)
+{
+    return 0;
+}
+
+PyAPI_FUNC(int)pthread_cond_destroy(pthread_cond_t *cond)
+{
+    return 0;
+}
+
+int
+pthread_cond_wait(pthread_cond_t *restrict cond,
+                  pthread_mutex_t *restrict mutex)
+{
+    return 0;
+}
+
+int
+pthread_cond_timedwait(pthread_cond_t *restrict cond,
+                       pthread_mutex_t *restrict mutex,
+                       const struct timespec *restrict abstime)
+{
+    return 0;
+}
+
+int
+pthread_cond_signal(pthread_cond_t *cond)
+{
+    return 0;
+}
+
+int
+pthread_condattr_init(pthread_condattr_t *attr)
+{
+    return 0;
+}
+
+int
+pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock_id)
+{
+    return 0;
+}
+
+// pthread
+int
+pthread_create(pthread_t *restrict thread,
+               const pthread_attr_t *restrict attr,
+               void *(*start_routine)(void *),
+               void *restrict arg)
+{
+    return EAGAIN;
+}
+
+int
+pthread_detach(pthread_t thread)
+{
+    return 0;
+}
+
+PyAPI_FUNC(pthread_t) pthread_self(void)
+{
+    return 0;
+}
+
+int
+pthread_exit(void *retval)
+{
+    exit(0);
+}
+
+int
+pthread_attr_init(pthread_attr_t *attr)
+{
+    return 0;
+}
+
+int
+pthread_attr_setstacksize(
+    pthread_attr_t *attr, size_t stacksize)
+{
+    return 0;
+}
+
+int
+pthread_attr_destroy(pthread_attr_t *attr)
+{
+    return 0;
+}
+
+// pthread_key
+typedef struct {
+    bool in_use;
+    void *value;
+} py_tls_entry;
+
+static py_tls_entry py_tls_entries[PTHREAD_KEYS_MAX] = {0};
+
+int
+pthread_key_create(pthread_key_t *key, void (*destr_function)(void *))
+{
+    if (!key) {
+        return EINVAL;
+    }
+    if (destr_function != NULL) {
+        Py_FatalError("pthread_key_create destructor is not supported");
+    }
+    for (pthread_key_t idx = 0; idx < PTHREAD_KEYS_MAX; idx++) {
+        if (!py_tls_entries[idx].in_use) {
+            py_tls_entries[idx].in_use = true;
+            *key = idx;
+            return 0;
+        }
+    }
+    return EAGAIN;
+}
+
+int
+pthread_key_delete(pthread_key_t key)
+{
+    if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) {
+        return EINVAL;
+    }
+    py_tls_entries[key].in_use = false;
+    py_tls_entries[key].value = NULL;
+    return 0;
+}
+
+
+void *
+pthread_getspecific(pthread_key_t key) {
+    if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) {
+        return NULL;
+    }
+    return py_tls_entries[key].value;
+}
+
+int
+pthread_setspecific(pthread_key_t key, const void *value)
+{
+    if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) {
+        return EINVAL;
+    }
+    py_tls_entries[key].value = (void *)value;
+    return 0;
+}
+
+// let thread_pthread define the Python API
+#include "thread_pthread.h"

Tools/wasm/README.md

@@ -226,16 +226,13 @@ AddType application/wasm wasm
 
 # WASI (wasm32-wasi)
 
-WASI builds require [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 15.0+
-and currently [wasix](https://github.com/singlestore-labs/wasix) for POSIX
-compatibility stubs.
+WASI builds require [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+.
 
 ## Cross-compile to wasm32-wasi
 
 The script ``wasi-env`` sets necessary compiler and linker flags as well as
 ``pkg-config`` overrides. The script assumes that WASI-SDK is installed in
-``/opt/wasi-sdk`` or ``$WASI_SDK_PATH`` and WASIX is installed in
-``/opt/wasix`` or ``$WASIX_PATH``.
+``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``.
 
 ```shell
 mkdir -p builddir/wasi
@@ -434,21 +431,15 @@ rm -f wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz
 
 ### Install [wasmtime](https://github.com/bytecodealliance/wasmtime) WASI runtime
 
-**NOTE**: wasmtime 0.37 has a bug. Newer versions should be fine again.
+wasmtime 0.38 or newer is required.
 
 ```shell
 curl -sSf -L -o ~/install-wasmtime.sh https://wasmtime.dev/install.sh
 chmod +x ~/install-wasmtime.sh
-~/install-wasmtime.sh --version v0.36.0
+~/install-wasmtime.sh --version v0.38.0
 ln -srf -t /usr/local/bin/ ~/.wasmtime/bin/wasmtime
 ```
 
-### Install [WASIX](https://github.com/singlestore-labs/wasix)
-
-```shell
-git clone https://github.com/singlestore-labs/wasix.git ~/wasix
-make install -C ~/wasix
-```
 
 ### WASI debugging
 

Tools/wasm/config.site-wasm32-wasi

@@ -32,14 +32,6 @@ ac_cv_func_makedev=no
 # OSError: [Errno 28] Invalid argument: '.'
 ac_cv_func_fdopendir=no
 
-# WASIX stubs we don't want to use.
-ac_cv_func_kill=no
-
-# WASI SDK 15.0 does not have chmod.
-# Ignore WASIX stubs for now.
-ac_cv_func_chmod=no
-ac_cv_func_fchmod=no
-
 # WASI sockets are limited to operations on given socket fd and inet sockets.
 # Disable AF_UNIX and AF_PACKET support, see socketmodule.h.
 ac_cv_header_sys_un_h=no

Tools/wasm/wasi-env

@@ -31,7 +31,6 @@ fi
 
 WASI_SDK_PATH="${WASI_SDK_PATH:-/opt/wasi-sdk}"
 WASI_SYSROOT="${WASI_SDK_PATH}/share/wasi-sysroot"
-WASIX_PATH="${WASIX_PATH:-/opt/wasix}"
 
 if ! test -x "${WASI_SDK_PATH}/bin/clang"; then
     echo "Error: ${WASI_SDK_PATH}/bin/clang does not exist." >&2
@@ -65,12 +64,6 @@ PKG_CONFIG_PATH=""
 PKG_CONFIG_LIBDIR="${WASI_SYSROOT}/lib/pkgconfig:${WASI_SYSROOT}/share/pkgconfig"
 PKG_CONFIG_SYSROOT_DIR="${WASI_SYSROOT}"
 
-# add WASIX (POSIX stubs for WASI) if WASIX is installed
-if test -f "${WASIX_PATH}/lib/libwasix.a"; then
-    CFLAGS="${CFLAGS} -isystem ${WASIX_PATH}/include"
-    LDFLAGS="${LDFLAGS} -L${WASIX_PATH}/lib -lwasix"
-fi
-
 PATH="${WASI_SDK_PATH}/bin:${PATH}"
 
 export WASI_SDK_PATH WASI_SYSROOT

configure

@@ -14748,11 +14748,16 @@ if test "x$ac_cv_lib_cma_pthread_create" = xyes; then :
 
 else
 
+    case $ac_sys_system in #(
+  WASI) :
+    posix_threads=stub ;; #(
+  *) :
     as_fn_error $? "could not find pthreads on your system" "$LINENO" 5
+     ;;
+esac
 
 fi
 
-
 fi
 
 fi
@@ -14908,6 +14913,14 @@ _ACEOF
 fi
 done
 
+fi
+
+if test "x$posix_threads" = xstub; then :
+
+
+$as_echo "#define HAVE_PTHREAD_STUBS 1" >>confdefs.h
+
+
 fi
 
 # Check for enable-ipv6

configure.ac

@@ -4207,9 +4207,11 @@ pthread_create (NULL, NULL, start_routine, NULL)]])],[
     posix_threads=yes
     LIBS="$LIBS -lcma"
     ],[
-    AC_MSG_ERROR([could not find pthreads on your system])
-    ])
-    ])])])])])
+    AS_CASE([$ac_sys_system],
+      [WASI], [posix_threads=stub],
+      [AC_MSG_ERROR([could not find pthreads on your system])]
+    )
+    ])])])])])])
 
     AC_CHECK_LIB(mpc, usconfig, [
     LIBS="$LIBS -lmpc"
@@ -4272,6 +4274,10 @@ if test "$posix_threads" = "yes"; then
       AC_CHECK_FUNCS(pthread_getcpuclockid)
 fi
 
+AS_VAR_IF([posix_threads], [stub], [
+  AC_DEFINE([HAVE_PTHREAD_STUBS], [1], [Define if platform requires stubbed pthreads support])
+])
+
 # Check for enable-ipv6
 AH_TEMPLATE(ENABLE_IPV6, [Define if --enable-ipv6 is specified])
 AC_MSG_CHECKING([if --enable-ipv6 is specified])

pyconfig.h.in

@@ -938,6 +938,9 @@
 /* Define to 1 if you have the `pthread_sigmask' function. */
 #undef HAVE_PTHREAD_SIGMASK
 
+/* Define if platform requires stubbed pthreads support */
+#undef HAVE_PTHREAD_STUBS
+
 /* Define to 1 if you have the <pty.h> header file. */
 #undef HAVE_PTY_H