gh-84461: Add --enable-wasm-pthreads and more file systems (GH-91820)

Doc/using/configure.rst

@@ -164,6 +164,12 @@ WebAssemby Options
 
    .. versionadded:: 3.11
 
+.. cmdoption:: --enable-wasm-pthreads
+
+   Turn on pthreads support for WASM.
+
+   .. versionadded:: 3.11
+
 
 Install Options
 ---------------

Misc/NEWS.d/next/Library/2022-04-22-13-01-20.gh-issue-84461.rsCiTH.rst

@@ -0,0 +1,3 @@
+Add :option:`--enable-wasm-pthreads` to enable pthreads support for WASM
+builds. ``Emscripten/node`` no longer has threading enabled by default.
+Include additional file systems.

Tools/wasm/README.md

@@ -121,18 +121,27 @@ functions.
 - The ``select`` module is limited. ``select.select()`` crashes the runtime
   due to lack of exectfd support.
 
-## processes, threads, signals
+## processes, signals
 
 - Processes are not supported. System calls like fork, popen, and subprocess
   fail with ``ENOSYS`` or ``ENOSUP``.
 - Signal support is limited. ``signal.alarm``, ``itimer``, ``sigaction``
   are not available or do not work correctly. ``SIGTERM`` exits the runtime.
 - Keyboard interrupt (CTRL+C) handling is not implemented yet.
-- Browser builds cannot start new threads. Node's web workers consume
-  extra file descriptors.
 - Resource-related functions like ``os.nice`` and most functions of the
   ``resource`` module are not available.
 
+## threading
+
+- Threading is disabled by default. The ``configure`` option
+  ``--enable-wasm-pthreads`` adds compiler flag ``-pthread`` and
+  linker flags ``-sUSE_PTHREADS -sPROXY_TO_PTHREAD``. 
+- pthread support requires WASM threads and SharedArrayBuffer (bulk memory).
+  The Node.JS runtime keeps a pool of web workers around. Each web worker
+  uses several file descriptors (eventfd, epoll, pipe).
+- It's not advised to enable threading when building for browsers or with
+  dynamic linking support; there are performance and stability issues.
+
 ## file system
 
 - Most user, group, and permission related function and modules are not
@@ -173,20 +182,16 @@ functions.
   distutils, multiprocessing, dbm, tests and similar modules
   are not shipped. All other modules are bundled as pre-compiled
   ``pyc`` files.
-- Threading is disabled.
 - In-memory file system (MEMFS) is not persistent and limited.
 - Test modules are disabled by default. Use ``--enable-test-modules`` build
   test modules like ``_testcapi``.
 
 ## wasm32-emscripten in node
 
-Node builds use ``NODERAWFS``, ``USE_PTHREADS`` and ``PROXY_TO_PTHREAD``
+Node builds use ``NODERAWFS``.
-linker options.
 
-- Node RawFS allows direct access to the host file system.
+- Node RawFS allows direct access to the host file system without need to
-- pthread support requires WASM threads and SharedArrayBuffer (bulk memory).
+  perform ``FS.mount()`` call.
-  The runtime keeps a pool of web workers around. Each web worker uses
-  several file descriptors (eventfd, epoll, pipe).
 
 # Hosting Python WASM builds
 

configure

@@ -1018,6 +1018,7 @@ enable_framework
 with_cxx_main
 with_emscripten_target
 enable_wasm_dynamic_linking
+enable_wasm_pthreads
 with_suffix
 enable_shared
 with_static_libpython
@@ -1734,6 +1735,8 @@ Optional Features:
   --enable-wasm-dynamic-linking
                           Enable dynamic linking support for WebAssembly
                           (default is no)
+  --enable-wasm-pthreads  Enable pthread emulation for WebAssembly (default is
+                          no)
   --enable-shared         enable building a shared Python library (default is
                           no)
   --enable-profiling      enable C-level code profiling with gprof (default is
@@ -6318,6 +6321,30 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_wasm_dynamic_linking" >&5
 $as_echo "$enable_wasm_dynamic_linking" >&6; }
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --enable-wasm-pthreads" >&5
+$as_echo_n "checking for --enable-wasm-pthreads... " >&6; }
+# Check whether --enable-wasm-pthreads was given.
+if test "${enable_wasm_pthreads+set}" = set; then :
+  enableval=$enable_wasm_pthreads;
+  case $ac_sys_system in #(
+  Emscripten) :
+     ;; #(
+  WASI) :
+    as_fn_error $? "WASI threading is not implemented yet." "$LINENO" 5 ;; #(
+  *) :
+    as_fn_error $? "--enable-wasm-pthreads only applies to Emscripten and WASI" "$LINENO" 5
+   ;;
+esac
+
+else
+
+  enable_wasm_pthreads=missing
+
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_wasm_pthreads" >&5
+$as_echo "$enable_wasm_pthreads" >&6; }
+
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-suffix" >&5
 $as_echo_n "checking for --with-suffix... " >&6; }
 
@@ -7788,43 +7815,70 @@ then
 fi
 
 # WASM flags
-case $ac_sys_system/$ac_sys_emscripten_target in #(
+case $ac_sys_system in #(
-  Emscripten/browser*) :
+  Emscripten) :
+
+            if test "x$Py_DEBUG" = xyes; then :
+  wasm_debug=yes
+else
+  wasm_debug=no
+fi
+
+        as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"
+
+        as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"
 
-    LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH"
-    LINKFORSHARED="--preload-file=\$(WASM_ASSETS_DIR)"
     if test "x$enable_wasm_dynamic_linking" = xyes; then :
 
       as_fn_append LINKFORSHARED " -sMAIN_MODULE"
 
 fi
-    WASM_ASSETS_DIR=".\$(prefix)"
-    WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py"
-        if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "browser-debug"; then
-      LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS"
-      LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map"
-    else
-      LINKFORSHARED="$LINKFORSHARED -O2 -g0"
-    fi
-   ;; #(
-  Emscripten/node*) :
 
-    LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH -sNODERAWFS -sUSE_PTHREADS"
+    if test "x$enable_wasm_pthreads" = xyes; then :
-    LINKFORSHARED="-sPROXY_TO_PTHREAD -sEXIT_RUNTIME"
-    if test "x$enable_wasm_dynamic_linking" = xyes; then :
 
-      as_fn_append LINKFORSHARED " -sMAIN_MODULE"
+      as_fn_append CFLAGS_NODIST " -pthread"
+      as_fn_append LDFLAGS_NODIST " -sUSE_PTHREADS"
+      as_fn_append LINKFORSHARED " -sPROXY_TO_PTHREAD"
+
+fi
+
+    case $ac_sys_emscripten_target in #(
+  browser*) :
+
+        if test "x$ac_sys_emscripten_target" = xbrowser-debug; then :
+  wasm_debug=yes
+fi
+        as_fn_append LINKFORSHARED " --preload-file=\$(WASM_ASSETS_DIR)"
+        WASM_ASSETS_DIR=".\$(prefix)"
+        WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py"
+                WASM_LINKFORSHARED_DEBUG="-gsource-map --emit-symbol-map"
+       ;; #(
+  node*) :
+
+        if test "x$ac_sys_emscripten_target" = xnode-debug; then :
+  wasm_debug=yes
+fi
+        as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sNODERAWFS"
+        as_fn_append LINKFORSHARED " -sEXIT_RUNTIME"
+        WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map"
+
+     ;; #(
+  *) :
+     ;;
+esac
+
+    if test "x$wasm_debug" = xyes; then :
+
+      as_fn_append LDFLAGS_NODIST " -sASSERTIONS"
+      as_fn_append LINKFORSHARED " $WASM_LINKFORSHARED_DEBUG"
+
+else
+
+      as_fn_append LINKFORSHARED " -O2 -g0"
 
 fi
-    CFLAGS_NODIST="$CFLAGS_NODIST -pthread"
-    if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "node-debug"; then
-      LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS"
-      LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map"
-    else
-      LINKFORSHARED="$LINKFORSHARED -O2 -g0"
-    fi
    ;; #(
-  WASI/*) :
+  WASI) :
 
 
 $as_echo "#define _WASI_EMULATED_SIGNAL 1" >>confdefs.h

configure.ac

@@ -1126,6 +1126,21 @@ AC_ARG_ENABLE([wasm-dynamic-linking],
 ])
 AC_MSG_RESULT([$enable_wasm_dynamic_linking])
 
+AC_MSG_CHECKING([for --enable-wasm-pthreads])
+AC_ARG_ENABLE([wasm-pthreads],
+  [AS_HELP_STRING([--enable-wasm-pthreads],
+                  [Enable pthread emulation for WebAssembly (default is no)])],
+[
+  AS_CASE([$ac_sys_system],
+    [Emscripten], [],
+    [WASI], [AC_MSG_ERROR([WASI threading is not implemented yet.])],
+    [AC_MSG_ERROR([--enable-wasm-pthreads only applies to Emscripten and WASI])]
+  )
+], [
+  enable_wasm_pthreads=missing
+])
+AC_MSG_RESULT([$enable_wasm_pthreads])
+
 AC_MSG_CHECKING([for --with-suffix])
 AC_ARG_WITH([suffix],
             [AS_HELP_STRING([--with-suffix=SUFFIX], [set executable suffix to SUFFIX (default is empty, yes is mapped to '.exe')])],
@@ -1909,38 +1924,53 @@ then
 fi
 
 # WASM flags
-AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
+AS_CASE([$ac_sys_system],
-  [Emscripten/browser*], [
+  [Emscripten], [
-    LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH"
+    dnl build with WASM debug info if either Py_DEBUG is set or the target is
-    LINKFORSHARED="--preload-file=\$(WASM_ASSETS_DIR)"
+    dnl node-debug or browser-debug.
+    AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no])
+    
+    dnl Start with 20 MB and allow to grow
+    AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"])
+
+    dnl Include file system support
+    AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"])
+
     AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
       AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"])
     ])
-    WASM_ASSETS_DIR=".\$(prefix)"
+
-    WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py"
+    AS_VAR_IF([enable_wasm_pthreads], [yes], [
-    dnl separate-dwarf does not seem to work in Chrome DevTools Support.
+      AS_VAR_APPEND([CFLAGS_NODIST], [" -pthread"])
-    if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "browser-debug"; then
+      AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"])
-      LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS"
+      AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"])
-      LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map"
+    ])
-    else
+
-      LINKFORSHARED="$LINKFORSHARED -O2 -g0"
+    AS_CASE([$ac_sys_emscripten_target],
-    fi
+      [browser*], [
-  ],
+        AS_VAR_IF([ac_sys_emscripten_target], [browser-debug], [wasm_debug=yes])
-  [Emscripten/node*], [
+        AS_VAR_APPEND([LINKFORSHARED], [" --preload-file=\$(WASM_ASSETS_DIR)"])
-    LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH -sNODERAWFS -sUSE_PTHREADS"
+        WASM_ASSETS_DIR=".\$(prefix)"
-    LINKFORSHARED="-sPROXY_TO_PTHREAD -sEXIT_RUNTIME"
+        WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py"
-    AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
+        dnl separate-dwarf does not seem to work in Chrome DevTools Support.
-      AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"])
+        WASM_LINKFORSHARED_DEBUG="-gsource-map --emit-symbol-map"
+      ],
+      [node*], [
+        AS_VAR_IF([ac_sys_emscripten_target], [node-debug], [wasm_debug=yes])
+        AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sNODERAWFS"])
+        AS_VAR_APPEND([LINKFORSHARED], [" -sEXIT_RUNTIME"])
+        WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map"
+      ]
+    )
+
+    AS_VAR_IF([wasm_debug], [yes], [
+      AS_VAR_APPEND([LDFLAGS_NODIST], [" -sASSERTIONS"])
+      AS_VAR_APPEND([LINKFORSHARED], [" $WASM_LINKFORSHARED_DEBUG"])
+    ], [
+      AS_VAR_APPEND([LINKFORSHARED], [" -O2 -g0"])
     ])
-    CFLAGS_NODIST="$CFLAGS_NODIST -pthread"
-    if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "node-debug"; then
-      LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS"
-      LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map"
-    else
-      LINKFORSHARED="$LINKFORSHARED -O2 -g0"
-    fi
   ],
-  [WASI/*], [
+  [WASI], [
     AC_DEFINE([_WASI_EMULATED_SIGNAL], [1], [Define to 1 if you want to emulate signals on WASI])
     AC_DEFINE([_WASI_EMULATED_GETPID], [1], [Define to 1 if you want to emulate getpid() on WASI])
     AC_DEFINE([_WASI_EMULATED_PROCESS_CLOCKS], [1], [Define to 1 if you want to emulate process clocks on WASI])