bpo-40280: Add Tools/wasm with helpers for cross building (GH-29984)

Co-authored-by: Ethan Smith <ethan@ethanhs.me>
Co-authored-by: Brett Cannon <brett@python.org>

Makefile.pre.in

@@ -830,6 +830,22 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
 	else true; \
 	fi
 
+# wasm32-emscripten build
+# wasm assets directory is relative to current build dir, e.g. "./usr/local".
+# --preload-file turns a relative asset path into an absolute path.
+WASM_ASSETS_DIR=".$(prefix)"
+WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py"
+
+$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
+                pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py
+	$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
+	    --builddir . --prefix $(prefix)
+
+python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB)
+	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \
+	    $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \
+	    -s ASSERTIONS=1 --preload-file $(WASM_ASSETS_DIR)
+
 ##########################################################################
 # Build static libmpdec.a
 LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@
@@ -938,6 +954,7 @@ Makefile Modules/config.c: Makefile.pre \
 	$(SHELL) $(MAKESETUP) -c $(srcdir)/Modules/config.c.in \
 				-s Modules \
 				Modules/Setup.local \
+				@MODULES_SETUP_STDLIB@ \
 				$(srcdir)/Modules/Setup.bootstrap \
 				$(srcdir)/Modules/Setup
 	@mv config.c Modules
@@ -2379,6 +2396,7 @@ clean-retain-profile: pycremoval
 	-rm -f pybuilddir.txt
 	-rm -f Lib/lib2to3/*Grammar*.pickle
 	-rm -f _bootstrap_python
+	-rm -f python.html python.js python.data
 	-rm -f Programs/_testembed Programs/_freeze_module
 	-rm -f Python/deepfreeze/*.[co]
 	-rm -f Python/frozen_modules/*.h

Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst

@@ -0,0 +1 @@
+A new directory ``Tools/wasm`` contains WebAssembly-related helpers like ``config.site`` override for wasm32-emscripten, wasm assets generator to bundle the stdlib, and a README.

Tools/wasm/README.md

@@ -0,0 +1,55 @@
+# Python WebAssembly (WASM) build
+
+This directory contains configuration and helpers to facilitate cross
+compilation of CPython to WebAssembly (WASM).
+
+## wasm32-emscripten build
+
+Cross compiling to wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/)
+tool chain and a build Python interpreter.
+All commands below are relative to a repository checkout.
+
+### Compile a build Python interpreter
+
+```shell
+mkdir -p builddir/build
+pushd builddir/build
+../../configure -C
+make -j$(nproc)
+popd
+```
+
+### Fetch and build additional emscripten ports
+
+```shell
+embuilder build zlib
+```
+
+### Cross compile to wasm32-emscripten
+
+```shell
+mkdir -p builddir/emscripten
+pushd builddir/emscripten
+
+CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
+  emconfigure ../../configure -C \
+    --host=wasm32-unknown-emscripten \
+    --build=$(../../config.guess) \
+    --with-build-python=$(pwd)/../build/python
+
+emmake make -j$(nproc) python.html
+```
+
+### Test in browser
+
+Serve `python.html` with a local webserver and open the file in a browser.
+
+```shell
+emrun python.html
+```
+
+or
+
+```shell
+python3 -m http.server
+```

Tools/wasm/config.site-wasm32-emscripten

@@ -0,0 +1,70 @@
+# config.site override for cross compiling to wasm32-emscripten platform
+#
+# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
+#     emconfigure ./configure --host=wasm32-unknown-emscripten --build=...
+#
+# Written by Christian Heimes <christian@python.org>
+# Partly based on pyodide's pyconfig.undefs.h file.
+#
+
+# cannot be detected in cross builds
+ac_cv_buggy_getaddrinfo=no
+
+# Emscripten has no /dev/pt*
+ac_cv_file__dev_ptmx=no
+ac_cv_file__dev_ptc=no
+
+# dummy readelf, Emscripten build does not need readelf.
+ac_cv_prog_ac_ct_READELF=true
+
+# new undefined symbols / unsupported features
+ac_cv_func_posix_spawn=no
+ac_cv_func_posix_spawnp=no
+ac_cv_func_eventfd=no
+ac_cv_func_memfd_create=no
+ac_cv_func_prlimit=no
+
+# unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393
+ac_cv_func_shutdown=no
+
+# breaks build, see https://github.com/ethanhs/python-wasm/issues/16
+ac_cv_lib_bz2_BZ2_bzCompress=no
+
+# The rest is based on pyodide
+# https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h
+
+ac_cv_func_epoll=no
+ac_cv_func_epoll_create1=no
+ac_cv_header_linux_vm_sockets_h=no
+ac_cv_func_socketpair=no
+ac_cv_func_utimensat=no
+ac_cv_func_sigaction=no
+
+# Untested syscalls in emscripten
+ac_cv_func_openat=no
+ac_cv_func_mkdirat=no
+ac_cv_func_fchownat=no
+ac_cv_func_renameat=no
+ac_cv_func_linkat=no
+ac_cv_func_symlinkat=no
+ac_cv_func_readlinkat=no
+ac_cv_func_fchmodat=no
+ac_cv_func_dup3=no
+
+# Syscalls not implemented in emscripten
+ac_cv_func_preadv2=no
+ac_cv_func_preadv=no
+ac_cv_func_pwritev2=no
+ac_cv_func_pwritev=no
+ac_cv_func_pipe2=no
+ac_cv_func_nice=no
+
+# Syscalls that resulted in a segfault
+ac_cv_func_utimensat=no
+ac_cv_header_sys_ioctl_h=no
+
+# sockets are supported, but only in non-blocking mode
+# ac_cv_header_sys_socket_h=no
+
+# Unsupported functionality
+#undef HAVE_PTHREAD_H

Tools/wasm/wasm_assets.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+"""Create a WASM asset bundle directory structure.
+
+The WASM asset bundles are pre-loaded by the final WASM build. The bundle
+contains:
+
+- a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip
+- os.py as marker module {PREFIX}/lib/python3.11/os.py
+- empty lib-dynload directory, to make sure it is copied into the bundle {PREFIX}/lib/python3.11/lib-dynload/.empty
+"""
+
+import argparse
+import pathlib
+import shutil
+import sys
+import zipfile
+
+# source directory
+SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
+SRCDIR_LIB = SRCDIR / "Lib"
+
+# sysconfig data relative to build dir.
+SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py"
+
+# Library directory relative to $(prefix).
+WASM_LIB = pathlib.PurePath("lib")
+WASM_STDLIB_ZIP = (
+    WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
+)
+WASM_STDLIB = (
+    WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
+)
+WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"
+
+
+# Don't ship large files / packages that are not particularly useful at
+# the moment.
+OMIT_FILES = (
+    # regression tests
+    "test/",
+    # user interfaces: TK, curses
+    "curses/",
+    "idlelib/",
+    "tkinter/",
+    "turtle.py",
+    "turtledemo/",
+    # package management
+    "ensurepip/",
+    "venv/",
+    # build system
+    "distutils/",
+    "lib2to3/",
+    # concurrency
+    "concurrent/",
+    "multiprocessing/",
+    # deprecated
+    "asyncore.py",
+    "asynchat.py",
+    # Synchronous network I/O and protocols are not supported; for example,
+    # socket.create_connection() raises an exception:
+    # "BlockingIOError: [Errno 26] Operation in progress".
+    "cgi.py",
+    "cgitb.py",
+    "email/",
+    "ftplib.py",
+    "http/",
+    "imaplib.py",
+    "nntplib.py",
+    "poplib.py",
+    "smtpd.py",
+    "smtplib.py",
+    "socketserver.py",
+    "telnetlib.py",
+    "urllib/",
+    "wsgiref/",
+    "xmlrpc/",
+    # dbm / gdbm
+    "dbm/",
+    # other platforms
+    "_aix_support.py",
+    "_bootsubprocess.py",
+    "_osx_support.py",
+    # webbrowser
+    "antigravity.py",
+    "webbrowser.py",
+    # ctypes
+    "ctypes/",
+    # Pure Python implementations of C extensions
+    "_pydecimal.py",
+    "_pyio.py",
+    # Misc unused or large files
+    "pydoc_data/",
+    "msilib/",
+)
+
+# regression test sub directories
+OMIT_SUBDIRS = (
+    "ctypes/test/",
+    "tkinter/test/",
+    "unittest/test/",
+)
+
+
+OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES}
+OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS)
+
+
+def filterfunc(name: str) -> bool:
+    return not name.startswith(OMIT_SUBDIRS_ABSOLUTE)
+
+
+def create_stdlib_zip(
+    args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0
+) -> None:
+    sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB))
+    if not sysconfig_data:
+        raise ValueError("No sysconfigdata file found")
+
+    with zipfile.PyZipFile(
+        args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0
+    ) as pzf:
+        for entry in sorted(args.srcdir_lib.iterdir()):
+            if entry.name == "__pycache__":
+                continue
+            if entry in OMIT_ABSOLUTE:
+                continue
+            if entry.name.endswith(".py") or entry.is_dir():
+                # writepy() writes .pyc files (bytecode).
+                pzf.writepy(entry, filterfunc=filterfunc)
+        for entry in sysconfig_data:
+            pzf.writepy(entry)
+
+
+def path(val: str) -> pathlib.Path:
+    return pathlib.Path(val).absolute()
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--builddir",
+    help="absolute build directory",
+    default=pathlib.Path(".").absolute(),
+    type=path,
+)
+parser.add_argument(
+    "--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path
+)
+
+
+def main():
+    args = parser.parse_args()
+
+    relative_prefix = args.prefix.relative_to(pathlib.Path("/"))
+    args.srcdir = SRCDIR
+    args.srcdir_lib = SRCDIR_LIB
+    args.wasm_root = args.builddir / relative_prefix
+    args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP
+    args.wasm_stdlib = args.wasm_root / WASM_STDLIB
+    args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
+
+    # Empty, unused directory for dynamic libs, but required for site initialization.
+    args.wasm_dynload.mkdir(parents=True, exist_ok=True)
+    marker = args.wasm_dynload / ".empty"
+    marker.touch()
+    # os.py is a marker for finding the correct lib directory.
+    shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
+    # The rest of stdlib that's useful in a WASM context.
+    create_stdlib_zip(args)
+    size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2)
+    parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
+
+
+if __name__ == "__main__":
+    main()

configure

@@ -772,6 +772,7 @@ MODULE_TIME_FALSE
 MODULE_TIME_TRUE
 MODULE__IO_FALSE
 MODULE__IO_TRUE
+MODULES_SETUP_STDLIB
 MODULE_BUILDTYPE
 TEST_MODULES
 LIBRARY_DEPS
@@ -13298,7 +13299,13 @@ fi
 
 if test -z "$with_pymalloc"
 then
+    case $ac_sys_system in #(
+  Emscripten) :
+    with_pymalloc="no" ;; #(
+  *) :
     with_pymalloc="yes"
+   ;;
+esac
 fi
 if test "$with_pymalloc" != "no"
 then
@@ -21165,12 +21172,22 @@ fi
 
 if test "$enable_test_modules" = no; then
     TEST_MODULES=no
-    { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
 else
+    case $ac_sys_system in #(
+  Emscripten) :
+    TEST_MODULES=no ;; #(
+  *) :
     TEST_MODULES=yes
-    { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+     ;;
+esac
+fi
+if test "x$TEST_MODULES" = xyes; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
 $as_echo "no" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
 fi
 
 
@@ -21189,7 +21206,7 @@ case $ac_sys_system in #(
     py_stdlib_not_available="_scproxy spwd" ;; #(
   Emscripten) :
 
-    py_stdlib_not_available="_curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _xxsubinterpreters fcntl grp nis ossaudiodev resource spwd syslog termios"
+    py_stdlib_not_available="_ctypes _curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _tkinter _xxsubinterpreters fcntl grp nis ossaudiodev resource readline spwd syslog termios"
    ;; #(
   *) :
     py_stdlib_not_available="_scproxy"
@@ -21205,6 +21222,20 @@ case $host_cpu in #(
 esac
 
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for additional Modules/Setup files" >&5
+$as_echo_n "checking for additional Modules/Setup files... " >&6; }
+case $ac_sys_system in #(
+  Emscripten) :
+    MODULES_SETUP_STDLIB=Modules/Setup.stdlib ;; #(
+  *) :
+    MODULES_SETUP_STDLIB=
+ ;;
+esac
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $MODULES_SETUP_STDLIB" >&5
+$as_echo "$MODULES_SETUP_STDLIB" >&6; }
+
+
+
 
 MODULE_BLOCK=
 
@@ -25100,7 +25131,7 @@ fi
 $as_echo "$as_me: creating Makefile" >&6;}
 $SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
 			-s Modules \
-			Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
+			Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
 mv config.c Modules
 
 if test -z "$PKG_CONFIG"; then

configure.ac

@@ -3865,7 +3865,11 @@ AC_ARG_WITH(pymalloc,
 
 if test -z "$with_pymalloc"
 then
-    with_pymalloc="yes"
+  dnl default to yes except for wasm32-emscripten
+  AS_CASE([$ac_sys_system],
+    [Emscripten], [with_pymalloc="no"],
+    [with_pymalloc="yes"]
+  )
 fi
 if test "$with_pymalloc" != "no"
 then
@@ -6253,11 +6257,15 @@ AC_ARG_ENABLE(test-modules,
               AS_HELP_STRING([--disable-test-modules], [don't build nor install test modules]))
 if test "$enable_test_modules" = no; then
     TEST_MODULES=no
-    AC_MSG_RESULT(yes)
 else
-    TEST_MODULES=yes
+    AS_CASE([$ac_sys_system],
-    AC_MSG_RESULT(no)
+      [Emscripten], [TEST_MODULES=no],
+      [TEST_MODULES=yes]
+    )
 fi
+AS_VAR_IF([TEST_MODULES], [yes],
+  [AC_MSG_RESULT(no)], [AC_MSG_RESULT(yes)]
+)
 AC_SUBST(TEST_MODULES)
 
 dnl Modules that are not available on some platforms
@@ -6272,6 +6280,7 @@ AS_CASE([$ac_sys_system],
   [FreeBSD*], [py_stdlib_not_available="_scproxy spwd"],
   [Emscripten], [
     py_stdlib_not_available="m4_normalize([
+      _ctypes
       _curses
       _curses_panel
       _dbm
@@ -6280,12 +6289,14 @@ AS_CASE([$ac_sys_system],
       _posixshmem
       _posixsubprocess
       _scproxy
+      _tkinter
       _xxsubinterpreters
       fcntl
       grp
       nis
       ossaudiodev
       resource
+      readline
       spwd
       syslog
       termios
@@ -6301,6 +6312,16 @@ AS_CASE([$host_cpu],
 )
 AC_SUBST([MODULE_BUILDTYPE])
 
+dnl Use Modules/Setup.stdlib as additional provider?
+AC_MSG_CHECKING([for additional Modules/Setup files])
+AS_CASE([$ac_sys_system],
+    [Emscripten], [MODULES_SETUP_STDLIB=Modules/Setup.stdlib],
+    [MODULES_SETUP_STDLIB=]
+)
+AC_MSG_RESULT([$MODULES_SETUP_STDLIB])
+AC_SUBST([MODULES_SETUP_STDLIB])
+
+
 dnl _MODULE_BLOCK_ADD([VAR], [VALUE])
 dnl internal: adds $1=quote($2) to MODULE_BLOCK
 AC_DEFUN([_MODULE_BLOCK_ADD], [AS_VAR_APPEND([MODULE_BLOCK], ["$1=_AS_QUOTE([$2])$as_nl"])])
@@ -6515,7 +6536,7 @@ fi
 AC_MSG_NOTICE([creating Makefile])
 $SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
 			-s Modules \
-			Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
+			Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
 mv config.c Modules
 
 if test -z "$PKG_CONFIG"; then