gh-95853: Multiple ops and debug for wasm_build.py (#96744)

Lib/distutils/tests/test_sysconfig.py

@@ -48,6 +48,7 @@ class SysconfigTestCase(support.EnvironGuard, unittest.TestCase):
         self.assertIsInstance(cvars, dict)
         self.assertTrue(cvars)
 
+    @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
     def test_srcdir(self):
         # See Issues #15322, #15364.
         srcdir = sysconfig.get_config_var('srcdir')

Lib/test/test_sysconfig.py

@@ -438,6 +438,7 @@ class TestSysConfig(unittest.TestCase):
         self.assertEqual(status, 0)
         self.assertEqual(my_platform, test_platform)
 
+    @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
     def test_srcdir(self):
         # See Issues #15322, #15364.
         srcdir = sysconfig.get_config_var('srcdir')

Makefile.pre.in

@@ -1718,6 +1718,10 @@ buildbottest: all
 		fi
 		$(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
+# Like testall, but run Python tests with HOSTRUNNER directly.
+hostrunnertest: all
+	$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS)
+
 pythoninfo: all
 		$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo
 

Tools/wasm/README.md

@@ -1,11 +1,16 @@
 # Python WebAssembly (WASM) build
 
-**WARNING: WASM support is highly experimental! Lots of features are not working yet.**
+**WARNING: WASM support is work-in-progress! Lots of features are not working yet.**
 
 This directory contains configuration and helpers to facilitate cross
-compilation of CPython to WebAssembly (WASM). For now we support
+compilation of CPython to WebAssembly (WASM). Python supports Emscripten
-*wasm32-emscripten* builds for modern browser and for *Node.js*. WASI
+(*wasm32-emscripten*) and WASI (*wasm32-wasi*) targets. Emscripten builds
-(*wasm32-wasi*) is work-in-progress
+run in modern browsers and JavaScript runtimes like *Node.js*. WASI builds
+use WASM runtimes such as *wasmtime*.
+
+Users and developers are encouraged to use the script
+`Tools/wasm/wasm_build.py`. The tool automates the build process and provides
+assistance with installation of SDKs.
 
 ## wasm32-emscripten build
 
@@ -17,7 +22,7 @@ access the file system directly.
 
 Cross compiling to the wasm32-emscripten platform needs the
 [Emscripten](https://emscripten.org/) SDK and a build Python interpreter.
-Emscripten 3.1.8 or newer are recommended. All commands below are relative
+Emscripten 3.1.19 or newer are recommended. All commands below are relative
 to a repository checkout.
 
 Christian Heimes maintains a container image with Emscripten SDK, Python
@@ -336,26 +341,46 @@ if os.name == "posix":
 ```python
 >>> import os, sys
 >>> os.uname()
-posix.uname_result(sysname='Emscripten', nodename='emscripten', release='1.0', version='#1', machine='wasm32')
+posix.uname_result(
+    sysname='Emscripten',
+    nodename='emscripten',
+    release='3.1.19',
+    version='#1',
+    machine='wasm32'
+)
 >>> os.name
 'posix'
 >>> sys.platform
 'emscripten'
 >>> sys._emscripten_info
 sys._emscripten_info(
-    emscripten_version=(3, 1, 8),
+    emscripten_version=(3, 1, 10),
-    runtime='Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0',
+    runtime='Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0',
     pthreads=False,
     shared_memory=False
 )
+```
+
+```python
 >>> sys._emscripten_info
-sys._emscripten_info(emscripten_version=(3, 1, 8), runtime='Node.js v14.18.2', pthreads=True, shared_memory=True)
+sys._emscripten_info(
+    emscripten_version=(3, 1, 19),
+    runtime='Node.js v14.18.2',
+    pthreads=True,
+    shared_memory=True
+)
 ```
 
 ```python
 >>> import os, sys
 >>> os.uname()
-posix.uname_result(sysname='wasi', nodename='(none)', release='0.0.0', version='0.0.0', machine='wasm32')
+posix.uname_result(
+    sysname='wasi',
+    nodename='(none)',
+    release='0.0.0',
+    version='0.0.0',
+    machine='wasm32'
+)
 >>> os.name
 'posix'
 >>> sys.platform
@@ -446,7 +471,8 @@ embuilder build --pic zlib bzip2 MINIMAL_PIC
 
 **NOTE**: WASI-SDK's clang may show a warning on Fedora:
 ``/lib64/libtinfo.so.6: no version information available``,
-[RHBZ#1875587](https://bugzilla.redhat.com/show_bug.cgi?id=1875587).
+[RHBZ#1875587](https://bugzilla.redhat.com/show_bug.cgi?id=1875587). The
+warning can be ignored.
 
 ```shell
 export WASI_VERSION=16
@@ -471,6 +497,8 @@ ln -srf -t /usr/local/bin/ ~/.wasmtime/bin/wasmtime
 
 ### WASI debugging
 
-* ``wasmtime run -g`` generates debugging symbols for gdb and lldb.
+* ``wasmtime run -g`` generates debugging symbols for gdb and lldb. The
+  feature is currently broken, see
+  https://github.com/bytecodealliance/wasmtime/issues/4669 .
 * The environment variable ``RUST_LOG=wasi_common`` enables debug and
   trace logging.

Tools/wasm/wasm_build.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 """Build script for Python on WebAssembly platforms.
 
-  $ ./Tools/wasm/wasm_builder.py emscripten-browser compile
+  $ ./Tools/wasm/wasm_builder.py emscripten-browser build repl
-  $ ./Tools/wasm/wasm_builder.py emscripten-node-dl test
+  $ ./Tools/wasm/wasm_builder.py emscripten-node-dl build test
-  $ ./Tools/wasm/wasm_builder.py wasi test
+  $ ./Tools/wasm/wasm_builder.py wasi build test
 
 Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
 "emscripten-browser", and "wasi".
@@ -14,23 +14,36 @@ activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages
 
 WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
 and falls back to /opt/wasi-sdk.
+
+The 'build' Python interpreter must be rebuilt every time Python's byte code
+changes.
+
+  ./Tools/wasm/wasm_builder.py --clean build build
+
 """
 import argparse
 import enum
 import dataclasses
+import logging
 import os
 import pathlib
 import re
 import shlex
 import shutil
+import socket
 import subprocess
+import sys
 import sysconfig
 import tempfile
+import time
 import warnings
+import webbrowser
 
 # for Python 3.8
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
+logger = logging.getLogger("wasm_build")
+
 SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
 WASMTOOLS = SRCDIR / "Tools" / "wasm"
 BUILDDIR = SRCDIR / "builddir"
@@ -45,8 +58,7 @@ WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk"))
 # path to Emscripten SDK config file.
 # auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
 EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten"))
-# 3.1.16 has broken utime()
+EMSDK_MIN_VERSION = (3, 1, 19)
-EMSDK_MIN_VERSION = (3, 1, 17)
 EMSDK_BROKEN_VERSION = {
     (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338",
     (3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393",
@@ -54,17 +66,25 @@ EMSDK_BROKEN_VERSION = {
 }
 _MISSING = pathlib.PurePath("MISSING")
 
-# WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py"
+WASM_WEBSERVER = WASMTOOLS / "wasm_webserver.py"
 
 CLEAN_SRCDIR = f"""
 Builds require a clean source directory. Please use a clean checkout or
 run "make clean -C '{SRCDIR}'".
 """
 
+INSTALL_NATIVE = f"""
+Builds require a C compiler (gcc, clang), make, pkg-config, and development
+headers for dependencies like zlib.
+
+Debian/Ubuntu: sudo apt install build-essential git curl pkg-config zlib1g-dev
+Fedora/CentOS: sudo dnf install gcc make git-core curl pkgconfig zlib-devel
+"""
+
 INSTALL_EMSDK = """
 wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
 https://emscripten.org/docs/getting_started/downloads.html how to install
-Emscripten and how to activate the SDK with ". /path/to/emsdk/emsdk_env.sh".
+Emscripten and how to activate the SDK with "emsdk_env.sh".
 
     git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
     cd /path/to/emsdk
@@ -182,6 +202,24 @@ def _check_clean_src():
             raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
 
 
+def _check_native():
+    if not any(shutil.which(cc) for cc in ["cc", "gcc", "clang"]):
+        raise MissingDependency("cc", INSTALL_NATIVE)
+    if not shutil.which("make"):
+        raise MissingDependency("make", INSTALL_NATIVE)
+    if sys.platform == "linux":
+        # skip pkg-config check on macOS
+        if not shutil.which("pkg-config"):
+            raise MissingDependency("pkg-config", INSTALL_NATIVE)
+        # zlib is needed to create zip files
+        for devel in ["zlib"]:
+            try:
+                subprocess.check_call(["pkg-config", "--exists", devel])
+            except subprocess.CalledProcessError:
+                raise MissingDependency(devel, INSTALL_NATIVE) from None
+    _check_clean_src()
+
+
 NATIVE = Platform(
     "native",
     # macOS has python.exe
@@ -192,7 +230,7 @@ NATIVE = Platform(
     cc=None,
     make_wrapper=None,
     environ={},
-    check=_check_clean_src,
+    check=_check_native,
 )
 
 
@@ -362,9 +400,9 @@ class EmscriptenTarget(enum.Enum):
     node_debug = "node-debug"
 
     @property
-    def can_execute(self) -> bool:
+    def is_browser(self):
         cls = type(self)
-        return self not in {cls.browser, cls.browser_debug}
+        return self in {cls.browser, cls.browser_debug}
 
     @property
     def emport_args(self) -> List[str]:
@@ -396,15 +434,12 @@ class BuildProfile:
     target: Union[EmscriptenTarget, None] = None
     dynamic_linking: Union[bool, None] = None
     pthreads: Union[bool, None] = None
-    testopts: str = "-j2"
+    default_testopts: str = "-j2"
 
     @property
-    def can_execute(self) -> bool:
+    def is_browser(self) -> bool:
-        """Can target run pythoninfo and tests?
+        """Is this a browser build?"""
-
+        return self.target is not None and self.target.is_browser
-        Disabled for browser, enabled for all other targets
-        """
-        return self.target is None or self.target.can_execute
 
     @property
     def builddir(self) -> pathlib.Path:
@@ -500,6 +535,7 @@ class BuildProfile:
         cmd.extend(args)
         if cwd is None:
             cwd = self.builddir
+        logger.info('Running "%s" in "%s"', shlex.join(cmd), cwd)
         return subprocess.check_call(
             cmd,
             cwd=os.fspath(cwd),
@@ -507,14 +543,15 @@ class BuildProfile:
         )
 
     def _check_execute(self):
-        if not self.can_execute:
+        if self.is_browser:
             raise ValueError(f"Cannot execute on {self.target}")
 
-    def run_build(self, force_configure: bool = False):
+    def run_build(self, *args):
         """Run configure (if necessary) and make"""
-        if force_configure or not self.makefile.exists():
+        if not self.makefile.exists():
-            self.run_configure()
+            logger.info("Makefile not found, running configure")
-        self.run_make()
+            self.run_configure(*args)
+        self.run_make("all", *args)
 
     def run_configure(self, *args):
         """Run configure script to generate Makefile"""
@@ -525,15 +562,17 @@ class BuildProfile:
         """Run make (defaults to build all)"""
         return self._run_cmd(self.make_cmd, args)
 
-    def run_pythoninfo(self):
+    def run_pythoninfo(self, *args):
         """Run 'make pythoninfo'"""
         self._check_execute()
-        return self.run_make("pythoninfo")
+        return self.run_make("pythoninfo", *args)
 
-    def run_test(self):
+    def run_test(self, target: str, testopts: Optional[str] = None):
         """Run buildbottests"""
         self._check_execute()
-        return self.run_make("buildbottest", f"TESTOPTS={self.testopts}")
+        if testopts is None:
+            testopts = self.default_testopts
+        return self.run_make(target, f"TESTOPTS={testopts}")
 
     def run_py(self, *args):
         """Run Python with hostrunner"""
@@ -542,6 +581,37 @@ class BuildProfile:
             "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run"
         )
 
+    def run_browser(self, bind="127.0.0.1", port=8000):
+        """Run WASM webserver and open build in browser"""
+        relbuilddir = self.builddir.relative_to(SRCDIR)
+        url = f"http://{bind}:{port}/{relbuilddir}/python.html"
+        args = [
+            sys.executable,
+            os.fspath(WASM_WEBSERVER),
+            "--bind",
+            bind,
+            "--port",
+            str(port),
+        ]
+        srv = subprocess.Popen(args, cwd=SRCDIR)
+        # wait for server
+        end = time.monotonic() + 3.0
+        while time.monotonic() < end and srv.returncode is None:
+            try:
+                with socket.create_connection((bind, port), timeout=0.1) as s:
+                    pass
+            except OSError:
+                time.sleep(0.01)
+            else:
+                break
+
+        webbrowser.open(url)
+
+        try:
+            srv.wait()
+        except KeyboardInterrupt:
+            pass
+
     def clean(self, all: bool = False):
         """Clean build directory"""
         if all:
@@ -570,19 +640,19 @@ class BuildProfile:
             # Trigger PIC build.
             ports_cmd.append("-sMAIN_MODULE")
             embuilder_cmd.append("--pic")
+
         if self.pthreads:
             # Trigger multi-threaded build.
             ports_cmd.append("-sUSE_PTHREADS")
-            # https://github.com/emscripten-core/emscripten/pull/17729
-            # embuilder_cmd.append("--pthreads")
 
         # Pre-build libbz2, libsqlite3, libz, and some system libs.
         ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"])
-        embuilder_cmd.extend(["build", "bzip2", "sqlite3", "zlib"])
+        # Multi-threaded sqlite3 has different suffix
+        embuilder_cmd.extend(
+            ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"]
+        )
 
-        if not self.pthreads:
+        self._run_cmd(embuilder_cmd, cwd=SRCDIR)
-            # Emscripten <= 3.1.20 has no option to build multi-threaded ports.
-            self._run_cmd(embuilder_cmd, cwd=SRCDIR)
 
         with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir:
             tmppath = pathlib.Path(tmpdir)
@@ -659,7 +729,7 @@ _profiles = [
         dynamic_linking=True,
         pthreads=True,
     ),
-    # wasm64-emscripten (requires unreleased Emscripten >= 3.1.21)
+    # wasm64-emscripten (requires Emscripten >= 3.1.21)
     BuildProfile(
         "wasm64-emscripten-node-debug",
         support_level=SupportLevel.experimental,
@@ -674,8 +744,6 @@ _profiles = [
         "wasi",
         support_level=SupportLevel.supported,
         host=Host.wasm32_wasi,
-        # skip sysconfig test_srcdir
-        testopts="-i '*.test_srcdir' -j2",
     ),
     # no SDK available yet
     # BuildProfile(
@@ -690,10 +758,36 @@ PROFILES = {p.name: p for p in _profiles}
 parser = argparse.ArgumentParser(
     "wasm_build.py",
     description=__doc__,
-    formatter_class=argparse.RawDescriptionHelpFormatter,
+    formatter_class=argparse.RawTextHelpFormatter,
 )
+
 parser.add_argument(
-    "--clean", "-c", help="Clean build directories first", action="store_true"
+    "--clean",
+    "-c",
+    help="Clean build directories first",
+    action="store_true",
+)
+
+parser.add_argument(
+    "--verbose",
+    "-v",
+    help="Verbose logging",
+    action="store_true",
+)
+
+parser.add_argument(
+    "--silent",
+    help="Run configure and make in silent mode",
+    action="store_true",
+)
+
+parser.add_argument(
+    "--testopts",
+    help=(
+        "Additional test options for 'test' and 'hostrunnertest', e.g. "
+        "--testopts='-v test_os'."
+    ),
+    default=None,
 )
 
 # Don't list broken and experimental variants in help
@@ -706,67 +800,104 @@ parser.add_argument(
     choices=platforms_choices,
 )
 
-ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall", "emports"]
+ops = dict(
+    build="auto build (build 'build' Python, emports, configure, compile)",
+    configure="run ./configure",
+    compile="run 'make all'",
+    pythoninfo="run 'make pythoninfo'",
+    test="run 'make buildbottest TESTOPTS=...' (supports parallel tests)",
+    hostrunnertest="run 'make hostrunnertest TESTOPTS=...'",
+    repl="start interactive REPL / webserver + browser session",
+    clean="run 'make clean'",
+    cleanall="remove all build directories",
+    emports="build Emscripten port with embuilder (only Emscripten)",
+)
+ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items())
 parser.add_argument(
-    "op",
+    "ops",
     metavar="OP",
-    help=f"operation: {', '.join(ops)}",
+    help=f"operation (default: build)\n\n{ops_help}",
-    choices=ops,
+    choices=tuple(ops),
-    default="compile",
+    default="build",
-    nargs="?",
+    nargs="*",
 )
 
 
 def main():
     args = parser.parse_args()
+    logging.basicConfig(
+        level=logging.INFO if args.verbose else logging.ERROR,
+        format="%(message)s",
+    )
+
     if args.platform == "cleanall":
         for builder in PROFILES.values():
             builder.clean(all=True)
         parser.exit(0)
 
+    # additional configure and make args
+    cm_args = ("--silent",) if args.silent else ()
+
+    # nargs=* with default quirk
+    if args.ops == "build":
+        args.ops = ["build"]
+
     builder = PROFILES[args.platform]
     try:
         builder.host.platform.check()
     except ConditionError as e:
         parser.error(str(e))
 
+    if args.clean:
+        builder.clean(all=False)
+
     # hack for WASI
     if builder.host.is_wasi and not SETUP_LOCAL.exists():
         SETUP_LOCAL.touch()
 
-    if args.op in {"compile", "pythoninfo", "repl", "test"}:
+    # auto-build
-        # all targets need a build Python
+    if "build" in args.ops:
+        # check and create build Python
         if builder is not BUILD:
+            logger.info("Auto-building 'build' Python.")
+            try:
+                BUILD.host.platform.check()
+            except ConditionError as e:
+                parser.error(str(e))
             if args.clean:
                 BUILD.clean(all=False)
-                BUILD.run_build()
+            BUILD.run_build(*cm_args)
-            elif not BUILD.python_cmd.exists():
+        # build Emscripten ports with embuilder
-                BUILD.run_build()
+        if builder.host.is_emscripten and "emports" not in args.ops:
+            builder.build_emports()
 
-        if args.clean:
+    for op in args.ops:
-            builder.clean(all=False)
+        logger.info("\n*** %s %s", args.platform, op)
-
+        if op == "build":
-        if args.op == "compile":
+            builder.run_build(*cm_args)
-            if builder.host.is_emscripten:
+        elif op == "configure":
-                builder.build_emports()
+            builder.run_configure(*cm_args)
-            builder.run_build(force_configure=True)
+        elif op == "compile":
-        else:
+            builder.run_make("all", *cm_args)
-            if not builder.makefile.exists():
+        elif op == "pythoninfo":
-                builder.run_configure()
+            builder.run_pythoninfo(*cm_args)
-            if args.op == "pythoninfo":
+        elif op == "repl":
-                builder.run_pythoninfo()
+            if builder.is_browser:
-            elif args.op == "repl":
+                builder.run_browser()
+            else:
                 builder.run_py()
-            elif args.op == "test":
+        elif op == "test":
-                builder.run_test()
+            builder.run_test("buildbottest", testopts=args.testopts)
-    elif args.op == "clean":
+        elif op == "hostrunnertest":
-        builder.clean(all=False)
+            builder.run_test("hostrunnertest", testopts=args.testopts)
-    elif args.op == "cleanall":
+        elif op == "clean":
-        builder.clean(all=True)
+            builder.clean(all=False)
-    elif args.op == "emports":
+        elif op == "cleanall":
-        builder.build_emports(force=args.clean)
+            builder.clean(all=True)
-    else:
+        elif op == "emports":
-        raise ValueError(args.op)
+            builder.build_emports(force=args.clean)
+        else:
+            raise ValueError(op)
 
     print(builder.builddir)
     parser.exit(0)