bpo-40280: Add Tools/wasm with helpers for cross building (GH-29984)

Co-authored-by: Ethan Smith <ethan@ethanhs.me>
Co-authored-by: Brett Cannon <brett@python.org>
This commit is contained in:
Christian Heimes 2021-12-18 16:54:02 +02:00 committed by GitHub
parent ae36cd1e79
commit 0339434835
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 380 additions and 10 deletions

View File

@ -830,6 +830,22 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
else true; \
fi
# wasm32-emscripten build
# wasm assets directory is relative to current build dir, e.g. "./usr/local".
# --preload-file turns a relative asset path into an absolute path.
WASM_ASSETS_DIR=".$(prefix)"
WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py"
$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py
$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
--builddir . --prefix $(prefix)
python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB)
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \
$(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \
-s ASSERTIONS=1 --preload-file $(WASM_ASSETS_DIR)
##########################################################################
# Build static libmpdec.a
LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@
@ -938,6 +954,7 @@ Makefile Modules/config.c: Makefile.pre \
$(SHELL) $(MAKESETUP) -c $(srcdir)/Modules/config.c.in \
-s Modules \
Modules/Setup.local \
@MODULES_SETUP_STDLIB@ \
$(srcdir)/Modules/Setup.bootstrap \
$(srcdir)/Modules/Setup
@mv config.c Modules
@ -2379,6 +2396,7 @@ clean-retain-profile: pycremoval
-rm -f pybuilddir.txt
-rm -f Lib/lib2to3/*Grammar*.pickle
-rm -f _bootstrap_python
-rm -f python.html python.js python.data
-rm -f Programs/_testembed Programs/_freeze_module
-rm -f Python/deepfreeze/*.[co]
-rm -f Python/frozen_modules/*.h

View File

@ -0,0 +1 @@
A new directory ``Tools/wasm`` contains WebAssembly-related helpers like ``config.site`` override for wasm32-emscripten, wasm assets generator to bundle the stdlib, and a README.

55
Tools/wasm/README.md Normal file
View File

@ -0,0 +1,55 @@
# Python WebAssembly (WASM) build
This directory contains configuration and helpers to facilitate cross
compilation of CPython to WebAssembly (WASM).
## wasm32-emscripten build
Cross compiling to wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/)
tool chain and a build Python interpreter.
All commands below are relative to a repository checkout.
### Compile a build Python interpreter
```shell
mkdir -p builddir/build
pushd builddir/build
../../configure -C
make -j$(nproc)
popd
```
### Fetch and build additional emscripten ports
```shell
embuilder build zlib
```
### Cross compile to wasm32-emscripten
```shell
mkdir -p builddir/emscripten
pushd builddir/emscripten
CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
emconfigure ../../configure -C \
--host=wasm32-unknown-emscripten \
--build=$(../../config.guess) \
--with-build-python=$(pwd)/../build/python
emmake make -j$(nproc) python.html
```
### Test in browser
Serve `python.html` with a local webserver and open the file in a browser.
```shell
emrun python.html
```
or
```shell
python3 -m http.server
```

View File

@ -0,0 +1,70 @@
# config.site override for cross compiling to wasm32-emscripten platform
#
# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
# emconfigure ./configure --host=wasm32-unknown-emscripten --build=...
#
# Written by Christian Heimes <christian@python.org>
# Partly based on pyodide's pyconfig.undefs.h file.
#
# cannot be detected in cross builds
ac_cv_buggy_getaddrinfo=no
# Emscripten has no /dev/pt*
ac_cv_file__dev_ptmx=no
ac_cv_file__dev_ptc=no
# dummy readelf, Emscripten build does not need readelf.
ac_cv_prog_ac_ct_READELF=true
# new undefined symbols / unsupported features
ac_cv_func_posix_spawn=no
ac_cv_func_posix_spawnp=no
ac_cv_func_eventfd=no
ac_cv_func_memfd_create=no
ac_cv_func_prlimit=no
# unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393
ac_cv_func_shutdown=no
# breaks build, see https://github.com/ethanhs/python-wasm/issues/16
ac_cv_lib_bz2_BZ2_bzCompress=no
# The rest is based on pyodide
# https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h
ac_cv_func_epoll=no
ac_cv_func_epoll_create1=no
ac_cv_header_linux_vm_sockets_h=no
ac_cv_func_socketpair=no
ac_cv_func_utimensat=no
ac_cv_func_sigaction=no
# Untested syscalls in emscripten
ac_cv_func_openat=no
ac_cv_func_mkdirat=no
ac_cv_func_fchownat=no
ac_cv_func_renameat=no
ac_cv_func_linkat=no
ac_cv_func_symlinkat=no
ac_cv_func_readlinkat=no
ac_cv_func_fchmodat=no
ac_cv_func_dup3=no
# Syscalls not implemented in emscripten
ac_cv_func_preadv2=no
ac_cv_func_preadv=no
ac_cv_func_pwritev2=no
ac_cv_func_pwritev=no
ac_cv_func_pipe2=no
ac_cv_func_nice=no
# Syscalls that resulted in a segfault
ac_cv_func_utimensat=no
ac_cv_header_sys_ioctl_h=no
# sockets are supported, but only in non-blocking mode
# ac_cv_header_sys_socket_h=no
# Unsupported functionality
#undef HAVE_PTHREAD_H

174
Tools/wasm/wasm_assets.py Executable file
View File

@ -0,0 +1,174 @@
#!/usr/bin/env python
"""Create a WASM asset bundle directory structure.
The WASM asset bundles are pre-loaded by the final WASM build. The bundle
contains:
- a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip
- os.py as marker module {PREFIX}/lib/python3.11/os.py
- empty lib-dynload directory, to make sure it is copied into the bundle {PREFIX}/lib/python3.11/lib-dynload/.empty
"""
import argparse
import pathlib
import shutil
import sys
import zipfile
# source directory
SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
SRCDIR_LIB = SRCDIR / "Lib"
# sysconfig data relative to build dir.
SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py"
# Library directory relative to $(prefix).
WASM_LIB = pathlib.PurePath("lib")
WASM_STDLIB_ZIP = (
WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
)
WASM_STDLIB = (
WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
)
WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"
# Don't ship large files / packages that are not particularly useful at
# the moment.
OMIT_FILES = (
# regression tests
"test/",
# user interfaces: TK, curses
"curses/",
"idlelib/",
"tkinter/",
"turtle.py",
"turtledemo/",
# package management
"ensurepip/",
"venv/",
# build system
"distutils/",
"lib2to3/",
# concurrency
"concurrent/",
"multiprocessing/",
# deprecated
"asyncore.py",
"asynchat.py",
# Synchronous network I/O and protocols are not supported; for example,
# socket.create_connection() raises an exception:
# "BlockingIOError: [Errno 26] Operation in progress".
"cgi.py",
"cgitb.py",
"email/",
"ftplib.py",
"http/",
"imaplib.py",
"nntplib.py",
"poplib.py",
"smtpd.py",
"smtplib.py",
"socketserver.py",
"telnetlib.py",
"urllib/",
"wsgiref/",
"xmlrpc/",
# dbm / gdbm
"dbm/",
# other platforms
"_aix_support.py",
"_bootsubprocess.py",
"_osx_support.py",
# webbrowser
"antigravity.py",
"webbrowser.py",
# ctypes
"ctypes/",
# Pure Python implementations of C extensions
"_pydecimal.py",
"_pyio.py",
# Misc unused or large files
"pydoc_data/",
"msilib/",
)
# regression test sub directories
OMIT_SUBDIRS = (
"ctypes/test/",
"tkinter/test/",
"unittest/test/",
)
OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES}
OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS)
def filterfunc(name: str) -> bool:
return not name.startswith(OMIT_SUBDIRS_ABSOLUTE)
def create_stdlib_zip(
args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0
) -> None:
sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB))
if not sysconfig_data:
raise ValueError("No sysconfigdata file found")
with zipfile.PyZipFile(
args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0
) as pzf:
for entry in sorted(args.srcdir_lib.iterdir()):
if entry.name == "__pycache__":
continue
if entry in OMIT_ABSOLUTE:
continue
if entry.name.endswith(".py") or entry.is_dir():
# writepy() writes .pyc files (bytecode).
pzf.writepy(entry, filterfunc=filterfunc)
for entry in sysconfig_data:
pzf.writepy(entry)
def path(val: str) -> pathlib.Path:
return pathlib.Path(val).absolute()
parser = argparse.ArgumentParser()
parser.add_argument(
"--builddir",
help="absolute build directory",
default=pathlib.Path(".").absolute(),
type=path,
)
parser.add_argument(
"--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path
)
def main():
args = parser.parse_args()
relative_prefix = args.prefix.relative_to(pathlib.Path("/"))
args.srcdir = SRCDIR
args.srcdir_lib = SRCDIR_LIB
args.wasm_root = args.builddir / relative_prefix
args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP
args.wasm_stdlib = args.wasm_root / WASM_STDLIB
args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
# Empty, unused directory for dynamic libs, but required for site initialization.
args.wasm_dynload.mkdir(parents=True, exist_ok=True)
marker = args.wasm_dynload / ".empty"
marker.touch()
# os.py is a marker for finding the correct lib directory.
shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
# The rest of stdlib that's useful in a WASM context.
create_stdlib_zip(args)
size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2)
parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
if __name__ == "__main__":
main()

41
configure vendored
View File

@ -772,6 +772,7 @@ MODULE_TIME_FALSE
MODULE_TIME_TRUE
MODULE__IO_FALSE
MODULE__IO_TRUE
MODULES_SETUP_STDLIB
MODULE_BUILDTYPE
TEST_MODULES
LIBRARY_DEPS
@ -13298,7 +13299,13 @@ fi
if test -z "$with_pymalloc"
then
case $ac_sys_system in #(
Emscripten) :
with_pymalloc="no" ;; #(
*) :
with_pymalloc="yes"
;;
esac
fi
if test "$with_pymalloc" != "no"
then
@ -21165,12 +21172,22 @@ fi
if test "$enable_test_modules" = no; then
TEST_MODULES=no
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
else
case $ac_sys_system in #(
Emscripten) :
TEST_MODULES=no ;; #(
*) :
TEST_MODULES=yes
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
;;
esac
fi
if test "x$TEST_MODULES" = xyes; then :
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
fi
@ -21189,7 +21206,7 @@ case $ac_sys_system in #(
py_stdlib_not_available="_scproxy spwd" ;; #(
Emscripten) :
py_stdlib_not_available="_curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _xxsubinterpreters fcntl grp nis ossaudiodev resource spwd syslog termios"
py_stdlib_not_available="_ctypes _curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _tkinter _xxsubinterpreters fcntl grp nis ossaudiodev resource readline spwd syslog termios"
;; #(
*) :
py_stdlib_not_available="_scproxy"
@ -21205,6 +21222,20 @@ case $host_cpu in #(
esac
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for additional Modules/Setup files" >&5
$as_echo_n "checking for additional Modules/Setup files... " >&6; }
case $ac_sys_system in #(
Emscripten) :
MODULES_SETUP_STDLIB=Modules/Setup.stdlib ;; #(
*) :
MODULES_SETUP_STDLIB=
;;
esac
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $MODULES_SETUP_STDLIB" >&5
$as_echo "$MODULES_SETUP_STDLIB" >&6; }
MODULE_BLOCK=
@ -25100,7 +25131,7 @@ fi
$as_echo "$as_me: creating Makefile" >&6;}
$SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
-s Modules \
Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
mv config.c Modules
if test -z "$PKG_CONFIG"; then

View File

@ -3865,7 +3865,11 @@ AC_ARG_WITH(pymalloc,
if test -z "$with_pymalloc"
then
with_pymalloc="yes"
dnl default to yes except for wasm32-emscripten
AS_CASE([$ac_sys_system],
[Emscripten], [with_pymalloc="no"],
[with_pymalloc="yes"]
)
fi
if test "$with_pymalloc" != "no"
then
@ -6253,11 +6257,15 @@ AC_ARG_ENABLE(test-modules,
AS_HELP_STRING([--disable-test-modules], [don't build nor install test modules]))
if test "$enable_test_modules" = no; then
TEST_MODULES=no
AC_MSG_RESULT(yes)
else
TEST_MODULES=yes
AC_MSG_RESULT(no)
AS_CASE([$ac_sys_system],
[Emscripten], [TEST_MODULES=no],
[TEST_MODULES=yes]
)
fi
AS_VAR_IF([TEST_MODULES], [yes],
[AC_MSG_RESULT(no)], [AC_MSG_RESULT(yes)]
)
AC_SUBST(TEST_MODULES)
dnl Modules that are not available on some platforms
@ -6272,6 +6280,7 @@ AS_CASE([$ac_sys_system],
[FreeBSD*], [py_stdlib_not_available="_scproxy spwd"],
[Emscripten], [
py_stdlib_not_available="m4_normalize([
_ctypes
_curses
_curses_panel
_dbm
@ -6280,12 +6289,14 @@ AS_CASE([$ac_sys_system],
_posixshmem
_posixsubprocess
_scproxy
_tkinter
_xxsubinterpreters
fcntl
grp
nis
ossaudiodev
resource
readline
spwd
syslog
termios
@ -6301,6 +6312,16 @@ AS_CASE([$host_cpu],
)
AC_SUBST([MODULE_BUILDTYPE])
dnl Use Modules/Setup.stdlib as additional provider?
AC_MSG_CHECKING([for additional Modules/Setup files])
AS_CASE([$ac_sys_system],
[Emscripten], [MODULES_SETUP_STDLIB=Modules/Setup.stdlib],
[MODULES_SETUP_STDLIB=]
)
AC_MSG_RESULT([$MODULES_SETUP_STDLIB])
AC_SUBST([MODULES_SETUP_STDLIB])
dnl _MODULE_BLOCK_ADD([VAR], [VALUE])
dnl internal: adds $1=quote($2) to MODULE_BLOCK
AC_DEFUN([_MODULE_BLOCK_ADD], [AS_VAR_APPEND([MODULE_BLOCK], ["$1=_AS_QUOTE([$2])$as_nl"])])
@ -6515,7 +6536,7 @@ fi
AC_MSG_NOTICE([creating Makefile])
$SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
-s Modules \
Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
mv config.c Modules
if test -z "$PKG_CONFIG"; then