mirror of https://github.com/python/cpython
bpo-40280: Add Tools/wasm with helpers for cross building (GH-29984)
Co-authored-by: Ethan Smith <ethan@ethanhs.me> Co-authored-by: Brett Cannon <brett@python.org>
This commit is contained in:
parent
ae36cd1e79
commit
0339434835
|
@ -830,6 +830,22 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
|
|||
else true; \
|
||||
fi
|
||||
|
||||
# wasm32-emscripten build
|
||||
# wasm assets directory is relative to current build dir, e.g. "./usr/local".
|
||||
# --preload-file turns a relative asset path into an absolute path.
|
||||
WASM_ASSETS_DIR=".$(prefix)"
|
||||
WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py"
|
||||
|
||||
$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
|
||||
pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py
|
||||
$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
|
||||
--builddir . --prefix $(prefix)
|
||||
|
||||
python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB)
|
||||
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \
|
||||
$(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \
|
||||
-s ASSERTIONS=1 --preload-file $(WASM_ASSETS_DIR)
|
||||
|
||||
##########################################################################
|
||||
# Build static libmpdec.a
|
||||
LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@
|
||||
|
@ -938,6 +954,7 @@ Makefile Modules/config.c: Makefile.pre \
|
|||
$(SHELL) $(MAKESETUP) -c $(srcdir)/Modules/config.c.in \
|
||||
-s Modules \
|
||||
Modules/Setup.local \
|
||||
@MODULES_SETUP_STDLIB@ \
|
||||
$(srcdir)/Modules/Setup.bootstrap \
|
||||
$(srcdir)/Modules/Setup
|
||||
@mv config.c Modules
|
||||
|
@ -2379,6 +2396,7 @@ clean-retain-profile: pycremoval
|
|||
-rm -f pybuilddir.txt
|
||||
-rm -f Lib/lib2to3/*Grammar*.pickle
|
||||
-rm -f _bootstrap_python
|
||||
-rm -f python.html python.js python.data
|
||||
-rm -f Programs/_testembed Programs/_freeze_module
|
||||
-rm -f Python/deepfreeze/*.[co]
|
||||
-rm -f Python/frozen_modules/*.h
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
A new directory ``Tools/wasm`` contains WebAssembly-related helpers like ``config.site`` override for wasm32-emscripten, wasm assets generator to bundle the stdlib, and a README.
|
|
@ -0,0 +1,55 @@
|
|||
# Python WebAssembly (WASM) build
|
||||
|
||||
This directory contains configuration and helpers to facilitate cross
|
||||
compilation of CPython to WebAssembly (WASM).
|
||||
|
||||
## wasm32-emscripten build
|
||||
|
||||
Cross compiling to wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/)
|
||||
tool chain and a build Python interpreter.
|
||||
All commands below are relative to a repository checkout.
|
||||
|
||||
### Compile a build Python interpreter
|
||||
|
||||
```shell
|
||||
mkdir -p builddir/build
|
||||
pushd builddir/build
|
||||
../../configure -C
|
||||
make -j$(nproc)
|
||||
popd
|
||||
```
|
||||
|
||||
### Fetch and build additional emscripten ports
|
||||
|
||||
```shell
|
||||
embuilder build zlib
|
||||
```
|
||||
|
||||
### Cross compile to wasm32-emscripten
|
||||
|
||||
```shell
|
||||
mkdir -p builddir/emscripten
|
||||
pushd builddir/emscripten
|
||||
|
||||
CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
|
||||
emconfigure ../../configure -C \
|
||||
--host=wasm32-unknown-emscripten \
|
||||
--build=$(../../config.guess) \
|
||||
--with-build-python=$(pwd)/../build/python
|
||||
|
||||
emmake make -j$(nproc) python.html
|
||||
```
|
||||
|
||||
### Test in browser
|
||||
|
||||
Serve `python.html` with a local webserver and open the file in a browser.
|
||||
|
||||
```shell
|
||||
emrun python.html
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```shell
|
||||
python3 -m http.server
|
||||
```
|
|
@ -0,0 +1,70 @@
|
|||
# config.site override for cross compiling to wasm32-emscripten platform
|
||||
#
|
||||
# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
|
||||
# emconfigure ./configure --host=wasm32-unknown-emscripten --build=...
|
||||
#
|
||||
# Written by Christian Heimes <christian@python.org>
|
||||
# Partly based on pyodide's pyconfig.undefs.h file.
|
||||
#
|
||||
|
||||
# cannot be detected in cross builds
|
||||
ac_cv_buggy_getaddrinfo=no
|
||||
|
||||
# Emscripten has no /dev/pt*
|
||||
ac_cv_file__dev_ptmx=no
|
||||
ac_cv_file__dev_ptc=no
|
||||
|
||||
# dummy readelf, Emscripten build does not need readelf.
|
||||
ac_cv_prog_ac_ct_READELF=true
|
||||
|
||||
# new undefined symbols / unsupported features
|
||||
ac_cv_func_posix_spawn=no
|
||||
ac_cv_func_posix_spawnp=no
|
||||
ac_cv_func_eventfd=no
|
||||
ac_cv_func_memfd_create=no
|
||||
ac_cv_func_prlimit=no
|
||||
|
||||
# unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393
|
||||
ac_cv_func_shutdown=no
|
||||
|
||||
# breaks build, see https://github.com/ethanhs/python-wasm/issues/16
|
||||
ac_cv_lib_bz2_BZ2_bzCompress=no
|
||||
|
||||
# The rest is based on pyodide
|
||||
# https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h
|
||||
|
||||
ac_cv_func_epoll=no
|
||||
ac_cv_func_epoll_create1=no
|
||||
ac_cv_header_linux_vm_sockets_h=no
|
||||
ac_cv_func_socketpair=no
|
||||
ac_cv_func_utimensat=no
|
||||
ac_cv_func_sigaction=no
|
||||
|
||||
# Untested syscalls in emscripten
|
||||
ac_cv_func_openat=no
|
||||
ac_cv_func_mkdirat=no
|
||||
ac_cv_func_fchownat=no
|
||||
ac_cv_func_renameat=no
|
||||
ac_cv_func_linkat=no
|
||||
ac_cv_func_symlinkat=no
|
||||
ac_cv_func_readlinkat=no
|
||||
ac_cv_func_fchmodat=no
|
||||
ac_cv_func_dup3=no
|
||||
|
||||
# Syscalls not implemented in emscripten
|
||||
ac_cv_func_preadv2=no
|
||||
ac_cv_func_preadv=no
|
||||
ac_cv_func_pwritev2=no
|
||||
ac_cv_func_pwritev=no
|
||||
ac_cv_func_pipe2=no
|
||||
ac_cv_func_nice=no
|
||||
|
||||
# Syscalls that resulted in a segfault
|
||||
ac_cv_func_utimensat=no
|
||||
ac_cv_header_sys_ioctl_h=no
|
||||
|
||||
# sockets are supported, but only in non-blocking mode
|
||||
# ac_cv_header_sys_socket_h=no
|
||||
|
||||
# Unsupported functionality
|
||||
#undef HAVE_PTHREAD_H
|
|
@ -0,0 +1,174 @@
|
|||
#!/usr/bin/env python
|
||||
"""Create a WASM asset bundle directory structure.
|
||||
|
||||
The WASM asset bundles are pre-loaded by the final WASM build. The bundle
|
||||
contains:
|
||||
|
||||
- a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip
|
||||
- os.py as marker module {PREFIX}/lib/python3.11/os.py
|
||||
- empty lib-dynload directory, to make sure it is copied into the bundle {PREFIX}/lib/python3.11/lib-dynload/.empty
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import shutil
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
# source directory
|
||||
SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
|
||||
SRCDIR_LIB = SRCDIR / "Lib"
|
||||
|
||||
# sysconfig data relative to build dir.
|
||||
SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py"
|
||||
|
||||
# Library directory relative to $(prefix).
|
||||
WASM_LIB = pathlib.PurePath("lib")
|
||||
WASM_STDLIB_ZIP = (
|
||||
WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
|
||||
)
|
||||
WASM_STDLIB = (
|
||||
WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
|
||||
)
|
||||
WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"
|
||||
|
||||
|
||||
# Don't ship large files / packages that are not particularly useful at
|
||||
# the moment.
|
||||
OMIT_FILES = (
|
||||
# regression tests
|
||||
"test/",
|
||||
# user interfaces: TK, curses
|
||||
"curses/",
|
||||
"idlelib/",
|
||||
"tkinter/",
|
||||
"turtle.py",
|
||||
"turtledemo/",
|
||||
# package management
|
||||
"ensurepip/",
|
||||
"venv/",
|
||||
# build system
|
||||
"distutils/",
|
||||
"lib2to3/",
|
||||
# concurrency
|
||||
"concurrent/",
|
||||
"multiprocessing/",
|
||||
# deprecated
|
||||
"asyncore.py",
|
||||
"asynchat.py",
|
||||
# Synchronous network I/O and protocols are not supported; for example,
|
||||
# socket.create_connection() raises an exception:
|
||||
# "BlockingIOError: [Errno 26] Operation in progress".
|
||||
"cgi.py",
|
||||
"cgitb.py",
|
||||
"email/",
|
||||
"ftplib.py",
|
||||
"http/",
|
||||
"imaplib.py",
|
||||
"nntplib.py",
|
||||
"poplib.py",
|
||||
"smtpd.py",
|
||||
"smtplib.py",
|
||||
"socketserver.py",
|
||||
"telnetlib.py",
|
||||
"urllib/",
|
||||
"wsgiref/",
|
||||
"xmlrpc/",
|
||||
# dbm / gdbm
|
||||
"dbm/",
|
||||
# other platforms
|
||||
"_aix_support.py",
|
||||
"_bootsubprocess.py",
|
||||
"_osx_support.py",
|
||||
# webbrowser
|
||||
"antigravity.py",
|
||||
"webbrowser.py",
|
||||
# ctypes
|
||||
"ctypes/",
|
||||
# Pure Python implementations of C extensions
|
||||
"_pydecimal.py",
|
||||
"_pyio.py",
|
||||
# Misc unused or large files
|
||||
"pydoc_data/",
|
||||
"msilib/",
|
||||
)
|
||||
|
||||
# regression test sub directories
|
||||
OMIT_SUBDIRS = (
|
||||
"ctypes/test/",
|
||||
"tkinter/test/",
|
||||
"unittest/test/",
|
||||
)
|
||||
|
||||
|
||||
OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES}
|
||||
OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS)
|
||||
|
||||
|
||||
def filterfunc(name: str) -> bool:
|
||||
return not name.startswith(OMIT_SUBDIRS_ABSOLUTE)
|
||||
|
||||
|
||||
def create_stdlib_zip(
|
||||
args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0
|
||||
) -> None:
|
||||
sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB))
|
||||
if not sysconfig_data:
|
||||
raise ValueError("No sysconfigdata file found")
|
||||
|
||||
with zipfile.PyZipFile(
|
||||
args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0
|
||||
) as pzf:
|
||||
for entry in sorted(args.srcdir_lib.iterdir()):
|
||||
if entry.name == "__pycache__":
|
||||
continue
|
||||
if entry in OMIT_ABSOLUTE:
|
||||
continue
|
||||
if entry.name.endswith(".py") or entry.is_dir():
|
||||
# writepy() writes .pyc files (bytecode).
|
||||
pzf.writepy(entry, filterfunc=filterfunc)
|
||||
for entry in sysconfig_data:
|
||||
pzf.writepy(entry)
|
||||
|
||||
|
||||
def path(val: str) -> pathlib.Path:
|
||||
return pathlib.Path(val).absolute()
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--builddir",
|
||||
help="absolute build directory",
|
||||
default=pathlib.Path(".").absolute(),
|
||||
type=path,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = parser.parse_args()
|
||||
|
||||
relative_prefix = args.prefix.relative_to(pathlib.Path("/"))
|
||||
args.srcdir = SRCDIR
|
||||
args.srcdir_lib = SRCDIR_LIB
|
||||
args.wasm_root = args.builddir / relative_prefix
|
||||
args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP
|
||||
args.wasm_stdlib = args.wasm_root / WASM_STDLIB
|
||||
args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
|
||||
|
||||
# Empty, unused directory for dynamic libs, but required for site initialization.
|
||||
args.wasm_dynload.mkdir(parents=True, exist_ok=True)
|
||||
marker = args.wasm_dynload / ".empty"
|
||||
marker.touch()
|
||||
# os.py is a marker for finding the correct lib directory.
|
||||
shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
|
||||
# The rest of stdlib that's useful in a WASM context.
|
||||
create_stdlib_zip(args)
|
||||
size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2)
|
||||
parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
|
@ -772,6 +772,7 @@ MODULE_TIME_FALSE
|
|||
MODULE_TIME_TRUE
|
||||
MODULE__IO_FALSE
|
||||
MODULE__IO_TRUE
|
||||
MODULES_SETUP_STDLIB
|
||||
MODULE_BUILDTYPE
|
||||
TEST_MODULES
|
||||
LIBRARY_DEPS
|
||||
|
@ -13298,7 +13299,13 @@ fi
|
|||
|
||||
if test -z "$with_pymalloc"
|
||||
then
|
||||
case $ac_sys_system in #(
|
||||
Emscripten) :
|
||||
with_pymalloc="no" ;; #(
|
||||
*) :
|
||||
with_pymalloc="yes"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
if test "$with_pymalloc" != "no"
|
||||
then
|
||||
|
@ -21165,12 +21172,22 @@ fi
|
|||
|
||||
if test "$enable_test_modules" = no; then
|
||||
TEST_MODULES=no
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
|
||||
$as_echo "yes" >&6; }
|
||||
else
|
||||
case $ac_sys_system in #(
|
||||
Emscripten) :
|
||||
TEST_MODULES=no ;; #(
|
||||
*) :
|
||||
TEST_MODULES=yes
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
if test "x$TEST_MODULES" = xyes; then :
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
|
||||
$as_echo "no" >&6; }
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
|
||||
$as_echo "yes" >&6; }
|
||||
|
||||
fi
|
||||
|
||||
|
||||
|
@ -21189,7 +21206,7 @@ case $ac_sys_system in #(
|
|||
py_stdlib_not_available="_scproxy spwd" ;; #(
|
||||
Emscripten) :
|
||||
|
||||
py_stdlib_not_available="_curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _xxsubinterpreters fcntl grp nis ossaudiodev resource spwd syslog termios"
|
||||
py_stdlib_not_available="_ctypes _curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _tkinter _xxsubinterpreters fcntl grp nis ossaudiodev resource readline spwd syslog termios"
|
||||
;; #(
|
||||
*) :
|
||||
py_stdlib_not_available="_scproxy"
|
||||
|
@ -21205,6 +21222,20 @@ case $host_cpu in #(
|
|||
esac
|
||||
|
||||
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for additional Modules/Setup files" >&5
|
||||
$as_echo_n "checking for additional Modules/Setup files... " >&6; }
|
||||
case $ac_sys_system in #(
|
||||
Emscripten) :
|
||||
MODULES_SETUP_STDLIB=Modules/Setup.stdlib ;; #(
|
||||
*) :
|
||||
MODULES_SETUP_STDLIB=
|
||||
;;
|
||||
esac
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $MODULES_SETUP_STDLIB" >&5
|
||||
$as_echo "$MODULES_SETUP_STDLIB" >&6; }
|
||||
|
||||
|
||||
|
||||
|
||||
MODULE_BLOCK=
|
||||
|
||||
|
@ -25100,7 +25131,7 @@ fi
|
|||
$as_echo "$as_me: creating Makefile" >&6;}
|
||||
$SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
|
||||
-s Modules \
|
||||
Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
|
||||
Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
|
||||
mv config.c Modules
|
||||
|
||||
if test -z "$PKG_CONFIG"; then
|
||||
|
|
31
configure.ac
31
configure.ac
|
@ -3865,7 +3865,11 @@ AC_ARG_WITH(pymalloc,
|
|||
|
||||
if test -z "$with_pymalloc"
|
||||
then
|
||||
with_pymalloc="yes"
|
||||
dnl default to yes except for wasm32-emscripten
|
||||
AS_CASE([$ac_sys_system],
|
||||
[Emscripten], [with_pymalloc="no"],
|
||||
[with_pymalloc="yes"]
|
||||
)
|
||||
fi
|
||||
if test "$with_pymalloc" != "no"
|
||||
then
|
||||
|
@ -6253,11 +6257,15 @@ AC_ARG_ENABLE(test-modules,
|
|||
AS_HELP_STRING([--disable-test-modules], [don't build nor install test modules]))
|
||||
if test "$enable_test_modules" = no; then
|
||||
TEST_MODULES=no
|
||||
AC_MSG_RESULT(yes)
|
||||
else
|
||||
TEST_MODULES=yes
|
||||
AC_MSG_RESULT(no)
|
||||
AS_CASE([$ac_sys_system],
|
||||
[Emscripten], [TEST_MODULES=no],
|
||||
[TEST_MODULES=yes]
|
||||
)
|
||||
fi
|
||||
AS_VAR_IF([TEST_MODULES], [yes],
|
||||
[AC_MSG_RESULT(no)], [AC_MSG_RESULT(yes)]
|
||||
)
|
||||
AC_SUBST(TEST_MODULES)
|
||||
|
||||
dnl Modules that are not available on some platforms
|
||||
|
@ -6272,6 +6280,7 @@ AS_CASE([$ac_sys_system],
|
|||
[FreeBSD*], [py_stdlib_not_available="_scproxy spwd"],
|
||||
[Emscripten], [
|
||||
py_stdlib_not_available="m4_normalize([
|
||||
_ctypes
|
||||
_curses
|
||||
_curses_panel
|
||||
_dbm
|
||||
|
@ -6280,12 +6289,14 @@ AS_CASE([$ac_sys_system],
|
|||
_posixshmem
|
||||
_posixsubprocess
|
||||
_scproxy
|
||||
_tkinter
|
||||
_xxsubinterpreters
|
||||
fcntl
|
||||
grp
|
||||
nis
|
||||
ossaudiodev
|
||||
resource
|
||||
readline
|
||||
spwd
|
||||
syslog
|
||||
termios
|
||||
|
@ -6301,6 +6312,16 @@ AS_CASE([$host_cpu],
|
|||
)
|
||||
AC_SUBST([MODULE_BUILDTYPE])
|
||||
|
||||
dnl Use Modules/Setup.stdlib as additional provider?
|
||||
AC_MSG_CHECKING([for additional Modules/Setup files])
|
||||
AS_CASE([$ac_sys_system],
|
||||
[Emscripten], [MODULES_SETUP_STDLIB=Modules/Setup.stdlib],
|
||||
[MODULES_SETUP_STDLIB=]
|
||||
)
|
||||
AC_MSG_RESULT([$MODULES_SETUP_STDLIB])
|
||||
AC_SUBST([MODULES_SETUP_STDLIB])
|
||||
|
||||
|
||||
dnl _MODULE_BLOCK_ADD([VAR], [VALUE])
|
||||
dnl internal: adds $1=quote($2) to MODULE_BLOCK
|
||||
AC_DEFUN([_MODULE_BLOCK_ADD], [AS_VAR_APPEND([MODULE_BLOCK], ["$1=_AS_QUOTE([$2])$as_nl"])])
|
||||
|
@ -6515,7 +6536,7 @@ fi
|
|||
AC_MSG_NOTICE([creating Makefile])
|
||||
$SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
|
||||
-s Modules \
|
||||
Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
|
||||
Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
|
||||
mv config.c Modules
|
||||
|
||||
if test -z "$PKG_CONFIG"; then
|
||||
|
|
Loading…
Reference in New Issue