gh-127111: Emscripten Make web example work again (#127113)

Moves the Emscripten web example into a standalone folder, and updates 
Makefile targets to build the web example. Instructions for usage have
also been added.
This commit is contained in:
Hood Chatham 2024-12-03 00:30:24 +01:00 committed by GitHub
parent edefb8678a
commit bfb0788bfc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 175 additions and 92 deletions

View File

@ -269,10 +269,6 @@ SRCDIRS= @SRCDIRS@
# Other subdirectories
SUBDIRSTOO= Include Lib Misc
# assets for Emscripten browser builds
WASM_ASSETS_DIR=.$(prefix)
WASM_STDLIB=$(WASM_ASSETS_DIR)/lib/python$(VERSION)/os.py
# Files and directories to be distributed
CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in
DISTFILES= README.rst ChangeLog $(CONFIGFILES)
@ -737,6 +733,9 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh
build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \
python-config checksharedmods
.PHONY: build_emscripten
build_emscripten: build_wasm web_example
# Check that the source is clean when building out of source.
.PHONY: check-clean-src
check-clean-src:
@ -1016,23 +1015,38 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
else true; \
fi
# wasm32-emscripten browser build
# wasm assets directory is relative to current build dir, e.g. "./usr/local".
# --preload-file turns a relative asset path into an absolute path.
# wasm32-emscripten browser web example
.PHONY: wasm_stdlib
wasm_stdlib: $(WASM_STDLIB)
$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
$(srcdir)/Tools/wasm/wasm_assets.py \
WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/
web_example/python.html: $(WEBEX_DIR)/python.html
@mkdir -p web_example
@cp $< $@
web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs
@mkdir -p web_example
@cp $< $@
web_example/server.py: $(WEBEX_DIR)/server.py
@mkdir -p web_example
@cp $< $@
WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip
$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
$(WEBEX_DIR)/wasm_assets.py \
Makefile pybuilddir.txt Modules/Setup.local
$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
--buildroot . --prefix $(prefix)
$(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \
--buildroot . --prefix $(prefix) -o $@
python.html: $(srcdir)/Tools/wasm/python.html python.worker.js
@cp $(srcdir)/Tools/wasm/python.html $@
web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON)
@if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \
echo "Can only build web_example when target is Emscripten" ;\
exit 1 ;\
fi
cp python.mjs web_example/python.mjs
cp python.wasm web_example/python.wasm
python.worker.js: $(srcdir)/Tools/wasm/python.worker.js
@cp $(srcdir)/Tools/wasm/python.worker.js $@
.PHONY: web_example
web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB)
############################################################################
# Header files
@ -3053,8 +3067,7 @@ clean-retain-profile: pycremoval
find build -name '*.py[co]' -exec rm -f {} ';' || true
-rm -f pybuilddir.txt
-rm -f _bootstrap_python
-rm -f python.html python*.js python.data python*.symbols python*.map
-rm -f $(WASM_STDLIB)
-rm -rf web_example python.mjs python.wasm python*.symbols python*.map
-rm -f Programs/_testembed Programs/_freeze_module
-rm -rf Python/deepfreeze
-rm -f Python/frozen_modules/*.h

View File

@ -0,0 +1,2 @@
Updated the Emscripten web example to use ES6 modules and be built into a
distinct ``web_example`` subfolder.

View File

@ -23,9 +23,9 @@ https://github.com/psf/webassembly for more information.
To cross compile to the ``wasm32-emscripten`` platform you need
[the Emscripten compiler toolchain](https://emscripten.org/),
a Python interpreter, and an installation of Node version 18 or newer. Emscripten
version 3.1.42 or newer is recommended. All commands below are relative to a checkout
of the Python repository.
a Python interpreter, and an installation of Node version 18 or newer.
Emscripten version 3.1.73 or newer is recommended. All commands below are
relative to a checkout of the Python repository.
#### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html)
@ -50,7 +50,7 @@ sourced. Otherwise the source script removes the environment variable.
export EM_COMPILER_WRAPPER=ccache
```
### Compile and build Python interpreter
#### Compile and build Python interpreter
You can use `python Tools/wasm/emscripten` to compile and build targetting
Emscripten. You can do everything at once with:
@ -70,6 +70,88 @@ instance, to do a debug build, you can use:
python Tools/wasm/emscripten build --with-py-debug
```
### Running from node
If you want to run the normal Python CLI, you can use `python.sh`. It takes the
same options as the normal Python CLI entrypoint, though the REPL does not
function and will crash.
`python.sh` invokes `node_entry.mjs` which imports the Emscripten module for the
Python process and starts it up with the appropriate settings. If you wish to
make a node application that "embeds" the interpreter instead of acting like the
CLI you will need to write your own alternative to `node_entry.mjs`.
### The Web Example
When building for Emscripten, the web example will be built automatically. It is
in the ``web_example`` directory. To run the web example, ``cd`` into the
``web_example`` directory, then run ``python server.py``. This will start a web
server; you can then visit ``http://localhost:8000/python.html`` in a browser to
see a simple REPL example.
The web example relies on a bug fix in Emscripten version 3.1.73 so if you build
with earlier versions of Emscripten it may not work. The web example uses
``SharedArrayBuffer``. For security reasons browsers only provide
``SharedArrayBuffer`` in secure environments with cross-origin isolation. The
webserver must send cross-origin headers and correct MIME types for the
JavaScript and WebAssembly files. Otherwise the terminal will fail to load with
an error message like ``ReferenceError: SharedArrayBuffer is not defined``. See
more information here:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements
Note that ``SharedArrayBuffer`` is _not required_ to use Python itself, only the
web example. If cross-origin isolation is not appropriate for your use case you
may make your own application embedding `python.mjs` which does not use
``SharedArrayBuffer`` and serve it without the cross-origin isolation headers.
### Embedding Python in a custom JavaScript application
You can look at `python.worker.mjs` and `node_entry.mjs` for inspiration. At a
minimum you must import ``createEmscriptenModule`` and you need to call
``createEmscriptenModule`` with an appropriate settings object. This settings
object will need a prerun hook that installs the Python standard library into
the Emscripten file system.
#### NodeJs
In Node, you can use the NodeFS to mount the standard library in your native
file system into the Emscripten file system:
```js
import createEmscriptenModule from "./python.mjs";
await createEmscriptenModule({
preRun(Module) {
Module.FS.mount(
Module.FS.filesystems.NODEFS,
{ root: "/path/to/python/stdlib" },
"/lib/",
);
},
});
```
#### Browser
In the browser, the simplest approach is to put the standard library in a zip
file it and install it. With Python 3.14 this could look like:
```js
import createEmscriptenModule from "./python.mjs";
await createEmscriptenModule({
async preRun(Module) {
Module.FS.mkdirTree("/lib/python3.14/lib-dynload/");
Module.addRunDependency("install-stdlib");
const resp = await fetch("python3.14.zip");
const stdlibBuffer = await resp.arrayBuffer();
Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), {
canOwn: true,
});
Module.removeRunDependency("install-stdlib");
},
});
```
### Limitations and issues
#### Network stack
@ -151,38 +233,6 @@ python Tools/wasm/emscripten build --with-py-debug
- Test modules are disabled by default. Use ``--enable-test-modules`` build
test modules like ``_testcapi``.
### wasm32-emscripten in node
Node builds use ``NODERAWFS``.
- Node RawFS allows direct access to the host file system without need to
perform ``FS.mount()`` call.
### Hosting Python WASM builds
The simple REPL terminal uses SharedArrayBuffer. For security reasons
browsers only provide the feature in secure environments with cross-origin
isolation. The webserver must send cross-origin headers and correct MIME types
for the JavaScript and WASM files. Otherwise the terminal will fail to load
with an error message like ``Browsers disable shared array buffer``.
#### Apache HTTP .htaccess
Place a ``.htaccess`` file in the same directory as ``python.wasm``.
```
# .htaccess
Header set Cross-Origin-Opener-Policy same-origin
Header set Cross-Origin-Embedder-Policy require-corp
AddType application/javascript js
AddType application/wasm wasm
<IfModule mod_deflate.c>
AddOutputFilterByType DEFLATE text/html application/javascript application/wasm
</IfModule>
```
## WASI (wasm32-wasi)
See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi).

View File

@ -47,7 +47,7 @@ class WorkerManager {
async initialiseWorker() {
if (!this.worker) {
this.worker = new Worker(this.workerURL)
this.worker = new Worker(this.workerURL, {type: "module"})
this.worker.addEventListener('message', this.handleMessageFromWorker)
}
}
@ -347,7 +347,7 @@ window.onload = () => {
programRunning(false)
}
const pythonWorkerManager = new WorkerManager('./python.worker.js', stdio, readyCallback, finishedCallback)
const pythonWorkerManager = new WorkerManager('./python.worker.mjs', stdio, readyCallback, finishedCallback)
}
</script>
</head>

View File

@ -1,3 +1,5 @@
import createEmscriptenModule from "./python.mjs";
class StdinBuffer {
constructor() {
this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT)
@ -59,24 +61,40 @@ const stderr = (charCode) => {
const stdinBuffer = new StdinBuffer()
var Module = {
const emscriptenSettings = {
noInitialRun: true,
stdin: stdinBuffer.stdin,
stdout: stdout,
stderr: stderr,
onRuntimeInitialized: () => {
postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab})
},
async preRun(Module) {
const versionHex = Module.HEAPU32[Module._Py_Version/4].toString(16);
const versionTuple = versionHex.padStart(8, "0").match(/.{1,2}/g).map((x) => parseInt(x, 16));
const [major, minor, ..._] = versionTuple;
// Prevent complaints about not finding exec-prefix by making a lib-dynload directory
Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`);
Module.addRunDependency("install-stdlib");
const resp = await fetch(`python${major}.${minor}.zip`);
const stdlibBuffer = await resp.arrayBuffer();
Module.FS.writeFile(`/lib/python${major}${minor}.zip`, new Uint8Array(stdlibBuffer), { canOwn: true });
Module.removeRunDependency("install-stdlib");
}
}
onmessage = (event) => {
const modulePromise = createEmscriptenModule(emscriptenSettings);
onmessage = async (event) => {
if (event.data.type === 'run') {
const Module = await modulePromise;
if (event.data.files) {
for (const [filename, contents] of Object.entries(event.data.files)) {
Module.FS.writeFile(filename, contents)
}
}
const ret = callMain(event.data.args)
const ret = Module.callMain(event.data.args);
postMessage({
type: 'finished',
returnCode: ret
@ -84,4 +102,3 @@ onmessage = (event) => {
}
}
importScripts('python.js')

View File

@ -14,13 +14,6 @@ parser.add_argument(
class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler):
extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy()
extensions_map.update(
{
".wasm": "application/wasm",
}
)
def end_headers(self) -> None:
self.send_my_headers()
super().end_headers()
@ -42,5 +35,6 @@ def main() -> None:
bind=args.bind,
)
if __name__ == "__main__":
main()

View File

@ -19,7 +19,7 @@ import zipfile
from typing import Dict
# source directory
SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
SRCDIR = pathlib.Path(__file__).parents[4].absolute()
SRCDIR_LIB = SRCDIR / "Lib"
@ -28,9 +28,7 @@ WASM_LIB = pathlib.PurePath("lib")
WASM_STDLIB_ZIP = (
WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
)
WASM_STDLIB = (
WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
)
WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"
@ -114,9 +112,7 @@ def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path:
assert isinstance(args.builddir, pathlib.Path)
data_name: str = sysconfig._get_sysconfigdata_name() # type: ignore[attr-defined]
if not data_name.startswith(SYSCONFIG_NAMES):
raise ValueError(
f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES
)
raise ValueError(f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES)
filename = data_name + ".py"
return args.builddir / filename
@ -131,7 +127,7 @@ def create_stdlib_zip(
return pathname not in args.omit_files_absolute
with zipfile.PyZipFile(
args.wasm_stdlib_zip,
args.output,
mode="w",
compression=args.compression,
optimize=optimize,
@ -195,6 +191,12 @@ parser.add_argument(
default=pathlib.Path("/usr/local"),
type=path,
)
parser.add_argument(
"-o",
"--output",
help="output file",
type=path,
)
def main() -> None:
@ -204,7 +206,6 @@ def main() -> None:
args.srcdir = SRCDIR
args.srcdir_lib = SRCDIR_LIB
args.wasm_root = args.buildroot / relative_prefix
args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP
args.wasm_stdlib = args.wasm_root / WASM_STDLIB
args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
@ -234,12 +235,10 @@ def main() -> None:
args.wasm_dynload.mkdir(parents=True, exist_ok=True)
marker = args.wasm_dynload / ".empty"
marker.touch()
# os.py is a marker for finding the correct lib directory.
shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
# The rest of stdlib that's useful in a WASM context.
create_stdlib_zip(args)
size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2)
parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
size = round(args.output.stat().st_size / 1024**2, 2)
parser.exit(0, f"Created {args.output} ({size} MiB)\n")
if __name__ == "__main__":

14
configure generated vendored
View File

@ -8333,8 +8333,12 @@ fi
fi
elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then
DEF_MAKE_ALL_RULE="build_wasm"
elif test "$ac_sys_system" = "Emscripten"; then
DEF_MAKE_ALL_RULE="build_emscripten"
REQUIRE_PGO="no"
DEF_MAKE_RULE="all"
elif test "$ac_sys_system" = "WASI"; then
DEF_MAKE_ALL_RULE="build_wasm"
REQUIRE_PGO="no"
DEF_MAKE_RULE="all"
else
@ -9425,12 +9429,13 @@ else $as_nop
wasm_debug=no
fi
as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"
as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"
as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT"
as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"
as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS"
as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,callMain"
as_fn_append LDFLAGS_NODIST " -sEXPORTED_FUNCTIONS=_main,_Py_Version"
if test "x$enable_wasm_dynamic_linking" = xyes
then :
@ -9447,7 +9452,6 @@ then :
as_fn_append LINKFORSHARED " -sPROXY_TO_PTHREAD"
fi
as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH"
as_fn_append LDFLAGS_NODIST " -sEXIT_RUNTIME"
WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map"

View File

@ -1854,9 +1854,13 @@ if test "$Py_OPT" = 'true' ; then
LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition"
], [], [-Werror])
])
elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then
dnl Emscripten does not support shared extensions yet. Build
dnl "python.[js,wasm]", "pybuilddir.txt", and "platform" files.
elif test "$ac_sys_system" = "Emscripten"; then
dnl Build "python.[js,wasm]", "pybuilddir.txt", and "platform" files.
DEF_MAKE_ALL_RULE="build_emscripten"
REQUIRE_PGO="no"
DEF_MAKE_RULE="all"
elif test "$ac_sys_system" = "WASI"; then
dnl Build "python.wasm", "pybuilddir.txt", and "platform" files.
DEF_MAKE_ALL_RULE="build_wasm"
REQUIRE_PGO="no"
DEF_MAKE_RULE="all"
@ -2321,14 +2325,15 @@ AS_CASE([$ac_sys_system],
AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no])
dnl Start with 20 MB and allow to grow
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"])
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"])
dnl map int64_t and uint64_t to JS bigint
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sWASM_BIGINT"])
dnl Include file system support
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"])
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS"])
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain"])
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"])
AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"])
@ -2339,7 +2344,6 @@ AS_CASE([$ac_sys_system],
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"])
AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"])
])
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH"])
dnl not completely sure whether or not we want -sEXIT_RUNTIME, keeping it for now.
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXIT_RUNTIME"])
WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map"