bpo-40280: Add --enable-wasm-dynamic-linking (GH-32253)

This commit is contained in:
Christian Heimes 2022-04-04 20:31:31 +03:00 committed by GitHub
parent 48269ea9fd
commit c9844cb8aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 194 additions and 49 deletions

View File

@ -52,11 +52,13 @@ General Options
Set the Python executable suffix to *SUFFIX*.
The default suffix is ``.exe`` on Windows and macOS (``python.exe``
executable), ``.wasm`` on Emscripten (``python.wasm`` executable), and
an empty string on other platforms (``python`` executable).
executable), ``.js`` on Emscripten node, ``.html`` on Emscripten browser,
``.wasm`` on WASI, and an empty string on other platforms (``python``
executable).
.. versionchanged:: 3.11
The default suffix on Emscripten platform is ``.wasm``.
The default suffix on WASM platform is one of ``.js``, ``.html``
or ``.wasm``.
.. cmdoption:: --with-tzpath=<list of absolute paths separated by pathsep>
@ -141,6 +143,27 @@ General Options
.. versionadded:: 3.11
WebAssemby Options
------------------
.. cmdoption:: --with-emscripten-target=[browser|node]
Set build flavor for ``wasm32-emscripten``.
* ``browser`` (default): preload minimal stdlib, default MEMFS.
* ``node``: NODERAWFS and pthread support.
.. versionadded:: 3.11
.. cmdoption:: --enable-wasm-dynamic-linking
Turn on dynamic linking support for WASM.
Dynamic linking enables ``dlopen``. File size of the executable
increases due to limited dead code elimination and additional features.
.. versionadded:: 3.11
Install Options
---------------

View File

@ -0,0 +1,2 @@
Add configure option :option:`--enable-wasm-dynamic-linking` to enable
``dlopen`` and MAIN_MODULE / SIDE_MODULE on ``wasm32-emscripten``.

View File

@ -81,24 +81,31 @@ node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscri
## wasm32-emscripten limitations and issues
- Heap and stack are limited.
- Most stdlib modules with a dependency on external libraries are missing:
``ctypes``, ``readline``, ``sqlite3``, ``ssl``, and more.
- Shared extension modules are not implemented yet. All extension modules
are statically linked into the main binary.
The experimental configure option ``--enable-wasm-dynamic-linking`` enables
dynamic extensions.
- Processes are not supported. System calls like fork, popen, and subprocess
fail with ``ENOSYS`` or ``ENOSUP``.
- Only ``AF_INET`` and ``AF_INET6`` with ``SOCK_STREAM`` (TCP) or
``SOCK_DGRAM`` (UDP) are available. ``AF_UNIX`` is not supported.
- ``socketpair`` does not work.
- Blocking sockets are not available and non-blocking sockets don't work
correctly, e.g. ``socket.accept`` crashes the runtime. ``gethostbyname``
does not resolve to a real IP address. IPv6 is not available.
- The ``select`` module is limited. ``select.select()`` crashes the runtime
due to lack of exectfd support.
- The ``*at`` variants of functions (e.g. ``openat``) are not available.
The ``dir_fd`` argument of *os* module functions can't be used.
- Signal support is limited. ``signal.alarm``, ``itimer``, ``sigaction``
are not available or do not work correctly. ``SIGTERM`` exits the runtime.
- Most user, group, and permission related function and modules are not
supported or don't work as expected, e.g.``pwd`` module, ``grp`` module,
``os.setgroups``, ``os.chown``, and so on.
``os.setgroups``, ``os.chown``, and so on. ``lchown`` and `lchmod`` are
not available.
- ``umask`` is a no-op.
- hard links (``os.link``) are not supported.
- Offset and iovec I/O functions (e.g. ``os.pread``, ``os.preadv``) are not
available.
- ``os.mknod`` and ``os.mkfifo``
@ -108,17 +115,9 @@ node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscri
- ``mmap`` module is unstable. flush (``msync``) can crash the runtime.
- Resource-related functions like ``os.nice`` and most functions of the
``resource`` module are not available.
- Some time and datetime features are broken. ``strftime`` and ``strptime``
have known bugs, e.g.
[%% quoting](https://github.com/emscripten-core/emscripten/issues/16155),
[%U off-by-one](https://github.com/emscripten-core/emscripten/issues/16156).
Extended glibc formatting features are not available.
- glibc extensions for date and time formatting are not available.
- ``locales`` module is affected by musl libc issues,
[bpo-46390](https://bugs.python.org/issue46390).
- ``uuid`` module is affected by
[memory leak](https://github.com/emscripten-core/emscripten/issues/16081)
and crasher in Emscripten's ``freeaddrinfo``,
- Recursive ``glob`` leaks file descriptors.
- Python's object allocator ``obmalloc`` is disabled by default.
- ``ensurepip`` is not available.

View File

@ -91,7 +91,3 @@ ac_cv_func_linkat=no
# alarm signal is not delivered, may need a callback into the event loop?
ac_cv_func_alarm=no
# To use dlopen, you need to use Emscripten's linking support,
# see https://emscripten.org/docs/compiling/Dynamic-Linking.html
ac_cv_func_dlopen=no

104
configure generated vendored
View File

@ -1017,6 +1017,7 @@ with_framework_name
enable_framework
with_cxx_main
with_emscripten_target
enable_wasm_dynamic_linking
with_suffix
enable_shared
with_static_libpython
@ -1730,6 +1731,9 @@ Optional Features:
Unix install. optional INSTALLDIR specifies the
installation path. see Mac/README.rst (default is
no)
--enable-wasm-dynamic-linking
Enable dynamic linking support for WebAssembly
(default is no)
--enable-shared enable building a shared Python library (default is
no)
--enable-profiling enable C-level code profiling with gprof (default is
@ -6290,6 +6294,30 @@ fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_sys_emscripten_target" >&5
$as_echo "$ac_sys_emscripten_target" >&6; }
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --enable-wasm-dynamic-linking" >&5
$as_echo_n "checking for --enable-wasm-dynamic-linking... " >&6; }
# Check whether --enable-wasm-dynamic-linking was given.
if test "${enable_wasm_dynamic_linking+set}" = set; then :
enableval=$enable_wasm_dynamic_linking;
case $ac_sys_system in #(
Emscripten) :
;; #(
WASI) :
as_fn_error $? "WASI dynamic linking is not implemented yet." "$LINENO" 5 ;; #(
*) :
as_fn_error $? "--enable-wasm-dynamic-linking only applies to Emscripten and WASI" "$LINENO" 5
;;
esac
else
enable_wasm_dynamic_linking=missing
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_wasm_dynamic_linking" >&5
$as_echo "$enable_wasm_dynamic_linking" >&6; }
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-suffix" >&5
$as_echo_n "checking for --with-suffix... " >&6; }
@ -7760,18 +7788,20 @@ then
fi
# WASM flags
# TODO: Add -s MAIN_MODULE=2 for dlopen() support.
# The option disables code elimination, which increases code size of main
# binary. All objects must be built with -fPIC.
case $ac_sys_system/$ac_sys_emscripten_target in #(
Emscripten/browser*) :
LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1"
LINKFORSHARED="--preload-file \$(WASM_ASSETS_DIR)"
LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH=1"
LINKFORSHARED="--preload-file=\$(WASM_ASSETS_DIR)"
if test "x$enable_wasm_dynamic_linking" = xyes; then :
as_fn_append LINKFORSHARED " -sMAIN_MODULE=1"
fi
WASM_ASSETS_DIR=".\$(prefix)"
WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py"
if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "browser-debug"; then
LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1"
LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS=1"
LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map"
else
LINKFORSHARED="$LINKFORSHARED -O2 -g0"
@ -7779,11 +7809,16 @@ case $ac_sys_system/$ac_sys_emscripten_target in #(
;; #(
Emscripten/node*) :
LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s USE_PTHREADS=1"
LINKFORSHARED="-s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1"
LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH=1 -sNODERAWFS=1 -sUSE_PTHREADS=1"
LINKFORSHARED="-sPROXY_TO_PTHREAD=1 -sEXIT_RUNTIME=1"
if test "x$enable_wasm_dynamic_linking" = xyes; then :
as_fn_append LINKFORSHARED " -sMAIN_MODULE=1"
fi
CFLAGS_NODIST="$CFLAGS_NODIST -pthread"
if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "node-debug"; then
LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1"
LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS=1"
LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map"
else
LINKFORSHARED="$LINKFORSHARED -O2 -g0"
@ -7808,6 +7843,18 @@ $as_echo "#define _WASI_EMULATED_PROCESS_CLOCKS 1" >>confdefs.h
;;
esac
case $enable_wasm_dynamic_linking in #(
yes) :
ac_cv_func_dlopen=yes ;; #(
no) :
ac_cv_func_dlopen=no ;; #(
missing) :
;; #(
*) :
;;
esac
@ -10469,10 +10516,6 @@ then
Linux*|GNU*|QNX*|VxWorks*|Haiku*)
LDSHARED='$(CC) -shared'
LDCXXSHARED='$(CXX) -shared';;
Emscripten*)
LDSHARED='$(CC) -shared -s SIDE_MODULE=1'
LDCXXSHARED='$(CXX) -shared -s SIDE_MODULE=1'
;;
FreeBSD*)
if [ "`$CC -dM -E - </dev/null | grep __ELF__`" != "" ]
then
@ -10511,16 +10554,32 @@ then
SCO_SV*)
LDSHARED='$(CC) -Wl,-G,-Bexport'
LDCXXSHARED='$(CXX) -Wl,-G,-Bexport';;
WASI*)
if test "x$enable_wasm_dynamic_linking" = xyes; then :
fi;;
CYGWIN*)
LDSHARED="gcc -shared -Wl,--enable-auto-image-base"
LDCXXSHARED="g++ -shared -Wl,--enable-auto-image-base";;
*) LDSHARED="ld";;
esac
fi
if test "$enable_wasm_dynamic_linking" = "yes" -a "$ac_sys_system" = "Emscripten"; then
BLDSHARED='$(CC) -shared -sSIDE_MODULE=1 -sWASM=1'
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $LDSHARED" >&5
$as_echo "$LDSHARED" >&6; }
LDCXXSHARED=${LDCXXSHARED-$LDSHARED}
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking BLDSHARED flags" >&5
$as_echo_n "checking BLDSHARED flags... " >&6; }
BLDSHARED=${BLDSHARED-$LDSHARED}
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $BLDSHARED" >&5
$as_echo "$BLDSHARED" >&6; }
# CCSHARED are the C *flags* used to create objects to go into a shared
# library (module) -- this is only needed for a few systems
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking CCSHARED" >&5
@ -10540,6 +10599,12 @@ then
fi;;
Linux-android*) ;;
Linux*|GNU*) CCSHARED="-fPIC";;
Emscripten*|WASI*)
if test "x$enable_wasm_dynamic_linking" = xyes; then :
CCSHARED="-fPIC"
fi;;
FreeBSD*|NetBSD*|OpenBSD*|DragonFly*) CCSHARED="-fPIC";;
Haiku*) CCSHARED="-fPIC";;
OpenUNIX*|UnixWare*)
@ -10647,6 +10712,13 @@ then
CFLAGSFORSHARED='$(CCSHARED)'
esac
fi
if test "x$enable_wasm_dynamic_linking" = xyes; then :
CFLAGSFORSHARED='$(CCSHARED)'
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CFLAGSFORSHARED" >&5
$as_echo "$CFLAGSFORSHARED" >&6; }
@ -15379,7 +15451,8 @@ $as_echo "yes" >&6; }
fi
if test "$have_zlib" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$ZLIB_LIBS" = "-lz"; then
ZLIB_LIBS="-s USE_ZLIB=1"
ZLIB_CFLAGS="-sUSE_ZLIB=1"
ZLIB_LIBS="-sUSE_ZLIB=1"
fi
if test "x$have_zlib" = xyes; then :
@ -15612,7 +15685,8 @@ $as_echo "yes" >&6; }
fi
if test "$have_bzip2" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$BZIP2_LIBS" = "-lbz2"; then
BZIP2_LIBS="-s USE_BZIP2=1"
BZIP2_CFLAGS="-sUSE_BZIP2=1"
BZIP2_LIBS="-sUSE_BZIP2=1"
fi

View File

@ -1107,6 +1107,25 @@ dnl are free to remove them in the future.
])
AC_MSG_RESULT([$ac_sys_emscripten_target])
dnl On Emscripten dlopen() requires -s MAIN_MODULE and -fPIC. The flags
dnl disables dead code elimination and increases the size of the WASM module
dnl by about 1.5 to 2MB. MAIN_MODULE defines __wasm_mutable_globals__.
dnl See https://emscripten.org/docs/compiling/Dynamic-Linking.html
AC_MSG_CHECKING([for --enable-wasm-dynamic-linking])
AC_ARG_ENABLE([wasm-dynamic-linking],
[AS_HELP_STRING([--enable-wasm-dynamic-linking],
[Enable dynamic linking support for WebAssembly (default is no)])],
[
AS_CASE([$ac_sys_system],
[Emscripten], [],
[WASI], [AC_MSG_ERROR([WASI dynamic linking is not implemented yet.])],
[AC_MSG_ERROR([--enable-wasm-dynamic-linking only applies to Emscripten and WASI])]
)
], [
enable_wasm_dynamic_linking=missing
])
AC_MSG_RESULT([$enable_wasm_dynamic_linking])
AC_MSG_CHECKING([for --with-suffix])
AC_ARG_WITH([suffix],
[AS_HELP_STRING([--with-suffix=SUFFIX], [set executable suffix to SUFFIX (default is empty, yes is mapped to '.exe')])],
@ -1890,29 +1909,32 @@ then
fi
# WASM flags
# TODO: Add -s MAIN_MODULE=2 for dlopen() support.
# The option disables code elimination, which increases code size of main
# binary. All objects must be built with -fPIC.
AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
[Emscripten/browser*], [
LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1"
LINKFORSHARED="--preload-file \$(WASM_ASSETS_DIR)"
LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH=1"
LINKFORSHARED="--preload-file=\$(WASM_ASSETS_DIR)"
AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE=1"])
])
WASM_ASSETS_DIR=".\$(prefix)"
WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py"
dnl separate-dwarf does not seem to work in Chrome DevTools Support.
if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "browser-debug"; then
LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1"
LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS=1"
LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map"
else
LINKFORSHARED="$LINKFORSHARED -O2 -g0"
fi
],
[Emscripten/node*], [
LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s USE_PTHREADS=1"
LINKFORSHARED="-s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1"
LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH=1 -sNODERAWFS=1 -sUSE_PTHREADS=1"
LINKFORSHARED="-sPROXY_TO_PTHREAD=1 -sEXIT_RUNTIME=1"
AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE=1"])
])
CFLAGS_NODIST="$CFLAGS_NODIST -pthread"
if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "node-debug"; then
LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1"
LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS=1"
LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map"
else
LINKFORSHARED="$LINKFORSHARED -O2 -g0"
@ -1927,6 +1949,12 @@ AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
]
)
AS_CASE([$enable_wasm_dynamic_linking],
[yes], [ac_cv_func_dlopen=yes],
[no], [ac_cv_func_dlopen=no],
[missing], []
)
AC_SUBST(BASECFLAGS)
AC_SUBST(CFLAGS_NODIST)
AC_SUBST(LDFLAGS_NODIST)
@ -2929,10 +2957,6 @@ then
Linux*|GNU*|QNX*|VxWorks*|Haiku*)
LDSHARED='$(CC) -shared'
LDCXXSHARED='$(CXX) -shared';;
Emscripten*)
LDSHARED='$(CC) -shared -s SIDE_MODULE=1'
LDCXXSHARED='$(CXX) -shared -s SIDE_MODULE=1'
;;
FreeBSD*)
if [[ "`$CC -dM -E - </dev/null | grep __ELF__`" != "" ]]
then
@ -2971,15 +2995,30 @@ then
SCO_SV*)
LDSHARED='$(CC) -Wl,-G,-Bexport'
LDCXXSHARED='$(CXX) -Wl,-G,-Bexport';;
WASI*)
AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
dnl not iplemented yet
]);;
CYGWIN*)
LDSHARED="gcc -shared -Wl,--enable-auto-image-base"
LDCXXSHARED="g++ -shared -Wl,--enable-auto-image-base";;
*) LDSHARED="ld";;
esac
fi
dnl Emscripten's emconfigure sets LDSHARED. Set BLDSHARED outside the
dnl test -z $LDSHARED block to configure BLDSHARED for side module support.
if test "$enable_wasm_dynamic_linking" = "yes" -a "$ac_sys_system" = "Emscripten"; then
BLDSHARED='$(CC) -shared -sSIDE_MODULE=1 -sWASM=1'
fi
AC_MSG_RESULT($LDSHARED)
LDCXXSHARED=${LDCXXSHARED-$LDSHARED}
AC_MSG_CHECKING([BLDSHARED flags])
BLDSHARED=${BLDSHARED-$LDSHARED}
AC_MSG_RESULT([$BLDSHARED])
# CCSHARED are the C *flags* used to create objects to go into a shared
# library (module) -- this is only needed for a few systems
AC_MSG_CHECKING(CCSHARED)
@ -2998,6 +3037,10 @@ then
fi;;
Linux-android*) ;;
Linux*|GNU*) CCSHARED="-fPIC";;
Emscripten*|WASI*)
AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
CCSHARED="-fPIC"
]);;
FreeBSD*|NetBSD*|OpenBSD*|DragonFly*) CCSHARED="-fPIC";;
Haiku*) CCSHARED="-fPIC";;
OpenUNIX*|UnixWare*)
@ -3099,6 +3142,12 @@ then
CFLAGSFORSHARED='$(CCSHARED)'
esac
fi
dnl WASM dynamic linking requires -fPIC.
AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
CFLAGSFORSHARED='$(CCSHARED)'
])
AC_MSG_RESULT($CFLAGSFORSHARED)
# SHLIBS are libraries (except -lc and -lm) to link to the python shared
@ -4460,7 +4509,8 @@ PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0], [
])
if test "$have_zlib" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$ZLIB_LIBS" = "-lz"; then
ZLIB_LIBS="-s USE_ZLIB=1"
ZLIB_CFLAGS="-sUSE_ZLIB=1"
ZLIB_LIBS="-sUSE_ZLIB=1"
fi
dnl binascii can use zlib for optimized crc32.
@ -4482,7 +4532,8 @@ PKG_CHECK_MODULES([BZIP2], [bzip2], [have_bzip2=yes], [
])
if test "$have_bzip2" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$BZIP2_LIBS" = "-lbz2"; then
BZIP2_LIBS="-s USE_BZIP2=1"
BZIP2_CFLAGS="-sUSE_BZIP2=1"
BZIP2_LIBS="-sUSE_BZIP2=1"
fi