[Python-checkins] bpo-40280: Add Tools/wasm with helpers for cross building (GH-29984)
tiran
webhook-mailer at python.org
Sat Dec 18 09:54:06 EST 2021
https://github.com/python/cpython/commit/0339434835aa74dc78a38ae12ea7d2973c144eb1
commit: 0339434835aa74dc78a38ae12ea7d2973c144eb1
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-18T15:54:02+01:00
summary:
bpo-40280: Add Tools/wasm with helpers for cross building (GH-29984)
Co-authored-by: Ethan Smith <ethan at ethanhs.me>
Co-authored-by: Brett Cannon <brett at python.org>
files:
A Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst
A Tools/wasm/README.md
A Tools/wasm/config.site-wasm32-emscripten
A Tools/wasm/wasm_assets.py
M Makefile.pre.in
M configure
M configure.ac
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 59c92a0568022..ed77bebfab901 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -830,6 +830,22 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
else true; \
fi
+# wasm32-emscripten build
+# wasm assets directory is relative to current build dir, e.g. "./usr/local".
+# --preload-file turns a relative asset path into an absolute path.
+WASM_ASSETS_DIR=".$(prefix)"
+WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py"
+
+$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
+ pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py
+ $(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
+ --builddir . --prefix $(prefix)
+
+python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB)
+ $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \
+ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \
+ -s ASSERTIONS=1 --preload-file $(WASM_ASSETS_DIR)
+
##########################################################################
# Build static libmpdec.a
LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@
@@ -938,6 +954,7 @@ Makefile Modules/config.c: Makefile.pre \
$(SHELL) $(MAKESETUP) -c $(srcdir)/Modules/config.c.in \
-s Modules \
Modules/Setup.local \
+ @MODULES_SETUP_STDLIB@ \
$(srcdir)/Modules/Setup.bootstrap \
$(srcdir)/Modules/Setup
@mv config.c Modules
@@ -2379,6 +2396,7 @@ clean-retain-profile: pycremoval
-rm -f pybuilddir.txt
-rm -f Lib/lib2to3/*Grammar*.pickle
-rm -f _bootstrap_python
+ -rm -f python.html python.js python.data
-rm -f Programs/_testembed Programs/_freeze_module
-rm -f Python/deepfreeze/*.[co]
-rm -f Python/frozen_modules/*.h
diff --git a/Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst b/Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst
new file mode 100644
index 0000000000000..905ee44680276
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst
@@ -0,0 +1 @@
+A new directory ``Tools/wasm`` contains WebAssembly-related helpers like ``config.site`` override for wasm32-emscripten, wasm assets generator to bundle the stdlib, and a README.
diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md
new file mode 100644
index 0000000000000..93c76b225db79
--- /dev/null
+++ b/Tools/wasm/README.md
@@ -0,0 +1,55 @@
+# Python WebAssembly (WASM) build
+
+This directory contains configuration and helpers to facilitate cross
+compilation of CPython to WebAssembly (WASM).
+
+## wasm32-emscripten build
+
+Cross compiling to wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/)
+tool chain and a build Python interpreter.
+All commands below are relative to a repository checkout.
+
+### Compile a build Python interpreter
+
+```shell
+mkdir -p builddir/build
+pushd builddir/build
+../../configure -C
+make -j$(nproc)
+popd
+```
+
+### Fetch and build additional emscripten ports
+
+```shell
+embuilder build zlib
+```
+
+### Cross compile to wasm32-emscripten
+
+```shell
+mkdir -p builddir/emscripten
+pushd builddir/emscripten
+
+CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
+ emconfigure ../../configure -C \
+ --host=wasm32-unknown-emscripten \
+ --build=$(../../config.guess) \
+ --with-build-python=$(pwd)/../build/python
+
+emmake make -j$(nproc) python.html
+```
+
+### Test in browser
+
+Serve `python.html` with a local webserver and open the file in a browser.
+
+```shell
+emrun python.html
+```
+
+or
+
+```shell
+python3 -m http.server
+```
diff --git a/Tools/wasm/config.site-wasm32-emscripten b/Tools/wasm/config.site-wasm32-emscripten
new file mode 100644
index 0000000000000..67304be060b52
--- /dev/null
+++ b/Tools/wasm/config.site-wasm32-emscripten
@@ -0,0 +1,70 @@
+# config.site override for cross compiling to wasm32-emscripten platform
+#
+# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
+# emconfigure ./configure --host=wasm32-unknown-emscripten --build=...
+#
+# Written by Christian Heimes <christian at python.org>
+# Partly based on pyodide's pyconfig.undefs.h file.
+#
+
+# cannot be detected in cross builds
+ac_cv_buggy_getaddrinfo=no
+
+# Emscripten has no /dev/pt*
+ac_cv_file__dev_ptmx=no
+ac_cv_file__dev_ptc=no
+
+# dummy readelf, Emscripten build does not need readelf.
+ac_cv_prog_ac_ct_READELF=true
+
+# new undefined symbols / unsupported features
+ac_cv_func_posix_spawn=no
+ac_cv_func_posix_spawnp=no
+ac_cv_func_eventfd=no
+ac_cv_func_memfd_create=no
+ac_cv_func_prlimit=no
+
+# unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393
+ac_cv_func_shutdown=no
+
+# breaks build, see https://github.com/ethanhs/python-wasm/issues/16
+ac_cv_lib_bz2_BZ2_bzCompress=no
+
+# The rest is based on pyodide
+# https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h
+
+ac_cv_func_epoll=no
+ac_cv_func_epoll_create1=no
+ac_cv_header_linux_vm_sockets_h=no
+ac_cv_func_socketpair=no
+ac_cv_func_utimensat=no
+ac_cv_func_sigaction=no
+
+# Untested syscalls in emscripten
+ac_cv_func_openat=no
+ac_cv_func_mkdirat=no
+ac_cv_func_fchownat=no
+ac_cv_func_renameat=no
+ac_cv_func_linkat=no
+ac_cv_func_symlinkat=no
+ac_cv_func_readlinkat=no
+ac_cv_func_fchmodat=no
+ac_cv_func_dup3=no
+
+# Syscalls not implemented in emscripten
+ac_cv_func_preadv2=no
+ac_cv_func_preadv=no
+ac_cv_func_pwritev2=no
+ac_cv_func_pwritev=no
+ac_cv_func_pipe2=no
+ac_cv_func_nice=no
+
+# Syscalls that resulted in a segfault
+ac_cv_func_utimensat=no
+ac_cv_header_sys_ioctl_h=no
+
+# sockets are supported, but only in non-blocking mode
+# ac_cv_header_sys_socket_h=no
+
+# Unsupported functionality
+#undef HAVE_PTHREAD_H
diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py
new file mode 100755
index 0000000000000..6a4027184030f
--- /dev/null
+++ b/Tools/wasm/wasm_assets.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+"""Create a WASM asset bundle directory structure.
+
+The WASM asset bundles are pre-loaded by the final WASM build. The bundle
+contains:
+
+- a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip
+- os.py as marker module {PREFIX}/lib/python3.11/os.py
+- empty lib-dynload directory, to make sure it is copied into the bundle {PREFIX}/lib/python3.11/lib-dynload/.empty
+"""
+
+import argparse
+import pathlib
+import shutil
+import sys
+import zipfile
+
+# source directory
+SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
+SRCDIR_LIB = SRCDIR / "Lib"
+
+# sysconfig data relative to build dir.
+SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py"
+
+# Library directory relative to $(prefix).
+WASM_LIB = pathlib.PurePath("lib")
+WASM_STDLIB_ZIP = (
+ WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
+)
+WASM_STDLIB = (
+ WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
+)
+WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"
+
+
+# Don't ship large files / packages that are not particularly useful at
+# the moment.
+OMIT_FILES = (
+ # regression tests
+ "test/",
+ # user interfaces: TK, curses
+ "curses/",
+ "idlelib/",
+ "tkinter/",
+ "turtle.py",
+ "turtledemo/",
+ # package management
+ "ensurepip/",
+ "venv/",
+ # build system
+ "distutils/",
+ "lib2to3/",
+ # concurrency
+ "concurrent/",
+ "multiprocessing/",
+ # deprecated
+ "asyncore.py",
+ "asynchat.py",
+ # Synchronous network I/O and protocols are not supported; for example,
+ # socket.create_connection() raises an exception:
+ # "BlockingIOError: [Errno 26] Operation in progress".
+ "cgi.py",
+ "cgitb.py",
+ "email/",
+ "ftplib.py",
+ "http/",
+ "imaplib.py",
+ "nntplib.py",
+ "poplib.py",
+ "smtpd.py",
+ "smtplib.py",
+ "socketserver.py",
+ "telnetlib.py",
+ "urllib/",
+ "wsgiref/",
+ "xmlrpc/",
+ # dbm / gdbm
+ "dbm/",
+ # other platforms
+ "_aix_support.py",
+ "_bootsubprocess.py",
+ "_osx_support.py",
+ # webbrowser
+ "antigravity.py",
+ "webbrowser.py",
+ # ctypes
+ "ctypes/",
+ # Pure Python implementations of C extensions
+ "_pydecimal.py",
+ "_pyio.py",
+ # Misc unused or large files
+ "pydoc_data/",
+ "msilib/",
+)
+
+# regression test sub directories
+OMIT_SUBDIRS = (
+ "ctypes/test/",
+ "tkinter/test/",
+ "unittest/test/",
+)
+
+
+OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES}
+OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS)
+
+
+def filterfunc(name: str) -> bool:
+ return not name.startswith(OMIT_SUBDIRS_ABSOLUTE)
+
+
+def create_stdlib_zip(
+ args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0
+) -> None:
+ sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB))
+ if not sysconfig_data:
+ raise ValueError("No sysconfigdata file found")
+
+ with zipfile.PyZipFile(
+ args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0
+ ) as pzf:
+ for entry in sorted(args.srcdir_lib.iterdir()):
+ if entry.name == "__pycache__":
+ continue
+ if entry in OMIT_ABSOLUTE:
+ continue
+ if entry.name.endswith(".py") or entry.is_dir():
+ # writepy() writes .pyc files (bytecode).
+ pzf.writepy(entry, filterfunc=filterfunc)
+ for entry in sysconfig_data:
+ pzf.writepy(entry)
+
+
+def path(val: str) -> pathlib.Path:
+ return pathlib.Path(val).absolute()
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--builddir",
+ help="absolute build directory",
+ default=pathlib.Path(".").absolute(),
+ type=path,
+)
+parser.add_argument(
+ "--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path
+)
+
+
+def main():
+ args = parser.parse_args()
+
+ relative_prefix = args.prefix.relative_to(pathlib.Path("/"))
+ args.srcdir = SRCDIR
+ args.srcdir_lib = SRCDIR_LIB
+ args.wasm_root = args.builddir / relative_prefix
+ args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP
+ args.wasm_stdlib = args.wasm_root / WASM_STDLIB
+ args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
+
+ # Empty, unused directory for dynamic libs, but required for site initialization.
+ args.wasm_dynload.mkdir(parents=True, exist_ok=True)
+ marker = args.wasm_dynload / ".empty"
+ marker.touch()
+ # os.py is a marker for finding the correct lib directory.
+ shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
+ # The rest of stdlib that's useful in a WASM context.
+ create_stdlib_zip(args)
+ size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2)
+ parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/configure b/configure
index 1ede29989d948..eca63518dbca3 100755
--- a/configure
+++ b/configure
@@ -772,6 +772,7 @@ MODULE_TIME_FALSE
MODULE_TIME_TRUE
MODULE__IO_FALSE
MODULE__IO_TRUE
+MODULES_SETUP_STDLIB
MODULE_BUILDTYPE
TEST_MODULES
LIBRARY_DEPS
@@ -13298,7 +13299,13 @@ fi
if test -z "$with_pymalloc"
then
+ case $ac_sys_system in #(
+ Emscripten) :
+ with_pymalloc="no" ;; #(
+ *) :
with_pymalloc="yes"
+ ;;
+esac
fi
if test "$with_pymalloc" != "no"
then
@@ -21165,12 +21172,22 @@ fi
if test "$enable_test_modules" = no; then
TEST_MODULES=no
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
else
+ case $ac_sys_system in #(
+ Emscripten) :
+ TEST_MODULES=no ;; #(
+ *) :
TEST_MODULES=yes
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+ ;;
+esac
+fi
+if test "x$TEST_MODULES" = xyes; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
fi
@@ -21189,7 +21206,7 @@ case $ac_sys_system in #(
py_stdlib_not_available="_scproxy spwd" ;; #(
Emscripten) :
- py_stdlib_not_available="_curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _xxsubinterpreters fcntl grp nis ossaudiodev resource spwd syslog termios"
+ py_stdlib_not_available="_ctypes _curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _tkinter _xxsubinterpreters fcntl grp nis ossaudiodev resource readline spwd syslog termios"
;; #(
*) :
py_stdlib_not_available="_scproxy"
@@ -21205,6 +21222,20 @@ case $host_cpu in #(
esac
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for additional Modules/Setup files" >&5
+$as_echo_n "checking for additional Modules/Setup files... " >&6; }
+case $ac_sys_system in #(
+ Emscripten) :
+ MODULES_SETUP_STDLIB=Modules/Setup.stdlib ;; #(
+ *) :
+ MODULES_SETUP_STDLIB=
+ ;;
+esac
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $MODULES_SETUP_STDLIB" >&5
+$as_echo "$MODULES_SETUP_STDLIB" >&6; }
+
+
+
MODULE_BLOCK=
@@ -25100,7 +25131,7 @@ fi
$as_echo "$as_me: creating Makefile" >&6;}
$SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
-s Modules \
- Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
+ Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
mv config.c Modules
if test -z "$PKG_CONFIG"; then
diff --git a/configure.ac b/configure.ac
index 86404bcadeaba..050b907ac8624 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3865,7 +3865,11 @@ AC_ARG_WITH(pymalloc,
if test -z "$with_pymalloc"
then
- with_pymalloc="yes"
+ dnl default to yes except for wasm32-emscripten
+ AS_CASE([$ac_sys_system],
+ [Emscripten], [with_pymalloc="no"],
+ [with_pymalloc="yes"]
+ )
fi
if test "$with_pymalloc" != "no"
then
@@ -6253,11 +6257,15 @@ AC_ARG_ENABLE(test-modules,
AS_HELP_STRING([--disable-test-modules], [don't build nor install test modules]))
if test "$enable_test_modules" = no; then
TEST_MODULES=no
- AC_MSG_RESULT(yes)
else
- TEST_MODULES=yes
- AC_MSG_RESULT(no)
+ AS_CASE([$ac_sys_system],
+ [Emscripten], [TEST_MODULES=no],
+ [TEST_MODULES=yes]
+ )
fi
+AS_VAR_IF([TEST_MODULES], [yes],
+ [AC_MSG_RESULT(no)], [AC_MSG_RESULT(yes)]
+)
AC_SUBST(TEST_MODULES)
dnl Modules that are not available on some platforms
@@ -6272,6 +6280,7 @@ AS_CASE([$ac_sys_system],
[FreeBSD*], [py_stdlib_not_available="_scproxy spwd"],
[Emscripten], [
py_stdlib_not_available="m4_normalize([
+ _ctypes
_curses
_curses_panel
_dbm
@@ -6280,12 +6289,14 @@ AS_CASE([$ac_sys_system],
_posixshmem
_posixsubprocess
_scproxy
+ _tkinter
_xxsubinterpreters
fcntl
grp
nis
ossaudiodev
resource
+ readline
spwd
syslog
termios
@@ -6301,6 +6312,16 @@ AS_CASE([$host_cpu],
)
AC_SUBST([MODULE_BUILDTYPE])
+dnl Use Modules/Setup.stdlib as additional provider?
+AC_MSG_CHECKING([for additional Modules/Setup files])
+AS_CASE([$ac_sys_system],
+ [Emscripten], [MODULES_SETUP_STDLIB=Modules/Setup.stdlib],
+ [MODULES_SETUP_STDLIB=]
+)
+AC_MSG_RESULT([$MODULES_SETUP_STDLIB])
+AC_SUBST([MODULES_SETUP_STDLIB])
+
+
dnl _MODULE_BLOCK_ADD([VAR], [VALUE])
dnl internal: adds $1=quote($2) to MODULE_BLOCK
AC_DEFUN([_MODULE_BLOCK_ADD], [AS_VAR_APPEND([MODULE_BLOCK], ["$1=_AS_QUOTE([$2])$as_nl"])])
@@ -6515,7 +6536,7 @@ fi
AC_MSG_NOTICE([creating Makefile])
$SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
-s Modules \
- Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
+ Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
mv config.c Modules
if test -z "$PKG_CONFIG"; then
More information about the Python-checkins
mailing list