[Python-checkins] gh-95853: Multiple ops and debug for wasm_build.py (#96744)
tiran
webhook-mailer at python.org
Sun Sep 11 03:51:44 EDT 2022
https://github.com/python/cpython/commit/1fc8bd3710670982bc7ce91a34d8cd4bcdf88b9a
commit: 1fc8bd3710670982bc7ce91a34d8cd4bcdf88b9a
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2022-09-11T09:51:23+02:00
summary:
gh-95853: Multiple ops and debug for wasm_build.py (#96744)
files:
M Lib/distutils/tests/test_sysconfig.py
M Lib/test/test_sysconfig.py
M Makefile.pre.in
M Tools/wasm/README.md
M Tools/wasm/wasm_build.py
diff --git a/Lib/distutils/tests/test_sysconfig.py b/Lib/distutils/tests/test_sysconfig.py
index 8f9f72f2b065..ae0eca897bc7 100644
--- a/Lib/distutils/tests/test_sysconfig.py
+++ b/Lib/distutils/tests/test_sysconfig.py
@@ -48,6 +48,7 @@ def test_get_config_vars(self):
self.assertIsInstance(cvars, dict)
self.assertTrue(cvars)
+ @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
def test_srcdir(self):
# See Issues #15322, #15364.
srcdir = sysconfig.get_config_var('srcdir')
diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py
index 114e144d52e9..b6dbf3d52cb4 100644
--- a/Lib/test/test_sysconfig.py
+++ b/Lib/test/test_sysconfig.py
@@ -438,6 +438,7 @@ def test_platform_in_subprocess(self):
self.assertEqual(status, 0)
self.assertEqual(my_platform, test_platform)
+ @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
def test_srcdir(self):
# See Issues #15322, #15364.
srcdir = sysconfig.get_config_var('srcdir')
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 107a7075ebf6..5201abb5ee5b 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1718,6 +1718,10 @@ buildbottest: all
fi
$(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --timeout=$(TESTTIMEOUT) $(TESTOPTS)
+# Like testall, but run Python tests with HOSTRUNNER directly.
+hostrunnertest: all
+ $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS)
+
pythoninfo: all
$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo
diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md
index c4c21b4f09dd..fe9a1dc99b30 100644
--- a/Tools/wasm/README.md
+++ b/Tools/wasm/README.md
@@ -1,11 +1,16 @@
# Python WebAssembly (WASM) build
-**WARNING: WASM support is highly experimental! Lots of features are not working yet.**
+**WARNING: WASM support is work-in-progress! Lots of features are not working yet.**
This directory contains configuration and helpers to facilitate cross
-compilation of CPython to WebAssembly (WASM). For now we support
-*wasm32-emscripten* builds for modern browser and for *Node.js*. WASI
-(*wasm32-wasi*) is work-in-progress
+compilation of CPython to WebAssembly (WASM). Python supports Emscripten
+(*wasm32-emscripten*) and WASI (*wasm32-wasi*) targets. Emscripten builds
+run in modern browsers and JavaScript runtimes like *Node.js*. WASI builds
+use WASM runtimes such as *wasmtime*.
+
+Users and developers are encouraged to use the script
+`Tools/wasm/wasm_build.py`. The tool automates the build process and provides
+assistance with installation of SDKs.
## wasm32-emscripten build
@@ -17,7 +22,7 @@ access the file system directly.
Cross compiling to the wasm32-emscripten platform needs the
[Emscripten](https://emscripten.org/) SDK and a build Python interpreter.
-Emscripten 3.1.8 or newer are recommended. All commands below are relative
+Emscripten 3.1.19 or newer are recommended. All commands below are relative
to a repository checkout.
Christian Heimes maintains a container image with Emscripten SDK, Python
@@ -336,26 +341,46 @@ if os.name == "posix":
```python
>>> import os, sys
>>> os.uname()
-posix.uname_result(sysname='Emscripten', nodename='emscripten', release='1.0', version='#1', machine='wasm32')
+posix.uname_result(
+ sysname='Emscripten',
+ nodename='emscripten',
+ release='3.1.19',
+ version='#1',
+ machine='wasm32'
+)
>>> os.name
'posix'
>>> sys.platform
'emscripten'
>>> sys._emscripten_info
sys._emscripten_info(
- emscripten_version=(3, 1, 8),
- runtime='Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0',
+ emscripten_version=(3, 1, 10),
+ runtime='Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0',
pthreads=False,
shared_memory=False
)
+```
+
+```python
>>> sys._emscripten_info
-sys._emscripten_info(emscripten_version=(3, 1, 8), runtime='Node.js v14.18.2', pthreads=True, shared_memory=True)
+sys._emscripten_info(
+ emscripten_version=(3, 1, 19),
+ runtime='Node.js v14.18.2',
+ pthreads=True,
+ shared_memory=True
+)
```
```python
>>> import os, sys
>>> os.uname()
-posix.uname_result(sysname='wasi', nodename='(none)', release='0.0.0', version='0.0.0', machine='wasm32')
+posix.uname_result(
+ sysname='wasi',
+ nodename='(none)',
+ release='0.0.0',
+ version='0.0.0',
+ machine='wasm32'
+)
>>> os.name
'posix'
>>> sys.platform
@@ -446,7 +471,8 @@ embuilder build --pic zlib bzip2 MINIMAL_PIC
**NOTE**: WASI-SDK's clang may show a warning on Fedora:
``/lib64/libtinfo.so.6: no version information available``,
-[RHBZ#1875587](https://bugzilla.redhat.com/show_bug.cgi?id=1875587).
+[RHBZ#1875587](https://bugzilla.redhat.com/show_bug.cgi?id=1875587). The
+warning can be ignored.
```shell
export WASI_VERSION=16
@@ -471,6 +497,8 @@ ln -srf -t /usr/local/bin/ ~/.wasmtime/bin/wasmtime
### WASI debugging
-* ``wasmtime run -g`` generates debugging symbols for gdb and lldb.
+* ``wasmtime run -g`` generates debugging symbols for gdb and lldb. The
+ feature is currently broken, see
+ https://github.com/bytecodealliance/wasmtime/issues/4669 .
* The environment variable ``RUST_LOG=wasi_common`` enables debug and
trace logging.
diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py
index 9054370e5e2f..63812c6f3153 100755
--- a/Tools/wasm/wasm_build.py
+++ b/Tools/wasm/wasm_build.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
"""Build script for Python on WebAssembly platforms.
- $ ./Tools/wasm/wasm_builder.py emscripten-browser compile
- $ ./Tools/wasm/wasm_builder.py emscripten-node-dl test
- $ ./Tools/wasm/wasm_builder.py wasi test
+ $ ./Tools/wasm/wasm_builder.py emscripten-browser build repl
+ $ ./Tools/wasm/wasm_builder.py emscripten-node-dl build test
+ $ ./Tools/wasm/wasm_builder.py wasi build test
Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
"emscripten-browser", and "wasi".
@@ -14,23 +14,36 @@
WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
and falls back to /opt/wasi-sdk.
+
+The 'build' Python interpreter must be rebuilt every time Python's byte code
+changes.
+
+ ./Tools/wasm/wasm_builder.py --clean build build
+
"""
import argparse
import enum
import dataclasses
+import logging
import os
import pathlib
import re
import shlex
import shutil
+import socket
import subprocess
+import sys
import sysconfig
import tempfile
+import time
import warnings
+import webbrowser
# for Python 3.8
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
+logger = logging.getLogger("wasm_build")
+
SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
WASMTOOLS = SRCDIR / "Tools" / "wasm"
BUILDDIR = SRCDIR / "builddir"
@@ -45,8 +58,7 @@
# path to Emscripten SDK config file.
# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten"))
-# 3.1.16 has broken utime()
-EMSDK_MIN_VERSION = (3, 1, 17)
+EMSDK_MIN_VERSION = (3, 1, 19)
EMSDK_BROKEN_VERSION = {
(3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338",
(3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393",
@@ -54,17 +66,25 @@
}
_MISSING = pathlib.PurePath("MISSING")
-# WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py"
+WASM_WEBSERVER = WASMTOOLS / "wasm_webserver.py"
CLEAN_SRCDIR = f"""
Builds require a clean source directory. Please use a clean checkout or
run "make clean -C '{SRCDIR}'".
"""
+INSTALL_NATIVE = f"""
+Builds require a C compiler (gcc, clang), make, pkg-config, and development
+headers for dependencies like zlib.
+
+Debian/Ubuntu: sudo apt install build-essential git curl pkg-config zlib1g-dev
+Fedora/CentOS: sudo dnf install gcc make git-core curl pkgconfig zlib-devel
+"""
+
INSTALL_EMSDK = """
wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
https://emscripten.org/docs/getting_started/downloads.html how to install
-Emscripten and how to activate the SDK with ". /path/to/emsdk/emsdk_env.sh".
+Emscripten and how to activate the SDK with "emsdk_env.sh".
git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
cd /path/to/emsdk
@@ -182,6 +202,24 @@ def _check_clean_src():
raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
+def _check_native():
+ if not any(shutil.which(cc) for cc in ["cc", "gcc", "clang"]):
+ raise MissingDependency("cc", INSTALL_NATIVE)
+ if not shutil.which("make"):
+ raise MissingDependency("make", INSTALL_NATIVE)
+ if sys.platform == "linux":
+ # skip pkg-config check on macOS
+ if not shutil.which("pkg-config"):
+ raise MissingDependency("pkg-config", INSTALL_NATIVE)
+ # zlib is needed to create zip files
+ for devel in ["zlib"]:
+ try:
+ subprocess.check_call(["pkg-config", "--exists", devel])
+ except subprocess.CalledProcessError:
+ raise MissingDependency(devel, INSTALL_NATIVE) from None
+ _check_clean_src()
+
+
NATIVE = Platform(
"native",
# macOS has python.exe
@@ -192,7 +230,7 @@ def _check_clean_src():
cc=None,
make_wrapper=None,
environ={},
- check=_check_clean_src,
+ check=_check_native,
)
@@ -362,9 +400,9 @@ class EmscriptenTarget(enum.Enum):
node_debug = "node-debug"
@property
- def can_execute(self) -> bool:
+ def is_browser(self):
cls = type(self)
- return self not in {cls.browser, cls.browser_debug}
+ return self in {cls.browser, cls.browser_debug}
@property
def emport_args(self) -> List[str]:
@@ -396,15 +434,12 @@ class BuildProfile:
target: Union[EmscriptenTarget, None] = None
dynamic_linking: Union[bool, None] = None
pthreads: Union[bool, None] = None
- testopts: str = "-j2"
+ default_testopts: str = "-j2"
@property
- def can_execute(self) -> bool:
- """Can target run pythoninfo and tests?
-
- Disabled for browser, enabled for all other targets
- """
- return self.target is None or self.target.can_execute
+ def is_browser(self) -> bool:
+ """Is this a browser build?"""
+ return self.target is not None and self.target.is_browser
@property
def builddir(self) -> pathlib.Path:
@@ -500,6 +535,7 @@ def _run_cmd(
cmd.extend(args)
if cwd is None:
cwd = self.builddir
+ logger.info('Running "%s" in "%s"', shlex.join(cmd), cwd)
return subprocess.check_call(
cmd,
cwd=os.fspath(cwd),
@@ -507,14 +543,15 @@ def _run_cmd(
)
def _check_execute(self):
- if not self.can_execute:
+ if self.is_browser:
raise ValueError(f"Cannot execute on {self.target}")
- def run_build(self, force_configure: bool = False):
+ def run_build(self, *args):
"""Run configure (if necessary) and make"""
- if force_configure or not self.makefile.exists():
- self.run_configure()
- self.run_make()
+ if not self.makefile.exists():
+ logger.info("Makefile not found, running configure")
+ self.run_configure(*args)
+ self.run_make("all", *args)
def run_configure(self, *args):
"""Run configure script to generate Makefile"""
@@ -525,15 +562,17 @@ def run_make(self, *args):
"""Run make (defaults to build all)"""
return self._run_cmd(self.make_cmd, args)
- def run_pythoninfo(self):
+ def run_pythoninfo(self, *args):
"""Run 'make pythoninfo'"""
self._check_execute()
- return self.run_make("pythoninfo")
+ return self.run_make("pythoninfo", *args)
- def run_test(self):
+ def run_test(self, target: str, testopts: Optional[str] = None):
"""Run buildbottests"""
self._check_execute()
- return self.run_make("buildbottest", f"TESTOPTS={self.testopts}")
+ if testopts is None:
+ testopts = self.default_testopts
+ return self.run_make(target, f"TESTOPTS={testopts}")
def run_py(self, *args):
"""Run Python with hostrunner"""
@@ -542,6 +581,37 @@ def run_py(self, *args):
"--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run"
)
+ def run_browser(self, bind="127.0.0.1", port=8000):
+ """Run WASM webserver and open build in browser"""
+ relbuilddir = self.builddir.relative_to(SRCDIR)
+ url = f"http://{bind}:{port}/{relbuilddir}/python.html"
+ args = [
+ sys.executable,
+ os.fspath(WASM_WEBSERVER),
+ "--bind",
+ bind,
+ "--port",
+ str(port),
+ ]
+ srv = subprocess.Popen(args, cwd=SRCDIR)
+ # wait for server
+ end = time.monotonic() + 3.0
+ while time.monotonic() < end and srv.returncode is None:
+ try:
+ with socket.create_connection((bind, port), timeout=0.1) as s:
+ pass
+ except OSError:
+ time.sleep(0.01)
+ else:
+ break
+
+ webbrowser.open(url)
+
+ try:
+ srv.wait()
+ except KeyboardInterrupt:
+ pass
+
def clean(self, all: bool = False):
"""Clean build directory"""
if all:
@@ -570,19 +640,19 @@ def build_emports(self, force: bool = False):
# Trigger PIC build.
ports_cmd.append("-sMAIN_MODULE")
embuilder_cmd.append("--pic")
+
if self.pthreads:
# Trigger multi-threaded build.
ports_cmd.append("-sUSE_PTHREADS")
- # https://github.com/emscripten-core/emscripten/pull/17729
- # embuilder_cmd.append("--pthreads")
# Pre-build libbz2, libsqlite3, libz, and some system libs.
ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"])
- embuilder_cmd.extend(["build", "bzip2", "sqlite3", "zlib"])
+ # Multi-threaded sqlite3 has different suffix
+ embuilder_cmd.extend(
+ ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"]
+ )
- if not self.pthreads:
- # Emscripten <= 3.1.20 has no option to build multi-threaded ports.
- self._run_cmd(embuilder_cmd, cwd=SRCDIR)
+ self._run_cmd(embuilder_cmd, cwd=SRCDIR)
with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir:
tmppath = pathlib.Path(tmpdir)
@@ -659,7 +729,7 @@ def build_emports(self, force: bool = False):
dynamic_linking=True,
pthreads=True,
),
- # wasm64-emscripten (requires unreleased Emscripten >= 3.1.21)
+ # wasm64-emscripten (requires Emscripten >= 3.1.21)
BuildProfile(
"wasm64-emscripten-node-debug",
support_level=SupportLevel.experimental,
@@ -674,8 +744,6 @@ def build_emports(self, force: bool = False):
"wasi",
support_level=SupportLevel.supported,
host=Host.wasm32_wasi,
- # skip sysconfig test_srcdir
- testopts="-i '*.test_srcdir' -j2",
),
# no SDK available yet
# BuildProfile(
@@ -690,10 +758,36 @@ def build_emports(self, force: bool = False):
parser = argparse.ArgumentParser(
"wasm_build.py",
description=__doc__,
- formatter_class=argparse.RawDescriptionHelpFormatter,
+ formatter_class=argparse.RawTextHelpFormatter,
+)
+
+parser.add_argument(
+ "--clean",
+ "-c",
+ help="Clean build directories first",
+ action="store_true",
)
+
+parser.add_argument(
+ "--verbose",
+ "-v",
+ help="Verbose logging",
+ action="store_true",
+)
+
parser.add_argument(
- "--clean", "-c", help="Clean build directories first", action="store_true"
+ "--silent",
+ help="Run configure and make in silent mode",
+ action="store_true",
+)
+
+parser.add_argument(
+ "--testopts",
+ help=(
+ "Additional test options for 'test' and 'hostrunnertest', e.g. "
+ "--testopts='-v test_os'."
+ ),
+ default=None,
)
# Don't list broken and experimental variants in help
@@ -706,67 +800,104 @@ def build_emports(self, force: bool = False):
choices=platforms_choices,
)
-ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall", "emports"]
+ops = dict(
+ build="auto build (build 'build' Python, emports, configure, compile)",
+ configure="run ./configure",
+ compile="run 'make all'",
+ pythoninfo="run 'make pythoninfo'",
+ test="run 'make buildbottest TESTOPTS=...' (supports parallel tests)",
+ hostrunnertest="run 'make hostrunnertest TESTOPTS=...'",
+ repl="start interactive REPL / webserver + browser session",
+ clean="run 'make clean'",
+ cleanall="remove all build directories",
+ emports="build Emscripten port with embuilder (only Emscripten)",
+)
+ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items())
parser.add_argument(
- "op",
+ "ops",
metavar="OP",
- help=f"operation: {', '.join(ops)}",
- choices=ops,
- default="compile",
- nargs="?",
+ help=f"operation (default: build)\n\n{ops_help}",
+ choices=tuple(ops),
+ default="build",
+ nargs="*",
)
def main():
args = parser.parse_args()
+ logging.basicConfig(
+ level=logging.INFO if args.verbose else logging.ERROR,
+ format="%(message)s",
+ )
+
if args.platform == "cleanall":
for builder in PROFILES.values():
builder.clean(all=True)
parser.exit(0)
+ # additional configure and make args
+ cm_args = ("--silent",) if args.silent else ()
+
+ # nargs=* with default quirk
+ if args.ops == "build":
+ args.ops = ["build"]
+
builder = PROFILES[args.platform]
try:
builder.host.platform.check()
except ConditionError as e:
parser.error(str(e))
+ if args.clean:
+ builder.clean(all=False)
+
# hack for WASI
if builder.host.is_wasi and not SETUP_LOCAL.exists():
SETUP_LOCAL.touch()
- if args.op in {"compile", "pythoninfo", "repl", "test"}:
- # all targets need a build Python
+ # auto-build
+ if "build" in args.ops:
+ # check and create build Python
if builder is not BUILD:
+ logger.info("Auto-building 'build' Python.")
+ try:
+ BUILD.host.platform.check()
+ except ConditionError as e:
+ parser.error(str(e))
if args.clean:
BUILD.clean(all=False)
- BUILD.run_build()
- elif not BUILD.python_cmd.exists():
- BUILD.run_build()
-
- if args.clean:
+ BUILD.run_build(*cm_args)
+ # build Emscripten ports with embuilder
+ if builder.host.is_emscripten and "emports" not in args.ops:
+ builder.build_emports()
+
+ for op in args.ops:
+ logger.info("\n*** %s %s", args.platform, op)
+ if op == "build":
+ builder.run_build(*cm_args)
+ elif op == "configure":
+ builder.run_configure(*cm_args)
+ elif op == "compile":
+ builder.run_make("all", *cm_args)
+ elif op == "pythoninfo":
+ builder.run_pythoninfo(*cm_args)
+ elif op == "repl":
+ if builder.is_browser:
+ builder.run_browser()
+ else:
+ builder.run_py()
+ elif op == "test":
+ builder.run_test("buildbottest", testopts=args.testopts)
+ elif op == "hostrunnertest":
+ builder.run_test("hostrunnertest", testopts=args.testopts)
+ elif op == "clean":
builder.clean(all=False)
-
- if args.op == "compile":
- if builder.host.is_emscripten:
- builder.build_emports()
- builder.run_build(force_configure=True)
+ elif op == "cleanall":
+ builder.clean(all=True)
+ elif op == "emports":
+ builder.build_emports(force=args.clean)
else:
- if not builder.makefile.exists():
- builder.run_configure()
- if args.op == "pythoninfo":
- builder.run_pythoninfo()
- elif args.op == "repl":
- builder.run_py()
- elif args.op == "test":
- builder.run_test()
- elif args.op == "clean":
- builder.clean(all=False)
- elif args.op == "cleanall":
- builder.clean(all=True)
- elif args.op == "emports":
- builder.build_emports(force=args.clean)
- else:
- raise ValueError(args.op)
+ raise ValueError(op)
print(builder.builddir)
parser.exit(0)
More information about the Python-checkins
mailing list