mirror of
https://github.com/python/cpython.git
synced 2025-08-04 08:59:19 +00:00
[3.14] GH-133600: Backport file reorg of Tools/wasm/wasi (GH-135950)
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
This commit is contained in:
Brett Cannon
GitHub
parent
2045453171
commit
4ba18128f9
7 changed files with 399 additions and 378 deletions
7
.github/CODEOWNERS
vendored
7
.github/CODEOWNERS
vendored
|
|
@ -302,7 +302,12 @@ Lib/concurrent/futures/interpreter.py @ericsnowcurrently
|
||||||
**/*-ios* @freakboy3742
|
**/*-ios* @freakboy3742
|
||||||
|
|
||||||
# WebAssembly
|
# WebAssembly
|
||||||
/Tools/wasm/ @brettcannon @freakboy3742
|
Tools/wasm/config.site-wasm32-emscripten @freakboy3742
|
||||||
|
/Tools/wasm/README.md @brettcannon @freakboy3742
|
||||||
|
/Tools/wasm/wasi-env @brettcannon
|
||||||
|
/Tools/wasm/wasi_build.py @brettcannon
|
||||||
|
/Tools/wasm/emscripten @freakboy3742
|
||||||
|
/Tools/wasm/wasi @brettcannon
|
||||||
|
|
||||||
# SBOM
|
# SBOM
|
||||||
/Misc/externals.spdx.json @sethmlarson
|
/Misc/externals.spdx.json @sethmlarson
|
||||||
|
|
|
||||||
8
.github/workflows/reusable-wasi.yml
vendored
8
.github/workflows/reusable-wasi.yml
vendored
|
|
@ -64,9 +64,9 @@ jobs:
|
||||||
# (Make sure to keep the key in sync with the other config.cache step below.)
|
# (Make sure to keep the key in sync with the other config.cache step below.)
|
||||||
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
|
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
|
||||||
- name: "Configure build Python"
|
- name: "Configure build Python"
|
||||||
run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug
|
run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug
|
||||||
- name: "Make build Python"
|
- name: "Make build Python"
|
||||||
run: python3 Tools/wasm/wasi.py make-build-python
|
run: python3 Tools/wasm/wasi make-build-python
|
||||||
- name: "Restore host config.cache"
|
- name: "Restore host config.cache"
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
|
|
@ -75,9 +75,9 @@ jobs:
|
||||||
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
|
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
|
||||||
- name: "Configure host"
|
- name: "Configure host"
|
||||||
# `--with-pydebug` inferred from configure-build-python
|
# `--with-pydebug` inferred from configure-build-python
|
||||||
run: python3 Tools/wasm/wasi.py configure-host -- --config-cache
|
run: python3 Tools/wasm/wasi configure-host -- --config-cache
|
||||||
- name: "Make host"
|
- name: "Make host"
|
||||||
run: python3 Tools/wasm/wasi.py make-host
|
run: python3 Tools/wasm/wasi make-host
|
||||||
- name: "Display build info"
|
- name: "Display build info"
|
||||||
run: make --directory "${CROSS_BUILD_WASI}" pythoninfo
|
run: make --directory "${CROSS_BUILD_WASI}" pythoninfo
|
||||||
- name: "Test"
|
- name: "Test"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
Backport file reorganization for Tools/wasm/wasi.
|
||||||
|
|
||||||
|
This should make backporting future code changes easier. It also simplifies
|
||||||
|
instructions around how to do WASI builds in the devguide.
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py .
|
# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py ;
|
||||||
|
# expected in Python 3.18 as 3.13 is when `wasi.py` was introduced.
|
||||||
|
|
||||||
# function
|
# function
|
||||||
usage() {
|
usage() {
|
||||||
|
|
|
||||||
|
|
@ -1,373 +1,10 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import contextlib
|
|
||||||
import functools
|
|
||||||
import os
|
|
||||||
try:
|
|
||||||
from os import process_cpu_count as cpu_count
|
|
||||||
except ImportError:
|
|
||||||
from os import cpu_count
|
|
||||||
import pathlib
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import sysconfig
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
|
|
||||||
CHECKOUT = pathlib.Path(__file__).parent.parent.parent
|
|
||||||
|
|
||||||
CROSS_BUILD_DIR = CHECKOUT / "cross-build"
|
|
||||||
BUILD_DIR = CROSS_BUILD_DIR / "build"
|
|
||||||
|
|
||||||
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
|
|
||||||
LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8")
|
|
||||||
|
|
||||||
WASMTIME_VAR_NAME = "WASMTIME"
|
|
||||||
WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
|
|
||||||
|
|
||||||
|
|
||||||
def updated_env(updates={}):
|
|
||||||
"""Create a new dict representing the environment to use.
|
|
||||||
|
|
||||||
The changes made to the execution environment are printed out.
|
|
||||||
"""
|
|
||||||
env_defaults = {}
|
|
||||||
# https://reproducible-builds.org/docs/source-date-epoch/
|
|
||||||
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
|
|
||||||
try:
|
|
||||||
epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip()
|
|
||||||
env_defaults["SOURCE_DATE_EPOCH"] = epoch
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
pass # Might be building from a tarball.
|
|
||||||
# This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
|
|
||||||
environment = env_defaults | os.environ | updates
|
|
||||||
|
|
||||||
env_diff = {}
|
|
||||||
for key, value in environment.items():
|
|
||||||
if os.environ.get(key) != value:
|
|
||||||
env_diff[key] = value
|
|
||||||
|
|
||||||
print("🌎 Environment changes:")
|
|
||||||
for key in sorted(env_diff.keys()):
|
|
||||||
print(f" {key}={env_diff[key]}")
|
|
||||||
|
|
||||||
return environment
|
|
||||||
|
|
||||||
|
|
||||||
def subdir(working_dir, *, clean_ok=False):
|
|
||||||
"""Decorator to change to a working directory."""
|
|
||||||
def decorator(func):
|
|
||||||
@functools.wraps(func)
|
|
||||||
def wrapper(context):
|
|
||||||
nonlocal working_dir
|
|
||||||
|
|
||||||
if callable(working_dir):
|
|
||||||
working_dir = working_dir(context)
|
|
||||||
try:
|
|
||||||
tput_output = subprocess.check_output(["tput", "cols"],
|
|
||||||
encoding="utf-8")
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
terminal_width = 80
|
|
||||||
else:
|
|
||||||
terminal_width = int(tput_output.strip())
|
|
||||||
print("⎯" * terminal_width)
|
|
||||||
print("📁", working_dir)
|
|
||||||
if (clean_ok and getattr(context, "clean", False) and
|
|
||||||
working_dir.exists()):
|
|
||||||
print(f"🚮 Deleting directory (--clean)...")
|
|
||||||
shutil.rmtree(working_dir)
|
|
||||||
|
|
||||||
working_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
with contextlib.chdir(working_dir):
|
|
||||||
return func(context, working_dir)
|
|
||||||
|
|
||||||
return wrapper
|
|
||||||
|
|
||||||
return decorator
|
|
||||||
|
|
||||||
|
|
||||||
def call(command, *, quiet, **kwargs):
|
|
||||||
"""Execute a command.
|
|
||||||
|
|
||||||
If 'quiet' is true, then redirect stdout and stderr to a temporary file.
|
|
||||||
"""
|
|
||||||
print("❯", " ".join(map(str, command)))
|
|
||||||
if not quiet:
|
|
||||||
stdout = None
|
|
||||||
stderr = None
|
|
||||||
else:
|
|
||||||
stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8",
|
|
||||||
delete=False,
|
|
||||||
prefix="cpython-wasi-",
|
|
||||||
suffix=".log")
|
|
||||||
stderr = subprocess.STDOUT
|
|
||||||
print(f"📝 Logging output to {stdout.name} (--quiet)...")
|
|
||||||
|
|
||||||
subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
|
|
||||||
|
|
||||||
|
|
||||||
def build_platform():
|
|
||||||
"""The name of the build/host platform."""
|
|
||||||
# Can also be found via `config.guess`.
|
|
||||||
return sysconfig.get_config_var("BUILD_GNU_TYPE")
|
|
||||||
|
|
||||||
|
|
||||||
def build_python_path():
|
|
||||||
"""The path to the build Python binary."""
|
|
||||||
binary = BUILD_DIR / "python"
|
|
||||||
if not binary.is_file():
|
|
||||||
binary = binary.with_suffix(".exe")
|
|
||||||
if not binary.is_file():
|
|
||||||
raise FileNotFoundError("Unable to find `python(.exe)` in "
|
|
||||||
f"{BUILD_DIR}")
|
|
||||||
|
|
||||||
return binary
|
|
||||||
|
|
||||||
|
|
||||||
def build_python_is_pydebug():
|
|
||||||
"""Find out if the build Python is a pydebug build."""
|
|
||||||
test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
|
|
||||||
result = subprocess.run([build_python_path(), "-c", test],
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE)
|
|
||||||
return bool(result.returncode)
|
|
||||||
|
|
||||||
|
|
||||||
@subdir(BUILD_DIR, clean_ok=True)
|
|
||||||
def configure_build_python(context, working_dir):
|
|
||||||
"""Configure the build/host Python."""
|
|
||||||
if LOCAL_SETUP.exists():
|
|
||||||
print(f"👍 {LOCAL_SETUP} exists ...")
|
|
||||||
else:
|
|
||||||
print(f"📝 Touching {LOCAL_SETUP} ...")
|
|
||||||
LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
|
|
||||||
|
|
||||||
configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)]
|
|
||||||
if context.args:
|
|
||||||
configure.extend(context.args)
|
|
||||||
|
|
||||||
call(configure, quiet=context.quiet)
|
|
||||||
|
|
||||||
|
|
||||||
@subdir(BUILD_DIR)
|
|
||||||
def make_build_python(context, working_dir):
|
|
||||||
"""Make/build the build Python."""
|
|
||||||
call(["make", "--jobs", str(cpu_count()), "all"],
|
|
||||||
quiet=context.quiet)
|
|
||||||
|
|
||||||
binary = build_python_path()
|
|
||||||
cmd = [binary, "-c",
|
|
||||||
"import sys; "
|
|
||||||
"print(f'{sys.version_info.major}.{sys.version_info.minor}')"]
|
|
||||||
version = subprocess.check_output(cmd, encoding="utf-8").strip()
|
|
||||||
|
|
||||||
print(f"🎉 {binary} {version}")
|
|
||||||
|
|
||||||
|
|
||||||
def find_wasi_sdk():
|
|
||||||
"""Find the path to wasi-sdk."""
|
|
||||||
if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"):
|
|
||||||
return pathlib.Path(wasi_sdk_path)
|
|
||||||
elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists():
|
|
||||||
return default_path
|
|
||||||
|
|
||||||
|
|
||||||
def wasi_sdk_env(context):
|
|
||||||
"""Calculate environment variables for building with wasi-sdk."""
|
|
||||||
wasi_sdk_path = context.wasi_sdk_path
|
|
||||||
sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
|
|
||||||
env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++",
|
|
||||||
"AR": "llvm-ar", "RANLIB": "ranlib"}
|
|
||||||
|
|
||||||
for env_var, binary_name in list(env.items()):
|
|
||||||
env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
|
|
||||||
|
|
||||||
if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"):
|
|
||||||
for compiler in ["CC", "CPP", "CXX"]:
|
|
||||||
env[compiler] += f" --sysroot={sysroot}"
|
|
||||||
|
|
||||||
env["PKG_CONFIG_PATH"] = ""
|
|
||||||
env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
|
|
||||||
map(os.fsdecode,
|
|
||||||
[sysroot / "lib" / "pkgconfig",
|
|
||||||
sysroot / "share" / "pkgconfig"]))
|
|
||||||
env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
|
|
||||||
|
|
||||||
env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
|
|
||||||
env["WASI_SYSROOT"] = os.fsdecode(sysroot)
|
|
||||||
|
|
||||||
env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"),
|
|
||||||
os.environ["PATH"]])
|
|
||||||
|
|
||||||
return env
|
|
||||||
|
|
||||||
|
|
||||||
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
|
|
||||||
def configure_wasi_python(context, working_dir):
|
|
||||||
"""Configure the WASI/host build."""
|
|
||||||
if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
|
|
||||||
raise ValueError("WASI-SDK not found; "
|
|
||||||
"download from "
|
|
||||||
"https://github.com/WebAssembly/wasi-sdk and/or "
|
|
||||||
"specify via $WASI_SDK_PATH or --wasi-sdk")
|
|
||||||
|
|
||||||
config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi")
|
|
||||||
|
|
||||||
wasi_build_dir = working_dir.relative_to(CHECKOUT)
|
|
||||||
|
|
||||||
python_build_dir = BUILD_DIR / "build"
|
|
||||||
lib_dirs = list(python_build_dir.glob("lib.*"))
|
|
||||||
assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}"
|
|
||||||
lib_dir = os.fsdecode(lib_dirs[0])
|
|
||||||
python_version = lib_dir.rpartition("-")[-1]
|
|
||||||
sysconfig_data_dir = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
|
|
||||||
|
|
||||||
# Use PYTHONPATH to include sysconfig data which must be anchored to the
|
|
||||||
# WASI guest's `/` directory.
|
|
||||||
args = {"GUEST_DIR": "/",
|
|
||||||
"HOST_DIR": CHECKOUT,
|
|
||||||
"ENV_VAR_NAME": "PYTHONPATH",
|
|
||||||
"ENV_VAR_VALUE": f"/{sysconfig_data_dir}",
|
|
||||||
"PYTHON_WASM": working_dir / "python.wasm"}
|
|
||||||
# Check dynamically for wasmtime in case it was specified manually via
|
|
||||||
# `--host-runner`.
|
|
||||||
if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
|
|
||||||
if wasmtime := shutil.which("wasmtime"):
|
|
||||||
args[WASMTIME_VAR_NAME] = wasmtime
|
|
||||||
else:
|
|
||||||
raise FileNotFoundError("wasmtime not found; download from "
|
|
||||||
"https://github.com/bytecodealliance/wasmtime")
|
|
||||||
host_runner = context.host_runner.format_map(args)
|
|
||||||
env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
|
|
||||||
build_python = os.fsdecode(build_python_path())
|
|
||||||
# The path to `configure` MUST be relative, else `python.wasm` is unable
|
|
||||||
# to find the stdlib due to Python not recognizing that it's being
|
|
||||||
# executed from within a checkout.
|
|
||||||
configure = [os.path.relpath(CHECKOUT / 'configure', working_dir),
|
|
||||||
f"--host={context.host_triple}",
|
|
||||||
f"--build={build_platform()}",
|
|
||||||
f"--with-build-python={build_python}"]
|
|
||||||
if build_python_is_pydebug():
|
|
||||||
configure.append("--with-pydebug")
|
|
||||||
if context.args:
|
|
||||||
configure.extend(context.args)
|
|
||||||
call(configure,
|
|
||||||
env=updated_env(env_additions | wasi_sdk_env(context)),
|
|
||||||
quiet=context.quiet)
|
|
||||||
|
|
||||||
python_wasm = working_dir / "python.wasm"
|
|
||||||
exec_script = working_dir / "python.sh"
|
|
||||||
with exec_script.open("w", encoding="utf-8") as file:
|
|
||||||
file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
|
|
||||||
exec_script.chmod(0o755)
|
|
||||||
print(f"🏃♀️ Created {exec_script} ... ")
|
|
||||||
sys.stdout.flush()
|
|
||||||
|
|
||||||
|
|
||||||
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
|
|
||||||
def make_wasi_python(context, working_dir):
|
|
||||||
"""Run `make` for the WASI/host build."""
|
|
||||||
call(["make", "--jobs", str(cpu_count()), "all"],
|
|
||||||
env=updated_env(),
|
|
||||||
quiet=context.quiet)
|
|
||||||
|
|
||||||
exec_script = working_dir / "python.sh"
|
|
||||||
subprocess.check_call([exec_script, "--version"])
|
|
||||||
print(
|
|
||||||
f"🎉 Use '{exec_script.relative_to(context.init_dir)}' "
|
|
||||||
"to run CPython in wasm runtime"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def build_all(context):
|
|
||||||
"""Build everything."""
|
|
||||||
steps = [configure_build_python, make_build_python, configure_wasi_python,
|
|
||||||
make_wasi_python]
|
|
||||||
for step in steps:
|
|
||||||
step(context)
|
|
||||||
|
|
||||||
def clean_contents(context):
|
|
||||||
"""Delete all files created by this script."""
|
|
||||||
if CROSS_BUILD_DIR.exists():
|
|
||||||
print(f"🧹 Deleting {CROSS_BUILD_DIR} ...")
|
|
||||||
shutil.rmtree(CROSS_BUILD_DIR)
|
|
||||||
|
|
||||||
if LOCAL_SETUP.exists():
|
|
||||||
with LOCAL_SETUP.open("rb") as file:
|
|
||||||
if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER:
|
|
||||||
print(f"🧹 Deleting generated {LOCAL_SETUP} ...")
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run "
|
|
||||||
# Make sure the stack size will work for a pydebug
|
|
||||||
# build.
|
|
||||||
# Use 16 MiB stack.
|
|
||||||
"--wasm max-wasm-stack=16777216 "
|
|
||||||
# Enable thread support; causes use of preview1.
|
|
||||||
#"--wasm threads=y --wasi threads=y "
|
|
||||||
# Map the checkout to / to load the stdlib from /Lib.
|
|
||||||
"--dir {HOST_DIR}::{GUEST_DIR} "
|
|
||||||
# Set PYTHONPATH to the sysconfig data.
|
|
||||||
"--env {ENV_VAR_NAME}={ENV_VAR_VALUE}")
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
subcommands = parser.add_subparsers(dest="subcommand")
|
|
||||||
build = subcommands.add_parser("build", help="Build everything")
|
|
||||||
configure_build = subcommands.add_parser("configure-build-python",
|
|
||||||
help="Run `configure` for the "
|
|
||||||
"build Python")
|
|
||||||
make_build = subcommands.add_parser("make-build-python",
|
|
||||||
help="Run `make` for the build Python")
|
|
||||||
configure_host = subcommands.add_parser("configure-host",
|
|
||||||
help="Run `configure` for the "
|
|
||||||
"host/WASI (pydebug builds "
|
|
||||||
"are inferred from the build "
|
|
||||||
"Python)")
|
|
||||||
make_host = subcommands.add_parser("make-host",
|
|
||||||
help="Run `make` for the host/WASI")
|
|
||||||
clean = subcommands.add_parser("clean", help="Delete files and directories "
|
|
||||||
"created by this script")
|
|
||||||
for subcommand in build, configure_build, make_build, configure_host, make_host:
|
|
||||||
subcommand.add_argument("--quiet", action="store_true", default=False,
|
|
||||||
dest="quiet",
|
|
||||||
help="Redirect output from subprocesses to a log file")
|
|
||||||
for subcommand in configure_build, configure_host:
|
|
||||||
subcommand.add_argument("--clean", action="store_true", default=False,
|
|
||||||
dest="clean",
|
|
||||||
help="Delete any relevant directories before building")
|
|
||||||
for subcommand in build, configure_build, configure_host:
|
|
||||||
subcommand.add_argument("args", nargs="*",
|
|
||||||
help="Extra arguments to pass to `configure`")
|
|
||||||
for subcommand in build, configure_host:
|
|
||||||
subcommand.add_argument("--wasi-sdk", type=pathlib.Path,
|
|
||||||
dest="wasi_sdk_path",
|
|
||||||
default=find_wasi_sdk(),
|
|
||||||
help="Path to wasi-sdk; defaults to "
|
|
||||||
"$WASI_SDK_PATH or /opt/wasi-sdk")
|
|
||||||
subcommand.add_argument("--host-runner", action="store",
|
|
||||||
default=default_host_runner, dest="host_runner",
|
|
||||||
help="Command template for running the WASI host "
|
|
||||||
"(default designed for wasmtime 14 or newer: "
|
|
||||||
f"`{default_host_runner}`)")
|
|
||||||
for subcommand in build, configure_host, make_host:
|
|
||||||
subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1",
|
|
||||||
help="The target triple for the WASI host build")
|
|
||||||
|
|
||||||
context = parser.parse_args()
|
|
||||||
context.init_dir = pathlib.Path().absolute()
|
|
||||||
|
|
||||||
dispatch = {"configure-build-python": configure_build_python,
|
|
||||||
"make-build-python": make_build_python,
|
|
||||||
"configure-host": configure_wasi_python,
|
|
||||||
"make-host": make_wasi_python,
|
|
||||||
"build": build_all,
|
|
||||||
"clean": clean_contents}
|
|
||||||
dispatch[context.subcommand](context)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
import pathlib
|
||||||
|
import runpy
|
||||||
|
import sys
|
||||||
|
|
||||||
|
print("⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; "
|
||||||
|
"execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n",
|
||||||
|
file=sys.stderr)
|
||||||
|
|
||||||
|
runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__")
|
||||||
|
|
|
||||||
374
Tools/wasm/wasi/__main__.py
Normal file
374
Tools/wasm/wasi/__main__.py
Normal file
|
|
@ -0,0 +1,374 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import contextlib
|
||||||
|
import functools
|
||||||
|
import os
|
||||||
|
try:
|
||||||
|
from os import process_cpu_count as cpu_count
|
||||||
|
except ImportError:
|
||||||
|
from os import cpu_count
|
||||||
|
import pathlib
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import sysconfig
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
|
||||||
|
CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent
|
||||||
|
assert (CHECKOUT / "configure").is_file(), "Please update the location of the file"
|
||||||
|
|
||||||
|
CROSS_BUILD_DIR = CHECKOUT / "cross-build"
|
||||||
|
BUILD_DIR = CROSS_BUILD_DIR / "build"
|
||||||
|
|
||||||
|
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
|
||||||
|
LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8")
|
||||||
|
|
||||||
|
WASMTIME_VAR_NAME = "WASMTIME"
|
||||||
|
WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
|
||||||
|
|
||||||
|
|
||||||
|
def updated_env(updates={}):
|
||||||
|
"""Create a new dict representing the environment to use.
|
||||||
|
|
||||||
|
The changes made to the execution environment are printed out.
|
||||||
|
"""
|
||||||
|
env_defaults = {}
|
||||||
|
# https://reproducible-builds.org/docs/source-date-epoch/
|
||||||
|
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
|
||||||
|
try:
|
||||||
|
epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip()
|
||||||
|
env_defaults["SOURCE_DATE_EPOCH"] = epoch
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
pass # Might be building from a tarball.
|
||||||
|
# This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
|
||||||
|
environment = env_defaults | os.environ | updates
|
||||||
|
|
||||||
|
env_diff = {}
|
||||||
|
for key, value in environment.items():
|
||||||
|
if os.environ.get(key) != value:
|
||||||
|
env_diff[key] = value
|
||||||
|
|
||||||
|
print("🌎 Environment changes:")
|
||||||
|
for key in sorted(env_diff.keys()):
|
||||||
|
print(f" {key}={env_diff[key]}")
|
||||||
|
|
||||||
|
return environment
|
||||||
|
|
||||||
|
|
||||||
|
def subdir(working_dir, *, clean_ok=False):
|
||||||
|
"""Decorator to change to a working directory."""
|
||||||
|
def decorator(func):
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrapper(context):
|
||||||
|
nonlocal working_dir
|
||||||
|
|
||||||
|
if callable(working_dir):
|
||||||
|
working_dir = working_dir(context)
|
||||||
|
try:
|
||||||
|
tput_output = subprocess.check_output(["tput", "cols"],
|
||||||
|
encoding="utf-8")
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
terminal_width = 80
|
||||||
|
else:
|
||||||
|
terminal_width = int(tput_output.strip())
|
||||||
|
print("⎯" * terminal_width)
|
||||||
|
print("📁", working_dir)
|
||||||
|
if (clean_ok and getattr(context, "clean", False) and
|
||||||
|
working_dir.exists()):
|
||||||
|
print(f"🚮 Deleting directory (--clean)...")
|
||||||
|
shutil.rmtree(working_dir)
|
||||||
|
|
||||||
|
working_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
with contextlib.chdir(working_dir):
|
||||||
|
return func(context, working_dir)
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
return decorator
|
||||||
|
|
||||||
|
|
||||||
|
def call(command, *, quiet, **kwargs):
|
||||||
|
"""Execute a command.
|
||||||
|
|
||||||
|
If 'quiet' is true, then redirect stdout and stderr to a temporary file.
|
||||||
|
"""
|
||||||
|
print("❯", " ".join(map(str, command)))
|
||||||
|
if not quiet:
|
||||||
|
stdout = None
|
||||||
|
stderr = None
|
||||||
|
else:
|
||||||
|
stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8",
|
||||||
|
delete=False,
|
||||||
|
prefix="cpython-wasi-",
|
||||||
|
suffix=".log")
|
||||||
|
stderr = subprocess.STDOUT
|
||||||
|
print(f"📝 Logging output to {stdout.name} (--quiet)...")
|
||||||
|
|
||||||
|
subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def build_platform():
|
||||||
|
"""The name of the build/host platform."""
|
||||||
|
# Can also be found via `config.guess`.
|
||||||
|
return sysconfig.get_config_var("BUILD_GNU_TYPE")
|
||||||
|
|
||||||
|
|
||||||
|
def build_python_path():
|
||||||
|
"""The path to the build Python binary."""
|
||||||
|
binary = BUILD_DIR / "python"
|
||||||
|
if not binary.is_file():
|
||||||
|
binary = binary.with_suffix(".exe")
|
||||||
|
if not binary.is_file():
|
||||||
|
raise FileNotFoundError("Unable to find `python(.exe)` in "
|
||||||
|
f"{BUILD_DIR}")
|
||||||
|
|
||||||
|
return binary
|
||||||
|
|
||||||
|
|
||||||
|
def build_python_is_pydebug():
|
||||||
|
"""Find out if the build Python is a pydebug build."""
|
||||||
|
test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
|
||||||
|
result = subprocess.run([build_python_path(), "-c", test],
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE)
|
||||||
|
return bool(result.returncode)
|
||||||
|
|
||||||
|
|
||||||
|
@subdir(BUILD_DIR, clean_ok=True)
|
||||||
|
def configure_build_python(context, working_dir):
|
||||||
|
"""Configure the build/host Python."""
|
||||||
|
if LOCAL_SETUP.exists():
|
||||||
|
print(f"👍 {LOCAL_SETUP} exists ...")
|
||||||
|
else:
|
||||||
|
print(f"📝 Touching {LOCAL_SETUP} ...")
|
||||||
|
LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
|
||||||
|
|
||||||
|
configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)]
|
||||||
|
if context.args:
|
||||||
|
configure.extend(context.args)
|
||||||
|
|
||||||
|
call(configure, quiet=context.quiet)
|
||||||
|
|
||||||
|
|
||||||
|
@subdir(BUILD_DIR)
|
||||||
|
def make_build_python(context, working_dir):
|
||||||
|
"""Make/build the build Python."""
|
||||||
|
call(["make", "--jobs", str(cpu_count()), "all"],
|
||||||
|
quiet=context.quiet)
|
||||||
|
|
||||||
|
binary = build_python_path()
|
||||||
|
cmd = [binary, "-c",
|
||||||
|
"import sys; "
|
||||||
|
"print(f'{sys.version_info.major}.{sys.version_info.minor}')"]
|
||||||
|
version = subprocess.check_output(cmd, encoding="utf-8").strip()
|
||||||
|
|
||||||
|
print(f"🎉 {binary} {version}")
|
||||||
|
|
||||||
|
|
||||||
|
def find_wasi_sdk():
|
||||||
|
"""Find the path to wasi-sdk."""
|
||||||
|
if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"):
|
||||||
|
return pathlib.Path(wasi_sdk_path)
|
||||||
|
elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists():
|
||||||
|
return default_path
|
||||||
|
|
||||||
|
|
||||||
|
def wasi_sdk_env(context):
|
||||||
|
"""Calculate environment variables for building with wasi-sdk."""
|
||||||
|
wasi_sdk_path = context.wasi_sdk_path
|
||||||
|
sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
|
||||||
|
env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++",
|
||||||
|
"AR": "llvm-ar", "RANLIB": "ranlib"}
|
||||||
|
|
||||||
|
for env_var, binary_name in list(env.items()):
|
||||||
|
env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
|
||||||
|
|
||||||
|
if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"):
|
||||||
|
for compiler in ["CC", "CPP", "CXX"]:
|
||||||
|
env[compiler] += f" --sysroot={sysroot}"
|
||||||
|
|
||||||
|
env["PKG_CONFIG_PATH"] = ""
|
||||||
|
env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
|
||||||
|
map(os.fsdecode,
|
||||||
|
[sysroot / "lib" / "pkgconfig",
|
||||||
|
sysroot / "share" / "pkgconfig"]))
|
||||||
|
env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
|
||||||
|
|
||||||
|
env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
|
||||||
|
env["WASI_SYSROOT"] = os.fsdecode(sysroot)
|
||||||
|
|
||||||
|
env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"),
|
||||||
|
os.environ["PATH"]])
|
||||||
|
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
|
||||||
|
def configure_wasi_python(context, working_dir):
|
||||||
|
"""Configure the WASI/host build."""
|
||||||
|
if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
|
||||||
|
raise ValueError("WASI-SDK not found; "
|
||||||
|
"download from "
|
||||||
|
"https://github.com/WebAssembly/wasi-sdk and/or "
|
||||||
|
"specify via $WASI_SDK_PATH or --wasi-sdk")
|
||||||
|
|
||||||
|
config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi")
|
||||||
|
|
||||||
|
wasi_build_dir = working_dir.relative_to(CHECKOUT)
|
||||||
|
|
||||||
|
python_build_dir = BUILD_DIR / "build"
|
||||||
|
lib_dirs = list(python_build_dir.glob("lib.*"))
|
||||||
|
assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}"
|
||||||
|
lib_dir = os.fsdecode(lib_dirs[0])
|
||||||
|
python_version = lib_dir.rpartition("-")[-1]
|
||||||
|
sysconfig_data_dir = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
|
||||||
|
|
||||||
|
# Use PYTHONPATH to include sysconfig data which must be anchored to the
|
||||||
|
# WASI guest's `/` directory.
|
||||||
|
args = {"GUEST_DIR": "/",
|
||||||
|
"HOST_DIR": CHECKOUT,
|
||||||
|
"ENV_VAR_NAME": "PYTHONPATH",
|
||||||
|
"ENV_VAR_VALUE": f"/{sysconfig_data_dir}",
|
||||||
|
"PYTHON_WASM": working_dir / "python.wasm"}
|
||||||
|
# Check dynamically for wasmtime in case it was specified manually via
|
||||||
|
# `--host-runner`.
|
||||||
|
if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
|
||||||
|
if wasmtime := shutil.which("wasmtime"):
|
||||||
|
args[WASMTIME_VAR_NAME] = wasmtime
|
||||||
|
else:
|
||||||
|
raise FileNotFoundError("wasmtime not found; download from "
|
||||||
|
"https://github.com/bytecodealliance/wasmtime")
|
||||||
|
host_runner = context.host_runner.format_map(args)
|
||||||
|
env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
|
||||||
|
build_python = os.fsdecode(build_python_path())
|
||||||
|
# The path to `configure` MUST be relative, else `python.wasm` is unable
|
||||||
|
# to find the stdlib due to Python not recognizing that it's being
|
||||||
|
# executed from within a checkout.
|
||||||
|
configure = [os.path.relpath(CHECKOUT / 'configure', working_dir),
|
||||||
|
f"--host={context.host_triple}",
|
||||||
|
f"--build={build_platform()}",
|
||||||
|
f"--with-build-python={build_python}"]
|
||||||
|
if build_python_is_pydebug():
|
||||||
|
configure.append("--with-pydebug")
|
||||||
|
if context.args:
|
||||||
|
configure.extend(context.args)
|
||||||
|
call(configure,
|
||||||
|
env=updated_env(env_additions | wasi_sdk_env(context)),
|
||||||
|
quiet=context.quiet)
|
||||||
|
|
||||||
|
python_wasm = working_dir / "python.wasm"
|
||||||
|
exec_script = working_dir / "python.sh"
|
||||||
|
with exec_script.open("w", encoding="utf-8") as file:
|
||||||
|
file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
|
||||||
|
exec_script.chmod(0o755)
|
||||||
|
print(f"🏃♀️ Created {exec_script} (--host-runner)... ")
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
|
||||||
|
def make_wasi_python(context, working_dir):
|
||||||
|
"""Run `make` for the WASI/host build."""
|
||||||
|
call(["make", "--jobs", str(cpu_count()), "all"],
|
||||||
|
env=updated_env(),
|
||||||
|
quiet=context.quiet)
|
||||||
|
|
||||||
|
exec_script = working_dir / "python.sh"
|
||||||
|
call([exec_script, "--version"], quiet=False)
|
||||||
|
print(
|
||||||
|
f"🎉 Use `{exec_script.relative_to(context.init_dir)}` "
|
||||||
|
"to run CPython w/ the WASI host specified by --host-runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_all(context):
|
||||||
|
"""Build everything."""
|
||||||
|
steps = [configure_build_python, make_build_python, configure_wasi_python,
|
||||||
|
make_wasi_python]
|
||||||
|
for step in steps:
|
||||||
|
step(context)
|
||||||
|
|
||||||
|
def clean_contents(context):
|
||||||
|
"""Delete all files created by this script."""
|
||||||
|
if CROSS_BUILD_DIR.exists():
|
||||||
|
print(f"🧹 Deleting {CROSS_BUILD_DIR} ...")
|
||||||
|
shutil.rmtree(CROSS_BUILD_DIR)
|
||||||
|
|
||||||
|
if LOCAL_SETUP.exists():
|
||||||
|
with LOCAL_SETUP.open("rb") as file:
|
||||||
|
if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER:
|
||||||
|
print(f"🧹 Deleting generated {LOCAL_SETUP} ...")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run "
|
||||||
|
# Make sure the stack size will work for a pydebug
|
||||||
|
# build.
|
||||||
|
# Use 16 MiB stack.
|
||||||
|
"--wasm max-wasm-stack=16777216 "
|
||||||
|
# Enable thread support; causes use of preview1.
|
||||||
|
#"--wasm threads=y --wasi threads=y "
|
||||||
|
# Map the checkout to / to load the stdlib from /Lib.
|
||||||
|
"--dir {HOST_DIR}::{GUEST_DIR} "
|
||||||
|
# Set PYTHONPATH to the sysconfig data.
|
||||||
|
"--env {ENV_VAR_NAME}={ENV_VAR_VALUE}")
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
subcommands = parser.add_subparsers(dest="subcommand")
|
||||||
|
build = subcommands.add_parser("build", help="Build everything")
|
||||||
|
configure_build = subcommands.add_parser("configure-build-python",
|
||||||
|
help="Run `configure` for the "
|
||||||
|
"build Python")
|
||||||
|
make_build = subcommands.add_parser("make-build-python",
|
||||||
|
help="Run `make` for the build Python")
|
||||||
|
configure_host = subcommands.add_parser("configure-host",
|
||||||
|
help="Run `configure` for the "
|
||||||
|
"host/WASI (pydebug builds "
|
||||||
|
"are inferred from the build "
|
||||||
|
"Python)")
|
||||||
|
make_host = subcommands.add_parser("make-host",
|
||||||
|
help="Run `make` for the host/WASI")
|
||||||
|
clean = subcommands.add_parser("clean", help="Delete files and directories "
|
||||||
|
"created by this script")
|
||||||
|
for subcommand in build, configure_build, make_build, configure_host, make_host:
|
||||||
|
subcommand.add_argument("--quiet", action="store_true", default=False,
|
||||||
|
dest="quiet",
|
||||||
|
help="Redirect output from subprocesses to a log file")
|
||||||
|
for subcommand in configure_build, configure_host:
|
||||||
|
subcommand.add_argument("--clean", action="store_true", default=False,
|
||||||
|
dest="clean",
|
||||||
|
help="Delete any relevant directories before building")
|
||||||
|
for subcommand in build, configure_build, configure_host:
|
||||||
|
subcommand.add_argument("args", nargs="*",
|
||||||
|
help="Extra arguments to pass to `configure`")
|
||||||
|
for subcommand in build, configure_host:
|
||||||
|
subcommand.add_argument("--wasi-sdk", type=pathlib.Path,
|
||||||
|
dest="wasi_sdk_path",
|
||||||
|
default=find_wasi_sdk(),
|
||||||
|
help="Path to wasi-sdk; defaults to "
|
||||||
|
"$WASI_SDK_PATH or /opt/wasi-sdk")
|
||||||
|
subcommand.add_argument("--host-runner", action="store",
|
||||||
|
default=default_host_runner, dest="host_runner",
|
||||||
|
help="Command template for running the WASI host "
|
||||||
|
"(default designed for wasmtime 14 or newer: "
|
||||||
|
f"`{default_host_runner}`)")
|
||||||
|
for subcommand in build, configure_host, make_host:
|
||||||
|
subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1",
|
||||||
|
help="The target triple for the WASI host build")
|
||||||
|
|
||||||
|
context = parser.parse_args()
|
||||||
|
context.init_dir = pathlib.Path().absolute()
|
||||||
|
|
||||||
|
dispatch = {"configure-build-python": configure_build_python,
|
||||||
|
"make-build-python": make_build_python,
|
||||||
|
"configure-host": configure_wasi_python,
|
||||||
|
"make-host": make_wasi_python,
|
||||||
|
"build": build_all,
|
||||||
|
"clean": clean_contents}
|
||||||
|
dispatch[context.subcommand](context)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Loading…
Add table
Add a link
Reference in a new issue