mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-07 19:35:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

GH-114809: Add support for macOS multi-arch builds with the JIT enabled (#131751)

Browse source 
Co-authored-by: Ronald Oussoren <ronaldoussoren@mac.com>
Co-authored-by: Brandt Bucher <brandtbucher@microsoft.com>
This commit is contained in:
Savannah Ostrowski 2025-04-30 11:03:57 -07:00 • committed by GitHub
parent 2b67db7ce3
commit 26c0248b54
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 152 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/jit.yml vendored
View file

@ -113,7 +113,7 @@ jobs:
find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
brew install llvm@${{ matrix.llvm }}
export SDKROOT="$(xcrun --show-sdk-path)"
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }}
./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }}
make all --jobs 4
./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3

2
.gitignore vendored
View file

@ -138,7 +138,7 @@ Tools/unicode/data/
# hendrikmuhs/ccache-action@v1
/.ccache
/cross-build/
/jit_stencils.h
/jit_stencils*.h
/platform
/profile-clean-stamp
/profile-run-stamp

1
Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst Normal file
View file

@ -0,0 +1 @@
Add support for macOS multi-arch builds with the JIT enabled

34
Tools/jit/_targets.py
View file

@ -25,6 +25,7 @@ TOOLS = TOOLS_JIT.parent
CPYTHON = TOOLS.parent
PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h"
TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c"
ASYNCIO_RUNNER = asyncio.Runner()
_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection)
_R = typing.TypeVar(
@ -35,6 +36,7 @@ _R = typing.TypeVar(
@dataclasses.dataclass
class _Target(typing.Generic[_S, _R]):
triple: str
condition: str
_: dataclasses.KW_ONLY
alignment: int = 1
args: typing.Sequence[str] = ()
@ -188,7 +190,12 @@ class _Target(typing.Generic[_S, _R]):
return stencil_groups
def build(
self, out: pathlib.Path, *, comment: str = "", force: bool = False
self,
out: pathlib.Path,
*,
comment: str = "",
force: bool = False,
stencils_h: str = "jit_stencils.h",
) -> None:
"""Build jit_stencils.h in the given directory."""
if not self.stable:
@ -197,14 +204,14 @@ class _Target(typing.Generic[_S, _R]):
outline = "=" * len(warning)
print("\n".join(["", outline, warning, request, outline, ""]))
digest = f"// {self._compute_digest(out)}\n"
jit_stencils = out / "jit_stencils.h"
jit_stencils = out / stencils_h
if (
not force
and jit_stencils.exists()
and jit_stencils.read_text().startswith(digest)
):
return
stencil_groups = asyncio.run(self._build_stencils())
stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils())
jit_stencils_new = out / "jit_stencils.h.new"
try:
with jit_stencils_new.open("w") as file:
@ -512,10 +519,12 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
"""Build a _Target for the given host "triple" and options."""
target: _COFF | _ELF | _MachO
if re.fullmatch(r"aarch64-apple-darwin.*", host):
target = _MachO(host, alignment=8, prefix="_")
condition = "defined(__aarch64__) && defined(__APPLE__)"
target = _MachO(host, condition, alignment=8, prefix="_")
elif re.fullmatch(r"aarch64-pc-windows-msvc", host):
args = ["-fms-runtime-lib=dll", "-fplt"]
target = _COFF(host, alignment=8, args=args)
condition = "defined(_M_ARM64)"
target = _COFF(host, condition, alignment=8, args=args)
elif re.fullmatch(r"aarch64-.*-linux-gnu", host):
args = [
"-fpic",
@ -523,22 +532,27 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
# was required to disable them.
"-mno-outline-atomics",
]
target = _ELF(host, alignment=8, args=args)
condition = "defined(__aarch64__) && defined(__linux__)"
target = _ELF(host, condition, alignment=8, args=args)
elif re.fullmatch(r"i686-pc-windows-msvc", host):
args = [
"-DPy_NO_ENABLE_SHARED",
# __attribute__((preserve_none)) is not supported
"-Wno-ignored-attributes",
]
target = _COFF(host, args=args, prefix="_")
condition = "defined(_M_IX86)"
target = _COFF(host, condition, args=args, prefix="_")
elif re.fullmatch(r"x86_64-apple-darwin.*", host):
target = _MachO(host, prefix="_")
condition = "defined(__x86_64__) && defined(__APPLE__)"
target = _MachO(host, condition, prefix="_")
elif re.fullmatch(r"x86_64-pc-windows-msvc", host):
args = ["-fms-runtime-lib=dll"]
target = _COFF(host, args=args)
condition = "defined(_M_X64)"
target = _COFF(host, condition, args=args)
elif re.fullmatch(r"x86_64-.*-linux-gnu", host):
args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0"]
target = _ELF(host, args=args)
condition = "defined(__x86_64__) && defined(__linux__)"
target = _ELF(host, condition, args=args)
else:
raise ValueError(host)
return target

27
Tools/jit/build.py
View file

@ -11,7 +11,10 @@ if __name__ == "__main__":
comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}"
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"target", type=_targets.get_target, help="a PEP 11 target triple to compile for"
"target",
nargs="+",
type=_targets.get_target,
help="a PEP 11 target triple to compile for",
)
parser.add_argument(
"-d", "--debug", action="store_true", help="compile for a debug build of Python"
@ -23,6 +26,22 @@ if __name__ == "__main__":
"-v", "--verbose", action="store_true", help="echo commands as they are run"
)
args = parser.parse_args()
args.target.debug = args.debug
args.target.verbose = args.verbose
args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force)
for target in args.target:
target.debug = args.debug
target.force = args.force
target.verbose = args.verbose
target.build(
pathlib.Path.cwd(),
comment=comment,
stencils_h=f"jit_stencils-{target.triple}.h",
force=args.force,
)
with open("jit_stencils.h", "w") as fp:
for idx, target in enumerate(args.target):
fp.write(f"#{'if' if idx == 0 else 'elif'} {target.condition}\n")
fp.write(f'#include "jit_stencils-{target.triple}.h"\n')
fp.write("#else\n")
fp.write('#error "unexpected target"\n')
fp.write("#endif\n")

110
configure generated vendored
View file

@ -900,6 +900,8 @@ LDSHARED
SHLIB_SUFFIX
DSYMUTIL_PATH
DSYMUTIL
JIT_STENCILS_H
REGEN_JIT_COMMAND
UNIVERSAL_ARCH_FLAGS
WASM_STDLIB
WASM_ASSETS_DIR
@ -928,8 +930,6 @@ LLVM_AR
PROFILE_TASK
DEF_MAKE_RULE
DEF_MAKE_ALL_RULE
JIT_STENCILS_H
REGEN_JIT_COMMAND
ABI_THREAD
ABIFLAGS
LN
@ -1092,13 +1092,13 @@ with_pydebug
with_trace_refs
enable_pystats
with_assertions
enable_experimental_jit
enable_optimizations
with_lto
enable_bolt
with_strict_overflow
enable_safety
enable_slower_safety
enable_experimental_jit
with_dsymutil
with_address_sanitizer
with_memory_sanitizer
@ -1823,9 +1823,6 @@ Optional Features:
--disable-gil enable experimental support for running without the
GIL (default is no)
--enable-pystats enable internal statistics gathering (default is no)
--enable-experimental-jit[=no|yes|yes-off|interpreter]
build the experimental just-in-time compiler
(default is no)
--enable-optimizations enable expensive, stable optimizations (PGO, etc.)
(default is no)
--enable-bolt enable usage of the llvm-bolt post-link optimizer
@ -1834,6 +1831,9 @@ Optional Features:
no performance overhead
--enable-slower-safety enable usage of the security compiler options with
performance overhead
--enable-experimental-jit[=no|yes|yes-off|interpreter]
build the experimental just-in-time compiler
(default is no)
--enable-loadable-sqlite-extensions
support loadable extensions in the sqlite3 module,
see Doc/library/sqlite3.rst (default is no)
@ -8367,51 +8367,6 @@ else
printf "%s\n" "no" >&6; }
fi
# Check for --enable-experimental-jit:
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5
printf %s "checking for --enable-experimental-jit... " >&6; }
# Check whether --enable-experimental-jit was given.
if test ${enable_experimental_jit+y}
then :
enableval=$enable_experimental_jit;
else case e in #(
e) enable_experimental_jit=no ;;
esac
fi
case $enable_experimental_jit in
no) jit_flags=""; tier2_flags="" ;;
yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;;
yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;;
interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;;
interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option
*) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;;
esac
if ${tier2_flags:+false} :
then :
else case e in #(
e) as_fn_append CFLAGS_NODIST " $tier2_flags" ;;
esac
fi
if ${jit_flags:+false} :
then :
else case e in #(
e) as_fn_append CFLAGS_NODIST " $jit_flags"
REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"
JIT_STENCILS_H="jit_stencils.h"
if test "x$Py_DEBUG" = xtrue
then :
as_fn_append REGEN_JIT_COMMAND " --debug"
fi ;;
esac
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5
printf "%s\n" "$tier2_flags $jit_flags" >&6; }
# Enable optimization flags
@ -10706,42 +10661,50 @@ fi
UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT=""
ARCH_TRIPLES=`echo {ppc,i386}-apple-darwin`
;;
64-bit)
UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT="true"
ARCH_TRIPLES=`echo {ppc64,x86_64}-apple-darwin`
;;
all)
UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64"
LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386"
ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc"
ARCH_TRIPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin`
;;
universal2)
UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64"
LIPO_32BIT_FLAGS=""
LIPO_INTEL64_FLAGS="-extract x86_64"
ARCH_RUN_32BIT="true"
ARCH_TRIPLES=`echo {aarch64,x86_64}-apple-darwin`
;;
intel)
UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64"
LIPO_32BIT_FLAGS="-extract i386"
ARCH_RUN_32BIT="/usr/bin/arch -i386"
ARCH_TRIPLES=`echo {i386,x86_64}-apple-darwin`
;;
intel-32)
UNIVERSAL_ARCH_FLAGS="-arch i386"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT=""
ARCH_TRIPLES=i386-apple-darwin
;;
intel-64)
UNIVERSAL_ARCH_FLAGS="-arch x86_64"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT="true"
ARCH_TRIPLES=x86_64-apple-darwin
;;
3-way)
UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64"
LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386"
ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc"
ARCH_TRIPLES=`echo {i386,ppc,x86_64}-apple-darwin`
;;
*)
as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5
@ -10858,6 +10821,51 @@ else case e in #(
esac
fi
# Check for --enable-experimental-jit:
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5
printf %s "checking for --enable-experimental-jit... " >&6; }
# Check whether --enable-experimental-jit was given.
if test ${enable_experimental_jit+y}
then :
enableval=$enable_experimental_jit;
else case e in #(
e) enable_experimental_jit=no ;;
esac
fi
case $enable_experimental_jit in
no) jit_flags=""; tier2_flags="" ;;
yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;;
yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;;
interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;;
interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option
*) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;;
esac
if ${tier2_flags:+false} :
then :
else case e in #(
e) as_fn_append CFLAGS_NODIST " $tier2_flags" ;;
esac
fi
if ${jit_flags:+false} :
then :
else case e in #(
e) as_fn_append CFLAGS_NODIST " $jit_flags"
REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}"
JIT_STENCILS_H="jit_stencils.h"
if test "x$Py_DEBUG" = xtrue
then :
as_fn_append REGEN_JIT_COMMAND " --debug"
fi ;;
esac
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5
printf "%s\n" "$tier2_flags $jit_flags" >&6; }
case "$ac_cv_cc_name" in
mpicc)
CFLAGS_NODIST="$CFLAGS_NODIST"

78
configure.ac
View file

@ -1802,41 +1802,6 @@ else
AC_MSG_RESULT([no])
fi
# Check for --enable-experimental-jit:
AC_MSG_CHECKING([for --enable-experimental-jit])
AC_ARG_ENABLE([experimental-jit],
[AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@],
[build the experimental just-in-time compiler (default is no)])],
[],
[enable_experimental_jit=no])
case $enable_experimental_jit in
no) jit_flags=""; tier2_flags="" ;;
yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;;
yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;;
interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;;
interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option
*) AC_MSG_ERROR(
[invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;;
esac
AS_VAR_IF([tier2_flags],
[],
[],
[AS_VAR_APPEND([CFLAGS_NODIST], [" $tier2_flags"])])
AS_VAR_IF([jit_flags],
[],
[],
[AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"])
AS_VAR_SET([REGEN_JIT_COMMAND],
["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"])
AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"])
AS_VAR_IF([Py_DEBUG],
[true],
[AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],
[])])
AC_SUBST([REGEN_JIT_COMMAND])
AC_SUBST([JIT_STENCILS_H])
AC_MSG_RESULT([$tier2_flags $jit_flags])
# Enable optimization flags
AC_SUBST([DEF_MAKE_ALL_RULE])
AC_SUBST([DEF_MAKE_RULE])
@ -2652,42 +2617,50 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [
UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT=""
ARCH_TRIPLES=`echo {ppc,i386}-apple-darwin`
;;
64-bit)
UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT="true"
ARCH_TRIPLES=`echo {ppc64,x86_64}-apple-darwin`
;;
all)
UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64"
LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386"
ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc"
ARCH_TRIPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin`
;;
universal2)
UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64"
LIPO_32BIT_FLAGS=""
LIPO_INTEL64_FLAGS="-extract x86_64"
ARCH_RUN_32BIT="true"
ARCH_TRIPLES=`echo {aarch64,x86_64}-apple-darwin`
;;
intel)
UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64"
LIPO_32BIT_FLAGS="-extract i386"
ARCH_RUN_32BIT="/usr/bin/arch -i386"
ARCH_TRIPLES=`echo {i386,x86_64}-apple-darwin`
;;
intel-32)
UNIVERSAL_ARCH_FLAGS="-arch i386"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT=""
ARCH_TRIPLES=i386-apple-darwin
;;
intel-64)
UNIVERSAL_ARCH_FLAGS="-arch x86_64"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT="true"
ARCH_TRIPLES=x86_64-apple-darwin
;;
3-way)
UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64"
LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386"
ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc"
ARCH_TRIPLES=`echo {i386,ppc,x86_64}-apple-darwin`
;;
*)
AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way])
@ -2778,6 +2751,41 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [
esac
])
# Check for --enable-experimental-jit:
AC_MSG_CHECKING([for --enable-experimental-jit])
AC_ARG_ENABLE([experimental-jit],
[AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@],
[build the experimental just-in-time compiler (default is no)])],
[],
[enable_experimental_jit=no])
case $enable_experimental_jit in
no) jit_flags=""; tier2_flags="" ;;
yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;;
yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;;
interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;;
interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option
*) AC_MSG_ERROR(
[invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;;
esac
AS_VAR_IF([tier2_flags],
[],
[],
[AS_VAR_APPEND([CFLAGS_NODIST], [" $tier2_flags"])])
AS_VAR_IF([jit_flags],
[],
[],
[AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"])
AS_VAR_SET([REGEN_JIT_COMMAND],
["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}"])
AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"])
AS_VAR_IF([Py_DEBUG],
[true],
[AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],
[])])
AC_SUBST([REGEN_JIT_COMMAND])
AC_SUBST([JIT_STENCILS_H])
AC_MSG_RESULT([$tier2_flags $jit_flags])
case "$ac_cv_cc_name" in
mpicc)
CFLAGS_NODIST="$CFLAGS_NODIST"