Add downloading of GraalPy (#13172)

## Summary

This adds GraalPy download metadata so that `uv python install graalpy`
works. See https://github.com/astral-sh/uv/issues/13114

## Test Plan

The existing integration test was changed to test this functionality.
This commit is contained in:
Tim Felgentreff 2025-05-06 18:02:27 +02:00 committed by GitHub
parent 9071e0eeac
commit 878c2acdf3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 416 additions and 34 deletions

View file

@ -985,7 +985,7 @@ jobs:
- name: "Install free-threaded Python via uv"
run: |
./uv python install -v 3.13t
./uv venv -p 3.13t --python-preference only-managed
./uv venv -p 3.13t --managed-python
- name: "Check version"
run: |
@ -1025,7 +1025,7 @@ jobs:
- name: "Create a virtual environment (uv)"
run: |
./uv venv -p 3.13t --python-preference only-managed
./uv venv -p 3.13t --managed-python
- name: "Check version (uv)"
run: |
@ -1065,7 +1065,7 @@ jobs:
- name: "Create a virtual environment"
run: |
./uv venv -p pypy3.9 --python-preference only-managed
./uv venv -p pypy3.9 --managed-python
- name: "Check for executables"
run: |
@ -1125,7 +1125,7 @@ jobs:
- name: "Create a virtual environment"
run: |
.\uv.exe venv -p pypy3.9 --python-preference only-managed
.\uv.exe venv -p pypy3.9 --managed-python
- name: "Check for executables"
shell: python
@ -1176,10 +1176,6 @@ jobs:
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "graalpy24.1"
- name: "Download binary"
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
@ -1188,13 +1184,12 @@ jobs:
- name: "Prepare binary"
run: chmod +x ./uv
- name: Graalpy info
run: |
which graalpy
- name: "Install GraalPy"
run: ./uv python install -v graalpy
- name: "Create a virtual environment"
run: |
./uv venv -p $(which graalpy)
./uv venv -p graalpy --managed-python
- name: "Check for executables"
run: |
@ -1244,22 +1239,17 @@ jobs:
runs-on: windows-latest
steps:
- uses: timfel/setup-python@fc9bcb4a04f5b1ea7d678c2ca7ea1c479a2468d7
with:
python-version: "graalpy24.1"
- name: "Download binary"
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: uv-windows-x86_64-${{ github.sha }}
- name: Graalpy info
run: Get-Command graalpy
- name: "Install GraalPy"
run: .\uv.exe python install graalpy
- name: "Create a virtual environment"
run: |
$graalpy = (Get-Command graalpy).source
.\uv.exe venv -p $graalpy
.\uv.exe venv -p graalpy --managed-python
- name: "Check for executables"
shell: python
@ -1742,6 +1732,32 @@ jobs:
- name: "Validate global Python install"
run: python3 scripts/check_system_python.py --uv ./uv
system-test-graalpy:
timeout-minutes: 10
needs: build-binary-linux-libc
name: "check system | graalpy on ubuntu"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "graalpy24.1"
- name: "Download binary"
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
name: uv-linux-libc-${{ github.sha }}
- name: "Prepare binary"
run: chmod +x ./uv
- name: "Print Python path"
run: echo $(which graalpy)
- name: "Validate global Python install"
run: graalpy scripts/check_system_python.py --uv ./uv
system-test-pypy:
timeout-minutes: 10
needs: build-binary-linux-libc

View file

@ -27822,5 +27822,261 @@
"url": "https://downloads.python.org/pypy/pypy3.8-v7.3.8-win64.zip",
"sha256": "0894c468e7de758c509a602a28ef0ba4fbf197ccdf946c7853a7283d9bb2a345",
"variant": null
},
"graalpy-3.11.0-darwin-aarch64-none": {
"name": "graalpy",
"arch": {
"family": "aarch64",
"variant": null
},
"os": "darwin",
"libc": "none",
"major": 3,
"minor": 11,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.2.1/graalpy-24.2.1-macos-aarch64.tar.gz",
"sha256": "61e11d5176d5bb709b919979ef3525f4db1e39c404b59aa54d887f56bf8fab44",
"variant": null
},
"graalpy-3.11.0-darwin-x86_64-none": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "darwin",
"libc": "none",
"major": 3,
"minor": 11,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.2.1/graalpy-24.2.1-macos-amd64.tar.gz",
"sha256": "4bc42b36117c9ab09c4f411ec5a7a85ed58521dd20b529d971bb0ed3d0b7c363",
"variant": null
},
"graalpy-3.11.0-linux-aarch64-gnu": {
"name": "graalpy",
"arch": {
"family": "aarch64",
"variant": null
},
"os": "linux",
"libc": "gnu",
"major": 3,
"minor": 11,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.2.1/graalpy-24.2.1-linux-aarch64.tar.gz",
"sha256": "2a80800a76ee6b737d6458ba9ab30ce386dfdd5b2b2bec3ee6bc51fd8e51e7c2",
"variant": null
},
"graalpy-3.11.0-linux-x86_64-gnu": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "linux",
"libc": "gnu",
"major": 3,
"minor": 11,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.2.1/graalpy-24.2.1-linux-amd64.tar.gz",
"sha256": "55872af24819cb99efa2338db057aeda0c8f9dd412a4a6f5ea19b256ee82fd9e",
"variant": null
},
"graalpy-3.11.0-windows-x86_64-none": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "windows",
"libc": "none",
"major": 3,
"minor": 11,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.2.1/graalpy-24.2.1-windows-amd64.zip",
"sha256": "bad923fb64fa2fc71bb424818aac8dcfe0cc9554abef5235d7c08e597ed778ae",
"variant": null
},
"graalpy-3.10.0-darwin-aarch64-none": {
"name": "graalpy",
"arch": {
"family": "aarch64",
"variant": null
},
"os": "darwin",
"libc": "none",
"major": 3,
"minor": 10,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.0.2/graalpy-24.0.2-macos-aarch64.tar.gz",
"sha256": "568f84b77865f5952b456840e8fa843811e0c32553a2ce777c7b460ad305f3e5",
"variant": null
},
"graalpy-3.10.0-darwin-x86_64-none": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "darwin",
"libc": "none",
"major": 3,
"minor": 10,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.0.2/graalpy-24.0.2-macos-amd64.tar.gz",
"sha256": "6fe7c46c9e4f958217f576afcab8bd65ad4fb7daabf2d25353ab7a9ca45c01d2",
"variant": null
},
"graalpy-3.10.0-linux-aarch64-gnu": {
"name": "graalpy",
"arch": {
"family": "aarch64",
"variant": null
},
"os": "linux",
"libc": "gnu",
"major": 3,
"minor": 10,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.0.2/graalpy-24.0.2-linux-aarch64.tar.gz",
"sha256": "cd7e17bb0a72aefbd3dbc81c340b20d1ab080a7072ccfa9568658bdc6152911f",
"variant": null
},
"graalpy-3.10.0-linux-x86_64-gnu": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "linux",
"libc": "gnu",
"major": 3,
"minor": 10,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.0.2/graalpy-24.0.2-linux-amd64.tar.gz",
"sha256": "510aa284d258e308bfa4d9df440f7739a2cf977cb9d2a0879269d9bbe485e5a4",
"variant": null
},
"graalpy-3.10.0-windows-x86_64-none": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "windows",
"libc": "none",
"major": 3,
"minor": 10,
"patch": 0,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/graal-24.0.2/graalpy-24.0.2-windows-amd64.zip",
"sha256": "eb82db48b43e040ca9b906a00a746dcb6c848f9cb5d0a1a6314224d478568538",
"variant": null
},
"graalpy-3.8.5-darwin-aarch64-none": {
"name": "graalpy",
"arch": {
"family": "aarch64",
"variant": null
},
"os": "darwin",
"libc": "none",
"major": 3,
"minor": 8,
"patch": 5,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/vm-22.3.1/graalpy-22.3.1-macos-aarch64.tar.gz",
"sha256": "01721ddd56094a185403099c0230f3bf1eeb5abbcc96dd3198b193da763329ab",
"variant": null
},
"graalpy-3.8.5-darwin-x86_64-none": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "darwin",
"libc": "none",
"major": 3,
"minor": 8,
"patch": 5,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/vm-22.3.1/graalpy-22.3.1-macos-amd64.tar.gz",
"sha256": "24af2f441082fad2aa553cde1a1d6356d6c5ca0a6791f7910512593dcd909d09",
"variant": null
},
"graalpy-3.8.5-linux-aarch64-gnu": {
"name": "graalpy",
"arch": {
"family": "aarch64",
"variant": null
},
"os": "linux",
"libc": "gnu",
"major": 3,
"minor": 8,
"patch": 5,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/vm-22.3.1/graalpy-22.3.1-linux-aarch64.tar.gz",
"sha256": "e051246c5a123fe8180fdfb072843224d54bb8b859533d275f8df21700171bb5",
"variant": null
},
"graalpy-3.8.5-linux-x86_64-gnu": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "linux",
"libc": "gnu",
"major": 3,
"minor": 8,
"patch": 5,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/vm-22.3.1/graalpy-22.3.1-linux-amd64.tar.gz",
"sha256": "9ef3885c8a498a70de53bc71409f3e60f36e9ce5d3d745b173fedcca1f4abd12",
"variant": null
},
"graalpy-3.8.2-darwin-x86_64-none": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "darwin",
"libc": "none",
"major": 3,
"minor": 8,
"patch": 2,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/vm-20.2.0/graalpython-20.2.0-macos-amd64.tar.gz",
"sha256": null,
"variant": null
},
"graalpy-3.8.2-linux-x86_64-gnu": {
"name": "graalpy",
"arch": {
"family": "x86_64",
"variant": null
},
"os": "linux",
"libc": "gnu",
"major": 3,
"minor": 8,
"patch": 2,
"prerelease": "",
"url": "https://github.com/oracle/graalpython/releases/download/vm-20.2.0/graalpython-20.2.0-linux-amd64.tar.gz",
"sha256": null,
"variant": null
}
}

View file

@ -137,6 +137,7 @@ class Version(NamedTuple):
class ImplementationName(StrEnum):
CPYTHON = "cpython"
PYPY = "pypy"
GRAALPY = "graalpy"
class Variant(StrEnum):
@ -540,6 +541,105 @@ class PyPyFinder(Finder):
download.sha256 = checksums.get(download.filename)
class GraalPyFinder(Finder):
implementation = ImplementationName.GRAALPY
RELEASE_URL = "https://api.github.com/repos/oracle/graalpython/releases"
PLATFORM_MAPPING = {
"windows": "windows",
"linux": "linux",
"macos": "darwin",
}
ARCH_MAPPING = {
"amd64": "x86_64",
"aarch64": "aarch64",
}
GRAALPY_VERSION_RE = re.compile(r"-(\d+\.\d+\.\d+)$", re.ASCII)
CPY_VERSION_RE = re.compile(r"Python (\d+\.\d+(\.\d+)?)", re.ASCII)
PLATFORM_RE = re.compile(r"(\w+)-(\w+)\.(?:zip|tar\.gz)$", re.ASCII)
def __init__(self, client: httpx.AsyncClient):
self.client = client
async def find(self) -> list[PythonDownload]:
downloads = await self._fetch_downloads()
await self._fetch_checksums(downloads, n=10)
return downloads
async def _fetch_downloads(self) -> list[PythonDownload]:
# This will only download the first page, i.e., ~30 releases of
# GraalPy. Since GraalPy releases 6 times a year and has a support
# window of 2 years this is plenty.
resp = await self.client.get(self.RELEASE_URL)
resp.raise_for_status()
releases = resp.json()
results = {}
for release in releases:
m = self.GRAALPY_VERSION_RE.search(release["tag_name"])
if not m:
continue
graalpy_version = m.group(1)
m = self.CPY_VERSION_RE.search(release["body"])
if not m:
continue
python_version_str = m.group(1)
if not m.group(2):
python_version_str += ".0"
python_version = Version.from_str(python_version_str)
for asset in release["assets"]:
url = asset["browser_download_url"]
m = self.PLATFORM_RE.search(url)
if not m:
continue
platform = self._normalize_os(m.group(1))
arch = self._normalize_arch(m.group(2))
libc = "gnu" if platform == "linux" else "none"
download = PythonDownload(
release=0,
version=python_version,
triple=PlatformTriple(
platform=platform,
arch=arch,
libc=libc,
),
flavor=graalpy_version,
implementation=self.implementation,
filename=asset["name"],
url=url,
)
# Only keep the latest GraalPy version of each arch/platform
if (python_version, arch, platform) not in results:
results[(python_version, arch, platform)] = download
return list(results.values())
def _normalize_arch(self, arch: str) -> Arch:
return Arch(self.ARCH_MAPPING.get(arch, arch), None)
def _normalize_os(self, os: str) -> str:
return self.PLATFORM_MAPPING.get(os, os)
async def _fetch_checksums(self, downloads: list[PythonDownload], n: int) -> None:
for idx, batch in enumerate(batched(downloads, n)):
logging.info("Fetching GraalPy checksums: %d/%d", idx * n, len(downloads))
checksum_requests = []
for download in batch:
url = download.url + ".sha256"
checksum_requests.append(self.client.get(url))
for download, resp in zip(batch, await asyncio.gather(*checksum_requests)):
try:
resp.raise_for_status()
except httpx.HTTPStatusError as e:
if e.response.status_code == 404:
continue
raise
download.sha256 = resp.text.strip()
def render(downloads: list[PythonDownload]) -> None:
"""Render `download-metadata.json`."""
@ -564,7 +664,11 @@ def render(downloads: list[PythonDownload]) -> None:
def sort_key(download: PythonDownload) -> tuple:
# Sort by implementation, version (latest first), and then by triple.
impl_order = [ImplementationName.CPYTHON, ImplementationName.PYPY]
impl_order = [
ImplementationName.CPYTHON,
ImplementationName.PYPY,
ImplementationName.GRAALPY,
]
prerelease = prerelease_sort_key(download.version.prerelease)
return (
impl_order.index(download.implementation),
@ -630,6 +734,7 @@ async def find() -> None:
finders = [
CPythonFinder(client),
PyPyFinder(client),
GraalPyFinder(client),
]
downloads = []

View file

@ -933,6 +933,7 @@ fn parse_json_downloads(
let implementation = match entry.name.as_str() {
"cpython" => LenientImplementationName::Known(ImplementationName::CPython),
"pypy" => LenientImplementationName::Known(ImplementationName::PyPy),
"graalpy" => LenientImplementationName::Known(ImplementationName::GraalPy),
_ => LenientImplementationName::Unknown(entry.name.clone()),
};

View file

@ -349,9 +349,7 @@ impl ManagedPythonInstallation {
let implementation = match self.implementation() {
ImplementationName::CPython => "python",
ImplementationName::PyPy => "pypy",
ImplementationName::GraalPy => {
unreachable!("Managed installations of GraalPy are not supported")
}
ImplementationName::GraalPy => "graalpy",
};
let version = match self.implementation() {
@ -364,13 +362,14 @@ impl ManagedPythonInstallation {
}
// PyPy uses a full version number, even on Windows.
ImplementationName::PyPy => format!("{}.{}", self.key.major, self.key.minor),
ImplementationName::GraalPy => {
unreachable!("Managed installations of GraalPy are not supported")
}
ImplementationName::GraalPy => String::new(),
};
// On Windows, the executable is just `python.exe` even for alternative variants
let variant = if cfg!(unix) {
// GraalPy always uses `graalpy.exe` as the main executable
let variant = if *self.implementation() == ImplementationName::GraalPy {
""
} else if cfg!(unix) {
self.key.variant.suffix()
} else if cfg!(windows) && windowed {
// Use windowed Python that doesn't open a terminal.
@ -384,10 +383,10 @@ impl ManagedPythonInstallation {
exe = std::env::consts::EXE_SUFFIX
);
let executable = if cfg!(windows) {
self.python_dir().join(name)
} else if cfg!(unix) {
let executable = if cfg!(unix) || *self.implementation() == ImplementationName::GraalPy {
self.python_dir().join("bin").join(name)
} else if cfg!(windows) {
self.python_dir().join(name)
} else {
unimplemented!("Only Windows and Unix systems are supported.")
};

View file

@ -298,7 +298,7 @@ impl TestContext {
// Filter platform keys
let platform_re = r"(?x)
( # We capture the group before the platform
(?:cpython|pypy) # Python implementation
(?:cpython|pypy|graalpy)# Python implementation
-
\d+\.\d+ # Major and minor version
(?: # The patch version is handled separately

View file

@ -367,6 +367,7 @@ fn python_list_downloads() {
----- stdout -----
cpython-3.10.17-[PLATFORM] <download available>
pypy-3.10.16-[PLATFORM] <download available>
graalpy-3.10.0-[PLATFORM] <download available>
----- stderr -----
");
@ -396,6 +397,7 @@ fn python_list_downloads() {
pypy-3.10.14-[PLATFORM] <download available>
pypy-3.10.13-[PLATFORM] <download available>
pypy-3.10.12-[PLATFORM] <download available>
graalpy-3.10.0-[PLATFORM] <download available>
----- stderr -----
");
@ -422,6 +424,7 @@ fn python_list_downloads_installed() {
----- stdout -----
cpython-3.10.17-[PLATFORM] <download available>
pypy-3.10.16-[PLATFORM] <download available>
graalpy-3.10.0-[PLATFORM] <download available>
----- stderr -----
");
@ -448,6 +451,7 @@ fn python_list_downloads_installed() {
----- stdout -----
cpython-3.10.17-[PLATFORM] managed/cpython-3.10.17-[PLATFORM]/[INSTALL-BIN]/python
pypy-3.10.16-[PLATFORM] <download available>
graalpy-3.10.0-[PLATFORM] <download available>
----- stderr -----
");
@ -459,6 +463,7 @@ fn python_list_downloads_installed() {
----- stdout -----
cpython-3.10.17-[PLATFORM] <download available>
pypy-3.10.16-[PLATFORM] <download available>
graalpy-3.10.0-[PLATFORM] <download available>
----- stderr -----
");

View file

@ -197,8 +197,8 @@ if __name__ == "__main__":
# Attempt to install NumPy.
# This ensures that we can successfully install a package with native libraries.
#
# NumPy doesn't distribute wheels for Python 3.13 (at time of writing).
if sys.version_info < (3, 13):
# NumPy doesn't distribute wheels for Python 3.13 or GraalPy (at time of writing).
if sys.version_info < (3, 13) and sys.implementation.name != "graalpy":
install_package(uv=uv, package="numpy")
# Attempt to install `pydantic_core`.