Allow download of Python distribution variants with newer CPU instruction sets (#9781)

Supersedes https://github.com/astral-sh/uv/pull/8517 with an alternative
approach of making all the variants available instead of replacing the
x86_64 (v1) variant with x86_64_v2.

Doesn't add automatic inference of the supported instructions, but that
should be doable per @charliermarsh's comment there. Going to do it as a
follow-up since this has been pretty time consuming.

e.g.,

```
❯ cargo run -q -- python install cpython-3.12.8-linux-x86_64_v3-gnu
Installed Python 3.12.8 in 2.72s
 + cpython-3.12.8-linux-x86_64_v3-gnu
```

Co-authored-by: j178 <10510431+j178@users.noreply.github.com>
This commit is contained in:
Zanie Blue 2024-12-10 14:26:45 -06:00 committed by GitHub
parent fd420db197
commit 761dafd0d1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 23040 additions and 2044 deletions

View file

@ -50,7 +50,7 @@ import json
import logging
import os
import re
from dataclasses import dataclass, field
from dataclasses import asdict, dataclass, field
from enum import StrEnum
from pathlib import Path
from typing import Generator, Iterable, NamedTuple, Self
@ -72,11 +72,40 @@ def batched(iterable: Iterable, n: int) -> Generator[tuple, None, None]:
yield batch
@dataclass(frozen=True)
class Arch:
# The architecture family, e.g. "x86_64", "aarch64".
family: str
# The architecture variant, e.g., "v2" in "x86_64_v2"
variant: str | None = None
def key(self) -> str:
return str(self)
def __str__(self) -> str:
return (self.family + "_" + self.variant) if self.variant else self.family
def __gt__(self, other) -> bool:
return (self.family, self.variant or "") > (other.family, other.variant or "")
def __lt__(self, other) -> bool:
return (self.family, self.variant or "") < (other.family, other.variant or "")
type PlatformTripleKey = tuple[str, str, str]
class PlatformTriple(NamedTuple):
# The operating system, e.g. "linux", "macos", "windows".
platform: str
arch: str
# The architecture, e.g. "x86_64", "aarch64".
arch: Arch
# The libc implementation, e.g. "gnu", "musl", "none".
libc: str
def key(self) -> PlatformTripleKey:
return (self.platform, self.arch.key(), self.libc)
class Version(NamedTuple):
major: int
@ -229,12 +258,12 @@ class CPythonFinder(Finder):
downloads = []
for version_downloads in downloads_by_version.values():
selected: dict[
tuple[PlatformTriple, Variant | None],
tuple[PlatformTripleKey, Variant | None],
tuple[PythonDownload, tuple[int, int]],
] = {}
for download in version_downloads:
priority = self._get_priority(download)
existing = selected.get((download.triple, download.variant))
existing = selected.get((download.triple.key(), download.variant))
if existing:
existing_download, existing_priority = existing
# Skip if we have a flavor with higher priority already (indicated by a smaller value)
@ -247,7 +276,7 @@ class CPythonFinder(Finder):
existing_download.flavor,
)
continue
selected[(download.triple, download.variant)] = (
selected[(download.triple.key(), download.variant)] = (
download,
priority,
)
@ -368,11 +397,12 @@ class CPythonFinder(Finder):
return PlatformTriple(operating_system, arch, libc)
def _normalize_arch(self, arch: str) -> str:
def _normalize_arch(self, arch: str) -> Arch:
arch = self.ARCH_MAP.get(arch, arch)
pieces = arch.split("_")
# Strip `_vN` from `x86_64`
return "_".join(pieces[:2])
family = "_".join(pieces[:2])
variant = pieces[2] if len(pieces) > 2 else None
return Arch(family, variant)
def _normalize_os(self, os: str) -> str:
return os
@ -472,8 +502,8 @@ class PyPyFinder(Finder):
return list(results.values())
def _normalize_arch(self, arch: str) -> str:
return self.ARCH_MAPPING.get(arch, arch)
def _normalize_arch(self, arch: str) -> Arch:
return Arch(self.ARCH_MAPPING.get(arch, arch), None)
def _normalize_os(self, os: str) -> str:
return self.PLATFORM_MAPPING.get(os, os)
@ -539,7 +569,7 @@ def render(downloads: list[PythonDownload]) -> None:
)
results[key] = {
"name": download.implementation,
"arch": download.triple.arch,
"arch": asdict(download.triple.arch),
"os": download.triple.platform,
"libc": download.triple.libc,
"major": download.version.major,