mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-31 07:47:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add support for installing pyodide Pythons (#14518)

Browse source 
- [x] Add tests

---------

Co-authored-by: Zanie Blue <contact@zanie.dev>
This commit is contained in:
Hood Chatham 2025-08-13 18:03:25 +02:00 • committed by GitHub
parent b38edb9b7d
commit c8d0bfba5c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 516 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

64
crates/uv-python/download-metadata.json
View file

@ -10511,6 +10511,22 @@
"sha256": "e5a904ecfb4061389773dd655d3b5665447c80cbf2948fcb1c07e92716eed955",
"variant": null
},
"cpython-3.13.2-emscripten-wasm32-musl": {
"name": "cpython",
"arch": {
"family": "wasm32",
"variant": null
},
"os": "emscripten",
"libc": "musl",
"major": 3,
"minor": 13,
"patch": 2,
"prerelease": "",
"url": "https://github.com/pyodide/pyodide/releases/download/0.28.0/xbuildenv-0.28.0.tar.bz2",
"sha256": null,
"variant": null
},
"cpython-3.13.2-linux-aarch64-gnu": {
"name": "cpython",
"arch": {
@ -15055,6 +15071,22 @@
"sha256": "848405b92bda20fad1f9bba99234c7d3f11e0b31e46f89835d1cb3d735e932aa",
"variant": null
},
"cpython-3.12.7-emscripten-wasm32-musl": {
"name": "cpython",
"arch": {
"family": "wasm32",
"variant": null
},
"os": "emscripten",
"libc": "musl",
"major": 3,
"minor": 12,
"patch": 7,
"prerelease": "",
"url": "https://github.com/pyodide/pyodide/releases/download/0.27.7/xbuildenv-0.27.7.tar.bz2",
"sha256": null,
"variant": null
},
"cpython-3.12.7-linux-aarch64-gnu": {
"name": "cpython",
"arch": {
@ -17103,6 +17135,22 @@
"sha256": "eca96158c1568dedd9a0b3425375637a83764d1fa74446438293089a8bfac1f8",
"variant": null
},
"cpython-3.12.1-emscripten-wasm32-musl": {
"name": "cpython",
"arch": {
"family": "wasm32",
"variant": null
},
"os": "emscripten",
"libc": "musl",
"major": 3,
"minor": 12,
"patch": 1,
"prerelease": "",
"url": "https://github.com/pyodide/pyodide/releases/download/0.26.4/xbuildenv-0.26.4.tar.bz2",
"sha256": null,
"variant": null
},
"cpython-3.12.1-linux-aarch64-gnu": {
"name": "cpython",
"arch": {
@ -21439,6 +21487,22 @@
"sha256": "f710b8d60621308149c100d5175fec39274ed0b9c99645484fd93d1716ef4310",
"variant": null
},
"cpython-3.11.3-emscripten-wasm32-musl": {
"name": "cpython",
"arch": {
"family": "wasm32",
"variant": null
},
"os": "emscripten",
"libc": "musl",
"major": 3,
"minor": 11,
"patch": 3,
"prerelease": "",
"url": "https://github.com/pyodide/pyodide/releases/download/0.25.1/xbuildenv-0.25.1.tar.bz2",
"sha256": null,
"variant": null
},
"cpython-3.11.3-linux-aarch64-gnu": {
"name": "cpython",
"arch": {

87
crates/uv-python/fetch-download-metadata.py
View file

@ -550,6 +550,92 @@ class PyPyFinder(Finder):
download.sha256 = checksums.get(download.filename)
class PyodideFinder(Finder):
implementation = ImplementationName.CPYTHON
RELEASE_URL = "https://api.github.com/repos/pyodide/pyodide/releases"
METADATA_URL = (
"https://pyodide.github.io/pyodide/api/pyodide-cross-build-environments.json"
)
TRIPLE = PlatformTriple(
platform="emscripten",
arch=Arch("wasm32"),
libc="musl",
)
def __init__(self, client: httpx.AsyncClient):
self.client = client
async def find(self) -> list[PythonDownload]:
downloads = await self._fetch_downloads()
await self._fetch_checksums(downloads, n=10)
return downloads
async def _fetch_downloads(self) -> list[PythonDownload]:
# This will only download the first page, i.e., ~30 releases
[release_resp, meta_resp] = await asyncio.gather(
self.client.get(self.RELEASE_URL), self.client.get(self.METADATA_URL)
)
release_resp.raise_for_status()
meta_resp.raise_for_status()
releases = release_resp.json()
metadata = meta_resp.json()["releases"]
maj_minor_seen = set()
results = []
for release in releases:
pyodide_version = release["tag_name"]
meta = metadata.get(pyodide_version, None)
if meta is None:
continue
maj_min = pyodide_version.rpartition(".")[0]
# Only keep latest
if maj_min in maj_minor_seen:
continue
maj_minor_seen.add(maj_min)
python_version = Version.from_str(meta["python_version"])
# Find xbuildenv asset
for asset in release["assets"]:
if asset["name"].startswith("xbuildenv"):
break
url = asset["browser_download_url"]
results.append(
PythonDownload(
release=0,
version=python_version,
triple=self.TRIPLE,
flavor=pyodide_version,
implementation=self.implementation,
filename=asset["name"],
url=url,
)
)
return results
async def _fetch_checksums(self, downloads: list[PythonDownload], n: int) -> None:
for idx, batch in enumerate(batched(downloads, n)):
logging.info("Fetching Pyodide checksums: %d/%d", idx * n, len(downloads))
checksum_requests = []
for download in batch:
url = download.url + ".sha256"
checksum_requests.append(self.client.get(url))
for download, resp in zip(
batch, await asyncio.gather(*checksum_requests), strict=False
):
try:
resp.raise_for_status()
except httpx.HTTPStatusError as e:
if e.response.status_code == 404:
continue
raise
download.sha256 = resp.text.strip()
class GraalPyFinder(Finder):
implementation = ImplementationName.GRAALPY
@ -751,6 +837,7 @@ async def find() -> None:
CPythonFinder(client),
PyPyFinder(client),
GraalPyFinder(client),
PyodideFinder(client),
]
downloads = []

19
crates/uv-python/src/discovery.rs
View file

@ -342,7 +342,8 @@ fn python_executables_from_installed<'a>(
installed_installations.root().user_display()
);
let installations = installed_installations.find_matching_current_platform()?;
// Check that the Python version and platform satisfy the request to avoid unnecessary interpreter queries later
// Check that the Python version and platform satisfy the request to avoid
// unnecessary interpreter queries later
Ok(installations
.into_iter()
.filter(move |installation| {
@ -351,7 +352,7 @@ fn python_executables_from_installed<'a>(
return false;
}
if !platform.matches(installation.platform()) {
debug!("Skipping managed installation `{installation}`: does not satisfy `{platform}`");
debug!("Skipping managed installation `{installation}`: does not satisfy requested platform `{platform}`");
return false;
}
true
@ -1259,6 +1260,7 @@ pub(crate) fn find_python_installation(
let mut first_prerelease = None;
let mut first_managed = None;
let mut first_error = None;
let mut emscripten_installation = None;
for result in installations {
// Iterate until the first critical error or happy result
if !result.as_ref().err().is_none_or(Error::is_critical) {
@ -1276,6 +1278,15 @@ pub(crate) fn find_python_installation(
return result;
};
if installation.os().is_emscripten() {
// We want to pick a native Python over an Emscripten Python if we
// can find any native Python.
if emscripten_installation.is_none() {
emscripten_installation = Some(installation.clone());
}
continue;
}
// Check if we need to skip the interpreter because it is "not allowed", e.g., if it is a
// pre-release version or an alternative implementation, using it requires opt-in.
@ -1352,6 +1363,10 @@ pub(crate) fn find_python_installation(
return Ok(Ok(installation));
}
if let Some(emscripten_python) = emscripten_installation {
return Ok(Ok(emscripten_python));
}
// If we found a Python, but it was unusable for some reason, report that instead of saying we
// couldn't find any Python interpreters.
if let Some(err) = first_error {

61
crates/uv-python/src/downloads.rs
View file

@ -176,7 +176,7 @@ impl PlatformRequest {
/// Check if this platform request is satisfied by a platform.
pub fn matches(&self, platform: &Platform) -> bool {
if let Some(os) = self.os {
if platform.os != os {
if !platform.os.supports(os) {
return false;
}
}
@ -452,24 +452,13 @@ impl PythonDownloadRequest {
return false;
}
}
if let Some(os) = self.os() {
if &interpreter.os() != os {
debug!(
"Skipping interpreter at `{executable}`: operating system `{}` does not match request `{os}`",
interpreter.os()
);
return false;
}
}
if let Some(arch) = self.arch() {
let interpreter_platform = Platform::from(interpreter.platform());
if !arch.satisfied_by(&interpreter_platform) {
debug!(
"Skipping interpreter at `{executable}`: architecture `{}` does not match request `{arch}`",
interpreter.arch()
);
return false;
}
let platform = self.platform();
let interpreter_platform = Platform::from(interpreter.platform());
if !platform.matches(&interpreter_platform) {
debug!(
"Skipping interpreter at `{executable}`: platform `{interpreter_platform}` does not match request `{platform}`",
);
return false;
}
if let Some(implementation) = self.implementation() {
let interpreter_implementation = interpreter.implementation_name();
@ -482,15 +471,6 @@ impl PythonDownloadRequest {
return false;
}
}
if let Some(libc) = self.libc() {
if &interpreter.libc() != libc {
debug!(
"Skipping interpreter at `{executable}`: libc `{}` does not match request `{libc}`",
interpreter.libc()
);
return false;
}
}
true
}
@ -1066,13 +1046,32 @@ impl ManagedPythonDownload {
// If the distribution is a `full` archive, the Python installation is in the `install` directory.
if extracted.join("install").is_dir() {
extracted = extracted.join("install");
// If the distribution is a Pyodide archive, the Python installation is in the `pyodide-root/dist` directory.
} else if self.os().is_emscripten() {
extracted = extracted.join("pyodide-root").join("dist");
}
// If the distribution is missing a `python`-to-`pythonX.Y` symlink, add it. PEP 394 permits
// it, and python-build-standalone releases after `20240726` include it, but releases prior
// to that date do not.
#[cfg(unix)]
{
// Pyodide distributions require all of the supporting files to be alongside the Python
// executable, so they don't have a `bin` directory. We create it and link
// `bin/pythonX.Y` to `dist/python`.
if self.os().is_emscripten() {
fs_err::create_dir_all(extracted.join("bin"))?;
fs_err::os::unix::fs::symlink(
"../python",
extracted
.join("bin")
.join(format!("python{}.{}", self.key.major, self.key.minor)),
)?;
}
// If the distribution is missing a `python` -> `pythonX.Y` symlink, add it.
//
// Pyodide releases never contain this link by default.
//
// PEP 394 permits it, and python-build-standalone releases after `20240726` include it,
// but releases prior to that date do not.
match fs_err::os::unix::fs::symlink(
format!("python{}.{}", self.key.major, self.key.minor),
extracted.join("bin").join("python"),

133
crates/uv-python/src/lib.rs
View file

@ -325,6 +325,89 @@ mod tests {
Ok(())
}
fn create_mock_pyodide_interpreter(path: &Path, version: &PythonVersion) -> Result<()> {
let json = indoc! {r##"
{
"result": "success",
"platform": {
"os": {
"name": "pyodide",
"major": 2025,
"minor": 0
},
"arch": "wasm32"
},
"manylinux_compatible": false,
"standalone": false,
"markers": {
"implementation_name": "cpython",
"implementation_version": "{FULL_VERSION}",
"os_name": "posix",
"platform_machine": "wasm32",
"platform_python_implementation": "CPython",
"platform_release": "4.0.9",
"platform_system": "Emscripten",
"platform_version": "#1",
"python_full_version": "{FULL_VERSION}",
"python_version": "{VERSION}",
"sys_platform": "emscripten"
},
"sys_base_exec_prefix": "/",
"sys_base_prefix": "/",
"sys_prefix": "/",
"sys_executable": "{PATH}",
"sys_path": [
"",
"/lib/python313.zip",
"/lib/python{VERSION}",
"/lib/python{VERSION}/lib-dynload",
"/lib/python{VERSION}/site-packages"
],
"site_packages": [
"/lib/python{VERSION}/site-packages"
],
"stdlib": "//lib/python{VERSION}",
"scheme": {
"platlib": "//lib/python{VERSION}/site-packages",
"purelib": "//lib/python{VERSION}/site-packages",
"include": "//include/python{VERSION}",
"scripts": "//bin",
"data": "/"
},
"virtualenv": {
"purelib": "lib/python{VERSION}/site-packages",
"platlib": "lib/python{VERSION}/site-packages",
"include": "include/site/python{VERSION}",
"scripts": "bin",
"data": ""
},
"pointer_size": "32",
"gil_disabled": false
}
"##};
let json = json
.replace(
"{PATH}",
path.to_str().expect("Path can be represented as string"),
)
.replace("{FULL_VERSION}", &version.to_string())
.replace("{VERSION}", &version.without_patch().to_string());
fs_err::create_dir_all(path.parent().unwrap())?;
fs_err::write(
path,
formatdoc! {r"
#!/bin/sh
echo '{json}'
"},
)?;
fs_err::set_permissions(path, std::os::unix::fs::PermissionsExt::from_mode(0o770))?;
Ok(())
}
/// Create a mock Python 2 interpreter executable which returns a fixed error message mocking
/// invocation of Python 2 with the `-I` flag as done by our query script.
fn create_mock_python2_interpreter(path: &Path) -> Result<()> {
@ -372,6 +455,16 @@ mod tests {
)
}
fn add_pyodide_version(&mut self, version: &'static str) -> Result<()> {
let path = self.new_search_path_directory(format!("pyodide-{version}"))?;
let python = format!("python{}", env::consts::EXE_SUFFIX);
Self::create_mock_pyodide_interpreter(
&path.join(python),
&PythonVersion::from_str(version).unwrap(),
)?;
Ok(())
}
/// Create fake Python interpreters the given Python versions.
///
/// Adds them to the test context search path.
@ -2606,4 +2699,44 @@ mod tests {
Ok(())
}
#[test]
fn find_python_pyodide() -> Result<()> {
let mut context = TestContext::new()?;
context.add_pyodide_version("3.13.2")?;
let python = context.run(|| {
find_python_installation(
&PythonRequest::Default,
EnvironmentPreference::Any,
PythonPreference::OnlySystem,
&context.cache,
Preview::default(),
)
})??;
// We should find the Pyodide interpreter
assert_eq!(
python.interpreter().python_full_version().to_string(),
"3.13.2"
);
// We should prefer any native Python to the Pyodide Python
context.add_python_versions(&["3.15.7"])?;
let python = context.run(|| {
find_python_installation(
&PythonRequest::Default,
EnvironmentPreference::Any,
PythonPreference::OnlySystem,
&context.cache,
Preview::default(),
)
})??;
assert_eq!(
python.interpreter().python_full_version().to_string(),
"3.15.7"
);
Ok(())
}
}

18
crates/uv-python/src/managed.rs
View file

@ -263,7 +263,13 @@ impl ManagedPythonInstallations {
let iter = Self::from_settings(None)?
.find_all()?
.filter(move |installation| platform.supports(installation.platform()));
.filter(move |installation| {
if !platform.supports(installation.platform()) {
debug!("Skipping managed installation `{installation}`: not supported by current platform `{platform}`");
return false;
}
true
});
Ok(iter)
}
@ -538,6 +544,11 @@ impl ManagedPythonInstallation {
/// Ensure the environment is marked as externally managed with the
/// standard `EXTERNALLY-MANAGED` file.
pub fn ensure_externally_managed(&self) -> Result<(), Error> {
if self.key.os().is_emscripten() {
// Emscripten's stdlib is a zip file so we can't put an
// EXTERNALLY-MANAGED inside.
return Ok(());
}
// Construct the path to the `stdlib` directory.
let stdlib = if self.key.os().is_windows() {
self.python_dir().join("Lib")
@ -563,6 +574,11 @@ impl ManagedPythonInstallation {
/// Ensure that the `sysconfig` data is patched to match the installation path.
pub fn ensure_sysconfig_patched(&self) -> Result<(), Error> {
if cfg!(unix) {
if self.key.os().is_emscripten() {
// Emscripten's stdlib is a zip file so we can't update the
// sysconfig directly
return Ok(());
}
if *self.implementation() == ImplementationName::CPython {
sysconfig::update_sysconfig(
self.path(),