From f9d3f24728d34e73cb0b01570ffd389dd2c1f92f Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 3 Jun 2025 10:01:26 -0700 Subject: [PATCH] Add Pyodide support (#12731) This includes some initial work on adding Pyodide support (issue #12729). It is enough to get ``` uv pip compile -p /path/to/pyodide --extra-index-url file:/path/to/simple-index ``` to work which should already be quite useful. ## Test Plan * added a unit test for `pyodide_platform` * integration tested manually with: ``` cargo run pip install \ -p /home/rchatham/Documents/programming/tmp/pyodide-venv-test/.pyodide-xbuildenv-0.29.3/0.27.4/xbuildenv/pyodide-root/dist/python \ --extra-index-url file:/home/rchatham/Documents/programming/tmp/pyodide-venv-test/.pyodide-xbuildenv-0.29.3/0.27.4/xbuildenv/pyodide-root/package_index \ --index-strategy unsafe-best-match --target blah --no-build \ numpy pydantic ``` --------- Co-authored-by: konsti Co-authored-by: Zanie Blue --- .github/workflows/ci.yml | 39 ++++++++++++++ crates/uv-configuration/src/target_triple.rs | 23 ++++++++ crates/uv-platform-tags/src/platform.rs | 7 ++- crates/uv-platform-tags/src/platform_tag.rs | 54 +++++++++++++++++++ crates/uv-platform-tags/src/tags.rs | 6 +++ .../uv-python/python/get_interpreter_info.py | 18 +++++++ crates/uv-python/src/interpreter.rs | 2 + crates/uv-python/src/platform.rs | 7 +++ crates/uv-torch/src/backend.rs | 3 +- docs/reference/cli.md | 4 ++ uv.schema.json | 7 +++ 11 files changed, 168 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5b14ab65..5463294cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1294,6 +1294,45 @@ jobs: run: | .\uv.exe pip install anyio + integration-test-pyodide-linux: + timeout-minutes: 10 + needs: build-binary-linux-libc + name: "integration test | pyodide on ubuntu" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: "Download binary" + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: uv-linux-libc-${{ github.sha }} + + - name: "Prepare binary" + run: chmod +x ./uv + + - name: "Create a native virtual environment" + run: | + ./uv venv venv-native -p 3.12 + # We use features added in 0.30.3 but there is no known breakage in + # newer versions. + ./uv pip install -p venv-native/bin/python pyodide-build==0.30.3 pip + + - name: "Install pyodide interpreter" + run: | + source ./venv-native/bin/activate + pyodide xbuildenv install 0.27.5 + PYODIDE_PYTHON=$(pyodide config get interpreter) + PYODIDE_INDEX=$(pyodide config get package_index) + echo "PYODIDE_PYTHON=$PYODIDE_PYTHON" >> $GITHUB_ENV + echo "PYODIDE_INDEX=$PYODIDE_INDEX" >> $GITHUB_ENV + + - name: "Create pyodide virtual environment" + run: | + ./uv venv -p $PYODIDE_PYTHON venv-pyodide + source ./venv-pyodide/bin/activate + ./uv pip install --extra-index-url=$PYODIDE_INDEX --no-build numpy + python -c 'import numpy' + integration-test-github-actions: timeout-minutes: 10 needs: build-binary-linux-libc diff --git a/crates/uv-configuration/src/target_triple.rs b/crates/uv-configuration/src/target_triple.rs index c0c651409..b9ca3fafe 100644 --- a/crates/uv-configuration/src/target_triple.rs +++ b/crates/uv-configuration/src/target_triple.rs @@ -226,6 +226,10 @@ pub enum TargetTriple { #[serde(rename = "aarch64-manylinux_2_40")] #[serde(alias = "aarch64-manylinux240")] Aarch64Manylinux240, + + /// A wasm32 target using the the Pyodide 2024 platform. Meant for use with Python 3.12. + #[cfg_attr(feature = "clap", value(name = "wasm32-pyodide2024"))] + Wasm32Pyodide2024, } impl TargetTriple { @@ -450,6 +454,13 @@ impl TargetTriple { }, Arch::Aarch64, ), + Self::Wasm32Pyodide2024 => Platform::new( + Os::Pyodide { + major: 2024, + minor: 0, + }, + Arch::Wasm32, + ), } } @@ -490,6 +501,7 @@ impl TargetTriple { Self::Aarch64Manylinux238 => "aarch64", Self::Aarch64Manylinux239 => "aarch64", Self::Aarch64Manylinux240 => "aarch64", + Self::Wasm32Pyodide2024 => "wasm32", } } @@ -530,6 +542,7 @@ impl TargetTriple { Self::Aarch64Manylinux238 => "Linux", Self::Aarch64Manylinux239 => "Linux", Self::Aarch64Manylinux240 => "Linux", + Self::Wasm32Pyodide2024 => "Emscripten", } } @@ -570,6 +583,10 @@ impl TargetTriple { Self::Aarch64Manylinux238 => "", Self::Aarch64Manylinux239 => "", Self::Aarch64Manylinux240 => "", + // This is the value Emscripten gives for its version: + // https://github.com/emscripten-core/emscripten/blob/4.0.8/system/lib/libc/emscripten_syscall_stubs.c#L63 + // It doesn't really seem to mean anything? But for completeness we include it here. + Self::Wasm32Pyodide2024 => "#1", } } @@ -610,6 +627,9 @@ impl TargetTriple { Self::Aarch64Manylinux238 => "", Self::Aarch64Manylinux239 => "", Self::Aarch64Manylinux240 => "", + // This is the Emscripten compiler version for Pyodide 2024. + // See https://pyodide.org/en/stable/development/abi.html#pyodide-2024-0 + Self::Wasm32Pyodide2024 => "3.1.58", } } @@ -650,6 +670,7 @@ impl TargetTriple { Self::Aarch64Manylinux238 => "posix", Self::Aarch64Manylinux239 => "posix", Self::Aarch64Manylinux240 => "posix", + Self::Wasm32Pyodide2024 => "posix", } } @@ -690,6 +711,7 @@ impl TargetTriple { Self::Aarch64Manylinux238 => "linux", Self::Aarch64Manylinux239 => "linux", Self::Aarch64Manylinux240 => "linux", + Self::Wasm32Pyodide2024 => "emscripten", } } @@ -730,6 +752,7 @@ impl TargetTriple { Self::Aarch64Manylinux238 => true, Self::Aarch64Manylinux239 => true, Self::Aarch64Manylinux240 => true, + Self::Wasm32Pyodide2024 => false, } } diff --git a/crates/uv-platform-tags/src/platform.rs b/crates/uv-platform-tags/src/platform.rs index c2b849e0f..88c36714b 100644 --- a/crates/uv-platform-tags/src/platform.rs +++ b/crates/uv-platform-tags/src/platform.rs @@ -43,6 +43,7 @@ pub enum Os { Manylinux { major: u16, minor: u16 }, Musllinux { major: u16, minor: u16 }, Windows, + Pyodide { major: u16, minor: u16 }, Macos { major: u16, minor: u16 }, FreeBsd { release: String }, NetBsd { release: String }, @@ -67,6 +68,7 @@ impl fmt::Display for Os { Self::Illumos { .. } => write!(f, "illumos"), Self::Haiku { .. } => write!(f, "haiku"), Self::Android { .. } => write!(f, "android"), + Self::Pyodide { .. } => write!(f, "pyodide"), } } } @@ -109,6 +111,7 @@ pub enum Arch { S390X, LoongArch64, Riscv64, + Wasm32, } impl fmt::Display for Arch { @@ -126,6 +129,7 @@ impl fmt::Display for Arch { Self::S390X => write!(f, "s390x"), Self::LoongArch64 => write!(f, "loongarch64"), Self::Riscv64 => write!(f, "riscv64"), + Self::Wasm32 => write!(f, "wasm32"), } } } @@ -168,7 +172,7 @@ impl Arch { // manylinux_2_36 Self::LoongArch64 => Some(36), // unsupported - Self::Powerpc | Self::Armv5TEL | Self::Armv6L => None, + Self::Powerpc | Self::Armv5TEL | Self::Armv6L | Self::Wasm32 => None, } } @@ -187,6 +191,7 @@ impl Arch { Self::S390X => "s390x", Self::LoongArch64 => "loongarch64", Self::Riscv64 => "riscv64", + Self::Wasm32 => "wasm32", } } diff --git a/crates/uv-platform-tags/src/platform_tag.rs b/crates/uv-platform-tags/src/platform_tag.rs index 0ba46200d..1162a83d7 100644 --- a/crates/uv-platform-tags/src/platform_tag.rs +++ b/crates/uv-platform-tags/src/platform_tag.rs @@ -71,6 +71,8 @@ pub enum PlatformTag { Illumos { release_arch: SmallString }, /// Ex) `solaris_11_4_x86_64` Solaris { release_arch: SmallString }, + /// Ex) `pyodide_2024_0_wasm32` + Pyodide { major: u16, minor: u16 }, } impl PlatformTag { @@ -97,6 +99,7 @@ impl PlatformTag { PlatformTag::Haiku { .. } => Some("Haiku"), PlatformTag::Illumos { .. } => Some("Illumos"), PlatformTag::Solaris { .. } => Some("Solaris"), + PlatformTag::Pyodide { .. } => Some("Pyodide"), } } } @@ -262,6 +265,7 @@ impl std::fmt::Display for PlatformTag { Self::Haiku { release_arch } => write!(f, "haiku_{release_arch}"), Self::Illumos { release_arch } => write!(f, "illumos_{release_arch}"), Self::Solaris { release_arch } => write!(f, "solaris_{release_arch}_64bit"), + Self::Pyodide { major, minor } => write!(f, "pyodide_{major}_{minor}_wasm32"), } } } @@ -616,6 +620,35 @@ impl FromStr for PlatformTag { }); } + if let Some(rest) = s.strip_prefix("pyodide_") { + let mid = + rest.strip_suffix("_wasm32") + .ok_or_else(|| ParsePlatformTagError::InvalidArch { + platform: "pyodide", + tag: s.to_string(), + })?; + let underscore = memchr::memchr(b'_', mid.as_bytes()).ok_or_else(|| { + ParsePlatformTagError::InvalidFormat { + platform: "pyodide", + tag: s.to_string(), + } + })?; + let major: u16 = mid[..underscore].parse().map_err(|_| { + ParsePlatformTagError::InvalidMajorVersion { + platform: "pyodide", + tag: s.to_string(), + } + })?; + + let minor: u16 = mid[underscore + 1..].parse().map_err(|_| { + ParsePlatformTagError::InvalidMinorVersion { + platform: "pyodide", + tag: s.to_string(), + } + })?; + return Ok(Self::Pyodide { major, minor }); + } + Err(ParsePlatformTagError::UnknownFormat(s.to_string())) } } @@ -900,6 +933,27 @@ mod tests { ); } + #[test] + fn pyodide_platform() { + let tag = PlatformTag::Pyodide { + major: 2024, + minor: 0, + }; + assert_eq!( + PlatformTag::from_str("pyodide_2024_0_wasm32").as_ref(), + Ok(&tag) + ); + assert_eq!(tag.to_string(), "pyodide_2024_0_wasm32"); + + assert_eq!( + PlatformTag::from_str("pyodide_2024_0_wasm64"), + Err(ParsePlatformTagError::InvalidArch { + platform: "pyodide", + tag: "pyodide_2024_0_wasm64".to_string() + }) + ); + } + #[test] fn unknown_platform() { assert_eq!( diff --git a/crates/uv-platform-tags/src/tags.rs b/crates/uv-platform-tags/src/tags.rs index 1e20c348c..7381f5dd5 100644 --- a/crates/uv-platform-tags/src/tags.rs +++ b/crates/uv-platform-tags/src/tags.rs @@ -617,6 +617,12 @@ fn compatible_tags(platform: &Platform) -> Result, PlatformErro arch, }] } + (Os::Pyodide { major, minor }, Arch::Wasm32) => { + vec![PlatformTag::Pyodide { + major: *major, + minor: *minor, + }] + } _ => { return Err(PlatformError::OsVersionDetectionError(format!( "Unsupported operating system and architecture combination: {os} {arch}" diff --git a/crates/uv-python/python/get_interpreter_info.py b/crates/uv-python/python/get_interpreter_info.py index 2b1ee09cc..0fe088819 100644 --- a/crates/uv-python/python/get_interpreter_info.py +++ b/crates/uv-python/python/get_interpreter_info.py @@ -510,6 +510,24 @@ def get_operating_system_and_architecture(): "major": int(version[0]), "minor": int(version[1]), } + elif operating_system == "emscripten": + pyodide_abi_version = sysconfig.get_config_var("PYODIDE_ABI_VERSION") + if not pyodide_abi_version: + print( + json.dumps( + { + "result": "error", + "kind": "emscripten_not_pyodide", + } + ) + ) + sys.exit(0) + version = pyodide_abi_version.split("_") + operating_system = { + "name": "pyodide", + "major": int(version[0]), + "minor": int(version[1]), + } elif operating_system in [ "freebsd", "netbsd", diff --git a/crates/uv-python/src/interpreter.rs b/crates/uv-python/src/interpreter.rs index 2f85fa042..4d6f1f81d 100644 --- a/crates/uv-python/src/interpreter.rs +++ b/crates/uv-python/src/interpreter.rs @@ -757,6 +757,8 @@ pub enum InterpreterInfoError { python_major: usize, python_minor: usize, }, + #[error("Only Pyodide is support for Emscripten Python")] + EmscriptenNotPyodide, } #[derive(Debug, Deserialize, Serialize, Clone)] diff --git a/crates/uv-python/src/platform.rs b/crates/uv-python/src/platform.rs index 1ccb0832e..025592c37 100644 --- a/crates/uv-python/src/platform.rs +++ b/crates/uv-python/src/platform.rs @@ -348,6 +348,10 @@ impl From<&uv_platform_tags::Arch> for Arch { ), variant: None, }, + uv_platform_tags::Arch::Wasm32 => Self { + family: target_lexicon::Architecture::Wasm32, + variant: None, + }, } } } @@ -380,6 +384,9 @@ impl From<&uv_platform_tags::Os> for Os { uv_platform_tags::Os::NetBsd { .. } => Self(target_lexicon::OperatingSystem::Netbsd), uv_platform_tags::Os::OpenBsd { .. } => Self(target_lexicon::OperatingSystem::Openbsd), uv_platform_tags::Os::Windows => Self(target_lexicon::OperatingSystem::Windows), + uv_platform_tags::Os::Pyodide { .. } => { + Self(target_lexicon::OperatingSystem::Emscripten) + } } } } diff --git a/crates/uv-torch/src/backend.rs b/crates/uv-torch/src/backend.rs index d3e5afc55..0df5bd844 100644 --- a/crates/uv-torch/src/backend.rs +++ b/crates/uv-torch/src/backend.rs @@ -220,7 +220,8 @@ impl TorchStrategy { | Os::Dragonfly { .. } | Os::Illumos { .. } | Os::Haiku { .. } - | Os::Android { .. } => { + | Os::Android { .. } + | Os::Pyodide { .. } => { Either::Right(std::iter::once(TorchBackend::Cpu.index_url())) } } diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ad0fce937..b7e41728a 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1702,6 +1702,7 @@ interpreter. Use --universal to display the tree for all platforms,
  • aarch64-manylinux_2_38: An ARM64 target for the manylinux_2_38 platform
  • aarch64-manylinux_2_39: An ARM64 target for the manylinux_2_39 platform
  • aarch64-manylinux_2_40: An ARM64 target for the manylinux_2_40 platform
    • +
  • wasm32-pyodide2024: A wasm32 target using the the Pyodide 2024 platform. Meant for use with Python 3.12
    • --python-version python-version

    The Python version to use when filtering the tree.

    For example, pass --python-version 3.10 to display the dependencies that would be included when installing on Python 3.10.

    Defaults to the version of the discovered Python interpreter.

    @@ -3295,6 +3296,7 @@ by --python-version.

  • aarch64-manylinux_2_38: An ARM64 target for the manylinux_2_38 platform
  • aarch64-manylinux_2_39: An ARM64 target for the manylinux_2_39 platform
  • aarch64-manylinux_2_40: An ARM64 target for the manylinux_2_40 platform
    • +
  • wasm32-pyodide2024: A wasm32 target using the the Pyodide 2024 platform. Meant for use with Python 3.12
    • --python-version python-version

    The Python version to use for resolution.

    For example, 3.8 or 3.8.17.

    Defaults to the version of the Python interpreter used for resolution.

    @@ -3533,6 +3535,7 @@ be used with caution, as it can modify the system Python installation.

  • aarch64-manylinux_2_38: An ARM64 target for the manylinux_2_38 platform
  • aarch64-manylinux_2_39: An ARM64 target for the manylinux_2_39 platform
  • aarch64-manylinux_2_40: An ARM64 target for the manylinux_2_40 platform
    • +
  • wasm32-pyodide2024: A wasm32 target using the the Pyodide 2024 platform. Meant for use with Python 3.12
    • --python-version python-version

    The minimum Python version that should be supported by the requirements (e.g., 3.7 or 3.7.9).

    If a patch version is omitted, the minimum patch version is assumed. For example, 3.7 is mapped to 3.7.0.

    --quiet, -q

    Use quiet output.

    @@ -3796,6 +3799,7 @@ should be used with caution, as it can modify the system Python installation.

    aarch64-manylinux_2_38: An ARM64 target for the manylinux_2_38 platform
  • aarch64-manylinux_2_39: An ARM64 target for the manylinux_2_39 platform
  • aarch64-manylinux_2_40: An ARM64 target for the manylinux_2_40 platform
    • +
  • wasm32-pyodide2024: A wasm32 target using the the Pyodide 2024 platform. Meant for use with Python 3.12
    • --python-version python-version

    The minimum Python version that should be supported by the requirements (e.g., 3.7 or 3.7.9).

    If a patch version is omitted, the minimum patch version is assumed. For example, 3.7 is mapped to 3.7.0.

    --quiet, -q

    Use quiet output.

    diff --git a/uv.schema.json b/uv.schema.json index 54d5a0c74..19a6bf4bd 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -2338,6 +2338,13 @@ "enum": [ "aarch64-manylinux_2_40" ] + }, + { + "description": "A wasm32 target using the the Pyodide 2024 platform. Meant for use with Python 3.12.", + "type": "string", + "enum": [ + "wasm32-pyodide2024" + ] } ] },