Add UV_PYTHON_DOWNLOADS_JSON_URL to set custom managed python sources (#10939)

## Summary

Add an option to overwrite the list of available Python downloads from a
local JSON file by using the environment variable
`UV_PYTHON_DOWNLOADS_JSON_URL`

as an experimental support for providing custom sources for Python
distribution binaries #8015

related #10203

I probably should make the JSON to be fetched from a remote URL instead
of a local file.
please let me know what you think and I will modify the code
accordingly.

## Test Plan

### normal run
```
root@75c66494ba8b:/# /code/target/release/uv python list
cpython-3.14.0a4+freethreaded-linux-x86_64-gnu    <download available>
cpython-3.14.0a4-linux-x86_64-gnu                 <download available>
cpython-3.13.1+freethreaded-linux-x86_64-gnu      <download available>
cpython-3.13.1-linux-x86_64-gnu                   <download available>
cpython-3.12.8-linux-x86_64-gnu                   <download available>
cpython-3.11.11-linux-x86_64-gnu                  <download available>
cpython-3.10.16-linux-x86_64-gnu                  <download available>
cpython-3.9.21-linux-x86_64-gnu                   <download available>
cpython-3.8.20-linux-x86_64-gnu                   <download available>
cpython-3.7.9-linux-x86_64-gnu                    <download available>
pypy-3.10.14-linux-x86_64-gnu                     <download available>
pypy-3.9.19-linux-x86_64-gnu                      <download available>
pypy-3.8.16-linux-x86_64-gnu                      <download available>
pypy-3.7.13-linux-x86_64-gnu                      <download available>
```

### empty JSON file
```sh
root@75c66494ba8b:/# export UV_PYTHON_DOWNLOADS_JSON_URL=/code/crates/uv-python/my-download-metadata.json 
root@75c66494ba8b:/# cat $UV_PYTHON_DOWNLOADS_JSON_URL 
{}
root@75c66494ba8b:/# /code/target/release/uv python list
root@75c66494ba8b:/# 
```

### JSON file with valid version
```sh
root@75c66494ba8b:/# export UV_PYTHON_DOWNLOADS_JSON_URL=/code/crates/uv-python/my-download-metadata.json 
root@75c66494ba8b:/# cat $UV_PYTHON_DOWNLOADS_JSON_URL 
{
  "cpython-3.11.9-linux-x86_64-gnu": {
    "name": "cpython",
    "arch": {
      "family": "x86_64",
      "variant": null
    },
    "os": "linux",
    "libc": "gnu",
    "major": 3,
    "minor": 11,
    "patch": 9,
    "prerelease": "",
    "url": "20240814/cpython-3.11.9%2B20240814-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz",
    "sha256": "daa487c7e73005c4426ac393273117cf0e2dc4ab9b2eeda366e04cd00eea00c9",
    "variant": null
  }
}
root@75c66494ba8b:/# /code/target/release/uv python list
cpython-3.11.9-linux-x86_64-gnu    <download available>
root@75c66494ba8b:/# 
```

### Remote Path

```sh
root@75c66494ba8b:/# export UV_PYTHON_DOWNLOADS_JSON_URL=http://a.com/file.json 
root@75c66494ba8b:/# /code/target/release/uv python list
error: Remote python downloads JSON is not yet supported, please use a local path (without `file://` prefix)
```

---------

Co-authored-by: Aria Desires <aria.desires@gmail.com>

crates/uv-python/src/downloads.inc.mustache

@@ -1,39 +0,0 @@
-// Generated with `{{generated_with}}`
-// From template at `{{generated_from}}`
-
-use uv_pep440::{Prerelease, PrereleaseKind};
-use crate::PythonVariant;
-use crate::platform::ArchVariant;
-
-pub(crate) const PYTHON_DOWNLOADS: &[ManagedPythonDownload] = &[
-    {{#versions}}
-    ManagedPythonDownload {
-        key: PythonInstallationKey {
-            major: {{value.major}},
-            minor: {{value.minor}},
-            patch: {{value.patch}},
-            prerelease: {{value.prerelease}},
-            implementation: LenientImplementationName::Known(ImplementationName::{{value.name}}),
-            arch: Arch{
-                family: target_lexicon::Architecture::{{value.arch_family}},
-                variant: {{value.arch_variant}},
-            },
-            os: Os(target_lexicon::OperatingSystem::{{value.os}}),
-            {{#value.libc}}
-            libc: Libc::Some(target_lexicon::Environment::{{.}}),
-            {{/value.libc}}
-            {{^value.libc}}
-            libc: Libc::None,
-            {{/value.libc}}
-            variant: {{value.variant}}
-        },
-        url: "{{value.url}}",
-        {{#value.sha256}}
-        sha256: Some("{{.}}")
-        {{/value.sha256}}
-        {{^value.sha256}}
-        sha256: None
-        {{/value.sha256}}
-    },
-    {{/versions}}
-];

crates/uv-python/src/downloads.rs

@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
 use std::fmt::Display;
 use std::io;
 use std::path::{Path, PathBuf};
@@ -7,8 +9,11 @@ use std::task::{Context, Poll};
 use std::time::{Duration, SystemTime};
 
 use futures::TryStreamExt;
+use itertools::Itertools;
+use once_cell::sync::OnceCell;
 use owo_colors::OwoColorize;
 use reqwest_retry::RetryPolicy;
+use serde::Deserialize;
 use thiserror::Error;
 use tokio::io::{AsyncRead, ReadBuf};
 use tokio_util::compat::FuturesAsyncReadCompatExt;
@@ -30,6 +35,7 @@ use crate::installation::PythonInstallationKey;
 use crate::libc::LibcDetectionError;
 use crate::managed::ManagedPythonInstallation;
 use crate::platform::{self, Arch, Libc, Os};
+use crate::PythonVariant;
 use crate::{Interpreter, PythonRequest, PythonVersion, VersionRequest};
 
 #[derive(Error, Debug)]
@@ -86,9 +92,13 @@ pub enum Error {
     Mirror(&'static str, &'static str),
     #[error(transparent)]
     LibcDetection(#[from] LibcDetectionError),
+    #[error("Remote python downloads JSON is not yet supported, please use a local path (without `file://` prefix)")]
+    RemoteJSONNotSupported(),
+    #[error("The json of the python downloads is invalid: {0}")]
+    InvalidPythonDownloadsJSON(String, #[source] serde_json::Error),
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct ManagedPythonDownload {
     key: PythonInstallationKey,
     url: &'static str,
@@ -245,9 +255,11 @@ impl PythonDownloadRequest {
     }
 
     /// Iterate over all [`PythonDownload`]'s that match this request.
-    pub fn iter_downloads(&self) -> impl Iterator<Item = &'static ManagedPythonDownload> + '_ {
-        ManagedPythonDownload::iter_all()
-            .filter(move |download| self.satisfied_by_download(download))
+    pub fn iter_downloads(
+        &self,
+    ) -> Result<impl Iterator<Item = &'static ManagedPythonDownload> + use<'_>, Error> {
+        Ok(ManagedPythonDownload::iter_all()?
+            .filter(move |download| self.satisfied_by_download(download)))
     }
 
     /// Whether this request is satisfied by an installation key.
@@ -445,7 +457,30 @@ impl FromStr for PythonDownloadRequest {
     }
 }
 
-include!("downloads.inc");
+const BUILTIN_PYTHON_DOWNLOADS_JSON: &str = include_str!("download-metadata-minified.json");
+static PYTHON_DOWNLOADS: OnceCell<std::borrow::Cow<'static, [ManagedPythonDownload]>> =
+    OnceCell::new();
+
+#[derive(Debug, Deserialize, Clone)]
+struct JsonPythonDownload {
+    name: String,
+    arch: JsonArch,
+    os: String,
+    libc: String,
+    major: u8,
+    minor: u8,
+    patch: u8,
+    prerelease: Option<String>,
+    url: String,
+    sha256: Option<String>,
+    variant: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+struct JsonArch {
+    family: String,
+    variant: Option<String>,
+}
 
 #[derive(Debug, Clone)]
 pub enum DownloadResult {
@@ -459,14 +494,40 @@ impl ManagedPythonDownload {
         request: &PythonDownloadRequest,
     ) -> Result<&'static ManagedPythonDownload, Error> {
         request
-            .iter_downloads()
+            .iter_downloads()?
             .next()
             .ok_or(Error::NoDownloadFound(request.clone()))
     }
 
     /// Iterate over all [`ManagedPythonDownload`]s.
-    pub fn iter_all() -> impl Iterator<Item = &'static ManagedPythonDownload> {
-        PYTHON_DOWNLOADS.iter()
+    pub fn iter_all() -> Result<impl Iterator<Item = &'static ManagedPythonDownload>, Error> {
+        let runtime_source = std::env::var(EnvVars::UV_PYTHON_DOWNLOADS_JSON_URL);
+
+        let downloads = PYTHON_DOWNLOADS.get_or_try_init(|| {
+            let json_downloads: HashMap<String, JsonPythonDownload> =
+                if let Ok(json_source) = &runtime_source {
+                    if Url::parse(json_source).is_ok() {
+                        return Err(Error::RemoteJSONNotSupported());
+                    }
+
+                    let file = match fs_err::File::open(json_source) {
+                        Ok(file) => file,
+                        Err(e) => { Err(Error::Io(e)) }?,
+                    };
+
+                    serde_json::from_reader(file)
+                        .map_err(|e| Error::InvalidPythonDownloadsJSON(json_source.clone(), e))?
+                } else {
+                    serde_json::from_str(BUILTIN_PYTHON_DOWNLOADS_JSON).map_err(|e| {
+                        Error::InvalidPythonDownloadsJSON("EMBEDDED IN THE BINARY".to_string(), e)
+                    })?
+                };
+
+            let result = parse_json_downloads(json_downloads);
+            Ok(Cow::Owned(result))
+        })?;
+
+        Ok(downloads.iter())
     }
 
     pub fn url(&self) -> &'static str {
@@ -702,6 +763,115 @@ impl ManagedPythonDownload {
     }
 }
 
+fn parse_json_downloads(
+    json_downloads: HashMap<String, JsonPythonDownload>,
+) -> Vec<ManagedPythonDownload> {
+    json_downloads
+        .into_iter()
+        .filter_map(|(key, entry)| {
+            let implementation = match entry.name.as_str() {
+                "cpython" => LenientImplementationName::Known(ImplementationName::CPython),
+                "pypy" => LenientImplementationName::Known(ImplementationName::PyPy),
+                _ => LenientImplementationName::Unknown(entry.name.clone()),
+            };
+
+            let arch_str = match entry.arch.family.as_str() {
+                "armv5tel" => "armv5te".to_string(),
+                // The `gc` variant of riscv64 is the common base instruction set and
+                // is the target in `python-build-standalone`
+                // See https://github.com/astral-sh/python-build-standalone/issues/504
+                "riscv64" => "riscv64gc".to_string(),
+                value => value.to_string(),
+            };
+
+            let arch_str = if let Some(variant) = entry.arch.variant {
+                format!("{arch_str}_{variant}")
+            } else {
+                arch_str
+            };
+
+            let arch = match Arch::from_str(&arch_str) {
+                Ok(arch) => arch,
+                Err(e) => {
+                    debug!("Skipping entry {key}: Invalid arch '{arch_str}' - {e}");
+                    return None;
+                }
+            };
+
+            let os = match Os::from_str(&entry.os) {
+                Ok(os) => os,
+                Err(e) => {
+                    debug!("Skipping entry {}: Invalid OS '{}' - {}", key, entry.os, e);
+                    return None;
+                }
+            };
+
+            let libc = match Libc::from_str(&entry.libc) {
+                Ok(libc) => libc,
+                Err(e) => {
+                    debug!(
+                        "Skipping entry {}: Invalid libc '{}' - {}",
+                        key, entry.libc, e
+                    );
+                    return None;
+                }
+            };
+
+            let variant = match entry
+                .variant
+                .as_deref()
+                .map(PythonVariant::from_str)
+                .transpose()
+            {
+                Ok(Some(variant)) => variant,
+                Ok(None) => PythonVariant::default(),
+                Err(()) => {
+                    debug!(
+                        "Skipping entry {key}: Unknown python variant - {}",
+                        entry.variant.unwrap_or_default()
+                    );
+                    return None;
+                }
+            };
+
+            let version_str = format!(
+                "{}.{}.{}{}",
+                entry.major,
+                entry.minor,
+                entry.patch,
+                entry.prerelease.as_deref().unwrap_or_default()
+            );
+
+            let version = match PythonVersion::from_str(&version_str) {
+                Ok(version) => version,
+                Err(e) => {
+                    debug!("Skipping entry {key}: Invalid version '{version_str}' - {e}");
+                    return None;
+                }
+            };
+
+            let url = Box::leak(entry.url.into_boxed_str()) as &'static str;
+            let sha256 = entry
+                .sha256
+                .map(|s| Box::leak(s.into_boxed_str()) as &'static str);
+
+            Some(ManagedPythonDownload {
+                key: PythonInstallationKey::new_from_version(
+                    implementation,
+                    &version,
+                    os,
+                    arch,
+                    libc,
+                    variant,
+                ),
+                url,
+                sha256,
+            })
+        })
+        .sorted_by(|a, b| Ord::cmp(&b.key, &a.key))
+        .collect()
+}
+
 impl Error {
     pub(crate) fn from_reqwest(url: Url, err: reqwest::Error) -> Self {
         Self::NetworkError(url, WrappedReqwestError::from(err))

crates/uv-python/src/installation.rs

@@ -294,7 +294,7 @@ impl PythonInstallationKey {
         }
     }
 
-    fn new_from_version(
+    pub fn new_from_version(
         implementation: LenientImplementationName,
         version: &PythonVersion,
         os: Os,
@@ -482,6 +482,6 @@ impl Ord for PythonInstallationKey {
             .then_with(|| self.os.to_string().cmp(&other.os.to_string()))
             .then_with(|| self.arch.to_string().cmp(&other.arch.to_string()))
             .then_with(|| self.libc.to_string().cmp(&other.libc.to_string()))
-            .then_with(|| self.variant.cmp(&other.variant))
+            .then_with(|| self.variant.cmp(&other.variant).reverse()) // we want Default to come first
     }
 }