From 326e4497da9fc64d88a5540ed0463be1db7d65e3 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 26 Jun 2025 16:17:42 -0400 Subject: [PATCH] Allow local indexes to reference remote files (#14294) ## Summary Previously, we assumed that local indexes only referenced local files. However, it's fine for a local index (like, a `file://`-based Simple API) to reference a remote file, and in fact Pyodide operates this way. Closes https://github.com/astral-sh/uv/issues/14227. ## Test Plan Ran `UV_INDEX=$(pyodide config get package_index) cargo run add anyio`, which produced this lockfile: ```toml version = 1 revision = 2 requires-python = ">=3.13.2" [[package]] name = "anyio" version = "4.9.0" source = { registry = "../../../Library/Caches/.pyodide-xbuildenv-0.30.5/0.27.7/xbuildenv/pyodide-root/package_index" } dependencies = [ { name = "idna" }, { name = "sniffio" }, ] wheels = [ { url = "https://cdn.jsdelivr.net/pyodide/v0.27.7/full/anyio-4.9.0-py3-none-any.whl", hash = "sha256:e1d9180d4361fd71d1bc4a7007fea6cae1d18792dba9d07eaad89f2a8562f71c" }, ] [[package]] name = "foo" version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "anyio" }, ] [package.metadata] requires-dist = [{ name = "anyio", specifier = ">=4.9.0" }] [[package]] name = "idna" version = "3.7" source = { registry = "../../../Library/Caches/.pyodide-xbuildenv-0.30.5/0.27.7/xbuildenv/pyodide-root/package_index" } wheels = [ { url = "https://cdn.jsdelivr.net/pyodide/v0.27.7/full/idna-3.7-py3-none-any.whl", hash = "sha256:9d4685891e3e37434e09b1becda7e96a284e660c7aea9222564d88b6c3527c09" }, ] [[package]] name = "sniffio" version = "1.3.1" source = { registry = "../../../Library/Caches/.pyodide-xbuildenv-0.30.5/0.27.7/xbuildenv/pyodide-root/package_index" } wheels = [ { url = "https://cdn.jsdelivr.net/pyodide/v0.27.7/full/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:9215f9917b34fc73152b134a3fc0a2eb0e4a49b0b956100cad75e84943412bb9" }, ] ``` --- crates/uv-resolver/src/lock/mod.rs | 198 +++++++++++++++++++---------- 1 file changed, 129 insertions(+), 69 deletions(-) diff --git a/crates/uv-resolver/src/lock/mod.rs b/crates/uv-resolver/src/lock/mod.rs index cee6364a9..9834ad845 100644 --- a/crates/uv-resolver/src/lock/mod.rs +++ b/crates/uv-resolver/src/lock/mod.rs @@ -2541,16 +2541,30 @@ impl Package { .as_ref() .expect("version for registry source"); - let file_path = sdist.path().ok_or_else(|| LockErrorKind::MissingPath { - name: name.clone(), - version: version.clone(), - })?; - let file_path = workspace_root.join(path).join(file_path); - let file_url = DisplaySafeUrl::from_file_path(&file_path).map_err(|()| { - LockErrorKind::PathToUrl { - path: file_path.into_boxed_path(), + let file_url = match sdist { + SourceDist::Url { url: file_url, .. } => { + FileLocation::AbsoluteUrl(file_url.clone()) } - })?; + SourceDist::Path { + path: file_path, .. + } => { + let file_path = workspace_root.join(path).join(file_path); + let file_url = + DisplaySafeUrl::from_file_path(&file_path).map_err(|()| { + LockErrorKind::PathToUrl { + path: file_path.into_boxed_path(), + } + })?; + FileLocation::AbsoluteUrl(UrlString::from(file_url)) + } + SourceDist::Metadata { .. } => { + return Err(LockErrorKind::MissingPath { + name: name.clone(), + version: version.clone(), + } + .into()); + } + }; let filename = sdist .filename() .ok_or_else(|| LockErrorKind::MissingFilename { @@ -2571,9 +2585,10 @@ impl Package { requires_python: None, size: sdist.size(), upload_time_utc_ms: sdist.upload_time().map(Timestamp::as_millisecond), - url: FileLocation::AbsoluteUrl(UrlString::from(file_url)), + url: file_url, yanked: None, }); + let index = IndexUrl::from( VerbatimUrl::from_absolute_path(workspace_root.join(path)) .map_err(LockErrorKind::RegistryVerbatimUrl)?, @@ -3688,14 +3703,6 @@ impl SourceDist { } } - fn path(&self) -> Option<&Path> { - match &self { - SourceDist::Metadata { .. } => None, - SourceDist::Url { .. } => None, - SourceDist::Path { path, .. } => Some(path), - } - } - pub(crate) fn hash(&self) -> Option<&Hash> { match &self { SourceDist::Metadata { metadata } => metadata.hash.as_ref(), @@ -3818,34 +3825,57 @@ impl SourceDist { let index_path = path .to_file_path() .map_err(|()| LockErrorKind::UrlToPath { url: path.to_url() })?; - let reg_dist_url = reg_dist + let url = reg_dist .file .url .to_url() .map_err(LockErrorKind::InvalidUrl)?; - let reg_dist_path = reg_dist_url - .to_file_path() - .map_err(|()| LockErrorKind::UrlToPath { url: reg_dist_url })?; - let path = relative_to(®_dist_path, index_path) - .or_else(|_| std::path::absolute(®_dist_path)) - .map_err(LockErrorKind::DistributionRelativePath)? - .into_boxed_path(); - let hash = reg_dist.file.hashes.iter().max().cloned().map(Hash::from); - let size = reg_dist.file.size; - let upload_time = reg_dist - .file - .upload_time_utc_ms - .map(Timestamp::from_millisecond) - .transpose() - .map_err(LockErrorKind::InvalidTimestamp)?; - Ok(Some(SourceDist::Path { - path, - metadata: SourceDistMetadata { - hash, - size, - upload_time, - }, - })) + + if url.scheme() == "file" { + let reg_dist_path = url + .to_file_path() + .map_err(|()| LockErrorKind::UrlToPath { url })?; + let path = relative_to(®_dist_path, index_path) + .or_else(|_| std::path::absolute(®_dist_path)) + .map_err(LockErrorKind::DistributionRelativePath)? + .into_boxed_path(); + let hash = reg_dist.file.hashes.iter().max().cloned().map(Hash::from); + let size = reg_dist.file.size; + let upload_time = reg_dist + .file + .upload_time_utc_ms + .map(Timestamp::from_millisecond) + .transpose() + .map_err(LockErrorKind::InvalidTimestamp)?; + Ok(Some(SourceDist::Path { + path, + metadata: SourceDistMetadata { + hash, + size, + upload_time, + }, + })) + } else { + let url = normalize_file_location(®_dist.file.url) + .map_err(LockErrorKind::InvalidUrl) + .map_err(LockError::from)?; + let hash = reg_dist.file.hashes.iter().max().cloned().map(Hash::from); + let size = reg_dist.file.size; + let upload_time = reg_dist + .file + .upload_time_utc_ms + .map(Timestamp::from_millisecond) + .transpose() + .map_err(LockErrorKind::InvalidTimestamp)?; + Ok(Some(SourceDist::Url { + url, + metadata: SourceDistMetadata { + hash, + size, + upload_time, + }, + })) + } } } } @@ -4152,20 +4182,42 @@ impl Wheel { .to_file_path() .map_err(|()| LockErrorKind::UrlToPath { url: path.to_url() })?; let wheel_url = wheel.file.url.to_url().map_err(LockErrorKind::InvalidUrl)?; - let wheel_path = wheel_url - .to_file_path() - .map_err(|()| LockErrorKind::UrlToPath { url: wheel_url })?; - let path = relative_to(&wheel_path, index_path) - .or_else(|_| std::path::absolute(&wheel_path)) - .map_err(LockErrorKind::DistributionRelativePath)? - .into_boxed_path(); - Ok(Wheel { - url: WheelWireSource::Path { path }, - hash: None, - size: None, - upload_time: None, - filename, - }) + + if wheel_url.scheme() == "file" { + let wheel_path = wheel_url + .to_file_path() + .map_err(|()| LockErrorKind::UrlToPath { url: wheel_url })?; + let path = relative_to(&wheel_path, index_path) + .or_else(|_| std::path::absolute(&wheel_path)) + .map_err(LockErrorKind::DistributionRelativePath)? + .into_boxed_path(); + Ok(Wheel { + url: WheelWireSource::Path { path }, + hash: None, + size: None, + upload_time: None, + filename, + }) + } else { + let url = normalize_file_location(&wheel.file.url) + .map_err(LockErrorKind::InvalidUrl) + .map_err(LockError::from)?; + let hash = wheel.file.hashes.iter().max().cloned().map(Hash::from); + let size = wheel.file.size; + let upload_time = wheel + .file + .upload_time_utc_ms + .map(Timestamp::from_millisecond) + .transpose() + .map_err(LockErrorKind::InvalidTimestamp)?; + Ok(Wheel { + url: WheelWireSource::Url { url }, + hash, + size, + filename, + upload_time, + }) + } } } } @@ -4203,8 +4255,10 @@ impl Wheel { match source { RegistrySource::Url(url) => { - let file_url = match &self.url { - WheelWireSource::Url { url } => url, + let file_location = match &self.url { + WheelWireSource::Url { url: file_url } => { + FileLocation::AbsoluteUrl(file_url.clone()) + } WheelWireSource::Path { .. } | WheelWireSource::Filename { .. } => { return Err(LockErrorKind::MissingUrl { name: filename.name, @@ -4220,7 +4274,7 @@ impl Wheel { requires_python: None, size: self.size, upload_time_utc_ms: self.upload_time.map(Timestamp::as_millisecond), - url: FileLocation::AbsoluteUrl(file_url.clone()), + url: file_location, yanked: None, }); let index = IndexUrl::from(VerbatimUrl::from_url( @@ -4233,9 +4287,21 @@ impl Wheel { }) } RegistrySource::Path(index_path) => { - let file_path = match &self.url { - WheelWireSource::Path { path } => path, - WheelWireSource::Url { .. } | WheelWireSource::Filename { .. } => { + let file_location = match &self.url { + WheelWireSource::Url { url: file_url } => { + FileLocation::AbsoluteUrl(file_url.clone()) + } + WheelWireSource::Path { path: file_path } => { + let file_path = root.join(index_path).join(file_path); + let file_url = + DisplaySafeUrl::from_file_path(&file_path).map_err(|()| { + LockErrorKind::PathToUrl { + path: file_path.into_boxed_path(), + } + })?; + FileLocation::AbsoluteUrl(UrlString::from(file_url)) + } + WheelWireSource::Filename { .. } => { return Err(LockErrorKind::MissingPath { name: filename.name, version: filename.version, @@ -4243,12 +4309,6 @@ impl Wheel { .into()); } }; - let file_path = root.join(index_path).join(file_path); - let file_url = DisplaySafeUrl::from_file_path(&file_path).map_err(|()| { - LockErrorKind::PathToUrl { - path: file_path.into_boxed_path(), - } - })?; let file = Box::new(uv_distribution_types::File { dist_info_metadata: false, filename: SmallString::from(filename.to_string()), @@ -4256,7 +4316,7 @@ impl Wheel { requires_python: None, size: self.size, upload_time_utc_ms: self.upload_time.map(Timestamp::as_millisecond), - url: FileLocation::AbsoluteUrl(UrlString::from(file_url)), + url: file_location, yanked: None, }); let index = IndexUrl::from(