mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 10:58:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Respect pyproject.toml credentials from user-provided requirements (#7474)

Browse source 
## Summary

When syncing a lockfile, we need to respect credentials defined in the
`pyproject.toml`, even if they won't be used for resolution.
Unfortunately, this includes credentials in `tool.uv.sources`,
`tool.uv.dev-dependencies`, `project.dependencies`, and
`project.optional-dependencies`.

Closes https://github.com/astral-sh/uv/issues/7453.
This commit is contained in:
Charlie Marsh 2024-09-17 15:09:11 -04:00 • committed by GitHub
parent 08a7c708d1
commit c2ad31aa58
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 317 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

25
crates/uv-git/src/credentials.rs
View file

@ -1,9 +1,15 @@
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use cache_key::RepositoryUrl;
use std::collections::HashMap;
use std::sync::{Arc, LazyLock, RwLock};
use tracing::trace;
use url::Url;
use uv_auth::Credentials;
/// Global authentication cache for a uv invocation.
///
/// This is used to share Git credentials within a single process.
pub static GIT_STORE: LazyLock<GitStore> = LazyLock::new(GitStore::default);
/// A store for Git credentials.
#[derive(Debug, Default)]
pub struct GitStore(RwLock<HashMap<RepositoryUrl, Arc<Credentials>>>);
@ -19,3 +25,16 @@ impl GitStore {
self.0.read().unwrap().get(url).cloned()
}
}
/// Populate the global authentication store with credentials on a Git URL, if there are any.
///
/// Returns `true` if the store was updated.
pub fn store_credentials_from_url(url: &Url) -> bool {
if let Some(credentials) = Credentials::from_url(url) {
trace!("Caching credentials for {url}");
GIT_STORE.insert(RepositoryUrl::new(url), credentials);
true
} else {
false
}
}

9
crates/uv-git/src/lib.rs
View file

@ -1,8 +1,6 @@
use std::sync::LazyLock;
use url::Url;
use crate::credentials::GitStore;
pub use crate::credentials::{store_credentials_from_url, GIT_STORE};
pub use crate::git::GitReference;
pub use crate::resolver::{
GitResolver, GitResolverError, RepositoryReference, ResolvedRepositoryReference,
@ -16,11 +14,6 @@ mod resolver;
mod sha;
mod source;
/// Global authentication cache for a uv invocation.
///
/// This is used to share Git credentials within a single process.
pub static GIT_STORE: LazyLock<GitStore> = LazyLock::new(GitStore::default);
/// A URL reference to a Git repository.
#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Hash, Ord)]
pub struct GitUrl {

2
crates/uv-workspace/src/pyproject.rs
View file

@ -120,6 +120,8 @@ pub struct Project {
pub version: Option<Version>,
/// The Python versions this project is compatible with.
pub requires_python: Option<VersionSpecifiers>,
/// The dependencies of the project.
pub dependencies: Option<Vec<String>>,
/// The optional dependencies of the project.
pub optional_dependencies: Option<BTreeMap<ExtraName, Vec<String>>>,

96
crates/uv-workspace/src/workspace.rs
View file

@ -531,6 +531,22 @@ impl Workspace {
&self.sources
}
/// Returns an iterator over all sources in the workspace.
pub fn iter_sources(&self) -> impl Iterator<Item = &Source> {
self.packages
.values()
.filter_map(|member| {
member.pyproject_toml().tool.as_ref().and_then(|tool| {
tool.uv
.as_ref()
.and_then(|uv| uv.sources.as_ref())
.map(ToolUvSources::inner)
.map(|sources| sources.values())
})
})
.flatten()
}
/// The `pyproject.toml` of the workspace.
pub fn pyproject_toml(&self) -> &PyProjectToml {
&self.pyproject_toml
@ -1608,6 +1624,9 @@ mod tests {
"name": "bird-feeder",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"anyio>=4.3.0,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1619,6 +1638,9 @@ mod tests {
"name": "bird-feeder",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"anyio>=4.3.0,<5"
],
"optional-dependencies": null
},
"tool": null
@ -1653,6 +1675,9 @@ mod tests {
"name": "bird-feeder",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"anyio>=4.3.0,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1664,6 +1689,9 @@ mod tests {
"name": "bird-feeder",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"anyio>=4.3.0,<5"
],
"optional-dependencies": null
},
"tool": null
@ -1697,6 +1725,10 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"bird-feeder",
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1707,6 +1739,10 @@ mod tests {
"name": "bird-feeder",
"version": "1.0.0",
"requires-python": ">=3.8",
"dependencies": [
"anyio>=4.3.0,<5",
"seeds"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1717,6 +1753,9 @@ mod tests {
"name": "seeds",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"idna==3.6"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1732,6 +1771,10 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"bird-feeder",
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"tool": {
@ -1786,6 +1829,10 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"bird-feeder",
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1796,6 +1843,10 @@ mod tests {
"name": "bird-feeder",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"anyio>=4.3.0,<5",
"seeds"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1806,6 +1857,9 @@ mod tests {
"name": "seeds",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"idna==3.6"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1861,6 +1915,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1872,6 +1929,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"tool": null
@ -1973,6 +2033,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1983,6 +2046,9 @@ mod tests {
"name": "seeds",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"idna==3.6"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -1994,6 +2060,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"tool": {
@ -2062,6 +2131,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -2072,6 +2144,9 @@ mod tests {
"name": "seeds",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"idna==3.6"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -2083,6 +2158,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"tool": {
@ -2152,6 +2230,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -2162,6 +2243,9 @@ mod tests {
"name": "bird-feeder",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"anyio>=4.3.0,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -2172,6 +2256,9 @@ mod tests {
"name": "seeds",
"version": "1.0.0",
"requires-python": ">=3.12",
"dependencies": [
"idna==3.6"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -2183,6 +2270,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"tool": {
@ -2252,6 +2342,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"pyproject_toml": "[PYPROJECT_TOML]"
@ -2263,6 +2356,9 @@ mod tests {
"name": "albatross",
"version": "0.1.0",
"requires-python": ">=3.12",
"dependencies": [
"tqdm>=4,<5"
],
"optional-dependencies": null
},
"tool": {

107
crates/uv/src/commands/project/sync.rs
View file

@ -1,9 +1,19 @@
use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger, InstallLogger};
use crate::commands::pip::operations::Modifications;
use crate::commands::project::lock::do_safe_lock;
use crate::commands::project::{ProjectError, SharedState};
use crate::commands::{pip, project, ExitStatus};
use crate::printer::Printer;
use crate::settings::{InstallerSettingsRef, ResolverInstallerSettings};
use anyhow::{Context, Result};
use itertools::Itertools;
use distribution_types::{DirectorySourceDist, Dist, ResolvedDist, SourceDist};
use pep508_rs::MarkerTree;
use uv_auth::store_credentials_from_url;
use itertools::Itertools;
use pep508_rs::{MarkerTree, Requirement, VersionOrUrl};
use pypi_types::{
LenientRequirement, ParsedArchiveUrl, ParsedGitUrl, ParsedUrl, VerbatimParsedUrl,
};
use std::borrow::Cow;
use std::str::FromStr;
use uv_cache::Cache;
use uv_client::{Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
@ -18,16 +28,9 @@ use uv_python::{PythonDownloads, PythonEnvironment, PythonPreference, PythonRequ
use uv_resolver::{FlatIndex, Lock};
use uv_types::{BuildIsolation, HashStrategy};
use uv_warnings::warn_user;
use uv_workspace::pyproject::{Source, ToolUvSources};
use uv_workspace::{DiscoveryOptions, InstallTarget, MemberDiscovery, VirtualProject, Workspace};
use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger, InstallLogger};
use crate::commands::pip::operations::Modifications;
use crate::commands::project::lock::do_safe_lock;
use crate::commands::project::{ProjectError, SharedState};
use crate::commands::{pip, project, ExitStatus};
use crate::printer::Printer;
use crate::settings::{InstallerSettingsRef, ResolverInstallerSettings};
/// Sync the project environment.
#[allow(clippy::fn_params_excessive_bools)]
pub(crate) async fn sync(
@ -250,9 +253,12 @@ pub(super) async fn do_sync(
// Add all authenticated sources to the cache.
for url in index_locations.urls() {
store_credentials_from_url(url);
uv_auth::store_credentials_from_url(url);
}
// Populate credentials from the workspace.
store_credentials_from_workspace(target.workspace());
// Initialize the registry client.
let client = RegistryClientBuilder::new(cache.clone())
.native_tls(native_tls)
@ -399,3 +405,78 @@ fn apply_editable_mode(
}),
}
}
fn store_credentials_from_workspace(workspace: &Workspace) {
for member in workspace.packages().values() {
// Iterate over the `tool.uv.sources`.
for source in member
.pyproject_toml()
.tool
.as_ref()
.and_then(|tool| tool.uv.as_ref())
.and_then(|uv| uv.sources.as_ref())
.map(ToolUvSources::inner)
.iter()
.flat_map(|sources| sources.values())
{
match source {
Source::Git { git, .. } => {
uv_git::store_credentials_from_url(git);
}
Source::Url { url, .. } => {
uv_auth::store_credentials_from_url(url);
}
_ => {}
}
}
// Iterate over all dependencies.
let dependencies = member
.pyproject_toml()
.project
.as_ref()
.and_then(|project| project.dependencies.as_ref())
.into_iter()
.flatten();
let optional_dependencies = member
.pyproject_toml()
.project
.as_ref()
.and_then(|project| project.optional_dependencies.as_ref())
.into_iter()
.flat_map(|optional| optional.values())
.flatten();
let dev_dependencies = member
.pyproject_toml()
.tool
.as_ref()
.and_then(|tool| tool.uv.as_ref())
.and_then(|uv| uv.dev_dependencies.as_ref())
.into_iter()
.flatten();
for requirement in dependencies
.chain(optional_dependencies)
.filter_map(|requires_dist| {
LenientRequirement::<VerbatimParsedUrl>::from_str(requires_dist)
.map(Requirement::from)
.map(Cow::Owned)
.ok()
})
.chain(dev_dependencies.map(Cow::Borrowed))
{
let Some(VersionOrUrl::Url(url)) = &requirement.version_or_url else {
continue;
};
match &url.parsed_url {
ParsedUrl::Git(ParsedGitUrl { url, .. }) => {
uv_git::store_credentials_from_url(url.repository());
}
ParsedUrl::Archive(ParsedArchiveUrl { url, .. }) => {
uv_auth::store_credentials_from_url(url);
}
_ => {}
}
}
}
}

4
crates/uv/tests/common/mod.rs
View file

@ -442,7 +442,7 @@ impl TestContext {
if cfg!(all(windows, debug_assertions)) {
// TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the
// default windows stack of 1MB
command.env("UV_STACK_SIZE", (2 * 1024 * 1024).to_string());
command.env("UV_STACK_SIZE", (4 * 1024 * 1024).to_string());
}
}
@ -533,7 +533,7 @@ impl TestContext {
if cfg!(all(windows, debug_assertions)) {
// TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the
// default windows stack of 1MB
command.env("UV_STACK_SIZE", (2 * 1024 * 1024).to_string());
command.env("UV_STACK_SIZE", (4 * 1024 * 1024).to_string());
}
command

101
crates/uv/tests/lock.rs
View file

@ -6034,7 +6034,7 @@ fn lock_redact_https() -> Result<()> {
/// However, we don't currently avoid persisting Git credentials in `uv.lock`.
#[test]
fn lock_redact_git() -> Result<()> {
fn lock_redact_git_pep508() -> Result<()> {
let context = TestContext::new("3.12");
let token = decode_token(common::READ_ONLY_GITHUB_TOKEN);
@ -6111,7 +6111,104 @@ fn lock_redact_git() -> Result<()> {
"###);
// Install from the lockfile.
uv_snapshot!(&filters, context.sync().arg("--frozen"), @r###"
uv_snapshot!(&filters, context.sync().arg("--frozen").arg("--reinstall").arg("--no-cache"), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Prepared 2 packages in [TIME]
Installed 2 packages in [TIME]
+ foo==0.1.0 (from file://[TEMP_DIR]/)
+ uv-private-pypackage==0.1.0 (from git+https://github.com/astral-test/uv-private-pypackage@d780faf0ac91257d4d5a4f0c5a0e4509608c0071)
"###);
Ok(())
}
/// However, we don't currently avoid persisting Git credentials in `uv.lock`.
#[test]
fn lock_redact_git_sources() -> Result<()> {
let context = TestContext::new("3.12");
let token = decode_token(common::READ_ONLY_GITHUB_TOKEN);
let filters: Vec<_> = [(token.as_str(), "***")]
.into_iter()
.chain(context.filters())
.collect();
let pyproject_toml = context.temp_dir.child("pyproject.toml");
pyproject_toml.write_str(&formatdoc! {
r#"
[project]
name = "foo"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = ["uv-private-pypackage"]
[build-system]
requires = ["setuptools>=42"]
build-backend = "setuptools.build_meta"
[tool.uv.sources]
uv-private-pypackage = {{ git = "https://{token}@github.com/astral-test/uv-private-pypackage" }}
"#,
token = token,
})?;
uv_snapshot!(&filters, context.lock(), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 2 packages in [TIME]
"###);
let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap();
insta::with_settings!({
filters => filters.clone(),
}, {
assert_snapshot!(
lock, @r###"
version = 1
requires-python = ">=3.12"
[options]
exclude-newer = "2024-03-25T00:00:00Z"
[[package]]
name = "foo"
version = "0.1.0"
source = { editable = "." }
dependencies = [
{ name = "uv-private-pypackage" },
]
[package.metadata]
requires-dist = [{ name = "uv-private-pypackage", git = "https://github.com/astral-test/uv-private-pypackage" }]
[[package]]
name = "uv-private-pypackage"
version = "0.1.0"
source = { git = "https://github.com/astral-test/uv-private-pypackage#d780faf0ac91257d4d5a4f0c5a0e4509608c0071" }
"###
);
});
// Re-run with `--locked`.
uv_snapshot!(&filters, context.lock().arg("--locked"), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 2 packages in [TIME]
"###);
// Install from the lockfile.
uv_snapshot!(&filters, context.sync().arg("--frozen").arg("--reinstall").arg("--no-cache"), @r###"
success: true
exit_code: 0
----- stdout -----

2
crates/uv/tests/pip_compile.rs
View file

@ -686,7 +686,7 @@ build-backend = "poetry.core.masonry.api"
----- stdout -----
----- stderr -----
error: Failed to extract static metadata from `pyproject.toml`
error: Failed to parse: `pyproject.toml`
Caused by: TOML parse error at line 13, column 1
|
13 | [project.dependencies]