Enable environment variable authentication for named indexes (#7741)

## Summary

This PR enables users to provide index credentials via named environment
variables.

For example, given an index named `internal` that requires a username
(`public`) and password
(`koala`), you can define the index (without credentials) in your
`pyproject.toml`:

```toml
[[tool.uv.index]]
name = "internal"
url = "https://pypi-proxy.corp.dev/simple"
```

Then set the `UV_INDEX_INTERNAL_USERNAME` and
`UV_INDEX_INTERNAL_PASSWORD`
environment variables, where `INTERNAL` is the uppercase version of the
index name:

```sh
export UV_INDEX_INTERNAL_USERNAME=public
export UV_INDEX_INTERNAL_PASSWORD=koala
```
This commit is contained in:
Charlie Marsh 2024-10-15 15:35:07 -07:00 committed by GitHub
parent 5b391770df
commit 1925922770
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 284 additions and 67 deletions

View file

@ -139,6 +139,21 @@ impl Credentials {
})
}
/// Extract the [`Credentials`] from the environment, given a named source.
///
/// For example, given a name of `"pytorch"`, search for `UV_HTTP_BASIC_PYTORCH_USERNAME` and
/// `UV_HTTP_BASIC_PYTORCH_PASSWORD`.
pub fn from_env(name: &str) -> Option<Self> {
let name = name.to_uppercase();
let username = std::env::var(format!("UV_HTTP_BASIC_{name}_USERNAME")).ok();
let password = std::env::var(format!("UV_HTTP_BASIC_{name}_PASSWORD")).ok();
if username.is_none() && password.is_none() {
None
} else {
Some(Self::new(username, password))
}
}
/// Parse [`Credentials`] from an HTTP request, if any.
///
/// Only HTTP Basic Authentication is supported.

View file

@ -35,3 +35,11 @@ pub fn store_credentials_from_url(url: &Url) -> bool {
false
}
}
/// Populate the global authentication store with credentials on a URL, if there are any.
///
/// Returns `true` if the store was updated.
pub fn store_credentials(url: &Url, credentials: Credentials) {
trace!("Caching credentials for {url}");
CREDENTIALS_CACHE.insert(url, Arc::new(credentials));
}

View file

@ -16,6 +16,7 @@ doctest = false
workspace = true
[dependencies]
uv-auth = { workspace = true }
uv-cache-info = { workspace = true }
uv-cache-key = { workspace = true }
uv-distribution-filename = { workspace = true }

View file

@ -2,6 +2,7 @@ use crate::{IndexUrl, IndexUrlError};
use std::str::FromStr;
use thiserror::Error;
use url::Url;
use uv_auth::Credentials;
#[derive(Debug, Clone, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
@ -102,6 +103,19 @@ impl Index {
pub fn raw_url(&self) -> &Url {
self.url.url()
}
/// Retrieve the credentials for the index, either from the environment, or from the URL itself.
pub fn credentials(&self) -> Option<Credentials> {
// If the index is named, and credentials are provided via the environment, prefer those.
if let Some(name) = self.name.as_deref() {
if let Some(credentials) = Credentials::from_env(name) {
return Some(credentials);
}
}
// Otherwise, extract the credentials from the URL.
Credentials::from_url(self.url.url())
}
}
impl FromStr for Index {

View file

@ -407,7 +407,7 @@ impl<'a> IndexLocations {
}
/// Return an iterator over the [`FlatIndexLocation`] entries.
pub fn flat_index(&'a self) -> impl Iterator<Item = &'a FlatIndexLocation> + 'a {
pub fn flat_indexes(&'a self) -> impl Iterator<Item = &'a FlatIndexLocation> + 'a {
self.flat_index.iter()
}
@ -424,9 +424,10 @@ impl<'a> IndexLocations {
}
}
/// Return an iterator over all allowed [`IndexUrl`] entries.
/// Return an iterator over all allowed [`Index`] entries.
///
/// This includes both explicit and implicit indexes, as well as the default index.
/// This includes both explicit and implicit indexes, as well as the default index (but _not_
/// the flat indexes).
///
/// If `no_index` was enabled, then this always returns an empty
/// iterator.
@ -435,18 +436,6 @@ impl<'a> IndexLocations {
.chain(self.implicit_indexes())
.chain(self.default_index())
}
/// Return an iterator over all allowed [`Url`] entries.
///
/// This includes both explicit and implicit index URLs, as well as the default index.
///
/// If `no_index` was enabled, then this always returns an empty
/// iterator.
pub fn allowed_urls(&'a self) -> impl Iterator<Item = &'a Url> + 'a {
self.allowed_indexes()
.map(Index::raw_url)
.chain(self.flat_index().map(FlatIndexLocation::url))
}
}
/// The index URLs to use for fetching packages.

View file

@ -83,7 +83,7 @@ impl<'a> RegistryWheelIndex<'a> {
let mut entries = vec![];
let flat_index_urls: Vec<Index> = index_locations
.flat_index()
.flat_indexes()
.map(|flat_index| Index::from_extra_index_url(IndexUrl::from(flat_index.clone())))
.collect();

View file

@ -1067,7 +1067,7 @@ impl Lock {
})
.chain(
locations
.flat_index()
.flat_indexes()
.filter_map(|index_url| match index_url {
FlatIndexLocation::Url(_) => {
Some(UrlString::from(index_url.redacted()))
@ -1093,7 +1093,7 @@ impl Lock {
})
.chain(
locations
.flat_index()
.flat_indexes()
.filter_map(|index_url| match index_url {
FlatIndexLocation::Url(_) => None,
FlatIndexLocation::Path(index_url) => {

View file

@ -626,7 +626,7 @@ impl PubGrubReportFormatter<'_> {
incomplete_packages: &FxHashMap<PackageName, BTreeMap<Version, IncompletePackage>>,
hints: &mut IndexSet<PubGrubHint>,
) {
let no_find_links = index_locations.flat_index().peekable().peek().is_none();
let no_find_links = index_locations.flat_indexes().peekable().peek().is_none();
// Add hints due to the package being entirely unavailable.
match unavailable_packages.get(name) {

View file

@ -10,7 +10,7 @@ use uv_distribution_filename::SourceDistExtension;
use uv_distribution_types::{DependencyMetadata, IndexLocations};
use uv_install_wheel::linker::LinkMode;
use uv_auth::store_credentials_from_url;
use uv_auth::{store_credentials, store_credentials_from_url};
use uv_cache::{Cache, CacheBucket};
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
@ -400,8 +400,13 @@ async fn build_package(
.into_interpreter();
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
store_credentials_from_url(index.url());
}
// Read build constraints.
@ -454,7 +459,7 @@ async fn build_package(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, None, &hasher, build_options)
};

View file

@ -6,7 +6,6 @@ use itertools::Itertools;
use owo_colors::OwoColorize;
use tracing::debug;
use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
@ -284,8 +283,13 @@ pub(crate) async fn pip_compile(
);
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Initialize the registry client.
@ -308,7 +312,7 @@ pub(crate) async fn pip_compile(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, tags.as_deref(), &hasher, &build_options)
};
@ -465,7 +469,7 @@ pub(crate) async fn pip_compile(
// If necessary, include the `--find-links` locations.
if include_find_links {
for flat_index in index_locations.flat_index() {
for flat_index in index_locations.flat_indexes() {
writeln!(writer, "--find-links {}", flat_index.verbatim())?;
wrote_preamble = true;
}

View file

@ -4,7 +4,6 @@ use itertools::Itertools;
use owo_colors::OwoColorize;
use tracing::{debug, enabled, Level};
use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
@ -286,8 +285,13 @@ pub(crate) async fn pip_install(
);
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Initialize the registry client.
@ -305,7 +309,7 @@ pub(crate) async fn pip_install(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, Some(&tags), &hasher, &build_options)
};

View file

@ -4,7 +4,6 @@ use anyhow::Result;
use owo_colors::OwoColorize;
use tracing::debug;
use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
@ -222,8 +221,13 @@ pub(crate) async fn pip_sync(
);
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Initialize the registry client.
@ -241,7 +245,7 @@ pub(crate) async fn pip_sync(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, Some(&tags), &hasher, &build_options)
};

View file

@ -9,7 +9,6 @@ use rustc_hash::{FxBuildHasher, FxHashMap};
use tracing::debug;
use url::Url;
use uv_auth::{store_credentials_from_url, Credentials};
use uv_cache::Cache;
use uv_cache_key::RepositoryUrl;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
@ -246,8 +245,13 @@ pub(crate) async fn add(
resolution_environment(python_version, python_platform, target.interpreter())?;
// Add all authenticated sources to the cache.
for url in settings.index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in settings.index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in settings.index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Initialize the registry client.
@ -276,7 +280,9 @@ pub(crate) async fn add(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(settings.index_locations.flat_index()).await?;
let entries = client
.fetch(settings.index_locations.flat_indexes())
.await?;
FlatIndex::from_entries(entries, Some(&tags), &hasher, &settings.build_options)
};
@ -424,7 +430,7 @@ pub(crate) async fn add(
branch,
marker,
}) => {
let credentials = Credentials::from_url(&git);
let credentials = uv_auth::Credentials::from_url(&git);
if let Some(credentials) = credentials {
debug!("Caching credentials for: {git}");
GIT_STORE.insert(RepositoryUrl::new(&git), credentials);

View file

@ -8,7 +8,7 @@ use anstream::eprint;
use owo_colors::OwoColorize;
use rustc_hash::{FxBuildHasher, FxHashMap};
use tracing::debug;
use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
@ -364,8 +364,13 @@ async fn do_lock(
PythonRequirement::from_requires_python(interpreter, requires_python.clone());
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Initialize the registry client.
@ -409,7 +414,7 @@ async fn do_lock(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, None, &hasher, build_options)
};

View file

@ -5,7 +5,6 @@ use itertools::Itertools;
use owo_colors::OwoColorize;
use tracing::debug;
use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{Concurrency, Constraints, ExtrasSpecification, Reinstall, Upgrade};
@ -647,8 +646,13 @@ pub(crate) async fn resolve_names(
} = settings;
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Initialize the registry client.
@ -794,8 +798,13 @@ pub(crate) async fn resolve_environment<'a>(
let python_requirement = PythonRequirement::from_interpreter(interpreter);
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Initialize the registry client.
@ -857,7 +866,7 @@ pub(crate) async fn resolve_environment<'a>(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, Some(tags), &hasher, build_options)
};
@ -952,8 +961,13 @@ pub(crate) async fn sync_environment(
let tags = venv.interpreter().tags()?;
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Initialize the registry client.
@ -987,7 +1001,7 @@ pub(crate) async fn sync_environment(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, Some(tags), &hasher, build_options)
};
@ -1140,8 +1154,13 @@ pub(crate) async fn update_environment(
}
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Initialize the registry client.
@ -1189,7 +1208,7 @@ pub(crate) async fn update_environment(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, Some(tags), &hasher, build_options)
};
@ -1368,7 +1387,7 @@ fn warn_on_requirements_txt_setting(
}
}
for find_link in find_links {
if !settings.index_locations.flat_index().contains(find_link) {
if !settings.index_locations.flat_indexes().contains(find_link) {
warn_user_once!(
"Ignoring `--find-links` from requirements file: `{find_link}`. Instead, use the `--find-links` command-line argument, or set `find-links` in a `uv.toml` or `pyproject.toml` file.`"
);

View file

@ -11,6 +11,7 @@ use itertools::Itertools;
use std::borrow::Cow;
use std::path::Path;
use std::str::FromStr;
use uv_auth::{store_credentials, store_credentials_from_url};
use uv_cache::Cache;
use uv_client::{Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
@ -276,8 +277,13 @@ pub(super) async fn do_sync(
let resolution = apply_editable_mode(resolution, editable);
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
uv_auth::store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
store_credentials_from_url(index.url());
}
// Populate credentials from the workspace.
@ -316,7 +322,7 @@ pub(super) async fn do_sync(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, Some(tags), &hasher, build_options)
};

View file

@ -9,7 +9,6 @@ use miette::{Diagnostic, IntoDiagnostic};
use owo_colors::OwoColorize;
use thiserror::Error;
use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
@ -229,8 +228,13 @@ async fn venv_impl(
let interpreter = python.into_interpreter();
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
if managed {
@ -278,8 +282,13 @@ async fn venv_impl(
let interpreter = venv.interpreter();
// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
uv_auth::store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
uv_auth::store_credentials_from_url(index.url());
}
// Instantiate a client.
@ -299,7 +308,7 @@ async fn venv_impl(
let tags = interpreter.tags().map_err(VenvError::Tags)?;
let client = FlatIndexClient::new(&client, cache);
let entries = client
.fetch(index_locations.flat_index())
.fetch(index_locations.flat_indexes())
.await
.map_err(VenvError::FlatIndex)?;
FlatIndex::from_entries(

View file

@ -6437,6 +6437,94 @@ fn lock_redact_git_sources() -> Result<()> {
Ok(())
}
/// Pass credentials for a named index via environment variables.
#[test]
fn lock_env_credentials() -> Result<()> {
let context = TestContext::new("3.12");
let pyproject_toml = context.temp_dir.child("pyproject.toml");
pyproject_toml.write_str(
r#"
[project]
name = "foo"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = ["iniconfig"]
[build-system]
requires = ["setuptools>=42"]
build-backend = "setuptools.build_meta"
[[tool.uv.index]]
name = "proxy"
url = "https://pypi-proxy.fly.dev/basic-auth/simple"
default = true
"#,
)?;
// Without credentials, the resolution should fail.
uv_snapshot!(context.filters(), context.lock(), @r###"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
× No solution found when resolving dependencies:
Because iniconfig was not found in the package registry and your project depends on iniconfig, we can conclude that your project's requirements are unsatisfiable.
"###);
// Provide credentials via environment variables.
uv_snapshot!(context.filters(), context.lock()
.env("UV_HTTP_BASIC_PROXY_USERNAME", "public")
.env("UV_HTTP_BASIC_PROXY_PASSWORD", "heron"), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 2 packages in [TIME]
"###);
let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap();
// The lockfile shout omit the credentials.
insta::with_settings!({
filters => context.filters(),
}, {
assert_snapshot!(
lock, @r###"
version = 1
requires-python = ">=3.12"
[options]
exclude-newer = "2024-03-25T00:00:00Z"
[[package]]
name = "foo"
version = "0.1.0"
source = { editable = "." }
dependencies = [
{ name = "iniconfig" },
]
[package.metadata]
requires-dist = [{ name = "iniconfig" }]
[[package]]
name = "iniconfig"
version = "2.0.0"
source = { registry = "https://pypi-proxy.fly.dev/basic-auth/simple" }
sdist = { url = "https://pypi-proxy.fly.dev/basic-auth/files/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646 }
wheels = [
{ url = "https://pypi-proxy.fly.dev/basic-auth/files/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 },
]
"###
);
});
Ok(())
}
/// Resolve against an index that uses relative links.
#[test]
fn lock_relative_index() -> Result<()> {