Opt-out tool.uv.sources support for uv add (#4406)

## Summary

After this change, `uv add` will try to use `tool.uv.sources` for all
source requirements. If a source cannot be resolved, i.e. an ambiguous
Git reference is provided, it will error. Git references can be
specified with the `--tag`, `--branch`, or `--rev` arguments. Editables
are also supported with `--editable`.

Users can opt-out of `tool.uv.sources` support with the `--raw` flag,
which will force uv to use `project.dependencies`.

Part of https://github.com/astral-sh/uv/issues/3959.
This commit is contained in:
Ibraheem Ahmed 2024-06-19 14:20:16 -04:00 committed by GitHub
parent 3c5b13695e
commit 7b72b55af8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 484 additions and 40 deletions

View file

@ -153,6 +153,15 @@ pub struct Requirement<T: Pep508Url = VerbatimUrl> {
pub origin: Option<RequirementOrigin>,
}
impl<T: Pep508Url> Requirement<T> {
/// Removes the URL specifier from this requirement.
pub fn clear_url(&mut self) {
if matches!(self.version_or_url, Some(VersionOrUrl::Url(_))) {
self.version_or_url = None;
}
}
}
impl<T: Pep508Url + Display> Display for Requirement<T> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.name)?;

View file

@ -6,15 +6,17 @@
//!
//! Then lowers them into a dependency specification.
use std::collections::BTreeMap;
use std::ops::Deref;
use std::{collections::BTreeMap, mem};
use glob::Pattern;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use url::Url;
use pep440_rs::VersionSpecifiers;
use pypi_types::VerbatimParsedUrl;
use pypi_types::{RequirementSource, VerbatimParsedUrl};
use uv_git::GitReference;
use uv_normalize::{ExtraName, PackageName};
/// A `pyproject.toml` as specified in PEP 517.
@ -182,6 +184,90 @@ pub enum Source {
},
}
#[derive(Error, Debug)]
pub enum SourceError {
#[error("Cannot resolve git reference `{0}`.")]
UnresolvedReference(String),
#[error("Workspace dependency must be a local path.")]
InvalidWorkspaceRequirement,
}
impl Source {
pub fn from_requirement(
source: RequirementSource,
workspace: bool,
editable: Option<bool>,
rev: Option<String>,
tag: Option<String>,
branch: Option<String>,
) -> Result<Option<Source>, SourceError> {
if workspace {
match source {
RequirementSource::Registry { .. } | RequirementSource::Directory { .. } => {}
_ => return Err(SourceError::InvalidWorkspaceRequirement),
}
return Ok(Some(Source::Workspace {
editable,
workspace: true,
}));
}
let source = match source {
RequirementSource::Registry { .. } => return Ok(None),
RequirementSource::Path { lock_path, .. } => Source::Path {
editable,
path: lock_path.to_string_lossy().into_owned(),
},
RequirementSource::Directory { lock_path, .. } => Source::Path {
editable,
path: lock_path.to_string_lossy().into_owned(),
},
RequirementSource::Url {
subdirectory, url, ..
} => Source::Url {
url: url.to_url(),
subdirectory: subdirectory.map(|path| path.to_string_lossy().into_owned()),
},
RequirementSource::Git {
repository,
mut reference,
subdirectory,
..
} => {
// We can only resolve a full commit hash from a pep508 URL, everything else is ambiguous.
let rev = match reference {
GitReference::FullCommit(ref mut rev) => Some(mem::take(rev)),
_ => None,
}
// Give precedence to an explicit argument.
.or(rev);
// Error if the user tried to specify a reference but didn't disambiguate.
if reference != GitReference::DefaultBranch
&& rev.is_none()
&& tag.is_none()
&& branch.is_none()
{
return Err(SourceError::UnresolvedReference(
reference.as_str().unwrap().to_owned(),
));
}
Source::Git {
rev,
tag,
branch,
git: repository,
subdirectory: subdirectory.map(|path| path.to_string_lossy().into_owned()),
}
}
};
Ok(Some(source))
}
}
/// <https://github.com/serde-rs/serde/issues/1316#issue-332908452>
mod serde_from_and_to_string {
use std::fmt::Display;

View file

@ -1,8 +1,8 @@
use std::fmt;
use std::str::FromStr;
use std::{fmt, mem};
use thiserror::Error;
use toml_edit::{Array, DocumentMut, InlineTable, Item, RawString, Table, TomlError, Value};
use toml_edit::{Array, DocumentMut, Item, RawString, Table, TomlError, Value};
use pep508_rs::{PackageName, Requirement};
use pypi_types::VerbatimParsedUrl;
@ -21,6 +21,8 @@ pub struct PyProjectTomlMut {
pub enum Error {
#[error("Failed to parse `pyproject.toml`")]
Parse(#[from] Box<TomlError>),
#[error("Failed to serialize `pyproject.toml`")]
Serialize(#[from] Box<toml::ser::Error>),
#[error("Dependencies in `pyproject.toml` are malformed")]
MalformedDependencies,
#[error("Sources in `pyproject.toml` are malformed")]
@ -72,7 +74,7 @@ impl PyProjectTomlMut {
.as_table_mut()
.ok_or(Error::MalformedSources)?;
add_source(req, source, sources);
add_source(req, source, sources)?;
}
Ok(())
@ -113,7 +115,7 @@ impl PyProjectTomlMut {
.as_table_mut()
.ok_or(Error::MalformedSources)?;
add_source(req, source, sources);
add_source(req, source, sources)?;
}
Ok(())
@ -244,21 +246,17 @@ fn find_dependencies(name: &PackageName, deps: &Array) -> Vec<usize> {
}
// Add a source to `tool.uv.sources`.
fn add_source(req: &Requirement, source: &Source, sources: &mut Table) {
match source {
Source::Workspace {
workspace,
editable,
} => {
let mut value = InlineTable::new();
value.insert("workspace", Value::from(*workspace));
if let Some(editable) = editable {
value.insert("editable", Value::from(*editable));
}
sources.insert(req.name.as_ref(), Item::Value(Value::InlineTable(value)));
}
_ => unimplemented!(),
}
fn add_source(req: &Requirement, source: &Source, sources: &mut Table) -> Result<(), Error> {
// Serialize as an inline table.
let mut doc = toml::to_string(source)
.map_err(Box::new)?
.parse::<DocumentMut>()
.unwrap();
let table = mem::take(doc.as_table_mut()).into_inline_table();
sources.insert(req.name.as_ref(), Item::Value(Value::InlineTable(table)));
Ok(())
}
impl fmt::Display for PyProjectTomlMut {

View file

@ -1638,6 +1638,28 @@ pub(crate) struct AddArgs {
#[arg(long)]
pub(crate) workspace: bool,
/// Add the requirements as editables.
#[arg(long, default_missing_value = "true", num_args(0..=1))]
pub(crate) editable: Option<bool>,
/// Add source requirements to the `project.dependencies` section of the `pyproject.toml`.
///
/// Without this flag uv will try to use `tool.uv.sources` for any sources.
#[arg(long)]
pub(crate) raw: bool,
/// Specific commit to use when adding from Git.
#[arg(long)]
pub(crate) rev: Option<String>,
/// Tag to use when adding from git.
#[arg(long)]
pub(crate) tag: Option<String>,
/// Branch to use when adding from git.
#[arg(long)]
pub(crate) branch: Option<String>,
#[command(flatten)]
pub(crate) installer: ResolverInstallerArgs,

View file

@ -1,7 +1,7 @@
use anyhow::Result;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_dispatch::BuildDispatch;
use uv_distribution::pyproject::Source;
use uv_distribution::pyproject::{Source, SourceError};
use uv_distribution::pyproject_mut::PyProjectTomlMut;
use uv_git::GitResolver;
use uv_requirements::{NamedRequirementsResolver, RequirementsSource, RequirementsSpecification};
@ -22,11 +22,16 @@ use crate::printer::Printer;
use crate::settings::ResolverInstallerSettings;
/// Add one or more packages to the project requirements.
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_arguments, clippy::fn_params_excessive_bools)]
pub(crate) async fn add(
requirements: Vec<RequirementsSource>,
workspace: bool,
dev: bool,
editable: Option<bool>,
raw: bool,
rev: Option<String>,
tag: Option<String>,
branch: Option<String>,
python: Option<String>,
settings: ResolverInstallerSettings,
preview: PreviewMode,
@ -135,14 +140,34 @@ pub(crate) async fn add(
// Add the requirements to the `pyproject.toml`.
let mut pyproject = PyProjectTomlMut::from_toml(project.current_project().pyproject_toml())?;
for req in requirements.into_iter().map(pep508_rs::Requirement::from) {
let source = if workspace {
Some(Source::Workspace {
workspace: true,
editable: None,
})
for req in requirements {
let (req, source) = if raw {
// Use the PEP 508 requirement directly.
(pep508_rs::Requirement::from(req), None)
} else {
None
// Otherwise, try to construct the source.
let result = Source::from_requirement(
req.source.clone(),
workspace,
editable,
rev.clone(),
tag.clone(),
branch.clone(),
);
let source = match result {
Ok(source) => source,
Err(SourceError::UnresolvedReference(rev)) => {
anyhow::bail!("Cannot resolve Git reference `{rev}` for requirement `{}`. Specify the reference with one of `--tag`, `--branch`, or `--rev`, or use the `--raw` flag.", req.name)
}
Err(err) => return Err(err.into()),
};
// Ignore the PEP 508 source.
let mut req = pep508_rs::Requirement::from(req);
req.clear_url();
(req, source)
};
if dev {

View file

@ -689,6 +689,11 @@ async fn run() -> Result<ExitStatus> {
args.requirements,
args.workspace,
args.dev,
args.editable,
args.raw,
args.rev,
args.tag,
args.branch,
args.python,
args.settings,
globals.preview,

View file

@ -368,8 +368,13 @@ impl LockSettings {
#[derive(Debug, Clone)]
pub(crate) struct AddSettings {
pub(crate) requirements: Vec<RequirementsSource>,
pub(crate) workspace: bool,
pub(crate) dev: bool,
pub(crate) workspace: bool,
pub(crate) editable: Option<bool>,
pub(crate) raw: bool,
pub(crate) rev: Option<String>,
pub(crate) tag: Option<String>,
pub(crate) branch: Option<String>,
pub(crate) python: Option<String>,
pub(crate) refresh: Refresh,
pub(crate) settings: ResolverInstallerSettings,
@ -383,6 +388,11 @@ impl AddSettings {
requirements,
dev,
workspace,
editable,
raw,
rev,
tag,
branch,
installer,
build,
refresh,
@ -398,6 +408,11 @@ impl AddSettings {
requirements,
workspace,
dev,
editable,
raw,
rev,
tag,
branch,
python,
refresh: Refresh::from(refresh),
settings: ResolverInstallerSettings::combine(

View file

@ -167,13 +167,179 @@ fn add_git() -> Result<()> {
+ sniffio==1.3.1
"###);
uv_snapshot!(context.filters(), context.add(&["uv-public-pypackage @ git+https://github.com/astral-test/uv-public-pypackage@0.0.1"]), @r###"
// Adding with an ambiguous Git reference will fail.
uv_snapshot!(context.filters(), context.add(&["uv-public-pypackage @ git+https://github.com/astral-test/uv-public-pypackage@0.0.1"]).arg("--preview"), @r###"
success: false
exit_code: 2
----- stdout -----
----- stderr -----
error: Cannot resolve Git reference `0.0.1` for requirement `uv-public-pypackage`. Specify the reference with one of `--tag`, `--branch`, or `--rev`, or use the `--raw` flag.
"###);
uv_snapshot!(context.filters(), context.add(&["uv-public-pypackage @ git+https://github.com/astral-test/uv-public-pypackage"]).arg("--tag=0.0.1").arg("--preview"), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 5 packages in [TIME]
Prepared 2 packages in [TIME]
Uninstalled 1 package in [TIME]
Installed 2 packages in [TIME]
- project==0.1.0 (from file://[TEMP_DIR]/)
+ project==0.1.0 (from file://[TEMP_DIR]/)
+ uv-public-pypackage==0.1.0 (from git+https://github.com/astral-test/uv-public-pypackage@0dacfd662c64cb4ceb16e6cf65a157a8b715b979?tag=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979)
"###);
let pyproject_toml = fs_err::read_to_string(context.temp_dir.join("pyproject.toml"))?;
insta::with_settings!({
filters => context.filters(),
}, {
assert_snapshot!(
pyproject_toml, @r###"
[project]
name = "project"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = [
"anyio==3.7.0",
"uv-public-pypackage",
]
[tool.uv.sources]
uv-public-pypackage = { git = "https://github.com/astral-test/uv-public-pypackage", tag = "0.0.1" }
"###
);
});
let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock"))?;
insta::with_settings!({
filters => context.filters(),
}, {
assert_snapshot!(
lock, @r###"
version = 1
requires-python = ">=3.12"
[[distribution]]
name = "anyio"
version = "3.7.0"
source = "registry+https://pypi.org/simple"
sdist = { url = "https://files.pythonhosted.org/packages/c6/b3/fefbf7e78ab3b805dec67d698dc18dd505af7a18a8dd08868c9b4fa736b5/anyio-3.7.0.tar.gz", hash = "sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce", size = 142737 }
wheels = [{ url = "https://files.pythonhosted.org/packages/68/fe/7ce1926952c8a403b35029e194555558514b365ad77d75125f521a2bec62/anyio-3.7.0-py3-none-any.whl", hash = "sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0", size = 80873 }]
[[distribution.dependencies]]
name = "idna"
version = "3.6"
source = "registry+https://pypi.org/simple"
[[distribution.dependencies]]
name = "sniffio"
version = "1.3.1"
source = "registry+https://pypi.org/simple"
[[distribution]]
name = "idna"
version = "3.6"
source = "registry+https://pypi.org/simple"
sdist = { url = "https://files.pythonhosted.org/packages/bf/3f/ea4b9117521a1e9c50344b909be7886dd00a519552724809bb1f486986c2/idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", size = 175426 }
wheels = [{ url = "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f", size = 61567 }]
[[distribution]]
name = "project"
version = "0.1.0"
source = "editable+."
sdist = { path = "." }
[[distribution.dependencies]]
name = "anyio"
version = "3.7.0"
source = "registry+https://pypi.org/simple"
[[distribution.dependencies]]
name = "uv-public-pypackage"
version = "0.1.0"
source = "git+https://github.com/astral-test/uv-public-pypackage?tag=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979"
[[distribution]]
name = "sniffio"
version = "1.3.1"
source = "registry+https://pypi.org/simple"
sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 }
wheels = [{ url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }]
[[distribution]]
name = "uv-public-pypackage"
version = "0.1.0"
source = "git+https://github.com/astral-test/uv-public-pypackage?tag=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979"
sdist = { url = "https://github.com/astral-test/uv-public-pypackage?tag=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979" }
"###
);
});
// Install from the lockfile.
uv_snapshot!(context.filters(), context.sync(), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
warning: `uv sync` is experimental and may change without warning.
Audited 5 packages in [TIME]
"###);
Ok(())
}
/// Add a Git requirement using the `--raw` API.
#[test]
fn add_git_raw() -> Result<()> {
let context = TestContext::new("3.12");
let pyproject_toml = context.temp_dir.child("pyproject.toml");
pyproject_toml.write_str(indoc! {r#"
[project]
name = "project"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = ["anyio==3.7.0"]
"#})?;
uv_snapshot!(context.filters(), context.lock(), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
warning: `uv lock` is experimental and may change without warning.
Resolved 4 packages in [TIME]
"###);
uv_snapshot!(context.filters(), context.sync(), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
warning: `uv sync` is experimental and may change without warning.
Prepared 4 packages in [TIME]
Installed 4 packages in [TIME]
+ anyio==3.7.0
+ idna==3.6
+ project==0.1.0 (from file://[TEMP_DIR]/)
+ sniffio==1.3.1
"###);
// Use an ambiguous tag reference, which would otherwise not resolve.
uv_snapshot!(context.filters(), context.add(&["uv-public-pypackage @ git+https://github.com/astral-test/uv-public-pypackage@0.0.1"]).arg("--raw").arg("--preview"), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
warning: `uv add` is experimental and may change without warning.
Resolved 5 packages in [TIME]
Prepared 2 packages in [TIME]
Uninstalled 1 package in [TIME]
@ -297,18 +463,17 @@ fn add_unnamed() -> Result<()> {
dependencies = []
"#})?;
uv_snapshot!(context.filters(), context.add(&["git+https://github.com/astral-test/uv-public-pypackage@0.0.1"]), @r###"
uv_snapshot!(context.filters(), context.add(&["git+https://github.com/astral-test/uv-public-pypackage"]).arg("--tag=0.0.1").arg("--preview"), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
warning: `uv add` is experimental and may change without warning.
Resolved 2 packages in [TIME]
Prepared 2 packages in [TIME]
Installed 2 packages in [TIME]
+ project==0.1.0 (from file://[TEMP_DIR]/)
+ uv-public-pypackage==0.1.0 (from git+https://github.com/astral-test/uv-public-pypackage@0dacfd662c64cb4ceb16e6cf65a157a8b715b979?rev=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979)
+ uv-public-pypackage==0.1.0 (from git+https://github.com/astral-test/uv-public-pypackage@0dacfd662c64cb4ceb16e6cf65a157a8b715b979?tag=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979)
"###);
let pyproject_toml = fs_err::read_to_string(context.temp_dir.join("pyproject.toml"))?;
@ -324,8 +489,11 @@ fn add_unnamed() -> Result<()> {
# ...
requires-python = ">=3.12"
dependencies = [
"uv-public-pypackage @ git+https://github.com/astral-test/uv-public-pypackage@0.0.1",
"uv-public-pypackage",
]
[tool.uv.sources]
uv-public-pypackage = { git = "https://github.com/astral-test/uv-public-pypackage", tag = "0.0.1" }
"###
);
});
@ -349,13 +517,13 @@ fn add_unnamed() -> Result<()> {
[[distribution.dependencies]]
name = "uv-public-pypackage"
version = "0.1.0"
source = "git+https://github.com/astral-test/uv-public-pypackage?rev=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979"
source = "git+https://github.com/astral-test/uv-public-pypackage?tag=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979"
[[distribution]]
name = "uv-public-pypackage"
version = "0.1.0"
source = "git+https://github.com/astral-test/uv-public-pypackage?rev=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979"
sdist = { url = "https://github.com/astral-test/uv-public-pypackage?rev=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979" }
source = "git+https://github.com/astral-test/uv-public-pypackage?tag=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979"
sdist = { url = "https://github.com/astral-test/uv-public-pypackage?tag=0.0.1#0dacfd662c64cb4ceb16e6cf65a157a8b715b979" }
"###
);
});
@ -763,6 +931,122 @@ fn add_remove_workspace() -> Result<()> {
Ok(())
}
/// Add a workspace dependency as an editable.
#[test]
fn add_workspace_editable() -> Result<()> {
let context = TestContext::new("3.12");
let workspace = context.temp_dir.child("pyproject.toml");
workspace.write_str(indoc! {r#"
[tool.uv.workspace]
members = ["child1", "child2"]
"#})?;
let pyproject_toml = context.temp_dir.child("child1/pyproject.toml");
pyproject_toml.write_str(indoc! {r#"
[project]
name = "child1"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = []
"#})?;
let pyproject_toml = context.temp_dir.child("child2/pyproject.toml");
pyproject_toml.write_str(indoc! {r#"
[project]
name = "child2"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = []
"#})?;
let child1 = context.temp_dir.join("child1");
let mut add_cmd = context.add(&["child2"]);
add_cmd
.arg("--editable")
.arg("--workspace")
.arg("--preview")
.current_dir(&child1);
uv_snapshot!(context.filters(), add_cmd, @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 2 packages in [TIME]
Prepared 2 packages in [TIME]
Installed 2 packages in [TIME]
+ child1==0.1.0 (from file://[TEMP_DIR]/child1)
+ child2==0.1.0 (from file://[TEMP_DIR]/child2)
"###);
let pyproject_toml = fs_err::read_to_string(child1.join("pyproject.toml"))?;
insta::with_settings!({
filters => context.filters(),
}, {
assert_snapshot!(
pyproject_toml, @r###"
[project]
name = "child1"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = [
"child2",
]
[tool.uv.sources]
child2 = { workspace = true, editable = true }
"###
);
});
// `uv add` implies a full lock and sync, including development dependencies.
let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock"))?;
insta::with_settings!({
filters => context.filters(),
}, {
assert_snapshot!(
lock, @r###"
version = 1
requires-python = ">=3.12"
[[distribution]]
name = "child1"
version = "0.1.0"
source = "editable+child1"
sdist = { path = "child1" }
[[distribution.dependencies]]
name = "child2"
version = "0.1.0"
source = "editable+child2"
[[distribution]]
name = "child2"
version = "0.1.0"
source = "editable+child2"
sdist = { path = "child2" }
"###
);
});
// Install from the lockfile.
uv_snapshot!(context.filters(), context.sync().current_dir(&child1), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
warning: `uv sync` is experimental and may change without warning.
Audited 2 packages in [TIME]
"###);
Ok(())
}
/// Update a PyPI requirement.
#[test]
fn update_registry() -> Result<()> {