Avoid re-parsing pyproject.toml when provided as a source (#15851)
Some checks are pending
CI / Determine changes (push) Waiting to run
CI / typos (push) Waiting to run
CI / lint (push) Waiting to run
CI / cargo clippy | ubuntu (push) Blocked by required conditions
CI / cargo clippy | windows (push) Blocked by required conditions
CI / cargo dev generate-all (push) Blocked by required conditions
CI / cargo shear (push) Waiting to run
CI / ecosystem test | pydantic/pydantic-core (push) Blocked by required conditions
CI / cargo test | ubuntu (push) Blocked by required conditions
CI / cargo test | macos (push) Blocked by required conditions
CI / cargo test | windows (push) Blocked by required conditions
CI / check windows trampoline | aarch64 (push) Blocked by required conditions
CI / check windows trampoline | i686 (push) Blocked by required conditions
CI / check windows trampoline | x86_64 (push) Blocked by required conditions
CI / test windows trampoline | aarch64 (push) Blocked by required conditions
CI / test windows trampoline | i686 (push) Blocked by required conditions
CI / test windows trampoline | x86_64 (push) Blocked by required conditions
CI / mkdocs (push) Waiting to run
CI / build binary | linux libc (push) Blocked by required conditions
CI / build binary | linux aarch64 (push) Blocked by required conditions
CI / build binary | linux musl (push) Blocked by required conditions
CI / build binary | freebsd (push) Blocked by required conditions
CI / ecosystem test | prefecthq/prefect (push) Blocked by required conditions
CI / ecosystem test | pallets/flask (push) Blocked by required conditions
CI / smoke test | linux (push) Blocked by required conditions
CI / smoke test | linux aarch64 (push) Blocked by required conditions
CI / check system | alpine (push) Blocked by required conditions
CI / smoke test | macos (push) Blocked by required conditions
CI / smoke test | windows x86_64 (push) Blocked by required conditions
CI / smoke test | windows aarch64 (push) Blocked by required conditions
CI / integration test | activate nushell venv (push) Blocked by required conditions
CI / integration test | conda on ubuntu (push) Blocked by required conditions
CI / integration test | deadsnakes python3.9 on ubuntu (push) Blocked by required conditions
CI / integration test | free-threaded on windows (push) Blocked by required conditions
CI / integration test | aarch64 windows implicit (push) Blocked by required conditions
CI / integration test | aarch64 windows explicit (push) Blocked by required conditions
CI / integration test | pypy on ubuntu (push) Blocked by required conditions
CI / integration test | pypy on windows (push) Blocked by required conditions
CI / integration test | graalpy on ubuntu (push) Blocked by required conditions
CI / integration test | graalpy on windows (push) Blocked by required conditions
CI / integration test | pyodide on ubuntu (push) Blocked by required conditions
CI / integration test | github actions (push) Blocked by required conditions
CI / integration test | free-threaded python on github actions (push) Blocked by required conditions
CI / integration test | pyenv on wsl x86-64 (push) Blocked by required conditions
CI / integration test | determine publish changes (push) Blocked by required conditions
CI / integration test | registries (push) Blocked by required conditions
CI / integration test | uv publish (push) Blocked by required conditions
CI / build binary | macos aarch64 (push) Blocked by required conditions
CI / build binary | macos x86_64 (push) Blocked by required conditions
CI / build binary | windows x86_64 (push) Blocked by required conditions
CI / build binary | windows aarch64 (push) Blocked by required conditions
CI / build binary | msrv (push) Blocked by required conditions
CI / check system | python on macos aarch64 (push) Blocked by required conditions
CI / check system | homebrew python on macos aarch64 (push) Blocked by required conditions
CI / check system | x86-64 python on macos aarch64 (push) Blocked by required conditions
CI / check system | python on macos x86-64 (push) Blocked by required conditions
CI / check system | python3.10 on windows x86-64 (push) Blocked by required conditions
CI / check system | python3.10 on windows x86 (push) Blocked by required conditions
CI / check system | python3.13 on windows x86-64 (push) Blocked by required conditions
CI / check system | x86-64 python3.13 on windows aarch64 (push) Blocked by required conditions
CI / check system | aarch64 python3.13 on windows aarch64 (push) Blocked by required conditions
CI / check system | windows registry (push) Blocked by required conditions
CI / check system | python3.12 via chocolatey (push) Blocked by required conditions
CI / check system | python3.9 via pyenv (push) Blocked by required conditions
CI / check system | python3.13 (push) Blocked by required conditions
CI / check system | conda3.11 on macos aarch64 (push) Blocked by required conditions
CI / check system | conda3.11 on linux x86-64 (push) Blocked by required conditions
CI / check system | conda3.8 on linux x86-64 (push) Blocked by required conditions
CI / integration test | uv_build (push) Blocked by required conditions
CI / check cache | ubuntu (push) Blocked by required conditions
CI / check cache | macos aarch64 (push) Blocked by required conditions
CI / check system | python on debian (push) Blocked by required conditions
CI / check system | python on fedora (push) Blocked by required conditions
CI / check system | python on ubuntu (push) Blocked by required conditions
CI / check system | python on rocky linux 8 (push) Blocked by required conditions
CI / check system | python on rocky linux 9 (push) Blocked by required conditions
CI / check system | graalpy on ubuntu (push) Blocked by required conditions
CI / check system | pypy on ubuntu (push) Blocked by required conditions
CI / check system | pyston (push) Blocked by required conditions
CI / check system | conda3.8 on macos aarch64 (push) Blocked by required conditions
CI / check system | conda3.11 on windows x86-64 (push) Blocked by required conditions
CI / check system | conda3.8 on windows x86-64 (push) Blocked by required conditions
CI / check system | amazonlinux (push) Blocked by required conditions
CI / check system | embedded python3.10 on windows x86-64 (push) Blocked by required conditions
CI / benchmarks | walltime aarch64 linux (push) Blocked by required conditions
CI / benchmarks | instrumented (push) Blocked by required conditions
zizmor / Run zizmor (push) Waiting to run

## Summary

In the process of making a different change, I noticed that we parse
this during source discovery, throw it away, then parse it again later.
This commit is contained in:
Charlie Marsh 2025-09-15 10:07:38 -04:00 committed by GitHub
parent ef17e7d0f4
commit d706c07ae3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 124 additions and 80 deletions

2
Cargo.lock generated
View file

@ -6235,7 +6235,6 @@ dependencies = [
"uv-scripts",
"uv-types",
"uv-warnings",
"uv-workspace",
]
[[package]]
@ -6279,6 +6278,7 @@ dependencies = [
"clap",
"dashmap",
"either",
"fs-err",
"futures",
"hashbrown 0.15.5",
"indexmap",

View file

@ -26,7 +26,7 @@ use uv_distribution_types::{
use uv_extract::hash::Hasher;
use uv_fs::write_atomic;
use uv_platform_tags::Tags;
use uv_pypi_types::{HashDigest, HashDigests};
use uv_pypi_types::{HashDigest, HashDigests, PyProjectToml};
use uv_redacted::DisplaySafeUrl;
use uv_types::{BuildContext, BuildStack};
@ -550,10 +550,11 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
/// Return the [`RequiresDist`] from a `pyproject.toml`, if it can be statically extracted.
pub async fn requires_dist(
&self,
source_tree: impl AsRef<Path>,
path: &Path,
pyproject_toml: &PyProjectToml,
) -> Result<Option<RequiresDist>, Error> {
self.builder
.source_tree_requires_dist(source_tree.as_ref())
.source_tree_requires_dist(path, pyproject_toml)
.await
}

View file

@ -459,7 +459,8 @@ mod test {
&WorkspaceCache::default(),
)
.await?;
let requires_dist = uv_pypi_types::RequiresDist::parse_pyproject_toml(contents)?;
let pyproject_toml = uv_pypi_types::PyProjectToml::from_toml(contents)?;
let requires_dist = uv_pypi_types::RequiresDist::from_pyproject_toml(pyproject_toml)?;
Ok(RequiresDist::from_project_workspace(
requires_dist,
&project_workspace,

View file

@ -1488,18 +1488,16 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
/// Return the [`RequiresDist`] from a `pyproject.toml`, if it can be statically extracted.
pub(crate) async fn source_tree_requires_dist(
&self,
source_tree: &Path,
path: &Path,
pyproject_toml: &PyProjectToml,
) -> Result<Option<RequiresDist>, Error> {
// Attempt to read static metadata from the `pyproject.toml`.
match read_requires_dist(source_tree).await {
match uv_pypi_types::RequiresDist::from_pyproject_toml(pyproject_toml.clone()) {
Ok(requires_dist) => {
debug!(
"Found static `requires-dist` for: {}",
source_tree.display()
);
debug!("Found static `requires-dist` for: {}", path.display());
let requires_dist = RequiresDist::from_project_maybe_workspace(
requires_dist,
source_tree,
path,
None,
self.build_context.locations(),
self.build_context.sources(),
@ -1509,21 +1507,18 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
Ok(Some(requires_dist))
}
Err(
err @ (Error::MissingPyprojectToml
| Error::PyprojectToml(
uv_pypi_types::MetadataError::Pep508Error(_)
| uv_pypi_types::MetadataError::DynamicField(_)
| uv_pypi_types::MetadataError::FieldNotFound(_)
| uv_pypi_types::MetadataError::PoetrySyntax,
)),
err @ (uv_pypi_types::MetadataError::Pep508Error(_)
| uv_pypi_types::MetadataError::DynamicField(_)
| uv_pypi_types::MetadataError::FieldNotFound(_)
| uv_pypi_types::MetadataError::PoetrySyntax),
) => {
debug!(
"No static `requires-dist` available for: {} ({err:?})",
source_tree.display()
path.display()
);
Ok(None)
}
Err(err) => Err(err),
Err(err) => Err(Error::PyprojectToml(err)),
}
}
@ -3025,25 +3020,6 @@ async fn read_pyproject_toml(
Ok(pyproject_toml)
}
/// Return the [`pypi_types::RequiresDist`] from a `pyproject.toml`, if it can be statically extracted.
async fn read_requires_dist(project_root: &Path) -> Result<uv_pypi_types::RequiresDist, Error> {
// Read the `pyproject.toml` file.
let pyproject_toml = project_root.join("pyproject.toml");
let content = match fs::read_to_string(pyproject_toml).await {
Ok(content) => content,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Err(Error::MissingPyprojectToml);
}
Err(err) => return Err(Error::CacheRead(err)),
};
// Parse the metadata.
let requires_dist = uv_pypi_types::RequiresDist::parse_pyproject_toml(&content)
.map_err(Error::PyprojectToml)?;
Ok(requires_dist)
}
/// Wheel metadata stored in the source distribution cache.
#[derive(Debug, Clone)]
struct CachedMetadata(ResolutionMetadata);

View file

@ -10,7 +10,7 @@ use uv_pep440::Version;
use crate::MetadataError;
/// A `pyproject.toml` as specified in PEP 517.
#[derive(Deserialize, Debug)]
#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
pub struct PyProjectToml {
pub project: Option<Project>,
@ -33,7 +33,7 @@ impl PyProjectToml {
/// relevant for dependency resolution.
///
/// See <https://packaging.python.org/en/latest/specifications/pyproject-toml>.
#[derive(Deserialize, Debug)]
#[derive(Deserialize, Debug, Clone)]
#[serde(try_from = "PyprojectTomlWire")]
pub struct Project {
/// The name of the project
@ -51,7 +51,7 @@ pub struct Project {
pub dynamic: Option<Vec<String>>,
}
#[derive(Deserialize, Debug)]
#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
struct PyprojectTomlWire {
name: Option<PackageName>,
@ -78,13 +78,13 @@ impl TryFrom<PyprojectTomlWire> for Project {
}
}
#[derive(Deserialize, Debug)]
#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
pub(super) struct Tool {
pub(super) poetry: Option<ToolPoetry>,
}
#[derive(Deserialize, Debug)]
#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
#[allow(clippy::empty_structs_with_brackets)]
pub(super) struct ToolPoetry {}

View file

@ -24,9 +24,7 @@ pub struct RequiresDist {
impl RequiresDist {
/// Extract the [`RequiresDist`] from a `pyproject.toml` file, as specified in PEP 621.
pub fn parse_pyproject_toml(contents: &str) -> Result<Self, MetadataError> {
let pyproject_toml = PyProjectToml::from_toml(contents)?;
pub fn from_pyproject_toml(pyproject_toml: PyProjectToml) -> Result<Self, MetadataError> {
let project = pyproject_toml
.project
.ok_or(MetadataError::FieldNotFound("project"))?;

View file

@ -34,7 +34,6 @@ uv-resolver = { workspace = true, features = ["clap"] }
uv-scripts = { workspace = true }
uv-types = { workspace = true }
uv-warnings = { workspace = true }
uv-workspace = { workspace = true }
anyhow = { workspace = true }
configparser = { workspace = true }

View file

@ -1,5 +1,5 @@
use std::borrow::Cow;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Context, Result};
@ -16,10 +16,37 @@ use uv_distribution_types::{
use uv_fs::Simplified;
use uv_normalize::{ExtraName, PackageName};
use uv_pep508::RequirementOrigin;
use uv_pypi_types::PyProjectToml;
use uv_redacted::DisplaySafeUrl;
use uv_resolver::{InMemoryIndex, MetadataResponse};
use uv_types::{BuildContext, HashStrategy};
#[derive(Debug, Clone)]
pub enum SourceTree {
PyProjectToml(PathBuf, PyProjectToml),
SetupPy(PathBuf),
SetupCfg(PathBuf),
}
impl SourceTree {
/// Return the [`Path`] to the file representing the source tree (e.g., the `pyproject.toml`).
pub fn path(&self) -> &Path {
match self {
Self::PyProjectToml(path, ..) => path,
Self::SetupPy(path) => path,
Self::SetupCfg(path) => path,
}
}
/// Return the [`PyProjectToml`] if this is a `pyproject.toml`-based source tree.
pub fn pyproject_toml(&self) -> Option<&PyProjectToml> {
match self {
Self::PyProjectToml(.., toml) => Some(toml),
_ => None,
}
}
}
#[derive(Debug, Clone)]
pub struct SourceTreeResolution {
/// The requirements sourced from the source trees.
@ -73,7 +100,7 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
/// Resolve the requirements from the provided source trees.
pub async fn resolve(
self,
source_trees: impl Iterator<Item = &Path>,
source_trees: impl Iterator<Item = &SourceTree>,
) -> Result<Vec<SourceTreeResolution>> {
let resolutions: Vec<_> = source_trees
.map(async |source_tree| self.resolve_source_tree(source_tree).await)
@ -84,9 +111,10 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
}
/// Infer the dependencies for a directory dependency.
async fn resolve_source_tree(&self, path: &Path) -> Result<SourceTreeResolution> {
let metadata = self.resolve_requires_dist(path).await?;
let origin = RequirementOrigin::Project(path.to_path_buf(), metadata.name.clone());
async fn resolve_source_tree(&self, source_tree: &SourceTree) -> Result<SourceTreeResolution> {
let metadata = self.resolve_requires_dist(source_tree).await?;
let origin =
RequirementOrigin::Project(source_tree.path().to_path_buf(), metadata.name.clone());
// Determine the extras to include when resolving the requirements.
let extras = self
@ -124,15 +152,15 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
/// requirements without building the distribution, even if the project contains (e.g.) a
/// dynamic version since, critically, we don't need to install the package itself; only its
/// dependencies.
async fn resolve_requires_dist(&self, path: &Path) -> Result<RequiresDist> {
async fn resolve_requires_dist(&self, source_tree: &SourceTree) -> Result<RequiresDist> {
// Convert to a buildable source.
let source_tree = fs_err::canonicalize(path).with_context(|| {
let path = fs_err::canonicalize(source_tree.path()).with_context(|| {
format!(
"Failed to canonicalize path to source tree: {}",
path.user_display()
source_tree.path().user_display()
)
})?;
let source_tree = source_tree.parent().ok_or_else(|| {
let path = path.parent().ok_or_else(|| {
anyhow::anyhow!(
"The file `{}` appears to be a `pyproject.toml`, `setup.py`, or `setup.cfg` file, which must be in a directory",
path.user_display()
@ -144,16 +172,18 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
// _only_ need the requirements. So, for example, even if the version is dynamic, we can
// still extract the requirements without performing a build, unlike in the database where
// we typically construct a "complete" metadata object.
if let Some(metadata) = self.database.requires_dist(source_tree).await? {
return Ok(metadata);
if let Some(pyproject_toml) = source_tree.pyproject_toml() {
if let Some(metadata) = self.database.requires_dist(path, pyproject_toml).await? {
return Ok(metadata);
}
}
let Ok(url) = Url::from_directory_path(source_tree).map(DisplaySafeUrl::from) else {
let Ok(url) = Url::from_directory_path(path).map(DisplaySafeUrl::from) else {
return Err(anyhow::anyhow!("Failed to convert path to URL"));
};
let source = SourceUrl::Directory(DirectorySourceUrl {
url: &url,
install_path: Cow::Borrowed(source_tree),
install_path: Cow::Borrowed(path),
editable: None,
});

View file

@ -44,12 +44,12 @@ use uv_distribution_types::{
};
use uv_fs::{CWD, Simplified};
use uv_normalize::{ExtraName, PackageName, PipGroupName};
use uv_pypi_types::PyProjectToml;
use uv_requirements_txt::{RequirementsTxt, RequirementsTxtRequirement};
use uv_scripts::{Pep723Error, Pep723Item, Pep723Script};
use uv_warnings::warn_user;
use uv_workspace::pyproject::PyProjectToml;
use crate::RequirementsSource;
use crate::{RequirementsSource, SourceTree};
#[derive(Debug, Default, Clone)]
pub struct RequirementsSpecification {
@ -64,7 +64,7 @@ pub struct RequirementsSpecification {
/// The `pylock.toml` file from which to extract the resolution.
pub pylock: Option<PathBuf>,
/// The source trees from which to extract requirements.
pub source_trees: Vec<PathBuf>,
pub source_trees: Vec<SourceTree>,
/// The groups to use for `source_trees`
pub groups: BTreeMap<PathBuf, DependencyGroups>,
/// The extras used to collect requirements.
@ -174,11 +174,11 @@ impl RequirementsSpecification {
));
}
};
let _ = toml::from_str::<PyProjectToml>(&contents)
let pyproject_toml = toml::from_str::<PyProjectToml>(&contents)
.with_context(|| format!("Failed to parse: `{}`", path.user_display()))?;
Self {
source_trees: vec![path.clone()],
source_trees: vec![SourceTree::PyProjectToml(path.clone(), pyproject_toml)],
..Self::default()
}
}
@ -301,13 +301,23 @@ impl RequirementsSpecification {
}
}
}
RequirementsSource::SetupPy(path) | RequirementsSource::SetupCfg(path) => {
RequirementsSource::SetupPy(path) => {
if !path.is_file() {
return Err(anyhow::anyhow!("File not found: `{}`", path.user_display()));
}
Self {
source_trees: vec![path.clone()],
source_trees: vec![SourceTree::SetupPy(path.clone())],
..Self::default()
}
}
RequirementsSource::SetupCfg(path) => {
if !path.is_file() {
return Err(anyhow::anyhow!("File not found: `{}`", path.user_display()));
}
Self {
source_trees: vec![SourceTree::SetupCfg(path.clone())],
..Self::default()
}
}

View file

@ -47,6 +47,7 @@ arcstr = { workspace = true }
clap = { workspace = true, features = ["derive"], optional = true }
dashmap = { workspace = true }
either = { workspace = true }
fs-err = { workspace = true, features = ["tokio"] }
futures = { workspace = true }
hashbrown = { workspace = true }
indexmap = { workspace = true }

View file

@ -42,7 +42,7 @@ use uv_platform_tags::{
};
use uv_pypi_types::{
ConflictKind, Conflicts, HashAlgorithm, HashDigest, HashDigests, Hashes, ParsedArchiveUrl,
ParsedGitUrl,
ParsedGitUrl, PyProjectToml,
};
use uv_redacted::DisplaySafeUrl;
use uv_small_str::SmallString;
@ -1808,13 +1808,29 @@ impl Lock {
// even if the version is dynamic, we can still extract the requirements without
// performing a build, unlike in the database where we typically construct a "complete"
// metadata object.
let metadata = database
.requires_dist(root.join(source_tree))
.await
.map_err(|err| LockErrorKind::Resolution {
id: package.id.clone(),
err,
})?;
let parent = root.join(source_tree);
let path = parent.join("pyproject.toml");
let metadata =
match fs_err::tokio::read_to_string(&path).await {
Ok(contents) => {
let pyproject_toml = toml::from_str::<PyProjectToml>(&contents)
.map_err(|err| LockErrorKind::InvalidPyprojectToml {
path: path.clone(),
err,
})?;
database
.requires_dist(&parent, &pyproject_toml)
.await
.map_err(|err| LockErrorKind::Resolution {
id: package.id.clone(),
err,
})?
}
Err(err) if err.kind() == io::ErrorKind::NotFound => None,
Err(err) => {
return Err(LockErrorKind::UnreadablePyprojectToml { path, err }.into());
}
};
let satisfied = metadata.is_some_and(|metadata| {
// Validate that the package is still dynamic.
@ -5898,6 +5914,18 @@ enum LockErrorKind {
},
#[error(transparent)]
GitUrlParse(#[from] GitUrlParseError),
#[error("Failed to read `{path}`")]
UnreadablePyprojectToml {
path: PathBuf,
#[source]
err: std::io::Error,
},
#[error("Failed to parse `{path}`")]
InvalidPyprojectToml {
path: PathBuf,
#[source]
err: toml::de::Error,
},
}
/// An error that occurs when a source string could not be parsed.

View file

@ -34,7 +34,7 @@ use uv_pypi_types::{Conflicts, ResolverMarkerEnvironment};
use uv_python::{PythonEnvironment, PythonInstallation};
use uv_requirements::{
GroupsSpecification, LookaheadResolver, NamedRequirementsResolver, RequirementsSource,
RequirementsSpecification, SourceTreeResolver,
RequirementsSpecification, SourceTree, SourceTreeResolver,
};
use uv_resolver::{
DependencyMode, Exclusions, FlatIndex, InMemoryIndex, Manifest, Options, Preference,
@ -103,7 +103,7 @@ pub(crate) async fn resolve<InstalledPackages: InstalledPackagesProvider>(
requirements: Vec<UnresolvedRequirementSpecification>,
constraints: Vec<NameRequirementSpecification>,
overrides: Vec<UnresolvedRequirementSpecification>,
source_trees: Vec<PathBuf>,
source_trees: Vec<SourceTree>,
mut project: Option<PackageName>,
workspace_members: BTreeSet<PackageName>,
extras: &ExtrasSpecification,
@ -167,7 +167,7 @@ pub(crate) async fn resolve<InstalledPackages: InstalledPackagesProvider>(
DistributionDatabase::new(client, build_dispatch, concurrency.downloads),
)
.with_reporter(Arc::new(ResolverReporter::from(printer)))
.resolve(source_trees.iter().map(PathBuf::as_path))
.resolve(source_trees.iter())
.await?;
// If we resolved a single project, use it for the project name.