mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 21:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Respect existing versions in "lockfile" (#187)

Browse source 
Like `pip-compile`, we now respect existing versions from the
`requirements.txt` provided via `--output-file`, unless you pass a
`--upgrade` flag.

Closes #166.
This commit is contained in:
Charlie Marsh 2023-10-25 21:28:58 -07:00 • committed by GitHub
parent 9f894213e0
commit 6faaf4bc24
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 347 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

53
crates/puffin-cli/src/commands/pip_compile.rs
View file

@ -7,15 +7,16 @@ use anyhow::Result;
use colored::Colorize;
use fs_err::File;
use itertools::Itertools;
use pubgrub::report::Reporter;
use tracing::debug;
use pep508_rs::Requirement;
use platform_host::Platform;
use platform_tags::Tags;
use pubgrub::report::Reporter;
use puffin_client::RegistryClientBuilder;
use puffin_dispatch::BuildDispatch;
use puffin_interpreter::Virtualenv;
use puffin_resolver::ResolutionMode;
use tracing::debug;
use puffin_resolver::{Manifest, ResolutionMode};
use crate::commands::{elapsed, ExitStatus};
use crate::index_urls::IndexUrls;
@ -25,11 +26,13 @@ use crate::requirements::RequirementsSource;
const VERSION: &str = env!("CARGO_PKG_VERSION");
/// Resolve a set of requirements into a set of pinned versions.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn pip_compile(
requirements: &[RequirementsSource],
constraints: &[RequirementsSource],
output_file: Option<&Path>,
mode: ResolutionMode,
resolution_mode: ResolutionMode,
upgrade_mode: UpgradeMode,
index_urls: Option<IndexUrls>,
cache: Option<&Path>,
mut printer: Printer,
@ -47,6 +50,19 @@ pub(crate) async fn pip_compile(
.map(RequirementsSource::requirements)
.flatten_ok()
.collect::<Result<Vec<Requirement>>>()?;
let preferences: Vec<Requirement> = output_file
.filter(|_| upgrade_mode.is_prefer_pinned())
.filter(|output_file| output_file.exists())
.map(Path::to_path_buf)
.map(RequirementsSource::from)
.as_ref()
.map(RequirementsSource::requirements)
.transpose()?
.map(Iterator::collect)
.unwrap_or_default();
// Create a manifest of the requirements.
let manifest = Manifest::new(requirements, constraints, preferences, resolution_mode);
// Detect the current Python interpreter.
let platform = Platform::current()?;
@ -88,9 +104,7 @@ pub(crate) async fn pip_compile(
// Resolve the dependencies.
let resolver = puffin_resolver::Resolver::new(
requirements,
constraints,
mode,
manifest,
venv.interpreter_info().markers(),
&tags,
&client,
@ -155,3 +169,28 @@ pub(crate) async fn pip_compile(
Ok(ExitStatus::Success)
}
/// Whether to allow package upgrades.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum UpgradeMode {
/// Allow package upgrades, ignoring the existing lockfile.
AllowUpgrades,
/// Prefer pinned versions from the existing lockfile, if possible.
PreferPinned,
}
impl UpgradeMode {
fn is_prefer_pinned(self) -> bool {
self == Self::PreferPinned
}
}
impl From<bool> for UpgradeMode {
fn from(value: bool) -> Self {
if value {
Self::AllowUpgrades
} else {
Self::PreferPinned
}
}
}

5
crates/puffin-cli/src/main.rs
View file

@ -89,6 +89,10 @@ struct PipCompileArgs {
/// Ignore the package index, instead relying on local archives and caches.
#[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")]
no_index: bool,
/// Allow package upgrades, ignoring pinned versions in the existing output file.
#[clap(long)]
upgrade: bool,
}
#[derive(Args)]
@ -196,6 +200,7 @@ async fn main() -> ExitCode {
&constraints,
args.output_file.as_deref(),
args.resolution.unwrap_or_default(),
args.upgrade.into(),
index_urls,
cache_dir,
printer,

7
crates/puffin-dispatch/src/lib.rs
View file

@ -18,7 +18,7 @@ use puffin_installer::{
uninstall, Downloader, Installer, PartitionedRequirements, RemoteDistribution, Unzipper,
};
use puffin_interpreter::{InterpreterInfo, Virtualenv};
use puffin_resolver::{ResolutionMode, Resolver, WheelFinder};
use puffin_resolver::{Manifest, ResolutionMode, Resolver, WheelFinder};
use puffin_traits::BuildContext;
use tracing::debug;
@ -70,9 +70,12 @@ impl BuildContext for BuildDispatch {
self.interpreter_info.simple_version(),
)?;
let resolver = Resolver::new(
Manifest::new(
requirements.to_vec(),
Vec::default(),
ResolutionMode::Highest,
Vec::default(),
ResolutionMode::default(),
),
self.interpreter_info.markers(),
&tags,
&self.client,

13
crates/puffin-resolver/src/distribution.rs
View file

@ -66,6 +66,19 @@ impl From<SdistFile> for DistributionFile {
}
}
impl From<File> for DistributionFile {
fn from(file: File) -> Self {
if std::path::Path::new(file.filename.as_str())
.extension()
.map_or(false, |ext| ext.eq_ignore_ascii_case("whl"))
{
Self::Wheel(WheelFile::from(file))
} else {
Self::Sdist(SdistFile::from(file))
}
}
}
impl DistributionFile {
pub(crate) fn filename(&self) -> &str {
match self {

2
crates/puffin-resolver/src/lib.rs
View file

@ -1,6 +1,6 @@
pub use error::ResolveError;
pub use resolution::PinnedPackage;
pub use resolver::Resolver;
pub use resolver::{Manifest, Resolver};
pub use selector::ResolutionMode;
pub use source_distribution::BuiltSourceDistributionCache;
pub use wheel_finder::{Reporter, WheelFinder};

5
crates/puffin-resolver/src/resolution.rs
View file

@ -69,6 +69,11 @@ impl Resolution {
self.0.into_values().map(|package| package.file)
}
/// Return the pinned package for the given package name, if it exists.
pub fn get(&self, package_name: &PackageName) -> Option<&PinnedPackage> {
self.0.get(package_name)
}
/// Return the number of pinned packages in this resolution.
pub fn len(&self) -> usize {
self.0.len()

49
crates/puffin-resolver/src/resolver.rs
View file

@ -34,15 +34,39 @@ use crate::error::ResolveError;
use crate::pubgrub::package::PubGrubPackage;
use crate::pubgrub::version::{PubGrubVersion, MIN_VERSION};
use crate::pubgrub::{iter_requirements, version_range};
use crate::resolution::{Graph, Resolution};
use crate::resolution::Graph;
use crate::selector::{CandidateSelector, ResolutionMode};
use crate::source_distribution::{download_and_build_sdist, read_dist_info};
use crate::BuiltSourceDistributionCache;
/// A manifest of requirements, constraints, and preferences.
#[derive(Debug)]
pub struct Manifest {
requirements: Vec<Requirement>,
constraints: Vec<Requirement>,
preferences: Vec<Requirement>,
mode: ResolutionMode,
}
impl Manifest {
pub fn new(
requirements: Vec<Requirement>,
constraints: Vec<Requirement>,
preferences: Vec<Requirement>,
mode: ResolutionMode,
) -> Self {
Self {
requirements,
constraints,
preferences,
mode,
}
}
}
pub struct Resolver<'a, Context: BuildContext> {
requirements: Vec<Requirement>,
constraints: Vec<Requirement>,
resolution: Option<Resolution>,
markers: &'a MarkerEnvironment,
tags: &'a Tags,
client: &'a RegistryClient,
@ -54,20 +78,21 @@ pub struct Resolver<'a, Context: BuildContext> {
impl<'a, Context: BuildContext> Resolver<'a, Context> {
/// Initialize a new resolver.
pub fn new(
requirements: Vec<Requirement>,
constraints: Vec<Requirement>,
mode: ResolutionMode,
manifest: Manifest,
markers: &'a MarkerEnvironment,
tags: &'a Tags,
client: &'a RegistryClient,
build_context: &'a Context,
) -> Self {
Self {
selector: CandidateSelector::from_mode(mode, &requirements),
selector: CandidateSelector::from_mode(
manifest.mode,
&manifest.requirements,
&manifest.preferences,
),
index: Arc::new(Index::default()),
resolution: None,
requirements,
constraints,
requirements: manifest.requirements,
constraints: manifest.constraints,
markers,
tags,
client,
@ -75,12 +100,6 @@ impl<'a, Context: BuildContext> Resolver<'a, Context> {
}
}
#[must_use]
pub fn with_resolution(mut self, resolution: Resolution) -> Self {
self.resolution = Some(resolution);
self
}
/// Resolve a set of requirements into a set of pinned versions.
pub async fn resolve(self) -> Result<Graph, ResolveError> {
// A channel to fetch package metadata (e.g., given `flask`, fetch all versions) and version

157
crates/puffin-resolver/src/selector.rs
View file

@ -1,11 +1,10 @@
use fxhash::FxHashSet;
use fxhash::{FxHashMap, FxHashSet};
use pubgrub::range::Range;
use crate::distribution::DistributionFile;
use pep508_rs::Requirement;
use pep508_rs::{Requirement, VersionOrUrl};
use puffin_package::package_name::PackageName;
use crate::distribution::DistributionFile;
use crate::pubgrub::version::PubGrubVersion;
use crate::resolver::VersionMap;
@ -22,8 +21,10 @@ pub enum ResolutionMode {
LowestDirect,
}
#[derive(Debug, Clone)]
pub(crate) enum CandidateSelector {
/// Like [`ResolutionMode`], but with any additional information required to select a candidate,
/// like the set of direct dependencies.
#[derive(Debug)]
enum ResolutionStrategy {
/// Resolve the highest compatible version of each package.
Highest,
/// Resolve the lowest compatible version of each package.
@ -33,9 +34,8 @@ pub(crate) enum CandidateSelector {
LowestDirect(FxHashSet<PackageName>),
}
impl CandidateSelector {
/// Return a candidate selector for the given resolution mode.
pub(crate) fn from_mode(mode: ResolutionMode, direct_dependencies: &[Requirement]) -> Self {
impl ResolutionStrategy {
fn from_mode(mode: ResolutionMode, direct_dependencies: &[Requirement]) -> Self {
match mode {
ResolutionMode::Highest => Self::Highest,
ResolutionMode::Lowest => Self::Lowest,
@ -49,26 +49,87 @@ impl CandidateSelector {
}
}
/// A set of pinned packages that should be preserved during resolution, if possible.
#[derive(Debug)]
struct Preferences(FxHashMap<PackageName, PubGrubVersion>);
impl Preferences {
fn get(&self, package_name: &PackageName) -> Option<&PubGrubVersion> {
self.0.get(package_name)
}
}
impl From<&[Requirement]> for Preferences {
fn from(requirements: &[Requirement]) -> Self {
Self(
requirements
.iter()
.filter_map(|requirement| {
let Some(VersionOrUrl::VersionSpecifier(version_specifiers)) =
requirement.version_or_url.as_ref()
else {
return None;
};
let [version_specifier] = &**version_specifiers else {
return None;
};
let package_name = PackageName::normalize(&requirement.name);
let version = PubGrubVersion::from(version_specifier.version().clone());
Some((package_name, version))
})
.collect(),
)
}
}
#[derive(Debug)]
pub(crate) struct CandidateSelector {
strategy: ResolutionStrategy,
preferences: Preferences,
}
impl CandidateSelector {
/// Return a candidate selector for the given resolution mode.
pub(crate) fn from_mode(
mode: ResolutionMode,
direct_dependencies: &[Requirement],
resolution: &[Requirement],
) -> Self {
Self {
strategy: ResolutionStrategy::from_mode(mode, direct_dependencies),
preferences: Preferences::from(resolution),
}
}
}
impl CandidateSelector {
/// Select a [`Candidate`] from a set of candidate versions and files.
pub(crate) fn select<'a>(
pub(crate) fn select(
&self,
package_name: &'a PackageName,
range: &'a Range<PubGrubVersion>,
version_map: &'a VersionMap,
) -> Option<Candidate<'a>> {
match self {
CandidateSelector::Highest => {
CandidateSelector::select_highest(package_name, range, version_map)
package_name: &PackageName,
range: &Range<PubGrubVersion>,
version_map: &VersionMap,
) -> Option<Candidate> {
if let Some(version) = self.preferences.get(package_name) {
if range.contains(version) {
if let Some(file) = version_map.get(version) {
return Some(Candidate {
package_name: package_name.clone(),
version: version.clone(),
file: file.clone(),
});
}
CandidateSelector::Lowest => {
CandidateSelector::select_lowest(package_name, range, version_map)
}
CandidateSelector::LowestDirect(direct_dependencies) => {
}
match &self.strategy {
ResolutionStrategy::Highest => Self::select_highest(package_name, range, version_map),
ResolutionStrategy::Lowest => Self::select_lowest(package_name, range, version_map),
ResolutionStrategy::LowestDirect(direct_dependencies) => {
if direct_dependencies.contains(package_name) {
CandidateSelector::select_lowest(package_name, range, version_map)
Self::select_lowest(package_name, range, version_map)
} else {
CandidateSelector::select_highest(package_name, range, version_map)
Self::select_highest(package_name, range, version_map)
}
}
}
@ -76,27 +137,27 @@ impl CandidateSelector {
/// Select the highest-compatible [`Candidate`] from a set of candidate versions and files,
/// preferring wheels over sdists.
fn select_highest<'a>(
package_name: &'a PackageName,
range: &'a Range<PubGrubVersion>,
version_map: &'a VersionMap,
) -> Option<Candidate<'a>> {
fn select_highest(
package_name: &PackageName,
range: &Range<PubGrubVersion>,
version_map: &VersionMap,
) -> Option<Candidate> {
let mut sdist = None;
for (version, file) in version_map.iter().rev() {
if range.contains(version) {
match file {
DistributionFile::Wheel(_) => {
return Some(Candidate {
package_name,
version,
file,
package_name: package_name.clone(),
version: version.clone(),
file: file.clone(),
});
}
DistributionFile::Sdist(_) => {
sdist = Some(Candidate {
package_name,
version,
file,
package_name: package_name.clone(),
version: version.clone(),
file: file.clone(),
});
}
}
@ -107,27 +168,27 @@ impl CandidateSelector {
/// Select the highest-compatible [`Candidate`] from a set of candidate versions and files,
/// preferring wheels over sdists.
fn select_lowest<'a>(
package_name: &'a PackageName,
range: &'a Range<PubGrubVersion>,
version_map: &'a VersionMap,
) -> Option<Candidate<'a>> {
fn select_lowest(
package_name: &PackageName,
range: &Range<PubGrubVersion>,
version_map: &VersionMap,
) -> Option<Candidate> {
let mut sdist = None;
for (version, file) in version_map {
if range.contains(version) {
match file {
DistributionFile::Wheel(_) => {
return Some(Candidate {
package_name,
version,
file,
package_name: package_name.clone(),
version: version.clone(),
file: file.clone(),
});
}
DistributionFile::Sdist(_) => {
sdist = Some(Candidate {
package_name,
version,
file,
package_name: package_name.clone(),
version: version.clone(),
file: file.clone(),
});
}
}
@ -138,11 +199,11 @@ impl CandidateSelector {
}
#[derive(Debug, Clone)]
pub(crate) struct Candidate<'a> {
pub(crate) struct Candidate {
/// The name of the package.
pub(crate) package_name: &'a PackageName,
pub(crate) package_name: PackageName,
/// The version of the package.
pub(crate) version: &'a PubGrubVersion,
pub(crate) version: PubGrubVersion,
/// The file of the package.
pub(crate) file: &'a DistributionFile,
pub(crate) file: DistributionFile,
}

140
crates/puffin-resolver/tests/resolver.rs
View file

@ -16,7 +16,7 @@ use platform_host::{Arch, Os, Platform};
use platform_tags::Tags;
use puffin_client::RegistryClientBuilder;
use puffin_interpreter::{InterpreterInfo, Virtualenv};
use puffin_resolver::{ResolutionMode, Resolver};
use puffin_resolver::{Manifest, ResolutionMode, Resolver};
use puffin_traits::BuildContext;
struct DummyContext;
@ -66,15 +66,15 @@ async fn pylint() -> Result<()> {
let requirements = vec![Requirement::from_str("pylint==2.3.0").unwrap()];
let constraints = vec![];
let resolver = Resolver::new(
let preferences = vec![];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::default(),
&MARKERS_311,
&TAGS_311,
&client,
&DummyContext,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
@ -90,15 +90,15 @@ async fn black() -> Result<()> {
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
let constraints = vec![];
let resolver = Resolver::new(
let preferences = vec![];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::default(),
&MARKERS_311,
&TAGS_311,
&client,
&DummyContext,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
@ -114,15 +114,15 @@ async fn black_colorama() -> Result<()> {
let requirements = vec![Requirement::from_str("black[colorama]<=23.9.1").unwrap()];
let constraints = vec![];
let resolver = Resolver::new(
let preferences = vec![];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::default(),
&MARKERS_311,
&TAGS_311,
&client,
&DummyContext,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
@ -138,15 +138,15 @@ async fn black_python_310() -> Result<()> {
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
let constraints = vec![];
let resolver = Resolver::new(
let preferences = vec![];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::default(),
&MARKERS_310,
&TAGS_310,
&client,
&DummyContext,
);
let resolver = Resolver::new(manifest, &MARKERS_310, &TAGS_310, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
@ -164,15 +164,15 @@ async fn black_mypy_extensions() -> Result<()> {
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
let constraints = vec![Requirement::from_str("mypy-extensions<1").unwrap()];
let resolver = Resolver::new(
let preferences = vec![];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::default(),
&MARKERS_311,
&TAGS_311,
&client,
&DummyContext,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
@ -190,15 +190,15 @@ async fn black_mypy_extensions_extra() -> Result<()> {
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
let constraints = vec![Requirement::from_str("mypy-extensions[extra]<1").unwrap()];
let resolver = Resolver::new(
let preferences = vec![];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::default(),
&MARKERS_311,
&TAGS_311,
&client,
&DummyContext,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
@ -216,15 +216,15 @@ async fn black_flake8() -> Result<()> {
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
let constraints = vec![Requirement::from_str("flake8<1").unwrap()];
let resolver = Resolver::new(
let preferences = vec![];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::default(),
&MARKERS_311,
&TAGS_311,
&client,
&DummyContext,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
@ -240,15 +240,15 @@ async fn black_lowest() -> Result<()> {
let requirements = vec![Requirement::from_str("black>21").unwrap()];
let constraints = vec![];
let resolver = Resolver::new(
let preferences = vec![];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::Lowest,
&MARKERS_311,
&TAGS_311,
&client,
&DummyContext,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
@ -264,15 +264,63 @@ async fn black_lowest_direct() -> Result<()> {
let requirements = vec![Requirement::from_str("black>21").unwrap()];
let constraints = vec![];
let resolver = Resolver::new(
let preferences = vec![];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::LowestDirect,
&MARKERS_311,
&TAGS_311,
&client,
&DummyContext,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
Ok(())
}
#[tokio::test]
async fn black_respect_preference() -> Result<()> {
colored::control::set_override(false);
let client = RegistryClientBuilder::default().build();
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
let constraints = vec![];
let preferences = vec![Requirement::from_str("black==23.9.0").unwrap()];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::default(),
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);
Ok(())
}
#[tokio::test]
async fn black_ignore_preference() -> Result<()> {
colored::control::set_override(false);
let client = RegistryClientBuilder::default().build();
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
let constraints = vec![];
let preferences = vec![Requirement::from_str("black==23.9.2").unwrap()];
let manifest = Manifest::new(
requirements,
constraints,
preferences,
ResolutionMode::default(),
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
insta::assert_display_snapshot!(resolution);

16
crates/puffin-resolver/tests/snapshots/resolver__black_ignore_preference.snap Normal file
View file

@ -0,0 +1,16 @@
---
source: crates/puffin-resolver/tests/resolver.rs
expression: resolution
---
black==23.9.1
click==8.1.7
# via black
mypy-extensions==1.0.0
# via black
packaging==23.2
# via black
pathspec==0.11.2
# via black
platformdirs==3.11.0
# via black

16
crates/puffin-resolver/tests/snapshots/resolver__black_respect_preference.snap Normal file
View file

@ -0,0 +1,16 @@
---
source: crates/puffin-resolver/tests/resolver.rs
expression: resolution
---
black==23.9.0
click==8.1.7
# via black
mypy-extensions==1.0.0
# via black
packaging==23.2
# via black
pathspec==0.11.2
# via black
platformdirs==3.11.0
# via black