mirror of
https://github.com/astral-sh/uv.git
synced 2025-07-07 21:35:00 +00:00
Use git command to fetch repositories instead of libgit2 for robust SSH support (#1781)
Closes https://github.com/astral-sh/uv/issues/1775 Closes https://github.com/astral-sh/uv/issues/1452 Closes https://github.com/astral-sh/uv/issues/1514 Follows https://github.com/astral-sh/uv/pull/1717 libgit2 does not support host names with extra identifiers during SSH lookup (e.g. [`github.com-some_identifier`]( https://docs.github.com/en/authentication/connecting-to-github-with-ssh/managing-deploy-keys#using-multiple-repositories-on-one-server)) so we use the `git` command instead for fetching. This is required for `pip` parity. See the [Cargo documentation](https://doc.rust-lang.org/nightly/cargo/reference/config.html#netgit-fetch-with-cli) for more details on using the `git` CLI instead of libgit2. We may want to try to use libgit2 first in the future, as it is more performant (#1786). We now support authentication with: ``` git+ssh://git@<hostname>/... git+ssh://git@<hostname>-<identifier>/... ``` Tested with a deploy key e.g. ``` cargo run -- \ pip install uv-private-pypackage@git+ssh://git@github.com-test-uv-private-pypackage/astral-test/uv-private-pypackage.git \ --reinstall --no-cache -v ``` and ``` cargo run -- \ pip install uv-private-pypackage@git+ssh://git@github.com/astral-test/uv-private-pypackage.git \ --reinstall --no-cache -v ``` with a ssh config like ``` Host github.com Hostname github.com IdentityFile=/Users/mz/.ssh/id_ed25519 Host github.com-test-uv-private-pypackage Hostname github.com IdentityFile=/Users/mz/.ssh/id_ed25519 ``` It seems quite hard to add test coverage for this to the test suite, as we'd need to add the SSH key and I don't know how to isolate that from affecting other developer's machines.
This commit is contained in:
Zanie Blue
GitHub
parent
b4bc40627c
commit
71ec568d0f
4 changed files with 179 additions and 12 deletions
|
|
@ -11,7 +11,7 @@ use cargo_util::{paths, ProcessBuilder};
|
|||
use git2::{self, ErrorClass, ObjectType};
|
||||
use reqwest::Client;
|
||||
use reqwest::StatusCode;
|
||||
use tracing::{debug, warn};
|
||||
use tracing::{debug, error, warn};
|
||||
use url::Url;
|
||||
use uv_fs::Normalized;
|
||||
|
||||
|
|
@ -43,6 +43,14 @@ pub(crate) enum GitReference {
|
|||
DefaultBranch,
|
||||
}
|
||||
|
||||
/// Strategy when fetching refspecs for a [`GitReference`]
|
||||
enum RefspecStrategy {
|
||||
// All refspecs should be fetched, if any fail then the fetch will fail
|
||||
All,
|
||||
// Stop after the first successful fetch, if none suceed then the fetch will fail
|
||||
First,
|
||||
}
|
||||
|
||||
impl GitReference {
|
||||
pub(crate) fn from_rev(rev: &str) -> Self {
|
||||
if rev.starts_with("refs/") {
|
||||
|
|
@ -908,6 +916,7 @@ pub(crate) fn fetch(
|
|||
// which need to get fetched. Additionally record if we're fetching tags.
|
||||
let mut refspecs = Vec::new();
|
||||
let mut tags = false;
|
||||
let mut refspec_strategy = RefspecStrategy::All;
|
||||
// The `+` symbol on the refspec means to allow a forced (fast-forward)
|
||||
// update which is needed if there is ever a force push that requires a
|
||||
// fast-forward.
|
||||
|
|
@ -929,6 +938,7 @@ pub(crate) fn fetch(
|
|||
refspecs.push(format!(
|
||||
"+refs/tags/{branch_or_tag}:refs/remotes/origin/tags/{branch_or_tag}"
|
||||
));
|
||||
refspec_strategy = RefspecStrategy::First;
|
||||
}
|
||||
|
||||
GitReference::DefaultBranch => {
|
||||
|
|
@ -969,13 +979,49 @@ pub(crate) fn fetch(
|
|||
|
||||
debug!("Performing a Git fetch for: {remote_url}");
|
||||
match strategy {
|
||||
FetchStrategy::Cli => fetch_with_cli(repo, remote_url, &refspecs, tags),
|
||||
FetchStrategy::Cli => {
|
||||
match refspec_strategy {
|
||||
RefspecStrategy::All => fetch_with_cli(repo, remote_url, refspecs.as_slice(), tags),
|
||||
RefspecStrategy::First => {
|
||||
let num_refspecs = refspecs.len();
|
||||
|
||||
// Try each refspec
|
||||
let errors = refspecs
|
||||
.into_iter()
|
||||
.map(|refspec| {
|
||||
(
|
||||
refspec.clone(),
|
||||
fetch_with_cli(repo, remote_url, &[refspec], tags),
|
||||
)
|
||||
})
|
||||
// Stop after the first success
|
||||
.take_while(|(_, result)| result.is_err())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if errors.len() == num_refspecs {
|
||||
// If all of the fetches failed, report to the user
|
||||
for (refspec, err) in errors {
|
||||
if let Err(err) = err {
|
||||
error!("failed to fetch refspec `{refspec}`: {err}");
|
||||
}
|
||||
}
|
||||
Err(anyhow!("failed to fetch all refspecs"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
FetchStrategy::Libgit2 => {
|
||||
// Libgit2 does not fail if a refspec is missing, so the `refspec_strategy`
|
||||
// is not handled here
|
||||
|
||||
let git_config = git2::Config::open_default()?;
|
||||
with_fetch_options(&git_config, remote_url, &mut |mut opts| {
|
||||
if tags {
|
||||
opts.download_tags(git2::AutotagOption::All);
|
||||
}
|
||||
|
||||
// The `fetch` operation here may fail spuriously due to a corrupt
|
||||
// repository. It could also fail, however, for a whole slew of other
|
||||
// reasons (aka network related reasons). We want Cargo to automatically
|
||||
|
|
@ -1057,7 +1103,11 @@ fn fetch_with_cli(
|
|||
.env_remove("GIT_OBJECT_DIRECTORY")
|
||||
.env_remove("GIT_ALTERNATE_OBJECT_DIRECTORIES")
|
||||
.cwd(repo.path());
|
||||
cmd.exec()?;
|
||||
|
||||
// We capture the output to avoid streaming it to the user's console during clones.
|
||||
// The required `on...line` callbacks currently do nothing.
|
||||
// The output appears to be included in error messages by default.
|
||||
cmd.exec_with_streaming(&mut |_| Ok(()), &mut |_| Ok(()), true)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ impl GitSource {
|
|||
Self {
|
||||
git,
|
||||
client: Client::new(),
|
||||
strategy: FetchStrategy::Libgit2,
|
||||
strategy: FetchStrategy::Cli,
|
||||
cache: cache.into(),
|
||||
reporter: None,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,6 @@
|
|||
// The `unreachable_pub` is to silence false positives in RustRover.
|
||||
#![allow(dead_code, unreachable_pub)]
|
||||
|
||||
use std::borrow::BorrowMut;
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Output;
|
||||
|
||||
use assert_cmd::assert::{Assert, OutputAssertExt};
|
||||
use assert_cmd::Command;
|
||||
use assert_fs::assert::PathAssert;
|
||||
|
|
@ -14,7 +9,12 @@ use assert_fs::fixture::PathChild;
|
|||
use fs_err::os::unix::fs::symlink as symlink_file;
|
||||
#[cfg(windows)]
|
||||
use fs_err::os::windows::fs::symlink_file;
|
||||
use regex::Regex;
|
||||
use regex::{self, Regex};
|
||||
use std::borrow::BorrowMut;
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Output;
|
||||
use uv_fs::Normalized;
|
||||
|
||||
use platform_host::Platform;
|
||||
use uv_cache::Cache;
|
||||
|
|
@ -44,17 +44,39 @@ pub struct TestContext {
|
|||
pub temp_dir: assert_fs::TempDir,
|
||||
pub cache_dir: assert_fs::TempDir,
|
||||
pub venv: PathBuf,
|
||||
|
||||
// Standard filters for this test context
|
||||
filters: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl TestContext {
|
||||
pub fn new(python_version: &str) -> Self {
|
||||
let temp_dir = assert_fs::TempDir::new().expect("Failed to create temp dir");
|
||||
let cache_dir = assert_fs::TempDir::new().expect("Failed to create temp dir");
|
||||
let cache_dir = assert_fs::TempDir::new().expect("Failed to create cache dir");
|
||||
let venv = create_venv(&temp_dir, &cache_dir, python_version);
|
||||
|
||||
let mut filters = Vec::new();
|
||||
filters.extend(
|
||||
Self::path_patterns(&cache_dir)
|
||||
.into_iter()
|
||||
.map(|pattern| (pattern, "[CACHE_DIR]/".to_string())),
|
||||
);
|
||||
filters.extend(
|
||||
Self::path_patterns(&temp_dir)
|
||||
.into_iter()
|
||||
.map(|pattern| (pattern, "[TEMP_DIR]/".to_string())),
|
||||
);
|
||||
filters.extend(
|
||||
Self::path_patterns(&venv)
|
||||
.into_iter()
|
||||
.map(|pattern| (pattern, "[VENV]/".to_string())),
|
||||
);
|
||||
|
||||
Self {
|
||||
temp_dir,
|
||||
cache_dir,
|
||||
venv,
|
||||
filters,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -96,6 +118,46 @@ impl TestContext {
|
|||
.success()
|
||||
.stdout(version);
|
||||
}
|
||||
|
||||
/// Generate an escaped regex pattern for the given path.
|
||||
fn path_patterns(path: impl AsRef<Path>) -> Vec<String> {
|
||||
vec![
|
||||
format!(
|
||||
// Trim the trailing separator for cross-platform directories filters
|
||||
r"{}\\?/?",
|
||||
regex::escape(
|
||||
&path
|
||||
.as_ref()
|
||||
.canonicalize()
|
||||
.expect("Failed to create canonical path")
|
||||
// Normalize the path to match display and remove UNC prefixes on Windows
|
||||
.normalized()
|
||||
.display()
|
||||
.to_string(),
|
||||
)
|
||||
// Make seprators platform agnostic because on Windows we will display
|
||||
// paths with Unix-style separators sometimes
|
||||
.replace(r"\\", r"(\\|\/)")
|
||||
),
|
||||
// Include a non-canonicalized version
|
||||
format!(
|
||||
r"{}\\?/?",
|
||||
regex::escape(&path.as_ref().normalized().display().to_string())
|
||||
.replace(r"\\", r"(\\|\/)")
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
/// Canonical snapshot filters for this test context.
|
||||
pub fn filters(&self) -> Vec<(&str, &str)> {
|
||||
let mut filters = INSTA_FILTERS.to_vec();
|
||||
|
||||
for (pattern, replacement) in &self.filters {
|
||||
filters.push((pattern, replacement));
|
||||
}
|
||||
|
||||
filters
|
||||
}
|
||||
}
|
||||
|
||||
pub fn venv_to_interpreter(venv: &Path) -> PathBuf {
|
||||
|
|
|
|||
|
|
@ -812,6 +812,28 @@ fn install_git_public_https() {
|
|||
context.assert_installed("uv_public_pypackage", "0.1.0");
|
||||
}
|
||||
|
||||
/// Install a package from a public GitHub repository at a ref that does not exist
|
||||
#[test]
|
||||
#[cfg(feature = "git")]
|
||||
fn install_git_public_https_missing_ref() {
|
||||
let context = TestContext::new("3.8");
|
||||
|
||||
uv_snapshot!(context.filters(), command(&context)
|
||||
// 2.0.0 does not exist
|
||||
.arg("uv-public-pypackage @ git+https://github.com/astral-test/uv-public-pypackage@2.0.0")
|
||||
, @r###"
|
||||
success: false
|
||||
exit_code: 2
|
||||
----- stdout -----
|
||||
|
||||
----- stderr -----
|
||||
error: Failed to download and build: uv-public-pypackage @ git+https://github.com/astral-test/uv-public-pypackage@2.0.0
|
||||
Caused by: Git operation failed
|
||||
Caused by: failed to clone into: [CACHE_DIR]/git-v0/db/8dab139913c4b566
|
||||
Caused by: failed to fetch all refspecs
|
||||
"###);
|
||||
}
|
||||
|
||||
/// Install a package from a private GitHub repository using a PAT
|
||||
#[test]
|
||||
#[cfg(all(not(windows), feature = "git"))]
|
||||
|
|
@ -886,7 +908,6 @@ fn install_git_private_https_pat_and_username() {
|
|||
|
||||
let mut filters = INSTA_FILTERS.to_vec();
|
||||
filters.insert(0, (&token, "***"));
|
||||
filters.push(("failed to clone into: .*", "failed to clone into: [PATH]"));
|
||||
|
||||
uv_snapshot!(filters, command(&context)
|
||||
.arg(format!("uv-private-pypackage @ git+https://{user}:{token}@github.com/astral-test/uv-private-pypackage"))
|
||||
|
|
@ -905,6 +926,40 @@ fn install_git_private_https_pat_and_username() {
|
|||
context.assert_installed("uv_private_pypackage", "0.1.0");
|
||||
}
|
||||
|
||||
/// Install a package from a private GitHub repository using a PAT
|
||||
#[test]
|
||||
#[cfg(all(not(windows), feature = "git"))]
|
||||
fn install_git_private_https_pat_not_authorized() {
|
||||
let context = TestContext::new("3.8");
|
||||
|
||||
// A revoked token
|
||||
let token = "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";
|
||||
|
||||
let mut filters = context.filters();
|
||||
filters.insert(0, (&token, "***"));
|
||||
|
||||
// We provide a username otherwise (since the token is invalid), the git cli will prompt for a password
|
||||
// and hang the test
|
||||
uv_snapshot!(filters, command(&context)
|
||||
.arg(format!("uv-private-pypackage @ git+https://git:{token}@github.com/astral-test/uv-private-pypackage"))
|
||||
, @r###"
|
||||
success: false
|
||||
exit_code: 2
|
||||
----- stdout -----
|
||||
|
||||
----- stderr -----
|
||||
error: Failed to download and build: uv-private-pypackage @ git+https://git:***@github.com/astral-test/uv-private-pypackage
|
||||
Caused by: Git operation failed
|
||||
Caused by: failed to clone into: [CACHE_DIR]/git-v0/db/2496970ed6fdf08f
|
||||
Caused by: process didn't exit successfully: `git fetch --force --update-head-ok 'https://git:***@github.com/astral-test/uv-private-pypackage' '+HEAD:refs/remotes/origin/HEAD'` (exit status: 128)
|
||||
--- stderr
|
||||
remote: Support for password authentication was removed on August 13, 2021.
|
||||
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
|
||||
fatal: Authentication failed for 'https://github.com/astral-test/uv-private-pypackage/'
|
||||
|
||||
"###);
|
||||
}
|
||||
|
||||
/// Install a package without using pre-built wheels.
|
||||
#[test]
|
||||
fn reinstall_no_binary() {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue