mirror of
https://github.com/astral-sh/uv.git
synced 2025-07-07 21:35:00 +00:00
Port all git functionality to use git CLI (#3833)
## Summary We currently rely on libgit2 for most git-related functionality. However, libgit2 has long-standing performance issues, as well as lags significantly behind git in terms of new features. For these reasons we now use the git CLI by default for fetching repositories (https://github.com/astral-sh/uv/pull/1781). This PR completely drops libgit2 in favor of the git CLI for all git-related functionality, which should allow us to use features such as partial clones and sparse checkouts in the future for performance. There is also a lot of technical debt in the current git code as it's mostly taken from Cargo. Switching to the git CLI *vastly* simplifies the `uv-git` codebase. Eventually we might want to look into switching to [`gitoxide`](https://github.com/Byron/gitoxide), but it's currently too immature for our use case.
This commit is contained in:
Ibraheem Ahmed
GitHub
parent
85183c1c36
commit
261aa2c70a
16 changed files with 300 additions and 2318 deletions
|
|
@ -71,7 +71,7 @@ Source distributions can run arbitrary code on build and can make unwanted modif
|
|||
```bash
|
||||
docker buildx build -t uv-builder -f builder.dockerfile --load .
|
||||
# Build for musl to avoid glibc errors, might not be required with your OS version
|
||||
cargo build --target x86_64-unknown-linux-musl --profile profiling --features vendored-openssl
|
||||
cargo build --target x86_64-unknown-linux-musl --profile profiling
|
||||
docker run --rm -it -v $(pwd):/app uv-builder /app/target/x86_64-unknown-linux-musl/profiling/uv-dev resolve-many --cache-dir /app/cache-docker /app/scripts/popular_packages/pypi_10k_most_dependents.txt
|
||||
```
|
||||
|
||||
|
|
|
|||
114
Cargo.lock
generated
114
Cargo.lock
generated
|
|
@ -1058,7 +1058,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1465,21 +1464,6 @@ version = "0.28.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
|
||||
|
||||
[[package]]
|
||||
name = "git2"
|
||||
version = "0.18.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70"
|
||||
dependencies = [
|
||||
"bitflags 2.5.0",
|
||||
"libc",
|
||||
"libgit2-sys",
|
||||
"log",
|
||||
"openssl-probe",
|
||||
"openssl-sys",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.1"
|
||||
|
|
@ -1578,15 +1562,6 @@ version = "0.4.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||
dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "home"
|
||||
version = "0.5.9"
|
||||
|
|
@ -2002,20 +1977,6 @@ version = "0.2.153"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
|
||||
|
||||
[[package]]
|
||||
name = "libgit2-sys"
|
||||
version = "0.16.2+1.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"libssh2-sys",
|
||||
"libz-sys",
|
||||
"openssl-sys",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.38"
|
||||
|
|
@ -2036,20 +1997,6 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libssh2-sys"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"libz-sys",
|
||||
"openssl-sys",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libz-ng-sys"
|
||||
version = "1.1.15"
|
||||
|
|
@ -2060,18 +2007,6 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libz-sys"
|
||||
version = "1.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linked-hash-map"
|
||||
version = "0.5.6"
|
||||
|
|
@ -2368,28 +2303,6 @@ version = "0.1.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
|
||||
|
||||
[[package]]
|
||||
name = "openssl-src"
|
||||
version = "300.2.3+3.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.102"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"openssl-src",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "option-ext"
|
||||
version = "0.2.0"
|
||||
|
|
@ -2866,7 +2779,6 @@ version = "0.0.1"
|
|||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"git2",
|
||||
"indexmap",
|
||||
"mailparse",
|
||||
"once_cell",
|
||||
|
|
@ -3573,17 +3485,6 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.8"
|
||||
|
|
@ -4832,17 +4733,10 @@ name = "uv-git"
|
|||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
"cache-key",
|
||||
"cargo-util",
|
||||
"fs-err",
|
||||
"git2",
|
||||
"glob",
|
||||
"hmac",
|
||||
"home",
|
||||
"rand",
|
||||
"reqwest",
|
||||
"sha1",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
|
|
@ -5120,12 +5014,6 @@ version = "0.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.4"
|
||||
|
|
|
|||
|
|
@ -80,11 +80,8 @@ flate2 = { version = "1.0.28", default-features = false }
|
|||
fs-err = { version = "2.11.0" }
|
||||
fs2 = { version = "0.4.3" }
|
||||
futures = { version = "0.3.30" }
|
||||
git2 = { version = "0.18.1" }
|
||||
glob = { version = "0.3.1" }
|
||||
hex = { version = "0.4.3" }
|
||||
hmac = { version = "0.12.1" }
|
||||
home = { version = "0.5.9" }
|
||||
html-escape = { version = "0.2.13" }
|
||||
http = { version = "1.1.0" }
|
||||
indexmap = { version = "2.2.5" }
|
||||
|
|
@ -106,7 +103,6 @@ platform-info = { version = "2.0.2" }
|
|||
pubgrub = { git = "https://github.com/astral-sh/pubgrub", rev = "0e684a874c9fb8f74738cd8875524c80e3d4820b" }
|
||||
pyo3 = { version = "0.21.0" }
|
||||
pyo3-log = { version = "0.10.0" }
|
||||
rand = { version = "0.8.5" }
|
||||
rayon = { version = "1.8.0" }
|
||||
reflink-copy = { version = "0.1.15" }
|
||||
regex = { version = "1.10.2" }
|
||||
|
|
@ -122,7 +118,6 @@ schemars = { version = "0.8.16", features = ["url"] }
|
|||
seahash = { version = "4.1.0" }
|
||||
serde = { version = "1.0.197", features = ["derive"] }
|
||||
serde_json = { version = "1.0.114" }
|
||||
sha1 = { version = "0.10.6" }
|
||||
sha2 = { version = "0.10.8" }
|
||||
sys-info = { version = "0.9.1" }
|
||||
tempfile = { version = "3.9.0" }
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ pep508_rs = { workspace = true }
|
|||
platform-tags = { workspace = true }
|
||||
pypi-types = { workspace = true }
|
||||
uv-fs = { workspace = true }
|
||||
uv-git = { workspace = true, features = ["vendored-openssl"] }
|
||||
uv-git = { workspace = true }
|
||||
uv-normalize = { workspace = true }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ uv-normalize = { workspace = true }
|
|||
uv-git = { workspace = true }
|
||||
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
git2 = { workspace = true }
|
||||
indexmap = { workspace = true, features = ["serde"] }
|
||||
mailparse = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use thiserror::Error;
|
|||
use url::{ParseError, Url};
|
||||
|
||||
use pep508_rs::{Pep508Url, UnnamedRequirementUrl, VerbatimUrl, VerbatimUrlError};
|
||||
use uv_git::GitUrl;
|
||||
use uv_git::{GitUrl, OidParseError};
|
||||
|
||||
use crate::{ArchiveInfo, DirInfo, DirectUrl, VcsInfo, VcsKind};
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ pub enum ParsedUrlError {
|
|||
#[error("Invalid path in file URL: `{0}`")]
|
||||
InvalidFileUrl(Url),
|
||||
#[error("Failed to parse Git reference from URL: `{0}`")]
|
||||
GitShaParse(Url, #[source] git2::Error),
|
||||
GitShaParse(Url, #[source] OidParseError),
|
||||
#[error("Not a valid URL: `{0}`")]
|
||||
UrlParse(String, #[source] ParseError),
|
||||
#[error(transparent)]
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ uv-cache = { workspace = true }
|
|||
uv-client = { workspace = true }
|
||||
uv-extract = { workspace = true }
|
||||
uv-fs = { workspace = true, features = ["tokio"] }
|
||||
uv-git = { workspace = true, features = ["vendored-openssl"] }
|
||||
uv-git = { workspace = true }
|
||||
uv-normalize = { workspace = true }
|
||||
uv-types = { workspace = true }
|
||||
uv-configuration = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -17,20 +17,9 @@ cache-key = { workspace = true }
|
|||
uv-fs = { workspace = true }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
cargo-util = { workspace = true }
|
||||
git2 = { workspace = true }
|
||||
glob = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
home = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
reqwest = { workspace = true, features = ["blocking"] }
|
||||
sha1 = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
url = { workspace = true }
|
||||
fs-err = { workspace = true }
|
||||
|
||||
[features]
|
||||
vendored-libgit2 = ["git2/vendored-libgit2"]
|
||||
vendored-openssl = ["git2/vendored-openssl"]
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,872 +0,0 @@
|
|||
//! Git support is derived from Cargo's implementation.
|
||||
//! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice.
|
||||
//! Source: <https://github.com/rust-lang/cargo/blob/23eb492cf920ce051abfc56bbaf838514dc8365c/src/cargo/sources/git/known_hosts.rs>
|
||||
//!
|
||||
//! SSH host key validation support.
|
||||
//!
|
||||
//! The only public item in this module is [`certificate_check`],
|
||||
//! which provides a callback to [`git2::RemoteCallbacks::certificate_check`].
|
||||
//!
|
||||
//! A primary goal with this implementation is to provide user-friendly error
|
||||
//! messages, guiding them to understand the issue and how to resolve it.
|
||||
//!
|
||||
//! Note that there are a lot of limitations here. This reads OpenSSH
|
||||
//! `known_hosts` files from well-known locations, but it does not read OpenSSH
|
||||
//! config files. The config file can change the behavior of how OpenSSH
|
||||
//! handles `known_hosts` files. For example, some things we don't handle:
|
||||
//!
|
||||
//! - `GlobalKnownHostsFile` — Changes the location of the global host file.
|
||||
//! - `UserKnownHostsFile` — Changes the location of the user's host file.
|
||||
//! - `KnownHostsCommand` — A command to fetch known hosts.
|
||||
//! - `CheckHostIP` — DNS spoofing checks.
|
||||
//! - `VisualHostKey` — Shows a visual ascii-art key.
|
||||
//! - `VerifyHostKeyDNS` — Uses SSHFP DNS records to fetch a host key.
|
||||
//!
|
||||
//! There's also a number of things that aren't supported but could be easily
|
||||
//! added (it just adds a little complexity). For example, hostname patterns,
|
||||
//! and revoked markers. See "FIXME" comments littered in this file.
|
||||
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use base64::engine::general_purpose::STANDARD_NO_PAD;
|
||||
use base64::Engine as _;
|
||||
use git2::cert::{Cert, SshHostKeyType};
|
||||
use git2::CertificateCheckStatus;
|
||||
use hmac::Mac;
|
||||
use std::collections::HashSet;
|
||||
use std::fmt::{Display, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// These are host keys that are hard-coded in cargo to provide convenience.
|
||||
///
|
||||
/// If GitHub ever publishes new keys, the user can add them to their own
|
||||
/// configuration file to use those instead.
|
||||
///
|
||||
/// The GitHub keys are sourced from <https://api.github.com/meta> or
|
||||
/// <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints>.
|
||||
///
|
||||
/// These will be ignored if the user adds their own entries for `github.com`,
|
||||
/// which can be useful if GitHub ever revokes their old keys.
|
||||
static BUNDLED_KEYS: &[(&str, &str, &str)] = &[
|
||||
("github.com", "ssh-ed25519", "AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"),
|
||||
("github.com", "ecdsa-sha2-nistp256", "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg="),
|
||||
("github.com", "ssh-rsa", "AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk="),
|
||||
];
|
||||
|
||||
/// List of keys that public hosts have rotated away from.
|
||||
///
|
||||
/// We explicitly distrust these keys as users with the old key in their
|
||||
/// local configuration will otherwise be vulnerable to MITM attacks if the
|
||||
/// attacker has access to the old key. As there is no other way to distribute
|
||||
/// revocations of ssh host keys, we need to bundle them with the client.
|
||||
///
|
||||
/// Unlike [`BUNDLED_KEYS`], these revocations will not be ignored if the user
|
||||
/// has their own entries: we *know* that these keys are bad.
|
||||
static BUNDLED_REVOCATIONS: &[(&str, &str, &str)] = &[
|
||||
// Used until March 24, 2023: https://github.blog/2023-03-23-we-updated-our-rsa-ssh-host-key/
|
||||
("github.com", "ssh-rsa", "AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="),
|
||||
];
|
||||
|
||||
enum KnownHostError {
|
||||
/// Some general error happened while validating the known hosts.
|
||||
CheckError(anyhow::Error),
|
||||
/// The host key was not found.
|
||||
HostKeyNotFound {
|
||||
hostname: String,
|
||||
key_type: SshHostKeyType,
|
||||
remote_host_key: String,
|
||||
remote_fingerprint: String,
|
||||
other_hosts: Vec<KnownHost>,
|
||||
},
|
||||
/// The host key was found, but does not match the remote's key.
|
||||
HostKeyHasChanged {
|
||||
hostname: String,
|
||||
key_type: SshHostKeyType,
|
||||
old_known_host: KnownHost,
|
||||
remote_host_key: String,
|
||||
remote_fingerprint: String,
|
||||
},
|
||||
/// The host key was found with a @revoked marker, it must not be accepted.
|
||||
HostKeyRevoked {
|
||||
hostname: String,
|
||||
key_type: SshHostKeyType,
|
||||
remote_host_key: String,
|
||||
location: KnownHostLocation,
|
||||
},
|
||||
/// The host key was not found, but there was a matching known host with a
|
||||
/// @cert-authority marker (which Cargo doesn't yet support).
|
||||
HostHasOnlyCertAuthority {
|
||||
hostname: String,
|
||||
location: KnownHostLocation,
|
||||
},
|
||||
}
|
||||
|
||||
impl From<anyhow::Error> for KnownHostError {
|
||||
fn from(err: anyhow::Error) -> Self {
|
||||
Self::CheckError(err)
|
||||
}
|
||||
}
|
||||
|
||||
/// The location where a host key was located.
|
||||
#[derive(Clone)]
|
||||
enum KnownHostLocation {
|
||||
/// Loaded from a file from disk.
|
||||
File { path: PathBuf, lineno: usize },
|
||||
/// Part of the hard-coded bundled keys in Cargo.
|
||||
Bundled,
|
||||
}
|
||||
|
||||
impl Display for KnownHostLocation {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let loc = match self {
|
||||
Self::File { path, lineno } => {
|
||||
format!("{} line {lineno}", path.display())
|
||||
}
|
||||
Self::Bundled => "bundled with cargo".to_string(),
|
||||
};
|
||||
f.write_str(&loc)
|
||||
}
|
||||
}
|
||||
|
||||
/// The git2 callback used to validate a certificate (only ssh known hosts are validated).
|
||||
pub(crate) fn certificate_check(
|
||||
cert: &Cert<'_>,
|
||||
host: &str,
|
||||
port: Option<u16>,
|
||||
) -> Result<CertificateCheckStatus, git2::Error> {
|
||||
let Some(host_key) = cert.as_hostkey() else {
|
||||
// Return passthrough for TLS X509 certificates to use whatever validation
|
||||
// was done in git2.
|
||||
return Ok(CertificateCheckStatus::CertificatePassthrough);
|
||||
};
|
||||
// If a nonstandard port is in use, check for that first.
|
||||
// The fallback to check without a port is handled in the HostKeyNotFound handler.
|
||||
let host_maybe_port = match port {
|
||||
Some(port) if port != 22 => format!("[{host}]:{port}"),
|
||||
_ => host.to_string(),
|
||||
};
|
||||
// The error message must be constructed as a string to pass through the libgit2 C API.
|
||||
let err_msg = match check_ssh_known_hosts(host_key, &host_maybe_port) {
|
||||
Ok(()) => {
|
||||
return Ok(CertificateCheckStatus::CertificateOk);
|
||||
}
|
||||
Err(KnownHostError::CheckError(e)) => {
|
||||
format!("error: failed to validate host key:\n{e:#}")
|
||||
}
|
||||
Err(KnownHostError::HostKeyNotFound {
|
||||
hostname,
|
||||
key_type,
|
||||
remote_host_key,
|
||||
remote_fingerprint,
|
||||
other_hosts,
|
||||
}) => {
|
||||
// Try checking without the port.
|
||||
if port.is_some()
|
||||
&& !matches!(port, Some(22))
|
||||
&& check_ssh_known_hosts(host_key, host).is_ok()
|
||||
{
|
||||
return Ok(CertificateCheckStatus::CertificateOk);
|
||||
}
|
||||
let key_type_short_name = key_type.short_name();
|
||||
let key_type_name = key_type.name();
|
||||
let other_hosts_message = if other_hosts.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
let mut msg = String::from(
|
||||
"Note: This host key was found, \
|
||||
but is associated with a different host:\n",
|
||||
);
|
||||
for known_host in other_hosts {
|
||||
writeln!(
|
||||
msg,
|
||||
" {loc}: {patterns}",
|
||||
loc = known_host.location,
|
||||
patterns = known_host.patterns
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
msg
|
||||
};
|
||||
format!("error: unknown SSH host key\n\
|
||||
The SSH host key for `{hostname}` is not known and cannot be validated.\n\
|
||||
\n\
|
||||
To resolve this issue, add the host key to the list of known hosts.\n\
|
||||
\n\
|
||||
The key to add is:\n\
|
||||
\n\
|
||||
{hostname} {key_type_name} {remote_host_key}\n\
|
||||
\n\
|
||||
The {key_type_short_name} key fingerprint is: SHA256:{remote_fingerprint}\n\
|
||||
This fingerprint should be validated with the server administrator that it is correct.\n\
|
||||
{other_hosts_message}\n\
|
||||
See https://doc.rust-lang.org/stable/cargo/appendix/git-authentication.html#ssh-known-hosts \
|
||||
for more information.\n\
|
||||
")
|
||||
}
|
||||
Err(KnownHostError::HostKeyHasChanged {
|
||||
hostname,
|
||||
key_type,
|
||||
old_known_host,
|
||||
remote_host_key,
|
||||
remote_fingerprint,
|
||||
}) => {
|
||||
let key_type_short_name = key_type.short_name();
|
||||
let key_type_name = key_type.name();
|
||||
let old_key_resolution = match old_known_host.location {
|
||||
KnownHostLocation::File { path, lineno } => {
|
||||
let old_key_location = path.display();
|
||||
format!(
|
||||
"removing the old {key_type_name} key for `{hostname}` \
|
||||
located at {old_key_location} line {lineno}, \
|
||||
and adding the new key to the list of known hosts.",
|
||||
)
|
||||
}
|
||||
KnownHostLocation::Bundled => "adding the new key to the list of known hosts.\n\
|
||||
The current host key is bundled as part of Cargo."
|
||||
.to_string(),
|
||||
};
|
||||
format!("error: SSH host key has changed for `{hostname}`\n\
|
||||
*********************************\n\
|
||||
* WARNING: HOST KEY HAS CHANGED *\n\
|
||||
*********************************\n\
|
||||
This may be caused by a man-in-the-middle attack, or the \
|
||||
server may have changed its host key.\n\
|
||||
\n\
|
||||
The {key_type_short_name} fingerprint for the key from the remote host is:\n\
|
||||
SHA256:{remote_fingerprint}\n\
|
||||
\n\
|
||||
You are strongly encouraged to contact the server \
|
||||
administrator for `{hostname}` to verify that this new key is \
|
||||
correct.\n\
|
||||
\n\
|
||||
If you can verify that the server has a new key, you can \
|
||||
resolve this error by {old_key_resolution}\n\
|
||||
\n\
|
||||
The key provided by the remote host is:\n\
|
||||
\n\
|
||||
{hostname} {key_type_name} {remote_host_key}\n\
|
||||
\n\
|
||||
See https://doc.rust-lang.org/stable/cargo/appendix/git-authentication.html#ssh-known-hosts \
|
||||
for more information.\n\
|
||||
")
|
||||
}
|
||||
Err(KnownHostError::HostKeyRevoked {
|
||||
hostname,
|
||||
key_type,
|
||||
remote_host_key,
|
||||
location,
|
||||
}) => {
|
||||
let key_type_short_name = key_type.short_name();
|
||||
format!(
|
||||
"error: Key has been revoked for `{hostname}`\n\
|
||||
**************************************\n\
|
||||
* WARNING: REVOKED HOST KEY DETECTED *\n\
|
||||
**************************************\n\
|
||||
This may indicate that the key provided by this host has been\n\
|
||||
compromised and should not be accepted.
|
||||
\n\
|
||||
The host key {key_type_short_name} {remote_host_key} is revoked\n\
|
||||
in {location} and has been rejected.\n\
|
||||
"
|
||||
)
|
||||
}
|
||||
Err(KnownHostError::HostHasOnlyCertAuthority { hostname, location }) => {
|
||||
format!("error: Found a `@cert-authority` marker for `{hostname}`\n\
|
||||
\n\
|
||||
Cargo doesn't support certificate authorities for host key verification. It is\n\
|
||||
recommended that the command line Git client is used instead. This can be achieved\n\
|
||||
by setting `net.git-fetch-with-cli` to `true` in the Cargo config.\n\
|
||||
\n
|
||||
The `@cert-authority` line was found in {location}.\n\
|
||||
\n\
|
||||
See https://doc.rust-lang.org/stable/cargo/appendix/git-authentication.html#ssh-known-hosts \
|
||||
for more information.\n\
|
||||
")
|
||||
}
|
||||
};
|
||||
Err(git2::Error::new(
|
||||
git2::ErrorCode::GenericError,
|
||||
git2::ErrorClass::Callback,
|
||||
err_msg,
|
||||
))
|
||||
}
|
||||
|
||||
/// Checks if the given host/host key pair is known.
|
||||
#[allow(clippy::result_large_err)]
|
||||
fn check_ssh_known_hosts(
|
||||
cert_host_key: &git2::cert::CertHostkey<'_>,
|
||||
host: &str,
|
||||
) -> Result<(), KnownHostError> {
|
||||
let Some(remote_host_key) = cert_host_key.hostkey() else {
|
||||
return Err(anyhow::format_err!("remote host key is not available").into());
|
||||
};
|
||||
let remote_key_type = cert_host_key.hostkey_type().unwrap();
|
||||
|
||||
// Collect all the known host entries from disk.
|
||||
let mut known_hosts = Vec::new();
|
||||
for path in known_host_files() {
|
||||
if !path.exists() {
|
||||
continue;
|
||||
}
|
||||
let hosts = load_hostfile(&path)?;
|
||||
known_hosts.extend(hosts);
|
||||
}
|
||||
// Load the bundled keys. Don't add keys for hosts that the user has
|
||||
// configured, which gives them the option to override them. This could be
|
||||
// useful if the keys are ever revoked.
|
||||
let configured_hosts: HashSet<_> = known_hosts
|
||||
.iter()
|
||||
.flat_map(|known_host| known_host.patterns.split(',').map(str::to_lowercase))
|
||||
.collect();
|
||||
for (patterns, key_type, key) in BUNDLED_KEYS {
|
||||
if !configured_hosts.contains(*patterns) {
|
||||
let key = STANDARD.decode(key).unwrap();
|
||||
known_hosts.push(KnownHost {
|
||||
location: KnownHostLocation::Bundled,
|
||||
patterns: (*patterns).to_string(),
|
||||
key_type: (*key_type).to_string(),
|
||||
key,
|
||||
line_type: KnownHostLineType::Key,
|
||||
});
|
||||
}
|
||||
}
|
||||
for (patterns, key_type, key) in BUNDLED_REVOCATIONS {
|
||||
let key = STANDARD.decode(key).unwrap();
|
||||
known_hosts.push(KnownHost {
|
||||
location: KnownHostLocation::Bundled,
|
||||
patterns: (*patterns).to_string(),
|
||||
key_type: (*key_type).to_string(),
|
||||
key,
|
||||
line_type: KnownHostLineType::Revoked,
|
||||
});
|
||||
}
|
||||
check_ssh_known_hosts_loaded(&known_hosts, host, remote_key_type, remote_host_key)
|
||||
}
|
||||
|
||||
/// Checks a host key against a loaded set of known hosts.
|
||||
#[allow(clippy::result_large_err)]
|
||||
fn check_ssh_known_hosts_loaded(
|
||||
known_hosts: &[KnownHost],
|
||||
host: &str,
|
||||
remote_key_type: SshHostKeyType,
|
||||
remote_host_key: &[u8],
|
||||
) -> Result<(), KnownHostError> {
|
||||
// `latent_error` keeps track of a potential error that will be returned
|
||||
// in case a matching host key isn't found.
|
||||
let mut latent_errors: Vec<KnownHostError> = Vec::new();
|
||||
|
||||
// `other_hosts` keeps track of any entries that have an identical key,
|
||||
// but a different hostname.
|
||||
let mut other_hosts = Vec::new();
|
||||
|
||||
// `accepted_known_host_found` keeps track of whether we've found a matching
|
||||
// line in the `known_hosts` file that we would accept. We can't return that
|
||||
// immediately, because there may be a subsequent @revoked key.
|
||||
let mut accepted_known_host_found = false;
|
||||
|
||||
// Older versions of OpenSSH (before 6.8, March 2015) showed MD5
|
||||
// fingerprints (see FingerprintHash ssh config option). Here we only
|
||||
// support SHA256.
|
||||
let mut remote_fingerprint = cargo_util::Sha256::new();
|
||||
remote_fingerprint.update(remote_host_key);
|
||||
let remote_fingerprint = STANDARD_NO_PAD.encode(remote_fingerprint.finish());
|
||||
let remote_host_key_encoded = STANDARD.encode(remote_host_key);
|
||||
|
||||
for known_host in known_hosts {
|
||||
// The key type from libgit2 needs to match the key type from the host file.
|
||||
if known_host.key_type != remote_key_type.name() {
|
||||
continue;
|
||||
}
|
||||
let key_matches = known_host.key == remote_host_key;
|
||||
if !known_host.host_matches(host) {
|
||||
if key_matches {
|
||||
other_hosts.push(known_host.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match known_host.line_type {
|
||||
KnownHostLineType::Key => {
|
||||
if key_matches {
|
||||
accepted_known_host_found = true;
|
||||
} else {
|
||||
// The host and key type matched, but the key itself did not.
|
||||
// This indicates the key has changed.
|
||||
// This is only reported as an error if no subsequent lines have a
|
||||
// correct key.
|
||||
latent_errors.push(KnownHostError::HostKeyHasChanged {
|
||||
hostname: host.to_string(),
|
||||
key_type: remote_key_type,
|
||||
old_known_host: known_host.clone(),
|
||||
remote_host_key: remote_host_key_encoded.clone(),
|
||||
remote_fingerprint: remote_fingerprint.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
KnownHostLineType::Revoked => {
|
||||
if key_matches {
|
||||
return Err(KnownHostError::HostKeyRevoked {
|
||||
hostname: host.to_string(),
|
||||
key_type: remote_key_type,
|
||||
remote_host_key: remote_host_key_encoded,
|
||||
location: known_host.location.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
KnownHostLineType::CertAuthority => {
|
||||
// The host matches a @cert-authority line, which is unsupported.
|
||||
latent_errors.push(KnownHostError::HostHasOnlyCertAuthority {
|
||||
hostname: host.to_string(),
|
||||
location: known_host.location.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We have an accepted host key and it hasn't been revoked.
|
||||
if accepted_known_host_found {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if latent_errors.is_empty() {
|
||||
// FIXME: Ideally the error message should include the IP address of the
|
||||
// remote host (to help the user validate that they are connecting to the
|
||||
// host they were expecting to). However, I don't see a way to obtain that
|
||||
// information from libgit2.
|
||||
Err(KnownHostError::HostKeyNotFound {
|
||||
hostname: host.to_string(),
|
||||
key_type: remote_key_type,
|
||||
remote_host_key: remote_host_key_encoded,
|
||||
remote_fingerprint,
|
||||
other_hosts,
|
||||
})
|
||||
} else {
|
||||
// We're going to take the first HostKeyHasChanged error if
|
||||
// we find one, otherwise we'll take the first error (which
|
||||
// we expect to be a CertAuthority error).
|
||||
if let Some(index) = latent_errors
|
||||
.iter()
|
||||
.position(|e| matches!(e, KnownHostError::HostKeyHasChanged { .. }))
|
||||
{
|
||||
Err(latent_errors.remove(index))
|
||||
} else {
|
||||
// Otherwise, we take the first error (which we expect to be
|
||||
// a CertAuthority error).
|
||||
Err(latent_errors.pop().unwrap())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a list of files to try loading OpenSSH-formatted known hosts.
|
||||
fn known_host_files() -> Vec<PathBuf> {
|
||||
let mut result = Vec::new();
|
||||
if cfg!(unix) {
|
||||
result.push(PathBuf::from("/etc/ssh/ssh_known_hosts"));
|
||||
} else if cfg!(windows) {
|
||||
// The msys/cygwin version of OpenSSH uses `/etc` from the posix root
|
||||
// filesystem there (such as `C:\msys64\etc\ssh\ssh_known_hosts`).
|
||||
// However, I do not know of a way to obtain that location from
|
||||
// Windows-land. The ProgramData version here is what the PowerShell
|
||||
// port of OpenSSH does.
|
||||
if let Some(progdata) = std::env::var_os("ProgramData") {
|
||||
let mut progdata = PathBuf::from(progdata);
|
||||
progdata.push("ssh");
|
||||
progdata.push("ssh_known_hosts");
|
||||
result.push(progdata);
|
||||
}
|
||||
}
|
||||
result.extend(user_known_host_location());
|
||||
result
|
||||
}
|
||||
|
||||
/// The location of the user's `known_hosts` file.
|
||||
fn user_known_host_location() -> Option<PathBuf> {
|
||||
// NOTE: This is a potentially inaccurate prediction of what the user
|
||||
// actually wants. The actual location depends on several factors:
|
||||
//
|
||||
// - Windows OpenSSH Powershell version: I believe this looks up the home
|
||||
// directory via ProfileImagePath in the registry, falling back to
|
||||
// `GetWindowsDirectoryW` if that fails.
|
||||
// - OpenSSH Portable (under msys): This is very complicated. I got lost
|
||||
// after following it through some ldap/active directory stuff.
|
||||
// - OpenSSH (most unix platforms): Uses `pw->pw_dir` from `getpwuid()`.
|
||||
//
|
||||
// This doesn't do anything close to that. home_dir's behavior is:
|
||||
// - Windows: $USERPROFILE, or SHGetKnownFolderPath()
|
||||
// - Unix: $HOME, or getpwuid_r()
|
||||
//
|
||||
// Since there is a mismatch here, the location returned here might be
|
||||
// different than what the user's `ssh` CLI command uses. We may want to
|
||||
// consider trying to align it better.
|
||||
home::home_dir().map(|mut home| {
|
||||
home.push(".ssh");
|
||||
home.push("known_hosts");
|
||||
home
|
||||
})
|
||||
}
|
||||
|
||||
const HASH_HOSTNAME_PREFIX: &str = "|1|";
|
||||
|
||||
#[derive(Clone)]
|
||||
enum KnownHostLineType {
|
||||
Key,
|
||||
CertAuthority,
|
||||
Revoked,
|
||||
}
|
||||
|
||||
/// A single known host entry.
|
||||
#[derive(Clone)]
|
||||
struct KnownHost {
|
||||
location: KnownHostLocation,
|
||||
/// The hostname. May be comma separated to match multiple hosts.
|
||||
patterns: String,
|
||||
key_type: String,
|
||||
key: Vec<u8>,
|
||||
line_type: KnownHostLineType,
|
||||
}
|
||||
|
||||
impl KnownHost {
|
||||
/// Returns whether or not the given host matches this known host entry.
|
||||
fn host_matches(&self, host: &str) -> bool {
|
||||
let mut match_found = false;
|
||||
let host = host.to_lowercase();
|
||||
if let Some(hashed) = self.patterns.strip_prefix(HASH_HOSTNAME_PREFIX) {
|
||||
return hashed_hostname_matches(&host, hashed);
|
||||
}
|
||||
for pattern in self.patterns.split(',') {
|
||||
let pattern = pattern.to_lowercase();
|
||||
// FIXME: support * and ? wildcards
|
||||
if let Some(pattern) = pattern.strip_prefix('!') {
|
||||
if pattern == host {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
match_found |= pattern == host;
|
||||
}
|
||||
}
|
||||
match_found
|
||||
}
|
||||
}
|
||||
|
||||
fn hashed_hostname_matches(host: &str, hashed: &str) -> bool {
|
||||
let Some((b64_salt, b64_host)) = hashed.split_once('|') else {
|
||||
return false;
|
||||
};
|
||||
let Ok(salt) = STANDARD.decode(b64_salt) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(hashed_host) = STANDARD.decode(b64_host) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(mut mac) = hmac::Hmac::<sha1::Sha1>::new_from_slice(&salt) else {
|
||||
return false;
|
||||
};
|
||||
mac.update(host.as_bytes());
|
||||
let result = mac.finalize().into_bytes();
|
||||
hashed_host == result[..]
|
||||
}
|
||||
|
||||
/// Loads an OpenSSH `known_hosts` file.
|
||||
fn load_hostfile(path: &Path) -> Result<Vec<KnownHost>, anyhow::Error> {
|
||||
let contents = cargo_util::paths::read(path)?;
|
||||
Ok(load_hostfile_contents(path, &contents))
|
||||
}
|
||||
|
||||
fn load_hostfile_contents(path: &Path, contents: &str) -> Vec<KnownHost> {
|
||||
let entries = contents
|
||||
.lines()
|
||||
.enumerate()
|
||||
.filter_map(|(lineno, line)| {
|
||||
let location = KnownHostLocation::File {
|
||||
path: path.to_path_buf(),
|
||||
lineno: lineno + 1,
|
||||
};
|
||||
parse_known_hosts_line(line, location)
|
||||
})
|
||||
.collect();
|
||||
entries
|
||||
}
|
||||
|
||||
fn parse_known_hosts_line(line: &str, location: KnownHostLocation) -> Option<KnownHost> {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
return None;
|
||||
}
|
||||
let mut parts = line.split([' ', '\t']).filter(|s| !s.is_empty());
|
||||
|
||||
let line_type = if line.starts_with('@') {
|
||||
let line_type = parts.next()?;
|
||||
|
||||
if line_type == "@cert-authority" {
|
||||
KnownHostLineType::CertAuthority
|
||||
} else if line_type == "@revoked" {
|
||||
KnownHostLineType::Revoked
|
||||
} else {
|
||||
// No other markers are defined
|
||||
return None;
|
||||
}
|
||||
} else {
|
||||
KnownHostLineType::Key
|
||||
};
|
||||
|
||||
let patterns = parts.next()?;
|
||||
let key_type = parts.next()?;
|
||||
let key = parts.next().map(|p| STANDARD.decode(p))?.ok()?;
|
||||
Some(KnownHost {
|
||||
line_type,
|
||||
location,
|
||||
patterns: patterns.to_string(),
|
||||
key_type: key_type.to_string(),
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
static COMMON_CONTENTS: &str = r"
|
||||
# Comments allowed at start of line
|
||||
|
||||
example.com,rust-lang.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5MzWIpZwpkpDjyCNiTIEVFhSA9OUUQvjFo7CgZBGCAj/cqeUIgiLsgtfmtBsfWIkAECQpM7ePP7NLZFGJcHvoyg5jXJiIX5s0eKo9IlcuTLLrMkW5MkHXE7bNklVbW1WdCfF2+y7Ao25B4L8FFRokMh0yp/H6+8xZ7PdVwL3FRPEg8ftZ5R0kuups6xiMHPRX+f/07vfJzA47YDPmXfhkn+JK8kL0JYw8iy8BtNBfRQL99d9iXJzWXnNce5NHMuKD5rOonD3aQHLDlwK+KhrFRrdaxQEM8ZWxNti0ux8yT4Dl5jJY0CrIu3Xl6+qroVgTqJGNkTbhs5DGWdFh6BLPTTH15rN4buisg7uMyLyHqx06ckborqD33gWu+Jig7O+PV6KJmL5mp1O1HXvZqkpBdTiT6GiDKG3oECCIXkUk0BSU9VG9VQcrMxxvgiHlyoXUAfYQoXv/lnxkTnm+Sr36kutsVOs7n5B43ZKAeuaxyQ11huJZpxamc0RA1HM641s= eric@host
|
||||
Example.net ssh-dss AAAAB3NzaC1kc3MAAACBAK2Ek3jVxisXmz5UcZ7W65BAj/nDJCCVvSe0Aytndn4PH6k7sVesut5OoY6PdksZ9tEfuFjjS9HR5SJb8j1GW0GxtaSHHbf+rNc36PeU75bffzyIWwpA8uZFONt5swUAXJXcsHOoapNbUFuhHsRhB2hXxz9QGNiiwIwRJeSHixKRAAAAFQChKfxO1z9H2/757697xP5nJ/Z5dwAAAIEAoc+HIWas+4WowtB/KtAp6XE0B9oHI+55wKtdcGwwb7zHKK9scWNXwxIcMhSvyB3Oe2I7dQQlvyIWxsdZlzOkX0wdsTHjIAnBAP68MyvMv4kq3+I5GAVcFsqoLZfZvh0dlcgUq1/YNYZwKlt89tnzk8Fp4KLWmuw8Bd8IShYVa78AAACAL3qd8kNTY7CthgsQ8iWdjbkGSF/1KCeFyt8UjurInp9wvPDjqagwakbyLOzN7y3/ItTPCaGuX+RjFP0zZTf8i9bsAVyjFJiJ7vzRXcWytuFWANrpzLTn1qzPfh63iK92Aw8AVBYvEA/4bxo+XReAvhNBB/m78G6OedTeu6ZoTsI= eric@host
|
||||
[example.net]:2222 ssh-dss AAAAB3NzaC1kc3MAAACBAJJN5kLZEpOJpXWyMT4KwYvLAj+b9ErNtglxOi86C6Kw7oZeYdDMCfD3lc3PJyX64udQcWGfO4abSESMiYdY43yFAZH279QGH5Q/B5CklVvTqYpfAUR+1r9TQxy3OVQHk7FB2wOi4xNQ3myO0vaYlBOB9il+P223aERbXx4JTWdvAAAAFQCTHWTcXxLK5Z6ZVPmfdSDyHzkF2wAAAIEAhp41/mTnM0Y0EWSyCXuETMW1QSpKGF8sqoZKp6wdzyhLXu0i32gLdXj4p24em/jObYh93hr+MwgxqWq+FHgD+D80Qg5f6vj4yEl4Uu5hqtTpCBFWUQoyEckbUkPf8uZ4/XzAne+tUSjZm09xATCmK9U2IGqZE+D+90eBkf1Svc8AAACAeKhi4EtfwenFYqKz60ZoEEhIsE1yI2jH73akHnfHpcW84w+fk3YlwjcfDfyYso+D0jZBdJeK5qIdkbUWhAX8wDjJVO0WL6r/YPr4yu/CgEyW1H59tAbujGJ4NR0JDqioulzYqNHnxpiw1RJukZnPBfSFKzRElvPOCq/NkQM/Mwk= eric@host
|
||||
nistp256.example.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ4iYGCcJrUIfrHfzlsv8e8kaF36qpcUpe3VNAKVCZX/BDptIdlEe8u8vKNRTPgUO9jqS0+tjTcPiQd8/8I9qng= eric@host
|
||||
nistp384.example.org ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBNuGT3TqMz2rcwOt2ZqkiNqq7dvWPE66W2qPCoZsh0pQhVU3BnhKIc6nEr6+Wts0Z3jdF3QWwxbbTjbVTVhdr8fMCFhDCWiQFm9xLerYPKnu9qHvx9K87/fjc5+0pu4hLA== eric@host
|
||||
nistp521.example.org ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAD35HH6OsK4DN75BrKipVj/GvZaUzjPNa1F8wMjUdPB1JlVcUfgzJjWSxrhmaNN3u0soiZw8WNRFINsGPCw5E7DywF1689WcIj2Ye2rcy99je15FknScTzBBD04JgIyOI50mCUaPCBoF14vFlN6BmO00cFo+yzy5N8GuQ2sx9kr21xmFQ== eric@host
|
||||
# Revoked is supported, but without Cert-Authority support, it will only negate some other fixed key.
|
||||
@revoked revoked.example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKtQsi+KPYispwm2rkMidQf30fG1Niy8XNkvASfePoca eric@host
|
||||
# Cert-Authority is not supported (below key should not be valid anyway)
|
||||
@cert-authority ca.example.com ssh-rsa AABBB5Wm
|
||||
example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAWkjI6XT2SZh3xNk5NhisA3o3sGzWR+VAKMSqHtI0aY eric@host
|
||||
192.168.42.12 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKVYJpa0yUGaNk0NXQTPWa0tHjqRpx+7hl2diReH6DtR eric@host
|
||||
|1|QxzZoTXIWLhUsuHAXjuDMIV3FjQ=|M6NCOIkjiWdCWqkh5+Q+/uFLGjs= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIHgN3O21U4LWtP5OzjTzPnUnSDmCNDvyvlaj6Hi65JC eric@host
|
||||
# Negation isn't terribly useful without globs.
|
||||
neg.example.com,!neg.example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOXfUnaAHTlo1Qi//rNk26OcmHikmkns1Z6WW/UuuS3K eric@host
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn known_hosts_parse() {
|
||||
let kh_path = Path::new("/home/abc/.known_hosts");
|
||||
let khs = load_hostfile_contents(kh_path, COMMON_CONTENTS);
|
||||
assert_eq!(khs.len(), 12);
|
||||
match &khs[0].location {
|
||||
KnownHostLocation::File { path, lineno } => {
|
||||
assert_eq!(path, kh_path);
|
||||
assert_eq!(*lineno, 4);
|
||||
}
|
||||
KnownHostLocation::Bundled => panic!("unexpected"),
|
||||
}
|
||||
assert_eq!(khs[0].patterns, "example.com,rust-lang.org");
|
||||
assert_eq!(khs[0].key_type, "ssh-rsa");
|
||||
assert_eq!(khs[0].key.len(), 407);
|
||||
assert_eq!(&khs[0].key[..30], b"\x00\x00\x00\x07ssh-rsa\x00\x00\x00\x03\x01\x00\x01\x00\x00\x01\x81\x00\xb935\x88\xa5\x9c)");
|
||||
match &khs[1].location {
|
||||
KnownHostLocation::File { path, lineno } => {
|
||||
assert_eq!(path, kh_path);
|
||||
assert_eq!(*lineno, 5);
|
||||
}
|
||||
KnownHostLocation::Bundled => panic!("unexpected"),
|
||||
}
|
||||
assert_eq!(khs[2].patterns, "[example.net]:2222");
|
||||
assert_eq!(khs[3].patterns, "nistp256.example.org");
|
||||
assert_eq!(khs[9].patterns, "192.168.42.12");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_matches() {
|
||||
let kh_path = Path::new("/home/abc/.known_hosts");
|
||||
let khs = load_hostfile_contents(kh_path, COMMON_CONTENTS);
|
||||
assert!(khs[0].host_matches("example.com"));
|
||||
assert!(khs[0].host_matches("rust-lang.org"));
|
||||
assert!(khs[0].host_matches("EXAMPLE.COM"));
|
||||
assert!(khs[1].host_matches("example.net"));
|
||||
assert!(!khs[0].host_matches("example.net"));
|
||||
assert!(khs[2].host_matches("[example.net]:2222"));
|
||||
assert!(!khs[2].host_matches("example.net"));
|
||||
assert!(khs[10].host_matches("hashed.example.com"));
|
||||
assert!(!khs[10].host_matches("example.com"));
|
||||
assert!(!khs[11].host_matches("neg.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_match() {
|
||||
let kh_path = Path::new("/home/abc/.known_hosts");
|
||||
let khs = load_hostfile_contents(kh_path, COMMON_CONTENTS);
|
||||
|
||||
assert!(check_ssh_known_hosts_loaded(
|
||||
&khs,
|
||||
"example.com",
|
||||
SshHostKeyType::Rsa,
|
||||
&khs[0].key
|
||||
)
|
||||
.is_ok());
|
||||
|
||||
match check_ssh_known_hosts_loaded(&khs, "example.com", SshHostKeyType::Dss, &khs[0].key) {
|
||||
Err(KnownHostError::HostKeyNotFound {
|
||||
hostname,
|
||||
remote_fingerprint,
|
||||
other_hosts,
|
||||
..
|
||||
}) => {
|
||||
assert_eq!(
|
||||
remote_fingerprint,
|
||||
"yn+pONDn0EcgdOCVptgB4RZd/wqmsVKrPnQMLtrvhw8"
|
||||
);
|
||||
assert_eq!(hostname, "example.com");
|
||||
assert_eq!(other_hosts.len(), 0);
|
||||
}
|
||||
_ => panic!("unexpected"),
|
||||
}
|
||||
|
||||
match check_ssh_known_hosts_loaded(
|
||||
&khs,
|
||||
"foo.example.com",
|
||||
SshHostKeyType::Rsa,
|
||||
&khs[0].key,
|
||||
) {
|
||||
Err(KnownHostError::HostKeyNotFound { other_hosts, .. }) => {
|
||||
assert_eq!(other_hosts.len(), 1);
|
||||
assert_eq!(other_hosts[0].patterns, "example.com,rust-lang.org");
|
||||
}
|
||||
_ => panic!("unexpected"),
|
||||
}
|
||||
|
||||
let mut modified_key = khs[0].key.clone();
|
||||
modified_key[0] = 1;
|
||||
match check_ssh_known_hosts_loaded(&khs, "example.com", SshHostKeyType::Rsa, &modified_key)
|
||||
{
|
||||
Err(KnownHostError::HostKeyHasChanged { old_known_host, .. }) => {
|
||||
assert!(matches!(
|
||||
old_known_host.location,
|
||||
KnownHostLocation::File { lineno: 4, .. }
|
||||
));
|
||||
}
|
||||
_ => panic!("unexpected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revoked() {
|
||||
let kh_path = Path::new("/home/abc/.known_hosts");
|
||||
let khs = load_hostfile_contents(kh_path, COMMON_CONTENTS);
|
||||
|
||||
match check_ssh_known_hosts_loaded(
|
||||
&khs,
|
||||
"revoked.example.com",
|
||||
SshHostKeyType::Ed255219,
|
||||
&khs[6].key,
|
||||
) {
|
||||
Err(KnownHostError::HostKeyRevoked {
|
||||
hostname, location, ..
|
||||
}) => {
|
||||
assert_eq!("revoked.example.com", hostname);
|
||||
assert!(matches!(
|
||||
location,
|
||||
KnownHostLocation::File { lineno: 11, .. }
|
||||
));
|
||||
}
|
||||
_ => panic!("Expected key to be revoked for revoked.example.com."),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cert_authority() {
|
||||
let kh_path = Path::new("/home/abc/.known_hosts");
|
||||
let khs = load_hostfile_contents(kh_path, COMMON_CONTENTS);
|
||||
|
||||
match check_ssh_known_hosts_loaded(
|
||||
&khs,
|
||||
"ca.example.com",
|
||||
SshHostKeyType::Rsa,
|
||||
&khs[0].key, // The key should not matter
|
||||
) {
|
||||
Err(KnownHostError::HostHasOnlyCertAuthority {
|
||||
hostname, location, ..
|
||||
}) => {
|
||||
assert_eq!("ca.example.com", hostname);
|
||||
assert!(matches!(
|
||||
location,
|
||||
KnownHostLocation::File { lineno: 13, .. }
|
||||
));
|
||||
}
|
||||
Err(KnownHostError::HostKeyNotFound { hostname, .. }) => {
|
||||
panic!("host key not found... {hostname}");
|
||||
}
|
||||
_ => panic!("Expected host to only have @cert-authority line (which is unsupported)."),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_errors() {
|
||||
let contents = r"
|
||||
not-used.example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAWkjI6XT2SZh3xNk5NhisA3o3sGzWR+VAKMSqHtI0aY eric@host
|
||||
# Cert-authority and changed key for the same host - changed key error should prevail
|
||||
@cert-authority example.com ssh-ed25519 AABBB5Wm
|
||||
example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKVYJpa0yUGaNk0NXQTPWa0tHjqRpx+7hl2diReH6DtR eric@host
|
||||
";
|
||||
|
||||
let kh_path = Path::new("/home/abc/.known_hosts");
|
||||
let khs = load_hostfile_contents(kh_path, contents);
|
||||
|
||||
match check_ssh_known_hosts_loaded(
|
||||
&khs,
|
||||
"example.com",
|
||||
SshHostKeyType::Ed255219,
|
||||
&khs[0].key,
|
||||
) {
|
||||
Err(KnownHostError::HostKeyHasChanged {
|
||||
hostname,
|
||||
old_known_host,
|
||||
remote_host_key,
|
||||
..
|
||||
}) => {
|
||||
assert_eq!("example.com", hostname);
|
||||
assert_eq!(
|
||||
"AAAAC3NzaC1lZDI1NTE5AAAAIAWkjI6XT2SZh3xNk5NhisA3o3sGzWR+VAKMSqHtI0aY",
|
||||
remote_host_key
|
||||
);
|
||||
assert!(matches!(
|
||||
old_known_host.location,
|
||||
KnownHostLocation::File { lineno: 5, .. }
|
||||
));
|
||||
}
|
||||
_ => panic!("Expected error to be of type HostKeyHasChanged."),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_host_and_revoked() {
|
||||
let contents = r"
|
||||
example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKVYJpa0yUGaNk0NXQTPWa0tHjqRpx+7hl2diReH6DtR eric@host
|
||||
# Later in the file the same host key is revoked
|
||||
@revoked example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKVYJpa0yUGaNk0NXQTPWa0tHjqRpx+7hl2diReH6DtR eric@host
|
||||
";
|
||||
|
||||
let kh_path = Path::new("/home/abc/.known_hosts");
|
||||
let khs = load_hostfile_contents(kh_path, contents);
|
||||
|
||||
match check_ssh_known_hosts_loaded(
|
||||
&khs,
|
||||
"example.com",
|
||||
SshHostKeyType::Ed255219,
|
||||
&khs[0].key,
|
||||
) {
|
||||
Err(KnownHostError::HostKeyRevoked {
|
||||
hostname,
|
||||
remote_host_key,
|
||||
location,
|
||||
..
|
||||
}) => {
|
||||
assert_eq!("example.com", hostname);
|
||||
assert_eq!(
|
||||
"AAAAC3NzaC1lZDI1NTE5AAAAIKVYJpa0yUGaNk0NXQTPWa0tHjqRpx+7hl2diReH6DtR",
|
||||
remote_host_key
|
||||
);
|
||||
assert!(matches!(
|
||||
location,
|
||||
KnownHostLocation::File { lineno: 4, .. }
|
||||
));
|
||||
}
|
||||
_ => panic!("Expected host key to be reject with error HostKeyRevoked."),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2,14 +2,12 @@ use std::str::FromStr;
|
|||
use url::Url;
|
||||
|
||||
pub use crate::git::GitReference;
|
||||
pub use crate::sha::GitSha;
|
||||
pub use crate::sha::{GitOid, GitSha, OidParseError};
|
||||
pub use crate::source::{Fetch, GitSource, Reporter};
|
||||
|
||||
mod git;
|
||||
mod known_hosts;
|
||||
mod sha;
|
||||
mod source;
|
||||
mod util;
|
||||
|
||||
/// A URL reference to a Git repository.
|
||||
#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Hash, Ord)]
|
||||
|
|
@ -55,7 +53,7 @@ impl GitUrl {
|
|||
}
|
||||
|
||||
impl TryFrom<Url> for GitUrl {
|
||||
type Error = git2::Error;
|
||||
type Error = OidParseError;
|
||||
|
||||
/// Initialize a [`GitUrl`] source from a URL.
|
||||
fn try_from(mut url: Url) -> Result<Self, Self::Error> {
|
||||
|
|
@ -121,11 +119,3 @@ impl std::fmt::Display for GitUrl {
|
|||
write!(f, "{}", self.repository)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum FetchStrategy {
|
||||
/// Fetch Git repositories using libgit2.
|
||||
Libgit2,
|
||||
/// Fetch Git repositories using the `git` CLI.
|
||||
Cli,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
use std::str::FromStr;
|
||||
use std::fmt::Display;
|
||||
use std::str::{self, FromStr};
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// A complete Git SHA, i.e., a 40-character hexadecimal representation of a Git commit.
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct GitSha(git2::Oid);
|
||||
pub struct GitSha(GitOid);
|
||||
|
||||
impl GitSha {
|
||||
/// Convert the SHA to a truncated representation, i.e., the first 16 characters of the SHA.
|
||||
|
|
@ -11,14 +14,14 @@ impl GitSha {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<GitSha> for git2::Oid {
|
||||
impl From<GitSha> for GitOid {
|
||||
fn from(value: GitSha) -> Self {
|
||||
value.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<git2::Oid> for GitSha {
|
||||
fn from(value: git2::Oid) -> Self {
|
||||
impl From<GitOid> for GitSha {
|
||||
fn from(value: GitOid) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
|
@ -30,9 +33,79 @@ impl std::fmt::Display for GitSha {
|
|||
}
|
||||
|
||||
impl FromStr for GitSha {
|
||||
type Err = git2::Error;
|
||||
type Err = OidParseError;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
Ok(Self(git2::Oid::from_str(value)?))
|
||||
Ok(Self(GitOid::from_str(value)?))
|
||||
}
|
||||
}
|
||||
|
||||
/// Unique identity of any Git object (commit, tree, blob, tag).
|
||||
///
|
||||
/// Note this type does not validate whether the input is a valid hash.
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct GitOid {
|
||||
len: usize,
|
||||
bytes: [u8; 40],
|
||||
}
|
||||
|
||||
impl GitOid {
|
||||
/// Return the string representation of an object ID.
|
||||
pub(crate) fn as_str(&self) -> &str {
|
||||
str::from_utf8(&self.bytes[..self.len]).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error, PartialEq)]
|
||||
pub enum OidParseError {
|
||||
#[error("Object ID can be at most 40 hex characters")]
|
||||
TooLong,
|
||||
#[error("Object ID cannot be parsed from empty string")]
|
||||
Empty,
|
||||
}
|
||||
|
||||
impl FromStr for GitOid {
|
||||
type Err = OidParseError;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
if s.is_empty() {
|
||||
return Err(OidParseError::Empty);
|
||||
}
|
||||
|
||||
if s.len() > 40 {
|
||||
return Err(OidParseError::TooLong);
|
||||
}
|
||||
|
||||
let mut out = [0; 40];
|
||||
out[..s.len()].copy_from_slice(s.as_bytes());
|
||||
|
||||
Ok(GitOid {
|
||||
len: s.len(),
|
||||
bytes: out,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for GitOid {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::str::FromStr;
|
||||
|
||||
use super::{GitOid, OidParseError};
|
||||
|
||||
#[test]
|
||||
fn git_oid() {
|
||||
GitOid::from_str("4a23745badf5bf5ef7928f1e346e9986bd696d82").unwrap();
|
||||
|
||||
assert_eq!(GitOid::from_str(""), Err(OidParseError::Empty));
|
||||
assert_eq!(
|
||||
GitOid::from_str(&str::repeat("a", 41)),
|
||||
Err(OidParseError::TooLong)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use url::Url;
|
|||
use cache_key::{digest, RepositoryUrl};
|
||||
|
||||
use crate::git::GitRemote;
|
||||
use crate::{FetchStrategy, GitSha, GitUrl};
|
||||
use crate::{GitOid, GitSha, GitUrl};
|
||||
|
||||
/// A remote Git source that can be checked out locally.
|
||||
pub struct GitSource {
|
||||
|
|
@ -19,8 +19,6 @@ pub struct GitSource {
|
|||
git: GitUrl,
|
||||
/// The HTTP client to use for fetching.
|
||||
client: Client,
|
||||
/// The fetch strategy to use when cloning.
|
||||
strategy: FetchStrategy,
|
||||
/// The path to the Git source database.
|
||||
cache: PathBuf,
|
||||
/// The reporter to use for this source.
|
||||
|
|
@ -33,7 +31,6 @@ impl GitSource {
|
|||
Self {
|
||||
git,
|
||||
client: Client::new(),
|
||||
strategy: FetchStrategy::Cli,
|
||||
cache: cache.into(),
|
||||
reporter: None,
|
||||
}
|
||||
|
|
@ -77,8 +74,7 @@ impl GitSource {
|
|||
&db_path,
|
||||
db,
|
||||
&self.git.reference,
|
||||
locked_rev.map(git2::Oid::from),
|
||||
self.strategy,
|
||||
locked_rev.map(GitOid::from),
|
||||
&self.client,
|
||||
)?;
|
||||
|
||||
|
|
@ -98,12 +94,8 @@ impl GitSource {
|
|||
.join("checkouts")
|
||||
.join(&ident)
|
||||
.join(short_id.as_str());
|
||||
db.copy_to(
|
||||
actual_rev.into(),
|
||||
&checkout_path,
|
||||
self.strategy,
|
||||
&self.client,
|
||||
)?;
|
||||
|
||||
db.copy_to(actual_rev.into(), &checkout_path)?;
|
||||
|
||||
// Report the checkout operation to the reporter.
|
||||
if let Some(task) = task {
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
//! Git support is derived from Cargo's implementation.
|
||||
//! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice.
|
||||
//! Source: <https://github.com/rust-lang/cargo/blob/23eb492cf920ce051abfc56bbaf838514dc8365c/src/cargo/util/errors.rs>
|
||||
use std::fmt::{self, Write};
|
||||
|
||||
use super::truncate_with_ellipsis;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct HttpNotSuccessful {
|
||||
pub(crate) code: u32,
|
||||
pub(crate) url: String,
|
||||
pub(crate) ip: Option<String>,
|
||||
pub(crate) body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl HttpNotSuccessful {
|
||||
fn render(&self) -> String {
|
||||
let mut result = String::new();
|
||||
let body = std::str::from_utf8(&self.body).map_or_else(
|
||||
|_| format!("[{} non-utf8 bytes]", self.body.len()),
|
||||
|s| truncate_with_ellipsis(s, 512),
|
||||
);
|
||||
|
||||
write!(
|
||||
result,
|
||||
"failed to get successful HTTP response from `{}`",
|
||||
self.url
|
||||
)
|
||||
.unwrap();
|
||||
if let Some(ip) = &self.ip {
|
||||
write!(result, " ({ip})").unwrap();
|
||||
}
|
||||
writeln!(result, ", got {}", self.code).unwrap();
|
||||
write!(result, "body:\n{body}").unwrap();
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for HttpNotSuccessful {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.render())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for HttpNotSuccessful {}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
//! Git support is derived from Cargo's implementation.
|
||||
//! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice.
|
||||
//! Source: <https://github.com/rust-lang/cargo/blob/23eb492cf920ce051abfc56bbaf838514dc8365c/src/cargo/util/mod.rs>
|
||||
pub(crate) mod errors;
|
||||
pub(crate) mod retry;
|
||||
|
||||
pub(crate) fn truncate_with_ellipsis(s: &str, max_width: usize) -> String {
|
||||
// We should truncate at grapheme-boundary and compute character-widths,
|
||||
// yet the dependencies on unicode-segmentation and unicode-width are
|
||||
// not worth it.
|
||||
let mut chars = s.chars();
|
||||
let mut prefix = (&mut chars).take(max_width - 1).collect::<String>();
|
||||
if chars.next().is_some() {
|
||||
prefix.push('…');
|
||||
}
|
||||
prefix
|
||||
}
|
||||
|
|
@ -1,188 +0,0 @@
|
|||
//! Utilities for retrying a network operation.
|
||||
//!
|
||||
//! Some network errors are considered "spurious", meaning it is not a real
|
||||
//! error (such as a 404 not found) and is likely a transient error (like a
|
||||
//! bad network connection) that we can hope will resolve itself shortly. The
|
||||
//! [`Retry`] type offers a way to repeatedly perform some kind of network
|
||||
//! operation with a delay if it detects one of these possibly transient
|
||||
//! errors.
|
||||
//!
|
||||
//! This supports errors from [`git2`], [`reqwest`], and [`HttpNotSuccessful`]
|
||||
//! 5xx HTTP errors.
|
||||
//!
|
||||
//! The number of retries can be configured by the user via the `net.retry`
|
||||
//! config option. This indicates the number of times to retry the operation
|
||||
//! (default 3 times for a total of 4 attempts).
|
||||
//!
|
||||
//! There are hard-coded constants that indicate how long to sleep between
|
||||
//! retries. The constants are tuned to balance a few factors, such as the
|
||||
//! responsiveness to the user (we don't want cargo to hang for too long
|
||||
//! retrying things), and accommodating things like Cloudfront's default
|
||||
//! negative TTL of 10 seconds (if Cloudfront gets a 5xx error for whatever
|
||||
//! reason it won't try to fetch again for 10 seconds).
|
||||
//!
|
||||
//! The timeout also implements a primitive form of random jitter. This is so
|
||||
//! that if multiple requests fail at the same time that they don't all flood
|
||||
//! the server at the same time when they are retried. This jitter still has
|
||||
//! some clumping behavior, but should be good enough.
|
||||
//!
|
||||
//! [`Retry`] is the core type for implementing retry logic. The
|
||||
//! [`Retry::try`] method can be called with a callback, and it will
|
||||
//! indicate if it needs to be called again sometime in the future if there
|
||||
//! was a possibly transient error. The caller is responsible for sleeping the
|
||||
//! appropriate amount of time and then calling [`Retry::try`] again.
|
||||
//!
|
||||
//! [`with_retry`] is a convenience function that will create a [`Retry`] and
|
||||
//! handle repeatedly running a callback until it succeeds, or it runs out of
|
||||
//! retries.
|
||||
//!
|
||||
//! Some interesting resources about retries:
|
||||
//! - <https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/>
|
||||
//! - <https://en.wikipedia.org/wiki/Exponential_backoff>
|
||||
//! - <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After>
|
||||
|
||||
//! Git support is derived from Cargo's implementation.
|
||||
//! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice.
|
||||
//! Source: <https://github.com/rust-lang/cargo/blob/23eb492cf920ce051abfc56bbaf838514dc8365c/src/cargo/util/network/retry.rs>
|
||||
use std::cmp::min;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Error, Result};
|
||||
use rand::Rng;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::util::errors::HttpNotSuccessful;
|
||||
|
||||
/// State for managing retrying a network operation.
|
||||
pub(crate) struct Retry {
|
||||
/// The number of failed attempts that have been done so far.
|
||||
///
|
||||
/// Starts at 0, and increases by one each time an attempt fails.
|
||||
retries: u64,
|
||||
/// The maximum number of times the operation should be retried.
|
||||
///
|
||||
/// 0 means it should never retry.
|
||||
max_retries: u64,
|
||||
}
|
||||
|
||||
/// The result of attempting some operation via [`Retry::try`].
|
||||
pub(crate) enum RetryResult<T> {
|
||||
/// The operation was successful.
|
||||
///
|
||||
/// The wrapped value is the return value of the callback function.
|
||||
Success(T),
|
||||
/// The operation was an error, and it should not be tried again.
|
||||
Err(Error),
|
||||
/// The operation failed, and should be tried again in the future.
|
||||
///
|
||||
/// The wrapped value is the number of milliseconds to wait before trying
|
||||
/// again. The caller is responsible for waiting this long and then
|
||||
/// calling [`Retry::try`] again.
|
||||
Retry(u64),
|
||||
}
|
||||
|
||||
/// Maximum amount of time a single retry can be delayed (milliseconds).
|
||||
const MAX_RETRY_SLEEP_MS: u64 = 10 * 1000;
|
||||
/// The minimum initial amount of time a retry will be delayed (milliseconds).
|
||||
///
|
||||
/// The actual amount of time will be a random value above this.
|
||||
const INITIAL_RETRY_SLEEP_BASE_MS: u64 = 500;
|
||||
/// The maximum amount of additional time the initial retry will take (milliseconds).
|
||||
///
|
||||
/// The initial delay will be [`INITIAL_RETRY_SLEEP_BASE_MS`] plus a random range
|
||||
/// from 0 to this value.
|
||||
const INITIAL_RETRY_JITTER_MS: u64 = 1000;
|
||||
|
||||
impl Retry {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
retries: 0,
|
||||
max_retries: 3,
|
||||
}
|
||||
}
|
||||
|
||||
/// Calls the given callback, and returns a [`RetryResult`] which
|
||||
/// indicates whether or not this needs to be called again at some point
|
||||
/// in the future to retry the operation if it failed.
|
||||
pub(crate) fn r#try<T>(&mut self, f: impl FnOnce() -> Result<T>) -> RetryResult<T> {
|
||||
match f() {
|
||||
Err(ref err) if maybe_spurious(err) && self.retries < self.max_retries => {
|
||||
let err_msg = err.downcast_ref::<HttpNotSuccessful>().map_or_else(
|
||||
|| err.root_cause().to_string(),
|
||||
HttpNotSuccessful::to_string,
|
||||
);
|
||||
warn!(
|
||||
"Spurious network error ({} tries remaining): {err_msg}",
|
||||
self.max_retries - self.retries,
|
||||
);
|
||||
self.retries += 1;
|
||||
RetryResult::Retry(self.next_sleep_ms())
|
||||
}
|
||||
Err(e) => RetryResult::Err(e),
|
||||
Ok(r) => RetryResult::Success(r),
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets the next sleep duration in milliseconds.
|
||||
fn next_sleep_ms(&self) -> u64 {
|
||||
if self.retries == 1 {
|
||||
let mut rng = rand::thread_rng();
|
||||
INITIAL_RETRY_SLEEP_BASE_MS + rng.gen_range(0..INITIAL_RETRY_JITTER_MS)
|
||||
} else {
|
||||
min(
|
||||
((self.retries - 1) * 3) * 1000 + INITIAL_RETRY_SLEEP_BASE_MS,
|
||||
MAX_RETRY_SLEEP_MS,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn maybe_spurious(err: &Error) -> bool {
|
||||
if let Some(git_err) = err.downcast_ref::<git2::Error>() {
|
||||
match git_err.class() {
|
||||
git2::ErrorClass::Net
|
||||
| git2::ErrorClass::Os
|
||||
| git2::ErrorClass::Zlib
|
||||
| git2::ErrorClass::Ssl
|
||||
| git2::ErrorClass::Http => return git_err.code() != git2::ErrorCode::Certificate,
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
if let Some(reqwest_err) = err.downcast_ref::<reqwest::Error>() {
|
||||
if reqwest_err.is_timeout()
|
||||
|| reqwest_err.is_connect()
|
||||
|| reqwest_err
|
||||
.status()
|
||||
.map_or(false, |status| status.is_server_error())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if let Some(not_200) = err.downcast_ref::<HttpNotSuccessful>() {
|
||||
if 500 <= not_200.code && not_200.code < 600 {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// Wrapper method for network call retry logic.
|
||||
///
|
||||
/// Retry counts provided by Config object `net.retry`. Config shell outputs
|
||||
/// a warning on per retry.
|
||||
///
|
||||
/// Closure must return a `Result`.
|
||||
pub(crate) fn with_retry<T, F>(mut callback: F) -> Result<T>
|
||||
where
|
||||
F: FnMut() -> Result<T>,
|
||||
{
|
||||
let mut retry = Retry::new();
|
||||
loop {
|
||||
match retry.r#try(&mut callback) {
|
||||
RetryResult::Success(r) => return Ok(r),
|
||||
RetryResult::Err(e) => return Err(e),
|
||||
RetryResult::Retry(sleep) => std::thread::sleep(Duration::from_millis(sleep)),
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue