From 3b6c70e5b25d1dc48ee849b7d837f22ebd6afd8e Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Tue, 13 May 2025 18:06:17 +0200 Subject: [PATCH] feat(ext/fetch): add support for fetch on unix sockets (#29154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds support for using Unix socket proxies in `fetch` API. This is facilitated by passing an appropriate `Deno.HttpClient` instance to the `fetch` API: ``` const client = Deno.createHttpClient({ proxy: { transport: "unix", path: "/path/to/unix.sock", }, }); await fetch("http://localhost/ping", { client }); ``` Closes https://github.com/denoland/deno/issues/8821 --------- Co-authored-by: Bartek Iwańczuk --- .devcontainer/Dockerfile | 2 +- cli/tsc/dts/lib.deno.ns.d.ts | 31 ++++++-- ext/fetch/22_http_client.js | 67 +++++++++++++++++- ext/fetch/lib.rs | 132 +++++++++++++++++++++++++++++++---- ext/fetch/proxy.rs | 53 ++++++++++++-- ext/fetch/tests.rs | 2 +- ext/tls/lib.rs | 17 +++-- runtime/snapshot_info.rs | 9 +++ tests/unit/fetch_test.ts | 88 +++++++++++++++++++++++ 9 files changed, 366 insertions(+), 35 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 142a3c286e..6cd22a14cc 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/vscode/devcontainers/rust:1-bullseye +FROM mcr.microsoft.com/devcontainers/rust:1 # Install cmake RUN apt-get update \ diff --git a/cli/tsc/dts/lib.deno.ns.d.ts b/cli/tsc/dts/lib.deno.ns.d.ts index 0aaeb6792d..0bb30fd6ac 100644 --- a/cli/tsc/dts/lib.deno.ns.d.ts +++ b/cli/tsc/dts/lib.deno.ns.d.ts @@ -6185,7 +6185,7 @@ declare namespace Deno { * * Must be in PEM format. */ caCerts?: string[]; - /** A HTTP proxy to use for new connections. */ + /** An alternative transport (a proxy) to use for new connections. */ proxy?: Proxy; /** Sets the maximum number of idle connections per host allowed in the pool. */ poolMaxIdlePerHost?: number; @@ -6213,17 +6213,38 @@ declare namespace Deno { } /** - * The definition of a proxy when specifying + * The definition for alternative transports (or proxies) in * {@linkcode Deno.CreateHttpClientOptions}. * + * Supported proxies: + * - HTTP/HTTPS proxy: this uses the HTTP CONNECT method to tunnel HTTP + * requests through a different server. + * - SOCKS5 proxy: this uses the SOCKS5 protocol to tunnel TCP connections + * through a different server. + * - Unix domain socket: this sends all requests to a local Unix domain + * socket rather than a TCP socket. *Not supported on Windows.* + * * @category Fetch */ - export interface Proxy { - /** The string URL of the proxy server to use. */ + export type Proxy = { + transport?: "http" | "https" | "socks5"; + /** + * The string URL of the proxy server to use. + * + * For `http` and `https` transports, the URL must start with `http://` or + * `https://` respectively, or be a plain hostname. + * + * For `socks` transport, the URL must start with `socks5://` or + * `socks5h://`. + */ url: string; /** The basic auth credentials to be used against the proxy server. */ basicAuth?: BasicAuth; - } + } | { + transport: "unix"; + /** The path to the unix domain socket to use. */ + path: string; + }; /** * Basic authentication credentials to be used with a {@linkcode Deno.Proxy} diff --git a/ext/fetch/22_http_client.js b/ext/fetch/22_http_client.js index c7ade7e2d2..1d28f611d1 100644 --- a/ext/fetch/22_http_client.js +++ b/ext/fetch/22_http_client.js @@ -17,7 +17,13 @@ import { op_fetch_custom_client } from "ext:core/ops"; import { loadTlsKeyPair } from "ext:deno_net/02_tls.js"; const { internalRidSymbol } = core; -const { ObjectDefineProperty } = primordials; +const { + JSONStringify, + ObjectDefineProperty, + ObjectHasOwn, + StringPrototypeStartsWith, + TypeError, +} = primordials; /** * @param {Deno.CreateHttpClientOptions} options @@ -25,6 +31,65 @@ const { ObjectDefineProperty } = primordials; */ function createHttpClient(options) { options.caCerts ??= []; + if (options.proxy) { + if (ObjectHasOwn(options.proxy, "transport")) { + switch (options.proxy.transport) { + case "http": { + const url = options.proxy.url; + if ( + StringPrototypeStartsWith(url, "https:") || + StringPrototypeStartsWith(url, "socks5:") || + StringPrototypeStartsWith(url, "socks5h:") + ) { + throw new TypeError( + `The url passed into 'proxy.url' has an invalid scheme for this transport.`, + ); + } + options.proxy.transport = "http"; + break; + } + case "https": { + const url = options.proxy.url; + if ( + StringPrototypeStartsWith(url, "http:") || + StringPrototypeStartsWith(url, "socks5:") || + StringPrototypeStartsWith(url, "socks5h:") + ) { + throw new TypeError( + `The url passed into 'proxy.url' has an invalid scheme for this transport.`, + ); + } + options.proxy.transport = "http"; + break; + } + case "socks5": { + const url = options.proxy.url; + if ( + !StringPrototypeStartsWith(url, "socks5:") || + !StringPrototypeStartsWith(url, "socks5h:") + ) { + throw new TypeError( + `The url passed into 'proxy.url' has an invalid scheme for this transport.`, + ); + } + options.proxy.transport = "http"; + break; + } + case "unix": { + break; + } + default: { + throw new TypeError( + `Invalid value for 'proxy.transport' option: ${ + JSONStringify(options.proxy.transport) + }`, + ); + } + } + } else { + options.proxy.transport = "http"; + } + } const keyPair = loadTlsKeyPair("Deno.createHttpClient", options); return new HttpClient( op_fetch_custom_client( diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index 41adc7c7f5..567faecd45 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -14,6 +14,8 @@ use std::future; use std::future::Future; use std::net::IpAddr; use std::path::Path; +#[cfg(not(windows))] +use std::path::PathBuf; use std::pin::Pin; use std::rc::Rc; use std::sync::Arc; @@ -47,10 +49,13 @@ use deno_core::RcRef; use deno_core::Resource; use deno_core::ResourceId; use deno_error::JsErrorBox; +use deno_fs::open_options_with_access_check; use deno_fs::CheckedPath; pub use deno_fs::FsError; +use deno_fs::OpenOptions; use deno_path_util::PathToUrlError; use deno_permissions::PermissionCheckError; +use deno_permissions::PermissionsContainer; use deno_tls::rustls::RootCertStore; use deno_tls::Proxy; use deno_tls::RootCertStoreProvider; @@ -407,6 +412,12 @@ pub trait FetchPermissions { api_name: &str, get_path: &'a dyn deno_fs::GetPath, ) -> Result, FsError>; + fn check_write<'a>( + &mut self, + path: Cow<'a, Path>, + api_name: &str, + get_path: &'a dyn deno_fs::GetPath, + ) -> Result, FsError>; } impl FetchPermissions for deno_permissions::PermissionsContainer { @@ -456,6 +467,42 @@ impl FetchPermissions for deno_permissions::PermissionsContainer { Ok(CheckedPath::Unresolved(path)) } } + + fn check_write<'a>( + &mut self, + path: Cow<'a, Path>, + api_name: &str, + get_path: &'a dyn deno_fs::GetPath, + ) -> Result, FsError> { + if self.allows_all() { + return Ok(deno_fs::CheckedPath::Unresolved(path)); + } + + let (needs_canonicalize, normalized_path) = get_path.normalized(path)?; + let path = deno_permissions::PermissionsContainer::check_write_path( + self, + normalized_path, + api_name, + ) + .map_err(|_| FsError::NotCapable("write"))?; + + let path = if needs_canonicalize { + let path = get_path.resolved(&path)?; + + Cow::Owned(path) + } else { + path + }; + self + .check_special_file(&path, api_name) + .map_err(FsError::NotCapable)?; + + if needs_canonicalize { + Ok(CheckedPath::Resolved(path)) + } else { + Ok(CheckedPath::Unresolved(path)) + } + } } #[op2(stack_trace)] @@ -894,8 +941,6 @@ pub struct CreateHttpClientArgs { proxy: Option, pool_max_idle_per_host: Option, pool_idle_timeout: Option, - #[serde(default)] - use_hickory_resolver: bool, #[serde(default = "default_true")] http1: bool, #[serde(default = "default_true")] @@ -909,6 +954,27 @@ fn default_true() -> bool { true } +fn sync_permission_check<'a, P: FetchPermissions + 'static>( + permissions: &'a mut P, + api_name: &'static str, +) -> impl deno_fs::AccessCheckFn + 'a { + move |path, _options, _resolve| { + let read_path = permissions.check_read(path.clone(), api_name, _resolve)?; + let write_path = permissions.check_write(path, api_name, _resolve)?; + match (&read_path, &write_path) { + (CheckedPath::Resolved(a), CheckedPath::Resolved(b)) + | (CheckedPath::Unresolved(a), CheckedPath::Resolved(b)) + | (CheckedPath::Resolved(a), CheckedPath::Unresolved(b)) + | (CheckedPath::Unresolved(a), CheckedPath::Unresolved(b)) + if a == b => + { + Ok(write_path) + } + _ => Err(FsError::NotCapable("write")), + } + } +} + #[op2(stack_trace)] #[smi] pub fn op_fetch_custom_client( @@ -919,10 +985,33 @@ pub fn op_fetch_custom_client( where FP: FetchPermissions + 'static, { - if let Some(proxy) = args.proxy.clone() { + if let Some(proxy) = &args.proxy { let permissions = state.borrow_mut::(); - let url = Url::parse(&proxy.url)?; - permissions.check_net_url(&url, "Deno.createHttpClient()")?; + match proxy { + Proxy::Http { url, .. } => { + let url = Url::parse(url)?; + permissions.check_net_url(&url, "Deno.createHttpClient()")?; + } + Proxy::Unix { path } => { + let path = Path::new(path); + let mut access_check = sync_permission_check::( + state.borrow_mut(), + "Deno.createHttpClient()", + ); + let (resolved_path, _) = open_options_with_access_check( + OpenOptions { + read: true, + write: true, + ..Default::default() + }, + path, + Some(&mut access_check), + )?; + if path != resolved_path { + return Err(FetchError::NotCapable("write")); + } + } + } } let options = state.borrow::(); @@ -940,11 +1029,7 @@ where .map_err(HttpClientCreateError::RootCertStore)?, ca_certs, proxy: args.proxy, - dns_resolver: if args.use_hickory_resolver { - dns::Resolver::hickory().map_err(FetchError::Dns)? - } else { - dns::Resolver::default() - }, + dns_resolver: dns::Resolver::default(), unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -1024,6 +1109,8 @@ pub enum HttpClientCreateError { #[class(inherit)] #[error(transparent)] RootCertStore(JsErrorBox), + #[error("Unix proxy is not supported on Windows")] + UnixProxyNotSupportedOnWindows, } /// Create new instance of async Client. This client supports @@ -1079,11 +1166,26 @@ pub fn create_http_client( let mut proxies = proxy::from_env(); if let Some(proxy) = options.proxy { - let mut intercept = proxy::Intercept::all(&proxy.url) - .ok_or_else(|| HttpClientCreateError::InvalidProxyUrl)?; - if let Some(basic_auth) = &proxy.basic_auth { - intercept.set_auth(&basic_auth.username, &basic_auth.password); - } + let intercept = match proxy { + Proxy::Http { url, basic_auth } => { + let target = proxy::Target::parse(&url) + .ok_or_else(|| HttpClientCreateError::InvalidProxyUrl)?; + let mut intercept = proxy::Intercept::all(target); + if let Some(basic_auth) = &basic_auth { + intercept.set_auth(&basic_auth.username, &basic_auth.password); + } + intercept + } + #[cfg(not(windows))] + Proxy::Unix { path } => { + let target = proxy::Target::new_unix(PathBuf::from(path)); + proxy::Intercept::all(target) + } + #[cfg(windows)] + Proxy::Unix { .. } => { + return Err(HttpClientCreateError::UnixProxyNotSupportedOnWindows); + } + }; proxies.prepend(intercept); } let proxies = Arc::new(proxies); diff --git a/ext/fetch/proxy.rs b/ext/fetch/proxy.rs index 9d85a21f41..105c96b9c8 100644 --- a/ext/fetch/proxy.rs +++ b/ext/fetch/proxy.rs @@ -6,6 +6,8 @@ use std::env; use std::future::Future; use std::net::IpAddr; +#[cfg(not(windows))] +use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use std::task::Context; @@ -24,6 +26,8 @@ use hyper_util::rt::TokioIo; use ipnet::IpNet; use percent_encoding::percent_decode_str; use tokio::net::TcpStream; +#[cfg(not(windows))] +use tokio::net::UnixStream; use tokio_rustls::client::TlsStream; use tokio_rustls::TlsConnector; use tokio_socks::tcp::Socks5Stream; @@ -54,7 +58,7 @@ pub(crate) struct Intercept { } #[derive(Clone)] -enum Target { +pub(crate) enum Target { Http { dst: Uri, auth: Option, @@ -67,6 +71,10 @@ enum Target { dst: Uri, auth: Option<(String, String)>, }, + #[cfg(not(windows))] + Unix { + path: PathBuf, + }, } #[derive(Debug, Clone, Copy)] @@ -133,12 +141,11 @@ fn parse_env_var(name: &str, filter: Filter) -> Option { } impl Intercept { - pub(crate) fn all(s: &str) -> Option { - let target = Target::parse(s)?; - Some(Intercept { + pub(crate) fn all(target: Target) -> Self { + Intercept { filter: Filter::All, target, - }) + } } pub(crate) fn set_auth(&mut self, user: &str, pass: &str) { @@ -152,6 +159,10 @@ impl Intercept { Target::Socks { ref mut auth, .. } => { *auth = Some((user.into(), pass.into())); } + #[cfg(not(windows))] + Target::Unix { .. } => { + // Auth not supported for Unix sockets + } } } } @@ -165,7 +176,7 @@ impl std::fmt::Debug for Intercept { } impl Target { - fn parse(val: &str) -> Option { + pub(crate) fn parse(val: &str) -> Option { let uri = val.parse::().ok()?; let mut builder = Uri::builder(); @@ -229,6 +240,11 @@ impl Target { Some(target) } + + #[cfg(not(windows))] + pub(crate) fn new_unix(path: PathBuf) -> Self { + Target::Unix { path } + } } #[derive(Debug)] @@ -429,6 +445,9 @@ pub enum Proxied { Socks(TokioIo), /// Tunneled through SOCKS and TLS SocksTls(TokioIo>>>), + /// Forwarded via Unix socket + #[cfg(not(windows))] + Unix(TokioIo), } impl Service for ProxyConnector @@ -525,6 +544,14 @@ where } }) } + #[cfg(not(windows))] + Target::Unix { path } => { + let path = path.clone(); + Box::pin(async move { + let io = UnixStream::connect(&path).await?; + Ok(Proxied::Unix(TokioIo::new(io))) + }) + } }; } @@ -634,6 +661,8 @@ where Proxied::HttpTunneled(ref mut p) => Pin::new(p).poll_read(cx, buf), Proxied::Socks(ref mut p) => Pin::new(p).poll_read(cx, buf), Proxied::SocksTls(ref mut p) => Pin::new(p).poll_read(cx, buf), + #[cfg(not(windows))] + Proxied::Unix(ref mut p) => Pin::new(p).poll_read(cx, buf), } } } @@ -653,6 +682,8 @@ where Proxied::HttpTunneled(ref mut p) => Pin::new(p).poll_write(cx, buf), Proxied::Socks(ref mut p) => Pin::new(p).poll_write(cx, buf), Proxied::SocksTls(ref mut p) => Pin::new(p).poll_write(cx, buf), + #[cfg(not(windows))] + Proxied::Unix(ref mut p) => Pin::new(p).poll_write(cx, buf), } } @@ -666,6 +697,8 @@ where Proxied::HttpTunneled(ref mut p) => Pin::new(p).poll_flush(cx), Proxied::Socks(ref mut p) => Pin::new(p).poll_flush(cx), Proxied::SocksTls(ref mut p) => Pin::new(p).poll_flush(cx), + #[cfg(not(windows))] + Proxied::Unix(ref mut p) => Pin::new(p).poll_flush(cx), } } @@ -679,6 +712,8 @@ where Proxied::HttpTunneled(ref mut p) => Pin::new(p).poll_shutdown(cx), Proxied::Socks(ref mut p) => Pin::new(p).poll_shutdown(cx), Proxied::SocksTls(ref mut p) => Pin::new(p).poll_shutdown(cx), + #[cfg(not(windows))] + Proxied::Unix(ref mut p) => Pin::new(p).poll_shutdown(cx), } } @@ -689,6 +724,8 @@ where Proxied::HttpTunneled(ref p) => p.is_write_vectored(), Proxied::Socks(ref p) => p.is_write_vectored(), Proxied::SocksTls(ref p) => p.is_write_vectored(), + #[cfg(not(windows))] + Proxied::Unix(ref p) => p.is_write_vectored(), } } @@ -709,6 +746,8 @@ where } Proxied::Socks(ref mut p) => Pin::new(p).poll_write_vectored(cx, bufs), Proxied::SocksTls(ref mut p) => Pin::new(p).poll_write_vectored(cx, bufs), + #[cfg(not(windows))] + Proxied::Unix(ref mut p) => Pin::new(p).poll_write_vectored(cx, bufs), } } } @@ -738,6 +777,8 @@ where tunneled_tls.0.connected() } } + #[cfg(not(windows))] + Proxied::Unix(_) => Connected::new(), } } } diff --git a/ext/fetch/tests.rs b/ext/fetch/tests.rs index 1c5981b0e3..ec7d940b1d 100644 --- a/ext/fetch/tests.rs +++ b/ext/fetch/tests.rs @@ -114,7 +114,7 @@ async fn rust_test_client_with_resolver( CreateHttpClientOptions { root_cert_store: None, ca_certs: vec![], - proxy: prx_addr.map(|p| deno_tls::Proxy { + proxy: prx_addr.map(|p| deno_tls::Proxy::Http { url: format!("{}://{}", proto, p), basic_auth: None, }), diff --git a/ext/tls/lib.rs b/ext/tls/lib.rs index a3e386052e..cf1194375b 100644 --- a/ext/tls/lib.rs +++ b/ext/tls/lib.rs @@ -158,12 +158,17 @@ impl ServerCertVerifier for NoCertificateVerification { } } -#[derive(Deserialize, Default, Debug, Clone)] -#[serde(rename_all = "camelCase")] -#[serde(default)] -pub struct Proxy { - pub url: String, - pub basic_auth: Option, +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase", tag = "transport")] +pub enum Proxy { + #[serde(rename_all = "camelCase")] + Http { + url: String, + basic_auth: Option, + }, + Unix { + path: String, + }, } #[derive(Deserialize, Default, Debug, Clone)] diff --git a/runtime/snapshot_info.rs b/runtime/snapshot_info.rs index 6bab3bd132..87cb1b1121 100644 --- a/runtime/snapshot_info.rs +++ b/runtime/snapshot_info.rs @@ -50,6 +50,15 @@ impl deno_fetch::FetchPermissions for Permissions { ) -> Result, FsError> { unreachable!("snapshotting!") } + + fn check_write<'a>( + &mut self, + _p: Cow<'a, Path>, + _api_name: &str, + _get_path: &'a dyn deno_fs::GetPath, + ) -> Result, FsError> { + unreachable!("snapshotting!") + } } impl deno_ffi::FfiPermissions for Permissions { diff --git a/tests/unit/fetch_test.ts b/tests/unit/fetch_test.ts index 272bdd7bd9..b7d972d42f 100644 --- a/tests/unit/fetch_test.ts +++ b/tests/unit/fetch_test.ts @@ -2222,3 +2222,91 @@ Deno.test("fetch string object", async () => { const res = new Response(Object("hello")); assertEquals(await res.text(), "hello"); }); + +Deno.test( + { + permissions: { net: true, read: true, write: true }, + ignore: Deno.build.os === "windows", + }, + async function fetchUnixSocket() { + const tempDir = await Deno.makeTempDir(); + const socketPath = `${tempDir}/unix.sock`; + + await using _server = Deno.serve({ + path: socketPath, + transport: "unix", + onListen: () => {}, + }, (req) => { + const url = new URL(req.url); + if (url.pathname === "/ping") { + return new Response(url.href, { + headers: { "content-type": "text/plain" }, + }); + } else { + return new Response("Not found", { status: 404 }); + } + }); + + // Canonicalize the path, because permission checks are done so that + // the symlink doesn't change in between the calls. + const resolvedPath = await Deno.realPath(socketPath); + using client = Deno.createHttpClient({ + proxy: { + transport: "unix", + path: resolvedPath, + }, + }); + + const resp1 = await fetch("http://localhost/ping", { client }); + assertEquals(resp1.status, 200); + assertEquals(resp1.headers.get("content-type"), "text/plain"); + assertEquals(await resp1.text(), "http+unix://localhost/ping"); + + const resp2 = await fetch("http://localhost/not-found", { client }); + assertEquals(resp2.status, 404); + assertEquals(await resp2.text(), "Not found"); + }, +); + +Deno.test( + { + permissions: { net: true }, + }, + function createHttpClientThrowsWhenProxyTransportMismatch() { + assertThrows( + () => { + Deno.createHttpClient({ + proxy: { + transport: "socks5", // Mismatch with "http://" URL + url: "http://localhost:8080", + }, + }); + }, + TypeError, + ); + + assertThrows( + () => { + Deno.createHttpClient({ + proxy: { + transport: "http", // Mismatch with "https://" URL + url: "https://localhost:8080", + }, + }); + }, + TypeError, + ); + + assertThrows( + () => { + Deno.createHttpClient({ + proxy: { + transport: "https", // Mismatch with "socks5://" URL + url: "socks5://localhost:1080", + }, + }); + }, + TypeError, + ); + }, +);