feat(ext/fetch): add support for fetch on unix sockets (#29154)

This commit adds support for using Unix socket proxies in `fetch` API.

This is facilitated by passing an appropriate `Deno.HttpClient` instance
to the `fetch` API:
```
const client = Deno.createHttpClient({
  proxy: {
    transport: "unix",
    path: "/path/to/unix.sock",
  },
});

await fetch("http://localhost/ping", { client });
```

Closes https://github.com/denoland/deno/issues/8821

---------

Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/vscode/devcontainers/rust:1-bullseye
+FROM mcr.microsoft.com/devcontainers/rust:1
 
 # Install cmake
 RUN apt-get update \

cli/tsc/dts/lib.deno.ns.d.ts

@@ -6185,7 +6185,7 @@ declare namespace Deno {
      *
      * Must be in PEM format. */
     caCerts?: string[];
-    /** A HTTP proxy to use for new connections. */
+    /** An alternative transport (a proxy) to use for new connections. */
     proxy?: Proxy;
     /** Sets the maximum number of idle connections per host allowed in the pool. */
     poolMaxIdlePerHost?: number;
@@ -6213,17 +6213,38 @@ declare namespace Deno {
   }
 
   /**
-   * The definition of a proxy when specifying
+   * The definition for alternative transports (or proxies) in
    * {@linkcode Deno.CreateHttpClientOptions}.
    *
+   * Supported proxies:
+   *  - HTTP/HTTPS proxy: this uses the HTTP CONNECT method to tunnel HTTP
+   *    requests through a different server.
+   *  - SOCKS5 proxy: this uses the SOCKS5 protocol to tunnel TCP connections
+   *    through a different server.
+   *  - Unix domain socket: this sends all requests to a local Unix domain
+   *    socket rather than a TCP socket. *Not supported on Windows.*
+   *
    * @category Fetch
    */
-  export interface Proxy {
-    /** The string URL of the proxy server to use. */
+  export type Proxy = {
+    transport?: "http" | "https" | "socks5";
+    /**
+     * The string URL of the proxy server to use.
+     *
+     * For `http` and `https` transports, the URL must start with `http://` or
+     * `https://` respectively, or be a plain hostname.
+     *
+     * For `socks` transport, the URL must start with `socks5://` or
+     * `socks5h://`.
+     */
     url: string;
     /** The basic auth credentials to be used against the proxy server. */
     basicAuth?: BasicAuth;
-  }
+  } | {
+    transport: "unix";
+    /** The path to the unix domain socket to use. */
+    path: string;
+  };
 
   /**
    * Basic authentication credentials to be used with a {@linkcode Deno.Proxy}

ext/fetch/22_http_client.js

@@ -17,7 +17,13 @@ import { op_fetch_custom_client } from "ext:core/ops";
 import { loadTlsKeyPair } from "ext:deno_net/02_tls.js";
 
 const { internalRidSymbol } = core;
-const { ObjectDefineProperty } = primordials;
+const {
+  JSONStringify,
+  ObjectDefineProperty,
+  ObjectHasOwn,
+  StringPrototypeStartsWith,
+  TypeError,
+} = primordials;
 
 /**
  * @param {Deno.CreateHttpClientOptions} options
@@ -25,6 +31,65 @@ const { ObjectDefineProperty } = primordials;
  */
 function createHttpClient(options) {
   options.caCerts ??= [];
+  if (options.proxy) {
+    if (ObjectHasOwn(options.proxy, "transport")) {
+      switch (options.proxy.transport) {
+        case "http": {
+          const url = options.proxy.url;
+          if (
+            StringPrototypeStartsWith(url, "https:") ||
+            StringPrototypeStartsWith(url, "socks5:") ||
+            StringPrototypeStartsWith(url, "socks5h:")
+          ) {
+            throw new TypeError(
+              `The url passed into 'proxy.url' has an invalid scheme for this transport.`,
+            );
+          }
+          options.proxy.transport = "http";
+          break;
+        }
+        case "https": {
+          const url = options.proxy.url;
+          if (
+            StringPrototypeStartsWith(url, "http:") ||
+            StringPrototypeStartsWith(url, "socks5:") ||
+            StringPrototypeStartsWith(url, "socks5h:")
+          ) {
+            throw new TypeError(
+              `The url passed into 'proxy.url' has an invalid scheme for this transport.`,
+            );
+          }
+          options.proxy.transport = "http";
+          break;
+        }
+        case "socks5": {
+          const url = options.proxy.url;
+          if (
+            !StringPrototypeStartsWith(url, "socks5:") ||
+            !StringPrototypeStartsWith(url, "socks5h:")
+          ) {
+            throw new TypeError(
+              `The url passed into 'proxy.url' has an invalid scheme for this transport.`,
+            );
+          }
+          options.proxy.transport = "http";
+          break;
+        }
+        case "unix": {
+          break;
+        }
+        default: {
+          throw new TypeError(
+            `Invalid value for 'proxy.transport' option: ${
+              JSONStringify(options.proxy.transport)
+            }`,
+          );
+        }
+      }
+    } else {
+      options.proxy.transport = "http";
+    }
+  }
   const keyPair = loadTlsKeyPair("Deno.createHttpClient", options);
   return new HttpClient(
     op_fetch_custom_client(

ext/fetch/lib.rs

@@ -14,6 +14,8 @@ use std::future;
 use std::future::Future;
 use std::net::IpAddr;
 use std::path::Path;
+#[cfg(not(windows))]
+use std::path::PathBuf;
 use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
@@ -47,10 +49,13 @@ use deno_core::RcRef;
 use deno_core::Resource;
 use deno_core::ResourceId;
 use deno_error::JsErrorBox;
+use deno_fs::open_options_with_access_check;
 use deno_fs::CheckedPath;
 pub use deno_fs::FsError;
+use deno_fs::OpenOptions;
 use deno_path_util::PathToUrlError;
 use deno_permissions::PermissionCheckError;
+use deno_permissions::PermissionsContainer;
 use deno_tls::rustls::RootCertStore;
 use deno_tls::Proxy;
 use deno_tls::RootCertStoreProvider;
@@ -407,6 +412,12 @@ pub trait FetchPermissions {
     api_name: &str,
     get_path: &'a dyn deno_fs::GetPath,
   ) -> Result<deno_fs::CheckedPath<'a>, FsError>;
+  fn check_write<'a>(
+    &mut self,
+    path: Cow<'a, Path>,
+    api_name: &str,
+    get_path: &'a dyn deno_fs::GetPath,
+  ) -> Result<deno_fs::CheckedPath<'a>, FsError>;
 }
 
 impl FetchPermissions for deno_permissions::PermissionsContainer {
@@ -456,6 +467,42 @@ impl FetchPermissions for deno_permissions::PermissionsContainer {
       Ok(CheckedPath::Unresolved(path))
     }
   }
+
+  fn check_write<'a>(
+    &mut self,
+    path: Cow<'a, Path>,
+    api_name: &str,
+    get_path: &'a dyn deno_fs::GetPath,
+  ) -> Result<deno_fs::CheckedPath<'a>, FsError> {
+    if self.allows_all() {
+      return Ok(deno_fs::CheckedPath::Unresolved(path));
+    }
+
+    let (needs_canonicalize, normalized_path) = get_path.normalized(path)?;
+    let path = deno_permissions::PermissionsContainer::check_write_path(
+      self,
+      normalized_path,
+      api_name,
+    )
+    .map_err(|_| FsError::NotCapable("write"))?;
+
+    let path = if needs_canonicalize {
+      let path = get_path.resolved(&path)?;
+
+      Cow::Owned(path)
+    } else {
+      path
+    };
+    self
+      .check_special_file(&path, api_name)
+      .map_err(FsError::NotCapable)?;
+
+    if needs_canonicalize {
+      Ok(CheckedPath::Resolved(path))
+    } else {
+      Ok(CheckedPath::Unresolved(path))
+    }
+  }
 }
 
 #[op2(stack_trace)]
@@ -894,8 +941,6 @@ pub struct CreateHttpClientArgs {
   proxy: Option<Proxy>,
   pool_max_idle_per_host: Option<usize>,
   pool_idle_timeout: Option<serde_json::Value>,
-  #[serde(default)]
-  use_hickory_resolver: bool,
   #[serde(default = "default_true")]
   http1: bool,
   #[serde(default = "default_true")]
@@ -909,6 +954,27 @@ fn default_true() -> bool {
   true
 }
 
+fn sync_permission_check<'a, P: FetchPermissions + 'static>(
+  permissions: &'a mut P,
+  api_name: &'static str,
+) -> impl deno_fs::AccessCheckFn + 'a {
+  move |path, _options, _resolve| {
+    let read_path = permissions.check_read(path.clone(), api_name, _resolve)?;
+    let write_path = permissions.check_write(path, api_name, _resolve)?;
+    match (&read_path, &write_path) {
+      (CheckedPath::Resolved(a), CheckedPath::Resolved(b))
+      | (CheckedPath::Unresolved(a), CheckedPath::Resolved(b))
+      | (CheckedPath::Resolved(a), CheckedPath::Unresolved(b))
+      | (CheckedPath::Unresolved(a), CheckedPath::Unresolved(b))
+        if a == b =>
+      {
+        Ok(write_path)
+      }
+      _ => Err(FsError::NotCapable("write")),
+    }
+  }
+}
+
 #[op2(stack_trace)]
 #[smi]
 pub fn op_fetch_custom_client<FP>(
@@ -919,10 +985,33 @@ pub fn op_fetch_custom_client<FP>(
 where
   FP: FetchPermissions + 'static,
 {
-  if let Some(proxy) = args.proxy.clone() {
+  if let Some(proxy) = &args.proxy {
     let permissions = state.borrow_mut::<FP>();
-    let url = Url::parse(&proxy.url)?;
-    permissions.check_net_url(&url, "Deno.createHttpClient()")?;
+    match proxy {
+      Proxy::Http { url, .. } => {
+        let url = Url::parse(url)?;
+        permissions.check_net_url(&url, "Deno.createHttpClient()")?;
+      }
+      Proxy::Unix { path } => {
+        let path = Path::new(path);
+        let mut access_check = sync_permission_check::<PermissionsContainer>(
+          state.borrow_mut(),
+          "Deno.createHttpClient()",
+        );
+        let (resolved_path, _) = open_options_with_access_check(
+          OpenOptions {
+            read: true,
+            write: true,
+            ..Default::default()
+          },
+          path,
+          Some(&mut access_check),
+        )?;
+        if path != resolved_path {
+          return Err(FetchError::NotCapable("write"));
+        }
+      }
+    }
   }
 
   let options = state.borrow::<Options>();
@@ -940,11 +1029,7 @@ where
         .map_err(HttpClientCreateError::RootCertStore)?,
       ca_certs,
       proxy: args.proxy,
-      dns_resolver: if args.use_hickory_resolver {
-        dns::Resolver::hickory().map_err(FetchError::Dns)?
-      } else {
-        dns::Resolver::default()
-      },
+      dns_resolver: dns::Resolver::default(),
       unsafely_ignore_certificate_errors: options
         .unsafely_ignore_certificate_errors
         .clone(),
@@ -1024,6 +1109,8 @@ pub enum HttpClientCreateError {
   #[class(inherit)]
   #[error(transparent)]
   RootCertStore(JsErrorBox),
+  #[error("Unix proxy is not supported on Windows")]
+  UnixProxyNotSupportedOnWindows,
 }
 
 /// Create new instance of async Client. This client supports
@@ -1079,11 +1166,26 @@ pub fn create_http_client(
 
   let mut proxies = proxy::from_env();
   if let Some(proxy) = options.proxy {
-    let mut intercept = proxy::Intercept::all(&proxy.url)
-      .ok_or_else(|| HttpClientCreateError::InvalidProxyUrl)?;
-    if let Some(basic_auth) = &proxy.basic_auth {
-      intercept.set_auth(&basic_auth.username, &basic_auth.password);
-    }
+    let intercept = match proxy {
+      Proxy::Http { url, basic_auth } => {
+        let target = proxy::Target::parse(&url)
+          .ok_or_else(|| HttpClientCreateError::InvalidProxyUrl)?;
+        let mut intercept = proxy::Intercept::all(target);
+        if let Some(basic_auth) = &basic_auth {
+          intercept.set_auth(&basic_auth.username, &basic_auth.password);
+        }
+        intercept
+      }
+      #[cfg(not(windows))]
+      Proxy::Unix { path } => {
+        let target = proxy::Target::new_unix(PathBuf::from(path));
+        proxy::Intercept::all(target)
+      }
+      #[cfg(windows)]
+      Proxy::Unix { .. } => {
+        return Err(HttpClientCreateError::UnixProxyNotSupportedOnWindows);
+      }
+    };
     proxies.prepend(intercept);
   }
   let proxies = Arc::new(proxies);

ext/fetch/proxy.rs

@@ -6,6 +6,8 @@
 use std::env;
 use std::future::Future;
 use std::net::IpAddr;
+#[cfg(not(windows))]
+use std::path::PathBuf;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::Context;
@@ -24,6 +26,8 @@ use hyper_util::rt::TokioIo;
 use ipnet::IpNet;
 use percent_encoding::percent_decode_str;
 use tokio::net::TcpStream;
+#[cfg(not(windows))]
+use tokio::net::UnixStream;
 use tokio_rustls::client::TlsStream;
 use tokio_rustls::TlsConnector;
 use tokio_socks::tcp::Socks5Stream;
@@ -54,7 +58,7 @@ pub(crate) struct Intercept {
 }
 
 #[derive(Clone)]
-enum Target {
+pub(crate) enum Target {
   Http {
     dst: Uri,
     auth: Option<HeaderValue>,
@@ -67,6 +71,10 @@ enum Target {
     dst: Uri,
     auth: Option<(String, String)>,
   },
+  #[cfg(not(windows))]
+  Unix {
+    path: PathBuf,
+  },
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -133,12 +141,11 @@ fn parse_env_var(name: &str, filter: Filter) -> Option<Intercept> {
 }
 
 impl Intercept {
-  pub(crate) fn all(s: &str) -> Option<Self> {
-    let target = Target::parse(s)?;
-    Some(Intercept {
+  pub(crate) fn all(target: Target) -> Self {
+    Intercept {
       filter: Filter::All,
       target,
-    })
+    }
   }
 
   pub(crate) fn set_auth(&mut self, user: &str, pass: &str) {
@@ -152,6 +159,10 @@ impl Intercept {
       Target::Socks { ref mut auth, .. } => {
         *auth = Some((user.into(), pass.into()));
       }
+      #[cfg(not(windows))]
+      Target::Unix { .. } => {
+        // Auth not supported for Unix sockets
+      }
     }
   }
 }
@@ -165,7 +176,7 @@ impl std::fmt::Debug for Intercept {
 }
 
 impl Target {
-  fn parse(val: &str) -> Option<Self> {
+  pub(crate) fn parse(val: &str) -> Option<Self> {
     let uri = val.parse::<Uri>().ok()?;
 
     let mut builder = Uri::builder();
@@ -229,6 +240,11 @@ impl Target {
 
     Some(target)
   }
+
+  #[cfg(not(windows))]
+  pub(crate) fn new_unix(path: PathBuf) -> Self {
+    Target::Unix { path }
+  }
 }
 
 #[derive(Debug)]
@@ -429,6 +445,9 @@ pub enum Proxied<T> {
   Socks(TokioIo<TcpStream>),
   /// Tunneled through SOCKS and TLS
   SocksTls(TokioIo<TlsStream<TokioIo<TokioIo<TcpStream>>>>),
+  /// Forwarded via Unix socket
+  #[cfg(not(windows))]
+  Unix(TokioIo<UnixStream>),
 }
 
 impl<C> Service<Uri> for ProxyConnector<C>
@@ -525,6 +544,14 @@ where
             }
           })
         }
+        #[cfg(not(windows))]
+        Target::Unix { path } => {
+          let path = path.clone();
+          Box::pin(async move {
+            let io = UnixStream::connect(&path).await?;
+            Ok(Proxied::Unix(TokioIo::new(io)))
+          })
+        }
       };
     }
 
@@ -634,6 +661,8 @@ where
       Proxied::HttpTunneled(ref mut p) => Pin::new(p).poll_read(cx, buf),
       Proxied::Socks(ref mut p) => Pin::new(p).poll_read(cx, buf),
       Proxied::SocksTls(ref mut p) => Pin::new(p).poll_read(cx, buf),
+      #[cfg(not(windows))]
+      Proxied::Unix(ref mut p) => Pin::new(p).poll_read(cx, buf),
     }
   }
 }
@@ -653,6 +682,8 @@ where
       Proxied::HttpTunneled(ref mut p) => Pin::new(p).poll_write(cx, buf),
       Proxied::Socks(ref mut p) => Pin::new(p).poll_write(cx, buf),
       Proxied::SocksTls(ref mut p) => Pin::new(p).poll_write(cx, buf),
+      #[cfg(not(windows))]
+      Proxied::Unix(ref mut p) => Pin::new(p).poll_write(cx, buf),
     }
   }
 
@@ -666,6 +697,8 @@ where
       Proxied::HttpTunneled(ref mut p) => Pin::new(p).poll_flush(cx),
       Proxied::Socks(ref mut p) => Pin::new(p).poll_flush(cx),
       Proxied::SocksTls(ref mut p) => Pin::new(p).poll_flush(cx),
+      #[cfg(not(windows))]
+      Proxied::Unix(ref mut p) => Pin::new(p).poll_flush(cx),
     }
   }
 
@@ -679,6 +712,8 @@ where
       Proxied::HttpTunneled(ref mut p) => Pin::new(p).poll_shutdown(cx),
       Proxied::Socks(ref mut p) => Pin::new(p).poll_shutdown(cx),
       Proxied::SocksTls(ref mut p) => Pin::new(p).poll_shutdown(cx),
+      #[cfg(not(windows))]
+      Proxied::Unix(ref mut p) => Pin::new(p).poll_shutdown(cx),
     }
   }
 
@@ -689,6 +724,8 @@ where
       Proxied::HttpTunneled(ref p) => p.is_write_vectored(),
       Proxied::Socks(ref p) => p.is_write_vectored(),
       Proxied::SocksTls(ref p) => p.is_write_vectored(),
+      #[cfg(not(windows))]
+      Proxied::Unix(ref p) => p.is_write_vectored(),
     }
   }
 
@@ -709,6 +746,8 @@ where
       }
       Proxied::Socks(ref mut p) => Pin::new(p).poll_write_vectored(cx, bufs),
       Proxied::SocksTls(ref mut p) => Pin::new(p).poll_write_vectored(cx, bufs),
+      #[cfg(not(windows))]
+      Proxied::Unix(ref mut p) => Pin::new(p).poll_write_vectored(cx, bufs),
     }
   }
 }
@@ -738,6 +777,8 @@ where
           tunneled_tls.0.connected()
         }
       }
+      #[cfg(not(windows))]
+      Proxied::Unix(_) => Connected::new(),
     }
   }
 }

ext/fetch/tests.rs

@@ -114,7 +114,7 @@ async fn rust_test_client_with_resolver(
     CreateHttpClientOptions {
       root_cert_store: None,
       ca_certs: vec![],
-      proxy: prx_addr.map(|p| deno_tls::Proxy {
+      proxy: prx_addr.map(|p| deno_tls::Proxy::Http {
         url: format!("{}://{}", proto, p),
         basic_auth: None,
       }),

ext/tls/lib.rs

@@ -158,12 +158,17 @@ impl ServerCertVerifier for NoCertificateVerification {
   }
 }
 
-#[derive(Deserialize, Default, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-#[serde(default)]
-pub struct Proxy {
-  pub url: String,
-  pub basic_auth: Option<BasicAuth>,
+#[derive(Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase", tag = "transport")]
+pub enum Proxy {
+  #[serde(rename_all = "camelCase")]
+  Http {
+    url: String,
+    basic_auth: Option<BasicAuth>,
+  },
+  Unix {
+    path: String,
+  },
 }
 
 #[derive(Deserialize, Default, Debug, Clone)]

runtime/snapshot_info.rs

@@ -50,6 +50,15 @@ impl deno_fetch::FetchPermissions for Permissions {
   ) -> Result<deno_fs::CheckedPath<'a>, FsError> {
     unreachable!("snapshotting!")
   }
+
+  fn check_write<'a>(
+    &mut self,
+    _p: Cow<'a, Path>,
+    _api_name: &str,
+    _get_path: &'a dyn deno_fs::GetPath,
+  ) -> Result<deno_fs::CheckedPath<'a>, FsError> {
+    unreachable!("snapshotting!")
+  }
 }
 
 impl deno_ffi::FfiPermissions for Permissions {

tests/unit/fetch_test.ts

@@ -2222,3 +2222,91 @@ Deno.test("fetch string object", async () => {
   const res = new Response(Object("hello"));
   assertEquals(await res.text(), "hello");
 });
+
+Deno.test(
+  {
+    permissions: { net: true, read: true, write: true },
+    ignore: Deno.build.os === "windows",
+  },
+  async function fetchUnixSocket() {
+    const tempDir = await Deno.makeTempDir();
+    const socketPath = `${tempDir}/unix.sock`;
+
+    await using _server = Deno.serve({
+      path: socketPath,
+      transport: "unix",
+      onListen: () => {},
+    }, (req) => {
+      const url = new URL(req.url);
+      if (url.pathname === "/ping") {
+        return new Response(url.href, {
+          headers: { "content-type": "text/plain" },
+        });
+      } else {
+        return new Response("Not found", { status: 404 });
+      }
+    });
+
+    // Canonicalize the path, because permission checks are done so that
+    // the symlink doesn't change in between the calls.
+    const resolvedPath = await Deno.realPath(socketPath);
+    using client = Deno.createHttpClient({
+      proxy: {
+        transport: "unix",
+        path: resolvedPath,
+      },
+    });
+
+    const resp1 = await fetch("http://localhost/ping", { client });
+    assertEquals(resp1.status, 200);
+    assertEquals(resp1.headers.get("content-type"), "text/plain");
+    assertEquals(await resp1.text(), "http+unix://localhost/ping");
+
+    const resp2 = await fetch("http://localhost/not-found", { client });
+    assertEquals(resp2.status, 404);
+    assertEquals(await resp2.text(), "Not found");
+  },
+);
+
+Deno.test(
+  {
+    permissions: { net: true },
+  },
+  function createHttpClientThrowsWhenProxyTransportMismatch() {
+    assertThrows(
+      () => {
+        Deno.createHttpClient({
+          proxy: {
+            transport: "socks5", // Mismatch with "http://" URL
+            url: "http://localhost:8080",
+          },
+        });
+      },
+      TypeError,
+    );
+
+    assertThrows(
+      () => {
+        Deno.createHttpClient({
+          proxy: {
+            transport: "http", // Mismatch with "https://" URL
+            url: "https://localhost:8080",
+          },
+        });
+      },
+      TypeError,
+    );
+
+    assertThrows(
+      () => {
+        Deno.createHttpClient({
+          proxy: {
+            transport: "https", // Mismatch with "socks5://" URL
+            url: "socks5://localhost:1080",
+          },
+        });
+      },
+      TypeError,
+    );
+  },
+);