feat(ext/fetch): allow embedders to use hickory_dns_resolver instead of default GaiResolver (#26740)

Allows embedders to use `hickory-dns-resolver` instead of threaded "getaddrinfo" resolver
in the `fetch()` implementation.

Cargo.lock

@@ -1566,6 +1566,7 @@ dependencies = [
  "dyn-clone",
  "error_reporter",
  "fast-socks5",
+ "hickory-resolver",
  "http 1.1.0",
  "http-body-util",
  "hyper 1.4.1",

Cargo.toml

@@ -127,6 +127,7 @@ fs3 = "0.5.0"
 futures = "0.3.21"
 glob = "0.3.1"
 h2 = "0.4.4"
+hickory-resolver = { version = "0.24", features = ["tokio-runtime", "serde-config"] }
 http = "1.0"
 http-body = "1.0"
 http-body-util = "0.1.2"

cli/worker.rs

@@ -547,6 +547,7 @@ impl CliMainWorkerFactory {
       npm_process_state_provider: Some(shared.npm_process_state_provider()),
       blob_store: shared.blob_store.clone(),
       broadcast_channel: shared.broadcast_channel.clone(),
+      fetch_dns_resolver: Default::default(),
       shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()),
       compiled_wasm_module_store: Some(
         shared.compiled_wasm_module_store.clone(),
@@ -855,6 +856,7 @@ mod tests {
         node_services: Default::default(),
         npm_process_state_provider: Default::default(),
         root_cert_store_provider: Default::default(),
+        fetch_dns_resolver: Default::default(),
         shared_array_buffer_store: Default::default(),
         compiled_wasm_module_store: Default::default(),
         v8_code_cache: Default::default(),

ext/fetch/Cargo.toml

@@ -22,6 +22,7 @@ deno_permissions.workspace = true
 deno_tls.workspace = true
 dyn-clone = "1"
 error_reporter = "1"
+hickory-resolver.workspace = true
 http.workspace = true
 http-body-util.workspace = true
 hyper.workspace = true

ext/fetch/dns.rs

@@ -0,0 +1,116 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+use std::future::Future;
+use std::io;
+use std::net::SocketAddr;
+use std::pin::Pin;
+use std::task::Poll;
+use std::task::{self};
+use std::vec;
+
+use hickory_resolver::error::ResolveError;
+use hickory_resolver::name_server::GenericConnector;
+use hickory_resolver::name_server::TokioRuntimeProvider;
+use hickory_resolver::AsyncResolver;
+use hyper_util::client::legacy::connect::dns::GaiResolver;
+use hyper_util::client::legacy::connect::dns::Name;
+use tokio::task::JoinHandle;
+use tower::Service;
+
+#[derive(Clone, Debug)]
+pub enum Resolver {
+  /// A resolver using blocking `getaddrinfo` calls in a threadpool.
+  Gai(GaiResolver),
+  /// hickory-resolver's userspace resolver.
+  Hickory(AsyncResolver<GenericConnector<TokioRuntimeProvider>>),
+}
+
+impl Default for Resolver {
+  fn default() -> Self {
+    Self::gai()
+  }
+}
+
+impl Resolver {
+  pub fn gai() -> Self {
+    Self::Gai(GaiResolver::new())
+  }
+
+  /// Create a [`AsyncResolver`] from system conf.
+  pub fn hickory() -> Result<Self, ResolveError> {
+    Ok(Self::Hickory(
+      hickory_resolver::AsyncResolver::tokio_from_system_conf()?,
+    ))
+  }
+
+  pub fn hickory_from_async_resolver(
+    resolver: AsyncResolver<GenericConnector<TokioRuntimeProvider>>,
+  ) -> Self {
+    Self::Hickory(resolver)
+  }
+}
+
+type SocketAddrs = vec::IntoIter<SocketAddr>;
+
+pub struct ResolveFut {
+  inner: JoinHandle<Result<SocketAddrs, io::Error>>,
+}
+
+impl Future for ResolveFut {
+  type Output = Result<SocketAddrs, io::Error>;
+
+  fn poll(
+    mut self: Pin<&mut Self>,
+    cx: &mut task::Context<'_>,
+  ) -> Poll<Self::Output> {
+    Pin::new(&mut self.inner).poll(cx).map(|res| match res {
+      Ok(Ok(addrs)) => Ok(addrs),
+      Ok(Err(e)) => Err(e),
+      Err(join_err) => {
+        if join_err.is_cancelled() {
+          Err(io::Error::new(io::ErrorKind::Interrupted, join_err))
+        } else {
+          Err(io::Error::new(io::ErrorKind::Other, join_err))
+        }
+      }
+    })
+  }
+}
+
+impl Service<Name> for Resolver {
+  type Response = SocketAddrs;
+  type Error = io::Error;
+  type Future = ResolveFut;
+
+  fn poll_ready(
+    &mut self,
+    _cx: &mut task::Context<'_>,
+  ) -> Poll<Result<(), io::Error>> {
+    Poll::Ready(Ok(()))
+  }
+
+  fn call(&mut self, name: Name) -> Self::Future {
+    let task = match self {
+      Resolver::Gai(gai_resolver) => {
+        let mut resolver = gai_resolver.clone();
+        tokio::spawn(async move {
+          let result = resolver.call(name).await?;
+          let x: Vec<_> = result.into_iter().collect();
+          let iter: SocketAddrs = x.into_iter();
+          Ok(iter)
+        })
+      }
+      Resolver::Hickory(async_resolver) => {
+        let resolver = async_resolver.clone();
+        tokio::spawn(async move {
+          let result = resolver.lookup_ip(name.as_str()).await?;
+
+          let x: Vec<_> =
+            result.into_iter().map(|x| SocketAddr::new(x, 0)).collect();
+          let iter: SocketAddrs = x.into_iter();
+          Ok(iter)
+        })
+      }
+    };
+    ResolveFut { inner: task }
+  }
+}

ext/fetch/lib.rs

@@ -1,5 +1,6 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
+pub mod dns;
 mod fs_fetch_handler;
 mod proxy;
 #[cfg(test)]
@@ -91,6 +92,7 @@ pub struct Options {
   pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
   pub client_cert_chain_and_key: TlsKeys,
   pub file_fetch_handler: Rc<dyn FetchHandler>,
+  pub resolver: dns::Resolver,
 }
 
 impl Options {
@@ -114,6 +116,7 @@ impl Default for Options {
       unsafely_ignore_certificate_errors: None,
       client_cert_chain_and_key: TlsKeys::Null,
       file_fetch_handler: Rc::new(DefaultFileFetchHandler),
+      resolver: dns::Resolver::default(),
     }
   }
 }
@@ -255,6 +258,7 @@ pub fn create_client_from_options(
         .map_err(HttpClientCreateError::RootCertStore)?,
       ca_certs: vec![],
       proxy: options.proxy.clone(),
+      dns_resolver: options.resolver.clone(),
       unsafely_ignore_certificate_errors: options
         .unsafely_ignore_certificate_errors
         .clone(),
@@ -835,6 +839,8 @@ pub struct CreateHttpClientArgs {
   proxy: Option<Proxy>,
   pool_max_idle_per_host: Option<usize>,
   pool_idle_timeout: Option<serde_json::Value>,
+  #[serde(default)]
+  use_hickory_resolver: bool,
   #[serde(default = "default_true")]
   http1: bool,
   #[serde(default = "default_true")]
@@ -878,6 +884,13 @@ where
         .map_err(HttpClientCreateError::RootCertStore)?,
       ca_certs,
       proxy: args.proxy,
+      dns_resolver: if args.use_hickory_resolver {
+        dns::Resolver::hickory()
+          .map_err(deno_core::error::AnyError::new)
+          .map_err(FetchError::Resource)?
+      } else {
+        dns::Resolver::default()
+      },
       unsafely_ignore_certificate_errors: options
         .unsafely_ignore_certificate_errors
         .clone(),
@@ -909,6 +922,7 @@ pub struct CreateHttpClientOptions {
   pub root_cert_store: Option<RootCertStore>,
   pub ca_certs: Vec<Vec<u8>>,
   pub proxy: Option<Proxy>,
+  pub dns_resolver: dns::Resolver,
   pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
   pub client_cert_chain_and_key: Option<TlsKey>,
   pub pool_max_idle_per_host: Option<usize>,
@@ -923,6 +937,7 @@ impl Default for CreateHttpClientOptions {
       root_cert_store: None,
       ca_certs: vec![],
       proxy: None,
+      dns_resolver: dns::Resolver::default(),
       unsafely_ignore_certificate_errors: None,
       client_cert_chain_and_key: None,
       pool_max_idle_per_host: None,
@@ -976,7 +991,8 @@ pub fn create_http_client(
   tls_config.alpn_protocols = alpn_protocols;
   let tls_config = Arc::from(tls_config);
 
-  let mut http_connector = HttpConnector::new();
+  let mut http_connector =
+    HttpConnector::new_with_resolver(options.dns_resolver.clone());
   http_connector.enforce_http(false);
 
   let user_agent = user_agent.parse::<HeaderValue>().map_err(|_| {
@@ -1051,7 +1067,7 @@ pub struct Client {
   user_agent: HeaderValue,
 }
 
-type Connector = proxy::ProxyConnector<HttpConnector>;
+type Connector = proxy::ProxyConnector<HttpConnector<dns::Resolver>>;
 
 // clippy is wrong here
 #[allow(clippy::declare_interior_mutable_const)]

ext/fetch/tests.rs

@@ -1,6 +1,8 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
 use std::net::SocketAddr;
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering::SeqCst;
 use std::sync::Arc;
 
 use bytes::Bytes;
@@ -10,6 +12,8 @@ use http_body_util::BodyExt;
 use tokio::io::AsyncReadExt;
 use tokio::io::AsyncWriteExt;
 
+use crate::dns;
+
 use super::create_http_client;
 use super::CreateHttpClientOptions;
 
@@ -17,6 +21,53 @@ static EXAMPLE_CRT: &[u8] = include_bytes!("../tls/testdata/example1_cert.der");
 static EXAMPLE_KEY: &[u8] =
   include_bytes!("../tls/testdata/example1_prikey.der");
 
+#[test]
+fn test_userspace_resolver() {
+  let thread_counter = Arc::new(AtomicUsize::new(0));
+
+  let thread_counter_ref = thread_counter.clone();
+  let rt = tokio::runtime::Builder::new_current_thread()
+    .enable_all()
+    .on_thread_start(move || {
+      thread_counter_ref.fetch_add(1, SeqCst);
+    })
+    .build()
+    .unwrap();
+
+  rt.block_on(async move {
+    assert_eq!(thread_counter.load(SeqCst), 0);
+    let src_addr = create_https_server(true).await;
+    assert_eq!(src_addr.ip().to_string(), "127.0.0.1");
+    // use `localhost` to ensure dns step happens.
+    let addr = format!("localhost:{}", src_addr.port());
+
+    let hickory = hickory_resolver::AsyncResolver::tokio(
+      Default::default(),
+      Default::default(),
+    );
+
+    assert_eq!(thread_counter.load(SeqCst), 0);
+    rust_test_client_with_resolver(
+      None,
+      addr.clone(),
+      "https",
+      http::Version::HTTP_2,
+      dns::Resolver::hickory_from_async_resolver(hickory),
+    )
+    .await;
+    assert_eq!(thread_counter.load(SeqCst), 0, "userspace resolver shouldn't spawn new threads.");
+    rust_test_client_with_resolver(
+      None,
+      addr.clone(),
+      "https",
+      http::Version::HTTP_2,
+      dns::Resolver::gai(),
+    )
+    .await;
+    assert_eq!(thread_counter.load(SeqCst), 1, "getaddrinfo is called inside spawn_blocking, so tokio spawn a new worker thread for it.");
+  });
+}
+
 #[tokio::test]
 async fn test_https_proxy_http11() {
   let src_addr = create_https_server(false).await;
@@ -52,25 +103,27 @@ async fn test_socks_proxy_h2() {
   run_test_client(prx_addr, src_addr, "socks5", http::Version::HTTP_2).await;
 }
 
-async fn run_test_client(
+async fn rust_test_client_with_resolver(
-  prx_addr: SocketAddr,
+  prx_addr: Option<SocketAddr>,
-  src_addr: SocketAddr,
+  src_addr: String,
   proto: &str,
   ver: http::Version,
+  resolver: dns::Resolver,
 ) {
   let client = create_http_client(
     "fetch/test",
     CreateHttpClientOptions {
       root_cert_store: None,
       ca_certs: vec![],
-      proxy: Some(deno_tls::Proxy {
+      proxy: prx_addr.map(|p| deno_tls::Proxy {
-        url: format!("{}://{}", proto, prx_addr),
+        url: format!("{}://{}", proto, p),
         basic_auth: None,
       }),
       unsafely_ignore_certificate_errors: Some(vec![]),
       client_cert_chain_and_key: None,
       pool_max_idle_per_host: None,
       pool_idle_timeout: None,
+      dns_resolver: resolver,
       http1: true,
       http2: true,
     },
@@ -92,6 +145,22 @@ async fn run_test_client(
   assert_eq!(hello, "hello from server");
 }
 
+async fn run_test_client(
+  prx_addr: SocketAddr,
+  src_addr: SocketAddr,
+  proto: &str,
+  ver: http::Version,
+) {
+  rust_test_client_with_resolver(
+    Some(prx_addr),
+    src_addr.to_string(),
+    proto,
+    ver,
+    Default::default(),
+  )
+  .await
+}
+
 async fn create_https_server(allow_h2: bool) -> SocketAddr {
   let mut tls_config = deno_tls::rustls::server::ServerConfig::builder()
     .with_no_client_auth()

ext/kv/remote.rs

@@ -197,6 +197,7 @@ impl<P: RemoteDbHandlerPermissions + 'static> DatabaseHandler
         root_cert_store: options.root_cert_store()?,
         ca_certs: vec![],
         proxy: options.proxy.clone(),
+        dns_resolver: Default::default(),
         unsafely_ignore_certificate_errors: options
           .unsafely_ignore_certificate_errors
           .clone(),

ext/net/Cargo.toml

@@ -18,7 +18,7 @@ deno_core.workspace = true
 deno_permissions.workspace = true
 deno_tls.workspace = true
 hickory-proto = "0.24"
-hickory-resolver = { version = "0.24", features = ["tokio-runtime", "serde-config"] }
+hickory-resolver.workspace = true
 pin-project.workspace = true
 rustls-tokio-stream.workspace = true
 serde.workspace = true

runtime/examples/extension/main.rs

@@ -50,6 +50,7 @@ async fn main() -> Result<(), AnyError> {
       node_services: Default::default(),
       npm_process_state_provider: Default::default(),
       root_cert_store_provider: Default::default(),
+      fetch_dns_resolver: Default::default(),
       shared_array_buffer_store: Default::default(),
       compiled_wasm_module_store: Default::default(),
       v8_code_cache: Default::default(),

runtime/worker.rs

@@ -143,6 +143,7 @@ pub struct WorkerServiceOptions {
   pub npm_process_state_provider: Option<NpmProcessStateProviderRc>,
   pub permissions: PermissionsContainer,
   pub root_cert_store_provider: Option<Arc<dyn RootCertStoreProvider>>,
+  pub fetch_dns_resolver: deno_fetch::dns::Resolver,
 
   /// The store to use for transferring SharedArrayBuffers between isolates.
   /// If multiple isolates should have the possibility of sharing
@@ -363,6 +364,7 @@ impl MainWorker {
             .unsafely_ignore_certificate_errors
             .clone(),
           file_fetch_handler: Rc::new(deno_fetch::FsFetchHandler),
+          resolver: services.fetch_dns_resolver,
           ..Default::default()
         },
       ),