From 032ae7fb19bd01c1de28515facd5c3b2ce821924 Mon Sep 17 00:00:00 2001 From: Sahand Akbarzadeh Date: Fri, 15 Nov 2024 14:14:11 +0330 Subject: [PATCH] feat(ext/fetch): allow embedders to use `hickory_dns_resolver` instead of default `GaiResolver` (#26740) Allows embedders to use `hickory-dns-resolver` instead of threaded "getaddrinfo" resolver in the `fetch()` implementation. --- Cargo.lock | 1 + Cargo.toml | 1 + cli/worker.rs | 2 + ext/fetch/Cargo.toml | 1 + ext/fetch/dns.rs | 116 +++++++++++++++++++++++++++++ ext/fetch/lib.rs | 20 ++++- ext/fetch/tests.rs | 79 ++++++++++++++++++-- ext/kv/remote.rs | 1 + ext/net/Cargo.toml | 2 +- runtime/examples/extension/main.rs | 1 + runtime/worker.rs | 2 + 11 files changed, 218 insertions(+), 8 deletions(-) create mode 100644 ext/fetch/dns.rs diff --git a/Cargo.lock b/Cargo.lock index 87265c02d8..fc5834da53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1566,6 +1566,7 @@ dependencies = [ "dyn-clone", "error_reporter", "fast-socks5", + "hickory-resolver", "http 1.1.0", "http-body-util", "hyper 1.4.1", diff --git a/Cargo.toml b/Cargo.toml index 4a78e7e466..36e59eab2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -127,6 +127,7 @@ fs3 = "0.5.0" futures = "0.3.21" glob = "0.3.1" h2 = "0.4.4" +hickory-resolver = { version = "0.24", features = ["tokio-runtime", "serde-config"] } http = "1.0" http-body = "1.0" http-body-util = "0.1.2" diff --git a/cli/worker.rs b/cli/worker.rs index c6cbf77f1b..24397b6bf0 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -547,6 +547,7 @@ impl CliMainWorkerFactory { npm_process_state_provider: Some(shared.npm_process_state_provider()), blob_store: shared.blob_store.clone(), broadcast_channel: shared.broadcast_channel.clone(), + fetch_dns_resolver: Default::default(), shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), compiled_wasm_module_store: Some( shared.compiled_wasm_module_store.clone(), @@ -855,6 +856,7 @@ mod tests { node_services: Default::default(), npm_process_state_provider: Default::default(), root_cert_store_provider: Default::default(), + fetch_dns_resolver: Default::default(), shared_array_buffer_store: Default::default(), compiled_wasm_module_store: Default::default(), v8_code_cache: Default::default(), diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml index 56d416bbb8..00c85f2aa1 100644 --- a/ext/fetch/Cargo.toml +++ b/ext/fetch/Cargo.toml @@ -22,6 +22,7 @@ deno_permissions.workspace = true deno_tls.workspace = true dyn-clone = "1" error_reporter = "1" +hickory-resolver.workspace = true http.workspace = true http-body-util.workspace = true hyper.workspace = true diff --git a/ext/fetch/dns.rs b/ext/fetch/dns.rs new file mode 100644 index 0000000000..9e21a4c342 --- /dev/null +++ b/ext/fetch/dns.rs @@ -0,0 +1,116 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::future::Future; +use std::io; +use std::net::SocketAddr; +use std::pin::Pin; +use std::task::Poll; +use std::task::{self}; +use std::vec; + +use hickory_resolver::error::ResolveError; +use hickory_resolver::name_server::GenericConnector; +use hickory_resolver::name_server::TokioRuntimeProvider; +use hickory_resolver::AsyncResolver; +use hyper_util::client::legacy::connect::dns::GaiResolver; +use hyper_util::client::legacy::connect::dns::Name; +use tokio::task::JoinHandle; +use tower::Service; + +#[derive(Clone, Debug)] +pub enum Resolver { + /// A resolver using blocking `getaddrinfo` calls in a threadpool. + Gai(GaiResolver), + /// hickory-resolver's userspace resolver. + Hickory(AsyncResolver>), +} + +impl Default for Resolver { + fn default() -> Self { + Self::gai() + } +} + +impl Resolver { + pub fn gai() -> Self { + Self::Gai(GaiResolver::new()) + } + + /// Create a [`AsyncResolver`] from system conf. + pub fn hickory() -> Result { + Ok(Self::Hickory( + hickory_resolver::AsyncResolver::tokio_from_system_conf()?, + )) + } + + pub fn hickory_from_async_resolver( + resolver: AsyncResolver>, + ) -> Self { + Self::Hickory(resolver) + } +} + +type SocketAddrs = vec::IntoIter; + +pub struct ResolveFut { + inner: JoinHandle>, +} + +impl Future for ResolveFut { + type Output = Result; + + fn poll( + mut self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> Poll { + Pin::new(&mut self.inner).poll(cx).map(|res| match res { + Ok(Ok(addrs)) => Ok(addrs), + Ok(Err(e)) => Err(e), + Err(join_err) => { + if join_err.is_cancelled() { + Err(io::Error::new(io::ErrorKind::Interrupted, join_err)) + } else { + Err(io::Error::new(io::ErrorKind::Other, join_err)) + } + } + }) + } +} + +impl Service for Resolver { + type Response = SocketAddrs; + type Error = io::Error; + type Future = ResolveFut; + + fn poll_ready( + &mut self, + _cx: &mut task::Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, name: Name) -> Self::Future { + let task = match self { + Resolver::Gai(gai_resolver) => { + let mut resolver = gai_resolver.clone(); + tokio::spawn(async move { + let result = resolver.call(name).await?; + let x: Vec<_> = result.into_iter().collect(); + let iter: SocketAddrs = x.into_iter(); + Ok(iter) + }) + } + Resolver::Hickory(async_resolver) => { + let resolver = async_resolver.clone(); + tokio::spawn(async move { + let result = resolver.lookup_ip(name.as_str()).await?; + + let x: Vec<_> = + result.into_iter().map(|x| SocketAddr::new(x, 0)).collect(); + let iter: SocketAddrs = x.into_iter(); + Ok(iter) + }) + } + }; + ResolveFut { inner: task } + } +} diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index 7ef26431c2..5949f9f75f 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +pub mod dns; mod fs_fetch_handler; mod proxy; #[cfg(test)] @@ -91,6 +92,7 @@ pub struct Options { pub unsafely_ignore_certificate_errors: Option>, pub client_cert_chain_and_key: TlsKeys, pub file_fetch_handler: Rc, + pub resolver: dns::Resolver, } impl Options { @@ -114,6 +116,7 @@ impl Default for Options { unsafely_ignore_certificate_errors: None, client_cert_chain_and_key: TlsKeys::Null, file_fetch_handler: Rc::new(DefaultFileFetchHandler), + resolver: dns::Resolver::default(), } } } @@ -255,6 +258,7 @@ pub fn create_client_from_options( .map_err(HttpClientCreateError::RootCertStore)?, ca_certs: vec![], proxy: options.proxy.clone(), + dns_resolver: options.resolver.clone(), unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -835,6 +839,8 @@ pub struct CreateHttpClientArgs { proxy: Option, pool_max_idle_per_host: Option, pool_idle_timeout: Option, + #[serde(default)] + use_hickory_resolver: bool, #[serde(default = "default_true")] http1: bool, #[serde(default = "default_true")] @@ -878,6 +884,13 @@ where .map_err(HttpClientCreateError::RootCertStore)?, ca_certs, proxy: args.proxy, + dns_resolver: if args.use_hickory_resolver { + dns::Resolver::hickory() + .map_err(deno_core::error::AnyError::new) + .map_err(FetchError::Resource)? + } else { + dns::Resolver::default() + }, unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -909,6 +922,7 @@ pub struct CreateHttpClientOptions { pub root_cert_store: Option, pub ca_certs: Vec>, pub proxy: Option, + pub dns_resolver: dns::Resolver, pub unsafely_ignore_certificate_errors: Option>, pub client_cert_chain_and_key: Option, pub pool_max_idle_per_host: Option, @@ -923,6 +937,7 @@ impl Default for CreateHttpClientOptions { root_cert_store: None, ca_certs: vec![], proxy: None, + dns_resolver: dns::Resolver::default(), unsafely_ignore_certificate_errors: None, client_cert_chain_and_key: None, pool_max_idle_per_host: None, @@ -976,7 +991,8 @@ pub fn create_http_client( tls_config.alpn_protocols = alpn_protocols; let tls_config = Arc::from(tls_config); - let mut http_connector = HttpConnector::new(); + let mut http_connector = + HttpConnector::new_with_resolver(options.dns_resolver.clone()); http_connector.enforce_http(false); let user_agent = user_agent.parse::().map_err(|_| { @@ -1051,7 +1067,7 @@ pub struct Client { user_agent: HeaderValue, } -type Connector = proxy::ProxyConnector; +type Connector = proxy::ProxyConnector>; // clippy is wrong here #[allow(clippy::declare_interior_mutable_const)] diff --git a/ext/fetch/tests.rs b/ext/fetch/tests.rs index dad1b34a9e..5cd1a35a5e 100644 --- a/ext/fetch/tests.rs +++ b/ext/fetch/tests.rs @@ -1,6 +1,8 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::net::SocketAddr; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::SeqCst; use std::sync::Arc; use bytes::Bytes; @@ -10,6 +12,8 @@ use http_body_util::BodyExt; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; +use crate::dns; + use super::create_http_client; use super::CreateHttpClientOptions; @@ -17,6 +21,53 @@ static EXAMPLE_CRT: &[u8] = include_bytes!("../tls/testdata/example1_cert.der"); static EXAMPLE_KEY: &[u8] = include_bytes!("../tls/testdata/example1_prikey.der"); +#[test] +fn test_userspace_resolver() { + let thread_counter = Arc::new(AtomicUsize::new(0)); + + let thread_counter_ref = thread_counter.clone(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .on_thread_start(move || { + thread_counter_ref.fetch_add(1, SeqCst); + }) + .build() + .unwrap(); + + rt.block_on(async move { + assert_eq!(thread_counter.load(SeqCst), 0); + let src_addr = create_https_server(true).await; + assert_eq!(src_addr.ip().to_string(), "127.0.0.1"); + // use `localhost` to ensure dns step happens. + let addr = format!("localhost:{}", src_addr.port()); + + let hickory = hickory_resolver::AsyncResolver::tokio( + Default::default(), + Default::default(), + ); + + assert_eq!(thread_counter.load(SeqCst), 0); + rust_test_client_with_resolver( + None, + addr.clone(), + "https", + http::Version::HTTP_2, + dns::Resolver::hickory_from_async_resolver(hickory), + ) + .await; + assert_eq!(thread_counter.load(SeqCst), 0, "userspace resolver shouldn't spawn new threads."); + rust_test_client_with_resolver( + None, + addr.clone(), + "https", + http::Version::HTTP_2, + dns::Resolver::gai(), + ) + .await; + assert_eq!(thread_counter.load(SeqCst), 1, "getaddrinfo is called inside spawn_blocking, so tokio spawn a new worker thread for it."); + }); +} + #[tokio::test] async fn test_https_proxy_http11() { let src_addr = create_https_server(false).await; @@ -52,25 +103,27 @@ async fn test_socks_proxy_h2() { run_test_client(prx_addr, src_addr, "socks5", http::Version::HTTP_2).await; } -async fn run_test_client( - prx_addr: SocketAddr, - src_addr: SocketAddr, +async fn rust_test_client_with_resolver( + prx_addr: Option, + src_addr: String, proto: &str, ver: http::Version, + resolver: dns::Resolver, ) { let client = create_http_client( "fetch/test", CreateHttpClientOptions { root_cert_store: None, ca_certs: vec![], - proxy: Some(deno_tls::Proxy { - url: format!("{}://{}", proto, prx_addr), + proxy: prx_addr.map(|p| deno_tls::Proxy { + url: format!("{}://{}", proto, p), basic_auth: None, }), unsafely_ignore_certificate_errors: Some(vec![]), client_cert_chain_and_key: None, pool_max_idle_per_host: None, pool_idle_timeout: None, + dns_resolver: resolver, http1: true, http2: true, }, @@ -92,6 +145,22 @@ async fn run_test_client( assert_eq!(hello, "hello from server"); } +async fn run_test_client( + prx_addr: SocketAddr, + src_addr: SocketAddr, + proto: &str, + ver: http::Version, +) { + rust_test_client_with_resolver( + Some(prx_addr), + src_addr.to_string(), + proto, + ver, + Default::default(), + ) + .await +} + async fn create_https_server(allow_h2: bool) -> SocketAddr { let mut tls_config = deno_tls::rustls::server::ServerConfig::builder() .with_no_client_auth() diff --git a/ext/kv/remote.rs b/ext/kv/remote.rs index 4930aacfe3..63146daf71 100644 --- a/ext/kv/remote.rs +++ b/ext/kv/remote.rs @@ -197,6 +197,7 @@ impl DatabaseHandler root_cert_store: options.root_cert_store()?, ca_certs: vec![], proxy: options.proxy.clone(), + dns_resolver: Default::default(), unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), diff --git a/ext/net/Cargo.toml b/ext/net/Cargo.toml index 245deedd2d..1febbd5338 100644 --- a/ext/net/Cargo.toml +++ b/ext/net/Cargo.toml @@ -18,7 +18,7 @@ deno_core.workspace = true deno_permissions.workspace = true deno_tls.workspace = true hickory-proto = "0.24" -hickory-resolver = { version = "0.24", features = ["tokio-runtime", "serde-config"] } +hickory-resolver.workspace = true pin-project.workspace = true rustls-tokio-stream.workspace = true serde.workspace = true diff --git a/runtime/examples/extension/main.rs b/runtime/examples/extension/main.rs index 9889b28dcf..1ff16ec83f 100644 --- a/runtime/examples/extension/main.rs +++ b/runtime/examples/extension/main.rs @@ -50,6 +50,7 @@ async fn main() -> Result<(), AnyError> { node_services: Default::default(), npm_process_state_provider: Default::default(), root_cert_store_provider: Default::default(), + fetch_dns_resolver: Default::default(), shared_array_buffer_store: Default::default(), compiled_wasm_module_store: Default::default(), v8_code_cache: Default::default(), diff --git a/runtime/worker.rs b/runtime/worker.rs index c7bfb1c5f3..99123463cf 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -143,6 +143,7 @@ pub struct WorkerServiceOptions { pub npm_process_state_provider: Option, pub permissions: PermissionsContainer, pub root_cert_store_provider: Option>, + pub fetch_dns_resolver: deno_fetch::dns::Resolver, /// The store to use for transferring SharedArrayBuffers between isolates. /// If multiple isolates should have the possibility of sharing @@ -363,6 +364,7 @@ impl MainWorker { .unsafely_ignore_certificate_errors .clone(), file_fetch_handler: Rc::new(deno_fetch::FsFetchHandler), + resolver: services.fetch_dns_resolver, ..Default::default() }, ),