fix(tls): throw meaningful error when hostname is invalid (#10387)

`InvalidDNSNameError` is thrown when a string is not a valid hostname,
e.g. it contains invalid characters, or starts with a numeric digit. It
does not involve a (failed) DNS lookup.

cli/tests/unit/tls_test.ts

@@ -32,7 +32,7 @@ unitTest(
 
     await assertThrowsAsync(async () => {
       await Deno.connectTls({ hostname: "127.0.0.1", port: 3567 });
-    }, Error);
+    }, TypeError);
 
     listener.close();
   },

core/error.rs

@@ -36,6 +36,10 @@ pub fn type_error(message: impl Into<Cow<'static, str>>) -> AnyError {
   custom_error("TypeError", message)
 }
 
+pub fn invalid_hostname(hostname: &str) -> AnyError {
+  type_error(format!("Invalid hostname: '{}'", hostname))
+}
+
 pub fn uri_error(message: impl Into<Cow<'static, str>>) -> AnyError {
   custom_error("URIError", message)
 }

op_crates/websocket/lib.rs

@@ -1,6 +1,7 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
 use deno_core::error::bad_resource_id;
+use deno_core::error::invalid_hostname;
 use deno_core::error::null_opbuf;
 use deno_core::error::type_error;
 use deno_core::error::AnyError;
@@ -175,8 +176,8 @@ where
       }
 
       let tls_connector = TlsConnector::from(Arc::new(config));
-      let dnsname =
+      let dnsname = DNSNameRef::try_from_ascii_str(domain)
-        DNSNameRef::try_from_ascii_str(&domain).expect("Invalid DNS lookup");
+        .map_err(|_| invalid_hostname(domain))?;
       let tls_socket = tls_connector.connect(dnsname, tcp_socket).await?;
       MaybeTlsStream::Rustls(tls_socket)
     }

runtime/ops/tls.rs

@@ -13,6 +13,7 @@ use deno_core::error::bad_resource;
 use deno_core::error::bad_resource_id;
 use deno_core::error::custom_error;
 use deno_core::error::generic_error;
+use deno_core::error::invalid_hostname;
 use deno_core::error::AnyError;
 use deno_core::AsyncRefCell;
 use deno_core::CancelHandle;
@@ -139,8 +140,8 @@ async fn op_start_tls(
   }
 
   let tls_connector = TlsConnector::from(Arc::new(config));
-  let dnsname = DNSNameRef::try_from_ascii_str(&domain)
+  let dnsname = DNSNameRef::try_from_ascii_str(domain)
-    .map_err(|_| generic_error("Invalid DNS lookup"))?;
+    .map_err(|_| invalid_hostname(domain))?;
   let tls_stream = tls_connector.connect(dnsname, tcp_stream).await?;
 
   let rid = {
@@ -169,20 +170,22 @@ async fn op_connect_tls(
 ) -> Result<OpConn, AnyError> {
   assert_eq!(args.transport, "tcp");
 
-  {
-    let mut s = state.borrow_mut();
-    let permissions = s.borrow_mut::<Permissions>();
-    permissions.net.check(&(&args.hostname, Some(args.port)))?;
-    if let Some(path) = &args.cert_file {
-      permissions.read.check(Path::new(&path))?;
-    }
-  }
   let mut domain = args.hostname.as_str();
   if domain.is_empty() {
     domain = "localhost";
   }
+  {
+    let mut s = state.borrow_mut();
+    let permissions = s.borrow_mut::<Permissions>();
+    permissions.net.check(&(domain, Some(args.port)))?;
+    if let Some(path) = &args.cert_file {
+      permissions.read.check(Path::new(&path))?;
+    }
+  }
 
-  let addr = resolve_addr(&args.hostname, args.port)
+  let dnsname = DNSNameRef::try_from_ascii_str(domain)
+    .map_err(|_| invalid_hostname(domain))?;
+  let addr = resolve_addr(domain, args.port)
     .await?
     .next()
     .ok_or_else(|| generic_error("No resolved address found"))?;
@@ -200,8 +203,6 @@ async fn op_connect_tls(
     config.root_store.add_pem_file(reader).unwrap();
   }
   let tls_connector = TlsConnector::from(Arc::new(config));
-  let dnsname = DNSNameRef::try_from_ascii_str(&domain)
-    .map_err(|_| generic_error("Invalid DNS lookup"))?;
   let tls_stream = tls_connector.connect(dnsname, tcp_stream).await?;
   let rid = {
     let mut state_ = state.borrow_mut();