feat(fetch): add programmatic proxy (#10907)

This commit adds new options to unstable "Deno.createHttpClient" API.

"proxy" and "basicAuth" options were added that allow to use custom proxy
when client instance is passed to "fetch" API.

cli/dts/lib.deno.unstable.d.ts

@@ -1089,6 +1089,17 @@ declare namespace Deno {
     /** A certificate authority to use when validating TLS certificates. Certificate data must be PEM encoded.
      */
     caData?: string;
+    proxy?: Proxy;
+  }
+
+  export interface Proxy {
+    url: string;
+    basicAuth?: BasicAuth;
+  }
+
+  export interface BasicAuth {
+    username: string;
+    password: string;
   }
 
   /** **UNSTABLE**: New API, yet to be vetted.
@@ -1096,7 +1107,12 @@ declare namespace Deno {
    *
    * ```ts
    * const client = Deno.createHttpClient({ caData: await Deno.readTextFile("./ca.pem") });
-   * const req = await fetch("https://myserver.com", { client });
+   * const response = await fetch("https://myserver.com", { client });
+   * ```
+   *
+   * ```ts
+   * const client = Deno.createHttpClient({ proxy: { url: "http://myproxy.com:8080" } });
+   * const response = await fetch("https://myserver.com", { client });
    * ```
    */
   export function createHttpClient(

cli/tests/045_programmatic_proxy_client.ts

@@ -0,0 +1,16 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+const client = Deno.createHttpClient({
+  proxy: {
+    url: "http://localhost:4555",
+    basicAuth: { username: "username", password: "password" },
+  },
+});
+
+const res = await fetch(
+  "http://localhost:4545/test_util/std/examples/colors.ts",
+  { client },
+);
+console.log(`Response http: ${await res.text()}`);
+
+client.close();

cli/tests/045_proxy_test.ts

@@ -15,6 +15,11 @@ async function proxyServer(): Promise<void> {
 
 async function proxyRequest(req: ServerRequest): Promise<void> {
   console.log(`Proxy request to: ${req.url}`);
+  const proxyAuthorization = req.headers.get("proxy-authorization");
+  if (proxyAuthorization) {
+    console.log(`proxy-authorization: ${proxyAuthorization}`);
+    req.headers.delete("proxy-authorization");
+  }
   const resp = await fetch(req.url, {
     method: req.method,
     headers: req.headers,
@@ -110,9 +115,28 @@ async function testModuleDownloadNoProxy(): Promise<void> {
   http.close();
 }
 
+async function testFetchProgrammaticProxy(): Promise<void> {
+  const c = Deno.run({
+    cmd: [
+      Deno.execPath(),
+      "run",
+      "--quiet",
+      "--reload",
+      "--allow-net=localhost:4545,localhost:4555",
+      "--unstable",
+      "045_programmatic_proxy_client.ts",
+    ],
+    stdout: "piped",
+  });
+  const status = await c.status();
+  assertEquals(status.code, 0);
+  c.close();
+}
+
 proxyServer();
 await testFetch();
 await testModuleDownload();
 await testFetchNoProxy();
 await testModuleDownloadNoProxy();
+await testFetchProgrammaticProxy();
 Deno.exit(0);

cli/tests/045_proxy_test.ts.out

@@ -2,3 +2,5 @@ Proxy server listening on [WILDCARD]
 Proxy request to: http://localhost:4545/test_util/std/examples/colors.ts
 Proxy request to: http://localhost:4545/test_util/std/examples/colors.ts
 Proxy request to: http://localhost:4545/test_util/std/fmt/colors.ts
+Proxy request to: http://localhost:4545/test_util/std/examples/colors.ts
+proxy-authorization: Basic dXNlcm5hbWU6cGFzc3dvcmQ=

extensions/fetch/lib.rs

@@ -56,6 +56,7 @@ pub use reqwest; // Re-export reqwest
 pub fn init<P: FetchPermissions + 'static>(
   user_agent: String,
   ca_data: Option<Vec<u8>>,
+  proxy: Option<Proxy>,
 ) -> Extension {
   Extension::builder()
     .js(include_js_files!(
@@ -78,11 +79,13 @@ pub fn init<P: FetchPermissions + 'static>(
     ])
     .state(move |state| {
       state.put::<reqwest::Client>({
-        create_http_client(user_agent.clone(), ca_data.clone()).unwrap()
+        create_http_client(user_agent.clone(), ca_data.clone(), proxy.clone())
+          .unwrap()
       });
       state.put::<HttpClientDefaults>(HttpClientDefaults {
         ca_data: ca_data.clone(),
         user_agent: user_agent.clone(),
+        proxy: proxy.clone(),
       });
       Ok(())
     })
@@ -92,6 +95,7 @@ pub fn init<P: FetchPermissions + 'static>(
 pub struct HttpClientDefaults {
   pub user_agent: String,
   pub ca_data: Option<Vec<u8>>,
+  pub proxy: Option<Proxy>,
 }
 
 pub trait FetchPermissions {
@@ -461,6 +465,22 @@ impl HttpClientResource {
 pub struct CreateHttpClientOptions {
   ca_file: Option<String>,
   ca_data: Option<String>,
+  proxy: Option<Proxy>,
+}
+
+#[derive(Deserialize, Default, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+#[serde(default)]
+pub struct Proxy {
+  pub url: String,
+  pub basic_auth: Option<BasicAuth>,
+}
+
+#[derive(Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct BasicAuth {
+  pub username: String,
+  pub password: String,
 }
 
 pub fn op_create_http_client<FP>(
@@ -476,6 +496,12 @@ where
     permissions.check_read(&PathBuf::from(ca_file))?;
   }
 
+  if let Some(proxy) = args.proxy.clone() {
+    let permissions = state.borrow_mut::<FP>();
+    let url = Url::parse(&proxy.url)?;
+    permissions.check_net_url(&url)?;
+  }
+
   let defaults = state.borrow::<HttpClientDefaults>();
 
   let cert_data =
@@ -483,6 +509,7 @@ where
   let client = create_http_client(
     defaults.user_agent.clone(),
     cert_data.or_else(|| defaults.ca_data.clone()),
+    args.proxy,
   )
   .unwrap();
 
@@ -510,6 +537,7 @@ fn get_cert_data(
 pub fn create_http_client(
   user_agent: String,
   ca_data: Option<Vec<u8>>,
+  proxy: Option<Proxy>,
 ) -> Result<Client, AnyError> {
   let mut headers = HeaderMap::new();
   headers.insert(USER_AGENT, user_agent.parse().unwrap());
@@ -523,6 +551,15 @@ pub fn create_http_client(
     builder = builder.add_root_certificate(cert);
   }
 
+  if let Some(proxy) = proxy {
+    let mut reqwest_proxy = reqwest::Proxy::all(&proxy.url)?;
+    if let Some(basic_auth) = &proxy.basic_auth {
+      reqwest_proxy =
+        reqwest_proxy.basic_auth(&basic_auth.username, &basic_auth.password);
+    }
+    builder = builder.proxy(reqwest_proxy);
+  }
+
   builder
     .build()
     .map_err(|e| generic_error(format!("Unable to build http client: {}", e)))

runtime/build.rs

@@ -42,7 +42,11 @@ fn create_runtime_snapshot(snapshot_path: &Path, files: Vec<PathBuf>) {
     deno_console::init(),
     deno_url::init(),
     deno_web::init(Default::default(), Default::default()),
-    deno_fetch::init::<deno_fetch::NoFetchPermissions>("".to_owned(), None),
+    deno_fetch::init::<deno_fetch::NoFetchPermissions>(
+      "".to_owned(),
+      None,
+      None,
+    ),
     deno_websocket::init::<deno_websocket::NoWebSocketPermissions>(
       "".to_owned(),
       None,

runtime/web_worker.rs

@@ -257,6 +257,7 @@ impl WebWorker {
       deno_fetch::init::<Permissions>(
         options.user_agent.clone(),
         options.ca_data.clone(),
+        None,
       ),
       deno_websocket::init::<Permissions>(
         options.user_agent.clone(),

runtime/worker.rs

@@ -99,6 +99,7 @@ impl MainWorker {
       deno_fetch::init::<Permissions>(
         options.user_agent.clone(),
         options.ca_data.clone(),
+        None,
       ),
       deno_websocket::init::<Permissions>(
         options.user_agent.clone(),