Use 'reqwest' to implement HTTP client (#2822)

Closes #2720

cli/file_fetcher.rs

@@ -5,6 +5,7 @@ use crate::deno_error::ErrorKind;
 use crate::deno_error::GetErrorKind;
 use crate::disk_cache::DiskCache;
 use crate::http_util;
+use crate::http_util::FetchOnceResult;
 use crate::msg;
 use crate::progress::Progress;
 use crate::tokio_util;
@@ -12,7 +13,6 @@ use deno::ErrBox;
 use deno::ModuleSpecifier;
 use futures::future::Either;
 use futures::Future;
-use http;
 use serde_json;
 use std;
 use std::collections::HashMap;
@@ -21,7 +21,6 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::result::Result;
 use std::str;
-use std::str::FromStr;
 use std::sync::Arc;
 use std::sync::Mutex;
 use url;
@@ -305,8 +304,6 @@ impl SourceFileFetcher {
     no_remote_fetch: bool,
     redirect_limit: i64,
   ) -> Box<SourceFileFuture> {
-    use crate::http_util::FetchOnceResult;
-
     if redirect_limit < 0 {
       return Box::new(futures::future::err(too_many_redirects()));
     }
@@ -342,20 +339,14 @@ impl SourceFileFetcher {
     }
 
     let download_job = self.progress.add("Download", &module_url.to_string());
-
-    let module_uri = url_into_uri(&module_url);
-
     let dir = self.clone();
     let module_url = module_url.clone();
 
     // Single pass fetch, either yields code or yields redirect.
-    let f =
+    let f = http_util::fetch_string_once(&module_url).and_then(move |r| {
-      http_util::fetch_string_once(module_uri).and_then(move |r| match r {
+      match r {
-        FetchOnceResult::Redirect(uri) => {
+        FetchOnceResult::Redirect(new_module_url) => {
           // If redirects, update module_name and filename for next looped call.
-          let new_module_url = Url::parse(&uri.to_string())
-            .expect("http::uri::Uri should be parseable as Url");
-
           dir
             .save_source_code_headers(
               &module_url,
@@ -409,6 +400,7 @@ impl SourceFileFetcher {
 
           Either::B(futures::future::ok(source_file))
         }
+      }
     });
 
     Box::new(f)
@@ -539,11 +531,6 @@ fn filter_shebang(bytes: Vec<u8>) -> Vec<u8> {
   }
 }
 
-fn url_into_uri(url: &url::Url) -> http::uri::Uri {
-  http::uri::Uri::from_str(&url.to_string())
-    .expect("url::Url should be parseable as http::uri::Uri")
-}
-
 #[derive(Debug, Default)]
 /// Header metadata associated with a particular "symbolic" source code file.
 /// (the associated source code file might not be cached, while remaining

cli/http_body.rs

@@ -1,27 +1,27 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 
+use futures::stream::Stream;
 use futures::Async;
 use futures::Poll;
-use hyper::body::Payload;
+use reqwest::r#async::Chunk;
-use hyper::Body;
+use reqwest::r#async::Decoder;
-use hyper::Chunk;
 use std::cmp::min;
 use std::io;
 use std::io::Read;
 use tokio::io::AsyncRead;
 
-/// Wraps `hyper::Body` so that it can be exposed as an `AsyncRead` and integrated
+/// Wraps `reqwest::Decoder` so that it can be exposed as an `AsyncRead` and integrated
 /// into resources more easily.
 pub struct HttpBody {
-  body: Body,
+  decoder: Decoder,
   chunk: Option<Chunk>,
   pos: usize,
 }
 
 impl HttpBody {
-  pub fn from(body: Body) -> Self {
+  pub fn from(body: Decoder) -> Self {
     Self {
-      body,
+      decoder: body,
       chunk: None,
       pos: 0,
     }
@@ -59,7 +59,7 @@ impl AsyncRead for HttpBody {
       assert_eq!(self.pos, 0);
     }
 
-    let p = self.body.poll_data();
+    let p = self.decoder.poll();
     match p {
       Err(e) => Err(
         // TODO Need to map hyper::Error into std::io::Error.
@@ -87,26 +87,3 @@ impl AsyncRead for HttpBody {
     }
   }
 }
-
-#[test]
-fn test_body_async_read() {
-  use std::str::from_utf8;
-  let body = Body::from("hello world");
-  let mut body = HttpBody::from(body);
-
-  let buf = &mut [0, 0, 0, 0, 0];
-  let r = body.poll_read(buf);
-  assert!(r.is_ok());
-  assert_eq!(r.unwrap(), Async::Ready(5));
-  assert_eq!(from_utf8(buf).unwrap(), "hello");
-
-  let r = body.poll_read(buf);
-  assert!(r.is_ok());
-  assert_eq!(r.unwrap(), Async::Ready(5));
-  assert_eq!(from_utf8(buf).unwrap(), " worl");
-
-  let r = body.poll_read(buf);
-  assert!(r.is_ok());
-  assert_eq!(r.unwrap(), Async::Ready(1));
-  assert_eq!(from_utf8(&buf[0..1]).unwrap(), "d");
-}

cli/http_util.rs

@@ -2,76 +2,56 @@
 use crate::deno_error;
 use crate::deno_error::DenoError;
 use deno::ErrBox;
-#[cfg(test)]
+use futures::{future, Future};
-use futures::future::{loop_fn, Loop};
+use reqwest;
-use futures::{future, Future, Stream};
+use reqwest::header::CONTENT_TYPE;
-use hyper;
+use reqwest::header::LOCATION;
-use hyper::client::{Client, HttpConnector};
+use reqwest::r#async::Client;
-use hyper::header::CONTENT_TYPE;
+use reqwest::RedirectPolicy;
-use hyper::Uri;
+use url::Url;
-use hyper_rustls;
 
-type Connector = hyper_rustls::HttpsConnector<HttpConnector>;
+/// Create new instance of async reqwest::Client. This client supports
-
+/// proxies and doesn't follow redirects.
-lazy_static! {
+pub fn get_client() -> Client {
-  static ref CONNECTOR: Connector = {
+  Client::builder()
-    let num_dns_threads = 4;
+    .redirect(RedirectPolicy::none())
-    Connector::new(num_dns_threads)
+    .use_sys_proxy()
-  };
+    .build()
-}
+    .unwrap()
-
-pub fn get_client() -> Client<Connector, hyper::Body> {
-  // TODO use Hyper's connection pool.
-  let c = CONNECTOR.clone();
-  Client::builder().build(c)
 }
 
 /// Construct the next uri based on base uri and location header fragment
 /// See <https://tools.ietf.org/html/rfc3986#section-4.2>
-fn resolve_uri_from_location(base_uri: &Uri, location: &str) -> Uri {
+fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {
   if location.starts_with("http://") || location.starts_with("https://") {
     // absolute uri
-    location
+    Url::parse(location).expect("provided redirect url should be a valid url")
-      .parse::<Uri>()
-      .expect("provided redirect url should be a valid url")
   } else if location.starts_with("//") {
     // "//" authority path-abempty
-    format!("{}:{}", base_uri.scheme_part().unwrap().as_str(), location)
+    Url::parse(&format!("{}:{}", base_url.scheme(), location))
-      .parse::<Uri>()
       .expect("provided redirect url should be a valid url")
   } else if location.starts_with('/') {
     // path-absolute
-    let mut new_uri_parts = base_uri.clone().into_parts();
+    base_url
-    new_uri_parts.path_and_query = Some(location.parse().unwrap());
+      .join(location)
-    Uri::from_parts(new_uri_parts).unwrap()
+      .expect("provided redirect url should be a valid url")
   } else {
     // assuming path-noscheme | path-empty
-    let mut new_uri_parts = base_uri.clone().into_parts();
+    let base_url_path_str = base_url.path().to_owned();
-    let base_uri_path_str = base_uri.path().to_owned();
+    // Pop last part or url (after last slash)
-    let segs: Vec<&str> = base_uri_path_str.rsplitn(2, '/').collect();
+    let segs: Vec<&str> = base_url_path_str.rsplitn(2, '/').collect();
-    new_uri_parts.path_and_query = Some(
+    let new_path = format!("{}/{}", segs.last().unwrap_or(&""), location);
-      format!("{}/{}", segs.last().unwrap_or(&""), location)
+    base_url
-        .parse()
+      .join(&new_path)
-        .unwrap(),
+      .expect("provided redirect url should be a valid url")
-    );
-    Uri::from_parts(new_uri_parts).unwrap()
   }
 }
 
-#[cfg(test)]
+#[derive(Debug, PartialEq)]
-use crate::tokio_util;
-#[cfg(test)]
-/// Synchronously fetchs the given HTTP URL. Returns (content, media_type).
-pub fn fetch_sync_string(
-  module_name: &str,
-) -> Result<(String, String), ErrBox> {
-  tokio_util::block_on(fetch_string(module_name))
-}
-
 pub enum FetchOnceResult {
   // (code, maybe_content_type)
   Code(String, Option<String>),
-  Redirect(http::uri::Uri),
+  Redirect(Url),
 }
 
 /// Asynchronously fetchs the given HTTP URL one pass only.
@@ -80,51 +60,57 @@ pub enum FetchOnceResult {
 /// If redirect occurs, does not follow and
 /// yields Redirect(url).
 pub fn fetch_string_once(
-  url: http::uri::Uri,
+  url: &Url,
 ) -> impl Future<Item = FetchOnceResult, Error = ErrBox> {
   type FetchAttempt = (Option<String>, Option<String>, Option<FetchOnceResult>);
+
+  let url = url.clone();
   let client = get_client();
+
   client
     .get(url.clone())
+    .send()
     .map_err(ErrBox::from)
     .and_then(
-      move |response| -> Box<
+      move |mut response| -> Box<
         dyn Future<Item = FetchAttempt, Error = ErrBox> + Send,
       > {
         if response.status().is_redirection() {
-          let location_string = response
+          let location_string = response.headers()
-            .headers()
+            .get(LOCATION)
-            .get("location")
             .expect("url redirection should provide 'location' header")
             .to_str()
-            .unwrap()
+            .unwrap();
-            .to_string();
+
-          debug!("Redirecting to {}...", &location_string);
+          debug!("Redirecting to {:?}...", &location_string);
-          let new_url = resolve_uri_from_location(&url, &location_string);
+          let new_url = resolve_url_from_location(&url, location_string);
           // Boxed trait object turns out to be the savior for 2+ types yielding same results.
           return Box::new(future::ok(None).join3(
             future::ok(None),
             future::ok(Some(FetchOnceResult::Redirect(new_url))),
           ));
-        } else if response.status().is_client_error()
+        }
-          || response.status().is_server_error()
+
-        {
+        if response.status().is_client_error() || response.status().is_server_error() {
           return Box::new(future::err(DenoError::new(
             deno_error::ErrorKind::Other,
             format!("Import '{}' failed: {}", &url, response.status()),
           ).into()));
         }
+
         let content_type = response
           .headers()
           .get(CONTENT_TYPE)
           .map(|content_type| content_type.to_str().unwrap().to_owned());
+
         let body = response
-          .into_body()
+          .text()
-          .concat2()
-          .map(|body| String::from_utf8(body.to_vec()).ok())
           .map_err(ErrBox::from);
-        Box::new(body.join3(future::ok(content_type), future::ok(None)))
+
-      },
+        Box::new(
+          Some(body).join3(future::ok(content_type), future::ok(None))
+        )
+      }
     )
     .and_then(move |(maybe_code, maybe_content_type, maybe_redirect)| {
       if let Some(redirect) = maybe_redirect {
@@ -140,131 +126,74 @@ pub fn fetch_string_once(
 }
 
 #[cfg(test)]
-/// Asynchronously fetchs the given HTTP URL. Returns (content, media_type).
+mod tests {
-pub fn fetch_string(
+  use super::*;
-  module_name: &str,
+  use crate::tokio_util;
-) -> impl Future<Item = (String, String), Error = ErrBox> {
+
-  let url = module_name.parse::<Uri>().unwrap();
+  pub fn fetch_string_once_sync(url: &Url) -> Result<FetchOnceResult, ErrBox> {
-  let client = get_client();
+    tokio_util::block_on(fetch_string_once(url))
-  // TODO(kevinkassimo): consider set a max redirection counter
-  // to avoid bouncing between 2 or more urls
-  loop_fn((client, url), |(client, url)| {
-    client
-      .get(url.clone())
-      .map_err(ErrBox::from)
-      .and_then(move |response| {
-        if response.status().is_redirection() {
-          let location_string = response
-            .headers()
-            .get("location")
-            .expect("url redirection should provide 'location' header")
-            .to_str()
-            .unwrap()
-            .to_string();
-          debug!("Redirecting to {}...", &location_string);
-          let new_url = resolve_uri_from_location(&url, &location_string);
-          return Ok(Loop::Continue((client, new_url)));
-        }
-        if !response.status().is_success() {
-          return Err(
-            DenoError::new(
-              deno_error::ErrorKind::NotFound,
-              "module not found".to_string(),
-            )
-            .into(),
-          );
-        }
-        Ok(Loop::Break(response))
-      })
-  })
-  .and_then(|response| {
-    let content_type = response
-      .headers()
-      .get(CONTENT_TYPE)
-      .map(|content_type| content_type.to_str().unwrap().to_string());
-    let body = response
-      .into_body()
-      .concat2()
-      .map(|body| String::from_utf8(body.to_vec()).unwrap())
-      .map_err(ErrBox::from);
-    body.join(future::ok(content_type))
-  })
-  .and_then(|(body_string, maybe_content_type)| {
-    future::ok((body_string, maybe_content_type.unwrap()))
-  })
   }
 
   #[test]
   fn test_fetch_sync_string() {
     // Relies on external http server. See tools/http_server.py
-  tokio_util::init(|| {
+    let url = Url::parse("http://127.0.0.1:4545/package.json").unwrap();
-    let (p, m) =
+    tokio_util::init(|| match fetch_string_once_sync(&url).unwrap() {
-      fetch_sync_string("http://127.0.0.1:4545/package.json").unwrap();
+      FetchOnceResult::Code(code, maybe_content_type) => {
-    println!("package.json len {}", p.len());
+        assert!(!code.is_empty());
-    assert!(p.len() > 1);
+        assert_eq!(maybe_content_type, Some("application/json".to_string()));
-    assert!(m == "application/json")
+      }
+      _ => unreachable!(),
     });
   }
 
   #[test]
-fn test_fetch_string() {
+  fn test_fetch_string_once_with_redirect() {
     // Relies on external http server. See tools/http_server.py
+    let url = Url::parse("http://127.0.0.1:4546/package.json").unwrap();
+    // Dns resolver substitutes `127.0.0.1` with `localhost`
+    let target_url = Url::parse("http://localhost:4545/package.json").unwrap();
     tokio_util::init(|| {
-    let (p, m) = fetch_string("http://127.0.0.1:4545/package.json")
+      let result = fetch_string_once_sync(&url).unwrap();
-      .wait()
+      assert_eq!(result, FetchOnceResult::Redirect(target_url));
-      .unwrap();
-    println!("package.json len {}", p.len());
-    assert!(p.len() > 1);
-    assert!(m == "application/json")
     });
   }
 
   #[test]
-fn test_fetch_sync_string_with_redirect() {
+  fn test_resolve_url_from_location_full_1() {
-  // Relies on external http server. See tools/http_server.py
+    let url = "http://deno.land".parse::<Url>().unwrap();
-  tokio_util::init(|| {
+    let new_uri = resolve_url_from_location(&url, "http://golang.org");
-    let (p, m) =
+    assert_eq!(new_uri.host_str().unwrap(), "golang.org");
-      fetch_sync_string("http://127.0.0.1:4546/package.json").unwrap();
-    println!("package.json len {}", p.len());
-    assert!(p.len() > 1);
-    assert!(m == "application/json")
-  });
   }
 
   #[test]
-fn test_resolve_uri_from_location_full_1() {
+  fn test_resolve_url_from_location_full_2() {
-  let url = "http://deno.land".parse::<Uri>().unwrap();
+    let url = "https://deno.land".parse::<Url>().unwrap();
-  let new_uri = resolve_uri_from_location(&url, "http://golang.org");
+    let new_uri = resolve_url_from_location(&url, "https://golang.org");
-  assert_eq!(new_uri.host().unwrap(), "golang.org");
+    assert_eq!(new_uri.host_str().unwrap(), "golang.org");
   }
 
   #[test]
-fn test_resolve_uri_from_location_full_2() {
+  fn test_resolve_url_from_location_relative_1() {
-  let url = "https://deno.land".parse::<Uri>().unwrap();
+    let url = "http://deno.land/x".parse::<Url>().unwrap();
-  let new_uri = resolve_uri_from_location(&url, "https://golang.org");
+    let new_uri = resolve_url_from_location(&url, "//rust-lang.org/en-US");
-  assert_eq!(new_uri.host().unwrap(), "golang.org");
+    assert_eq!(new_uri.host_str().unwrap(), "rust-lang.org");
-}
-
-#[test]
-fn test_resolve_uri_from_location_relative_1() {
-  let url = "http://deno.land/x".parse::<Uri>().unwrap();
-  let new_uri = resolve_uri_from_location(&url, "//rust-lang.org/en-US");
-  assert_eq!(new_uri.host().unwrap(), "rust-lang.org");
     assert_eq!(new_uri.path(), "/en-US");
   }
 
   #[test]
-fn test_resolve_uri_from_location_relative_2() {
+  fn test_resolve_url_from_location_relative_2() {
-  let url = "http://deno.land/x".parse::<Uri>().unwrap();
+    let url = "http://deno.land/x".parse::<Url>().unwrap();
-  let new_uri = resolve_uri_from_location(&url, "/y");
+    let new_uri = resolve_url_from_location(&url, "/y");
-  assert_eq!(new_uri.host().unwrap(), "deno.land");
+    assert_eq!(new_uri.host_str().unwrap(), "deno.land");
     assert_eq!(new_uri.path(), "/y");
   }
 
   #[test]
-fn test_resolve_uri_from_location_relative_3() {
+  fn test_resolve_url_from_location_relative_3() {
-  let url = "http://deno.land/x".parse::<Uri>().unwrap();
+    let url = "http://deno.land/x".parse::<Url>().unwrap();
-  let new_uri = resolve_uri_from_location(&url, "z");
+    let new_uri = resolve_url_from_location(&url, "z");
-  assert_eq!(new_uri.host().unwrap(), "deno.land");
+    assert_eq!(new_uri.host_str().unwrap(), "deno.land");
     assert_eq!(new_uri.path(), "/z");
   }
+}

cli/ops/fetch.rs

@@ -1,19 +1,16 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 use super::dispatch_json::{Deserialize, JsonOp, Value};
-use crate::http_util;
+use crate::http_util::get_client;
 use crate::resources;
 use crate::state::ThreadSafeState;
 use deno::*;
 use http::header::HeaderName;
-use http::uri::Uri;
+use http::header::HeaderValue;
 use http::Method;
 use hyper;
-use hyper::header::HeaderValue;
 use hyper::rt::Future;
-use hyper::Request;
 use std;
 use std::convert::From;
-use std::str::FromStr;
 
 #[derive(Deserialize)]
 struct FetchArgs {
@@ -30,44 +27,37 @@ pub fn op_fetch(
   let args: FetchArgs = serde_json::from_value(args)?;
   let url = args.url;
 
-  let body = match data {
+  let client = get_client();
-    None => hyper::Body::empty(),
+
-    Some(buf) => hyper::Body::from(Vec::from(&*buf)),
+  let method = match args.method {
+    Some(method_str) => Method::from_bytes(method_str.as_bytes())?,
+    None => Method::GET,
   };
 
-  let mut req = Request::new(body);
-  let uri = Uri::from_str(&url).map_err(ErrBox::from)?;
-  *req.uri_mut() = uri;
-
-  if let Some(method) = args.method {
-    let method = Method::from_str(&method).unwrap();
-    *req.method_mut() = method;
-  }
-
-  let headers = req.headers_mut();
-  for header_pair in args.headers {
-    let name = HeaderName::from_bytes(header_pair.0.as_bytes()).unwrap();
-    let v = HeaderValue::from_str(&header_pair.1).unwrap();
-    headers.insert(name, v);
-  }
-
   let url_ = url::Url::parse(&url).map_err(ErrBox::from)?;
   state.check_net_url(&url_)?;
 
-  let client = http_util::get_client();
+  let mut request = client.request(method, url_);
 
+  if let Some(buf) = data {
+    request = request.body(Vec::from(&*buf));
+  }
+
+  for (key, value) in args.headers {
+    let name = HeaderName::from_bytes(key.as_bytes()).unwrap();
+    let v = HeaderValue::from_str(&value).unwrap();
+    request = request.header(name, v);
+  }
   debug!("Before fetch {}", url);
-  let future = client
+  let future = request.send().map_err(ErrBox::from).and_then(move |res| {
-    .request(req)
-    .map_err(ErrBox::from)
-    .and_then(move |res| {
     let status = res.status().as_u16();
     let mut res_headers = Vec::new();
     for (key, val) in res.headers().iter() {
       res_headers.push((key.to_string(), val.to_str().unwrap().to_owned()));
     }
+
     let body = res.into_body();
-      let body_resource = resources::add_hyper_body(body);
+    let body_resource = resources::add_reqwest_body(body);
 
     let json_res = json!({
       "bodyRid": body_resource.rid,

cli/resources.rs

@@ -22,7 +22,7 @@ use futures::Future;
 use futures::Poll;
 use futures::Sink;
 use futures::Stream;
-use hyper;
+use reqwest::r#async::Decoder as ReqwestDecoder;
 use std;
 use std::collections::BTreeMap;
 use std::io::{Error, Read, Seek, SeekFrom, Write};
@@ -317,7 +317,7 @@ pub fn add_tcp_stream(stream: tokio::net::TcpStream) -> Resource {
   Resource { rid }
 }
 
-pub fn add_hyper_body(body: hyper::Body) -> Resource {
+pub fn add_reqwest_body(body: ReqwestDecoder) -> Resource {
   let rid = new_rid();
   let mut tg = RESOURCE_TABLE.lock().unwrap();
   let body = HttpBody::from(body);

js/fetch_test.ts

@@ -69,8 +69,8 @@ testPerm({ net: true }, async function fetchEmptyInvalid(): Promise<void> {
   } catch (err_) {
     err = err_;
   }
-  assertEquals(err.kind, Deno.ErrorKind.InvalidUri);
+  assertEquals(err.kind, Deno.ErrorKind.RelativeUrlWithoutBase);
-  assertEquals(err.name, "InvalidUri");
+  assertEquals(err.name, "RelativeUrlWithoutBase");
 });
 
 testPerm({ net: true }, async function fetchMultipartFormDataSuccess(): Promise<

website/manual.md

@@ -866,6 +866,16 @@ installation command to your repository:
 $ deno install awesome_cli https://example.com/awesome/cli.ts
 ```
 
+## Proxies
+
+Deno supports proxies.
+
+`HTTP_PROXY` and `HTTPS_PROXY` environmental variables are used to configure
+them.
+
+For Windows if environmental variables are not found Deno will fall back to
+reading proxies from registry.
+
 ## Import maps
 
 Deno supports [import maps](https://github.com/WICG/import-maps).