diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 97fd0ad8fc..ae5454cd93 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -5,6 +5,7 @@ use crate::deno_error::ErrorKind; use crate::deno_error::GetErrorKind; use crate::disk_cache::DiskCache; use crate::http_util; +use crate::http_util::FetchOnceResult; use crate::msg; use crate::progress::Progress; use crate::tokio_util; @@ -12,7 +13,6 @@ use deno::ErrBox; use deno::ModuleSpecifier; use futures::future::Either; use futures::Future; -use http; use serde_json; use std; use std::collections::HashMap; @@ -21,7 +21,6 @@ use std::path::Path; use std::path::PathBuf; use std::result::Result; use std::str; -use std::str::FromStr; use std::sync::Arc; use std::sync::Mutex; use url; @@ -305,8 +304,6 @@ impl SourceFileFetcher { no_remote_fetch: bool, redirect_limit: i64, ) -> Box { - use crate::http_util::FetchOnceResult; - if redirect_limit < 0 { return Box::new(futures::future::err(too_many_redirects())); } @@ -342,20 +339,14 @@ impl SourceFileFetcher { } let download_job = self.progress.add("Download", &module_url.to_string()); - - let module_uri = url_into_uri(&module_url); - let dir = self.clone(); let module_url = module_url.clone(); // Single pass fetch, either yields code or yields redirect. - let f = - http_util::fetch_string_once(module_uri).and_then(move |r| match r { - FetchOnceResult::Redirect(uri) => { + let f = http_util::fetch_string_once(&module_url).and_then(move |r| { + match r { + FetchOnceResult::Redirect(new_module_url) => { // If redirects, update module_name and filename for next looped call. - let new_module_url = Url::parse(&uri.to_string()) - .expect("http::uri::Uri should be parseable as Url"); - dir .save_source_code_headers( &module_url, @@ -409,7 +400,8 @@ impl SourceFileFetcher { Either::B(futures::future::ok(source_file)) } - }); + } + }); Box::new(f) } @@ -539,11 +531,6 @@ fn filter_shebang(bytes: Vec) -> Vec { } } -fn url_into_uri(url: &url::Url) -> http::uri::Uri { - http::uri::Uri::from_str(&url.to_string()) - .expect("url::Url should be parseable as http::uri::Uri") -} - #[derive(Debug, Default)] /// Header metadata associated with a particular "symbolic" source code file. /// (the associated source code file might not be cached, while remaining diff --git a/cli/http_body.rs b/cli/http_body.rs index 235463ff1c..c03dfd637b 100644 --- a/cli/http_body.rs +++ b/cli/http_body.rs @@ -1,27 +1,27 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +use futures::stream::Stream; use futures::Async; use futures::Poll; -use hyper::body::Payload; -use hyper::Body; -use hyper::Chunk; +use reqwest::r#async::Chunk; +use reqwest::r#async::Decoder; use std::cmp::min; use std::io; use std::io::Read; use tokio::io::AsyncRead; -/// Wraps `hyper::Body` so that it can be exposed as an `AsyncRead` and integrated +/// Wraps `reqwest::Decoder` so that it can be exposed as an `AsyncRead` and integrated /// into resources more easily. pub struct HttpBody { - body: Body, + decoder: Decoder, chunk: Option, pos: usize, } impl HttpBody { - pub fn from(body: Body) -> Self { + pub fn from(body: Decoder) -> Self { Self { - body, + decoder: body, chunk: None, pos: 0, } @@ -59,7 +59,7 @@ impl AsyncRead for HttpBody { assert_eq!(self.pos, 0); } - let p = self.body.poll_data(); + let p = self.decoder.poll(); match p { Err(e) => Err( // TODO Need to map hyper::Error into std::io::Error. @@ -87,26 +87,3 @@ impl AsyncRead for HttpBody { } } } - -#[test] -fn test_body_async_read() { - use std::str::from_utf8; - let body = Body::from("hello world"); - let mut body = HttpBody::from(body); - - let buf = &mut [0, 0, 0, 0, 0]; - let r = body.poll_read(buf); - assert!(r.is_ok()); - assert_eq!(r.unwrap(), Async::Ready(5)); - assert_eq!(from_utf8(buf).unwrap(), "hello"); - - let r = body.poll_read(buf); - assert!(r.is_ok()); - assert_eq!(r.unwrap(), Async::Ready(5)); - assert_eq!(from_utf8(buf).unwrap(), " worl"); - - let r = body.poll_read(buf); - assert!(r.is_ok()); - assert_eq!(r.unwrap(), Async::Ready(1)); - assert_eq!(from_utf8(&buf[0..1]).unwrap(), "d"); -} diff --git a/cli/http_util.rs b/cli/http_util.rs index 511c67df0f..ec035d6bf6 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -2,76 +2,56 @@ use crate::deno_error; use crate::deno_error::DenoError; use deno::ErrBox; -#[cfg(test)] -use futures::future::{loop_fn, Loop}; -use futures::{future, Future, Stream}; -use hyper; -use hyper::client::{Client, HttpConnector}; -use hyper::header::CONTENT_TYPE; -use hyper::Uri; -use hyper_rustls; +use futures::{future, Future}; +use reqwest; +use reqwest::header::CONTENT_TYPE; +use reqwest::header::LOCATION; +use reqwest::r#async::Client; +use reqwest::RedirectPolicy; +use url::Url; -type Connector = hyper_rustls::HttpsConnector; - -lazy_static! { - static ref CONNECTOR: Connector = { - let num_dns_threads = 4; - Connector::new(num_dns_threads) - }; -} - -pub fn get_client() -> Client { - // TODO use Hyper's connection pool. - let c = CONNECTOR.clone(); - Client::builder().build(c) +/// Create new instance of async reqwest::Client. This client supports +/// proxies and doesn't follow redirects. +pub fn get_client() -> Client { + Client::builder() + .redirect(RedirectPolicy::none()) + .use_sys_proxy() + .build() + .unwrap() } /// Construct the next uri based on base uri and location header fragment /// See -fn resolve_uri_from_location(base_uri: &Uri, location: &str) -> Uri { +fn resolve_url_from_location(base_url: &Url, location: &str) -> Url { if location.starts_with("http://") || location.starts_with("https://") { // absolute uri - location - .parse::() - .expect("provided redirect url should be a valid url") + Url::parse(location).expect("provided redirect url should be a valid url") } else if location.starts_with("//") { // "//" authority path-abempty - format!("{}:{}", base_uri.scheme_part().unwrap().as_str(), location) - .parse::() + Url::parse(&format!("{}:{}", base_url.scheme(), location)) .expect("provided redirect url should be a valid url") } else if location.starts_with('/') { // path-absolute - let mut new_uri_parts = base_uri.clone().into_parts(); - new_uri_parts.path_and_query = Some(location.parse().unwrap()); - Uri::from_parts(new_uri_parts).unwrap() + base_url + .join(location) + .expect("provided redirect url should be a valid url") } else { // assuming path-noscheme | path-empty - let mut new_uri_parts = base_uri.clone().into_parts(); - let base_uri_path_str = base_uri.path().to_owned(); - let segs: Vec<&str> = base_uri_path_str.rsplitn(2, '/').collect(); - new_uri_parts.path_and_query = Some( - format!("{}/{}", segs.last().unwrap_or(&""), location) - .parse() - .unwrap(), - ); - Uri::from_parts(new_uri_parts).unwrap() + let base_url_path_str = base_url.path().to_owned(); + // Pop last part or url (after last slash) + let segs: Vec<&str> = base_url_path_str.rsplitn(2, '/').collect(); + let new_path = format!("{}/{}", segs.last().unwrap_or(&""), location); + base_url + .join(&new_path) + .expect("provided redirect url should be a valid url") } } -#[cfg(test)] -use crate::tokio_util; -#[cfg(test)] -/// Synchronously fetchs the given HTTP URL. Returns (content, media_type). -pub fn fetch_sync_string( - module_name: &str, -) -> Result<(String, String), ErrBox> { - tokio_util::block_on(fetch_string(module_name)) -} - +#[derive(Debug, PartialEq)] pub enum FetchOnceResult { // (code, maybe_content_type) Code(String, Option), - Redirect(http::uri::Uri), + Redirect(Url), } /// Asynchronously fetchs the given HTTP URL one pass only. @@ -80,51 +60,57 @@ pub enum FetchOnceResult { /// If redirect occurs, does not follow and /// yields Redirect(url). pub fn fetch_string_once( - url: http::uri::Uri, + url: &Url, ) -> impl Future { type FetchAttempt = (Option, Option, Option); + + let url = url.clone(); let client = get_client(); + client .get(url.clone()) + .send() .map_err(ErrBox::from) .and_then( - move |response| -> Box< + move |mut response| -> Box< dyn Future + Send, > { if response.status().is_redirection() { - let location_string = response - .headers() - .get("location") + let location_string = response.headers() + .get(LOCATION) .expect("url redirection should provide 'location' header") .to_str() - .unwrap() - .to_string(); - debug!("Redirecting to {}...", &location_string); - let new_url = resolve_uri_from_location(&url, &location_string); + .unwrap(); + + debug!("Redirecting to {:?}...", &location_string); + let new_url = resolve_url_from_location(&url, location_string); // Boxed trait object turns out to be the savior for 2+ types yielding same results. return Box::new(future::ok(None).join3( future::ok(None), future::ok(Some(FetchOnceResult::Redirect(new_url))), )); - } else if response.status().is_client_error() - || response.status().is_server_error() - { + } + + if response.status().is_client_error() || response.status().is_server_error() { return Box::new(future::err(DenoError::new( deno_error::ErrorKind::Other, format!("Import '{}' failed: {}", &url, response.status()), ).into())); } + let content_type = response .headers() .get(CONTENT_TYPE) .map(|content_type| content_type.to_str().unwrap().to_owned()); + let body = response - .into_body() - .concat2() - .map(|body| String::from_utf8(body.to_vec()).ok()) + .text() .map_err(ErrBox::from); - Box::new(body.join3(future::ok(content_type), future::ok(None))) - }, + + Box::new( + Some(body).join3(future::ok(content_type), future::ok(None)) + ) + } ) .and_then(move |(maybe_code, maybe_content_type, maybe_redirect)| { if let Some(redirect) = maybe_redirect { @@ -140,131 +126,74 @@ pub fn fetch_string_once( } #[cfg(test)] -/// Asynchronously fetchs the given HTTP URL. Returns (content, media_type). -pub fn fetch_string( - module_name: &str, -) -> impl Future { - let url = module_name.parse::().unwrap(); - let client = get_client(); - // TODO(kevinkassimo): consider set a max redirection counter - // to avoid bouncing between 2 or more urls - loop_fn((client, url), |(client, url)| { - client - .get(url.clone()) - .map_err(ErrBox::from) - .and_then(move |response| { - if response.status().is_redirection() { - let location_string = response - .headers() - .get("location") - .expect("url redirection should provide 'location' header") - .to_str() - .unwrap() - .to_string(); - debug!("Redirecting to {}...", &location_string); - let new_url = resolve_uri_from_location(&url, &location_string); - return Ok(Loop::Continue((client, new_url))); - } - if !response.status().is_success() { - return Err( - DenoError::new( - deno_error::ErrorKind::NotFound, - "module not found".to_string(), - ) - .into(), - ); - } - Ok(Loop::Break(response)) - }) - }) - .and_then(|response| { - let content_type = response - .headers() - .get(CONTENT_TYPE) - .map(|content_type| content_type.to_str().unwrap().to_string()); - let body = response - .into_body() - .concat2() - .map(|body| String::from_utf8(body.to_vec()).unwrap()) - .map_err(ErrBox::from); - body.join(future::ok(content_type)) - }) - .and_then(|(body_string, maybe_content_type)| { - future::ok((body_string, maybe_content_type.unwrap())) - }) -} +mod tests { + use super::*; + use crate::tokio_util; -#[test] -fn test_fetch_sync_string() { - // Relies on external http server. See tools/http_server.py - tokio_util::init(|| { - let (p, m) = - fetch_sync_string("http://127.0.0.1:4545/package.json").unwrap(); - println!("package.json len {}", p.len()); - assert!(p.len() > 1); - assert!(m == "application/json") - }); -} + pub fn fetch_string_once_sync(url: &Url) -> Result { + tokio_util::block_on(fetch_string_once(url)) + } -#[test] -fn test_fetch_string() { - // Relies on external http server. See tools/http_server.py - tokio_util::init(|| { - let (p, m) = fetch_string("http://127.0.0.1:4545/package.json") - .wait() - .unwrap(); - println!("package.json len {}", p.len()); - assert!(p.len() > 1); - assert!(m == "application/json") - }); -} + #[test] + fn test_fetch_sync_string() { + // Relies on external http server. See tools/http_server.py + let url = Url::parse("http://127.0.0.1:4545/package.json").unwrap(); + tokio_util::init(|| match fetch_string_once_sync(&url).unwrap() { + FetchOnceResult::Code(code, maybe_content_type) => { + assert!(!code.is_empty()); + assert_eq!(maybe_content_type, Some("application/json".to_string())); + } + _ => unreachable!(), + }); + } -#[test] -fn test_fetch_sync_string_with_redirect() { - // Relies on external http server. See tools/http_server.py - tokio_util::init(|| { - let (p, m) = - fetch_sync_string("http://127.0.0.1:4546/package.json").unwrap(); - println!("package.json len {}", p.len()); - assert!(p.len() > 1); - assert!(m == "application/json") - }); -} + #[test] + fn test_fetch_string_once_with_redirect() { + // Relies on external http server. See tools/http_server.py + let url = Url::parse("http://127.0.0.1:4546/package.json").unwrap(); + // Dns resolver substitutes `127.0.0.1` with `localhost` + let target_url = Url::parse("http://localhost:4545/package.json").unwrap(); + tokio_util::init(|| { + let result = fetch_string_once_sync(&url).unwrap(); + assert_eq!(result, FetchOnceResult::Redirect(target_url)); + }); + } -#[test] -fn test_resolve_uri_from_location_full_1() { - let url = "http://deno.land".parse::().unwrap(); - let new_uri = resolve_uri_from_location(&url, "http://golang.org"); - assert_eq!(new_uri.host().unwrap(), "golang.org"); -} + #[test] + fn test_resolve_url_from_location_full_1() { + let url = "http://deno.land".parse::().unwrap(); + let new_uri = resolve_url_from_location(&url, "http://golang.org"); + assert_eq!(new_uri.host_str().unwrap(), "golang.org"); + } -#[test] -fn test_resolve_uri_from_location_full_2() { - let url = "https://deno.land".parse::().unwrap(); - let new_uri = resolve_uri_from_location(&url, "https://golang.org"); - assert_eq!(new_uri.host().unwrap(), "golang.org"); -} + #[test] + fn test_resolve_url_from_location_full_2() { + let url = "https://deno.land".parse::().unwrap(); + let new_uri = resolve_url_from_location(&url, "https://golang.org"); + assert_eq!(new_uri.host_str().unwrap(), "golang.org"); + } -#[test] -fn test_resolve_uri_from_location_relative_1() { - let url = "http://deno.land/x".parse::().unwrap(); - let new_uri = resolve_uri_from_location(&url, "//rust-lang.org/en-US"); - assert_eq!(new_uri.host().unwrap(), "rust-lang.org"); - assert_eq!(new_uri.path(), "/en-US"); -} + #[test] + fn test_resolve_url_from_location_relative_1() { + let url = "http://deno.land/x".parse::().unwrap(); + let new_uri = resolve_url_from_location(&url, "//rust-lang.org/en-US"); + assert_eq!(new_uri.host_str().unwrap(), "rust-lang.org"); + assert_eq!(new_uri.path(), "/en-US"); + } -#[test] -fn test_resolve_uri_from_location_relative_2() { - let url = "http://deno.land/x".parse::().unwrap(); - let new_uri = resolve_uri_from_location(&url, "/y"); - assert_eq!(new_uri.host().unwrap(), "deno.land"); - assert_eq!(new_uri.path(), "/y"); -} + #[test] + fn test_resolve_url_from_location_relative_2() { + let url = "http://deno.land/x".parse::().unwrap(); + let new_uri = resolve_url_from_location(&url, "/y"); + assert_eq!(new_uri.host_str().unwrap(), "deno.land"); + assert_eq!(new_uri.path(), "/y"); + } -#[test] -fn test_resolve_uri_from_location_relative_3() { - let url = "http://deno.land/x".parse::().unwrap(); - let new_uri = resolve_uri_from_location(&url, "z"); - assert_eq!(new_uri.host().unwrap(), "deno.land"); - assert_eq!(new_uri.path(), "/z"); + #[test] + fn test_resolve_url_from_location_relative_3() { + let url = "http://deno.land/x".parse::().unwrap(); + let new_uri = resolve_url_from_location(&url, "z"); + assert_eq!(new_uri.host_str().unwrap(), "deno.land"); + assert_eq!(new_uri.path(), "/z"); + } } diff --git a/cli/ops/fetch.rs b/cli/ops/fetch.rs index e2ab81a81a..afda07f751 100644 --- a/cli/ops/fetch.rs +++ b/cli/ops/fetch.rs @@ -1,19 +1,16 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. use super::dispatch_json::{Deserialize, JsonOp, Value}; -use crate::http_util; +use crate::http_util::get_client; use crate::resources; use crate::state::ThreadSafeState; use deno::*; use http::header::HeaderName; -use http::uri::Uri; +use http::header::HeaderValue; use http::Method; use hyper; -use hyper::header::HeaderValue; use hyper::rt::Future; -use hyper::Request; use std; use std::convert::From; -use std::str::FromStr; #[derive(Deserialize)] struct FetchArgs { @@ -30,53 +27,46 @@ pub fn op_fetch( let args: FetchArgs = serde_json::from_value(args)?; let url = args.url; - let body = match data { - None => hyper::Body::empty(), - Some(buf) => hyper::Body::from(Vec::from(&*buf)), + let client = get_client(); + + let method = match args.method { + Some(method_str) => Method::from_bytes(method_str.as_bytes())?, + None => Method::GET, }; - let mut req = Request::new(body); - let uri = Uri::from_str(&url).map_err(ErrBox::from)?; - *req.uri_mut() = uri; - - if let Some(method) = args.method { - let method = Method::from_str(&method).unwrap(); - *req.method_mut() = method; - } - - let headers = req.headers_mut(); - for header_pair in args.headers { - let name = HeaderName::from_bytes(header_pair.0.as_bytes()).unwrap(); - let v = HeaderValue::from_str(&header_pair.1).unwrap(); - headers.insert(name, v); - } - let url_ = url::Url::parse(&url).map_err(ErrBox::from)?; state.check_net_url(&url_)?; - let client = http_util::get_client(); + let mut request = client.request(method, url_); + if let Some(buf) = data { + request = request.body(Vec::from(&*buf)); + } + + for (key, value) in args.headers { + let name = HeaderName::from_bytes(key.as_bytes()).unwrap(); + let v = HeaderValue::from_str(&value).unwrap(); + request = request.header(name, v); + } debug!("Before fetch {}", url); - let future = client - .request(req) - .map_err(ErrBox::from) - .and_then(move |res| { - let status = res.status().as_u16(); - let mut res_headers = Vec::new(); - for (key, val) in res.headers().iter() { - res_headers.push((key.to_string(), val.to_str().unwrap().to_owned())); - } - let body = res.into_body(); - let body_resource = resources::add_hyper_body(body); + let future = request.send().map_err(ErrBox::from).and_then(move |res| { + let status = res.status().as_u16(); + let mut res_headers = Vec::new(); + for (key, val) in res.headers().iter() { + res_headers.push((key.to_string(), val.to_str().unwrap().to_owned())); + } - let json_res = json!({ - "bodyRid": body_resource.rid, - "status": status, - "headers": res_headers - }); + let body = res.into_body(); + let body_resource = resources::add_reqwest_body(body); - futures::future::ok(json_res) + let json_res = json!({ + "bodyRid": body_resource.rid, + "status": status, + "headers": res_headers }); + futures::future::ok(json_res) + }); + Ok(JsonOp::Async(Box::new(future))) } diff --git a/cli/resources.rs b/cli/resources.rs index c796465ea2..5d4ba33bf4 100644 --- a/cli/resources.rs +++ b/cli/resources.rs @@ -22,7 +22,7 @@ use futures::Future; use futures::Poll; use futures::Sink; use futures::Stream; -use hyper; +use reqwest::r#async::Decoder as ReqwestDecoder; use std; use std::collections::BTreeMap; use std::io::{Error, Read, Seek, SeekFrom, Write}; @@ -317,7 +317,7 @@ pub fn add_tcp_stream(stream: tokio::net::TcpStream) -> Resource { Resource { rid } } -pub fn add_hyper_body(body: hyper::Body) -> Resource { +pub fn add_reqwest_body(body: ReqwestDecoder) -> Resource { let rid = new_rid(); let mut tg = RESOURCE_TABLE.lock().unwrap(); let body = HttpBody::from(body); diff --git a/js/fetch_test.ts b/js/fetch_test.ts index 1b59e068a1..542d691470 100644 --- a/js/fetch_test.ts +++ b/js/fetch_test.ts @@ -69,8 +69,8 @@ testPerm({ net: true }, async function fetchEmptyInvalid(): Promise { } catch (err_) { err = err_; } - assertEquals(err.kind, Deno.ErrorKind.InvalidUri); - assertEquals(err.name, "InvalidUri"); + assertEquals(err.kind, Deno.ErrorKind.RelativeUrlWithoutBase); + assertEquals(err.name, "RelativeUrlWithoutBase"); }); testPerm({ net: true }, async function fetchMultipartFormDataSuccess(): Promise< diff --git a/website/manual.md b/website/manual.md index b491a1f176..a3678ca4ab 100644 --- a/website/manual.md +++ b/website/manual.md @@ -866,6 +866,16 @@ installation command to your repository: $ deno install awesome_cli https://example.com/awesome/cli.ts ``` +## Proxies + +Deno supports proxies. + +`HTTP_PROXY` and `HTTPS_PROXY` environmental variables are used to configure +them. + +For Windows if environmental variables are not found Deno will fall back to +reading proxies from registry. + ## Import maps Deno supports [import maps](https://github.com/WICG/import-maps).