// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. use crate::deno_error; use crate::deno_error::DenoError; use crate::version; use deno::ErrBox; use futures::future; use futures::future::FutureExt; use futures::future::TryFutureExt; use reqwest; use reqwest::header::HeaderMap; use reqwest::header::CONTENT_TYPE; use reqwest::header::LOCATION; use reqwest::header::USER_AGENT; use reqwest::r#async::Client; use reqwest::RedirectPolicy; use std::future::Future; use std::pin::Pin; use url::Url; /// Create new instance of async reqwest::Client. This client supports /// proxies and doesn't follow redirects. pub fn get_client() -> Client { let mut headers = HeaderMap::new(); headers.insert( USER_AGENT, format!("Deno/{}", version::DENO).parse().unwrap(), ); Client::builder() .redirect(RedirectPolicy::none()) .default_headers(headers) .use_sys_proxy() .build() .unwrap() } /// Construct the next uri based on base uri and location header fragment /// See fn resolve_url_from_location(base_url: &Url, location: &str) -> Url { if location.starts_with("http://") || location.starts_with("https://") { // absolute uri Url::parse(location).expect("provided redirect url should be a valid url") } else if location.starts_with("//") { // "//" authority path-abempty Url::parse(&format!("{}:{}", base_url.scheme(), location)) .expect("provided redirect url should be a valid url") } else if location.starts_with('/') { // path-absolute base_url .join(location) .expect("provided redirect url should be a valid url") } else { // assuming path-noscheme | path-empty let base_url_path_str = base_url.path().to_owned(); // Pop last part or url (after last slash) let segs: Vec<&str> = base_url_path_str.rsplitn(2, '/').collect(); let new_path = format!("{}/{}", segs.last().unwrap_or(&""), location); base_url .join(&new_path) .expect("provided redirect url should be a valid url") } } #[derive(Debug, PartialEq)] pub enum FetchOnceResult { // (code, maybe_content_type) Code(String, Option), Redirect(Url), } /// Asynchronously fetchs the given HTTP URL one pass only. /// If no redirect is present and no error occurs, /// yields Code(code, maybe_content_type). /// If redirect occurs, does not follow and /// yields Redirect(url). pub fn fetch_string_once( url: &Url, ) -> impl Future> { type FetchAttempt = (Option, Option, Option); let url = url.clone(); let client = get_client(); futures::compat::Compat01As03::new(client.get(url.clone()).send()) .map_err(ErrBox::from) .and_then( move |mut response| -> Pin< Box> + Send>, > { if response.status().is_redirection() { let location_string = response .headers() .get(LOCATION) .expect("url redirection should provide 'location' header") .to_str() .unwrap(); debug!("Redirecting to {:?}...", &location_string); let new_url = resolve_url_from_location(&url, location_string); // Boxed trait object turns out to be the savior for 2+ types yielding same results. return futures::future::try_join3( future::ok(None), future::ok(None), future::ok(Some(FetchOnceResult::Redirect(new_url))), ) .boxed(); } if response.status().is_client_error() || response.status().is_server_error() { return future::err( DenoError::new( deno_error::ErrorKind::Other, format!("Import '{}' failed: {}", &url, response.status()), ) .into(), ) .boxed(); } let content_type = response .headers() .get(CONTENT_TYPE) .map(|content_type| content_type.to_str().unwrap().to_owned()); let body = futures::compat::Compat01As03::new(response.text()) .map_ok(Some) .map_err(ErrBox::from); futures::future::try_join3( body, future::ok(content_type), future::ok(None), ) .boxed() }, ) .and_then(move |(maybe_code, maybe_content_type, maybe_redirect)| { if let Some(redirect) = maybe_redirect { future::ok(redirect) } else { // maybe_code should always contain code here! future::ok(FetchOnceResult::Code( maybe_code.unwrap(), maybe_content_type, )) } }) } #[cfg(test)] mod tests { use super::*; use crate::tokio_util; #[test] fn test_fetch_sync_string() { let http_server_guard = crate::test_util::http_server(); // Relies on external http server. See tools/http_server.py let url = Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap(); let fut = fetch_string_once(&url).then(|result| match result { Ok(FetchOnceResult::Code(code, maybe_content_type)) => { assert!(!code.is_empty()); assert_eq!(maybe_content_type, Some("application/json".to_string())); futures::future::ok(()) } _ => panic!(), }); tokio_util::run(fut); drop(http_server_guard); } #[test] fn test_fetch_string_once_with_redirect() { let http_server_guard = crate::test_util::http_server(); // Relies on external http server. See tools/http_server.py let url = Url::parse("http://127.0.0.1:4546/cli/tests/fixture.json").unwrap(); // Dns resolver substitutes `127.0.0.1` with `localhost` let target_url = Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap(); let fut = fetch_string_once(&url).then(move |result| match result { Ok(FetchOnceResult::Redirect(url)) => { assert_eq!(url, target_url); futures::future::ok(()) } _ => panic!(), }); tokio_util::run(fut); drop(http_server_guard); } #[test] fn test_resolve_url_from_location_full_1() { let url = "http://deno.land".parse::().unwrap(); let new_uri = resolve_url_from_location(&url, "http://golang.org"); assert_eq!(new_uri.host_str().unwrap(), "golang.org"); } #[test] fn test_resolve_url_from_location_full_2() { let url = "https://deno.land".parse::().unwrap(); let new_uri = resolve_url_from_location(&url, "https://golang.org"); assert_eq!(new_uri.host_str().unwrap(), "golang.org"); } #[test] fn test_resolve_url_from_location_relative_1() { let url = "http://deno.land/x".parse::().unwrap(); let new_uri = resolve_url_from_location(&url, "//rust-lang.org/en-US"); assert_eq!(new_uri.host_str().unwrap(), "rust-lang.org"); assert_eq!(new_uri.path(), "/en-US"); } #[test] fn test_resolve_url_from_location_relative_2() { let url = "http://deno.land/x".parse::().unwrap(); let new_uri = resolve_url_from_location(&url, "/y"); assert_eq!(new_uri.host_str().unwrap(), "deno.land"); assert_eq!(new_uri.path(), "/y"); } #[test] fn test_resolve_url_from_location_relative_3() { let url = "http://deno.land/x".parse::().unwrap(); let new_uri = resolve_url_from_location(&url, "z"); assert_eq!(new_uri.host_str().unwrap(), "deno.land"); assert_eq!(new_uri.path(), "/z"); } }