foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-21 15:04:11 -05:00
Code Issues Packages 1 Wiki Activity

Follow redirect location as new referrers for nested module imports (#2031)

Browse source 
Fixes #1742
Fixes #2021
This commit is contained in:
Kevin (Kun) "Kassimo" Qian 2019-04-01 18:46:40 -07:00 committed by Ryan Dahl
parent e44084c90d
commit 534b8d3021
28 changed files with 812 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cli/compiler.rs
View file

@ -104,6 +104,7 @@ impl WorkerBehavior for CompilerBehavior {
#[derive(Debug, Clone)]
pub struct ModuleMetaData {
pub module_name: String,
pub module_redirect_source_name: Option<String>, // source of redirect
pub filename: String,
pub media_type: msg::MediaType,
pub source_code: Vec<u8>,
@ -250,6 +251,9 @@ pub fn compile_sync(
) {
Ok(serde_json::Value::Object(map)) => ModuleMetaData {
module_name: module_meta_data_.module_name.clone(),
module_redirect_source_name: module_meta_data_
.module_redirect_source_name
.clone(),
filename: module_meta_data_.filename.clone(),
media_type: module_meta_data_.media_type,
source_code: module_meta_data_.source_code.clone(),
@ -342,6 +346,7 @@ mod tests {
let mut out = ModuleMetaData {
module_name: "xxx".to_owned(),
module_redirect_source_name: None,
filename: "/tests/002_hello.ts".to_owned(),
media_type: msg::MediaType::TypeScript,
source_code: include_bytes!("../tests/002_hello.ts").to_vec(),

615
cli/deno_dir.rs
View file

@ -11,9 +11,11 @@ use crate::msg;
use crate::tokio_util;
use crate::version;
use dirs;
use futures::future::Either;
use futures::future::{loop_fn, Either, Loop};
use futures::Future;
use http;
use ring;
use serde_json;
use std;
use std::fmt::Write;
use std::fs;
@ -34,6 +36,7 @@ fn extmap(ext: &str) -> msg::MediaType {
}
}
#[derive(Clone)]
pub struct DenoDir {
// Example: /Users/rld/.deno/
pub root: PathBuf,
@ -154,8 +157,12 @@ impl DenoDir {
let gen = self.gen.clone();
Either::B(
get_source_code_async(module_name.as_str(), filename.as_str(), use_cache)
.then(move |result| {
get_source_code_async(
self,
module_name.as_str(),
filename.as_str(),
use_cache,
).then(move |result| {
let mut out = match result {
Ok(out) => out,
Err(err) => {
@ -318,7 +325,7 @@ impl DenoDir {
get_cache_filename(self.deps_http.as_path(), &j).as_ref(),
)
}
// TODO(kevinkassimo): change this to support other protocols than http
// TODO(kevinkassimo): change this to support other protocols than http.
_ => unimplemented!(),
}
@ -353,6 +360,7 @@ impl SourceMapGetter for DenoDir {
/// download will be written to "filename". This happens no matter the value of
/// use_cache.
fn get_source_code_async(
deno_dir: &DenoDir,
module_name: &str,
filename: &str,
use_cache: bool,
@ -361,15 +369,15 @@ fn get_source_code_async(
let module_name = module_name.to_string();
let is_module_remote = is_remote(&module_name);
// We try fetch local. Two cases:
// 1. This is a remote module and we're allowed to use cached downloads
// 2. This is a local module
// 1. This is a remote module and we're allowed to use cached downloads.
// 2. This is a local module.
if !is_module_remote || use_cache {
debug!(
"fetch local or reload {} is_module_remote {}",
module_name, is_module_remote
);
// Note that local fetch is done synchronously.
match fetch_local_source(&module_name, &filename) {
match fetch_local_source(deno_dir, &module_name, &filename, None) {
Ok(Some(output)) => {
debug!("found local source ");
return Either::A(futures::future::ok(output));
@ -396,8 +404,9 @@ fn get_source_code_async(
debug!("is remote but didn't find module");
// not cached/local, try remote
Either::B(fetch_remote_source_async(&module_name, &filename).and_then(
// not cached/local, try remote.
Either::B(
fetch_remote_source_async(deno_dir, &module_name, &filename).and_then(
move |maybe_remote_source| match maybe_remote_source {
Some(output) => Ok(output),
None => Err(DenoError::from(std::io::Error::new(
@ -405,18 +414,25 @@ fn get_source_code_async(
format!("cannot find remote file '{}'", &filename),
))),
},
))
),
)
}
#[cfg(test)]
/// Synchronous version of get_source_code_async
/// This function is deprecated.
fn get_source_code(
deno_dir: &DenoDir,
module_name: &str,
filename: &str,
use_cache: bool,
) -> DenoResult<ModuleMetaData> {
tokio_util::block_on(get_source_code_async(module_name, filename, use_cache))
tokio_util::block_on(get_source_code_async(
deno_dir,
module_name,
filename,
use_cache,
))
}
fn get_cache_filename(basedir: &Path, url: &Url) -> PathBuf {
@ -537,71 +553,176 @@ fn filter_shebang(bytes: Vec<u8>) -> Vec<u8> {
/// Asynchronously fetch remote source file specified by the URL `module_name`
/// and write it to disk at `filename`.
fn fetch_remote_source_async(
deno_dir: &DenoDir,
module_name: &str,
filename: &str,
) -> impl Future<Item = Option<ModuleMetaData>, Error = DenoError> {
let filename = filename.to_string();
let module_name = module_name.to_string();
use crate::http_util::FetchOnceResult;
{
eprintln!("Downloading {}", module_name);
}
let filename = filename.to_owned();
let module_name = module_name.to_owned();
// We write a special ".headers.json" file into the `.deno/deps` directory along side the
// cached file, containing just the media type and possible redirect target (both are http headers).
// If redirect target is present, the file itself if not cached.
// In future resolutions, we would instead follow this redirect target ("redirect_to").
loop_fn(
(
deno_dir.clone(),
None,
None,
module_name.clone(),
filename.clone(),
),
|(
dir,
maybe_initial_module_name,
maybe_initial_filename,
module_name,
filename,
)| {
let url = module_name.parse::<http::uri::Uri>().unwrap();
// Single pass fetch, either yields code or yields redirect.
http_util::fetch_string_once(url).and_then(move |fetch_once_result| {
match fetch_once_result {
FetchOnceResult::Redirect(url) => {
// If redirects, update module_name and filename for next looped call.
let resolve_result = dir
.resolve_module(&(url.to_string()), ".")
.map_err(DenoError::from);
match resolve_result {
Ok((new_module_name, new_filename)) => {
let mut maybe_initial_module_name = maybe_initial_module_name;
let mut maybe_initial_filename = maybe_initial_filename;
if maybe_initial_module_name.is_none() {
maybe_initial_module_name = Some(module_name.clone());
maybe_initial_filename = Some(filename.clone());
}
// Not yet completed. Follow the redirect and loop.
Ok(Loop::Continue((
dir,
maybe_initial_module_name,
maybe_initial_filename,
new_module_name,
new_filename,
)))
}
Err(e) => Err(e),
}
}
FetchOnceResult::Code(source, maybe_content_type) => {
// We land on the code.
let p = PathBuf::from(filename.clone());
// We write a special ".mime" file into the `.deno/deps` directory along side the
// cached file, containing just the media type.
let media_type_filename = [&filename, ".mime"].concat();
let mt = PathBuf::from(&media_type_filename);
eprintln!("Downloading {}", &module_name);
http_util::fetch_string(&module_name).and_then(
move |(source, content_type)| {
match p.parent() {
Some(ref parent) => fs::create_dir_all(parent),
None => Ok(()),
}?;
// Write file and create .headers.json for the file.
deno_fs::write_file(&p, &source, 0o666)?;
// Remove possibly existing stale .mime file
// may not exist. DON'T unwrap
let _ = std::fs::remove_file(&media_type_filename);
// Create .mime file only when content type different from extension
let resolved_content_type = map_content_type(&p, Some(&content_type));
let ext = p
.extension()
.map(|x| x.to_str().unwrap_or(""))
.unwrap_or("");
let media_type = extmap(&ext);
if media_type == msg::MediaType::Unknown
|| media_type != resolved_content_type
{
deno_fs::write_file(&mt, content_type.as_bytes(), 0o666)?
save_source_code_headers(&filename, maybe_content_type.clone(), None);
}
Ok(Some(ModuleMetaData {
// Check if this file is downloaded due to some old redirect request.
if maybe_initial_filename.is_some() {
// If yes, record down the headers for redirect.
// Also create its containing folder.
let pp = PathBuf::from(filename.clone());
match pp.parent() {
Some(ref parent) => fs::create_dir_all(parent),
None => Ok(()),
}?;
{
save_source_code_headers(
&maybe_initial_filename.clone().unwrap(),
maybe_content_type.clone(),
Some(module_name.clone()),
);
}
}
Ok(Loop::Break(Some(ModuleMetaData {
module_name: module_name.to_string(),
module_redirect_source_name: maybe_initial_module_name,
filename: filename.to_string(),
media_type: map_content_type(&p, Some(&content_type)),
media_type: map_content_type(
&p,
maybe_content_type.as_ref().map(|s| s.as_str()),
),
source_code: source.as_bytes().to_owned(),
maybe_output_code_filename: None,
maybe_output_code: None,
maybe_source_map_filename: None,
maybe_source_map: None,
}))
})))
}
}
})
},
)
}
// Prototype https://github.com/denoland/deno/blob/golang/deno_dir.go#L37-L73
/// Fetch remote source code.
#[cfg(test)]
fn fetch_remote_source(
deno_dir: &DenoDir,
module_name: &str,
filename: &str,
) -> DenoResult<Option<ModuleMetaData>> {
tokio_util::block_on(fetch_remote_source_async(module_name, filename))
tokio_util::block_on(fetch_remote_source_async(
deno_dir,
module_name,
filename,
))
}
/// Fetch local or cached source code.
/// This is a recursive operation if source file has redirection.
/// It will keep reading filename.headers.json for information about redirection.
/// module_initial_source_name would be None on first call,
/// and becomes the name of the very first module that initiates the call
/// in subsequent recursions.
/// AKA if redirection occurs, module_initial_source_name is the source path
/// that user provides, and the final module_name is the resolved path
/// after following all redirections.
fn fetch_local_source(
deno_dir: &DenoDir,
module_name: &str,
filename: &str,
module_initial_source_name: Option<String>,
) -> DenoResult<Option<ModuleMetaData>> {
let p = Path::new(&filename);
let media_type_filename = [&filename, ".mime"].concat();
let mt = Path::new(&media_type_filename);
let source_code_headers = get_source_code_headers(&filename);
// If source code headers says that it would redirect elsewhere,
// (meaning that the source file might not exist; only .headers.json is present)
// Abort reading attempts to the cached source file and and follow the redirect.
if let Some(redirect_to) = source_code_headers.redirect_to {
// E.g.
// module_name https://import-meta.now.sh/redirect.js
// filename /Users/kun/Library/Caches/deno/deps/https/import-meta.now.sh/redirect.js
// redirect_to https://import-meta.now.sh/sub/final1.js
// real_filename /Users/kun/Library/Caches/deno/deps/https/import-meta.now.sh/sub/final1.js
// real_module_name = https://import-meta.now.sh/sub/final1.js
let (real_module_name, real_filename) =
deno_dir.resolve_module(&redirect_to, ".")?;
let mut module_initial_source_name = module_initial_source_name;
// If this is the first redirect attempt,
// then module_initial_source_name should be None.
// In that case, use current module name as module_initial_source_name.
if module_initial_source_name.is_none() {
module_initial_source_name = Some(module_name.to_owned());
}
// Recurse.
return fetch_local_source(
deno_dir,
&real_module_name,
&real_filename,
module_initial_source_name,
);
}
// No redirect needed or end of redirects.
// We can try read the file
let source_code = match fs::read(p) {
Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound {
@ -612,16 +733,14 @@ fn fetch_local_source(
}
Ok(c) => c,
};
// .mime file might not exists
// this is okay for local source: maybe_content_type_str will be None
let maybe_content_type_string = fs::read_to_string(&mt).ok();
// Option<String> -> Option<&str>
let maybe_content_type_str =
maybe_content_type_string.as_ref().map(String::as_str);
Ok(Some(ModuleMetaData {
module_name: module_name.to_string(),
module_redirect_source_name: module_initial_source_name,
filename: filename.to_string(),
media_type: map_content_type(&p, maybe_content_type_str),
media_type: map_content_type(
&p,
source_code_headers.mime_type.as_ref().map(String::as_str),
),
source_code,
maybe_output_code_filename: None,
maybe_output_code: None,
@ -630,6 +749,106 @@ fn fetch_local_source(
}))
}
#[derive(Debug)]
/// Header metadata associated with a particular "symbolic" source code file.
/// (the associated source code file might not be cached, while remaining
/// a user accessible entity through imports (due to redirects)).
pub struct SourceCodeHeaders {
/// MIME type of the source code.
pub mime_type: Option<String>,
/// Where should we actually look for source code.
/// This should be an absolute path!
pub redirect_to: Option<String>,
}
static MIME_TYPE: &'static str = "mime_type";
static REDIRECT_TO: &'static str = "redirect_to";
fn source_code_headers_filename(filename: &str) -> String {
[&filename, ".headers.json"].concat()
}
/// Get header metadata associated with a single source code file.
/// NOTICE: chances are that the source code itself is not downloaded due to redirects.
/// In this case, the headers file provides info about where we should go and get
/// the source code that redirect eventually points to (which should be cached).
fn get_source_code_headers(filename: &str) -> SourceCodeHeaders {
let headers_filename = source_code_headers_filename(filename);
let hd = Path::new(&headers_filename);
// .headers.json file might not exists.
// This is okay for local source.
let maybe_headers_string = fs::read_to_string(&hd).ok();
if let Some(headers_string) = maybe_headers_string {
// TODO(kevinkassimo): consider introduce serde::Deserialize to make things simpler.
let maybe_headers: serde_json::Result<serde_json::Value> =
serde_json::from_str(&headers_string);
if let Ok(headers) = maybe_headers {
return SourceCodeHeaders {
mime_type: headers[MIME_TYPE].as_str().map(String::from),
redirect_to: headers[REDIRECT_TO].as_str().map(String::from),
};
}
}
SourceCodeHeaders {
mime_type: None,
redirect_to: None,
}
}
/// Save headers related to source filename to {filename}.headers.json file,
/// only when there is actually something necessary to save.
/// For example, if the extension ".js" already mean JS file and we have
/// content type of "text/javascript", then we would not save the mime type.
/// If nothing needs to be saved, the headers file is not created.
fn save_source_code_headers(
filename: &str,
mime_type: Option<String>,
redirect_to: Option<String>,
) {
let headers_filename = source_code_headers_filename(filename);
// Remove possibly existing stale .headers.json file.
// May not exist. DON'T unwrap.
let _ = std::fs::remove_file(&headers_filename);
let p = PathBuf::from(filename);
// TODO(kevinkassimo): consider introduce serde::Deserialize to make things simpler.
// This is super ugly at this moment...
// Had trouble to make serde_derive work: I'm unable to build proc-macro2.
let mut value_map = serde_json::map::Map::new();
if mime_type.is_some() {
let mime_type_string = mime_type.clone().unwrap();
let resolved_mime_type =
{ map_content_type(Path::new(""), Some(mime_type_string.as_str())) };
let ext = p
.extension()
.map(|x| x.to_str().unwrap_or(""))
.unwrap_or("");
let ext_based_mime_type = extmap(&ext);
// Add mime to headers only when content type is different from extension.
if ext_based_mime_type == msg::MediaType::Unknown
|| resolved_mime_type != ext_based_mime_type
{
value_map.insert(MIME_TYPE.to_string(), json!(mime_type_string));
}
}
if redirect_to.is_some() {
value_map.insert(REDIRECT_TO.to_string(), json!(redirect_to.unwrap()));
}
// Only save to file when there is actually data.
if value_map.len() > 0 {
let _ = serde_json::to_string(&value_map).map(|s| {
// It is possible that we need to create file
// with parent folders not yet created.
// (Due to .headers.json feature for redirection)
let hd = PathBuf::from(&headers_filename);
let _ = match hd.parent() {
Some(ref parent) => fs::create_dir_all(parent),
None => Ok(()),
};
let _ = deno_fs::write_file(&(hd.as_path()), s, 0o666);
});
}
}
#[cfg(test)]
mod tests {
use super::*;
@ -709,6 +928,7 @@ mod tests {
filename: filename.to_owned(),
source_code: source_code[..].to_owned(),
module_name: "hello.js".to_owned(),
module_redirect_source_name: None,
media_type: msg::MediaType::TypeScript,
maybe_output_code: Some(output_code[..].to_owned()),
maybe_output_code_filename: None,
@ -745,6 +965,35 @@ mod tests {
);
}
#[test]
fn test_source_code_headers_get_and_save() {
let (temp_dir, _deno_dir) = test_setup();
let filename =
deno_fs::normalize_path(temp_dir.into_path().join("f.js").as_ref());
let headers_file_name = source_code_headers_filename(&filename);
assert_eq!(headers_file_name, [&filename, ".headers.json"].concat());
let _ = deno_fs::write_file(&PathBuf::from(&headers_file_name),
"{\"mime_type\":\"text/javascript\",\"redirect_to\":\"http://example.com/a.js\"}", 0o666);
let headers = get_source_code_headers(&filename);
assert_eq!(headers.mime_type.clone().unwrap(), "text/javascript");
assert_eq!(
headers.redirect_to.clone().unwrap(),
"http://example.com/a.js"
);
save_source_code_headers(
&filename,
Some("text/typescript".to_owned()),
Some("http://deno.land/a.js".to_owned()),
);
let headers2 = get_source_code_headers(&filename);
assert_eq!(headers2.mime_type.clone().unwrap(), "text/typescript");
assert_eq!(
headers2.redirect_to.clone().unwrap(),
"http://deno.land/a.js"
);
}
#[test]
fn test_get_source_code_1() {
let (_temp_dir, deno_dir) = test_setup();
@ -757,9 +1006,9 @@ mod tests {
.join("localhost_PORT4545/tests/subdir/mod2.ts")
.as_ref(),
);
let mime_file_name = format!("{}.mime", &filename);
let headers_file_name = source_code_headers_filename(&filename);
let result = get_source_code(module_name, &filename, true);
let result = get_source_code(&deno_dir, module_name, &filename, true);
assert!(result.is_ok());
let r = result.unwrap();
assert_eq!(
@ -767,12 +1016,13 @@ mod tests {
"export { printHello } from \"./print_hello.ts\";\n".as_bytes()
);
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
// Should not create .mime file due to matching ext
assert!(fs::read_to_string(&mime_file_name).is_err());
// Should not create .headers.json file due to matching ext
assert!(fs::read_to_string(&headers_file_name).is_err());
// Modify .mime
let _ = fs::write(&mime_file_name, "text/javascript");
let result2 = get_source_code(module_name, &filename, true);
// Modify .headers.json, write using fs write and read using save_source_code_headers
let _ =
fs::write(&headers_file_name, "{ \"mime_type\": \"text/javascript\" }");
let result2 = get_source_code(&deno_dir, module_name, &filename, true);
assert!(result2.is_ok());
let r2 = result2.unwrap();
assert_eq!(
@ -780,23 +1030,45 @@ mod tests {
"export { printHello } from \"./print_hello.ts\";\n".as_bytes()
);
// If get_source_code does not call remote, this should be JavaScript
// as we modified before! (we do not overwrite .mime due to no http fetch)
// as we modified before! (we do not overwrite .headers.json due to no http fetch)
assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
assert_eq!(
fs::read_to_string(&mime_file_name).unwrap(),
get_source_code_headers(&filename).mime_type.unwrap(),
"text/javascript"
);
// Don't use_cache
let result3 = get_source_code(module_name, &filename, false);
// Modify .headers.json again, but the other way around
save_source_code_headers(
&filename,
Some("application/json".to_owned()),
None,
);
let result3 = get_source_code(&deno_dir, module_name, &filename, true);
assert!(result3.is_ok());
let r3 = result3.unwrap();
let expected3 =
assert_eq!(
r3.source_code,
"export { printHello } from \"./print_hello.ts\";\n".as_bytes()
);
// If get_source_code does not call remote, this should be JavaScript
// as we modified before! (we do not overwrite .headers.json due to no http fetch)
assert_eq!(&(r3.media_type), &msg::MediaType::Json);
assert!(
fs::read_to_string(&headers_file_name)
.unwrap()
.contains("application/json")
);
// Don't use_cache
let result4 = get_source_code(&deno_dir, module_name, &filename, false);
assert!(result4.is_ok());
let r4 = result4.unwrap();
let expected4 =
"export { printHello } from \"./print_hello.ts\";\n".as_bytes();
assert_eq!(r3.source_code, expected3);
// Now the old .mime file should have gone! Resolved back to TypeScript
assert_eq!(&(r3.media_type), &msg::MediaType::TypeScript);
assert!(fs::read_to_string(&mime_file_name).is_err());
assert_eq!(r4.source_code, expected4);
// Now the old .headers.json file should have gone! Resolved back to TypeScript
assert_eq!(&(r4.media_type), &msg::MediaType::TypeScript);
assert!(fs::read_to_string(&headers_file_name).is_err());
});
}
@ -812,51 +1084,173 @@ mod tests {
.join("localhost_PORT4545/tests/subdir/mismatch_ext.ts")
.as_ref(),
);
let mime_file_name = format!("{}.mime", &filename);
let headers_file_name = source_code_headers_filename(&filename);
let result = get_source_code(module_name, &filename, true);
let result = get_source_code(&deno_dir, module_name, &filename, true);
assert!(result.is_ok());
let r = result.unwrap();
let expected = "export const loaded = true;\n".as_bytes();
assert_eq!(r.source_code, expected);
// Mismatch ext with content type, create .mime
// Mismatch ext with content type, create .headers.json
assert_eq!(&(r.media_type), &msg::MediaType::JavaScript);
assert_eq!(
fs::read_to_string(&mime_file_name).unwrap(),
get_source_code_headers(&filename).mime_type.unwrap(),
"text/javascript"
);
// Modify .mime
let _ = fs::write(&mime_file_name, "text/typescript");
let result2 = get_source_code(module_name, &filename, true);
// Modify .headers.json
save_source_code_headers(
&filename,
Some("text/typescript".to_owned()),
None,
);
let result2 = get_source_code(&deno_dir, module_name, &filename, true);
assert!(result2.is_ok());
let r2 = result2.unwrap();
let expected2 = "export const loaded = true;\n".as_bytes();
assert_eq!(r2.source_code, expected2);
// If get_source_code does not call remote, this should be TypeScript
// as we modified before! (we do not overwrite .mime due to no http fetch)
// as we modified before! (we do not overwrite .headers.json due to no http fetch)
assert_eq!(&(r2.media_type), &msg::MediaType::TypeScript);
assert_eq!(
fs::read_to_string(&mime_file_name).unwrap(),
"text/typescript"
);
assert!(fs::read_to_string(&headers_file_name).is_err());
// Don't use_cache
let result3 = get_source_code(module_name, &filename, false);
let result3 = get_source_code(&deno_dir, module_name, &filename, false);
assert!(result3.is_ok());
let r3 = result3.unwrap();
let expected3 = "export const loaded = true;\n".as_bytes();
assert_eq!(r3.source_code, expected3);
// Now the old .mime file should be overwritten back to JavaScript!
// Now the old .headers.json file should be overwritten back to JavaScript!
// (due to http fetch)
assert_eq!(&(r3.media_type), &msg::MediaType::JavaScript);
assert_eq!(
fs::read_to_string(&mime_file_name).unwrap(),
get_source_code_headers(&filename).mime_type.unwrap(),
"text/javascript"
);
});
}
#[test]
fn test_get_source_code_3() {
let (_temp_dir, deno_dir) = test_setup();
// Test basic follow and headers recording
tokio_util::init(|| {
let redirect_module_name =
"http://localhost:4546/tests/subdir/redirects/redirect1.js";
let redirect_source_filename = deno_fs::normalize_path(
deno_dir
.deps_http
.join("localhost_PORT4546/tests/subdir/redirects/redirect1.js")
.as_ref(),
);
let target_module_name =
"http://localhost:4545/tests/subdir/redirects/redirect1.js";
let redirect_target_filename = deno_fs::normalize_path(
deno_dir
.deps_http
.join("localhost_PORT4545/tests/subdir/redirects/redirect1.js")
.as_ref(),
);
let mod_meta = get_source_code(
&deno_dir,
redirect_module_name,
&redirect_source_filename,
true,
).unwrap();
// File that requires redirection is not downloaded.
assert!(fs::read_to_string(&redirect_source_filename).is_err());
// ... but its .headers.json is created.
let redirect_source_headers =
get_source_code_headers(&redirect_source_filename);
assert_eq!(
redirect_source_headers.redirect_to.unwrap(),
"http://localhost:4545/tests/subdir/redirects/redirect1.js"
);
// The target of redirection is downloaded instead.
assert_eq!(
fs::read_to_string(&redirect_target_filename).unwrap(),
"export const redirect = 1;\n"
);
let redirect_target_headers =
get_source_code_headers(&redirect_target_filename);
assert!(redirect_target_headers.redirect_to.is_none());
// Examine the meta result.
assert_eq!(&mod_meta.module_name, target_module_name);
assert_eq!(
&mod_meta.module_redirect_source_name.clone().unwrap(),
redirect_module_name
);
});
}
#[test]
fn test_get_source_code_4() {
let (_temp_dir, deno_dir) = test_setup();
// Test double redirects and headers recording
tokio_util::init(|| {
let redirect_module_name =
"http://localhost:4548/tests/subdir/redirects/redirect1.js";
let redirect_source_filename = deno_fs::normalize_path(
deno_dir
.deps_http
.join("localhost_PORT4548/tests/subdir/redirects/redirect1.js")
.as_ref(),
);
let redirect_source_filename_intermediate = deno_fs::normalize_path(
deno_dir
.deps_http
.join("localhost_PORT4546/tests/subdir/redirects/redirect1.js")
.as_ref(),
);
let target_module_name =
"http://localhost:4545/tests/subdir/redirects/redirect1.js";
let redirect_target_filename = deno_fs::normalize_path(
deno_dir
.deps_http
.join("localhost_PORT4545/tests/subdir/redirects/redirect1.js")
.as_ref(),
);
let mod_meta = get_source_code(
&deno_dir,
redirect_module_name,
&redirect_source_filename,
true,
).unwrap();
// File that requires redirection is not downloaded.
assert!(fs::read_to_string(&redirect_source_filename).is_err());
// ... but its .headers.json is created.
let redirect_source_headers =
get_source_code_headers(&redirect_source_filename);
assert_eq!(
redirect_source_headers.redirect_to.unwrap(),
target_module_name
);
// In the intermediate redirection step, file is also not downloaded.
assert!(
fs::read_to_string(&redirect_source_filename_intermediate).is_err()
);
// The target of redirection is downloaded instead.
assert_eq!(
fs::read_to_string(&redirect_target_filename).unwrap(),
"export const redirect = 1;\n"
);
let redirect_target_headers =
get_source_code_headers(&redirect_target_filename);
assert!(redirect_target_headers.redirect_to.is_none());
// Examine the meta result.
assert_eq!(&mod_meta.module_name, target_module_name);
assert_eq!(
&mod_meta.module_redirect_source_name.clone().unwrap(),
redirect_module_name
);
});
}
#[test]
fn test_fetch_source_async_1() {
use crate::tokio_util;
@ -871,9 +1265,10 @@ mod tests {
.join("127.0.0.1_PORT4545/tests/subdir/mt_video_mp2t.t3.ts")
.as_ref(),
);
let mime_file_name = format!("{}.mime", &filename);
let headers_file_name = source_code_headers_filename(&filename);
let result = tokio_util::block_on(fetch_remote_source_async(
&deno_dir,
&module_name,
&filename,
));
@ -881,16 +1276,21 @@ mod tests {
let r = result.unwrap().unwrap();
assert_eq!(r.source_code, b"export const loaded = true;\n");
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
// matching ext, no .mime file created
assert!(fs::read_to_string(&mime_file_name).is_err());
// matching ext, no .headers.json file created
assert!(fs::read_to_string(&headers_file_name).is_err());
// Modify .mime, make sure read from local
let _ = fs::write(&mime_file_name, "text/javascript");
let result2 = fetch_local_source(&module_name, &filename);
// Modify .headers.json, make sure read from local
save_source_code_headers(
&filename,
Some("text/javascript".to_owned()),
None,
);
let result2 =
fetch_local_source(&deno_dir, &module_name, &filename, None);
assert!(result2.is_ok());
let r2 = result2.unwrap().unwrap();
assert_eq!(r2.source_code, b"export const loaded = true;\n");
// Not MediaType::TypeScript due to .mime modification
// Not MediaType::TypeScript due to .headers.json modification
assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
});
}
@ -909,23 +1309,27 @@ mod tests {
.join("localhost_PORT4545/tests/subdir/mt_video_mp2t.t3.ts")
.as_ref(),
);
let mime_file_name = format!("{}.mime", &filename);
let headers_file_name = source_code_headers_filename(&filename);
let result = fetch_remote_source(module_name, &filename);
let result = fetch_remote_source(&deno_dir, module_name, &filename);
assert!(result.is_ok());
let r = result.unwrap().unwrap();
assert_eq!(r.source_code, "export const loaded = true;\n".as_bytes());
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
// matching ext, no .mime file created
assert!(fs::read_to_string(&mime_file_name).is_err());
// matching ext, no .headers.json file created
assert!(fs::read_to_string(&headers_file_name).is_err());
// Modify .mime, make sure read from local
let _ = fs::write(&mime_file_name, "text/javascript");
let result2 = fetch_local_source(module_name, &filename);
// Modify .headers.json, make sure read from local
save_source_code_headers(
&filename,
Some("text/javascript".to_owned()),
None,
);
let result2 = fetch_local_source(&deno_dir, module_name, &filename, None);
assert!(result2.is_ok());
let r2 = result2.unwrap().unwrap();
assert_eq!(r2.source_code, "export const loaded = true;\n".as_bytes());
// Not MediaType::TypeScript due to .mime modification
// Not MediaType::TypeScript due to .headers.json modification
assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
});
}
@ -943,15 +1347,14 @@ mod tests {
.join("localhost_PORT4545/tests/subdir/no_ext")
.as_ref(),
);
let mime_file_name = format!("{}.mime", &filename);
let result = fetch_remote_source(module_name, &filename);
let result = fetch_remote_source(&deno_dir, module_name, &filename);
assert!(result.is_ok());
let r = result.unwrap().unwrap();
assert_eq!(r.source_code, "export const loaded = true;\n".as_bytes());
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
// no ext, should create .mime file
// no ext, should create .headers.json file
assert_eq!(
fs::read_to_string(&mime_file_name).unwrap(),
get_source_code_headers(&filename).mime_type.unwrap(),
"text/typescript"
);
@ -962,15 +1365,14 @@ mod tests {
.join("localhost_PORT4545/tests/subdir/mismatch_ext.ts")
.as_ref(),
);
let mime_file_name_2 = format!("{}.mime", &filename_2);
let result_2 = fetch_remote_source(module_name_2, &filename_2);
let result_2 = fetch_remote_source(&deno_dir, module_name_2, &filename_2);
assert!(result_2.is_ok());
let r2 = result_2.unwrap().unwrap();
assert_eq!(r2.source_code, "export const loaded = true;\n".as_bytes());
assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
// mismatch ext, should create .mime file
// mismatch ext, should create .headers.json file
assert_eq!(
fs::read_to_string(&mime_file_name_2).unwrap(),
get_source_code_headers(&filename_2).mime_type.unwrap(),
"text/javascript"
);
@ -982,15 +1384,14 @@ mod tests {
.join("localhost_PORT4545/tests/subdir/unknown_ext.deno")
.as_ref(),
);
let mime_file_name_3 = format!("{}.mime", &filename_3);
let result_3 = fetch_remote_source(module_name_3, &filename_3);
let result_3 = fetch_remote_source(&deno_dir, module_name_3, &filename_3);
assert!(result_3.is_ok());
let r3 = result_3.unwrap().unwrap();
assert_eq!(r3.source_code, "export const loaded = true;\n".as_bytes());
assert_eq!(&(r3.media_type), &msg::MediaType::TypeScript);
// unknown ext, should create .mime file
// unknown ext, should create .headers.json file
assert_eq!(
fs::read_to_string(&mime_file_name_3).unwrap(),
get_source_code_headers(&filename_3).mime_type.unwrap(),
"text/typescript"
);
});
@ -999,14 +1400,14 @@ mod tests {
#[test]
fn test_fetch_source_3() {
// only local, no http_util::fetch_sync_string called
let (_temp_dir, _deno_dir) = test_setup();
let (_temp_dir, deno_dir) = test_setup();
let cwd = std::env::current_dir().unwrap();
let cwd_string = cwd.to_str().unwrap();
let module_name = "http://example.com/mt_text_typescript.t1.ts"; // not used
let filename =
format!("{}/tests/subdir/mt_text_typescript.t1.ts", &cwd_string);
let result = fetch_local_source(module_name, &filename);
let result = fetch_local_source(&deno_dir, module_name, &filename, None);
assert!(result.is_ok());
let r = result.unwrap().unwrap();
assert_eq!(r.source_code, "export const loaded = true;\n".as_bytes());

80
cli/http_util.rs
View file

@ -1,6 +1,7 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
use crate::errors;
use crate::errors::DenoError;
#[cfg(test)]
use futures::future::{loop_fn, Loop};
use futures::{future, Future, Stream};
use hyper;
@ -45,8 +46,13 @@ fn resolve_uri_from_location(base_uri: &Uri, location: &str) -> Uri {
} else {
// assuming path-noscheme | path-empty
let mut new_uri_parts = base_uri.clone().into_parts();
new_uri_parts.path_and_query =
Some(format!("{}/{}", base_uri.path(), location).parse().unwrap());
let base_uri_path_str = base_uri.path().to_owned();
let segs: Vec<&str> = base_uri_path_str.rsplitn(2, "/").collect();
new_uri_parts.path_and_query = Some(
format!("{}/{}", segs.last().unwrap_or(&""), location)
.parse()
.unwrap(),
);
Uri::from_parts(new_uri_parts).unwrap()
}
}
@ -61,6 +67,74 @@ pub fn fetch_sync_string(module_name: &str) -> DenoResult<(String, String)> {
tokio_util::block_on(fetch_string(module_name))
}
pub enum FetchOnceResult {
// (code, maybe_content_type)
Code(String, Option<String>),
Redirect(http::uri::Uri),
}
/// Asynchronously fetchs the given HTTP URL one pass only.
/// If no redirect is present and no error occurs,
/// yields Code(code, maybe_content_type).
/// If redirect occurs, does not follow and
/// yields Redirect(url).
pub fn fetch_string_once(
url: http::uri::Uri,
) -> impl Future<Item = FetchOnceResult, Error = DenoError> {
type FetchAttempt = (Option<String>, Option<String>, Option<FetchOnceResult>);
let client = get_client();
client
.get(url.clone())
.map_err(DenoError::from)
.and_then(move |response| -> Box<dyn Future<Item = FetchAttempt, Error = DenoError> + Send> {
if response.status().is_redirection() {
let location_string = response
.headers()
.get("location")
.expect("url redirection should provide 'location' header")
.to_str()
.unwrap()
.to_string();
debug!("Redirecting to {}...", &location_string);
let new_url = resolve_uri_from_location(&url, &location_string);
// Boxed trait object turns out to be the savior for 2+ types yielding same results.
return Box::new(
future::ok(None).join3(
future::ok(None),
future::ok(Some(FetchOnceResult::Redirect(new_url))
))
);
} else if response.status().is_client_error() || response.status().is_server_error() {
return Box::new(future::err(
errors::new(errors::ErrorKind::Other,
format!("Import '{}' failed: {}", &url, response.status()))
));
}
let content_type = response
.headers()
.get(CONTENT_TYPE)
.map(|content_type| content_type.to_str().unwrap().to_owned());
let body = response
.into_body()
.concat2()
.map(|body| String::from_utf8(body.to_vec()).ok())
.map_err(DenoError::from);
Box::new(body.join3(
future::ok(content_type),
future::ok(None)
))
})
.and_then(move |(maybe_code, maybe_content_type, maybe_redirect)| {
if let Some(redirect) = maybe_redirect {
future::ok(redirect)
} else {
// maybe_code should always contain code here!
future::ok(FetchOnceResult::Code(maybe_code.unwrap(), maybe_content_type))
}
})
}
#[cfg(test)]
/// Asynchronously fetchs the given HTTP URL. Returns (content, media_type).
pub fn fetch_string(
module_name: &str,
@ -182,5 +256,5 @@ fn test_resolve_uri_from_location_relative_3() {
let url = "http://deno.land/x".parse::<Uri>().unwrap();
let new_uri = resolve_uri_from_location(&url, "z");
assert_eq!(new_uri.host().unwrap(), "deno.land");
assert_eq!(new_uri.path(), "/x/z");
assert_eq!(new_uri.path(), "/z");
}

29
cli/isolate.rs
View file

@ -84,6 +84,15 @@ impl<B: DenoBehavior> Isolate<B> {
&out.js_source(),
)?;
// The resolved module is an alias to another module (due to redirects).
// Save such alias to the module map.
if out.module_redirect_source_name.is_some() {
self.mod_alias(
&out.module_redirect_source_name.clone().unwrap(),
&out.module_name,
);
}
self.mod_load_deps(child_id)?;
}
}
@ -117,10 +126,23 @@ impl<B: DenoBehavior> Isolate<B> {
let out = fetch_module_meta_data_and_maybe_compile(&self.state, url, ".")
.map_err(RustOrJsError::from)?;
// Be careful.
// url might not match the actual out.module_name
// due to the mechanism of redirection.
let id = self
.mod_new_and_register(true, &out.module_name.clone(), &out.js_source())
.map_err(RustOrJsError::from)?;
// The resolved module is an alias to another module (due to redirects).
// Save such alias to the module map.
if out.module_redirect_source_name.is_some() {
self.mod_alias(
&out.module_redirect_source_name.clone().unwrap(),
&out.module_name,
);
}
self.mod_load_deps(id)?;
let state = self.state.clone();
@ -153,6 +175,13 @@ impl<B: DenoBehavior> Isolate<B> {
Ok(id)
}
/// Create an alias for another module.
/// The alias could later be used to grab the module
/// which `target` points to.
fn mod_alias(&self, name: &str, target: &str) {
self.state.modules.lock().unwrap().alias(name, target);
}
pub fn print_file_info(&self, module: &str) {
let m = self.state.modules.lock().unwrap();
m.print_file_info(&self.state.dir, module.to_string());

68
cli/modules.rs
View file

@ -12,23 +12,78 @@ pub struct ModuleInfo {
children: Vec<deno_mod>,
}
/// A symbolic module entity.
pub enum SymbolicModule {
/// This module is an alias to another module.
/// This is useful such that multiple names could point to
/// the same underlying module (particularly due to redirects).
Alias(String),
/// This module associates with a V8 module by id.
Mod(deno_mod),
}
#[derive(Default)]
/// Alias-able module name map
pub struct ModuleNameMap {
inner: HashMap<String, SymbolicModule>,
}
impl ModuleNameMap {
pub fn new() -> Self {
ModuleNameMap {
inner: HashMap::new(),
}
}
/// Get the id of a module.
/// If this module is internally represented as an alias,
/// follow the alias chain to get the final module id.
pub fn get(&self, name: &str) -> Option<deno_mod> {
let mut mod_name = name;
loop {
let cond = self.inner.get(mod_name);
match cond {
Some(SymbolicModule::Alias(target)) => {
mod_name = target;
}
Some(SymbolicModule::Mod(mod_id)) => {
return Some(*mod_id);
}
_ => {
return None;
}
}
}
}
/// Insert a name assocated module id.
pub fn insert(&mut self, name: String, id: deno_mod) {
self.inner.insert(name, SymbolicModule::Mod(id));
}
/// Create an alias to another module.
pub fn alias(&mut self, name: String, target: String) {
self.inner.insert(name, SymbolicModule::Alias(target));
}
}
/// A collection of JS modules.
#[derive(Default)]
pub struct Modules {
pub info: HashMap<deno_mod, ModuleInfo>,
pub by_name: HashMap<String, deno_mod>,
pub by_name: ModuleNameMap,
}
impl Modules {
pub fn new() -> Modules {
Self {
info: HashMap::new(),
by_name: HashMap::new(),
by_name: ModuleNameMap::new(),
}
}
pub fn get_id(&self, name: &str) -> Option<deno_mod> {
self.by_name.get(name).cloned()
self.by_name.get(name)
}
pub fn get_children(&self, id: deno_mod) -> Option<&Vec<deno_mod>> {
@ -56,6 +111,10 @@ impl Modules {
);
}
pub fn alias(&mut self, name: &str, target: &str) {
self.by_name.alias(name.to_owned(), target.to_owned());
}
pub fn resolve_cb(
&mut self,
deno_dir: &DenoDir,
@ -78,8 +137,7 @@ impl Modules {
}
let (name, _local_filename) = r.unwrap();
if let Some(id) = self.by_name.get(&name) {
let child_id = *id;
if let Some(child_id) = self.by_name.get(&name) {
info.children.push(child_id);
return child_id;
} else {

0
tests/023_no_ext_with_mime → tests/023_no_ext_with_headers
View file

1
tests/023_no_ext_with_headers.headers.json Normal file
View file

@ -0,0 +1 @@
{ "mime_type": "application/javascript" }

0
tests/023_no_ext_with_mime.out → tests/023_no_ext_with_headers.out
View file

2
tests/023_no_ext_with_headers.test Normal file
View file

@ -0,0 +1,2 @@
args: tests/023_no_ext_with_headers --reload
output: tests/023_no_ext_with_headers.out

1
tests/023_no_ext_with_mime.mime
View file

@ -1 +0,0 @@
application/javascript

2
tests/023_no_ext_with_mime.test
View file

@ -1,2 +0,0 @@
args: tests/023_no_ext_with_mime --reload
output: tests/023_no_ext_with_mime.out

2
tests/024_import_no_ext_with_headers.test Normal file
View file

@ -0,0 +1,2 @@
args: tests/024_import_no_ext_with_headers.ts --reload
output: tests/024_import_no_ext_with_headers.ts.out

1
tests/024_import_no_ext_with_headers.ts Normal file
View file

@ -0,0 +1 @@
import "./023_no_ext_with_headers";

0
tests/024_import_no_ext_with_mime.ts.out → tests/024_import_no_ext_with_headers.ts.out
View file

2
tests/024_import_no_ext_with_mime.test
View file

@ -1,2 +0,0 @@
args: tests/024_import_no_ext_with_mime.ts --reload
output: tests/024_import_no_ext_with_mime.ts.out

1
tests/024_import_no_ext_with_mime.ts
View file

@ -1 +0,0 @@
import "./023_no_ext_with_mime";

2
tests/026_redirect_javascript.js Normal file
View file

@ -0,0 +1,2 @@
import { value } from "http://localhost:4547/redirects/redirect3.js";
console.log(value);

1
tests/026_redirect_javascript.js.out Normal file
View file

@ -0,0 +1 @@
3 imports 1

2
tests/026_redirect_javascript.js.test Normal file
View file

@ -0,0 +1,2 @@
args: tests/026_redirect_javascript.js --reload
output: tests/026_redirect_javascript.js.out

2
tests/027_redirect_typescript.ts Normal file
View file

@ -0,0 +1,2 @@
import { value } from "http://localhost:4547/redirects/redirect4.ts";
console.log(value);

1
tests/027_redirect_typescript.ts.out Normal file
View file

@ -0,0 +1 @@
4 imports 1

2
tests/027_redirect_typescript.ts.test Normal file
View file

@ -0,0 +1,2 @@
args: tests/027_redirect_typescript.ts --reload
output: tests/027_redirect_typescript.ts.out

1
tests/subdir/redirects/redirect1.js Normal file
View file

@ -0,0 +1 @@
export const redirect = 1;

1
tests/subdir/redirects/redirect1.ts Normal file
View file

@ -0,0 +1 @@
export const redirect = 1;

1
tests/subdir/redirects/redirect2.js Normal file
View file

@ -0,0 +1 @@
import "./redirect1.js";

2
tests/subdir/redirects/redirect3.js Normal file
View file

@ -0,0 +1,2 @@
import { redirect } from "./redirect1.js";
export const value = `3 imports ${redirect}`;

2
tests/subdir/redirects/redirect4.ts Normal file
View file

@ -0,0 +1,2 @@
import { redirect } from "./redirect1.ts";
export const value: string = `4 imports ${redirect}`;

40
tools/http_server.py
View file

@ -12,6 +12,8 @@ from time import sleep
PORT = 4545
REDIRECT_PORT = 4546
ANOTHER_REDIRECT_PORT = 4547
DOUBLE_REDIRECTS_PORT = 4548
class ContentTypeHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
@ -99,24 +101,42 @@ def server():
return s
def redirect_server():
def base_redirect_server(host_port, target_port, extra_path_segment=""):
os.chdir(root_path)
target_host = "http://localhost:%d" % PORT
target_host = "http://localhost:%d" % target_port
class RedirectHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_GET(self):
self.send_response(301)
self.send_header('Location', target_host + self.path)
self.send_header('Location',
target_host + extra_path_segment + self.path)
self.end_headers()
Handler = RedirectHandler
SocketServer.TCPServer.allow_reuse_address = True
s = SocketServer.TCPServer(("", REDIRECT_PORT), Handler)
s = SocketServer.TCPServer(("", host_port), Handler)
print "redirect server http://localhost:%d/ -> http://localhost:%d/" % (
REDIRECT_PORT, PORT)
host_port, target_port)
return s
# redirect server
def redirect_server():
return base_redirect_server(REDIRECT_PORT, PORT)
# another redirect server pointing to the same port as the one above
# BUT with an extra subdir path
def another_redirect_server():
return base_redirect_server(
ANOTHER_REDIRECT_PORT, PORT, extra_path_segment="/tests/subdir")
# redirect server that points to another redirect server
def double_redirects_server():
return base_redirect_server(DOUBLE_REDIRECTS_PORT, REDIRECT_PORT)
def spawn():
# Main http server
s = server()
@ -128,6 +148,16 @@ def spawn():
r_thread = Thread(target=rs.serve_forever)
r_thread.daemon = True
r_thread.start()
# Another redirect server
ars = another_redirect_server()
ar_thread = Thread(target=ars.serve_forever)
ar_thread.daemon = True
ar_thread.start()
# Double redirects server
drs = double_redirects_server()
dr_thread = Thread(target=drs.serve_forever)
dr_thread.daemon = True
dr_thread.start()
sleep(1) # TODO I'm too lazy to figure out how to do this properly.
return thread