mirror of
https://github.com/denoland/deno.git
synced 2024-12-23 07:44:48 -05:00
Add more async module loading function (#1974)
* get_source_code_async * fetch_module_meta_data_async * fetch_module_meta_data_and_maybe_compile_async
This commit is contained in:
Ryan Dahl
GitHub
parent
48bf419669
commit
4c831f1eb5
3 changed files with 226 additions and 147 deletions
290
cli/deno_dir.rs
290
cli/deno_dir.rs
|
|
@ -11,6 +11,7 @@ use crate::msg;
|
|||
use crate::tokio_util;
|
||||
use crate::version;
|
||||
use dirs;
|
||||
use futures::future::Either;
|
||||
use futures::Future;
|
||||
use ring;
|
||||
use std;
|
||||
|
|
@ -117,22 +118,6 @@ impl DenoDir {
|
|||
)
|
||||
}
|
||||
|
||||
fn load_cache(
|
||||
self: &Self,
|
||||
filename: &str,
|
||||
source_code: &[u8],
|
||||
) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> {
|
||||
let (output_code, source_map) = self.cache_path(filename, source_code);
|
||||
debug!(
|
||||
"load_cache code: {} map: {}",
|
||||
output_code.display(),
|
||||
source_map.display()
|
||||
);
|
||||
let read_output_code = fs::read(&output_code)?;
|
||||
let read_source_map = fs::read(&source_map)?;
|
||||
Ok((read_output_code, read_source_map))
|
||||
}
|
||||
|
||||
pub fn code_cache(
|
||||
self: &Self,
|
||||
module_meta_data: &ModuleMetaData,
|
||||
|
|
@ -158,13 +143,14 @@ impl DenoDir {
|
|||
}
|
||||
}
|
||||
|
||||
// Prototype: https://github.com/denoland/deno/blob/golang/os.go#L122-L138
|
||||
fn get_source_code(
|
||||
fn get_source_code_async(
|
||||
self: &Self,
|
||||
module_name: &str,
|
||||
filename: &str,
|
||||
) -> DenoResult<ModuleMetaData> {
|
||||
let is_module_remote = is_remote(module_name);
|
||||
) -> impl Future<Item = ModuleMetaData, Error = DenoError> {
|
||||
let filename = filename.to_string();
|
||||
let module_name = module_name.to_string();
|
||||
let is_module_remote = is_remote(&module_name);
|
||||
// We try fetch local. Two cases:
|
||||
// 1. This is a remote module, but no reload provided
|
||||
// 2. This is a local module
|
||||
|
|
@ -173,114 +159,165 @@ impl DenoDir {
|
|||
"fetch local or reload {} is_module_remote {}",
|
||||
module_name, is_module_remote
|
||||
);
|
||||
match fetch_local_source(&module_name, &filename)? {
|
||||
Some(output) => {
|
||||
// Note that local fetch is done synchronously.
|
||||
match fetch_local_source(&module_name, &filename) {
|
||||
Ok(Some(output)) => {
|
||||
debug!("found local source ");
|
||||
return Ok(output);
|
||||
return Either::A(futures::future::ok(output));
|
||||
}
|
||||
None => {
|
||||
Ok(None) => {
|
||||
debug!("fetch_local_source returned None");
|
||||
}
|
||||
Err(err) => {
|
||||
return Either::A(futures::future::err(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If not remote file, stop here!
|
||||
if !is_module_remote {
|
||||
debug!("not remote file stop here");
|
||||
return Err(DenoError::from(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("cannot find local file '{}'", filename),
|
||||
return Either::A(futures::future::err(DenoError::from(
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("cannot find local file '{}'", &filename),
|
||||
),
|
||||
)));
|
||||
}
|
||||
|
||||
debug!("is remote but didn't find module");
|
||||
|
||||
let filename2 = filename.clone();
|
||||
// not cached/local, try remote
|
||||
let maybe_remote_source = fetch_remote_source(&module_name, &filename)?;
|
||||
if let Some(output) = maybe_remote_source {
|
||||
return Ok(output);
|
||||
}
|
||||
Err(DenoError::from(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("cannot find remote file '{}'", filename),
|
||||
)))
|
||||
let f = fetch_remote_source_async(&module_name, &filename).and_then(
|
||||
move |maybe_remote_source| match maybe_remote_source {
|
||||
Some(output) => Ok(output),
|
||||
None => Err(DenoError::from(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("cannot find remote file '{}'", &filename2),
|
||||
))),
|
||||
},
|
||||
);
|
||||
Either::B(f)
|
||||
}
|
||||
|
||||
pub fn fetch_module_meta_data(
|
||||
#[cfg(test)]
|
||||
/// Synchronous version of get_source_code_async
|
||||
/// This function is deprecated.
|
||||
fn get_source_code(
|
||||
self: &Self,
|
||||
module_name: &str,
|
||||
filename: &str,
|
||||
) -> DenoResult<ModuleMetaData> {
|
||||
tokio_util::block_on(self.get_source_code_async(module_name, filename))
|
||||
}
|
||||
|
||||
pub fn fetch_module_meta_data_async(
|
||||
self: &Self,
|
||||
specifier: &str,
|
||||
referrer: &str,
|
||||
) -> Result<ModuleMetaData, errors::DenoError> {
|
||||
) -> impl Future<Item = ModuleMetaData, Error = errors::DenoError> {
|
||||
debug!(
|
||||
"fetch_module_meta_data. specifier {} referrer {}",
|
||||
specifier, referrer
|
||||
);
|
||||
|
||||
let (module_name, filename) = self.resolve_module(specifier, referrer)?;
|
||||
let specifier = specifier.to_string();
|
||||
let referrer = referrer.to_string();
|
||||
|
||||
let result = self.get_source_code(module_name.as_str(), filename.as_str());
|
||||
let mut out = match result {
|
||||
Ok(out) => out,
|
||||
Err(err) => {
|
||||
if err.kind() == ErrorKind::NotFound {
|
||||
// For NotFound, change the message to something better.
|
||||
return Err(errors::new(
|
||||
ErrorKind::NotFound,
|
||||
format!(
|
||||
"Cannot resolve module \"{}\" from \"{}\"",
|
||||
specifier, referrer
|
||||
),
|
||||
));
|
||||
} else {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if out.source_code.starts_with("#!".as_bytes()) {
|
||||
out.source_code = filter_shebang(out.source_code);
|
||||
let result = self.resolve_module(&specifier, &referrer);
|
||||
if let Err(err) = result {
|
||||
return Either::A(futures::future::err(DenoError::from(err)));
|
||||
}
|
||||
let (module_name, filename) = result.unwrap();
|
||||
|
||||
if out.media_type != msg::MediaType::TypeScript {
|
||||
return Ok(out);
|
||||
}
|
||||
let gen = self.gen.clone();
|
||||
let recompile = self.recompile;
|
||||
|
||||
let (output_code_filename, output_source_map_filename) =
|
||||
self.cache_path(&out.filename, &out.source_code);
|
||||
let mut maybe_output_code = None;
|
||||
let mut maybe_source_map = None;
|
||||
Either::B(
|
||||
self
|
||||
.get_source_code_async(module_name.as_str(), filename.as_str())
|
||||
.then(move |result| {
|
||||
let mut out = match result {
|
||||
Ok(out) => out,
|
||||
Err(err) => {
|
||||
if err.kind() == ErrorKind::NotFound {
|
||||
// For NotFound, change the message to something better.
|
||||
return Err(errors::new(
|
||||
ErrorKind::NotFound,
|
||||
format!(
|
||||
"Cannot resolve module \"{}\" from \"{}\"",
|
||||
specifier, referrer
|
||||
),
|
||||
));
|
||||
} else {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if !self.recompile {
|
||||
let result = self.load_cache(out.filename.as_str(), &out.source_code);
|
||||
match result {
|
||||
Err(err) => {
|
||||
if err.kind() == std::io::ErrorKind::NotFound {
|
||||
return Ok(out);
|
||||
} else {
|
||||
return Err(err.into());
|
||||
if out.source_code.starts_with("#!".as_bytes()) {
|
||||
out.source_code = filter_shebang(out.source_code);
|
||||
}
|
||||
}
|
||||
Ok((output_code, source_map)) => {
|
||||
maybe_output_code = Some(output_code);
|
||||
maybe_source_map = Some(source_map);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ModuleMetaData {
|
||||
module_name: out.module_name,
|
||||
filename: out.filename,
|
||||
media_type: out.media_type,
|
||||
source_code: out.source_code,
|
||||
maybe_output_code_filename: output_code_filename
|
||||
.to_str()
|
||||
.map(String::from),
|
||||
maybe_output_code,
|
||||
maybe_source_map_filename: output_source_map_filename
|
||||
.to_str()
|
||||
.map(String::from),
|
||||
maybe_source_map,
|
||||
})
|
||||
if out.media_type != msg::MediaType::TypeScript {
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
let cache_key =
|
||||
source_code_hash(&out.filename, &out.source_code, version::DENO);
|
||||
let (output_code_filename, output_source_map_filename) = (
|
||||
gen.join(cache_key.to_string() + ".js"),
|
||||
gen.join(cache_key.to_string() + ".js.map"),
|
||||
);
|
||||
|
||||
let mut maybe_output_code = None;
|
||||
let mut maybe_source_map = None;
|
||||
|
||||
if !recompile {
|
||||
let result =
|
||||
load_cache2(&output_code_filename, &output_source_map_filename);
|
||||
match result {
|
||||
Err(err) => {
|
||||
if err.kind() == std::io::ErrorKind::NotFound {
|
||||
return Ok(out);
|
||||
} else {
|
||||
return Err(err.into());
|
||||
}
|
||||
}
|
||||
Ok((output_code, source_map)) => {
|
||||
maybe_output_code = Some(output_code);
|
||||
maybe_source_map = Some(source_map);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ModuleMetaData {
|
||||
module_name: out.module_name,
|
||||
filename: out.filename,
|
||||
media_type: out.media_type,
|
||||
source_code: out.source_code,
|
||||
maybe_output_code_filename: output_code_filename
|
||||
.to_str()
|
||||
.map(String::from),
|
||||
maybe_output_code,
|
||||
maybe_source_map_filename: output_source_map_filename
|
||||
.to_str()
|
||||
.map(String::from),
|
||||
maybe_source_map,
|
||||
})
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
/// Synchronous version of fetch_module_meta_data_async
|
||||
/// This function is deprecated.
|
||||
pub fn fetch_module_meta_data(
|
||||
self: &Self,
|
||||
specifier: &str,
|
||||
referrer: &str,
|
||||
) -> Result<ModuleMetaData, errors::DenoError> {
|
||||
tokio_util::block_on(self.fetch_module_meta_data_async(specifier, referrer))
|
||||
}
|
||||
|
||||
// Prototype: https://github.com/denoland/deno/blob/golang/os.go#L56-L68
|
||||
|
|
@ -409,6 +446,20 @@ fn get_cache_filename(basedir: &Path, url: &Url) -> PathBuf {
|
|||
out
|
||||
}
|
||||
|
||||
fn load_cache2(
|
||||
js_filename: &PathBuf,
|
||||
map_filename: &PathBuf,
|
||||
) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> {
|
||||
debug!(
|
||||
"load_cache code: {} map: {}",
|
||||
js_filename.display(),
|
||||
map_filename.display()
|
||||
);
|
||||
let read_output_code = fs::read(&js_filename)?;
|
||||
let read_source_map = fs::read(&map_filename)?;
|
||||
Ok((read_output_code, read_source_map))
|
||||
}
|
||||
|
||||
fn source_code_hash(
|
||||
filename: &str,
|
||||
source_code: &[u8],
|
||||
|
|
@ -546,6 +597,7 @@ fn fetch_remote_source_async(
|
|||
|
||||
// Prototype https://github.com/denoland/deno/blob/golang/deno_dir.go#L37-L73
|
||||
/// Fetch remote source code.
|
||||
#[cfg(test)]
|
||||
fn fetch_remote_source(
|
||||
module_name: &str,
|
||||
filename: &str,
|
||||
|
|
@ -988,19 +1040,21 @@ mod tests {
|
|||
let cwd = std::env::current_dir().unwrap();
|
||||
let cwd_string = String::from(cwd.to_str().unwrap()) + "/";
|
||||
|
||||
// Test failure case.
|
||||
let specifier = "hello.ts";
|
||||
let referrer = add_root!("/baddir/badfile.ts");
|
||||
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
|
||||
assert!(r.is_err());
|
||||
tokio_util::init(|| {
|
||||
// Test failure case.
|
||||
let specifier = "hello.ts";
|
||||
let referrer = add_root!("/baddir/badfile.ts");
|
||||
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
|
||||
assert!(r.is_err());
|
||||
|
||||
// Assuming cwd is the deno repo root.
|
||||
let specifier = "./js/main.ts";
|
||||
let referrer = cwd_string.as_str();
|
||||
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
|
||||
assert!(r.is_ok());
|
||||
//let fetch_module_meta_data_output = r.unwrap();
|
||||
//println!("fetch_module_meta_data_output {:?}", fetch_module_meta_data_output);
|
||||
// Assuming cwd is the deno repo root.
|
||||
let specifier = "./js/main.ts";
|
||||
let referrer = cwd_string.as_str();
|
||||
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
|
||||
assert!(r.is_ok());
|
||||
//let fetch_module_meta_data_output = r.unwrap();
|
||||
//println!("fetch_module_meta_data_output {:?}", fetch_module_meta_data_output);
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -1011,17 +1065,19 @@ mod tests {
|
|||
let cwd = std::env::current_dir().unwrap();
|
||||
let cwd_string = String::from(cwd.to_str().unwrap()) + "/";
|
||||
|
||||
// Test failure case.
|
||||
let specifier = "hello.ts";
|
||||
let referrer = add_root!("/baddir/badfile.ts");
|
||||
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
|
||||
assert!(r.is_err());
|
||||
tokio_util::init(|| {
|
||||
// Test failure case.
|
||||
let specifier = "hello.ts";
|
||||
let referrer = add_root!("/baddir/badfile.ts");
|
||||
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
|
||||
assert!(r.is_err());
|
||||
|
||||
// Assuming cwd is the deno repo root.
|
||||
let specifier = "./js/main.ts";
|
||||
let referrer = cwd_string.as_str();
|
||||
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
|
||||
assert!(r.is_ok());
|
||||
// Assuming cwd is the deno repo root.
|
||||
let specifier = "./js/main.ts";
|
||||
let referrer = cwd_string.as_str();
|
||||
let r = deno_dir.fetch_module_meta_data(specifier, referrer);
|
||||
assert!(r.is_ok());
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use crate::isolate_state::IsolateState;
|
|||
use crate::isolate_state::IsolateStateContainer;
|
||||
use crate::js_errors;
|
||||
use crate::msg;
|
||||
use crate::tokio_util;
|
||||
use deno_core;
|
||||
use deno_core::deno_mod;
|
||||
use deno_core::Behavior;
|
||||
|
|
@ -162,23 +163,39 @@ impl<B: DenoBehavior> Future for Isolate<B> {
|
|||
self.inner.poll().map_err(|err| self.apply_source_map(err))
|
||||
}
|
||||
}
|
||||
fn fetch_module_meta_data_and_maybe_compile_async(
|
||||
state: &Arc<IsolateState>,
|
||||
specifier: &str,
|
||||
referrer: &str,
|
||||
) -> impl Future<Item = ModuleMetaData, Error = DenoError> {
|
||||
let state_ = state.clone();
|
||||
let specifier = specifier.to_string();
|
||||
let referrer = referrer.to_string();
|
||||
state
|
||||
.dir
|
||||
.fetch_module_meta_data_async(&specifier, &referrer)
|
||||
.and_then(move |mut out| {
|
||||
if (out.media_type == msg::MediaType::TypeScript
|
||||
&& out.maybe_output_code.is_none())
|
||||
|| state_.flags.recompile
|
||||
{
|
||||
debug!(">>>>> compile_sync START");
|
||||
out = compile_sync(state_.clone(), &specifier, &referrer, &out);
|
||||
debug!(">>>>> compile_sync END");
|
||||
state_.dir.code_cache(&out)?;
|
||||
}
|
||||
Ok(out)
|
||||
})
|
||||
}
|
||||
|
||||
fn fetch_module_meta_data_and_maybe_compile(
|
||||
state: &Arc<IsolateState>,
|
||||
specifier: &str,
|
||||
referrer: &str,
|
||||
) -> Result<ModuleMetaData, DenoError> {
|
||||
let mut out = state.dir.fetch_module_meta_data(specifier, referrer)?;
|
||||
if (out.media_type == msg::MediaType::TypeScript
|
||||
&& out.maybe_output_code.is_none())
|
||||
|| state.flags.recompile
|
||||
{
|
||||
debug!(">>>>> compile_sync START");
|
||||
out = compile_sync(state.clone(), specifier, &referrer, &out);
|
||||
debug!(">>>>> compile_sync END");
|
||||
state.dir.code_cache(&out)?;
|
||||
}
|
||||
Ok(out)
|
||||
tokio_util::block_on(fetch_module_meta_data_and_maybe_compile_async(
|
||||
state, specifier, referrer,
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -186,7 +203,6 @@ mod tests {
|
|||
use super::*;
|
||||
use crate::cli_behavior::CliBehavior;
|
||||
use crate::flags;
|
||||
use crate::tokio_util;
|
||||
use futures::future::lazy;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
|
|
|
|||
43
cli/ops.rs
43
cli/ops.rs
|
|
@ -1952,12 +1952,14 @@ mod tests {
|
|||
msg::finish_base_buffer(builder, base);
|
||||
let data = builder.finished_data();
|
||||
let final_msg = msg::get_root_as_base(&data);
|
||||
let fetch_result =
|
||||
op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
|
||||
match fetch_result {
|
||||
Ok(_) => assert!(true),
|
||||
Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
|
||||
}
|
||||
tokio_util::init(move || {
|
||||
let fetch_result =
|
||||
op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
|
||||
match fetch_result {
|
||||
Ok(_) => assert!(true),
|
||||
Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -1986,12 +1988,14 @@ mod tests {
|
|||
msg::finish_base_buffer(builder, base);
|
||||
let data = builder.finished_data();
|
||||
let final_msg = msg::get_root_as_base(&data);
|
||||
let fetch_result =
|
||||
op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
|
||||
match fetch_result {
|
||||
Ok(_) => assert!(true),
|
||||
Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
|
||||
}
|
||||
tokio_util::init(move || {
|
||||
let fetch_result =
|
||||
op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
|
||||
match fetch_result {
|
||||
Ok(_) => assert!(true),
|
||||
Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -2019,12 +2023,15 @@ mod tests {
|
|||
msg::finish_base_buffer(builder, base);
|
||||
let data = builder.finished_data();
|
||||
let final_msg = msg::get_root_as_base(&data);
|
||||
let fetch_result =
|
||||
op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
|
||||
match fetch_result {
|
||||
Ok(_) => assert!(true),
|
||||
Err(e) => assert!(e.to_string() != permission_denied().to_string()),
|
||||
}
|
||||
|
||||
tokio_util::init(move || {
|
||||
let fetch_result =
|
||||
op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
|
||||
match fetch_result {
|
||||
Ok(_) => assert!(true),
|
||||
Err(e) => assert!(e.to_string() != permission_denied().to_string()),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
|
|
|||
Loading…
Reference in a new issue