diff --git a/cli/state.rs b/cli/state.rs index 32c027c030..8003ea732f 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -11,11 +11,13 @@ use crate::permissions::DenoPermissions; use crate::web_worker::WebWorkerHandle; use deno_core::Buf; use deno_core::ErrBox; +use deno_core::ModuleLoadId; use deno_core::ModuleLoader; use deno_core::ModuleSpecifier; use deno_core::Op; use deno_core::ZeroCopyBuf; use futures::future::FutureExt; +use futures::Future; use rand::rngs::StdRng; use rand::SeedableRng; use serde_json::Value; @@ -28,7 +30,6 @@ use std::rc::Rc; use std::str; use std::thread::JoinHandle; use std::time::Instant; - #[derive(Copy, Clone, Eq, PartialEq)] pub enum DebugType { /// Can be debugged, will wait for debugger when --inspect-brk given. @@ -309,6 +310,27 @@ impl ModuleLoader for State { fut.boxed_local() } + + fn prepare_load( + &self, + _load_id: ModuleLoadId, + _module_specifier: &ModuleSpecifier, + _maybe_referrer: Option, + _is_dyn_import: bool, + ) -> Pin>>> { + // TODO(bartlomieju): + // 1. recursively: + // a) resolve specifier + // b) check permission if dynamic import + // c) fetch/download source code + // d) parse the source code and extract all import/exports (dependencies) + // e) add discovered deps and loop algorithm until no new dependencies + // are discovered + // 2. run through appropriate compiler giving it access only to + // discovered files + + async { Ok(()) }.boxed_local() + } } impl State { diff --git a/core/es_isolate.rs b/core/es_isolate.rs index be5e2c0c65..2ca344d2ad 100644 --- a/core/es_isolate.rs +++ b/core/es_isolate.rs @@ -35,6 +35,7 @@ use crate::modules::LoadState; use crate::modules::ModuleLoader; use crate::modules::ModuleSource; use crate::modules::Modules; +use crate::modules::PrepareLoadFuture; use crate::modules::RecursiveModuleLoad; pub type ModuleId = i32; @@ -53,6 +54,7 @@ pub struct EsIsolate { pub(crate) dyn_import_map: HashMap>, + preparing_dyn_imports: FuturesUnordered>>, pending_dyn_imports: FuturesUnordered>, waker: AtomicWaker, } @@ -93,6 +95,7 @@ impl EsIsolate { loader, core_isolate, dyn_import_map: HashMap::new(), + preparing_dyn_imports: FuturesUnordered::new(), pending_dyn_imports: FuturesUnordered::new(), waker: AtomicWaker::new(), }; @@ -315,7 +318,8 @@ impl EsIsolate { ); self.dyn_import_map.insert(load.id, resolver_handle); self.waker.wake(); - self.pending_dyn_imports.push(load.into_future()); + let fut = load.prepare().boxed_local(); + self.preparing_dyn_imports.push(fut); } fn dyn_import_error( @@ -387,6 +391,33 @@ impl EsIsolate { Ok(()) } + fn prepare_dyn_imports( + &mut self, + cx: &mut Context, + ) -> Poll> { + loop { + match self.preparing_dyn_imports.poll_next_unpin(cx) { + Poll::Pending | Poll::Ready(None) => { + // There are no active dynamic import loaders, or none are ready. + return Poll::Ready(Ok(())); + } + Poll::Ready(Some(prepare_poll)) => { + let dyn_import_id = prepare_poll.0; + let prepare_result = prepare_poll.1; + + match prepare_result { + Ok(load) => { + self.pending_dyn_imports.push(load.into_future()); + } + Err(err) => { + self.dyn_import_error(dyn_import_id, err)?; + } + } + } + } + } + } + fn poll_dyn_imports(&mut self, cx: &mut Context) -> Poll> { loop { match self.pending_dyn_imports.poll_next_unpin(cx) { @@ -511,11 +542,14 @@ impl EsIsolate { specifier: &ModuleSpecifier, code: Option, ) -> Result { - let mut load = RecursiveModuleLoad::main( + let load = RecursiveModuleLoad::main( &specifier.to_string(), code, self.loader.clone(), ); + let (_load_id, prepare_result) = load.prepare().await; + + let mut load = prepare_result?; while let Some(info_result) = load.next().await { let info = info_result?; @@ -535,7 +569,11 @@ impl Future for EsIsolate { inner.waker.register(cx.waker()); - // If there are any pending dyn_import futures, do those first. + if !inner.preparing_dyn_imports.is_empty() { + let poll_imports = inner.prepare_dyn_imports(cx)?; + assert!(poll_imports.is_ready()); + } + if !inner.pending_dyn_imports.is_empty() { let poll_imports = inner.poll_dyn_imports(cx)?; assert!(poll_imports.is_ready()); @@ -543,7 +581,9 @@ impl Future for EsIsolate { match ready!(inner.core_isolate.poll_unpin(cx)) { Ok(()) => { - if inner.pending_dyn_imports.is_empty() { + if inner.pending_dyn_imports.is_empty() + && inner.preparing_dyn_imports.is_empty() + { Poll::Ready(Ok(())) } else { Poll::Pending @@ -720,7 +760,7 @@ pub mod tests { if let Poll::Ready(Ok(_)) = result { unreachable!(); } - assert_eq!(count.load(Ordering::Relaxed), 1); + assert_eq!(count.load(Ordering::Relaxed), 2); }) } @@ -816,6 +856,7 @@ pub mod tests { fn dyn_import_ok() { #[derive(Clone, Default)] struct DynImportOkLoader { + pub prepare_load_count: Arc, pub resolve_count: Arc, pub load_count: Arc, } @@ -828,11 +869,8 @@ pub mod tests { _is_main: bool, ) -> Result { let c = self.resolve_count.fetch_add(1, Ordering::Relaxed); - match c { - 0 => assert_eq!(specifier, "./b.js"), - 1 => assert_eq!(specifier, "./b.js"), - _ => unreachable!(), - } + assert!(c < 4); + assert_eq!(specifier, "./b.js"); assert_eq!(referrer, "file:///dyn_import3.js"); let s = ModuleSpecifier::resolve_import(specifier, referrer).unwrap(); Ok(s) @@ -852,10 +890,22 @@ pub mod tests { }; async move { Ok(info) }.boxed() } + + fn prepare_load( + &self, + _load_id: ModuleLoadId, + _module_specifier: &ModuleSpecifier, + _maybe_referrer: Option, + _is_dyn_import: bool, + ) -> Pin>>> { + self.prepare_load_count.fetch_add(1, Ordering::Relaxed); + async { Ok(()) }.boxed_local() + } } run_in_task(|cx| { let loader = Rc::new(DynImportOkLoader::default()); + let prepare_load_count = loader.prepare_load_count.clone(); let resolve_count = loader.resolve_count.clone(); let load_count = loader.load_count.clone(); let mut isolate = EsIsolate::new(loader, StartupData::None, false); @@ -878,17 +928,25 @@ pub mod tests { "#, )); + // First poll runs `prepare_load` hook. + assert!(match isolate.poll_unpin(cx) { + Poll::Pending => true, + _ => false, + }); + assert_eq!(prepare_load_count.load(Ordering::Relaxed), 1); + + // Second poll actually loads modules into the isolate. assert!(match isolate.poll_unpin(cx) { Poll::Ready(Ok(_)) => true, _ => false, }); - assert_eq!(resolve_count.load(Ordering::Relaxed), 2); + assert_eq!(resolve_count.load(Ordering::Relaxed), 4); assert_eq!(load_count.load(Ordering::Relaxed), 2); assert!(match isolate.poll_unpin(cx) { Poll::Ready(Ok(_)) => true, _ => false, }); - assert_eq!(resolve_count.load(Ordering::Relaxed), 2); + assert_eq!(resolve_count.load(Ordering::Relaxed), 4); assert_eq!(load_count.load(Ordering::Relaxed), 2); }) } diff --git a/core/modules.rs b/core/modules.rs index 7e548ec824..632df2dd00 100644 --- a/core/modules.rs +++ b/core/modules.rs @@ -47,6 +47,8 @@ pub struct ModuleSource { pub module_url_found: String, } +pub type PrepareLoadFuture = + dyn Future)>; pub type ModuleSourceFuture = dyn Future>; pub trait ModuleLoader { @@ -74,6 +76,24 @@ pub trait ModuleLoader { maybe_referrer: Option, is_dyn_import: bool, ) -> Pin>; + + /// This hook can be used by implementors to do some preparation + /// work before starting loading of modules. + /// + /// For example implementor might download multiple modules in + /// parallel and transpile them to final JS sources before + /// yielding control back to Isolate. + /// + /// It's not required to implement this method. + fn prepare_load( + &self, + _load_id: ModuleLoadId, + _module_specifier: &ModuleSpecifier, + _maybe_referrer: Option, + _is_dyn_import: bool, + ) -> Pin>>> { + async { Ok(()) }.boxed_local() + } } #[derive(Debug, Eq, PartialEq)] @@ -95,6 +115,8 @@ pub enum LoadState { /// that is consumed by the isolate. pub struct RecursiveModuleLoad { kind: Kind, + // TODO(bartlomieju): in future this value should + // be randomized pub id: ModuleLoadId, pub root_module_id: Option, pub state: LoadState, @@ -142,6 +164,41 @@ impl RecursiveModuleLoad { } } + pub async fn prepare(self) -> (ModuleLoadId, Result) { + let (module_specifier, maybe_referrer) = match self.state { + LoadState::ResolveMain(ref specifier, _) => { + let spec = match self.loader.resolve(specifier, ".", true) { + Ok(spec) => spec, + Err(e) => return (self.id, Err(e)), + }; + (spec, None) + } + LoadState::ResolveImport(ref specifier, ref referrer) => { + let spec = match self.loader.resolve(specifier, referrer, false) { + Ok(spec) => spec, + Err(e) => return (self.id, Err(e)), + }; + (spec, Some(referrer.to_string())) + } + _ => unreachable!(), + }; + + let prepare_result = self + .loader + .prepare_load( + self.id, + &module_specifier, + maybe_referrer, + self.is_dynamic_import(), + ) + .await; + + match prepare_result { + Ok(()) => (self.id, Ok(self)), + Err(e) => (self.id, Err(e)), + } + } + fn add_root(&mut self) -> Result<(), ErrBox> { let module_specifier = match self.state { LoadState::ResolveMain(ref specifier, _) => {