foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-31 11:34:15 -05:00
Code Issues Packages 1 Wiki Activity

refactor(core): add "prepare_load" hook to ModuleLoader trait (#4866)

Browse source 
This PR adds prepare_load hook method to ModuleLoader trait. It allows implementors to perform preparation work before starting actual module loading into isolate. It's meant to be used in CLI; where "transpilation" step will be explicitly performed during prepare_load instead of doing it adhoc for each module if needed.
This commit is contained in:
Bartek Iwańczuk 2020-04-30 14:37:06 +02:00 committed by GitHub
parent 5f8c4d9b68
commit 46bfcbbaa8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 150 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
cli/state.rs
View file

@ -11,11 +11,13 @@ use crate::permissions::DenoPermissions;
use crate::web_worker::WebWorkerHandle;
use deno_core::Buf;
use deno_core::ErrBox;
use deno_core::ModuleLoadId;
use deno_core::ModuleLoader;
use deno_core::ModuleSpecifier;
use deno_core::Op;
use deno_core::ZeroCopyBuf;
use futures::future::FutureExt;
use futures::Future;
use rand::rngs::StdRng;
use rand::SeedableRng;
use serde_json::Value;
@ -28,7 +30,6 @@ use std::rc::Rc;
use std::str;
use std::thread::JoinHandle;
use std::time::Instant;
#[derive(Copy, Clone, Eq, PartialEq)]
pub enum DebugType {
/// Can be debugged, will wait for debugger when --inspect-brk given.
@ -309,6 +310,27 @@ impl ModuleLoader for State {
fut.boxed_local()
}
fn prepare_load(
&self,
_load_id: ModuleLoadId,
_module_specifier: &ModuleSpecifier,
_maybe_referrer: Option<String>,
_is_dyn_import: bool,
) -> Pin<Box<dyn Future<Output = Result<(), ErrBox>>>> {
// TODO(bartlomieju):
// 1. recursively:
// a) resolve specifier
// b) check permission if dynamic import
// c) fetch/download source code
// d) parse the source code and extract all import/exports (dependencies)
// e) add discovered deps and loop algorithm until no new dependencies
// are discovered
// 2. run through appropriate compiler giving it access only to
// discovered files
async { Ok(()) }.boxed_local()
}
}
impl State {

82
core/es_isolate.rs
View file

@ -35,6 +35,7 @@ use crate::modules::LoadState;
use crate::modules::ModuleLoader;
use crate::modules::ModuleSource;
use crate::modules::Modules;
use crate::modules::PrepareLoadFuture;
use crate::modules::RecursiveModuleLoad;
pub type ModuleId = i32;
@ -53,6 +54,7 @@ pub struct EsIsolate {
pub(crate) dyn_import_map:
HashMap<ModuleLoadId, v8::Global<v8::PromiseResolver>>,
preparing_dyn_imports: FuturesUnordered<Pin<Box<PrepareLoadFuture>>>,
pending_dyn_imports: FuturesUnordered<StreamFuture<RecursiveModuleLoad>>,
waker: AtomicWaker,
}
@ -93,6 +95,7 @@ impl EsIsolate {
loader,
core_isolate,
dyn_import_map: HashMap::new(),
preparing_dyn_imports: FuturesUnordered::new(),
pending_dyn_imports: FuturesUnordered::new(),
waker: AtomicWaker::new(),
};
@ -315,7 +318,8 @@ impl EsIsolate {
);
self.dyn_import_map.insert(load.id, resolver_handle);
self.waker.wake();
self.pending_dyn_imports.push(load.into_future());
let fut = load.prepare().boxed_local();
self.preparing_dyn_imports.push(fut);
}
fn dyn_import_error(
@ -387,6 +391,33 @@ impl EsIsolate {
Ok(())
}
fn prepare_dyn_imports(
&mut self,
cx: &mut Context,
) -> Poll<Result<(), ErrBox>> {
loop {
match self.preparing_dyn_imports.poll_next_unpin(cx) {
Poll::Pending | Poll::Ready(None) => {
// There are no active dynamic import loaders, or none are ready.
return Poll::Ready(Ok(()));
}
Poll::Ready(Some(prepare_poll)) => {
let dyn_import_id = prepare_poll.0;
let prepare_result = prepare_poll.1;
match prepare_result {
Ok(load) => {
self.pending_dyn_imports.push(load.into_future());
}
Err(err) => {
self.dyn_import_error(dyn_import_id, err)?;
}
}
}
}
}
}
fn poll_dyn_imports(&mut self, cx: &mut Context) -> Poll<Result<(), ErrBox>> {
loop {
match self.pending_dyn_imports.poll_next_unpin(cx) {
@ -511,11 +542,14 @@ impl EsIsolate {
specifier: &ModuleSpecifier,
code: Option<String>,
) -> Result<ModuleId, ErrBox> {
let mut load = RecursiveModuleLoad::main(
let load = RecursiveModuleLoad::main(
&specifier.to_string(),
code,
self.loader.clone(),
);
let (_load_id, prepare_result) = load.prepare().await;
let mut load = prepare_result?;
while let Some(info_result) = load.next().await {
let info = info_result?;
@ -535,7 +569,11 @@ impl Future for EsIsolate {
inner.waker.register(cx.waker());
// If there are any pending dyn_import futures, do those first.
if !inner.preparing_dyn_imports.is_empty() {
let poll_imports = inner.prepare_dyn_imports(cx)?;
assert!(poll_imports.is_ready());
}
if !inner.pending_dyn_imports.is_empty() {
let poll_imports = inner.poll_dyn_imports(cx)?;
assert!(poll_imports.is_ready());
@ -543,7 +581,9 @@ impl Future for EsIsolate {
match ready!(inner.core_isolate.poll_unpin(cx)) {
Ok(()) => {
if inner.pending_dyn_imports.is_empty() {
if inner.pending_dyn_imports.is_empty()
&& inner.preparing_dyn_imports.is_empty()
{
Poll::Ready(Ok(()))
} else {
Poll::Pending
@ -720,7 +760,7 @@ pub mod tests {
if let Poll::Ready(Ok(_)) = result {
unreachable!();
}
assert_eq!(count.load(Ordering::Relaxed), 1);
assert_eq!(count.load(Ordering::Relaxed), 2);
})
}
@ -816,6 +856,7 @@ pub mod tests {
fn dyn_import_ok() {
#[derive(Clone, Default)]
struct DynImportOkLoader {
pub prepare_load_count: Arc<AtomicUsize>,
pub resolve_count: Arc<AtomicUsize>,
pub load_count: Arc<AtomicUsize>,
}
@ -828,11 +869,8 @@ pub mod tests {
_is_main: bool,
) -> Result<ModuleSpecifier, ErrBox> {
let c = self.resolve_count.fetch_add(1, Ordering::Relaxed);
match c {
0 => assert_eq!(specifier, "./b.js"),
1 => assert_eq!(specifier, "./b.js"),
_ => unreachable!(),
}
assert!(c < 4);
assert_eq!(specifier, "./b.js");
assert_eq!(referrer, "file:///dyn_import3.js");
let s = ModuleSpecifier::resolve_import(specifier, referrer).unwrap();
Ok(s)
@ -852,10 +890,22 @@ pub mod tests {
};
async move { Ok(info) }.boxed()
}
fn prepare_load(
&self,
_load_id: ModuleLoadId,
_module_specifier: &ModuleSpecifier,
_maybe_referrer: Option<String>,
_is_dyn_import: bool,
) -> Pin<Box<dyn Future<Output = Result<(), ErrBox>>>> {
self.prepare_load_count.fetch_add(1, Ordering::Relaxed);
async { Ok(()) }.boxed_local()
}
}
run_in_task(|cx| {
let loader = Rc::new(DynImportOkLoader::default());
let prepare_load_count = loader.prepare_load_count.clone();
let resolve_count = loader.resolve_count.clone();
let load_count = loader.load_count.clone();
let mut isolate = EsIsolate::new(loader, StartupData::None, false);
@ -878,17 +928,25 @@ pub mod tests {
"#,
));
// First poll runs `prepare_load` hook.
assert!(match isolate.poll_unpin(cx) {
Poll::Pending => true,
_ => false,
});
assert_eq!(prepare_load_count.load(Ordering::Relaxed), 1);
// Second poll actually loads modules into the isolate.
assert!(match isolate.poll_unpin(cx) {
Poll::Ready(Ok(_)) => true,
_ => false,
});
assert_eq!(resolve_count.load(Ordering::Relaxed), 2);
assert_eq!(resolve_count.load(Ordering::Relaxed), 4);
assert_eq!(load_count.load(Ordering::Relaxed), 2);
assert!(match isolate.poll_unpin(cx) {
Poll::Ready(Ok(_)) => true,
_ => false,
});
assert_eq!(resolve_count.load(Ordering::Relaxed), 2);
assert_eq!(resolve_count.load(Ordering::Relaxed), 4);
assert_eq!(load_count.load(Ordering::Relaxed), 2);
})
}

57
core/modules.rs
View file

@ -47,6 +47,8 @@ pub struct ModuleSource {
pub module_url_found: String,
}
pub type PrepareLoadFuture =
dyn Future<Output = (ModuleLoadId, Result<RecursiveModuleLoad, ErrBox>)>;
pub type ModuleSourceFuture = dyn Future<Output = Result<ModuleSource, ErrBox>>;
pub trait ModuleLoader {
@ -74,6 +76,24 @@ pub trait ModuleLoader {
maybe_referrer: Option<ModuleSpecifier>,
is_dyn_import: bool,
) -> Pin<Box<ModuleSourceFuture>>;
/// This hook can be used by implementors to do some preparation
/// work before starting loading of modules.
///
/// For example implementor might download multiple modules in
/// parallel and transpile them to final JS sources before
/// yielding control back to Isolate.
///
/// It's not required to implement this method.
fn prepare_load(
&self,
_load_id: ModuleLoadId,
_module_specifier: &ModuleSpecifier,
_maybe_referrer: Option<String>,
_is_dyn_import: bool,
) -> Pin<Box<dyn Future<Output = Result<(), ErrBox>>>> {
async { Ok(()) }.boxed_local()
}
}
#[derive(Debug, Eq, PartialEq)]
@ -95,6 +115,8 @@ pub enum LoadState {
/// that is consumed by the isolate.
pub struct RecursiveModuleLoad {
kind: Kind,
// TODO(bartlomieju): in future this value should
// be randomized
pub id: ModuleLoadId,
pub root_module_id: Option<ModuleId>,
pub state: LoadState,
@ -142,6 +164,41 @@ impl RecursiveModuleLoad {
}
}
pub async fn prepare(self) -> (ModuleLoadId, Result<Self, ErrBox>) {
let (module_specifier, maybe_referrer) = match self.state {
LoadState::ResolveMain(ref specifier, _) => {
let spec = match self.loader.resolve(specifier, ".", true) {
Ok(spec) => spec,
Err(e) => return (self.id, Err(e)),
};
(spec, None)
}
LoadState::ResolveImport(ref specifier, ref referrer) => {
let spec = match self.loader.resolve(specifier, referrer, false) {
Ok(spec) => spec,
Err(e) => return (self.id, Err(e)),
};
(spec, Some(referrer.to_string()))
}
_ => unreachable!(),
};
let prepare_result = self
.loader
.prepare_load(
self.id,
&module_specifier,
maybe_referrer,
self.is_dynamic_import(),
)
.await;
match prepare_result {
Ok(()) => (self.id, Ok(self)),
Err(e) => (self.id, Err(e)),
}
}
fn add_root(&mut self) -> Result<(), ErrBox> {
let module_specifier = match self.state {
LoadState::ResolveMain(ref specifier, _) => {