mirror of
https://github.com/denoland/deno.git
synced 2024-12-23 15:49:44 -05:00
fix(node): seperate worker module cache (#23634)
Construct a new module graph container for workers instead of sharing it with the main worker. Fixes #17248 Fixes #23461 --------- Co-authored-by: David Sherret <dsherret@gmail.com>
This commit is contained in:
parent
bba553bea5
commit
88983fb3eb
17 changed files with 503 additions and 401 deletions
|
@ -12,6 +12,7 @@ use self::package_json::PackageJsonDeps;
|
|||
use ::import_map::ImportMap;
|
||||
use deno_ast::SourceMapOption;
|
||||
use deno_core::resolve_url_or_path;
|
||||
use deno_graph::GraphKind;
|
||||
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
|
||||
use deno_npm::NpmSystemInfo;
|
||||
use deno_runtime::deno_tls::RootCertStoreProvider;
|
||||
|
@ -873,6 +874,14 @@ impl CliOptions {
|
|||
self.maybe_config_file.as_ref().map(|f| f.specifier.clone())
|
||||
}
|
||||
|
||||
pub fn graph_kind(&self) -> GraphKind {
|
||||
match self.sub_command() {
|
||||
DenoSubcommand::Cache(_) => GraphKind::All,
|
||||
DenoSubcommand::Check(_) => GraphKind::TypesOnly,
|
||||
_ => self.type_check_mode().as_graph_kind(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn ts_type_lib_window(&self) -> TsTypeLib {
|
||||
TsTypeLib::DenoWindow
|
||||
}
|
||||
|
|
|
@ -21,9 +21,9 @@ use crate::cache::NodeAnalysisCache;
|
|||
use crate::cache::ParsedSourceCache;
|
||||
use crate::emit::Emitter;
|
||||
use crate::file_fetcher::FileFetcher;
|
||||
use crate::graph_container::MainModuleGraphContainer;
|
||||
use crate::graph_util::FileWatcherReporter;
|
||||
use crate::graph_util::ModuleGraphBuilder;
|
||||
use crate::graph_util::ModuleGraphContainer;
|
||||
use crate::graph_util::ModuleGraphCreator;
|
||||
use crate::http_util::HttpClient;
|
||||
use crate::module_loader::CliModuleLoaderFactory;
|
||||
|
@ -60,7 +60,6 @@ use deno_core::futures::FutureExt;
|
|||
use deno_core::parking_lot::Mutex;
|
||||
use deno_core::FeatureChecker;
|
||||
|
||||
use deno_graph::GraphKind;
|
||||
use deno_lockfile::WorkspaceMemberConfig;
|
||||
use deno_runtime::deno_fs;
|
||||
use deno_runtime::deno_node::analyze::NodeCodeTranslator;
|
||||
|
@ -157,7 +156,7 @@ struct CliFactoryServices {
|
|||
emit_cache: Deferred<EmitCache>,
|
||||
emitter: Deferred<Arc<Emitter>>,
|
||||
fs: Deferred<Arc<dyn deno_fs::FileSystem>>,
|
||||
graph_container: Deferred<Arc<ModuleGraphContainer>>,
|
||||
main_graph_container: Deferred<Arc<MainModuleGraphContainer>>,
|
||||
lockfile: Deferred<Option<Arc<Mutex<Lockfile>>>>,
|
||||
maybe_import_map: Deferred<Option<Arc<ImportMap>>>,
|
||||
maybe_inspector_server: Deferred<Option<Arc<InspectorServer>>>,
|
||||
|
@ -673,17 +672,19 @@ impl CliFactory {
|
|||
.await
|
||||
}
|
||||
|
||||
pub fn graph_container(&self) -> &Arc<ModuleGraphContainer> {
|
||||
self.services.graph_container.get_or_init(|| {
|
||||
let graph_kind = match self.options.sub_command() {
|
||||
// todo(dsherret): ideally the graph container would not be used
|
||||
// for deno cache because it doesn't dynamically load modules
|
||||
DenoSubcommand::Cache(_) => GraphKind::All,
|
||||
DenoSubcommand::Check(_) => GraphKind::TypesOnly,
|
||||
_ => self.options.type_check_mode().as_graph_kind(),
|
||||
};
|
||||
Arc::new(ModuleGraphContainer::new(graph_kind))
|
||||
})
|
||||
pub async fn main_module_graph_container(
|
||||
&self,
|
||||
) -> Result<&Arc<MainModuleGraphContainer>, AnyError> {
|
||||
self
|
||||
.services
|
||||
.main_graph_container
|
||||
.get_or_try_init_async(async {
|
||||
Ok(Arc::new(MainModuleGraphContainer::new(
|
||||
self.cli_options().clone(),
|
||||
self.module_load_preparer().await?.clone(),
|
||||
)))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn maybe_inspector_server(
|
||||
|
@ -706,7 +707,6 @@ impl CliFactory {
|
|||
.get_or_try_init_async(async {
|
||||
Ok(Arc::new(ModuleLoadPreparer::new(
|
||||
self.options.clone(),
|
||||
self.graph_container().clone(),
|
||||
self.maybe_lockfile().clone(),
|
||||
self.module_graph_builder().await?.clone(),
|
||||
self.text_only_progress_bar().clone(),
|
||||
|
@ -791,11 +791,15 @@ impl CliFactory {
|
|||
self.blob_store().clone(),
|
||||
Box::new(CliModuleLoaderFactory::new(
|
||||
&self.options,
|
||||
if self.options.code_cache_enabled() {
|
||||
Some(self.code_cache()?.clone())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
self.emitter()?.clone(),
|
||||
self.graph_container().clone(),
|
||||
self.main_module_graph_container().await?.clone(),
|
||||
self.module_info_cache()?.clone(),
|
||||
self.module_load_preparer().await?.clone(),
|
||||
self.parsed_source_cache().clone(),
|
||||
self.resolver().await?.clone(),
|
||||
cli_node_resolver.clone(),
|
||||
NpmModuleLoader::new(
|
||||
self.cjs_resolutions().clone(),
|
||||
|
@ -803,12 +807,8 @@ impl CliFactory {
|
|||
fs.clone(),
|
||||
cli_node_resolver.clone(),
|
||||
),
|
||||
if self.options.code_cache_enabled() {
|
||||
Some(self.code_cache()?.clone())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
self.module_info_cache()?.clone(),
|
||||
self.parsed_source_cache().clone(),
|
||||
self.resolver().await?.clone(),
|
||||
)),
|
||||
self.root_cert_store_provider().clone(),
|
||||
self.fs().clone(),
|
||||
|
|
157
cli/graph_container.rs
Normal file
157
cli/graph_container.rs
Normal file
|
@ -0,0 +1,157 @@
|
|||
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use deno_ast::ModuleSpecifier;
|
||||
use deno_core::error::AnyError;
|
||||
use deno_core::parking_lot::RwLock;
|
||||
use deno_core::resolve_url_or_path;
|
||||
use deno_graph::ModuleGraph;
|
||||
use deno_runtime::colors;
|
||||
use deno_runtime::permissions::PermissionsContainer;
|
||||
|
||||
use crate::args::CliOptions;
|
||||
use crate::module_loader::ModuleLoadPreparer;
|
||||
|
||||
pub trait ModuleGraphContainer: Clone + 'static {
|
||||
/// Acquires a permit to modify the module graph without other code
|
||||
/// having the chance to modify it. In the meantime, other code may
|
||||
/// still read from the existing module graph.
|
||||
async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit;
|
||||
/// Gets a copy of the graph.
|
||||
fn graph(&self) -> Arc<ModuleGraph>;
|
||||
}
|
||||
|
||||
/// A permit for updating the module graph. When complete and
|
||||
/// everything looks fine, calling `.commit()` will store the
|
||||
/// new graph in the ModuleGraphContainer.
|
||||
pub trait ModuleGraphUpdatePermit {
|
||||
/// Gets the module graph for mutation.
|
||||
fn graph_mut(&mut self) -> &mut ModuleGraph;
|
||||
/// Saves the mutated module graph in the container.
|
||||
fn commit(self);
|
||||
}
|
||||
|
||||
/// Holds the `ModuleGraph` for the main worker.
|
||||
#[derive(Clone)]
|
||||
pub struct MainModuleGraphContainer {
|
||||
// Allow only one request to update the graph data at a time,
|
||||
// but allow other requests to read from it at any time even
|
||||
// while another request is updating the data.
|
||||
update_queue: Arc<crate::util::sync::TaskQueue>,
|
||||
inner: Arc<RwLock<Arc<ModuleGraph>>>,
|
||||
cli_options: Arc<CliOptions>,
|
||||
module_load_preparer: Arc<ModuleLoadPreparer>,
|
||||
}
|
||||
|
||||
impl MainModuleGraphContainer {
|
||||
pub fn new(
|
||||
cli_options: Arc<CliOptions>,
|
||||
module_load_preparer: Arc<ModuleLoadPreparer>,
|
||||
) -> Self {
|
||||
Self {
|
||||
update_queue: Default::default(),
|
||||
inner: Arc::new(RwLock::new(Arc::new(ModuleGraph::new(
|
||||
cli_options.graph_kind(),
|
||||
)))),
|
||||
cli_options,
|
||||
module_load_preparer,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn check_specifiers(
|
||||
&self,
|
||||
specifiers: &[ModuleSpecifier],
|
||||
) -> Result<(), AnyError> {
|
||||
let mut graph_permit = self.acquire_update_permit().await;
|
||||
let graph = graph_permit.graph_mut();
|
||||
self
|
||||
.module_load_preparer
|
||||
.prepare_module_load(
|
||||
graph,
|
||||
specifiers,
|
||||
false,
|
||||
self.cli_options.ts_type_lib_window(),
|
||||
PermissionsContainer::allow_all(),
|
||||
)
|
||||
.await?;
|
||||
graph_permit.commit();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper around prepare_module_load that loads and type checks
|
||||
/// the provided files.
|
||||
pub async fn load_and_type_check_files(
|
||||
&self,
|
||||
files: &[String],
|
||||
) -> Result<(), AnyError> {
|
||||
let specifiers = self.collect_specifiers(files)?;
|
||||
|
||||
if specifiers.is_empty() {
|
||||
log::warn!("{} No matching files found.", colors::yellow("Warning"));
|
||||
}
|
||||
|
||||
self.check_specifiers(&specifiers).await
|
||||
}
|
||||
|
||||
pub fn collect_specifiers(
|
||||
&self,
|
||||
files: &[String],
|
||||
) -> Result<Vec<ModuleSpecifier>, AnyError> {
|
||||
let excludes = self.cli_options.resolve_config_excludes()?;
|
||||
Ok(
|
||||
files
|
||||
.iter()
|
||||
.filter_map(|file| {
|
||||
let file_url =
|
||||
resolve_url_or_path(file, self.cli_options.initial_cwd()).ok()?;
|
||||
if file_url.scheme() != "file" {
|
||||
return Some(file_url);
|
||||
}
|
||||
// ignore local files that match any of files listed in `exclude` option
|
||||
let file_path = file_url.to_file_path().ok()?;
|
||||
if excludes.matches_path(&file_path) {
|
||||
None
|
||||
} else {
|
||||
Some(file_url)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl ModuleGraphContainer for MainModuleGraphContainer {
|
||||
async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit {
|
||||
let permit = self.update_queue.acquire().await;
|
||||
MainModuleGraphUpdatePermit {
|
||||
permit,
|
||||
inner: self.inner.clone(),
|
||||
graph: (**self.inner.read()).clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn graph(&self) -> Arc<ModuleGraph> {
|
||||
self.inner.read().clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// A permit for updating the module graph. When complete and
|
||||
/// everything looks fine, calling `.commit()` will store the
|
||||
/// new graph in the ModuleGraphContainer.
|
||||
pub struct MainModuleGraphUpdatePermit<'a> {
|
||||
permit: crate::util::sync::TaskQueuePermit<'a>,
|
||||
inner: Arc<RwLock<Arc<ModuleGraph>>>,
|
||||
graph: ModuleGraph,
|
||||
}
|
||||
|
||||
impl<'a> ModuleGraphUpdatePermit for MainModuleGraphUpdatePermit<'a> {
|
||||
fn graph_mut(&mut self) -> &mut ModuleGraph {
|
||||
&mut self.graph
|
||||
}
|
||||
|
||||
fn commit(self) {
|
||||
*self.inner.write() = Arc::new(self.graph);
|
||||
drop(self.permit); // explicit drop for clarity
|
||||
}
|
||||
}
|
|
@ -18,8 +18,6 @@ use crate::tools::check;
|
|||
use crate::tools::check::TypeChecker;
|
||||
use crate::util::file_watcher::WatcherCommunicator;
|
||||
use crate::util::fs::canonicalize_path;
|
||||
use crate::util::sync::TaskQueue;
|
||||
use crate::util::sync::TaskQueuePermit;
|
||||
use deno_runtime::fs_util::specifier_to_file_path;
|
||||
|
||||
use deno_config::WorkspaceMemberConfig;
|
||||
|
@ -27,7 +25,6 @@ use deno_core::anyhow::bail;
|
|||
use deno_core::error::custom_error;
|
||||
use deno_core::error::AnyError;
|
||||
use deno_core::parking_lot::Mutex;
|
||||
use deno_core::parking_lot::RwLock;
|
||||
use deno_core::ModuleSpecifier;
|
||||
use deno_graph::source::Loader;
|
||||
use deno_graph::source::ResolutionMode;
|
||||
|
@ -762,40 +759,6 @@ fn get_resolution_error_bare_specifier(
|
|||
}
|
||||
}
|
||||
|
||||
/// Holds the `ModuleGraph` and what parts of it are type checked.
|
||||
pub struct ModuleGraphContainer {
|
||||
// Allow only one request to update the graph data at a time,
|
||||
// but allow other requests to read from it at any time even
|
||||
// while another request is updating the data.
|
||||
update_queue: Arc<TaskQueue>,
|
||||
inner: Arc<RwLock<Arc<ModuleGraph>>>,
|
||||
}
|
||||
|
||||
impl ModuleGraphContainer {
|
||||
pub fn new(graph_kind: GraphKind) -> Self {
|
||||
Self {
|
||||
update_queue: Default::default(),
|
||||
inner: Arc::new(RwLock::new(Arc::new(ModuleGraph::new(graph_kind)))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquires a permit to modify the module graph without other code
|
||||
/// having the chance to modify it. In the meantime, other code may
|
||||
/// still read from the existing module graph.
|
||||
pub async fn acquire_update_permit(&self) -> ModuleGraphUpdatePermit {
|
||||
let permit = self.update_queue.acquire().await;
|
||||
ModuleGraphUpdatePermit {
|
||||
permit,
|
||||
inner: self.inner.clone(),
|
||||
graph: (**self.inner.read()).clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn graph(&self) -> Arc<ModuleGraph> {
|
||||
self.inner.read().clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets if any of the specified root's "file:" dependents are in the
|
||||
/// provided changed set.
|
||||
pub fn has_graph_root_local_dependent_changed(
|
||||
|
@ -828,31 +791,6 @@ pub fn has_graph_root_local_dependent_changed(
|
|||
false
|
||||
}
|
||||
|
||||
/// A permit for updating the module graph. When complete and
|
||||
/// everything looks fine, calling `.commit()` will store the
|
||||
/// new graph in the ModuleGraphContainer.
|
||||
pub struct ModuleGraphUpdatePermit<'a> {
|
||||
permit: TaskQueuePermit<'a>,
|
||||
inner: Arc<RwLock<Arc<ModuleGraph>>>,
|
||||
graph: ModuleGraph,
|
||||
}
|
||||
|
||||
impl<'a> ModuleGraphUpdatePermit<'a> {
|
||||
/// Gets the module graph for mutation.
|
||||
pub fn graph_mut(&mut self) -> &mut ModuleGraph {
|
||||
&mut self.graph
|
||||
}
|
||||
|
||||
/// Saves the mutated module graph in the container
|
||||
/// and returns an Arc to the new module graph.
|
||||
pub fn commit(self) -> Arc<ModuleGraph> {
|
||||
let graph = Arc::new(self.graph);
|
||||
*self.inner.write() = graph.clone();
|
||||
drop(self.permit); // explicit drop for clarity
|
||||
graph
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FileWatcherReporter {
|
||||
watcher_communicator: Arc<WatcherCommunicator>,
|
||||
|
|
|
@ -219,10 +219,10 @@ impl TestRun {
|
|||
// file would have impact on other files, which is undesirable.
|
||||
let permissions =
|
||||
Permissions::from_options(&factory.cli_options().permissions_options()?)?;
|
||||
let main_graph_container = factory.main_module_graph_container().await?;
|
||||
test::check_specifiers(
|
||||
factory.cli_options(),
|
||||
factory.file_fetcher()?,
|
||||
factory.module_load_preparer().await?,
|
||||
main_graph_container,
|
||||
self
|
||||
.queue
|
||||
.iter()
|
||||
|
|
15
cli/main.rs
15
cli/main.rs
|
@ -8,6 +8,7 @@ mod emit;
|
|||
mod errors;
|
||||
mod factory;
|
||||
mod file_fetcher;
|
||||
mod graph_container;
|
||||
mod graph_util;
|
||||
mod http_util;
|
||||
mod js;
|
||||
|
@ -30,6 +31,7 @@ use crate::args::flags_from_vec;
|
|||
use crate::args::DenoSubcommand;
|
||||
use crate::args::Flags;
|
||||
use crate::args::DENO_FUTURE;
|
||||
use crate::graph_container::ModuleGraphContainer;
|
||||
use crate::util::display;
|
||||
use crate::util::v8::get_v8_flags_from_env;
|
||||
use crate::util::v8::init_v8_flags;
|
||||
|
@ -112,18 +114,19 @@ async fn run_subcommand(flags: Flags) -> Result<i32, AnyError> {
|
|||
}),
|
||||
DenoSubcommand::Cache(cache_flags) => spawn_subcommand(async move {
|
||||
let factory = CliFactory::from_flags(flags)?;
|
||||
let module_load_preparer = factory.module_load_preparer().await?;
|
||||
let emitter = factory.emitter()?;
|
||||
let graph_container = factory.graph_container();
|
||||
module_load_preparer
|
||||
let main_graph_container =
|
||||
factory.main_module_graph_container().await?;
|
||||
main_graph_container
|
||||
.load_and_type_check_files(&cache_flags.files)
|
||||
.await?;
|
||||
emitter.cache_module_emits(&graph_container.graph())
|
||||
emitter.cache_module_emits(&main_graph_container.graph())
|
||||
}),
|
||||
DenoSubcommand::Check(check_flags) => spawn_subcommand(async move {
|
||||
let factory = CliFactory::from_flags(flags)?;
|
||||
let module_load_preparer = factory.module_load_preparer().await?;
|
||||
module_load_preparer
|
||||
let main_graph_container =
|
||||
factory.main_module_graph_container().await?;
|
||||
main_graph_container
|
||||
.load_and_type_check_files(&check_flags.files)
|
||||
.await
|
||||
}),
|
||||
|
|
|
@ -1,5 +1,12 @@
|
|||
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::cell::RefCell;
|
||||
use std::pin::Pin;
|
||||
use std::rc::Rc;
|
||||
use std::str;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::args::jsr_url;
|
||||
use crate::args::CliOptions;
|
||||
use crate::args::DenoSubcommand;
|
||||
|
@ -9,10 +16,12 @@ use crate::cache::ModuleInfoCache;
|
|||
use crate::cache::ParsedSourceCache;
|
||||
use crate::emit::Emitter;
|
||||
use crate::factory::CliFactory;
|
||||
use crate::graph_container::MainModuleGraphContainer;
|
||||
use crate::graph_container::ModuleGraphContainer;
|
||||
use crate::graph_container::ModuleGraphUpdatePermit;
|
||||
use crate::graph_util::graph_lock_or_exit;
|
||||
use crate::graph_util::CreateGraphOptions;
|
||||
use crate::graph_util::ModuleGraphBuilder;
|
||||
use crate::graph_util::ModuleGraphContainer;
|
||||
use crate::node;
|
||||
use crate::resolver::CliGraphResolver;
|
||||
use crate::resolver::CliNodeResolver;
|
||||
|
@ -23,6 +32,7 @@ use crate::tools::check::TypeChecker;
|
|||
use crate::util::progress_bar::ProgressBar;
|
||||
use crate::util::text_encoding::code_without_source_map;
|
||||
use crate::util::text_encoding::source_map_from_code;
|
||||
use crate::worker::ModuleLoaderAndSourceMapGetter;
|
||||
use crate::worker::ModuleLoaderFactory;
|
||||
|
||||
use deno_ast::MediaType;
|
||||
|
@ -36,7 +46,6 @@ use deno_core::futures::future::FutureExt;
|
|||
use deno_core::futures::Future;
|
||||
use deno_core::parking_lot::Mutex;
|
||||
use deno_core::resolve_url;
|
||||
use deno_core::resolve_url_or_path;
|
||||
use deno_core::ModuleCodeString;
|
||||
use deno_core::ModuleLoader;
|
||||
use deno_core::ModuleSource;
|
||||
|
@ -48,9 +57,11 @@ use deno_core::ResolutionKind;
|
|||
use deno_core::SourceMapGetter;
|
||||
use deno_graph::source::ResolutionMode;
|
||||
use deno_graph::source::Resolver;
|
||||
use deno_graph::GraphKind;
|
||||
use deno_graph::JsModule;
|
||||
use deno_graph::JsonModule;
|
||||
use deno_graph::Module;
|
||||
use deno_graph::ModuleGraph;
|
||||
use deno_graph::Resolution;
|
||||
use deno_lockfile::Lockfile;
|
||||
use deno_runtime::code_cache;
|
||||
|
@ -58,12 +69,6 @@ use deno_runtime::deno_node::NodeResolutionMode;
|
|||
use deno_runtime::fs_util::code_timestamp;
|
||||
use deno_runtime::permissions::PermissionsContainer;
|
||||
use deno_semver::npm::NpmPackageReqReference;
|
||||
use deno_terminal::colors;
|
||||
use std::borrow::Cow;
|
||||
use std::pin::Pin;
|
||||
use std::rc::Rc;
|
||||
use std::str;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub async fn load_top_level_deps(factory: &CliFactory) -> Result<(), AnyError> {
|
||||
let npm_resolver = factory.npm_resolver().await?;
|
||||
|
@ -83,12 +88,19 @@ pub async fn load_top_level_deps(factory: &CliFactory) -> Result<(), AnyError> {
|
|||
entry.value.cloned()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
.collect::<Vec<_>>();
|
||||
let mut graph_permit = factory
|
||||
.main_module_graph_container()
|
||||
.await?
|
||||
.acquire_update_permit()
|
||||
.await;
|
||||
let graph = graph_permit.graph_mut();
|
||||
factory
|
||||
.module_load_preparer()
|
||||
.await?
|
||||
.prepare_module_load(
|
||||
roots,
|
||||
graph,
|
||||
&roots,
|
||||
false,
|
||||
factory.cli_options().ts_type_lib_window(),
|
||||
deno_runtime::permissions::PermissionsContainer::allow_all(),
|
||||
|
@ -101,7 +113,6 @@ pub async fn load_top_level_deps(factory: &CliFactory) -> Result<(), AnyError> {
|
|||
|
||||
pub struct ModuleLoadPreparer {
|
||||
options: Arc<CliOptions>,
|
||||
graph_container: Arc<ModuleGraphContainer>,
|
||||
lockfile: Option<Arc<Mutex<Lockfile>>>,
|
||||
module_graph_builder: Arc<ModuleGraphBuilder>,
|
||||
progress_bar: ProgressBar,
|
||||
|
@ -112,7 +123,6 @@ impl ModuleLoadPreparer {
|
|||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
options: Arc<CliOptions>,
|
||||
graph_container: Arc<ModuleGraphContainer>,
|
||||
lockfile: Option<Arc<Mutex<Lockfile>>>,
|
||||
module_graph_builder: Arc<ModuleGraphBuilder>,
|
||||
progress_bar: ProgressBar,
|
||||
|
@ -120,7 +130,6 @@ impl ModuleLoadPreparer {
|
|||
) -> Self {
|
||||
Self {
|
||||
options,
|
||||
graph_container,
|
||||
lockfile,
|
||||
module_graph_builder,
|
||||
progress_bar,
|
||||
|
@ -135,7 +144,8 @@ impl ModuleLoadPreparer {
|
|||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn prepare_module_load(
|
||||
&self,
|
||||
roots: Vec<ModuleSpecifier>,
|
||||
graph: &mut ModuleGraph,
|
||||
roots: &[ModuleSpecifier],
|
||||
is_dynamic: bool,
|
||||
lib: TsTypeLib,
|
||||
permissions: PermissionsContainer,
|
||||
|
@ -144,10 +154,7 @@ impl ModuleLoadPreparer {
|
|||
let _pb_clear_guard = self.progress_bar.clear_guard();
|
||||
|
||||
let mut cache = self.module_graph_builder.create_fetch_cacher(permissions);
|
||||
log::debug!("Creating module graph.");
|
||||
let mut graph_update_permit =
|
||||
self.graph_container.acquire_update_permit().await;
|
||||
let graph = graph_update_permit.graph_mut();
|
||||
log::debug!("Building module graph.");
|
||||
let has_type_checked = !graph.roots.is_empty();
|
||||
|
||||
self
|
||||
|
@ -157,13 +164,13 @@ impl ModuleLoadPreparer {
|
|||
CreateGraphOptions {
|
||||
is_dynamic,
|
||||
graph_kind: graph.graph_kind(),
|
||||
roots: roots.clone(),
|
||||
roots: roots.to_vec(),
|
||||
loader: Some(&mut cache),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.module_graph_builder.graph_roots_valid(graph, &roots)?;
|
||||
self.module_graph_builder.graph_roots_valid(graph, roots)?;
|
||||
|
||||
// If there is a lockfile...
|
||||
if let Some(lockfile) = &self.lockfile {
|
||||
|
@ -174,9 +181,6 @@ impl ModuleLoadPreparer {
|
|||
lockfile.write().context("Failed writing lockfile.")?;
|
||||
}
|
||||
|
||||
// save the graph and get a reference to the new graph
|
||||
let graph = graph_update_permit.commit();
|
||||
|
||||
drop(_pb_clear_guard);
|
||||
|
||||
// type check if necessary
|
||||
|
@ -188,7 +192,7 @@ impl ModuleLoadPreparer {
|
|||
// created, we could avoid the clone of the graph here by providing
|
||||
// the actual graph on the first run and then getting the Arc<ModuleGraph>
|
||||
// back from the return value.
|
||||
(*graph).clone(),
|
||||
graph.clone(),
|
||||
check::CheckOptions {
|
||||
build_fast_check_graph: true,
|
||||
lib,
|
||||
|
@ -204,154 +208,23 @@ impl ModuleLoadPreparer {
|
|||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper around prepare_module_load that loads and type checks
|
||||
/// the provided files.
|
||||
pub async fn load_and_type_check_files(
|
||||
&self,
|
||||
files: &[String],
|
||||
) -> Result<(), AnyError> {
|
||||
let lib = self.options.ts_type_lib_window();
|
||||
|
||||
let specifiers = self.collect_specifiers(files)?;
|
||||
|
||||
if specifiers.is_empty() {
|
||||
log::warn!("{} No matching files found.", colors::yellow("Warning"));
|
||||
}
|
||||
|
||||
self
|
||||
.prepare_module_load(
|
||||
specifiers,
|
||||
false,
|
||||
lib,
|
||||
PermissionsContainer::allow_all(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn collect_specifiers(
|
||||
&self,
|
||||
files: &[String],
|
||||
) -> Result<Vec<ModuleSpecifier>, AnyError> {
|
||||
let excludes = self.options.resolve_config_excludes()?;
|
||||
Ok(
|
||||
files
|
||||
.iter()
|
||||
.filter_map(|file| {
|
||||
let file_url =
|
||||
resolve_url_or_path(file, self.options.initial_cwd()).ok()?;
|
||||
if file_url.scheme() != "file" {
|
||||
return Some(file_url);
|
||||
}
|
||||
// ignore local files that match any of files listed in `exclude` option
|
||||
let file_path = file_url.to_file_path().ok()?;
|
||||
if excludes.matches_path(&file_path) {
|
||||
None
|
||||
} else {
|
||||
Some(file_url)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
struct PreparedModuleLoader {
|
||||
emitter: Arc<Emitter>,
|
||||
graph_container: Arc<ModuleGraphContainer>,
|
||||
parsed_source_cache: Arc<ParsedSourceCache>,
|
||||
}
|
||||
|
||||
impl PreparedModuleLoader {
|
||||
pub fn load_prepared_module(
|
||||
&self,
|
||||
specifier: &ModuleSpecifier,
|
||||
maybe_referrer: Option<&ModuleSpecifier>,
|
||||
) -> Result<ModuleCodeStringSource, AnyError> {
|
||||
if specifier.scheme() == "node" {
|
||||
unreachable!(); // Node built-in modules should be handled internally.
|
||||
}
|
||||
|
||||
let graph = self.graph_container.graph();
|
||||
match graph.get(specifier) {
|
||||
Some(deno_graph::Module::Json(JsonModule {
|
||||
source,
|
||||
media_type,
|
||||
specifier,
|
||||
..
|
||||
})) => Ok(ModuleCodeStringSource {
|
||||
code: source.clone().into(),
|
||||
found_url: specifier.clone(),
|
||||
media_type: *media_type,
|
||||
}),
|
||||
Some(deno_graph::Module::Js(JsModule {
|
||||
source,
|
||||
media_type,
|
||||
specifier,
|
||||
..
|
||||
})) => {
|
||||
let code: ModuleCodeString = match media_type {
|
||||
MediaType::JavaScript
|
||||
| MediaType::Unknown
|
||||
| MediaType::Cjs
|
||||
| MediaType::Mjs
|
||||
| MediaType::Json => source.clone().into(),
|
||||
MediaType::Dts | MediaType::Dcts | MediaType::Dmts => {
|
||||
Default::default()
|
||||
}
|
||||
MediaType::TypeScript
|
||||
| MediaType::Mts
|
||||
| MediaType::Cts
|
||||
| MediaType::Jsx
|
||||
| MediaType::Tsx => {
|
||||
// get emit text
|
||||
self
|
||||
.emitter
|
||||
.emit_parsed_source(specifier, *media_type, source)?
|
||||
}
|
||||
MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => {
|
||||
panic!("Unexpected media type {media_type} for {specifier}")
|
||||
}
|
||||
};
|
||||
|
||||
// at this point, we no longer need the parsed source in memory, so free it
|
||||
self.parsed_source_cache.free(specifier);
|
||||
|
||||
Ok(ModuleCodeStringSource {
|
||||
code,
|
||||
found_url: specifier.clone(),
|
||||
media_type: *media_type,
|
||||
})
|
||||
}
|
||||
Some(
|
||||
deno_graph::Module::External(_)
|
||||
| deno_graph::Module::Node(_)
|
||||
| deno_graph::Module::Npm(_),
|
||||
)
|
||||
| None => {
|
||||
let mut msg = format!("Loading unprepared module: {specifier}");
|
||||
if let Some(referrer) = maybe_referrer {
|
||||
msg = format!("{}, imported from: {}", msg, referrer.as_str());
|
||||
}
|
||||
Err(anyhow!(msg))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedCliModuleLoaderState {
|
||||
graph_kind: GraphKind,
|
||||
lib_window: TsTypeLib,
|
||||
lib_worker: TsTypeLib,
|
||||
is_inspecting: bool,
|
||||
is_repl: bool,
|
||||
graph_container: Arc<ModuleGraphContainer>,
|
||||
code_cache: Option<Arc<CodeCache>>,
|
||||
emitter: Arc<Emitter>,
|
||||
main_module_graph_container: Arc<MainModuleGraphContainer>,
|
||||
module_info_cache: Arc<ModuleInfoCache>,
|
||||
module_load_preparer: Arc<ModuleLoadPreparer>,
|
||||
prepared_module_loader: PreparedModuleLoader,
|
||||
resolver: Arc<CliGraphResolver>,
|
||||
node_resolver: Arc<CliNodeResolver>,
|
||||
npm_module_loader: NpmModuleLoader,
|
||||
code_cache: Option<Arc<CodeCache>>,
|
||||
module_info_cache: Arc<ModuleInfoCache>,
|
||||
parsed_source_cache: Arc<ParsedSourceCache>,
|
||||
resolver: Arc<CliGraphResolver>,
|
||||
}
|
||||
|
||||
pub struct CliModuleLoaderFactory {
|
||||
|
@ -362,18 +235,19 @@ impl CliModuleLoaderFactory {
|
|||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
options: &CliOptions,
|
||||
code_cache: Option<Arc<CodeCache>>,
|
||||
emitter: Arc<Emitter>,
|
||||
graph_container: Arc<ModuleGraphContainer>,
|
||||
main_module_graph_container: Arc<MainModuleGraphContainer>,
|
||||
module_info_cache: Arc<ModuleInfoCache>,
|
||||
module_load_preparer: Arc<ModuleLoadPreparer>,
|
||||
parsed_source_cache: Arc<ParsedSourceCache>,
|
||||
resolver: Arc<CliGraphResolver>,
|
||||
node_resolver: Arc<CliNodeResolver>,
|
||||
npm_module_loader: NpmModuleLoader,
|
||||
code_cache: Option<Arc<CodeCache>>,
|
||||
module_info_cache: Arc<ModuleInfoCache>,
|
||||
parsed_source_cache: Arc<ParsedSourceCache>,
|
||||
resolver: Arc<CliGraphResolver>,
|
||||
) -> Self {
|
||||
Self {
|
||||
shared: Arc::new(SharedCliModuleLoaderState {
|
||||
graph_kind: options.graph_kind(),
|
||||
lib_window: options.ts_type_lib_window(),
|
||||
lib_worker: options.ts_type_lib_worker(),
|
||||
is_inspecting: options.is_inspecting(),
|
||||
|
@ -381,34 +255,39 @@ impl CliModuleLoaderFactory {
|
|||
options.sub_command(),
|
||||
DenoSubcommand::Repl(_) | DenoSubcommand::Jupyter(_)
|
||||
),
|
||||
prepared_module_loader: PreparedModuleLoader {
|
||||
emitter,
|
||||
graph_container: graph_container.clone(),
|
||||
parsed_source_cache,
|
||||
},
|
||||
graph_container,
|
||||
code_cache,
|
||||
emitter,
|
||||
main_module_graph_container,
|
||||
module_info_cache,
|
||||
module_load_preparer,
|
||||
resolver,
|
||||
node_resolver,
|
||||
npm_module_loader,
|
||||
code_cache,
|
||||
module_info_cache,
|
||||
parsed_source_cache,
|
||||
resolver,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_with_lib(
|
||||
fn create_with_lib<TGraphContainer: ModuleGraphContainer>(
|
||||
&self,
|
||||
graph_container: TGraphContainer,
|
||||
lib: TsTypeLib,
|
||||
root_permissions: PermissionsContainer,
|
||||
dynamic_permissions: PermissionsContainer,
|
||||
) -> Rc<dyn ModuleLoader> {
|
||||
Rc::new(CliModuleLoader {
|
||||
) -> ModuleLoaderAndSourceMapGetter {
|
||||
let loader = Rc::new(CliModuleLoader {
|
||||
lib,
|
||||
root_permissions,
|
||||
dynamic_permissions,
|
||||
graph_container,
|
||||
emitter: self.shared.emitter.clone(),
|
||||
parsed_source_cache: self.shared.parsed_source_cache.clone(),
|
||||
shared: self.shared.clone(),
|
||||
})
|
||||
});
|
||||
ModuleLoaderAndSourceMapGetter {
|
||||
module_loader: loader.clone(),
|
||||
source_map_getter: Some(loader),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -417,8 +296,9 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory {
|
|||
&self,
|
||||
root_permissions: PermissionsContainer,
|
||||
dynamic_permissions: PermissionsContainer,
|
||||
) -> Rc<dyn ModuleLoader> {
|
||||
) -> ModuleLoaderAndSourceMapGetter {
|
||||
self.create_with_lib(
|
||||
(*self.shared.main_module_graph_container).clone(),
|
||||
self.shared.lib_window,
|
||||
root_permissions,
|
||||
dynamic_permissions,
|
||||
|
@ -429,22 +309,20 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory {
|
|||
&self,
|
||||
root_permissions: PermissionsContainer,
|
||||
dynamic_permissions: PermissionsContainer,
|
||||
) -> Rc<dyn ModuleLoader> {
|
||||
) -> ModuleLoaderAndSourceMapGetter {
|
||||
self.create_with_lib(
|
||||
// create a fresh module graph for the worker
|
||||
WorkerModuleGraphContainer::new(Arc::new(ModuleGraph::new(
|
||||
self.shared.graph_kind,
|
||||
))),
|
||||
self.shared.lib_worker,
|
||||
root_permissions,
|
||||
dynamic_permissions,
|
||||
)
|
||||
}
|
||||
|
||||
fn create_source_map_getter(&self) -> Option<Rc<dyn SourceMapGetter>> {
|
||||
Some(Rc::new(CliSourceMapGetter {
|
||||
shared: self.shared.clone(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
struct CliModuleLoader {
|
||||
struct CliModuleLoader<TGraphContainer: ModuleGraphContainer> {
|
||||
lib: TsTypeLib,
|
||||
/// The initial set of permissions used to resolve the static imports in the
|
||||
/// worker. These are "allow all" for main worker, and parent thread
|
||||
|
@ -454,9 +332,12 @@ struct CliModuleLoader {
|
|||
/// "root permissions" for Web Worker.
|
||||
dynamic_permissions: PermissionsContainer,
|
||||
shared: Arc<SharedCliModuleLoaderState>,
|
||||
emitter: Arc<Emitter>,
|
||||
parsed_source_cache: Arc<ParsedSourceCache>,
|
||||
graph_container: TGraphContainer,
|
||||
}
|
||||
|
||||
impl CliModuleLoader {
|
||||
impl<TGraphContainer: ModuleGraphContainer> CliModuleLoader<TGraphContainer> {
|
||||
fn load_sync(
|
||||
&self,
|
||||
specifier: &ModuleSpecifier,
|
||||
|
@ -476,10 +357,7 @@ impl CliModuleLoader {
|
|||
{
|
||||
result?
|
||||
} else {
|
||||
self
|
||||
.shared
|
||||
.prepared_module_loader
|
||||
.load_prepared_module(specifier, maybe_referrer)?
|
||||
self.load_prepared_module(specifier, maybe_referrer)?
|
||||
};
|
||||
let code = if self.shared.is_inspecting {
|
||||
// we need the code with the source map in order for
|
||||
|
@ -581,7 +459,7 @@ impl CliModuleLoader {
|
|||
};
|
||||
}
|
||||
|
||||
let graph = self.shared.graph_container.graph();
|
||||
let graph = self.graph_container.graph();
|
||||
let maybe_resolved = match graph.get(referrer) {
|
||||
Some(Module::Js(module)) => {
|
||||
module.dependencies.get(specifier).map(|d| &d.maybe_code)
|
||||
|
@ -695,9 +573,86 @@ impl CliModuleLoader {
|
|||
.map(|timestamp| timestamp.to_string())?;
|
||||
Ok(Some(timestamp))
|
||||
}
|
||||
|
||||
fn load_prepared_module(
|
||||
&self,
|
||||
specifier: &ModuleSpecifier,
|
||||
maybe_referrer: Option<&ModuleSpecifier>,
|
||||
) -> Result<ModuleCodeStringSource, AnyError> {
|
||||
if specifier.scheme() == "node" {
|
||||
unreachable!(); // Node built-in modules should be handled internally.
|
||||
}
|
||||
|
||||
let graph = self.graph_container.graph();
|
||||
match graph.get(specifier) {
|
||||
Some(deno_graph::Module::Json(JsonModule {
|
||||
source,
|
||||
media_type,
|
||||
specifier,
|
||||
..
|
||||
})) => Ok(ModuleCodeStringSource {
|
||||
code: source.clone().into(),
|
||||
found_url: specifier.clone(),
|
||||
media_type: *media_type,
|
||||
}),
|
||||
Some(deno_graph::Module::Js(JsModule {
|
||||
source,
|
||||
media_type,
|
||||
specifier,
|
||||
..
|
||||
})) => {
|
||||
let code: ModuleCodeString = match media_type {
|
||||
MediaType::JavaScript
|
||||
| MediaType::Unknown
|
||||
| MediaType::Cjs
|
||||
| MediaType::Mjs
|
||||
| MediaType::Json => source.clone().into(),
|
||||
MediaType::Dts | MediaType::Dcts | MediaType::Dmts => {
|
||||
Default::default()
|
||||
}
|
||||
MediaType::TypeScript
|
||||
| MediaType::Mts
|
||||
| MediaType::Cts
|
||||
| MediaType::Jsx
|
||||
| MediaType::Tsx => {
|
||||
// get emit text
|
||||
self
|
||||
.emitter
|
||||
.emit_parsed_source(specifier, *media_type, source)?
|
||||
}
|
||||
MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => {
|
||||
panic!("Unexpected media type {media_type} for {specifier}")
|
||||
}
|
||||
};
|
||||
|
||||
// at this point, we no longer need the parsed source in memory, so free it
|
||||
self.parsed_source_cache.free(specifier);
|
||||
|
||||
Ok(ModuleCodeStringSource {
|
||||
code,
|
||||
found_url: specifier.clone(),
|
||||
media_type: *media_type,
|
||||
})
|
||||
}
|
||||
Some(
|
||||
deno_graph::Module::External(_)
|
||||
| deno_graph::Module::Node(_)
|
||||
| deno_graph::Module::Npm(_),
|
||||
)
|
||||
| None => {
|
||||
let mut msg = format!("Loading unprepared module: {specifier}");
|
||||
if let Some(referrer) = maybe_referrer {
|
||||
msg = format!("{}, imported from: {}", msg, referrer.as_str());
|
||||
}
|
||||
Err(anyhow!(msg))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ModuleLoader for CliModuleLoader {
|
||||
impl<TGraphContainer: ModuleGraphContainer> ModuleLoader
|
||||
for CliModuleLoader<TGraphContainer>
|
||||
{
|
||||
fn resolve(
|
||||
&self,
|
||||
specifier: &str,
|
||||
|
@ -747,13 +702,12 @@ impl ModuleLoader for CliModuleLoader {
|
|||
_maybe_referrer: Option<String>,
|
||||
is_dynamic: bool,
|
||||
) -> Pin<Box<dyn Future<Output = Result<(), AnyError>>>> {
|
||||
if let Some(result) =
|
||||
self.shared.npm_module_loader.maybe_prepare_load(specifier)
|
||||
{
|
||||
return Box::pin(deno_core::futures::future::ready(result));
|
||||
if self.shared.node_resolver.in_npm_package(specifier) {
|
||||
return Box::pin(deno_core::futures::future::ready(Ok(())));
|
||||
}
|
||||
|
||||
let specifier = specifier.clone();
|
||||
let graph_container = self.graph_container.clone();
|
||||
let module_load_preparer = self.shared.module_load_preparer.clone();
|
||||
|
||||
let root_permissions = if is_dynamic {
|
||||
|
@ -764,9 +718,19 @@ impl ModuleLoader for CliModuleLoader {
|
|||
let lib = self.lib;
|
||||
|
||||
async move {
|
||||
let mut update_permit = graph_container.acquire_update_permit().await;
|
||||
let graph = update_permit.graph_mut();
|
||||
module_load_preparer
|
||||
.prepare_module_load(vec![specifier], is_dynamic, lib, root_permissions)
|
||||
.await
|
||||
.prepare_module_load(
|
||||
graph,
|
||||
&[specifier],
|
||||
is_dynamic,
|
||||
lib,
|
||||
root_permissions,
|
||||
)
|
||||
.await?;
|
||||
update_permit.commit();
|
||||
Ok(())
|
||||
}
|
||||
.boxed_local()
|
||||
}
|
||||
|
@ -795,15 +759,13 @@ impl ModuleLoader for CliModuleLoader {
|
|||
);
|
||||
}
|
||||
}
|
||||
async {}.boxed_local()
|
||||
std::future::ready(()).boxed_local()
|
||||
}
|
||||
}
|
||||
|
||||
struct CliSourceMapGetter {
|
||||
shared: Arc<SharedCliModuleLoaderState>,
|
||||
}
|
||||
|
||||
impl SourceMapGetter for CliSourceMapGetter {
|
||||
impl<TGraphContainer: ModuleGraphContainer> SourceMapGetter
|
||||
for CliModuleLoader<TGraphContainer>
|
||||
{
|
||||
fn get_source_map(&self, file_name: &str) -> Option<Vec<u8>> {
|
||||
let specifier = resolve_url(file_name).ok()?;
|
||||
match specifier.scheme() {
|
||||
|
@ -812,11 +774,7 @@ impl SourceMapGetter for CliSourceMapGetter {
|
|||
"wasm" | "file" | "http" | "https" | "data" | "blob" => (),
|
||||
_ => return None,
|
||||
}
|
||||
let source = self
|
||||
.shared
|
||||
.prepared_module_loader
|
||||
.load_prepared_module(&specifier, None)
|
||||
.ok()?;
|
||||
let source = self.load_prepared_module(&specifier, None).ok()?;
|
||||
source_map_from_code(&source.code)
|
||||
}
|
||||
|
||||
|
@ -825,7 +783,7 @@ impl SourceMapGetter for CliSourceMapGetter {
|
|||
file_name: &str,
|
||||
line_number: usize,
|
||||
) -> Option<String> {
|
||||
let graph = self.shared.graph_container.graph();
|
||||
let graph = self.graph_container.graph();
|
||||
let code = match graph.get(&resolve_url(file_name).ok()?) {
|
||||
Some(deno_graph::Module::Js(module)) => &module.source,
|
||||
Some(deno_graph::Module::Json(module)) => &module.source,
|
||||
|
@ -844,3 +802,54 @@ impl SourceMapGetter for CliSourceMapGetter {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Holds the `ModuleGraph` in workers.
|
||||
#[derive(Clone)]
|
||||
struct WorkerModuleGraphContainer {
|
||||
// Allow only one request to update the graph data at a time,
|
||||
// but allow other requests to read from it at any time even
|
||||
// while another request is updating the data.
|
||||
update_queue: Rc<deno_core::unsync::TaskQueue>,
|
||||
inner: Rc<RefCell<Arc<ModuleGraph>>>,
|
||||
}
|
||||
|
||||
impl WorkerModuleGraphContainer {
|
||||
pub fn new(module_graph: Arc<ModuleGraph>) -> Self {
|
||||
Self {
|
||||
update_queue: Default::default(),
|
||||
inner: Rc::new(RefCell::new(module_graph)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ModuleGraphContainer for WorkerModuleGraphContainer {
|
||||
async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit {
|
||||
let permit = self.update_queue.acquire().await;
|
||||
WorkerModuleGraphUpdatePermit {
|
||||
permit,
|
||||
inner: self.inner.clone(),
|
||||
graph: (**self.inner.borrow()).clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn graph(&self) -> Arc<ModuleGraph> {
|
||||
self.inner.borrow().clone()
|
||||
}
|
||||
}
|
||||
|
||||
struct WorkerModuleGraphUpdatePermit {
|
||||
permit: deno_core::unsync::TaskQueuePermit,
|
||||
inner: Rc<RefCell<Arc<ModuleGraph>>>,
|
||||
graph: ModuleGraph,
|
||||
}
|
||||
|
||||
impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit {
|
||||
fn graph_mut(&mut self) -> &mut ModuleGraph {
|
||||
&mut self.graph
|
||||
}
|
||||
|
||||
fn commit(self) {
|
||||
*self.inner.borrow_mut() = Arc::new(self.graph);
|
||||
drop(self.permit); // explicit drop for clarity
|
||||
}
|
||||
}
|
||||
|
|
|
@ -91,8 +91,8 @@ impl CliNodeResolver {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn in_npm_package(&self, referrer: &ModuleSpecifier) -> bool {
|
||||
self.npm_resolver.in_npm_package(referrer)
|
||||
pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool {
|
||||
self.npm_resolver.in_npm_package(specifier)
|
||||
}
|
||||
|
||||
pub fn get_closest_package_json(
|
||||
|
@ -249,6 +249,7 @@ impl CliNodeResolver {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NpmModuleLoader {
|
||||
cjs_resolutions: Arc<CjsResolutionStore>,
|
||||
node_code_translator: Arc<CliNodeCodeTranslator>,
|
||||
|
@ -271,18 +272,6 @@ impl NpmModuleLoader {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn maybe_prepare_load(
|
||||
&self,
|
||||
specifier: &ModuleSpecifier,
|
||||
) -> Option<Result<(), AnyError>> {
|
||||
if self.node_resolver.in_npm_package(specifier) {
|
||||
// nothing to prepare
|
||||
Some(Ok(()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load_sync_if_in_npm_package(
|
||||
&self,
|
||||
specifier: &ModuleSpecifier,
|
||||
|
|
|
@ -27,6 +27,7 @@ use crate::util::progress_bar::ProgressBarStyle;
|
|||
use crate::util::v8::construct_v8_flags;
|
||||
use crate::worker::CliMainWorkerFactory;
|
||||
use crate::worker::CliMainWorkerOptions;
|
||||
use crate::worker::ModuleLoaderAndSourceMapGetter;
|
||||
use crate::worker::ModuleLoaderFactory;
|
||||
use deno_ast::MediaType;
|
||||
use deno_core::anyhow::Context;
|
||||
|
@ -282,30 +283,30 @@ impl ModuleLoaderFactory for StandaloneModuleLoaderFactory {
|
|||
&self,
|
||||
root_permissions: PermissionsContainer,
|
||||
dynamic_permissions: PermissionsContainer,
|
||||
) -> Rc<dyn ModuleLoader> {
|
||||
Rc::new(EmbeddedModuleLoader {
|
||||
shared: self.shared.clone(),
|
||||
root_permissions,
|
||||
dynamic_permissions,
|
||||
})
|
||||
) -> ModuleLoaderAndSourceMapGetter {
|
||||
ModuleLoaderAndSourceMapGetter {
|
||||
module_loader: Rc::new(EmbeddedModuleLoader {
|
||||
shared: self.shared.clone(),
|
||||
root_permissions,
|
||||
dynamic_permissions,
|
||||
}),
|
||||
source_map_getter: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn create_for_worker(
|
||||
&self,
|
||||
root_permissions: PermissionsContainer,
|
||||
dynamic_permissions: PermissionsContainer,
|
||||
) -> Rc<dyn ModuleLoader> {
|
||||
Rc::new(EmbeddedModuleLoader {
|
||||
shared: self.shared.clone(),
|
||||
root_permissions,
|
||||
dynamic_permissions,
|
||||
})
|
||||
}
|
||||
|
||||
fn create_source_map_getter(
|
||||
&self,
|
||||
) -> Option<Rc<dyn deno_core::SourceMapGetter>> {
|
||||
None
|
||||
) -> ModuleLoaderAndSourceMapGetter {
|
||||
ModuleLoaderAndSourceMapGetter {
|
||||
module_loader: Rc::new(EmbeddedModuleLoader {
|
||||
shared: self.shared.clone(),
|
||||
root_permissions,
|
||||
dynamic_permissions,
|
||||
}),
|
||||
source_map_getter: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -8,7 +8,6 @@ use crate::display::write_json_to_stdout;
|
|||
use crate::factory::CliFactory;
|
||||
use crate::factory::CliFactoryBuilder;
|
||||
use crate::graph_util::has_graph_root_local_dependent_changed;
|
||||
use crate::module_loader::ModuleLoadPreparer;
|
||||
use crate::ops;
|
||||
use crate::tools::test::format_test_error;
|
||||
use crate::tools::test::TestFilter;
|
||||
|
@ -145,24 +144,6 @@ fn create_reporter(
|
|||
Box::new(ConsoleReporter::new(show_output))
|
||||
}
|
||||
|
||||
/// Type check a collection of module and document specifiers.
|
||||
async fn check_specifiers(
|
||||
cli_options: &CliOptions,
|
||||
module_load_preparer: &ModuleLoadPreparer,
|
||||
specifiers: Vec<ModuleSpecifier>,
|
||||
) -> Result<(), AnyError> {
|
||||
let lib = cli_options.ts_type_lib_window();
|
||||
module_load_preparer
|
||||
.prepare_module_load(
|
||||
specifiers,
|
||||
false,
|
||||
lib,
|
||||
PermissionsContainer::allow_all(),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run a single specifier as an executable bench module.
|
||||
async fn bench_specifier(
|
||||
worker_factory: Arc<CliMainWorkerFactory>,
|
||||
|
@ -445,12 +426,8 @@ pub async fn run_benchmarks(
|
|||
return Err(generic_error("No bench modules found"));
|
||||
}
|
||||
|
||||
check_specifiers(
|
||||
cli_options,
|
||||
factory.module_load_preparer().await?,
|
||||
specifiers.clone(),
|
||||
)
|
||||
.await?;
|
||||
let main_graph_container = factory.main_module_graph_container().await?;
|
||||
main_graph_container.check_specifiers(&specifiers).await?;
|
||||
|
||||
if bench_options.no_run {
|
||||
return Ok(());
|
||||
|
@ -507,7 +484,6 @@ pub async fn run_benchmarks_with_watch(
|
|||
|
||||
let graph_kind = cli_options.type_check_mode().as_graph_kind();
|
||||
let module_graph_creator = factory.module_graph_creator().await?;
|
||||
let module_load_preparer = factory.module_load_preparer().await?;
|
||||
|
||||
let bench_modules = collect_specifiers(
|
||||
bench_options.files.clone(),
|
||||
|
@ -559,7 +535,10 @@ pub async fn run_benchmarks_with_watch(
|
|||
.filter(|specifier| bench_modules_to_reload.contains(specifier))
|
||||
.collect::<Vec<ModuleSpecifier>>();
|
||||
|
||||
check_specifiers(cli_options, module_load_preparer, specifiers.clone())
|
||||
factory
|
||||
.main_module_graph_container()
|
||||
.await?
|
||||
.check_specifiers(&specifiers)
|
||||
.await?;
|
||||
|
||||
if bench_options.no_run {
|
||||
|
|
|
@ -284,8 +284,9 @@ pub async fn install_command(
|
|||
};
|
||||
|
||||
// ensure the module is cached
|
||||
CliFactory::from_flags(flags.clone())?
|
||||
.module_load_preparer()
|
||||
let factory = CliFactory::from_flags(flags.clone())?;
|
||||
factory
|
||||
.main_module_graph_container()
|
||||
.await?
|
||||
.load_and_type_check_files(&[install_flags_global.module_url.clone()])
|
||||
.await?;
|
||||
|
|
|
@ -10,8 +10,8 @@ use crate::factory::CliFactory;
|
|||
use crate::factory::CliFactoryBuilder;
|
||||
use crate::file_fetcher::File;
|
||||
use crate::file_fetcher::FileFetcher;
|
||||
use crate::graph_container::MainModuleGraphContainer;
|
||||
use crate::graph_util::has_graph_root_local_dependent_changed;
|
||||
use crate::module_loader::ModuleLoadPreparer;
|
||||
use crate::ops;
|
||||
use crate::util::file_watcher;
|
||||
use crate::util::fs::collect_specifiers;
|
||||
|
@ -1305,12 +1305,10 @@ async fn fetch_inline_files(
|
|||
|
||||
/// Type check a collection of module and document specifiers.
|
||||
pub async fn check_specifiers(
|
||||
cli_options: &CliOptions,
|
||||
file_fetcher: &FileFetcher,
|
||||
module_load_preparer: &ModuleLoadPreparer,
|
||||
main_graph_container: &Arc<MainModuleGraphContainer>,
|
||||
specifiers: Vec<(ModuleSpecifier, TestMode)>,
|
||||
) -> Result<(), AnyError> {
|
||||
let lib = cli_options.ts_type_lib_window();
|
||||
let inline_files = fetch_inline_files(
|
||||
file_fetcher,
|
||||
specifiers
|
||||
|
@ -1346,13 +1344,8 @@ pub async fn check_specifiers(
|
|||
}
|
||||
}
|
||||
|
||||
module_load_preparer
|
||||
.prepare_module_load(
|
||||
module_specifiers,
|
||||
false,
|
||||
lib,
|
||||
PermissionsContainer::allow_all(),
|
||||
)
|
||||
main_graph_container
|
||||
.check_specifiers(&module_specifiers)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
|
@ -1701,7 +1694,6 @@ pub async fn run_tests(
|
|||
let cli_options = factory.cli_options();
|
||||
let test_options = cli_options.resolve_test_options(test_flags)?;
|
||||
let file_fetcher = factory.file_fetcher()?;
|
||||
let module_load_preparer = factory.module_load_preparer().await?;
|
||||
// Various test files should not share the same permissions in terms of
|
||||
// `PermissionsContainer` - otherwise granting/revoking permissions in one
|
||||
// file would have impact on other files, which is undesirable.
|
||||
|
@ -1721,10 +1713,11 @@ pub async fn run_tests(
|
|||
return Err(generic_error("No test modules found"));
|
||||
}
|
||||
|
||||
let main_graph_container = factory.main_module_graph_container().await?;
|
||||
|
||||
check_specifiers(
|
||||
cli_options,
|
||||
file_fetcher,
|
||||
module_load_preparer,
|
||||
main_graph_container,
|
||||
specifiers_with_mode.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
@ -1863,7 +1856,6 @@ pub async fn run_tests_with_watch(
|
|||
|
||||
let worker_factory =
|
||||
Arc::new(factory.create_cli_main_worker_factory().await?);
|
||||
let module_load_preparer = factory.module_load_preparer().await?;
|
||||
let specifiers_with_mode = fetch_specifiers_with_test_mode(
|
||||
&cli_options,
|
||||
file_fetcher,
|
||||
|
@ -1875,10 +1867,11 @@ pub async fn run_tests_with_watch(
|
|||
.filter(|(specifier, _)| test_modules_to_reload.contains(specifier))
|
||||
.collect::<Vec<(ModuleSpecifier, TestMode)>>();
|
||||
|
||||
let main_graph_container =
|
||||
factory.main_module_graph_container().await?;
|
||||
check_specifiers(
|
||||
&cli_options,
|
||||
file_fetcher,
|
||||
module_load_preparer,
|
||||
main_graph_container,
|
||||
specifiers_with_mode.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
|
|
@ -58,20 +58,23 @@ use crate::util::file_watcher::WatcherCommunicator;
|
|||
use crate::util::file_watcher::WatcherRestartMode;
|
||||
use crate::version;
|
||||
|
||||
pub struct ModuleLoaderAndSourceMapGetter {
|
||||
pub module_loader: Rc<dyn ModuleLoader>,
|
||||
pub source_map_getter: Option<Rc<dyn SourceMapGetter>>,
|
||||
}
|
||||
|
||||
pub trait ModuleLoaderFactory: Send + Sync {
|
||||
fn create_for_main(
|
||||
&self,
|
||||
root_permissions: PermissionsContainer,
|
||||
dynamic_permissions: PermissionsContainer,
|
||||
) -> Rc<dyn ModuleLoader>;
|
||||
) -> ModuleLoaderAndSourceMapGetter;
|
||||
|
||||
fn create_for_worker(
|
||||
&self,
|
||||
root_permissions: PermissionsContainer,
|
||||
dynamic_permissions: PermissionsContainer,
|
||||
) -> Rc<dyn ModuleLoader>;
|
||||
|
||||
fn create_source_map_getter(&self) -> Option<Rc<dyn SourceMapGetter>>;
|
||||
) -> ModuleLoaderAndSourceMapGetter;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait(?Send)]
|
||||
|
@ -549,11 +552,12 @@ impl CliMainWorkerFactory {
|
|||
(main_module, false)
|
||||
};
|
||||
|
||||
let module_loader = shared
|
||||
let ModuleLoaderAndSourceMapGetter {
|
||||
module_loader,
|
||||
source_map_getter,
|
||||
} = shared
|
||||
.module_loader_factory
|
||||
.create_for_main(PermissionsContainer::allow_all(), permissions.clone());
|
||||
let maybe_source_map_getter =
|
||||
shared.module_loader_factory.create_source_map_getter();
|
||||
let maybe_inspector_server = shared.maybe_inspector_server.clone();
|
||||
|
||||
let create_web_worker_cb =
|
||||
|
@ -627,7 +631,7 @@ impl CliMainWorkerFactory {
|
|||
.clone(),
|
||||
root_cert_store_provider: Some(shared.root_cert_store_provider.clone()),
|
||||
seed: shared.options.seed,
|
||||
source_map_getter: maybe_source_map_getter,
|
||||
source_map_getter,
|
||||
format_js_error_fn: Some(Arc::new(format_js_error)),
|
||||
create_web_worker_cb,
|
||||
maybe_inspector_server,
|
||||
|
@ -769,12 +773,13 @@ fn create_web_worker_callback(
|
|||
Arc::new(move |args| {
|
||||
let maybe_inspector_server = shared.maybe_inspector_server.clone();
|
||||
|
||||
let module_loader = shared.module_loader_factory.create_for_worker(
|
||||
let ModuleLoaderAndSourceMapGetter {
|
||||
module_loader,
|
||||
source_map_getter,
|
||||
} = shared.module_loader_factory.create_for_worker(
|
||||
args.parent_permissions.clone(),
|
||||
args.permissions.clone(),
|
||||
);
|
||||
let maybe_source_map_getter =
|
||||
shared.module_loader_factory.create_source_map_getter();
|
||||
let create_web_worker_cb =
|
||||
create_web_worker_callback(mode, shared.clone(), stdio.clone());
|
||||
|
||||
|
@ -839,7 +844,7 @@ fn create_web_worker_callback(
|
|||
seed: shared.options.seed,
|
||||
create_web_worker_cb,
|
||||
format_js_error_fn: Some(Arc::new(format_js_error)),
|
||||
source_map_getter: maybe_source_map_getter,
|
||||
source_map_getter,
|
||||
module_loader,
|
||||
fs: shared.fs.clone(),
|
||||
npm_resolver: Some(shared.npm_resolver.clone().into_npm_resolver()),
|
||||
|
|
5
tests/specs/node/worker_threads_cache/__test__.jsonc
Normal file
5
tests/specs/node/worker_threads_cache/__test__.jsonc
Normal file
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"tempDir": true,
|
||||
"args": "run -A main.ts",
|
||||
"output": "main.out"
|
||||
}
|
2
tests/specs/node/worker_threads_cache/main.out
Normal file
2
tests/specs/node/worker_threads_cache/main.out
Normal file
|
@ -0,0 +1,2 @@
|
|||
[Module: null prototype] { default: true }
|
||||
[Module: null prototype] { default: false }
|
13
tests/specs/node/worker_threads_cache/main.ts
Normal file
13
tests/specs/node/worker_threads_cache/main.ts
Normal file
|
@ -0,0 +1,13 @@
|
|||
import fs from "node:fs/promises";
|
||||
import { isMainThread, Worker } from "node:worker_threads";
|
||||
|
||||
await fs.writeFile("mod.mjs", "export default " + isMainThread);
|
||||
|
||||
const path = new URL("mod.mjs", import.meta.url);
|
||||
const i = await import(path.href);
|
||||
console.log(i);
|
||||
|
||||
if (isMainThread) {
|
||||
const worker = new Worker(new URL("main.ts", import.meta.url));
|
||||
worker.on("message", (msg) => console.log(msg));
|
||||
}
|
|
@ -8739,9 +8739,7 @@
|
|||
"blob-url.any.worker-module.html": [
|
||||
"Revoking a blob URL immediately after calling import will not fail"
|
||||
],
|
||||
"blob-url-workers.window.html": [
|
||||
"A revoked blob URL will not resolve in a worker even if it's in the window's module graph"
|
||||
],
|
||||
"blob-url-workers.window.html": [],
|
||||
"microtasks": {
|
||||
"basic.any.html": [
|
||||
"import() should not drain the microtask queue if it fails during specifier resolution",
|
||||
|
|
Loading…
Reference in a new issue