0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-10-29 08:58:01 -04:00
denoland-deno/cli/proc_state.rs
2022-08-23 10:39:19 -04:00

731 lines
23 KiB
Rust

// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::Flags;
use crate::args::TypeCheckMode;
use crate::cache;
use crate::cache::EmitCache;
use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::cache::TypeCheckCache;
use crate::compat;
use crate::compat::NodeEsmResolver;
use crate::deno_dir;
use crate::emit;
use crate::emit::emit_parsed_source;
use crate::emit::TsConfigType;
use crate::emit::TsTypeLib;
use crate::file_fetcher::FileFetcher;
use crate::graph_util::graph_lock_or_exit;
use crate::graph_util::GraphData;
use crate::graph_util::ModuleEntry;
use crate::http_cache;
use crate::lockfile::as_maybe_locker;
use crate::lockfile::Lockfile;
use crate::node;
use crate::npm::GlobalNpmPackageResolver;
use crate::npm::NpmPackageReference;
use crate::npm::NpmPackageResolver;
use crate::resolver::ImportMapResolver;
use crate::resolver::JsxResolver;
use deno_ast::MediaType;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::parking_lot::Mutex;
use deno_core::parking_lot::RwLock;
use deno_core::url::Url;
use deno_core::CompiledWasmModuleStore;
use deno_core::ModuleSpecifier;
use deno_core::SharedArrayBufferStore;
use deno_graph::create_graph;
use deno_graph::source::CacheInfo;
use deno_graph::source::LoadFuture;
use deno_graph::source::Loader;
use deno_graph::source::ResolveResponse;
use deno_graph::ModuleKind;
use deno_graph::Resolved;
use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel;
use deno_runtime::deno_tls::rustls::RootCertStore;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::inspector_server::InspectorServer;
use deno_runtime::permissions::Permissions;
use import_map::ImportMap;
use log::warn;
use std::collections::HashSet;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
/// This structure represents state of single "deno" program.
///
/// It is shared by all created workers (thus V8 isolates).
#[derive(Clone)]
pub struct ProcState(Arc<Inner>);
pub struct Inner {
pub dir: deno_dir::DenoDir,
pub file_fetcher: FileFetcher,
pub options: Arc<CliOptions>,
pub emit_cache: EmitCache,
pub emit_options: deno_ast::EmitOptions,
pub emit_options_hash: u64,
pub graph_data: Arc<RwLock<GraphData>>,
pub lockfile: Option<Arc<Mutex<Lockfile>>>,
pub maybe_import_map: Option<Arc<ImportMap>>,
pub maybe_inspector_server: Option<Arc<InspectorServer>>,
pub root_cert_store: RootCertStore,
pub blob_store: BlobStore,
pub broadcast_channel: InMemoryBroadcastChannel,
pub shared_array_buffer_store: SharedArrayBufferStore,
pub compiled_wasm_module_store: CompiledWasmModuleStore,
pub parsed_source_cache: ParsedSourceCache,
maybe_resolver: Option<Arc<dyn deno_graph::source::Resolver + Send + Sync>>,
maybe_file_watcher_reporter: Option<FileWatcherReporter>,
pub npm_resolver: GlobalNpmPackageResolver,
pub cjs_resolutions: Mutex<HashSet<ModuleSpecifier>>,
}
impl Deref for ProcState {
type Target = Arc<Inner>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl ProcState {
pub async fn build(flags: Flags) -> Result<Self, AnyError> {
Self::from_options(Arc::new(CliOptions::from_flags(flags)?)).await
}
pub async fn from_options(
options: Arc<CliOptions>,
) -> Result<Self, AnyError> {
Self::build_with_sender(options, None).await
}
pub async fn build_for_file_watcher(
flags: Flags,
files_to_watch_sender: tokio::sync::mpsc::UnboundedSender<Vec<PathBuf>>,
) -> Result<Self, AnyError> {
// resolve the config each time
let cli_options = Arc::new(CliOptions::from_flags(flags)?);
let ps =
Self::build_with_sender(cli_options, Some(files_to_watch_sender.clone()))
.await?;
// Add the extra files listed in the watch flag
if let Some(watch_paths) = ps.options.watch_paths() {
files_to_watch_sender.send(watch_paths.clone()).unwrap();
}
if let Ok(Some(import_map_path)) = ps
.options
.resolve_import_map_specifier()
.map(|ms| ms.and_then(|ref s| s.to_file_path().ok()))
{
files_to_watch_sender.send(vec![import_map_path]).unwrap();
}
Ok(ps)
}
async fn build_with_sender(
cli_options: Arc<CliOptions>,
maybe_sender: Option<tokio::sync::mpsc::UnboundedSender<Vec<PathBuf>>>,
) -> Result<Self, AnyError> {
let blob_store = BlobStore::default();
let broadcast_channel = InMemoryBroadcastChannel::default();
let shared_array_buffer_store = SharedArrayBufferStore::default();
let compiled_wasm_module_store = CompiledWasmModuleStore::default();
let dir = cli_options.resolve_deno_dir()?;
let deps_cache_location = dir.root.join("deps");
let http_cache = http_cache::HttpCache::new(&deps_cache_location);
let root_cert_store = cli_options.resolve_root_cert_store()?;
let cache_usage = cli_options.cache_setting();
let file_fetcher = FileFetcher::new(
http_cache,
cache_usage,
!cli_options.no_remote(),
Some(root_cert_store.clone()),
blob_store.clone(),
cli_options
.unsafely_ignore_certificate_errors()
.map(ToOwned::to_owned),
)?;
let lockfile = cli_options
.resolve_lock_file()?
.map(|f| Arc::new(Mutex::new(f)));
let maybe_import_map_specifier =
cli_options.resolve_import_map_specifier()?;
let maybe_import_map =
if let Some(import_map_specifier) = maybe_import_map_specifier {
let file = file_fetcher
.fetch(&import_map_specifier, &mut Permissions::allow_all())
.await
.context(format!(
"Unable to load '{}' import map",
import_map_specifier
))?;
let import_map =
import_map_from_text(&import_map_specifier, &file.source)?;
Some(Arc::new(import_map))
} else {
None
};
let maybe_inspector_server =
cli_options.resolve_inspector_server().map(Arc::new);
// FIXME(bartlomieju): `NodeEsmResolver` is not aware of JSX resolver
// created below
let node_resolver = NodeEsmResolver::new(
maybe_import_map.clone().map(ImportMapResolver::new),
);
let maybe_import_map_resolver =
maybe_import_map.clone().map(ImportMapResolver::new);
let maybe_jsx_resolver = cli_options
.to_maybe_jsx_import_source_module()
.map(|im| JsxResolver::new(im, maybe_import_map_resolver.clone()));
let maybe_resolver: Option<
Arc<dyn deno_graph::source::Resolver + Send + Sync>,
> = if cli_options.compat() {
Some(Arc::new(node_resolver))
} else if let Some(jsx_resolver) = maybe_jsx_resolver {
// the JSX resolver offloads to the import map if present, otherwise uses
// the default Deno explicit import resolution.
Some(Arc::new(jsx_resolver))
} else if let Some(import_map_resolver) = maybe_import_map_resolver {
Some(Arc::new(import_map_resolver))
} else {
None
};
let maybe_file_watcher_reporter =
maybe_sender.map(|sender| FileWatcherReporter {
sender,
file_paths: Arc::new(Mutex::new(vec![])),
});
let ts_config_result =
cli_options.resolve_ts_config_for_emit(TsConfigType::Emit)?;
if let Some(ignored_options) = ts_config_result.maybe_ignored_options {
warn!("{}", ignored_options);
}
let emit_cache = EmitCache::new(dir.gen_cache.clone());
let parsed_source_cache =
ParsedSourceCache::new(Some(dir.dep_analysis_db_file_path()));
let npm_resolver = GlobalNpmPackageResolver::from_deno_dir(
&dir,
cli_options.reload_flag(),
cli_options.cache_setting(),
)?;
Ok(ProcState(Arc::new(Inner {
dir,
options: cli_options,
emit_cache,
emit_options_hash: FastInsecureHasher::new()
// todo(dsherret): use hash of emit options instead as it's more specific
.write(&ts_config_result.ts_config.as_bytes())
.finish(),
emit_options: ts_config_result.ts_config.into(),
file_fetcher,
graph_data: Default::default(),
lockfile,
maybe_import_map,
maybe_inspector_server,
root_cert_store,
blob_store,
broadcast_channel,
shared_array_buffer_store,
compiled_wasm_module_store,
parsed_source_cache,
maybe_resolver,
maybe_file_watcher_reporter,
npm_resolver,
cjs_resolutions: Default::default(),
})))
}
/// This method must be called for a module or a static importer of that
/// module before attempting to `load()` it from a `JsRuntime`. It will
/// populate `self.graph_data` in memory with the necessary source code, write
/// emits where necessary or report any module graph / type checking errors.
pub async fn prepare_module_load(
&self,
roots: Vec<ModuleSpecifier>,
is_dynamic: bool,
lib: TsTypeLib,
root_permissions: Permissions,
dynamic_permissions: Permissions,
reload_on_watch: bool,
) -> Result<(), AnyError> {
let maybe_resolver: Option<&dyn deno_graph::source::Resolver> =
if let Some(resolver) = &self.maybe_resolver {
Some(resolver.as_ref())
} else {
None
};
// NOTE(@bartlomieju):
// Even though `roots` are fully resolved at this point, we are going
// to resolve them through `maybe_resolver` to get module kind for the graph
// or default to ESM.
//
// One might argue that this is a code smell, and I would agree. However
// due to flux in "Node compatibility" it's not clear where it should be
// decided what `ModuleKind` is decided for root specifier.
let roots = roots
.into_iter()
.map(|r| {
if let Some(resolver) = &maybe_resolver {
let response =
resolver.resolve(r.as_str(), &Url::parse("unused:").unwrap());
// TODO(bartlomieju): this should be implemented in `deno_graph`
match response {
ResolveResponse::CommonJs(_) => (r, ModuleKind::CommonJs),
ResolveResponse::Err(_) => unreachable!(),
_ => (r, ModuleKind::Esm),
}
} else {
(r, ModuleKind::Esm)
}
})
.collect();
// TODO(bartlomieju): this is very make-shift, is there an existing API
// that we could include it like with "maybe_imports"?
let roots = if self.options.compat() {
let mut r = vec![(compat::GLOBAL_URL.clone(), ModuleKind::Esm)];
r.extend(roots);
r
} else {
roots
};
if !reload_on_watch {
let graph_data = self.graph_data.read();
if self.options.type_check_mode() == TypeCheckMode::None
|| graph_data.is_type_checked(&roots, &lib)
{
if let Some(result) = graph_data.check(
&roots,
self.options.type_check_mode() != TypeCheckMode::None,
false,
) {
return result;
}
}
}
let mut cache = cache::FetchCacher::new(
self.emit_cache.clone(),
self.file_fetcher.clone(),
root_permissions.clone(),
dynamic_permissions.clone(),
);
let maybe_locker = as_maybe_locker(self.lockfile.clone());
let maybe_imports = self.options.to_maybe_imports()?;
struct ProcStateLoader<'a> {
inner: &'a mut cache::FetchCacher,
graph_data: Arc<RwLock<GraphData>>,
reload: bool,
}
impl Loader for ProcStateLoader<'_> {
fn get_cache_info(
&self,
specifier: &ModuleSpecifier,
) -> Option<CacheInfo> {
self.inner.get_cache_info(specifier)
}
fn load(
&mut self,
specifier: &ModuleSpecifier,
is_dynamic: bool,
) -> LoadFuture {
let graph_data = self.graph_data.read();
let found_specifier = graph_data.follow_redirect(specifier);
match graph_data.get(&found_specifier) {
Some(_) if !self.reload => {
Box::pin(futures::future::ready(Err(anyhow!(""))))
}
_ => self.inner.load(specifier, is_dynamic),
}
}
}
let mut loader = ProcStateLoader {
inner: &mut cache,
graph_data: self.graph_data.clone(),
reload: reload_on_watch,
};
let maybe_file_watcher_reporter: Option<&dyn deno_graph::source::Reporter> =
if let Some(reporter) = &self.maybe_file_watcher_reporter {
Some(reporter)
} else {
None
};
let analyzer = self.parsed_source_cache.as_analyzer();
let graph = create_graph(
roots.clone(),
is_dynamic,
maybe_imports,
&mut loader,
maybe_resolver,
maybe_locker,
Some(&*analyzer),
maybe_file_watcher_reporter,
)
.await;
let needs_cjs_esm_translation = graph
.modules()
.iter()
.any(|m| m.kind == ModuleKind::CommonJs);
if needs_cjs_esm_translation {
for module in graph.modules() {
// TODO(bartlomieju): this is overly simplistic heuristic, once we are
// in compat mode, all files ending with plain `.js` extension are
// considered CommonJs modules. Which leads to situation where valid
// ESM modules with `.js` extension might undergo translation (it won't
// work in this situation).
if module.kind == ModuleKind::CommonJs {
let translated_source = compat::translate_cjs_to_esm(
&self.file_fetcher,
&module.specifier,
module.maybe_source.as_ref().unwrap().to_string(),
module.media_type,
)?;
let mut graph_data = self.graph_data.write();
graph_data
.add_cjs_esm_translation(&module.specifier, translated_source);
}
}
}
// If there was a locker, validate the integrity of all the modules in the
// locker.
graph_lock_or_exit(&graph);
// Determine any modules that have already been emitted this session and
// should be skipped.
let reload_exclusions: HashSet<ModuleSpecifier> = {
let graph_data = self.graph_data.read();
graph_data.entries().map(|(s, _)| s).cloned().collect()
};
let npm_package_references = {
let mut graph_data = self.graph_data.write();
graph_data.add_graph(&graph, reload_on_watch);
let check_js = self.options.check_js();
graph_data
.check(
&roots,
self.options.type_check_mode() != TypeCheckMode::None,
check_js,
)
.unwrap()?;
graph_data.npm_package_reqs()
};
if !npm_package_references.is_empty() {
self
.npm_resolver
.add_package_reqs(npm_package_references)
.await?;
self.npm_resolver.cache_packages().await?;
self.prepare_node_std_graph().await?;
}
// type check if necessary
if self.options.type_check_mode() != TypeCheckMode::None {
let maybe_config_specifier = self.options.maybe_config_file_specifier();
let roots = roots.clone();
let options = emit::CheckOptions {
type_check_mode: self.options.type_check_mode(),
debug: self.options.log_level() == Some(log::Level::Debug),
maybe_config_specifier,
ts_config: self
.options
.resolve_ts_config_for_emit(TsConfigType::Check { lib })?
.ts_config,
log_checks: true,
reload: self.options.reload_flag()
&& !roots.iter().all(|r| reload_exclusions.contains(&r.0)),
};
let check_cache =
TypeCheckCache::new(&self.dir.type_checking_cache_db_file_path());
let graph_data = self.graph_data.clone();
let check_result =
emit::check(&roots, graph_data, &check_cache, options)?;
if !check_result.diagnostics.is_empty() {
return Err(anyhow!(check_result.diagnostics));
}
log::debug!("{}", check_result.stats);
}
if self.options.type_check_mode() != TypeCheckMode::None {
let mut graph_data = self.graph_data.write();
graph_data.set_type_checked(&roots, lib);
}
// any updates to the lockfile should be updated now
if let Some(ref lockfile) = self.lockfile {
let g = lockfile.lock();
g.write()?;
}
Ok(())
}
/// Add the builtin node modules to the graph data.
pub async fn prepare_node_std_graph(&self) -> Result<(), AnyError> {
let node_std_graph = self
.create_graph(vec![(compat::MODULE_ALL_URL.clone(), ModuleKind::Esm)])
.await?;
self.graph_data.write().add_graph(&node_std_graph, false);
Ok(())
}
fn handle_node_resolve_result(
&self,
result: Result<Option<ResolveResponse>, AnyError>,
) -> Result<ModuleSpecifier, AnyError> {
let response = match result? {
Some(response) => response,
None => bail!("Not found."),
};
if let ResolveResponse::CommonJs(specifier) = &response {
// remember that this was a common js resolution
self.cjs_resolutions.lock().insert(specifier.clone());
}
response.to_result()
}
pub fn resolve(
&self,
specifier: &str,
referrer: &str,
) -> Result<ModuleSpecifier, AnyError> {
if let Ok(referrer) = deno_core::resolve_url_or_path(referrer) {
if self.npm_resolver.in_npm_package(&referrer) {
// we're in an npm package, so use node resolution
return self
.handle_node_resolve_result(node::node_resolve(
specifier,
&referrer,
&self.npm_resolver,
))
.with_context(|| {
format!(
"Could not resolve '{}' from '{}'.",
specifier,
self
.npm_resolver
.resolve_package_from_specifier(&referrer)
.unwrap()
.id
)
});
}
let graph_data = self.graph_data.read();
let found_referrer = graph_data.follow_redirect(&referrer);
let maybe_resolved = match graph_data.get(&found_referrer) {
Some(ModuleEntry::Module { dependencies, .. }) => {
dependencies.get(specifier).map(|d| &d.maybe_code)
}
_ => None,
};
match maybe_resolved {
Some(Resolved::Ok { specifier, .. }) => {
if let Ok(reference) = NpmPackageReference::from_specifier(specifier)
{
return self
.handle_node_resolve_result(node::node_resolve_npm_reference(
&reference,
&self.npm_resolver,
))
.with_context(|| format!("Could not resolve '{}'.", reference));
} else {
return Ok(specifier.clone());
}
}
Some(Resolved::Err(err)) => {
return Err(custom_error(
"TypeError",
format!("{}\n", err.to_string_with_range()),
))
}
Some(Resolved::None) | None => {}
}
}
// FIXME(bartlomieju): this is a hacky way to provide compatibility with REPL
// and `Deno.core.evalContext` API. Ideally we should always have a referrer filled
// but sadly that's not the case due to missing APIs in V8.
let referrer = if referrer.is_empty()
&& matches!(self.options.sub_command(), DenoSubcommand::Repl(_))
{
deno_core::resolve_url_or_path("./$deno$repl.ts").unwrap()
} else {
deno_core::resolve_url_or_path(referrer).unwrap()
};
let maybe_resolver: Option<&dyn deno_graph::source::Resolver> =
if let Some(resolver) = &self.maybe_resolver {
Some(resolver.as_ref())
} else {
None
};
if let Some(resolver) = &maybe_resolver {
resolver.resolve(specifier, &referrer).to_result()
} else {
deno_core::resolve_import(specifier, referrer.as_str())
.map_err(|err| err.into())
}
}
pub fn cache_module_emits(&self) -> Result<(), AnyError> {
let graph_data = self.graph_data.read();
for (specifier, entry) in graph_data.entries() {
if let ModuleEntry::Module {
code, media_type, ..
} = entry
{
let is_emittable = matches!(
media_type,
MediaType::TypeScript
| MediaType::Mts
| MediaType::Cts
| MediaType::Jsx
| MediaType::Tsx
);
if is_emittable {
emit_parsed_source(
&self.emit_cache,
&self.parsed_source_cache,
specifier,
*media_type,
code,
&self.emit_options,
self.emit_options_hash,
)?;
}
}
}
Ok(())
}
pub async fn create_graph(
&self,
roots: Vec<(ModuleSpecifier, ModuleKind)>,
) -> Result<deno_graph::ModuleGraph, AnyError> {
let mut cache = cache::FetchCacher::new(
self.emit_cache.clone(),
self.file_fetcher.clone(),
Permissions::allow_all(),
Permissions::allow_all(),
);
let maybe_locker = as_maybe_locker(self.lockfile.clone());
let maybe_import_map_resolver =
self.maybe_import_map.clone().map(ImportMapResolver::new);
let maybe_imports = self.options.to_maybe_imports()?;
let maybe_jsx_resolver = self
.options
.to_maybe_jsx_import_source_module()
.map(|im| JsxResolver::new(im, maybe_import_map_resolver.clone()));
let maybe_resolver = if maybe_jsx_resolver.is_some() {
maybe_jsx_resolver.as_ref().map(|jr| jr.as_resolver())
} else {
maybe_import_map_resolver
.as_ref()
.map(|im| im.as_resolver())
};
let analyzer = self.parsed_source_cache.as_analyzer();
let graph = create_graph(
roots,
false,
maybe_imports,
&mut cache,
maybe_resolver,
maybe_locker,
Some(&*analyzer),
None,
)
.await;
// add the found npm package references to the npm resolver and cache them
let mut package_reqs = Vec::new();
for (specifier, _) in graph.specifiers() {
if let Ok(reference) = NpmPackageReference::from_specifier(&specifier) {
package_reqs.push(reference.req);
}
}
if !package_reqs.is_empty() {
self.npm_resolver.add_package_reqs(package_reqs).await?;
self.npm_resolver.cache_packages().await?;
}
Ok(graph)
}
}
pub fn import_map_from_text(
specifier: &Url,
json_text: &str,
) -> Result<ImportMap, AnyError> {
debug_assert!(
!specifier.as_str().contains("../"),
"Import map specifier incorrectly contained ../: {}",
specifier.as_str()
);
let result = import_map::parse_from_json(specifier, json_text)?;
if !result.diagnostics.is_empty() {
warn!(
"Import map diagnostics:\n{}",
result
.diagnostics
.into_iter()
.map(|d| format!(" - {}", d))
.collect::<Vec<_>>()
.join("\n")
);
}
Ok(result.import_map)
}
#[derive(Debug)]
struct FileWatcherReporter {
sender: tokio::sync::mpsc::UnboundedSender<Vec<PathBuf>>,
file_paths: Arc<Mutex<Vec<PathBuf>>>,
}
impl deno_graph::source::Reporter for FileWatcherReporter {
fn on_load(
&self,
specifier: &ModuleSpecifier,
modules_done: usize,
modules_total: usize,
) {
let mut file_paths = self.file_paths.lock();
if specifier.scheme() == "file" {
file_paths.push(specifier.to_file_path().unwrap());
}
if modules_done == modules_total {
self.sender.send(file_paths.drain(..).collect()).unwrap();
}
}
}