From fc739dc5eb2769e4608ccf08d23ca8ff0fcc19c5 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 28 Sep 2024 07:55:01 -0400 Subject: [PATCH 001/302] refactor: use deno_path_util (#25918) --- Cargo.lock | 17 +++++++ Cargo.toml | 1 + cli/Cargo.toml | 1 + cli/args/flags.rs | 6 +-- cli/args/mod.rs | 2 +- cli/file_fetcher.rs | 4 +- cli/graph_util.rs | 4 +- cli/lsp/analysis.rs | 4 +- cli/lsp/cache.rs | 10 ++-- cli/lsp/completions.rs | 4 +- cli/lsp/config.rs | 14 +++--- cli/lsp/documents.rs | 8 ++-- cli/lsp/language_server.rs | 16 +++---- cli/lsp/resolver.rs | 4 +- cli/lsp/tsc.rs | 4 +- cli/npm/byonm.rs | 6 +-- cli/resolver.rs | 4 +- cli/tools/task.rs | 2 +- cli/util/fs.rs | 2 +- ext/fs/Cargo.toml | 1 + ext/fs/in_memory_fs.rs | 2 +- ext/fs/std_fs.rs | 2 +- ext/kv/Cargo.toml | 1 + ext/kv/sqlite.rs | 2 +- ext/node/Cargo.toml | 1 + ext/node/ops/require.rs | 4 +- runtime/Cargo.toml | 1 + runtime/fs_util.rs | 4 +- runtime/ops/os/mod.rs | 2 +- runtime/ops/process.rs | 2 +- runtime/permissions.rs | 2 +- runtime/permissions/Cargo.toml | 1 + runtime/permissions/lib.rs | 85 ++-------------------------------- 33 files changed, 83 insertions(+), 140 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ef9cd2e62..3d8641e168 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1174,6 +1174,7 @@ dependencies = [ "deno_lockfile", "deno_npm", "deno_package_json", + "deno_path_util", "deno_runtime", "deno_semver", "deno_task_shell", @@ -1570,6 +1571,7 @@ dependencies = [ "base32", "deno_core", "deno_io", + "deno_path_util", "deno_permissions", "filetime", "junction", @@ -1682,6 +1684,7 @@ dependencies = [ "chrono", "deno_core", "deno_fetch", + "deno_path_util", "deno_permissions", "deno_tls", "denokv_proto", @@ -1800,6 +1803,7 @@ dependencies = [ "deno_media_type", "deno_net", "deno_package_json", + "deno_path_util", "deno_permissions", "deno_whoami", "der", @@ -1916,11 +1920,23 @@ dependencies = [ "url", ] +[[package]] +name = "deno_path_util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625b490bc4734ff778926d7034650a2ddcef2fd42b71e30bb722249156d5a144" +dependencies = [ + "percent-encoding", + "thiserror", + "url", +] + [[package]] name = "deno_permissions" version = "0.28.0" dependencies = [ "deno_core", + "deno_path_util", "deno_terminal 0.2.0", "fqdn", "libc", @@ -1953,6 +1969,7 @@ dependencies = [ "deno_napi", "deno_net", "deno_node", + "deno_path_util", "deno_permissions", "deno_terminal 0.2.0", "deno_tls", diff --git a/Cargo.toml b/Cargo.toml index 81cbf8b4c6..3bb9f2d46e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ deno_core = { version = "0.311.0" } deno_bench_util = { version = "0.162.0", path = "./bench_util" } deno_lockfile = "=0.23.1" deno_media_type = { version = "0.1.4", features = ["module_specifier"] } +deno_path_util = "=0.1.1" deno_permissions = { version = "0.28.0", path = "./runtime/permissions" } deno_runtime = { version = "0.177.0", path = "./runtime" } deno_semver = "=0.5.13" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index f5a8624e1d..ddcf7119f4 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -73,6 +73,7 @@ deno_lint = { version = "=0.67.0", features = ["docs"] } deno_lockfile.workspace = true deno_npm = "=0.25.2" deno_package_json.workspace = true +deno_path_util.workspace = true deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_semver.workspace = true deno_task_shell = "=0.17.0" diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 9938a09552..13c93fa831 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -29,13 +29,13 @@ use deno_config::glob::PathOrPatternSet; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; -use deno_core::normalize_path; use deno_core::resolve_url_or_path; use deno_core::url::Url; use deno_graph::GraphKind; +use deno_path_util::normalize_path; +use deno_path_util::url_to_file_path; use deno_runtime::deno_permissions::parse_sys_kind; use deno_runtime::deno_permissions::PermissionsOptions; -use deno_runtime::fs_util::specifier_to_file_path; use log::debug; use log::Level; use serde::Deserialize; @@ -1002,7 +1002,7 @@ impl Flags { if module_specifier.scheme() == "file" || module_specifier.scheme() == "npm" { - if let Ok(p) = specifier_to_file_path(&module_specifier) { + if let Ok(p) = url_to_file_path(&module_specifier) { Some(vec![p.parent().unwrap().to_path_buf()]) } else { Some(vec![current_dir.to_path_buf()]) diff --git a/cli/args/mod.rs b/cli/args/mod.rs index c3a4c29377..3ff2a427fb 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -20,13 +20,13 @@ use deno_config::workspace::WorkspaceDiscoverOptions; use deno_config::workspace::WorkspaceDiscoverStart; use deno_config::workspace::WorkspaceLintConfig; use deno_config::workspace::WorkspaceResolver; -use deno_core::normalize_path; use deno_core::resolve_url_or_path; use deno_graph::GraphKind; use deno_npm::npm_rc::NpmRc; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmSystemInfo; +use deno_path_util::normalize_path; use deno_semver::npm::NpmPackageReqReference; use import_map::resolve_import_map_value_from_specifier; diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index ca11449393..1bf7635949 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -21,9 +21,9 @@ use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::source::LoaderChecksum; +use deno_path_util::url_to_file_path; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_web::BlobStore; -use deno_runtime::fs_util::specifier_to_file_path; use log::debug; use std::borrow::Cow; use std::collections::HashMap; @@ -136,7 +136,7 @@ impl MemoryFiles { /// Fetch a source file from the local file system. fn fetch_local(specifier: &ModuleSpecifier) -> Result { - let local = specifier_to_file_path(specifier).map_err(|_| { + let local = url_to_file_path(specifier).map_err(|_| { uri_error(format!("Invalid file path.\n Specifier: {specifier}")) })?; // If it doesnt have a extension, we want to treat it as typescript by default diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 17f7d6739f..7d03d3c0b2 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -39,10 +39,10 @@ use deno_graph::ModuleGraph; use deno_graph::ModuleGraphError; use deno_graph::ResolutionError; use deno_graph::SpecifierError; +use deno_path_util::url_to_file_path; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node; use deno_runtime::deno_permissions::PermissionsContainer; -use deno_runtime::fs_util::specifier_to_file_path; use deno_semver::jsr::JsrDepPackageReq; use deno_semver::package::PackageNv; use import_map::ImportMapError; @@ -948,7 +948,7 @@ pub fn has_graph_root_local_dependent_changed( }, ); while let Some((s, _)) = dependent_specifiers.next() { - if let Ok(path) = specifier_to_file_path(s) { + if let Ok(path) = url_to_file_path(s) { if let Ok(path) = canonicalize_path(&path) { if canonicalized_changed_paths.contains(&path) { return true; diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index fee9c86cbe..80db66d64c 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -23,8 +23,8 @@ use deno_core::serde::Serialize; use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::ModuleSpecifier; +use deno_path_util::url_to_file_path; use deno_runtime::deno_node::PathClean; -use deno_runtime::fs_util::specifier_to_file_path; use deno_semver::jsr::JsrPackageNvReference; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; @@ -401,7 +401,7 @@ impl<'a> TsResponseImportMapper<'a> { .flatten()?; let root_folder = package_json.path.parent()?; - let specifier_path = specifier_to_file_path(specifier).ok()?; + let specifier_path = url_to_file_path(specifier).ok()?; let mut search_paths = vec![specifier_path.clone()]; // TypeScript will provide a .js extension for quick fixes, so do // a search for the .d.ts file instead diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index a32087842d..db10dc9677 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -10,7 +10,7 @@ use crate::lsp::logging::lsp_warn; use deno_core::url::Url; use deno_core::ModuleSpecifier; -use deno_runtime::fs_util::specifier_to_file_path; +use deno_path_util::url_to_file_path; use std::collections::BTreeMap; use std::fs; use std::path::Path; @@ -24,7 +24,7 @@ pub fn calculate_fs_version( ) -> Option { match specifier.scheme() { "npm" | "node" | "data" | "blob" => None, - "file" => specifier_to_file_path(specifier) + "file" => url_to_file_path(specifier) .ok() .and_then(|path| calculate_fs_version_at_path(&path)), _ => calculate_fs_version_in_cache(cache, specifier, file_referrer), @@ -82,7 +82,7 @@ impl Default for LspCache { impl LspCache { pub fn new(global_cache_url: Option) -> Self { let global_cache_path = global_cache_url.and_then(|s| { - specifier_to_file_path(&s) + url_to_file_path(&s) .inspect(|p| { lsp_log!("Resolved global cache path: \"{}\"", p.to_string_lossy()); }) @@ -165,7 +165,7 @@ impl LspCache { &self, specifier: &ModuleSpecifier, ) -> Option { - let path = specifier_to_file_path(specifier).ok()?; + let path = url_to_file_path(specifier).ok()?; let vendor = self .vendors_by_scope .iter() @@ -176,7 +176,7 @@ impl LspCache { } pub fn is_valid_file_referrer(&self, specifier: &ModuleSpecifier) -> bool { - if let Ok(path) = specifier_to_file_path(specifier) { + if let Ok(path) = url_to_file_path(specifier) { if !path.starts_with(&self.deno_dir().root) { return true; } diff --git a/cli/lsp/completions.rs b/cli/lsp/completions.rs index b3d6fbbd04..a040be5691 100644 --- a/cli/lsp/completions.rs +++ b/cli/lsp/completions.rs @@ -29,7 +29,7 @@ use deno_core::serde::Serialize; use deno_core::serde_json::json; use deno_core::url::Position; use deno_core::ModuleSpecifier; -use deno_runtime::fs_util::specifier_to_file_path; +use deno_path_util::url_to_file_path; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::package::PackageNv; use import_map::ImportMap; @@ -380,7 +380,7 @@ fn get_local_completions( ResolutionMode::Execution, ) .ok()?; - let resolved_parent_path = specifier_to_file_path(&resolved_parent).ok()?; + let resolved_parent_path = url_to_file_path(&resolved_parent).ok()?; let raw_parent = &text[..text.char_indices().rfind(|(_, c)| *c == '/')?.0 + 1]; if resolved_parent_path.is_dir() { diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index e55e7b734d..dcb6120a45 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -36,8 +36,8 @@ use deno_core::ModuleSpecifier; use deno_lint::linter::LintConfig as DenoLintConfig; use deno_npm::npm_rc::ResolvedNpmRc; use deno_package_json::PackageJsonCache; +use deno_path_util::url_to_file_path; use deno_runtime::deno_node::PackageJson; -use deno_runtime::fs_util::specifier_to_file_path; use indexmap::IndexSet; use lsp_types::ClientCapabilities; use std::collections::BTreeMap; @@ -801,7 +801,7 @@ impl Settings { /// Returns `None` if the value should be deferred to the presence of a /// `deno.json` file. pub fn specifier_enabled(&self, specifier: &ModuleSpecifier) -> Option { - let Ok(path) = specifier_to_file_path(specifier) else { + let Ok(path) = url_to_file_path(specifier) else { // Non-file URLs are not disabled by these settings. return Some(true); }; @@ -810,7 +810,7 @@ impl Settings { let mut disable_paths = vec![]; let mut enable_paths = None; if let Some(folder_uri) = folder_uri { - if let Ok(folder_path) = specifier_to_file_path(folder_uri) { + if let Ok(folder_path) = url_to_file_path(folder_uri) { disable_paths = settings .disable_paths .iter() @@ -847,12 +847,12 @@ impl Settings { &self, specifier: &ModuleSpecifier, ) -> (&WorkspaceSettings, Option<&ModuleSpecifier>) { - let Ok(path) = specifier_to_file_path(specifier) else { + let Ok(path) = url_to_file_path(specifier) else { return (&self.unscoped, self.first_folder.as_ref()); }; for (folder_uri, settings) in self.by_workspace_folder.iter().rev() { if let Some(settings) = settings { - let Ok(folder_path) = specifier_to_file_path(folder_uri) else { + let Ok(folder_path) = url_to_file_path(folder_uri) else { continue; }; if path.starts_with(folder_path) { @@ -1767,7 +1767,7 @@ impl ConfigTree { let config_file_path = (|| { let config_setting = ws_settings.config.as_ref()?; let config_uri = folder_uri.join(config_setting).ok()?; - specifier_to_file_path(&config_uri).ok() + url_to_file_path(&config_uri).ok() })(); if config_file_path.is_some() || ws_settings.import_map.is_some() { scopes.insert( @@ -1844,7 +1844,7 @@ impl ConfigTree { let scope = config_file.specifier.join(".").unwrap(); let json_text = serde_json::to_string(&config_file.json).unwrap(); let test_fs = deno_runtime::deno_fs::InMemoryFs::default(); - let config_path = specifier_to_file_path(&config_file.specifier).unwrap(); + let config_path = url_to_file_path(&config_file.specifier).unwrap(); test_fs.setup_text_files(vec![( config_path.to_string_lossy().to_string(), json_text, diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index e96b8831bb..7d1ca6810d 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -26,8 +26,8 @@ use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; use deno_graph::source::ResolutionMode; use deno_graph::Resolution; +use deno_path_util::url_to_file_path; use deno_runtime::deno_node; -use deno_runtime::fs_util::specifier_to_file_path; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageReq; @@ -849,7 +849,7 @@ impl FileSystemDocuments { file_referrer: Option<&ModuleSpecifier>, ) -> Option> { let doc = if specifier.scheme() == "file" { - let path = specifier_to_file_path(specifier).ok()?; + let path = url_to_file_path(specifier).ok()?; let bytes = fs::read(path).ok()?; let content = deno_graph::source::decode_owned_source(specifier, bytes, None).ok()?; @@ -1136,7 +1136,7 @@ impl Documents { return true; } if specifier.scheme() == "file" { - return specifier_to_file_path(&specifier) + return url_to_file_path(&specifier) .map(|p| p.is_file()) .unwrap_or(false); } @@ -1325,7 +1325,7 @@ impl Documents { let fs_docs = &self.file_system_docs; // Clean up non-existent documents. fs_docs.docs.retain(|specifier, _| { - let Ok(path) = specifier_to_file_path(specifier) else { + let Ok(path) = url_to_file_path(specifier) else { // Remove non-file schemed docs (deps). They may not be dependencies // anymore after updating resolvers. return false; diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 216b14de62..3fe969ba3d 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -15,9 +15,9 @@ use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::GraphKind; use deno_graph::Resolution; +use deno_path_util::url_to_file_path; use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_tls::RootCertStoreProvider; -use deno_runtime::fs_util::specifier_to_file_path; use deno_semver::jsr::JsrPackageReqReference; use indexmap::Equivalent; use indexmap::IndexSet; @@ -626,7 +626,7 @@ impl Inner { let maybe_root_path = self .config .root_uri() - .and_then(|uri| specifier_to_file_path(uri).ok()); + .and_then(|uri| url_to_file_path(uri).ok()); let root_cert_store = get_root_cert_store( maybe_root_path, workspace_settings.certificate_stores.clone(), @@ -802,7 +802,7 @@ impl Inner { let mut roots = config .workspace_folders .iter() - .filter_map(|p| specifier_to_file_path(&p.0).ok()) + .filter_map(|p| url_to_file_path(&p.0).ok()) .collect::>(); roots.sort(); let roots = roots @@ -1124,7 +1124,7 @@ impl Inner { { return; } - match specifier_to_file_path(&specifier) { + match url_to_file_path(&specifier) { Ok(path) if is_importable_ext(&path) => {} _ => return, } @@ -1362,7 +1362,7 @@ impl Inner { { specifier = uri_to_url(¶ms.text_document.uri); } - let file_path = specifier_to_file_path(&specifier).map_err(|err| { + let file_path = url_to_file_path(&specifier).map_err(|err| { error!("{:#}", err); LspError::invalid_request() })?; @@ -2508,7 +2508,7 @@ impl Inner { let maybe_root_path_owned = self .config .root_uri() - .and_then(|uri| specifier_to_file_path(uri).ok()); + .and_then(|uri| url_to_file_path(uri).ok()); let mut resolved_items = Vec::::new(); for item in incoming_calls.iter() { if let Some(resolved) = item.try_resolve_call_hierarchy_incoming_call( @@ -2554,7 +2554,7 @@ impl Inner { let maybe_root_path_owned = self .config .root_uri() - .and_then(|uri| specifier_to_file_path(uri).ok()); + .and_then(|uri| url_to_file_path(uri).ok()); let mut resolved_items = Vec::::new(); for item in outgoing_calls.iter() { if let Some(resolved) = item.try_resolve_call_hierarchy_outgoing_call( @@ -2603,7 +2603,7 @@ impl Inner { let maybe_root_path_owned = self .config .root_uri() - .and_then(|uri| specifier_to_file_path(uri).ok()); + .and_then(|uri| url_to_file_path(uri).ok()); let mut resolved_items = Vec::::new(); match one_or_many { tsc::OneOrMany::One(item) => { diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index 2844eb6f98..f98b23a3f9 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -10,10 +10,10 @@ use deno_graph::source::Resolver; use deno_graph::GraphImport; use deno_graph::ModuleSpecifier; use deno_npm::NpmSystemInfo; +use deno_path_util::url_to_file_path; use deno_runtime::deno_fs; use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_node::PackageJson; -use deno_runtime::fs_util::specifier_to_file_path; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; @@ -443,7 +443,7 @@ async fn create_npm_resolver( fs: Arc::new(deno_fs::RealFs), root_node_modules_dir: config_data.and_then(|config_data| { config_data.node_modules_dir.clone().or_else(|| { - specifier_to_file_path(&config_data.scope) + url_to_file_path(&config_data.scope) .ok() .map(|p| p.join("node_modules/")) }) diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index fe3708d3cb..c8b5c47f83 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -62,7 +62,7 @@ use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_core::PollEventLoopOptions; use deno_core::RuntimeOptions; -use deno_runtime::fs_util::specifier_to_file_path; +use deno_path_util::url_to_file_path; use deno_runtime::inspector_server::InspectorServer; use deno_runtime::tokio_util::create_basic_runtime; use indexmap::IndexMap; @@ -3191,7 +3191,7 @@ impl CallHierarchyItem { let use_file_name = self.is_source_file_item(); let maybe_file_path = if uri.scheme().is_some_and(|s| s.as_str() == "file") { - specifier_to_file_path(&uri_to_url(&uri)).ok() + url_to_file_path(&uri_to_url(&uri)).ok() } else { None }; diff --git a/cli/npm/byonm.rs b/cli/npm/byonm.rs index 4c8bbd3948..83c4067655 100644 --- a/cli/npm/byonm.rs +++ b/cli/npm/byonm.rs @@ -10,12 +10,12 @@ use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_core::serde_json; use deno_package_json::PackageJsonDepValue; +use deno_path_util::url_to_file_path; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::DenoPkgJsonFsAdapter; use deno_runtime::deno_node::NodePermissions; use deno_runtime::deno_node::NodeRequireResolver; use deno_runtime::deno_node::PackageJson; -use deno_runtime::fs_util::specifier_to_file_path; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageReq; use deno_semver::Version; @@ -126,7 +126,7 @@ impl ByonmCliNpmResolver { } // attempt to resolve the npm specifier from the referrer's package.json, - if let Ok(file_path) = specifier_to_file_path(referrer) { + if let Ok(file_path) = url_to_file_path(referrer) { let mut current_path = file_path.as_path(); while let Some(dir_path) = current_path.parent() { let package_json_path = dir_path.join("package.json"); @@ -221,7 +221,7 @@ impl NpmResolver for ByonmCliNpmResolver { name: &str, referrer: &ModuleSpecifier, ) -> Result { - let maybe_referrer_file = specifier_to_file_path(referrer).ok(); + let maybe_referrer_file = url_to_file_path(referrer).ok(); let maybe_start_folder = maybe_referrer_file.as_ref().and_then(|f| f.parent()); if let Some(start_folder) = maybe_start_folder { diff --git a/cli/resolver.rs b/cli/resolver.rs index 07cb6931df..cf4cd8b74a 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -22,12 +22,12 @@ use deno_graph::NpmLoadError; use deno_graph::NpmResolvePkgReqsResult; use deno_npm::resolution::NpmResolutionError; use deno_package_json::PackageJsonDepValue; +use deno_path_util::url_to_file_path; use deno_runtime::colors; use deno_runtime::deno_fs; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::is_builtin_node_module; use deno_runtime::deno_node::NodeResolver; -use deno_runtime::fs_util::specifier_to_file_path; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageReq; use node_resolver::errors::ClosestPkgJsonError; @@ -992,7 +992,7 @@ impl SloppyImportsResolver { return None; } - let path = specifier_to_file_path(specifier).ok()?; + let path = url_to_file_path(specifier).ok()?; #[derive(Clone, Copy)] enum SloppyImportsResolutionReason { diff --git a/cli/tools/task.rs b/cli/tools/task.rs index ae91f53a7a..a5a5027d0f 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -16,7 +16,7 @@ use deno_core::anyhow::anyhow; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; -use deno_core::normalize_path; +use deno_path_util::normalize_path; use deno_task_shell::ShellCommand; use crate::args::CliOptions; diff --git a/cli/util/fs.rs b/cli/util/fs.rs index fdf6035ecd..7cf91bbb59 100644 --- a/cli/util/fs.rs +++ b/cli/util/fs.rs @@ -708,8 +708,8 @@ pub fn specifier_from_file_path( mod tests { use super::*; use deno_core::futures; - use deno_core::normalize_path; use deno_core::parking_lot::Mutex; + use deno_path_util::normalize_path; use pretty_assertions::assert_eq; use test_util::PathRef; use test_util::TempDir; diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml index 894b307efa..42717dccb0 100644 --- a/ext/fs/Cargo.toml +++ b/ext/fs/Cargo.toml @@ -21,6 +21,7 @@ async-trait.workspace = true base32.workspace = true deno_core.workspace = true deno_io.workspace = true +deno_path_util.workspace = true deno_permissions.workspace = true filetime.workspace = true libc.workspace = true diff --git a/ext/fs/in_memory_fs.rs b/ext/fs/in_memory_fs.rs index 027539e849..2692023860 100644 --- a/ext/fs/in_memory_fs.rs +++ b/ext/fs/in_memory_fs.rs @@ -12,12 +12,12 @@ use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; -use deno_core::normalize_path; use deno_core::parking_lot::Mutex; use deno_io::fs::File; use deno_io::fs::FsError; use deno_io::fs::FsResult; use deno_io::fs::FsStat; +use deno_path_util::normalize_path; use crate::interface::AccessCheckCb; use crate::interface::FsDirEntry; diff --git a/ext/fs/std_fs.rs b/ext/fs/std_fs.rs index d8d5f65027..443c26819d 100644 --- a/ext/fs/std_fs.rs +++ b/ext/fs/std_fs.rs @@ -11,13 +11,13 @@ use std::path::Path; use std::path::PathBuf; use std::rc::Rc; -use deno_core::normalize_path; use deno_core::unsync::spawn_blocking; use deno_io::fs::File; use deno_io::fs::FsError; use deno_io::fs::FsResult; use deno_io::fs::FsStat; use deno_io::StdFileResourceInner; +use deno_path_util::normalize_path; use crate::interface::AccessCheckCb; use crate::interface::FsDirEntry; diff --git a/ext/kv/Cargo.toml b/ext/kv/Cargo.toml index eee4762eb4..9b2d596d06 100644 --- a/ext/kv/Cargo.toml +++ b/ext/kv/Cargo.toml @@ -21,6 +21,7 @@ bytes.workspace = true chrono = { workspace = true, features = ["now"] } deno_core.workspace = true deno_fetch.workspace = true +deno_path_util.workspace = true deno_permissions.workspace = true deno_tls.workspace = true denokv_proto.workspace = true diff --git a/ext/kv/sqlite.rs b/ext/kv/sqlite.rs index 8027ff03d4..0b4a3693c4 100644 --- a/ext/kv/sqlite.rs +++ b/ext/kv/sqlite.rs @@ -16,9 +16,9 @@ use std::sync::OnceLock; use async_trait::async_trait; use deno_core::error::type_error; use deno_core::error::AnyError; -use deno_core::normalize_path; use deno_core::unsync::spawn_blocking; use deno_core::OpState; +use deno_path_util::normalize_path; pub use denokv_sqlite::SqliteBackendError; use denokv_sqlite::SqliteConfig; use denokv_sqlite::SqliteNotifier; diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index 17da17cd81..06a91c501f 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -34,6 +34,7 @@ deno_io.workspace = true deno_media_type.workspace = true deno_net.workspace = true deno_package_json.workspace = true +deno_path_util.workspace = true deno_permissions.workspace = true deno_whoami = "0.1.0" der = { version = "0.7.9", features = ["derive"] } diff --git a/ext/node/ops/require.rs b/ext/node/ops/require.rs index 3578719d0c..15667aae73 100644 --- a/ext/node/ops/require.rs +++ b/ext/node/ops/require.rs @@ -3,7 +3,6 @@ use deno_core::anyhow::Context; use deno_core::error::generic_error; use deno_core::error::AnyError; -use deno_core::normalize_path; use deno_core::op2; use deno_core::url::Url; use deno_core::v8; @@ -12,6 +11,7 @@ use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_fs::FileSystemRc; use deno_package_json::PackageJsonRc; +use deno_path_util::normalize_path; use node_resolver::NodeModuleKind; use node_resolver::NodeResolutionMode; use node_resolver::REQUIRE_CONDITIONS; @@ -104,7 +104,7 @@ where } else { let current_dir = &(fs.cwd().map_err(AnyError::from)).context("Unable to get CWD")?; - deno_core::normalize_path(current_dir.join(from)) + deno_path_util::normalize_path(current_dir.join(from)) }; ensure_read_permission::

(state, &from)?; diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index ea2978c49b..d999513d28 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -88,6 +88,7 @@ deno_kv.workspace = true deno_napi.workspace = true deno_net.workspace = true deno_node.workspace = true +deno_path_util.workspace = true deno_permissions.workspace = true deno_terminal.workspace = true deno_tls.workspace = true diff --git a/runtime/fs_util.rs b/runtime/fs_util.rs index 15ebafdf61..a858e9770d 100644 --- a/runtime/fs_util.rs +++ b/runtime/fs_util.rs @@ -2,12 +2,10 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; +use deno_path_util::normalize_path; use std::path::Path; use std::path::PathBuf; -pub use deno_core::normalize_path; -pub use deno_permissions::specifier_to_file_path; - #[inline] pub fn resolve_from_cwd(path: &Path) -> Result { if path.is_absolute() { diff --git a/runtime/ops/os/mod.rs b/runtime/ops/os/mod.rs index 544031dd7b..bd9260e97e 100644 --- a/runtime/ops/os/mod.rs +++ b/runtime/ops/os/mod.rs @@ -4,11 +4,11 @@ use super::utils::into_string; use crate::worker::ExitCode; use deno_core::error::type_error; use deno_core::error::AnyError; -use deno_core::normalize_path; use deno_core::op2; use deno_core::v8; use deno_core::OpState; use deno_node::NODE_ENV_VAR_ALLOWLIST; +use deno_path_util::normalize_path; use deno_permissions::PermissionsContainer; use serde::Serialize; use std::collections::HashMap; diff --git a/runtime/ops/process.rs b/runtime/ops/process.rs index 530dcf49be..f6555e9324 100644 --- a/runtime/ops/process.rs +++ b/runtime/ops/process.rs @@ -689,7 +689,7 @@ fn resolve_cmd(cmd: &str, env: &RunEnv) -> Result { } fn resolve_path(path: &str, cwd: &Path) -> PathBuf { - deno_core::normalize_path(cwd.join(path)) + deno_path_util::normalize_path(cwd.join(path)) } fn check_run_permission( diff --git a/runtime/permissions.rs b/runtime/permissions.rs index 9b2737b4fc..533319c4ef 100644 --- a/runtime/permissions.rs +++ b/runtime/permissions.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; -use deno_core::normalize_path; +use deno_path_util::normalize_path; use deno_permissions::AllowRunDescriptor; use deno_permissions::AllowRunDescriptorParseResult; use deno_permissions::DenyRunDescriptor; diff --git a/runtime/permissions/Cargo.toml b/runtime/permissions/Cargo.toml index 9821b61484..37500d3ebb 100644 --- a/runtime/permissions/Cargo.toml +++ b/runtime/permissions/Cargo.toml @@ -15,6 +15,7 @@ path = "lib.rs" [dependencies] deno_core.workspace = true +deno_path_util.workspace = true deno_terminal.workspace = true fqdn = "0.3.4" libc.workspace = true diff --git a/runtime/permissions/lib.rs b/runtime/permissions/lib.rs index 5d8e76125e..e919b81b60 100644 --- a/runtime/permissions/lib.rs +++ b/runtime/permissions/lib.rs @@ -6,7 +6,6 @@ use deno_core::error::custom_error; use deno_core::error::type_error; use deno_core::error::uri_error; use deno_core::error::AnyError; -use deno_core::normalize_path; use deno_core::parking_lot::Mutex; use deno_core::serde::de; use deno_core::serde::Deserialize; @@ -16,6 +15,8 @@ use deno_core::serde_json; use deno_core::unsync::sync::AtomicFlag; use deno_core::url::Url; use deno_core::ModuleSpecifier; +use deno_path_util::normalize_path; +use deno_path_util::url_to_file_path; use deno_terminal::colors; use fqdn::FQDN; use once_cell::sync::Lazy; @@ -2032,52 +2033,6 @@ impl Permissions { } } -/// Attempts to convert a specifier to a file path. By default, uses the Url -/// crate's `to_file_path()` method, but falls back to try and resolve unix-style -/// paths on Windows. -pub fn specifier_to_file_path( - specifier: &ModuleSpecifier, -) -> Result { - let result = if specifier.scheme() != "file" { - Err(()) - } else if cfg!(windows) { - if specifier.host().is_some() { - Err(()) - } else { - match specifier.to_file_path() { - Ok(path) => Ok(path), - Err(()) => { - // This might be a unix-style path which is used in the tests even on Windows. - // Attempt to see if we can convert it to a `PathBuf`. This code should be removed - // once/if https://github.com/servo/rust-url/issues/730 is implemented. - if specifier.scheme() == "file" - && specifier.port().is_none() - && specifier.path_segments().is_some() - { - let path_str = specifier.path(); - match String::from_utf8( - percent_encoding::percent_decode(path_str.as_bytes()).collect(), - ) { - Ok(path_str) => Ok(PathBuf::from(path_str)), - Err(_) => Err(()), - } - } else { - Err(()) - } - } - } - } - } else { - specifier.to_file_path() - }; - match result { - Ok(path) => Ok(path), - Err(()) => Err(uri_error(format!( - "Invalid file path.\n Specifier: {specifier}" - ))), - } -} - #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum CheckSpecifierKind { Static, @@ -2130,7 +2085,7 @@ impl PermissionsContainer { return Ok(()); } - match specifier_to_file_path(specifier) { + match url_to_file_path(specifier) { Ok(path) => inner.read.check( &PathQueryDescriptor { requested: path.to_string_lossy().into_owned(), @@ -4453,38 +4408,4 @@ mod tests { ); } } - - #[test] - fn test_specifier_to_file_path() { - run_success_test("file:///", "/"); - run_success_test("file:///test", "/test"); - run_success_test("file:///dir/test/test.txt", "/dir/test/test.txt"); - run_success_test( - "file:///dir/test%20test/test.txt", - "/dir/test test/test.txt", - ); - - assert_no_panic_specifier_to_file_path("file:/"); - assert_no_panic_specifier_to_file_path("file://"); - assert_no_panic_specifier_to_file_path("file://asdf/"); - assert_no_panic_specifier_to_file_path("file://asdf/66666/a.ts"); - - fn run_success_test(specifier: &str, expected_path: &str) { - let result = - specifier_to_file_path(&ModuleSpecifier::parse(specifier).unwrap()) - .unwrap(); - assert_eq!(result, PathBuf::from(expected_path)); - } - fn assert_no_panic_specifier_to_file_path(specifier: &str) { - let result = - specifier_to_file_path(&ModuleSpecifier::parse(specifier).unwrap()); - match result { - Ok(_) => (), - Err(err) => assert_eq!( - err.to_string(), - format!("Invalid file path.\n Specifier: {specifier}") - ), - } - } - } } From 1bb47805d6331ad048bd5e7ea2581aa230e8fc93 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 28 Sep 2024 08:50:16 -0400 Subject: [PATCH 002/302] refactor: move NpmCacheDir to deno_cache_dir (#25916) Part of the ongoing work to move more of Deno's resolution out of the CLI crate (for use in Wasm and other things) Includes: * https://github.com/denoland/deno_cache_dir/pull/60 --- Cargo.lock | 5 +- Cargo.toml | 2 +- cli/cache/mod.rs | 77 ++++++++ cli/npm/cache_dir.rs | 295 ----------------------------- cli/npm/managed/cache/mod.rs | 44 ++++- cli/npm/managed/mod.rs | 4 +- cli/npm/managed/resolvers/local.rs | 4 +- cli/npm/mod.rs | 2 - cli/standalone/file_system.rs | 4 +- cli/standalone/mod.rs | 4 +- cli/util/fs.rs | 157 +++++++++++---- cli/util/path.rs | 42 ---- ext/fs/in_memory_fs.rs | 8 +- ext/fs/interface.rs | 10 +- ext/fs/ops.rs | 12 +- ext/fs/std_fs.rs | 8 +- 16 files changed, 267 insertions(+), 411 deletions(-) delete mode 100644 cli/npm/cache_dir.rs diff --git a/Cargo.lock b/Cargo.lock index 3d8641e168..1372ac191a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1348,10 +1348,11 @@ dependencies = [ [[package]] name = "deno_cache_dir" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df43311cb7703fa3242c282823a850e4c8d0c06b9527d8209b55bd695452ea5" +checksum = "87900cfcd07bdbf3597bc36b77da0c0e7b6c2e65213faa2ed43d9a1ec12bd31d" dependencies = [ + "base32", "deno_media_type", "indexmap", "log", diff --git a/Cargo.toml b/Cargo.toml index 3bb9f2d46e..4b27f85b82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,7 +104,7 @@ chrono = { version = "0.4", default-features = false, features = ["std", "serde" console_static_text = "=0.8.1" data-encoding = "2.3.3" data-url = "=0.3.0" -deno_cache_dir = "=0.11.1" +deno_cache_dir = "=0.12.0" deno_package_json = { version = "=0.1.1", default-features = false } dlopen2 = "0.6.1" ecb = "=0.1.2" diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 2d88137165..2296bce010 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -10,6 +10,8 @@ use crate::file_fetcher::FileFetcher; use crate::file_fetcher::FileOrRedirect; use crate::npm::CliNpmResolver; use crate::util::fs::atomic_write_file_with_retries; +use crate::util::fs::atomic_write_file_with_retries_and_fs; +use crate::util::fs::AtomicWriteFileFsAdapter; use crate::util::path::specifier_has_extension; use deno_ast::MediaType; @@ -77,6 +79,14 @@ impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv { atomic_write_file_with_retries(path, bytes, CACHE_PERM) } + fn canonicalize_path(&self, path: &Path) -> std::io::Result { + crate::util::fs::canonicalize_path(path) + } + + fn create_dir_all(&self, path: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(path) + } + fn remove_file(&self, path: &Path) -> std::io::Result<()> { std::fs::remove_file(path) } @@ -100,6 +110,73 @@ impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv { } } +#[derive(Debug, Clone)] +pub struct DenoCacheEnvFsAdapter<'a>( + pub &'a dyn deno_runtime::deno_fs::FileSystem, +); + +impl<'a> deno_cache_dir::DenoCacheEnv for DenoCacheEnvFsAdapter<'a> { + fn read_file_bytes(&self, path: &Path) -> std::io::Result> { + self + .0 + .read_file_sync(path, None) + .map_err(|err| err.into_io_error()) + } + + fn atomic_write_file( + &self, + path: &Path, + bytes: &[u8], + ) -> std::io::Result<()> { + atomic_write_file_with_retries_and_fs( + &AtomicWriteFileFsAdapter { + fs: self.0, + write_mode: CACHE_PERM, + }, + path, + bytes, + ) + } + + fn canonicalize_path(&self, path: &Path) -> std::io::Result { + self.0.realpath_sync(path).map_err(|e| e.into_io_error()) + } + + fn create_dir_all(&self, path: &Path) -> std::io::Result<()> { + self + .0 + .mkdir_sync(path, true, None) + .map_err(|e| e.into_io_error()) + } + + fn remove_file(&self, path: &Path) -> std::io::Result<()> { + self + .0 + .remove_sync(path, false) + .map_err(|e| e.into_io_error()) + } + + fn modified(&self, path: &Path) -> std::io::Result> { + self + .0 + .stat_sync(path) + .map(|stat| { + stat + .mtime + .map(|ts| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(ts)) + }) + .map_err(|e| e.into_io_error()) + } + + fn is_file(&self, path: &Path) -> bool { + self.0.is_file_sync(path) + } + + fn time_now(&self) -> SystemTime { + SystemTime::now() + } +} + pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache; pub type LocalHttpCache = deno_cache_dir::LocalHttpCache; pub type LocalLspHttpCache = diff --git a/cli/npm/cache_dir.rs b/cli/npm/cache_dir.rs deleted file mode 100644 index 4467d685ec..0000000000 --- a/cli/npm/cache_dir.rs +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use std::path::Path; -use std::path::PathBuf; - -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::Context; -use deno_core::error::AnyError; -use deno_core::url::Url; -use deno_npm::NpmPackageCacheFolderId; -use deno_semver::package::PackageNv; -use deno_semver::Version; - -use crate::util::fs::canonicalize_path; -use crate::util::path::root_url_to_safe_local_dirname; - -/// The global cache directory of npm packages. -#[derive(Clone, Debug)] -pub struct NpmCacheDir { - root_dir: PathBuf, - // cached url representation of the root directory - root_dir_url: Url, - // A list of all registry that were discovered via `.npmrc` files - // turned into a safe directory names. - known_registries_dirnames: Vec, -} - -impl NpmCacheDir { - pub fn new(root_dir: PathBuf, known_registries_urls: Vec) -> Self { - fn try_get_canonicalized_root_dir( - root_dir: &Path, - ) -> Result { - if !root_dir.exists() { - std::fs::create_dir_all(root_dir) - .with_context(|| format!("Error creating {}", root_dir.display()))?; - } - Ok(canonicalize_path(root_dir)?) - } - - // this may fail on readonly file systems, so just ignore if so - let root_dir = - try_get_canonicalized_root_dir(&root_dir).unwrap_or(root_dir); - let root_dir_url = Url::from_directory_path(&root_dir).unwrap(); - - let known_registries_dirnames: Vec<_> = known_registries_urls - .into_iter() - .map(|url| { - root_url_to_safe_local_dirname(&url) - .to_string_lossy() - .replace('\\', "/") - }) - .collect(); - - Self { - root_dir, - root_dir_url, - known_registries_dirnames, - } - } - - pub fn root_dir(&self) -> &Path { - &self.root_dir - } - - pub fn root_dir_url(&self) -> &Url { - &self.root_dir_url - } - - pub fn package_folder_for_id( - &self, - folder_id: &NpmPackageCacheFolderId, - registry_url: &Url, - ) -> PathBuf { - if folder_id.copy_index == 0 { - self.package_folder_for_nv(&folder_id.nv, registry_url) - } else { - self - .package_name_folder(&folder_id.nv.name, registry_url) - .join(format!("{}_{}", folder_id.nv.version, folder_id.copy_index)) - } - } - - pub fn package_folder_for_nv( - &self, - package: &PackageNv, - registry_url: &Url, - ) -> PathBuf { - self - .package_name_folder(&package.name, registry_url) - .join(package.version.to_string()) - } - - pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { - let mut dir = self.registry_folder(registry_url); - if name.to_lowercase() != name { - let encoded_name = mixed_case_package_name_encode(name); - // Using the encoded directory may have a collision with an actual package name - // so prefix it with an underscore since npm packages can't start with that - dir.join(format!("_{encoded_name}")) - } else { - // ensure backslashes are used on windows - for part in name.split('/') { - dir = dir.join(part); - } - dir - } - } - - fn registry_folder(&self, registry_url: &Url) -> PathBuf { - self - .root_dir - .join(root_url_to_safe_local_dirname(registry_url)) - } - - pub fn resolve_package_folder_id_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Option { - let mut maybe_relative_url = None; - - // Iterate through known registries and try to get a match. - for registry_dirname in &self.known_registries_dirnames { - let registry_root_dir = self - .root_dir_url - .join(&format!("{}/", registry_dirname)) - // this not succeeding indicates a fatal issue, so unwrap - .unwrap(); - - let Some(relative_url) = registry_root_dir.make_relative(specifier) - else { - continue; - }; - - if relative_url.starts_with("../") { - continue; - } - - maybe_relative_url = Some(relative_url); - break; - } - - let mut relative_url = maybe_relative_url?; - - // base32 decode the url if it starts with an underscore - // * Ex. _{base32(package_name)}/ - if let Some(end_url) = relative_url.strip_prefix('_') { - let mut parts = end_url - .split('/') - .map(ToOwned::to_owned) - .collect::>(); - match mixed_case_package_name_decode(&parts[0]) { - Some(part) => { - parts[0] = part; - } - None => return None, - } - relative_url = parts.join("/"); - } - - // examples: - // * chalk/5.0.1/ - // * @types/chalk/5.0.1/ - // * some-package/5.0.1_1/ -- where the `_1` (/_\d+/) is a copy of the folder for peer deps - let is_scoped_package = relative_url.starts_with('@'); - let mut parts = relative_url - .split('/') - .enumerate() - .take(if is_scoped_package { 3 } else { 2 }) - .map(|(_, part)| part) - .collect::>(); - if parts.len() < 2 { - return None; - } - let version_part = parts.pop().unwrap(); - let name = parts.join("/"); - let (version, copy_index) = - if let Some((version, copy_count)) = version_part.split_once('_') { - (version, copy_count.parse::().ok()?) - } else { - (version_part, 0) - }; - Some(NpmPackageCacheFolderId { - nv: PackageNv { - name, - version: Version::parse_from_npm(version).ok()?, - }, - copy_index, - }) - } - - pub fn get_cache_location(&self) -> PathBuf { - self.root_dir.clone() - } -} - -pub fn mixed_case_package_name_encode(name: &str) -> String { - // use base32 encoding because it's reversible and the character set - // only includes the characters within 0-9 and A-Z so it can be lower cased - base32::encode( - base32::Alphabet::Rfc4648Lower { padding: false }, - name.as_bytes(), - ) - .to_lowercase() -} - -pub fn mixed_case_package_name_decode(name: &str) -> Option { - base32::decode(base32::Alphabet::Rfc4648Lower { padding: false }, name) - .and_then(|b| String::from_utf8(b).ok()) -} - -#[cfg(test)] -mod test { - use deno_core::url::Url; - use deno_semver::package::PackageNv; - use deno_semver::Version; - - use super::NpmCacheDir; - use crate::npm::cache_dir::NpmPackageCacheFolderId; - - #[test] - fn should_get_package_folder() { - let deno_dir = crate::cache::DenoDir::new(None).unwrap(); - let root_dir = deno_dir.npm_folder_path(); - let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); - let cache = NpmCacheDir::new(root_dir.clone(), vec![registry_url.clone()]); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "json".to_string(), - version: Version::parse_from_npm("1.2.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("json") - .join("1.2.5"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "json".to_string(), - version: Version::parse_from_npm("1.2.5").unwrap(), - }, - copy_index: 1, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("json") - .join("1.2.5_1"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "JSON".to_string(), - version: Version::parse_from_npm("2.1.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("_jjju6tq") - .join("2.1.5"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "@types/JSON".to_string(), - version: Version::parse_from_npm("2.1.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("_ib2hs4dfomxuuu2pjy") - .join("2.1.5"), - ); - } -} diff --git a/cli/npm/managed/cache/mod.rs b/cli/npm/managed/cache/mod.rs index f409744b98..fa0e8c8a59 100644 --- a/cli/npm/managed/cache/mod.rs +++ b/cli/npm/managed/cache/mod.rs @@ -8,6 +8,7 @@ use std::path::PathBuf; use std::sync::Arc; use deno_ast::ModuleSpecifier; +use deno_cache_dir::npm::NpmCacheDir; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; @@ -18,10 +19,10 @@ use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; use deno_npm::NpmPackageCacheFolderId; use deno_semver::package::PackageNv; +use deno_semver::Version; use crate::args::CacheSetting; use crate::cache::CACHE_PERM; -use crate::npm::NpmCacheDir; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::hard_link_dir_recursive; @@ -87,9 +88,12 @@ impl NpmCache { ) -> Result<(), AnyError> { let registry_url = self.npmrc.get_registry_url(&folder_id.nv.name); assert_ne!(folder_id.copy_index, 0); - let package_folder = self - .cache_dir - .package_folder_for_id(folder_id, registry_url); + let package_folder = self.cache_dir.package_folder_for_id( + &folder_id.nv.name, + &folder_id.nv.version.to_string(), + folder_id.copy_index, + registry_url, + ); if package_folder.exists() // if this file exists, then the package didn't successfully initialize @@ -100,9 +104,12 @@ impl NpmCache { return Ok(()); } - let original_package_folder = self - .cache_dir - .package_folder_for_nv(&folder_id.nv, registry_url); + let original_package_folder = self.cache_dir.package_folder_for_id( + &folder_id.nv.name, + &folder_id.nv.version.to_string(), + 0, // original copy index + registry_url, + ); // it seems Windows does an "AccessDenied" error when moving a // directory with hard links, so that's why this solution is done @@ -114,7 +121,12 @@ impl NpmCache { pub fn package_folder_for_id(&self, id: &NpmPackageCacheFolderId) -> PathBuf { let registry_url = self.npmrc.get_registry_url(&id.nv.name); - self.cache_dir.package_folder_for_id(id, registry_url) + self.cache_dir.package_folder_for_id( + &id.nv.name, + &id.nv.version.to_string(), + id.copy_index, + registry_url, + ) } pub fn package_folder_for_nv(&self, package: &PackageNv) -> PathBuf { @@ -127,7 +139,12 @@ impl NpmCache { package: &PackageNv, registry_url: &Url, ) -> PathBuf { - self.cache_dir.package_folder_for_nv(package, registry_url) + self.cache_dir.package_folder_for_id( + &package.name, + &package.version.to_string(), + 0, // original copy_index + registry_url, + ) } pub fn package_name_folder(&self, name: &str) -> PathBuf { @@ -146,6 +163,15 @@ impl NpmCache { self .cache_dir .resolve_package_folder_id_from_specifier(specifier) + .and_then(|cache_id| { + Some(NpmPackageCacheFolderId { + nv: PackageNv { + name: cache_id.name, + version: Version::parse_from_npm(&cache_id.version).ok()?, + }, + copy_index: cache_id.copy_index, + }) + }) } pub fn load_package_info( diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index 40c92cd46c..e3ac5e1af6 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use cache::RegistryInfoDownloader; use cache::TarballCache; use deno_ast::ModuleSpecifier; +use deno_cache_dir::npm::NpmCacheDir; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::serde_json; @@ -35,6 +36,7 @@ use crate::args::LifecycleScriptsConfig; use crate::args::NpmInstallDepsProvider; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; +use crate::cache::DenoCacheEnvFsAdapter; use crate::cache::FastInsecureHasher; use crate::http_util::HttpClientProvider; use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; @@ -50,7 +52,6 @@ use self::resolvers::NpmPackageFsResolver; use super::CliNpmResolver; use super::InnerCliNpmResolverRef; -use super::NpmCacheDir; mod cache; mod registry; @@ -188,6 +189,7 @@ fn create_inner( fn create_cache(options: &CliNpmResolverManagedCreateOptions) -> Arc { Arc::new(NpmCache::new( NpmCacheDir::new( + &DenoCacheEnvFsAdapter(options.fs.as_ref()), options.npm_global_cache_dir.clone(), options.npmrc.get_all_known_registries_urls(), ), diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index 5a90f252de..297fcab230 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -19,6 +19,8 @@ use crate::args::LifecycleScriptsConfig; use crate::colors; use async_trait::async_trait; use deno_ast::ModuleSpecifier; +use deno_cache_dir::npm::mixed_case_package_name_decode; +use deno_cache_dir::npm::mixed_case_package_name_encode; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; @@ -42,8 +44,6 @@ use serde::Serialize; use crate::args::NpmInstallDepsProvider; use crate::cache::CACHE_PERM; -use crate::npm::cache_dir::mixed_case_package_name_decode; -use crate::npm::cache_dir::mixed_case_package_name_encode; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; use crate::util::fs::clone_dir_recursive; diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 15ac8ebb28..2c9ee20bc8 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -1,7 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. mod byonm; -mod cache_dir; mod common; mod managed; @@ -24,7 +23,6 @@ use crate::file_fetcher::FileFetcher; pub use self::byonm::ByonmCliNpmResolver; pub use self::byonm::CliNpmResolverByonmCreateOptions; -pub use self::cache_dir::NpmCacheDir; pub use self::managed::CliNpmResolverManagedCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::ManagedCliNpmResolver; diff --git a/cli/standalone/file_system.rs b/cli/standalone/file_system.rs index 536b17f277..314444630b 100644 --- a/cli/standalone/file_system.rs +++ b/cli/standalone/file_system.rs @@ -102,7 +102,7 @@ impl FileSystem for DenoCompileFileSystem { &self, path: &Path, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { self.error_if_in_vfs(path)?; RealFs.mkdir_sync(path, recursive, mode) @@ -111,7 +111,7 @@ impl FileSystem for DenoCompileFileSystem { &self, path: PathBuf, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { self.error_if_in_vfs(&path)?; RealFs.mkdir_async(path, recursive, mode).await diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index c501d2d6bc..93ac6002bf 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -6,6 +6,7 @@ #![allow(unused_imports)] use deno_ast::MediaType; +use deno_cache_dir::npm::NpmCacheDir; use deno_config::workspace::MappedResolution; use deno_config::workspace::MappedResolutionError; use deno_config::workspace::ResolverWorkspaceJsrPackage; @@ -55,6 +56,7 @@ use crate::args::StorageKeyResolver; use crate::cache::Caches; use crate::cache::DenoDirProvider; use crate::cache::NodeAnalysisCache; +use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClientProvider; use crate::node::CliCjsCodeAnalyzer; use crate::npm::create_cli_npm_resolver; @@ -62,7 +64,6 @@ use crate::npm::CliNpmResolverByonmCreateOptions; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; -use crate::npm::NpmCacheDir; use crate::resolver::CjsResolutionStore; use crate::resolver::CliNodeResolver; use crate::resolver::NpmModuleLoader; @@ -464,6 +465,7 @@ pub async fn run( let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap(); let root_node_modules_path = root_path.join("node_modules"); let npm_cache_dir = NpmCacheDir::new( + &RealDenoCacheEnv, root_node_modules_path.clone(), vec![npm_registry_url.clone()], ); diff --git a/cli/util/fs.rs b/cli/util/fs.rs index 7cf91bbb59..9734d417e5 100644 --- a/cli/util/fs.rs +++ b/cli/util/fs.rs @@ -37,10 +37,98 @@ pub fn atomic_write_file_with_retries>( file_path: &Path, data: T, mode: u32, +) -> std::io::Result<()> { + struct RealAtomicWriteFileFs { + mode: u32, + } + + impl AtomicWriteFileFs for RealAtomicWriteFileFs { + fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> { + write_file(path, bytes, self.mode) + } + fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> { + std::fs::rename(from, to) + } + fn remove_file(&self, path: &Path) -> std::io::Result<()> { + std::fs::remove_file(path) + } + fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(dir_path) + } + fn path_exists(&self, path: &Path) -> bool { + path.exists() + } + } + + atomic_write_file_with_retries_and_fs( + &RealAtomicWriteFileFs { mode }, + file_path, + data.as_ref(), + ) +} + +pub trait AtomicWriteFileFs { + fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()>; + fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()>; + fn remove_file(&self, path: &Path) -> std::io::Result<()>; + fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()>; + fn path_exists(&self, path: &Path) -> bool; +} + +pub struct AtomicWriteFileFsAdapter<'a> { + pub fs: &'a dyn FileSystem, + pub write_mode: u32, +} + +impl<'a> AtomicWriteFileFs for AtomicWriteFileFsAdapter<'a> { + fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> { + self + .fs + .write_file_sync( + path, + deno_runtime::deno_fs::OpenOptions::write( + true, + false, + false, + Some(self.write_mode), + ), + None, + bytes, + ) + .map_err(|e| e.into_io_error()) + } + + fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> { + self.fs.rename_sync(from, to).map_err(|e| e.into_io_error()) + } + + fn remove_file(&self, path: &Path) -> std::io::Result<()> { + self + .fs + .remove_sync(path, false) + .map_err(|e| e.into_io_error()) + } + + fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> { + self + .fs + .mkdir_sync(dir_path, /* recursive */ true, None) + .map_err(|e| e.into_io_error()) + } + + fn path_exists(&self, path: &Path) -> bool { + self.fs.exists_sync(path) + } +} + +pub fn atomic_write_file_with_retries_and_fs>( + fs: &impl AtomicWriteFileFs, + file_path: &Path, + data: T, ) -> std::io::Result<()> { let mut count = 0; loop { - match atomic_write_file(file_path, data.as_ref(), mode) { + match atomic_write_file(fs, file_path, data.as_ref()) { Ok(()) => return Ok(()), Err(err) => { if count >= 5 { @@ -61,63 +149,54 @@ pub fn atomic_write_file_with_retries>( /// /// This also handles creating the directory if a NotFound error /// occurs. -fn atomic_write_file>( +fn atomic_write_file( + fs: &impl AtomicWriteFileFs, file_path: &Path, - data: T, - mode: u32, + data: &[u8], ) -> std::io::Result<()> { fn atomic_write_file_raw( + fs: &impl AtomicWriteFileFs, temp_file_path: &Path, file_path: &Path, data: &[u8], - mode: u32, ) -> std::io::Result<()> { - write_file(temp_file_path, data, mode)?; - std::fs::rename(temp_file_path, file_path).map_err(|err| { + fs.write_file(temp_file_path, data)?; + fs.rename_file(temp_file_path, file_path).map_err(|err| { // clean up the created temp file on error - let _ = std::fs::remove_file(temp_file_path); + let _ = fs.remove_file(temp_file_path); err }) } - fn inner(file_path: &Path, data: &[u8], mode: u32) -> std::io::Result<()> { - let temp_file_path = get_atomic_file_path(file_path); + let temp_file_path = get_atomic_file_path(file_path); - if let Err(write_err) = - atomic_write_file_raw(&temp_file_path, file_path, data, mode) - { - if write_err.kind() == ErrorKind::NotFound { - let parent_dir_path = file_path.parent().unwrap(); - match std::fs::create_dir_all(parent_dir_path) { - Ok(()) => { - return atomic_write_file_raw( - &temp_file_path, - file_path, - data, - mode, - ) + if let Err(write_err) = + atomic_write_file_raw(fs, &temp_file_path, file_path, data) + { + if write_err.kind() == ErrorKind::NotFound { + let parent_dir_path = file_path.parent().unwrap(); + match fs.create_dir_all(parent_dir_path) { + Ok(()) => { + return atomic_write_file_raw(fs, &temp_file_path, file_path, data) .map_err(|err| add_file_context_to_err(file_path, err)); - } - Err(create_err) => { - if !parent_dir_path.exists() { - return Err(Error::new( - create_err.kind(), - format!( - "{:#} (for '{}')\nCheck the permission of the directory.", - create_err, - parent_dir_path.display() - ), - )); - } + } + Err(create_err) => { + if !fs.path_exists(parent_dir_path) { + return Err(Error::new( + create_err.kind(), + format!( + "{:#} (for '{}')\nCheck the permission of the directory.", + create_err, + parent_dir_path.display() + ), + )); } } } - return Err(add_file_context_to_err(file_path, write_err)); } - Ok(()) + return Err(add_file_context_to_err(file_path, write_err)); } - - inner(file_path, data.as_ref(), mode) + Ok(()) } /// Creates a std::fs::File handling if the parent does not exist. diff --git a/cli/util/path.rs b/cli/util/path.rs index 6f09cf1eac..e4ae6e7cb1 100644 --- a/cli/util/path.rs +++ b/cli/util/path.rs @@ -165,48 +165,6 @@ pub fn relative_path(from: &Path, to: &Path) -> Option { pathdiff::diff_paths(to, from) } -/// Gets if the provided character is not supported on all -/// kinds of file systems. -pub fn is_banned_path_char(c: char) -> bool { - matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*') -} - -/// Gets a safe local directory name for the provided url. -/// -/// For example: -/// https://deno.land:8080/path -> deno.land_8080/path -pub fn root_url_to_safe_local_dirname(root: &ModuleSpecifier) -> PathBuf { - fn sanitize_segment(text: &str) -> String { - text - .chars() - .map(|c| if is_banned_segment_char(c) { '_' } else { c }) - .collect() - } - - fn is_banned_segment_char(c: char) -> bool { - matches!(c, '/' | '\\') || is_banned_path_char(c) - } - - let mut result = String::new(); - if let Some(domain) = root.domain() { - result.push_str(&sanitize_segment(domain)); - } - if let Some(port) = root.port() { - if !result.is_empty() { - result.push('_'); - } - result.push_str(&port.to_string()); - } - let mut result = PathBuf::from(result); - if let Some(segments) = root.path_segments() { - for segment in segments.filter(|s| !s.is_empty()) { - result = result.join(sanitize_segment(segment)); - } - } - - result -} - /// Slightly different behaviour than the default matching /// where an exact path needs to be matched to be opted-in /// rather than just a partial directory match. diff --git a/ext/fs/in_memory_fs.rs b/ext/fs/in_memory_fs.rs index 2692023860..e29b9d50c6 100644 --- a/ext/fs/in_memory_fs.rs +++ b/ext/fs/in_memory_fs.rs @@ -44,7 +44,7 @@ impl InMemoryFs { pub fn setup_text_files(&self, files: Vec<(String, String)>) { for (path, text) in files { let path = PathBuf::from(path); - self.mkdir_sync(path.parent().unwrap(), true, 0).unwrap(); + self.mkdir_sync(path.parent().unwrap(), true, None).unwrap(); self .write_file_sync( &path, @@ -101,7 +101,7 @@ impl FileSystem for InMemoryFs { &self, path: &Path, recursive: bool, - _mode: u32, + _mode: Option, ) -> FsResult<()> { let path = normalize_path(path); @@ -119,7 +119,7 @@ impl FileSystem for InMemoryFs { }, None => { if recursive { - self.mkdir_sync(parent, true, 0)?; + self.mkdir_sync(parent, true, None)?; } else { return Err(FsError::Io(Error::new( ErrorKind::NotFound, @@ -149,7 +149,7 @@ impl FileSystem for InMemoryFs { &self, path: PathBuf, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { self.mkdir_sync(&path, recursive, mode) } diff --git a/ext/fs/interface.rs b/ext/fs/interface.rs index af4beb248b..73333b0fd1 100644 --- a/ext/fs/interface.rs +++ b/ext/fs/interface.rs @@ -121,13 +121,17 @@ pub trait FileSystem: std::fmt::Debug + MaybeSend + MaybeSync { access_check: Option>, ) -> FsResult>; - fn mkdir_sync(&self, path: &Path, recursive: bool, mode: u32) - -> FsResult<()>; + fn mkdir_sync( + &self, + path: &Path, + recursive: bool, + mode: Option, + ) -> FsResult<()>; async fn mkdir_async( &self, path: PathBuf, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()>; fn chmod_sync(&self, path: &Path, mode: u32) -> FsResult<()>; diff --git a/ext/fs/ops.rs b/ext/fs/ops.rs index 150d3b9555..b13d3a7d1a 100644 --- a/ext/fs/ops.rs +++ b/ext/fs/ops.rs @@ -197,7 +197,7 @@ where .check_write(&path, "Deno.mkdirSync()")?; let fs = state.borrow::(); - fs.mkdir_sync(&path, recursive, mode) + fs.mkdir_sync(&path, recursive, Some(mode)) .context_path("mkdir", &path)?; Ok(()) @@ -221,7 +221,7 @@ where (state.borrow::().clone(), path) }; - fs.mkdir_async(path.clone(), recursive, mode) + fs.mkdir_async(path.clone(), recursive, Some(mode)) .await .context_path("mkdir", &path)?; @@ -886,7 +886,7 @@ where const MAX_TRIES: u32 = 10; for _ in 0..MAX_TRIES { let path = tmp_name(&mut rng, &dir, prefix.as_deref(), suffix.as_deref())?; - match fs.mkdir_sync(&path, false, 0o700) { + match fs.mkdir_sync(&path, false, Some(0o700)) { Ok(_) => { // PERMISSIONS: ensure the absolute path is not leaked let path = strip_dir_prefix(&dir, dir_arg.as_deref(), path)?; @@ -928,7 +928,11 @@ where const MAX_TRIES: u32 = 10; for _ in 0..MAX_TRIES { let path = tmp_name(&mut rng, &dir, prefix.as_deref(), suffix.as_deref())?; - match fs.clone().mkdir_async(path.clone(), false, 0o700).await { + match fs + .clone() + .mkdir_async(path.clone(), false, Some(0o700)) + .await + { Ok(_) => { // PERMISSIONS: ensure the absolute path is not leaked let path = strip_dir_prefix(&dir, dir_arg.as_deref(), path)?; diff --git a/ext/fs/std_fs.rs b/ext/fs/std_fs.rs index 443c26819d..41a8569ba8 100644 --- a/ext/fs/std_fs.rs +++ b/ext/fs/std_fs.rs @@ -101,7 +101,7 @@ impl FileSystem for RealFs { &self, path: &Path, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { mkdir(path, recursive, mode) } @@ -109,7 +109,7 @@ impl FileSystem for RealFs { &self, path: PathBuf, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { spawn_blocking(move || mkdir(&path, recursive, mode)).await? } @@ -407,11 +407,11 @@ impl FileSystem for RealFs { } } -fn mkdir(path: &Path, recursive: bool, mode: u32) -> FsResult<()> { +fn mkdir(path: &Path, recursive: bool, mode: Option) -> FsResult<()> { let mut builder = fs::DirBuilder::new(); builder.recursive(recursive); #[cfg(unix)] - { + if let Some(mode) = mode { use std::os::unix::fs::DirBuilderExt; builder.mode(mode); } From 3138478f66823348eb745c7f0c2d34eed378a3f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 28 Sep 2024 21:57:01 +0100 Subject: [PATCH 003/302] Revert "ci: use macos-14-xlarge on 'main' branch (#25908)" (#25913) This reverts commit 0f617be84a8e9edf73803210c24af43f729a97de. Reverting because it was an experiment. --- .github/workflows/ci.generate.ts | 9 +-------- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index e9fcacc823..35020a5f8c 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -14,7 +14,6 @@ const windowsX86Runner = "windows-2022"; const windowsX86XlRunner = "windows-2022-xl"; const macosX86Runner = "macos-13"; const macosArmRunner = "macos-14"; -const macosArmXlRunner = "macos-14-xlarge"; const Runners = { linuxX86: { @@ -43,12 +42,6 @@ const Runners = { arch: "aarch64", runner: macosArmRunner, }, - macosArmXl: { - os: "macos", - arch: "aarch64", - runner: - `\${{ github.repository == 'denoland/deno' && '${macosArmXlRunner}' || '${macosArmRunner}' }}`, - }, windowsX86: { os: "windows", arch: "x86_64", @@ -385,7 +378,7 @@ const ci = { job: "test", profile: "debug", }, { - ...Runners.macosArmXl, + ...Runners.macosArm, job: "test", profile: "release", skip_pr: true, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 035a4244fc..280eb7cb84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: profile: debug - os: macos arch: aarch64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || github.repository == ''denoland/deno'' && ''macos-14-xlarge'' || ''macos-14'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || ''macos-14'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' From 5faf769ac61b627d14710cdf487de7cd4eb3f9d3 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 28 Sep 2024 19:17:48 -0400 Subject: [PATCH 004/302] refactor: extract out sloppy imports resolution from CLI crate (#25920) This is slow progress towards creating a `deno_resolver` crate. Waiting on: * https://github.com/denoland/deno/pull/25918 * https://github.com/denoland/deno/pull/25916 --- Cargo.lock | 12 +- Cargo.toml | 10 +- cli/Cargo.toml | 6 +- cli/factory.rs | 18 +- cli/graph_util.rs | 9 +- cli/lsp/config.rs | 13 +- cli/lsp/diagnostics.rs | 11 +- cli/resolver.rs | 525 ++---------------- cli/tools/lint/rules/mod.rs | 6 +- cli/tools/lint/rules/no_sloppy_imports.rs | 19 +- cli/tools/registry/mod.rs | 7 +- cli/tools/registry/unfurl.rs | 15 +- resolvers/deno/Cargo.toml | 24 + resolvers/deno/README.md | 3 + resolvers/deno/lib.rs | 3 + resolvers/deno/sloppy_imports.rs | 511 +++++++++++++++++ .../node}/Cargo.toml | 0 .../node}/README.md | 0 .../node}/analyze.rs | 0 .../node}/clippy.toml | 0 {ext/node_resolver => resolvers/node}/env.rs | 0 .../node}/errors.rs | 0 {ext/node_resolver => resolvers/node}/lib.rs | 0 {ext/node_resolver => resolvers/node}/npm.rs | 0 .../node}/package_json.rs | 0 {ext/node_resolver => resolvers/node}/path.rs | 0 .../node}/resolution.rs | 0 {ext/node_resolver => resolvers/node}/sync.rs | 0 28 files changed, 665 insertions(+), 527 deletions(-) create mode 100644 resolvers/deno/Cargo.toml create mode 100644 resolvers/deno/README.md create mode 100644 resolvers/deno/lib.rs create mode 100644 resolvers/deno/sloppy_imports.rs rename {ext/node_resolver => resolvers/node}/Cargo.toml (100%) rename {ext/node_resolver => resolvers/node}/README.md (100%) rename {ext/node_resolver => resolvers/node}/analyze.rs (100%) rename {ext/node_resolver => resolvers/node}/clippy.toml (100%) rename {ext/node_resolver => resolvers/node}/env.rs (100%) rename {ext/node_resolver => resolvers/node}/errors.rs (100%) rename {ext/node_resolver => resolvers/node}/lib.rs (100%) rename {ext/node_resolver => resolvers/node}/npm.rs (100%) rename {ext/node_resolver => resolvers/node}/package_json.rs (100%) rename {ext/node_resolver => resolvers/node}/path.rs (100%) rename {ext/node_resolver => resolvers/node}/resolution.rs (100%) rename {ext/node_resolver => resolvers/node}/sync.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 1372ac191a..bdde5d5410 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1150,7 +1150,6 @@ version = "2.0.0-rc.7" dependencies = [ "anstream", "async-trait", - "base32", "base64 0.21.7", "bincode", "bytes", @@ -1175,6 +1174,7 @@ dependencies = [ "deno_npm", "deno_package_json", "deno_path_util", + "deno_resolver", "deno_runtime", "deno_semver", "deno_task_shell", @@ -1949,6 +1949,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "deno_resolver" +version = "0.0.1" +dependencies = [ + "deno_media_type", + "deno_path_util", + "test_server", + "url", +] + [[package]] name = "deno_runtime" version = "0.177.0" diff --git a/Cargo.toml b/Cargo.toml index 4b27f85b82..2f53c8999d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,13 +21,14 @@ members = [ "ext/napi", "ext/net", "ext/node", - "ext/node_resolver", "ext/url", "ext/web", "ext/webgpu", "ext/webidl", "ext/websocket", "ext/webstorage", + "resolvers/deno", + "resolvers/node", "runtime", "runtime/permissions", "tests", @@ -50,6 +51,7 @@ deno_core = { version = "0.311.0" } deno_bench_util = { version = "0.162.0", path = "./bench_util" } deno_lockfile = "=0.23.1" deno_media_type = { version = "0.1.4", features = ["module_specifier"] } +deno_npm = "=0.25.2" deno_path_util = "=0.1.1" deno_permissions = { version = "0.28.0", path = "./runtime/permissions" } deno_runtime = { version = "0.177.0", path = "./runtime" } @@ -86,7 +88,10 @@ deno_webgpu = { version = "0.135.0", path = "./ext/webgpu" } deno_webidl = { version = "0.168.0", path = "./ext/webidl" } deno_websocket = { version = "0.173.0", path = "./ext/websocket" } deno_webstorage = { version = "0.163.0", path = "./ext/webstorage" } -node_resolver = { version = "0.7.0", path = "./ext/node_resolver" } + +# resolvers +deno_resolver = { version = "0.0.1", path = "./resolvers/deno" } +node_resolver = { version = "0.7.0", path = "./resolvers/node" } aes = "=0.8.3" anyhow = "1.0.57" @@ -102,6 +107,7 @@ cbc = { version = "=0.1.2", features = ["alloc"] } # Instead use util::time::utc_now() chrono = { version = "0.4", default-features = false, features = ["std", "serde"] } console_static_text = "=0.8.1" +dashmap = "5.5.3" data-encoding = "2.3.3" data-url = "=0.3.0" deno_cache_dir = "=0.12.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ddcf7119f4..32e0651b2f 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -71,9 +71,10 @@ deno_doc = { version = "0.150.0", features = ["html", "syntect"] } deno_graph = { version = "=0.82.3" } deno_lint = { version = "=0.67.0", features = ["docs"] } deno_lockfile.workspace = true -deno_npm = "=0.25.2" +deno_npm.workspace = true deno_package_json.workspace = true deno_path_util.workspace = true +deno_resolver.workspace = true deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_semver.workspace = true deno_task_shell = "=0.17.0" @@ -85,7 +86,6 @@ node_resolver.workspace = true anstream = "0.6.14" async-trait.workspace = true -base32.workspace = true base64.workspace = true bincode = "=1.3.3" bytes.workspace = true @@ -96,7 +96,7 @@ clap_complete = "=4.5.24" clap_complete_fig = "=4.5.2" color-print = "0.3.5" console_static_text.workspace = true -dashmap = "5.5.3" +dashmap.workspace = true data-encoding.workspace = true dissimilar = "=1.0.4" dotenvy = "0.15.7" diff --git a/cli/factory.rs b/cli/factory.rs index 8aea635d2d..5e525ee32b 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -41,8 +41,9 @@ use crate::resolver::CjsResolutionStore; use crate::resolver::CliGraphResolver; use crate::resolver::CliGraphResolverOptions; use crate::resolver::CliNodeResolver; +use crate::resolver::CliSloppyImportsResolver; use crate::resolver::NpmModuleLoader; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::standalone::DenoCompileBinaryWriter; use crate::tools::check::TypeChecker; use crate::tools::coverage::CoverageCollector; @@ -186,7 +187,7 @@ struct CliFactoryServices { npm_resolver: Deferred>, permission_desc_parser: Deferred>, root_permissions_container: Deferred, - sloppy_imports_resolver: Deferred>>, + sloppy_imports_resolver: Deferred>>, text_only_progress_bar: Deferred, type_checker: Deferred>, cjs_resolutions: Deferred>, @@ -404,17 +405,16 @@ impl CliFactory { pub fn sloppy_imports_resolver( &self, - ) -> Result>, AnyError> { + ) -> Result>, AnyError> { self .services .sloppy_imports_resolver .get_or_try_init(|| { - Ok( - self - .cli_options()? - .unstable_sloppy_imports() - .then(|| Arc::new(SloppyImportsResolver::new(self.fs().clone()))), - ) + Ok(self.cli_options()?.unstable_sloppy_imports().then(|| { + Arc::new(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new( + self.fs().clone(), + ))) + })) }) .map(|maybe| maybe.as_ref()) } diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 7d03d3c0b2..f7194ac11b 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -14,7 +14,8 @@ use crate::errors::get_error_class_name; use crate::file_fetcher::FileFetcher; use crate::npm::CliNpmResolver; use crate::resolver::CliGraphResolver; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::tools::check; use crate::tools::check::TypeChecker; use crate::util::file_watcher::WatcherCommunicator; @@ -31,7 +32,6 @@ use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; use deno_graph::source::Loader; -use deno_graph::source::ResolutionMode; use deno_graph::source::ResolveError; use deno_graph::GraphKind; use deno_graph::ModuleError; @@ -40,6 +40,7 @@ use deno_graph::ModuleGraphError; use deno_graph::ResolutionError; use deno_graph::SpecifierError; use deno_path_util::url_to_file_path; +use deno_resolver::sloppy_imports::SloppyImportsResolutionMode; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node; use deno_runtime::deno_permissions::PermissionsContainer; @@ -765,8 +766,8 @@ fn enhanced_sloppy_imports_error_message( match error { ModuleError::LoadingErr(specifier, _, ModuleLoadError::Loader(_)) // ex. "Is a directory" error | ModuleError::Missing(specifier, _) => { - let additional_message = SloppyImportsResolver::new(fs.clone()) - .resolve(specifier, ResolutionMode::Execution)? + let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(fs.clone())) + .resolve(specifier, SloppyImportsResolutionMode::Execution)? .as_suggestion_message(); Some(format!( "{} {} or run with --unstable-sloppy-imports", diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index dcb6120a45..c54de3a235 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -59,7 +59,8 @@ use crate::args::LintOptions; use crate::cache::FastInsecureHasher; use crate::file_fetcher::FileFetcher; use crate::lsp::logging::lsp_warn; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::tools::lint::CliLinter; use crate::tools::lint::CliLinterOptions; use crate::tools::lint::LintRuleProvider; @@ -1181,7 +1182,7 @@ pub struct ConfigData { pub lockfile: Option>, pub npmrc: Option>, pub resolver: Arc, - pub sloppy_imports_resolver: Option>, + pub sloppy_imports_resolver: Option>, pub import_map_from_settings: Option, watched_files: HashMap, } @@ -1584,9 +1585,11 @@ impl ConfigData { .is_ok() || member_dir.workspace.has_unstable("sloppy-imports"); let sloppy_imports_resolver = unstable_sloppy_imports.then(|| { - Arc::new(SloppyImportsResolver::new_without_stat_cache(Arc::new( - deno_runtime::deno_fs::RealFs, - ))) + Arc::new(CliSloppyImportsResolver::new( + SloppyImportsCachedFs::new_without_stat_cache(Arc::new( + deno_runtime::deno_fs::RealFs, + )), + )) }); let resolver = Arc::new(resolver); let lint_rule_provider = LintRuleProvider::new( diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs index 1aebaf56fc..e57681f3ff 100644 --- a/cli/lsp/diagnostics.rs +++ b/cli/lsp/diagnostics.rs @@ -19,8 +19,8 @@ use super::urls::LspUrlMap; use crate::graph_util; use crate::graph_util::enhanced_resolution_error_message; use crate::lsp::lsp_custom::DiagnosticBatchNotificationParams; -use crate::resolver::SloppyImportsResolution; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::tools::lint::CliLinter; use crate::tools::lint::CliLinterOptions; use crate::tools::lint::LintRuleProvider; @@ -40,11 +40,12 @@ use deno_core::unsync::spawn_blocking; use deno_core::unsync::JoinHandle; use deno_core::url::Url; use deno_core::ModuleSpecifier; -use deno_graph::source::ResolutionMode; use deno_graph::source::ResolveError; use deno_graph::Resolution; use deno_graph::ResolutionError; use deno_graph::SpecifierError; +use deno_resolver::sloppy_imports::SloppyImportsResolution; +use deno_resolver::sloppy_imports::SloppyImportsResolutionMode; use deno_runtime::deno_fs; use deno_runtime::deno_node; use deno_runtime::tokio_util::create_basic_runtime; @@ -1263,7 +1264,9 @@ impl DenoDiagnostic { Self::NotInstalledJsr(pkg_req, specifier) => (lsp::DiagnosticSeverity::ERROR, format!("JSR package \"{pkg_req}\" is not installed or doesn't exist."), Some(json!({ "specifier": specifier }))), Self::NotInstalledNpm(pkg_req, specifier) => (lsp::DiagnosticSeverity::ERROR, format!("NPM package \"{pkg_req}\" is not installed or doesn't exist."), Some(json!({ "specifier": specifier }))), Self::NoLocal(specifier) => { - let maybe_sloppy_resolution = SloppyImportsResolver::new(Arc::new(deno_fs::RealFs)).resolve(specifier, ResolutionMode::Execution); + let maybe_sloppy_resolution = CliSloppyImportsResolver::new( + SloppyImportsCachedFs::new(Arc::new(deno_fs::RealFs)) + ).resolve(specifier, SloppyImportsResolutionMode::Execution); let data = maybe_sloppy_resolution.as_ref().map(|res| { json!({ "specifier": specifier, diff --git a/cli/resolver.rs b/cli/resolver.rs index cf4cd8b74a..d6e14c39d0 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -22,7 +22,8 @@ use deno_graph::NpmLoadError; use deno_graph::NpmResolvePkgReqsResult; use deno_npm::resolution::NpmResolutionError; use deno_package_json::PackageJsonDepValue; -use deno_path_util::url_to_file_path; +use deno_resolver::sloppy_imports::SloppyImportsResolutionMode; +use deno_resolver::sloppy_imports::SloppyImportsResolver; use deno_runtime::colors; use deno_runtime::deno_fs; use deno_runtime::deno_fs::FileSystem; @@ -421,13 +422,16 @@ impl CjsResolutionStore { } } +pub type CliSloppyImportsResolver = + SloppyImportsResolver; + /// A resolver that takes care of resolution, taking into account loaded /// import map, JSX settings. #[derive(Debug)] pub struct CliGraphResolver { node_resolver: Option>, npm_resolver: Option>, - sloppy_imports_resolver: Option>, + sloppy_imports_resolver: Option>, workspace_resolver: Arc, maybe_default_jsx_import_source: Option, maybe_default_jsx_import_source_types: Option, @@ -441,7 +445,7 @@ pub struct CliGraphResolver { pub struct CliGraphResolverOptions<'a> { pub node_resolver: Option>, pub npm_resolver: Option>, - pub sloppy_imports_resolver: Option>, + pub sloppy_imports_resolver: Option>, pub workspace_resolver: Arc, pub bare_node_builtins_enabled: bool, pub maybe_jsx_import_source_config: Option, @@ -565,7 +569,15 @@ impl Resolver for CliGraphResolver { if let Some(sloppy_imports_resolver) = &self.sloppy_imports_resolver { Ok( sloppy_imports_resolver - .resolve(&specifier, mode) + .resolve( + &specifier, + match mode { + ResolutionMode::Execution => { + SloppyImportsResolutionMode::Execution + } + ResolutionMode::Types => SloppyImportsResolutionMode::Types, + }, + ) .map(|s| s.into_specifier()) .unwrap_or(specifier), ) @@ -847,96 +859,18 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SloppyImportsFsEntry { - File, - Dir, -} - -impl SloppyImportsFsEntry { - pub fn from_fs_stat( - stat: &deno_runtime::deno_io::fs::FsStat, - ) -> Option { - if stat.is_file { - Some(SloppyImportsFsEntry::File) - } else if stat.is_directory { - Some(SloppyImportsFsEntry::Dir) - } else { - None - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SloppyImportsResolution { - /// Ex. `./file.js` to `./file.ts` - JsToTs(ModuleSpecifier), - /// Ex. `./file` to `./file.ts` - NoExtension(ModuleSpecifier), - /// Ex. `./dir` to `./dir/index.ts` - Directory(ModuleSpecifier), -} - -impl SloppyImportsResolution { - pub fn as_specifier(&self) -> &ModuleSpecifier { - match self { - Self::JsToTs(specifier) => specifier, - Self::NoExtension(specifier) => specifier, - Self::Directory(specifier) => specifier, - } - } - - pub fn into_specifier(self) -> ModuleSpecifier { - match self { - Self::JsToTs(specifier) => specifier, - Self::NoExtension(specifier) => specifier, - Self::Directory(specifier) => specifier, - } - } - - pub fn as_suggestion_message(&self) -> String { - format!("Maybe {}", self.as_base_message()) - } - - pub fn as_quick_fix_message(&self) -> String { - let message = self.as_base_message(); - let mut chars = message.chars(); - format!( - "{}{}.", - chars.next().unwrap().to_uppercase(), - chars.as_str() - ) - } - - fn as_base_message(&self) -> String { - match self { - SloppyImportsResolution::JsToTs(specifier) => { - let media_type = MediaType::from_specifier(specifier); - format!("change the extension to '{}'", media_type.as_ts_extension()) - } - SloppyImportsResolution::NoExtension(specifier) => { - let media_type = MediaType::from_specifier(specifier); - format!("add a '{}' extension", media_type.as_ts_extension()) - } - SloppyImportsResolution::Directory(specifier) => { - let file_name = specifier - .path() - .rsplit_once('/') - .map(|(_, file_name)| file_name) - .unwrap_or(specifier.path()); - format!("specify path to '{}' file in directory instead", file_name) - } - } - } -} - #[derive(Debug)] -pub struct SloppyImportsResolver { - fs: Arc, - cache: Option>>, +pub struct SloppyImportsCachedFs { + fs: Arc, + cache: Option< + DashMap< + PathBuf, + Option, + >, + >, } -impl SloppyImportsResolver { +impl SloppyImportsCachedFs { pub fn new(fs: Arc) -> Self { Self { fs, @@ -947,409 +881,34 @@ impl SloppyImportsResolver { pub fn new_without_stat_cache(fs: Arc) -> Self { Self { fs, cache: None } } +} - pub fn resolve( +impl deno_resolver::sloppy_imports::SloppyImportResolverFs + for SloppyImportsCachedFs +{ + fn stat_sync( &self, - specifier: &ModuleSpecifier, - mode: ResolutionMode, - ) -> Option { - fn path_without_ext( - path: &Path, - media_type: MediaType, - ) -> Option> { - let old_path_str = path.to_string_lossy(); - match media_type { - MediaType::Unknown => Some(old_path_str), - _ => old_path_str - .strip_suffix(media_type.as_ts_extension()) - .map(|s| Cow::Owned(s.to_string())), - } - } - - fn media_types_to_paths( - path_no_ext: &str, - original_media_type: MediaType, - probe_media_type_types: Vec, - reason: SloppyImportsResolutionReason, - ) -> Vec<(PathBuf, SloppyImportsResolutionReason)> { - probe_media_type_types - .into_iter() - .filter(|media_type| *media_type != original_media_type) - .map(|media_type| { - ( - PathBuf::from(format!( - "{}{}", - path_no_ext, - media_type.as_ts_extension() - )), - reason, - ) - }) - .collect::>() - } - - if specifier.scheme() != "file" { - return None; - } - - let path = url_to_file_path(specifier).ok()?; - - #[derive(Clone, Copy)] - enum SloppyImportsResolutionReason { - JsToTs, - NoExtension, - Directory, - } - - let probe_paths: Vec<(PathBuf, SloppyImportsResolutionReason)> = - match self.stat_sync(&path) { - Some(SloppyImportsFsEntry::File) => { - if mode.is_types() { - let media_type = MediaType::from_specifier(specifier); - // attempt to resolve the .d.ts file before the .js file - let probe_media_type_types = match media_type { - MediaType::JavaScript => { - vec![(MediaType::Dts), MediaType::JavaScript] - } - MediaType::Mjs => { - vec![MediaType::Dmts, MediaType::Dts, MediaType::Mjs] - } - MediaType::Cjs => { - vec![MediaType::Dcts, MediaType::Dts, MediaType::Cjs] - } - _ => return None, - }; - let path_no_ext = path_without_ext(&path, media_type)?; - media_types_to_paths( - &path_no_ext, - media_type, - probe_media_type_types, - SloppyImportsResolutionReason::JsToTs, - ) - } else { - return None; - } - } - entry @ None | entry @ Some(SloppyImportsFsEntry::Dir) => { - let media_type = MediaType::from_specifier(specifier); - let probe_media_type_types = match media_type { - MediaType::JavaScript => ( - if mode.is_types() { - vec![MediaType::TypeScript, MediaType::Tsx, MediaType::Dts] - } else { - vec![MediaType::TypeScript, MediaType::Tsx] - }, - SloppyImportsResolutionReason::JsToTs, - ), - MediaType::Jsx => { - (vec![MediaType::Tsx], SloppyImportsResolutionReason::JsToTs) - } - MediaType::Mjs => ( - if mode.is_types() { - vec![MediaType::Mts, MediaType::Dmts, MediaType::Dts] - } else { - vec![MediaType::Mts] - }, - SloppyImportsResolutionReason::JsToTs, - ), - MediaType::Cjs => ( - if mode.is_types() { - vec![MediaType::Cts, MediaType::Dcts, MediaType::Dts] - } else { - vec![MediaType::Cts] - }, - SloppyImportsResolutionReason::JsToTs, - ), - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Dts - | MediaType::Dmts - | MediaType::Dcts - | MediaType::Tsx - | MediaType::Json - | MediaType::Wasm - | MediaType::TsBuildInfo - | MediaType::SourceMap => { - return None; - } - MediaType::Unknown => ( - if mode.is_types() { - vec![ - MediaType::TypeScript, - MediaType::Tsx, - MediaType::Mts, - MediaType::Dts, - MediaType::Dmts, - MediaType::Dcts, - MediaType::JavaScript, - MediaType::Jsx, - MediaType::Mjs, - ] - } else { - vec![ - MediaType::TypeScript, - MediaType::JavaScript, - MediaType::Tsx, - MediaType::Jsx, - MediaType::Mts, - MediaType::Mjs, - ] - }, - SloppyImportsResolutionReason::NoExtension, - ), - }; - let mut probe_paths = match path_without_ext(&path, media_type) { - Some(path_no_ext) => media_types_to_paths( - &path_no_ext, - media_type, - probe_media_type_types.0, - probe_media_type_types.1, - ), - None => vec![], - }; - - if matches!(entry, Some(SloppyImportsFsEntry::Dir)) { - // try to resolve at the index file - if mode.is_types() { - probe_paths.push(( - path.join("index.ts"), - SloppyImportsResolutionReason::Directory, - )); - - probe_paths.push(( - path.join("index.mts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.d.ts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.d.mts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.js"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.mjs"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.tsx"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.jsx"), - SloppyImportsResolutionReason::Directory, - )); - } else { - probe_paths.push(( - path.join("index.ts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.mts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.tsx"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.js"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.mjs"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.jsx"), - SloppyImportsResolutionReason::Directory, - )); - } - } - if probe_paths.is_empty() { - return None; - } - probe_paths - } - }; - - for (probe_path, reason) in probe_paths { - if self.stat_sync(&probe_path) == Some(SloppyImportsFsEntry::File) { - if let Ok(specifier) = ModuleSpecifier::from_file_path(probe_path) { - match reason { - SloppyImportsResolutionReason::JsToTs => { - return Some(SloppyImportsResolution::JsToTs(specifier)); - } - SloppyImportsResolutionReason::NoExtension => { - return Some(SloppyImportsResolution::NoExtension(specifier)); - } - SloppyImportsResolutionReason::Directory => { - return Some(SloppyImportsResolution::Directory(specifier)); - } - } - } - } - } - - None - } - - fn stat_sync(&self, path: &Path) -> Option { + path: &Path, + ) -> Option { if let Some(cache) = &self.cache { if let Some(entry) = cache.get(path) { return *entry; } } - let entry = self - .fs - .stat_sync(path) - .ok() - .and_then(|stat| SloppyImportsFsEntry::from_fs_stat(&stat)); + let entry = self.fs.stat_sync(path).ok().and_then(|stat| { + if stat.is_file { + Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::File) + } else if stat.is_directory { + Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::Dir) + } else { + None + } + }); + if let Some(cache) = &self.cache { cache.insert(path.to_owned(), entry); } entry } } - -#[cfg(test)] -mod test { - use test_util::TestContext; - - use super::*; - - #[test] - fn test_unstable_sloppy_imports() { - fn resolve(specifier: &ModuleSpecifier) -> Option { - resolve_with_mode(specifier, ResolutionMode::Execution) - } - - fn resolve_types( - specifier: &ModuleSpecifier, - ) -> Option { - resolve_with_mode(specifier, ResolutionMode::Types) - } - - fn resolve_with_mode( - specifier: &ModuleSpecifier, - mode: ResolutionMode, - ) -> Option { - SloppyImportsResolver::new(Arc::new(deno_fs::RealFs)) - .resolve(specifier, mode) - } - - let context = TestContext::default(); - let temp_dir = context.temp_dir().path(); - - // scenarios like resolving ./example.js to ./example.ts - for (ext_from, ext_to) in [("js", "ts"), ("js", "tsx"), ("mjs", "mts")] { - let ts_file = temp_dir.join(format!("file.{}", ext_to)); - ts_file.write(""); - assert_eq!(resolve(&ts_file.url_file()), None); - assert_eq!( - resolve( - &temp_dir - .url_dir() - .join(&format!("file.{}", ext_from)) - .unwrap() - ), - Some(SloppyImportsResolution::JsToTs(ts_file.url_file())), - ); - ts_file.remove_file(); - } - - // no extension scenarios - for ext in ["js", "ts", "js", "tsx", "jsx", "mjs", "mts"] { - let file = temp_dir.join(format!("file.{}", ext)); - file.write(""); - assert_eq!( - resolve( - &temp_dir - .url_dir() - .join("file") // no ext - .unwrap() - ), - Some(SloppyImportsResolution::NoExtension(file.url_file())) - ); - file.remove_file(); - } - - // .ts and .js exists, .js specified (goes to specified) - { - let ts_file = temp_dir.join("file.ts"); - ts_file.write(""); - let js_file = temp_dir.join("file.js"); - js_file.write(""); - assert_eq!(resolve(&js_file.url_file()), None); - } - - // only js exists, .js specified - { - let js_only_file = temp_dir.join("js_only.js"); - js_only_file.write(""); - assert_eq!(resolve(&js_only_file.url_file()), None); - assert_eq!(resolve_types(&js_only_file.url_file()), None); - } - - // resolving a directory to an index file - { - let routes_dir = temp_dir.join("routes"); - routes_dir.create_dir_all(); - let index_file = routes_dir.join("index.ts"); - index_file.write(""); - assert_eq!( - resolve(&routes_dir.url_file()), - Some(SloppyImportsResolution::Directory(index_file.url_file())), - ); - } - - // both a directory and a file with specifier is present - { - let api_dir = temp_dir.join("api"); - api_dir.create_dir_all(); - let bar_file = api_dir.join("bar.ts"); - bar_file.write(""); - let api_file = temp_dir.join("api.ts"); - api_file.write(""); - assert_eq!( - resolve(&api_dir.url_file()), - Some(SloppyImportsResolution::NoExtension(api_file.url_file())), - ); - } - } - - #[test] - fn test_sloppy_import_resolution_suggestion_message() { - // directory - assert_eq!( - SloppyImportsResolution::Directory( - ModuleSpecifier::parse("file:///dir/index.js").unwrap() - ) - .as_suggestion_message(), - "Maybe specify path to 'index.js' file in directory instead" - ); - // no ext - assert_eq!( - SloppyImportsResolution::NoExtension( - ModuleSpecifier::parse("file:///dir/index.mjs").unwrap() - ) - .as_suggestion_message(), - "Maybe add a '.mjs' extension" - ); - // js to ts - assert_eq!( - SloppyImportsResolution::JsToTs( - ModuleSpecifier::parse("file:///dir/index.mts").unwrap() - ) - .as_suggestion_message(), - "Maybe change the extension to '.mts'" - ); - } -} diff --git a/cli/tools/lint/rules/mod.rs b/cli/tools/lint/rules/mod.rs index 2669ffda15..dd723ad159 100644 --- a/cli/tools/lint/rules/mod.rs +++ b/cli/tools/lint/rules/mod.rs @@ -14,7 +14,7 @@ use deno_graph::ModuleGraph; use deno_lint::diagnostic::LintDiagnostic; use deno_lint::rules::LintRule; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; mod no_sloppy_imports; mod no_slow_types; @@ -144,13 +144,13 @@ impl ConfiguredRules { } pub struct LintRuleProvider { - sloppy_imports_resolver: Option>, + sloppy_imports_resolver: Option>, workspace_resolver: Option>, } impl LintRuleProvider { pub fn new( - sloppy_imports_resolver: Option>, + sloppy_imports_resolver: Option>, workspace_resolver: Option>, ) -> Self { Self { diff --git a/cli/tools/lint/rules/no_sloppy_imports.rs b/cli/tools/lint/rules/no_sloppy_imports.rs index 4180be5be1..2f60875885 100644 --- a/cli/tools/lint/rules/no_sloppy_imports.rs +++ b/cli/tools/lint/rules/no_sloppy_imports.rs @@ -16,24 +16,25 @@ use deno_lint::diagnostic::LintDiagnosticRange; use deno_lint::diagnostic::LintFix; use deno_lint::diagnostic::LintFixChange; use deno_lint::rules::LintRule; +use deno_resolver::sloppy_imports::SloppyImportsResolution; +use deno_resolver::sloppy_imports::SloppyImportsResolutionMode; use text_lines::LineAndColumnIndex; use crate::graph_util::CliJsrUrlProvider; -use crate::resolver::SloppyImportsResolution; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; use super::ExtendedLintRule; #[derive(Debug)] pub struct NoSloppyImportsRule { - sloppy_imports_resolver: Option>, + sloppy_imports_resolver: Option>, // None for making printing out the lint rules easy workspace_resolver: Option>, } impl NoSloppyImportsRule { pub fn new( - sloppy_imports_resolver: Option>, + sloppy_imports_resolver: Option>, workspace_resolver: Option>, ) -> Self { NoSloppyImportsRule { @@ -172,7 +173,7 @@ impl LintRule for NoSloppyImportsRule { #[derive(Debug)] struct SloppyImportCaptureResolver<'a> { workspace_resolver: &'a WorkspaceResolver, - sloppy_imports_resolver: &'a SloppyImportsResolver, + sloppy_imports_resolver: &'a CliSloppyImportsResolver, captures: RefCell>, } @@ -194,7 +195,13 @@ impl<'a> deno_graph::source::Resolver for SloppyImportCaptureResolver<'a> { } | deno_config::workspace::MappedResolution::ImportMap { specifier, .. - } => match self.sloppy_imports_resolver.resolve(&specifier, mode) { + } => match self.sloppy_imports_resolver.resolve( + &specifier, + match mode { + ResolutionMode::Execution => SloppyImportsResolutionMode::Execution, + ResolutionMode::Types => SloppyImportsResolutionMode::Types, + }, + ) { Some(res) => { self .captures diff --git a/cli/tools/registry/mod.rs b/cli/tools/registry/mod.rs index 941514b045..4098d62e37 100644 --- a/cli/tools/registry/mod.rs +++ b/cli/tools/registry/mod.rs @@ -43,7 +43,8 @@ use crate::cache::ParsedSourceCache; use crate::factory::CliFactory; use crate::graph_util::ModuleGraphCreator; use crate::http_util::HttpClient; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::tools::check::CheckOptions; use crate::tools::lint::collect_no_slow_type_diagnostics; use crate::tools::registry::diagnostics::PublishDiagnostic; @@ -108,7 +109,9 @@ pub async fn publish( } let specifier_unfurler = Arc::new(SpecifierUnfurler::new( if cli_options.unstable_sloppy_imports() { - Some(SloppyImportsResolver::new(cli_factory.fs().clone())) + Some(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new( + cli_factory.fs().clone(), + ))) } else { None }, diff --git a/cli/tools/registry/unfurl.rs b/cli/tools/registry/unfurl.rs index 0f5b9fdd32..5ec726a640 100644 --- a/cli/tools/registry/unfurl.rs +++ b/cli/tools/registry/unfurl.rs @@ -12,9 +12,10 @@ use deno_graph::DynamicTemplatePart; use deno_graph::ParserModuleAnalyzer; use deno_graph::TypeScriptReference; use deno_package_json::PackageJsonDepValue; +use deno_resolver::sloppy_imports::SloppyImportsResolutionMode; use deno_runtime::deno_node::is_builtin_node_module; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; #[derive(Debug, Clone)] pub enum SpecifierUnfurlerDiagnostic { @@ -42,14 +43,14 @@ impl SpecifierUnfurlerDiagnostic { } pub struct SpecifierUnfurler { - sloppy_imports_resolver: Option, + sloppy_imports_resolver: Option, workspace_resolver: WorkspaceResolver, bare_node_builtins: bool, } impl SpecifierUnfurler { pub fn new( - sloppy_imports_resolver: Option, + sloppy_imports_resolver: Option, workspace_resolver: WorkspaceResolver, bare_node_builtins: bool, ) -> Self { @@ -179,7 +180,7 @@ impl SpecifierUnfurler { let resolved = if let Some(sloppy_imports_resolver) = &self.sloppy_imports_resolver { sloppy_imports_resolver - .resolve(&resolved, deno_graph::source::ResolutionMode::Execution) + .resolve(&resolved, SloppyImportsResolutionMode::Execution) .map(|res| res.into_specifier()) .unwrap_or(resolved) } else { @@ -388,6 +389,8 @@ fn to_range( mod tests { use std::sync::Arc; + use crate::resolver::SloppyImportsCachedFs; + use super::*; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; @@ -455,7 +458,9 @@ mod tests { ); let fs = Arc::new(RealFs); let unfurler = SpecifierUnfurler::new( - Some(SloppyImportsResolver::new(fs)), + Some(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new( + fs, + ))), workspace_resolver, true, ); diff --git a/resolvers/deno/Cargo.toml b/resolvers/deno/Cargo.toml new file mode 100644 index 0000000000..23c43810a0 --- /dev/null +++ b/resolvers/deno/Cargo.toml @@ -0,0 +1,24 @@ +# Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +[package] +name = "deno_resolver" +version = "0.0.1" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "Deno resolution algorithm" + +[lib] +path = "lib.rs" + +[features] + +[dependencies] +deno_media_type.workspace = true +deno_path_util.workspace = true +url.workspace = true + +[dev-dependencies] +test_util.workspace = true diff --git a/resolvers/deno/README.md b/resolvers/deno/README.md new file mode 100644 index 0000000000..f51619a314 --- /dev/null +++ b/resolvers/deno/README.md @@ -0,0 +1,3 @@ +# deno_resolver + +Deno resolution algorithm. diff --git a/resolvers/deno/lib.rs b/resolvers/deno/lib.rs new file mode 100644 index 0000000000..7d7796d776 --- /dev/null +++ b/resolvers/deno/lib.rs @@ -0,0 +1,3 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +pub mod sloppy_imports; diff --git a/resolvers/deno/sloppy_imports.rs b/resolvers/deno/sloppy_imports.rs new file mode 100644 index 0000000000..e4d0898e5d --- /dev/null +++ b/resolvers/deno/sloppy_imports.rs @@ -0,0 +1,511 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; + +use deno_media_type::MediaType; +use deno_path_util::url_to_file_path; +use url::Url; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SloppyImportsFsEntry { + File, + Dir, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SloppyImportsResolution { + /// Ex. `./file.js` to `./file.ts` + JsToTs(Url), + /// Ex. `./file` to `./file.ts` + NoExtension(Url), + /// Ex. `./dir` to `./dir/index.ts` + Directory(Url), +} + +impl SloppyImportsResolution { + pub fn as_specifier(&self) -> &Url { + match self { + Self::JsToTs(specifier) => specifier, + Self::NoExtension(specifier) => specifier, + Self::Directory(specifier) => specifier, + } + } + + pub fn into_specifier(self) -> Url { + match self { + Self::JsToTs(specifier) => specifier, + Self::NoExtension(specifier) => specifier, + Self::Directory(specifier) => specifier, + } + } + + pub fn as_suggestion_message(&self) -> String { + format!("Maybe {}", self.as_base_message()) + } + + pub fn as_quick_fix_message(&self) -> String { + let message = self.as_base_message(); + let mut chars = message.chars(); + format!( + "{}{}.", + chars.next().unwrap().to_uppercase(), + chars.as_str() + ) + } + + fn as_base_message(&self) -> String { + match self { + SloppyImportsResolution::JsToTs(specifier) => { + let media_type = MediaType::from_specifier(specifier); + format!("change the extension to '{}'", media_type.as_ts_extension()) + } + SloppyImportsResolution::NoExtension(specifier) => { + let media_type = MediaType::from_specifier(specifier); + format!("add a '{}' extension", media_type.as_ts_extension()) + } + SloppyImportsResolution::Directory(specifier) => { + let file_name = specifier + .path() + .rsplit_once('/') + .map(|(_, file_name)| file_name) + .unwrap_or(specifier.path()); + format!("specify path to '{}' file in directory instead", file_name) + } + } + } +} + +/// The kind of resolution currently being done. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SloppyImportsResolutionMode { + /// Resolving for code that will be executed. + Execution, + /// Resolving for code that will be used for type information. + Types, +} + +impl SloppyImportsResolutionMode { + pub fn is_types(&self) -> bool { + *self == SloppyImportsResolutionMode::Types + } +} + +pub trait SloppyImportResolverFs { + fn stat_sync(&self, path: &Path) -> Option; + + fn is_file(&self, path: &Path) -> bool { + self.stat_sync(path) == Some(SloppyImportsFsEntry::File) + } +} + +#[derive(Debug)] +pub struct SloppyImportsResolver { + fs: Fs, +} + +impl SloppyImportsResolver { + pub fn new(fs: Fs) -> Self { + Self { fs } + } + + pub fn resolve( + &self, + specifier: &Url, + mode: SloppyImportsResolutionMode, + ) -> Option { + fn path_without_ext( + path: &Path, + media_type: MediaType, + ) -> Option> { + let old_path_str = path.to_string_lossy(); + match media_type { + MediaType::Unknown => Some(old_path_str), + _ => old_path_str + .strip_suffix(media_type.as_ts_extension()) + .map(|s| Cow::Owned(s.to_string())), + } + } + + fn media_types_to_paths( + path_no_ext: &str, + original_media_type: MediaType, + probe_media_type_types: Vec, + reason: SloppyImportsResolutionReason, + ) -> Vec<(PathBuf, SloppyImportsResolutionReason)> { + probe_media_type_types + .into_iter() + .filter(|media_type| *media_type != original_media_type) + .map(|media_type| { + ( + PathBuf::from(format!( + "{}{}", + path_no_ext, + media_type.as_ts_extension() + )), + reason, + ) + }) + .collect::>() + } + + if specifier.scheme() != "file" { + return None; + } + + let path = url_to_file_path(specifier).ok()?; + + #[derive(Clone, Copy)] + enum SloppyImportsResolutionReason { + JsToTs, + NoExtension, + Directory, + } + + let probe_paths: Vec<(PathBuf, SloppyImportsResolutionReason)> = + match self.fs.stat_sync(&path) { + Some(SloppyImportsFsEntry::File) => { + if mode.is_types() { + let media_type = MediaType::from_specifier(specifier); + // attempt to resolve the .d.ts file before the .js file + let probe_media_type_types = match media_type { + MediaType::JavaScript => { + vec![(MediaType::Dts), MediaType::JavaScript] + } + MediaType::Mjs => { + vec![MediaType::Dmts, MediaType::Dts, MediaType::Mjs] + } + MediaType::Cjs => { + vec![MediaType::Dcts, MediaType::Dts, MediaType::Cjs] + } + _ => return None, + }; + let path_no_ext = path_without_ext(&path, media_type)?; + media_types_to_paths( + &path_no_ext, + media_type, + probe_media_type_types, + SloppyImportsResolutionReason::JsToTs, + ) + } else { + return None; + } + } + entry @ None | entry @ Some(SloppyImportsFsEntry::Dir) => { + let media_type = MediaType::from_specifier(specifier); + let probe_media_type_types = match media_type { + MediaType::JavaScript => ( + if mode.is_types() { + vec![MediaType::TypeScript, MediaType::Tsx, MediaType::Dts] + } else { + vec![MediaType::TypeScript, MediaType::Tsx] + }, + SloppyImportsResolutionReason::JsToTs, + ), + MediaType::Jsx => { + (vec![MediaType::Tsx], SloppyImportsResolutionReason::JsToTs) + } + MediaType::Mjs => ( + if mode.is_types() { + vec![MediaType::Mts, MediaType::Dmts, MediaType::Dts] + } else { + vec![MediaType::Mts] + }, + SloppyImportsResolutionReason::JsToTs, + ), + MediaType::Cjs => ( + if mode.is_types() { + vec![MediaType::Cts, MediaType::Dcts, MediaType::Dts] + } else { + vec![MediaType::Cts] + }, + SloppyImportsResolutionReason::JsToTs, + ), + MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Dts + | MediaType::Dmts + | MediaType::Dcts + | MediaType::Tsx + | MediaType::Json + | MediaType::Wasm + | MediaType::TsBuildInfo + | MediaType::SourceMap => { + return None; + } + MediaType::Unknown => ( + if mode.is_types() { + vec![ + MediaType::TypeScript, + MediaType::Tsx, + MediaType::Mts, + MediaType::Dts, + MediaType::Dmts, + MediaType::Dcts, + MediaType::JavaScript, + MediaType::Jsx, + MediaType::Mjs, + ] + } else { + vec![ + MediaType::TypeScript, + MediaType::JavaScript, + MediaType::Tsx, + MediaType::Jsx, + MediaType::Mts, + MediaType::Mjs, + ] + }, + SloppyImportsResolutionReason::NoExtension, + ), + }; + let mut probe_paths = match path_without_ext(&path, media_type) { + Some(path_no_ext) => media_types_to_paths( + &path_no_ext, + media_type, + probe_media_type_types.0, + probe_media_type_types.1, + ), + None => vec![], + }; + + if matches!(entry, Some(SloppyImportsFsEntry::Dir)) { + // try to resolve at the index file + if mode.is_types() { + probe_paths.push(( + path.join("index.ts"), + SloppyImportsResolutionReason::Directory, + )); + + probe_paths.push(( + path.join("index.mts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.d.ts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.d.mts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.js"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.mjs"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.tsx"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.jsx"), + SloppyImportsResolutionReason::Directory, + )); + } else { + probe_paths.push(( + path.join("index.ts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.mts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.tsx"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.js"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.mjs"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.jsx"), + SloppyImportsResolutionReason::Directory, + )); + } + } + if probe_paths.is_empty() { + return None; + } + probe_paths + } + }; + + for (probe_path, reason) in probe_paths { + if self.fs.is_file(&probe_path) { + if let Ok(specifier) = Url::from_file_path(probe_path) { + match reason { + SloppyImportsResolutionReason::JsToTs => { + return Some(SloppyImportsResolution::JsToTs(specifier)); + } + SloppyImportsResolutionReason::NoExtension => { + return Some(SloppyImportsResolution::NoExtension(specifier)); + } + SloppyImportsResolutionReason::Directory => { + return Some(SloppyImportsResolution::Directory(specifier)); + } + } + } + } + } + + None + } +} + +#[cfg(test)] +mod test { + use test_util::TestContext; + + use super::*; + + #[test] + fn test_unstable_sloppy_imports() { + fn resolve(specifier: &Url) -> Option { + resolve_with_mode(specifier, SloppyImportsResolutionMode::Execution) + } + + fn resolve_types(specifier: &Url) -> Option { + resolve_with_mode(specifier, SloppyImportsResolutionMode::Types) + } + + fn resolve_with_mode( + specifier: &Url, + mode: SloppyImportsResolutionMode, + ) -> Option { + struct RealSloppyImportsResolverFs; + impl SloppyImportResolverFs for RealSloppyImportsResolverFs { + fn stat_sync(&self, path: &Path) -> Option { + let stat = std::fs::metadata(path).ok()?; + if stat.is_dir() { + Some(SloppyImportsFsEntry::Dir) + } else if stat.is_file() { + Some(SloppyImportsFsEntry::File) + } else { + None + } + } + } + + SloppyImportsResolver::new(RealSloppyImportsResolverFs) + .resolve(specifier, mode) + } + + let context = TestContext::default(); + let temp_dir = context.temp_dir().path(); + + // scenarios like resolving ./example.js to ./example.ts + for (ext_from, ext_to) in [("js", "ts"), ("js", "tsx"), ("mjs", "mts")] { + let ts_file = temp_dir.join(format!("file.{}", ext_to)); + ts_file.write(""); + assert_eq!(resolve(&ts_file.url_file()), None); + assert_eq!( + resolve( + &temp_dir + .url_dir() + .join(&format!("file.{}", ext_from)) + .unwrap() + ), + Some(SloppyImportsResolution::JsToTs(ts_file.url_file())), + ); + ts_file.remove_file(); + } + + // no extension scenarios + for ext in ["js", "ts", "js", "tsx", "jsx", "mjs", "mts"] { + let file = temp_dir.join(format!("file.{}", ext)); + file.write(""); + assert_eq!( + resolve( + &temp_dir + .url_dir() + .join("file") // no ext + .unwrap() + ), + Some(SloppyImportsResolution::NoExtension(file.url_file())) + ); + file.remove_file(); + } + + // .ts and .js exists, .js specified (goes to specified) + { + let ts_file = temp_dir.join("file.ts"); + ts_file.write(""); + let js_file = temp_dir.join("file.js"); + js_file.write(""); + assert_eq!(resolve(&js_file.url_file()), None); + } + + // only js exists, .js specified + { + let js_only_file = temp_dir.join("js_only.js"); + js_only_file.write(""); + assert_eq!(resolve(&js_only_file.url_file()), None); + assert_eq!(resolve_types(&js_only_file.url_file()), None); + } + + // resolving a directory to an index file + { + let routes_dir = temp_dir.join("routes"); + routes_dir.create_dir_all(); + let index_file = routes_dir.join("index.ts"); + index_file.write(""); + assert_eq!( + resolve(&routes_dir.url_file()), + Some(SloppyImportsResolution::Directory(index_file.url_file())), + ); + } + + // both a directory and a file with specifier is present + { + let api_dir = temp_dir.join("api"); + api_dir.create_dir_all(); + let bar_file = api_dir.join("bar.ts"); + bar_file.write(""); + let api_file = temp_dir.join("api.ts"); + api_file.write(""); + assert_eq!( + resolve(&api_dir.url_file()), + Some(SloppyImportsResolution::NoExtension(api_file.url_file())), + ); + } + } + + #[test] + fn test_sloppy_import_resolution_suggestion_message() { + // directory + assert_eq!( + SloppyImportsResolution::Directory( + Url::parse("file:///dir/index.js").unwrap() + ) + .as_suggestion_message(), + "Maybe specify path to 'index.js' file in directory instead" + ); + // no ext + assert_eq!( + SloppyImportsResolution::NoExtension( + Url::parse("file:///dir/index.mjs").unwrap() + ) + .as_suggestion_message(), + "Maybe add a '.mjs' extension" + ); + // js to ts + assert_eq!( + SloppyImportsResolution::JsToTs( + Url::parse("file:///dir/index.mts").unwrap() + ) + .as_suggestion_message(), + "Maybe change the extension to '.mts'" + ); + } +} diff --git a/ext/node_resolver/Cargo.toml b/resolvers/node/Cargo.toml similarity index 100% rename from ext/node_resolver/Cargo.toml rename to resolvers/node/Cargo.toml diff --git a/ext/node_resolver/README.md b/resolvers/node/README.md similarity index 100% rename from ext/node_resolver/README.md rename to resolvers/node/README.md diff --git a/ext/node_resolver/analyze.rs b/resolvers/node/analyze.rs similarity index 100% rename from ext/node_resolver/analyze.rs rename to resolvers/node/analyze.rs diff --git a/ext/node_resolver/clippy.toml b/resolvers/node/clippy.toml similarity index 100% rename from ext/node_resolver/clippy.toml rename to resolvers/node/clippy.toml diff --git a/ext/node_resolver/env.rs b/resolvers/node/env.rs similarity index 100% rename from ext/node_resolver/env.rs rename to resolvers/node/env.rs diff --git a/ext/node_resolver/errors.rs b/resolvers/node/errors.rs similarity index 100% rename from ext/node_resolver/errors.rs rename to resolvers/node/errors.rs diff --git a/ext/node_resolver/lib.rs b/resolvers/node/lib.rs similarity index 100% rename from ext/node_resolver/lib.rs rename to resolvers/node/lib.rs diff --git a/ext/node_resolver/npm.rs b/resolvers/node/npm.rs similarity index 100% rename from ext/node_resolver/npm.rs rename to resolvers/node/npm.rs diff --git a/ext/node_resolver/package_json.rs b/resolvers/node/package_json.rs similarity index 100% rename from ext/node_resolver/package_json.rs rename to resolvers/node/package_json.rs diff --git a/ext/node_resolver/path.rs b/resolvers/node/path.rs similarity index 100% rename from ext/node_resolver/path.rs rename to resolvers/node/path.rs diff --git a/ext/node_resolver/resolution.rs b/resolvers/node/resolution.rs similarity index 100% rename from ext/node_resolver/resolution.rs rename to resolvers/node/resolution.rs diff --git a/ext/node_resolver/sync.rs b/resolvers/node/sync.rs similarity index 100% rename from ext/node_resolver/sync.rs rename to resolvers/node/sync.rs From 183130ff31d49d4774dc9c6ca1098567b04b1bcf Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 29 Sep 2024 20:07:50 -0400 Subject: [PATCH 005/302] refactor: cleanup for creating worker structs (#25933) --- cli/factory.rs | 2 + cli/module_loader.rs | 5 +- cli/resolver.rs | 3 +- cli/worker.rs | 111 ++++++++++++++--------- runtime/examples/extension/main.rs | 25 +++++- runtime/web_worker.rs | 140 ++++++++++++++--------------- runtime/worker.rs | 140 +++++++++++++---------------- 7 files changed, 229 insertions(+), 197 deletions(-) diff --git a/cli/factory.rs b/cli/factory.rs index 5e525ee32b..770aeefa14 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -782,6 +782,7 @@ impl CliFactory { let npm_resolver = self.npm_resolver().await?; let fs = self.fs(); let cli_node_resolver = self.cli_node_resolver().await?; + let cli_npm_resolver = self.npm_resolver().await?.clone(); let maybe_file_watcher_communicator = if cli_options.has_hmr() { Some(self.watcher_communicator.clone().unwrap()) } else { @@ -811,6 +812,7 @@ impl CliFactory { self.main_module_graph_container().await?.clone(), self.module_load_preparer().await?.clone(), cli_node_resolver.clone(), + cli_npm_resolver.clone(), NpmModuleLoader::new( self.cjs_resolutions().clone(), self.node_code_translator().await?.clone(), diff --git a/cli/module_loader.rs b/cli/module_loader.rs index f4e219bea1..293f41ddac 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -23,6 +23,7 @@ use crate::graph_container::ModuleGraphUpdatePermit; use crate::graph_util::CreateGraphOptions; use crate::graph_util::ModuleGraphBuilder; use crate::node; +use crate::npm::CliNpmResolver; use crate::resolver::CliGraphResolver; use crate::resolver::CliNodeResolver; use crate::resolver::ModuleCodeStringSource; @@ -203,6 +204,7 @@ struct SharedCliModuleLoaderState { main_module_graph_container: Arc, module_load_preparer: Arc, node_resolver: Arc, + npm_resolver: Arc, npm_module_loader: NpmModuleLoader, parsed_source_cache: Arc, resolver: Arc, @@ -221,6 +223,7 @@ impl CliModuleLoaderFactory { main_module_graph_container: Arc, module_load_preparer: Arc, node_resolver: Arc, + npm_resolver: Arc, npm_module_loader: NpmModuleLoader, parsed_source_cache: Arc, resolver: Arc, @@ -241,6 +244,7 @@ impl CliModuleLoaderFactory { main_module_graph_container, module_load_preparer, node_resolver, + npm_resolver, npm_module_loader, parsed_source_cache, resolver, @@ -478,7 +482,6 @@ impl Some(Module::Npm(module)) => { let package_folder = self .shared - .node_resolver .npm_resolver .as_managed() .unwrap() // byonm won't create a Module::Npm diff --git a/cli/resolver.rs b/cli/resolver.rs index d6e14c39d0..211f8aba1c 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -66,8 +66,7 @@ pub struct CliNodeResolver { cjs_resolutions: Arc, fs: Arc, node_resolver: Arc, - // todo(dsherret): remove this pub(crate) - pub(crate) npm_resolver: Arc, + npm_resolver: Arc, } impl CliNodeResolver { diff --git a/cli/worker.rs b/cli/worker.rs index c355d18bd5..71bdfd661f 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -34,8 +34,10 @@ use deno_runtime::ops::worker_host::CreateWebWorkerCb; use deno_runtime::permissions::RuntimePermissionDescriptorParser; use deno_runtime::web_worker::WebWorker; use deno_runtime::web_worker::WebWorkerOptions; +use deno_runtime::web_worker::WebWorkerServiceOptions; use deno_runtime::worker::MainWorker; use deno_runtime::worker::WorkerOptions; +use deno_runtime::worker::WorkerServiceOptions; use deno_runtime::BootstrapOptions; use deno_runtime::WorkerExecutionMode; use deno_runtime::WorkerLogLevel; @@ -570,6 +572,23 @@ impl CliMainWorkerFactory { } } + let services = WorkerServiceOptions { + root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), + module_loader, + fs: shared.fs.clone(), + node_services: Some(shared.create_node_init_services()), + npm_process_state_provider: Some(shared.npm_process_state_provider()), + blob_store: shared.blob_store.clone(), + broadcast_channel: shared.broadcast_channel.clone(), + shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some( + shared.compiled_wasm_module_store.clone(), + ), + feature_checker, + permissions, + permission_desc_parser: shared.permission_desc_parser.clone(), + v8_code_cache: shared.code_cache.clone(), + }; let options = WorkerOptions { bootstrap: BootstrapOptions { deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), @@ -604,7 +623,6 @@ impl CliMainWorkerFactory { .options .unsafely_ignore_certificate_errors .clone(), - root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), seed: shared.options.seed, format_js_error_fn: Some(Arc::new(format_js_error)), create_web_worker_cb, @@ -612,29 +630,16 @@ impl CliMainWorkerFactory { should_break_on_first_statement: shared.options.inspect_brk, should_wait_for_inspector_session: shared.options.inspect_wait, strace_ops: shared.options.strace_ops.clone(), - module_loader, - fs: shared.fs.clone(), - node_services: Some(shared.create_node_init_services()), - npm_process_state_provider: Some(shared.npm_process_state_provider()), get_error_class_fn: Some(&errors::get_error_class_name), cache_storage_dir, origin_storage_dir, - blob_store: shared.blob_store.clone(), - broadcast_channel: shared.broadcast_channel.clone(), - shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some( - shared.compiled_wasm_module_store.clone(), - ), stdio, - feature_checker, - permission_desc_parser: shared.permission_desc_parser.clone(), skip_op_registration: shared.options.skip_op_registration, - v8_code_cache: shared.code_cache.clone(), }; let mut worker = MainWorker::bootstrap_from_options( main_module.clone(), - permissions, + services, options, ); @@ -766,7 +771,27 @@ fn create_web_worker_callback( } } + let services = WebWorkerServiceOptions { + root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), + module_loader, + fs: shared.fs.clone(), + node_services: Some(shared.create_node_init_services()), + blob_store: shared.blob_store.clone(), + broadcast_channel: shared.broadcast_channel.clone(), + shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some( + shared.compiled_wasm_module_store.clone(), + ), + maybe_inspector_server, + feature_checker, + permission_desc_parser: shared.permission_desc_parser.clone(), + npm_process_state_provider: Some(shared.npm_process_state_provider()), + permissions: args.permissions, + }; let options = WebWorkerOptions { + name: args.name, + main_module: args.main_module.clone(), + worker_id: args.worker_id, bootstrap: BootstrapOptions { deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), args: shared.options.argv.clone(), @@ -777,7 +802,7 @@ fn create_web_worker_callback( enable_op_summary_metrics: shared.options.enable_op_summary_metrics, enable_testing_features: shared.options.enable_testing_features, locale: deno_core::v8::icu::get_language_tag(), - location: Some(args.main_module.clone()), + location: Some(args.main_module), no_color: !colors::use_color(), color_level: colors::get_color_level(), is_stdout_tty: deno_terminal::is_stdout_tty(), @@ -799,39 +824,19 @@ fn create_web_worker_callback( .options .unsafely_ignore_certificate_errors .clone(), - root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), seed: shared.options.seed, create_web_worker_cb, format_js_error_fn: Some(Arc::new(format_js_error)), - module_loader, - fs: shared.fs.clone(), - node_services: Some(shared.create_node_init_services()), worker_type: args.worker_type, - maybe_inspector_server, get_error_class_fn: Some(&errors::get_error_class_name), - blob_store: shared.blob_store.clone(), - broadcast_channel: shared.broadcast_channel.clone(), - shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some( - shared.compiled_wasm_module_store.clone(), - ), stdio: stdio.clone(), cache_storage_dir, - feature_checker, - permission_desc_parser: shared.permission_desc_parser.clone(), strace_ops: shared.options.strace_ops.clone(), close_on_idle: args.close_on_idle, maybe_worker_metadata: args.maybe_worker_metadata, - npm_process_state_provider: Some(shared.npm_process_state_provider()), }; - WebWorker::bootstrap_from_options( - args.name, - args.permissions, - args.main_module, - args.worker_id, - options, - ) + WebWorker::bootstrap_from_options(services, options) }) } @@ -841,23 +846,43 @@ fn create_web_worker_callback( mod tests { use super::*; use deno_core::resolve_path; + use deno_core::FsModuleLoader; use deno_fs::RealFs; use deno_runtime::deno_permissions::Permissions; fn create_test_worker() -> MainWorker { let main_module = resolve_path("./hello.js", &std::env::current_dir().unwrap()).unwrap(); - let permissions = PermissionsContainer::new( - Arc::new(RuntimePermissionDescriptorParser::new(Arc::new(RealFs))), - Permissions::none_without_prompt(), - ); - + let fs = Arc::new(RealFs); + let permission_desc_parser = + Arc::new(RuntimePermissionDescriptorParser::new(fs.clone())); let options = WorkerOptions { startup_snapshot: crate::js::deno_isolate_init(), ..Default::default() }; - MainWorker::bootstrap_from_options(main_module, permissions, options) + MainWorker::bootstrap_from_options( + main_module, + WorkerServiceOptions { + module_loader: Rc::new(FsModuleLoader), + permissions: PermissionsContainer::new( + permission_desc_parser.clone(), + Permissions::none_without_prompt(), + ), + blob_store: Default::default(), + broadcast_channel: Default::default(), + feature_checker: Default::default(), + node_services: Default::default(), + npm_process_state_provider: Default::default(), + permission_desc_parser, + root_cert_store_provider: Default::default(), + shared_array_buffer_store: Default::default(), + compiled_wasm_module_store: Default::default(), + v8_code_cache: Default::default(), + fs, + }, + options, + ) } #[tokio::test] diff --git a/runtime/examples/extension/main.rs b/runtime/examples/extension/main.rs index 6f4f02508b..71fbdfd6a0 100644 --- a/runtime/examples/extension/main.rs +++ b/runtime/examples/extension/main.rs @@ -16,6 +16,7 @@ use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::permissions::RuntimePermissionDescriptorParser; use deno_runtime::worker::MainWorker; use deno_runtime::worker::WorkerOptions; +use deno_runtime::worker::WorkerServiceOptions; #[op2(fast)] fn op_hello(#[string] text: &str) { @@ -35,13 +36,29 @@ async fn main() -> Result<(), AnyError> { Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/extension/main.js"); let main_module = ModuleSpecifier::from_file_path(js_path).unwrap(); eprintln!("Running {main_module}..."); + let fs = Arc::new(RealFs); + let permission_desc_parser = + Arc::new(RuntimePermissionDescriptorParser::new(fs.clone())); let mut worker = MainWorker::bootstrap_from_options( main_module.clone(), - PermissionsContainer::allow_all(Arc::new( - RuntimePermissionDescriptorParser::new(Arc::new(RealFs)), - )), - WorkerOptions { + WorkerServiceOptions { module_loader: Rc::new(FsModuleLoader), + permissions: PermissionsContainer::allow_all( + permission_desc_parser.clone(), + ), + blob_store: Default::default(), + broadcast_channel: Default::default(), + feature_checker: Default::default(), + node_services: Default::default(), + npm_process_state_provider: Default::default(), + permission_desc_parser, + root_cert_store_provider: Default::default(), + shared_array_buffer_store: Default::default(), + compiled_wasm_module_store: Default::default(), + v8_code_cache: Default::default(), + fs, + }, + WorkerOptions { extensions: vec![hello_runtime::init_ops_and_esm()], ..Default::default() }, diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index 8892d5bc6e..f13516268f 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -337,6 +337,42 @@ fn create_handles( (internal_handle, external_handle) } +pub struct WebWorkerServiceOptions { + pub blob_store: Arc, + pub broadcast_channel: InMemoryBroadcastChannel, + pub compiled_wasm_module_store: Option, + pub feature_checker: Arc, + pub fs: Arc, + pub maybe_inspector_server: Option>, + pub module_loader: Rc, + pub node_services: Option, + pub npm_process_state_provider: Option, + pub permissions: PermissionsContainer, + pub permission_desc_parser: Arc, + pub root_cert_store_provider: Option>, + pub shared_array_buffer_store: Option, +} + +pub struct WebWorkerOptions { + pub name: String, + pub main_module: ModuleSpecifier, + pub worker_id: WorkerId, + pub bootstrap: BootstrapOptions, + pub extensions: Vec, + pub startup_snapshot: Option<&'static [u8]>, + pub unsafely_ignore_certificate_errors: Option>, + pub seed: Option, + pub create_web_worker_cb: Arc, + pub format_js_error_fn: Option>, + pub worker_type: WebWorkerType, + pub get_error_class_fn: Option, + pub cache_storage_dir: Option, + pub stdio: Stdio, + pub strace_ops: Option>, + pub close_on_idle: bool, + pub maybe_worker_metadata: Option, +} + /// This struct is an implementation of `Worker` Web API /// /// Each `WebWorker` is either a child of `MainWorker` or other @@ -357,58 +393,21 @@ pub struct WebWorker { maybe_worker_metadata: Option, } -pub struct WebWorkerOptions { - // todo(dsherret): extract out the service structs from this options bag - pub bootstrap: BootstrapOptions, - pub extensions: Vec, - pub startup_snapshot: Option<&'static [u8]>, - pub unsafely_ignore_certificate_errors: Option>, - pub root_cert_store_provider: Option>, - pub seed: Option, - pub fs: Arc, - pub module_loader: Rc, - pub node_services: Option, - pub create_web_worker_cb: Arc, - pub format_js_error_fn: Option>, - pub worker_type: WebWorkerType, - pub maybe_inspector_server: Option>, - pub get_error_class_fn: Option, - pub blob_store: Arc, - pub broadcast_channel: InMemoryBroadcastChannel, - pub shared_array_buffer_store: Option, - pub compiled_wasm_module_store: Option, - pub cache_storage_dir: Option, - pub stdio: Stdio, - pub feature_checker: Arc, - pub permission_desc_parser: Arc, - pub strace_ops: Option>, - pub close_on_idle: bool, - pub maybe_worker_metadata: Option, - pub npm_process_state_provider: Option, -} - impl WebWorker { pub fn bootstrap_from_options( - name: String, - permissions: PermissionsContainer, - main_module: ModuleSpecifier, - worker_id: WorkerId, + services: WebWorkerServiceOptions, options: WebWorkerOptions, ) -> (Self, SendableWebWorkerHandle) { - let bootstrap_options = options.bootstrap.clone(); - let (mut worker, handle) = - Self::from_options(name, permissions, main_module, worker_id, options); + let (mut worker, handle, bootstrap_options) = + Self::from_options(services, options); worker.bootstrap(&bootstrap_options); (worker, handle) } - pub fn from_options( - name: String, - permissions: PermissionsContainer, - main_module: ModuleSpecifier, - worker_id: WorkerId, + fn from_options( + services: WebWorkerServiceOptions, mut options: WebWorkerOptions, - ) -> (Self, SendableWebWorkerHandle) { + ) -> (Self, SendableWebWorkerHandle, BootstrapOptions) { deno_core::extension!(deno_permissions_web_worker, options = { permissions: PermissionsContainer, @@ -436,15 +435,15 @@ impl WebWorker { deno_console::deno_console::init_ops_and_esm(), deno_url::deno_url::init_ops_and_esm(), deno_web::deno_web::init_ops_and_esm::( - options.blob_store.clone(), - Some(main_module.clone()), + services.blob_store, + Some(options.main_module.clone()), ), deno_webgpu::deno_webgpu::init_ops_and_esm(), deno_canvas::deno_canvas::init_ops_and_esm(), deno_fetch::deno_fetch::init_ops_and_esm::( deno_fetch::Options { user_agent: options.bootstrap.user_agent.clone(), - root_cert_store_provider: options.root_cert_store_provider.clone(), + root_cert_store_provider: services.root_cert_store_provider.clone(), unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -457,17 +456,17 @@ impl WebWorker { ), deno_websocket::deno_websocket::init_ops_and_esm::( options.bootstrap.user_agent.clone(), - options.root_cert_store_provider.clone(), + services.root_cert_store_provider.clone(), options.unsafely_ignore_certificate_errors.clone(), ), deno_webstorage::deno_webstorage::init_ops_and_esm(None).disable(), deno_crypto::deno_crypto::init_ops_and_esm(options.seed), deno_broadcast_channel::deno_broadcast_channel::init_ops_and_esm( - options.broadcast_channel.clone(), + services.broadcast_channel, ), deno_ffi::deno_ffi::init_ops_and_esm::(), deno_net::deno_net::init_ops_and_esm::( - options.root_cert_store_provider.clone(), + services.root_cert_store_provider.clone(), options.unsafely_ignore_certificate_errors.clone(), ), deno_tls::deno_tls::init_ops_and_esm(), @@ -477,7 +476,7 @@ impl WebWorker { options.seed, deno_kv::remote::HttpOptions { user_agent: options.bootstrap.user_agent.clone(), - root_cert_store_provider: options.root_cert_store_provider.clone(), + root_cert_store_provider: services.root_cert_store_provider, unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -492,25 +491,25 @@ impl WebWorker { deno_http::deno_http::init_ops_and_esm::(), deno_io::deno_io::init_ops_and_esm(Some(options.stdio)), deno_fs::deno_fs::init_ops_and_esm::( - options.fs.clone(), + services.fs.clone(), ), deno_node::deno_node::init_ops_and_esm::( - options.node_services, - options.fs, + services.node_services, + services.fs, ), // Runtime ops that are always initialized for WebWorkers - ops::runtime::deno_runtime::init_ops_and_esm(main_module.clone()), + ops::runtime::deno_runtime::init_ops_and_esm(options.main_module.clone()), ops::worker_host::deno_worker_host::init_ops_and_esm( - options.create_web_worker_cb.clone(), - options.format_js_error_fn.clone(), + options.create_web_worker_cb, + options.format_js_error_fn, ), ops::fs_events::deno_fs_events::init_ops_and_esm(), ops::os::deno_os_worker::init_ops_and_esm(), ops::permissions::deno_permissions::init_ops_and_esm( - options.permission_desc_parser.clone(), + services.permission_desc_parser, ), ops::process::deno_process::init_ops_and_esm( - options.npm_process_state_provider, + services.npm_process_state_provider, ), ops::signal::deno_signal::init_ops_and_esm(), ops::tty::deno_tty::init_ops_and_esm(), @@ -523,7 +522,7 @@ impl WebWorker { }, ), deno_permissions_web_worker::init_ops_and_esm( - permissions, + services.permissions, enable_testing_features, ), runtime::init_ops_and_esm(), @@ -556,17 +555,17 @@ impl WebWorker { ); let mut js_runtime = JsRuntime::new(RuntimeOptions { - module_loader: Some(options.module_loader.clone()), + module_loader: Some(services.module_loader), startup_snapshot: options.startup_snapshot, get_error_class_fn: options.get_error_class_fn, - shared_array_buffer_store: options.shared_array_buffer_store.clone(), - compiled_wasm_module_store: options.compiled_wasm_module_store.clone(), + shared_array_buffer_store: services.shared_array_buffer_store, + compiled_wasm_module_store: services.compiled_wasm_module_store, extensions, extension_transpiler: Some(Rc::new(|specifier, source| { maybe_transpile_source(specifier, source) })), - inspector: options.maybe_inspector_server.is_some(), - feature_checker: Some(options.feature_checker.clone()), + inspector: services.maybe_inspector_server.is_some(), + feature_checker: Some(services.feature_checker), op_metrics_factory_fn, import_meta_resolve_callback: Some(Box::new( import_meta_resolve_callback, @@ -582,9 +581,9 @@ impl WebWorker { js_runtime.op_state().borrow_mut().put(op_summary_metrics); } - if let Some(server) = options.maybe_inspector_server.clone() { + if let Some(server) = services.maybe_inspector_server { server.register_inspector( - main_module.to_string(), + options.main_module.to_string(), &mut js_runtime, false, ); @@ -599,7 +598,7 @@ impl WebWorker { let (internal_handle, external_handle) = { let handle = js_runtime.v8_isolate().thread_safe_handle(); let (internal_handle, external_handle) = - create_handles(handle, name.clone(), options.worker_type); + create_handles(handle, options.name.clone(), options.worker_type); let op_state = js_runtime.op_state(); let mut op_state = op_state.borrow_mut(); op_state.put(internal_handle.clone()); @@ -630,12 +629,12 @@ impl WebWorker { ( Self { - id: worker_id, + id: options.worker_id, js_runtime, - name, + name: options.name, internal_handle, worker_type: options.worker_type, - main_module, + main_module: options.main_module, poll_for_messages_fn: None, has_message_event_listener_fn: None, bootstrap_fn_global: Some(bootstrap_fn_global), @@ -644,6 +643,7 @@ impl WebWorker { maybe_worker_metadata: options.maybe_worker_metadata, }, external_handle, + options.bootstrap, ) } diff --git a/runtime/worker.rs b/runtime/worker.rs index f72e6d7c09..3ce5562fa2 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -19,7 +19,6 @@ use deno_core::v8; use deno_core::CompiledWasmModuleStore; use deno_core::Extension; use deno_core::FeatureChecker; -use deno_core::FsModuleLoader; use deno_core::GetErrorClassFn; use deno_core::JsRuntime; use deno_core::LocalInspectorSession; @@ -50,7 +49,6 @@ use crate::code_cache::CodeCacheType; use crate::inspector_server::InspectorServer; use crate::ops; use crate::ops::process::NpmProcessStateProviderRc; -use crate::permissions::RuntimePermissionDescriptorParser; use crate::shared::maybe_transpile_source; use crate::shared::runtime; use crate::BootstrapOptions; @@ -128,6 +126,43 @@ pub struct MainWorker { dispatch_process_exit_event_fn_global: v8::Global, } +pub struct WorkerServiceOptions { + pub blob_store: Arc, + pub broadcast_channel: InMemoryBroadcastChannel, + pub feature_checker: Arc, + pub fs: Arc, + /// Implementation of `ModuleLoader` which will be + /// called when V8 requests to load ES modules. + /// + /// If not provided runtime will error if code being + /// executed tries to load modules. + pub module_loader: Rc, + pub node_services: Option, + pub npm_process_state_provider: Option, + pub permission_desc_parser: + Arc, + pub permissions: PermissionsContainer, + pub root_cert_store_provider: Option>, + + /// The store to use for transferring SharedArrayBuffers between isolates. + /// If multiple isolates should have the possibility of sharing + /// SharedArrayBuffers, they should use the same [SharedArrayBufferStore]. If + /// no [SharedArrayBufferStore] is specified, SharedArrayBuffer can not be + /// serialized. + pub shared_array_buffer_store: Option, + + /// The store to use for transferring `WebAssembly.Module` objects between + /// isolates. + /// If multiple isolates should have the possibility of sharing + /// `WebAssembly.Module` objects, they should use the same + /// [CompiledWasmModuleStore]. If no [CompiledWasmModuleStore] is specified, + /// `WebAssembly.Module` objects cannot be serialized. + pub compiled_wasm_module_store: Option, + + /// V8 code cache for module and script source code. + pub v8_code_cache: Option>, +} + pub struct WorkerOptions { pub bootstrap: BootstrapOptions, @@ -148,20 +183,8 @@ pub struct WorkerOptions { pub create_params: Option, pub unsafely_ignore_certificate_errors: Option>, - pub root_cert_store_provider: Option>, pub seed: Option, - pub fs: Arc, - /// Implementation of `ModuleLoader` which will be - /// called when V8 requests to load ES modules. - /// - /// If not provided runtime will error if code being - /// executed tries to load modules. - pub module_loader: Rc, - pub node_services: Option, - pub npm_process_state_provider: Option, - pub permission_desc_parser: - Arc, // Callbacks invoked when creating new instance of WebWorker pub create_web_worker_cb: Arc, pub format_js_error_fn: Option>, @@ -182,69 +205,31 @@ pub struct WorkerOptions { pub get_error_class_fn: Option, pub cache_storage_dir: Option, pub origin_storage_dir: Option, - pub blob_store: Arc, - pub broadcast_channel: InMemoryBroadcastChannel, - - /// The store to use for transferring SharedArrayBuffers between isolates. - /// If multiple isolates should have the possibility of sharing - /// SharedArrayBuffers, they should use the same [SharedArrayBufferStore]. If - /// no [SharedArrayBufferStore] is specified, SharedArrayBuffer can not be - /// serialized. - pub shared_array_buffer_store: Option, - - /// The store to use for transferring `WebAssembly.Module` objects between - /// isolates. - /// If multiple isolates should have the possibility of sharing - /// `WebAssembly.Module` objects, they should use the same - /// [CompiledWasmModuleStore]. If no [CompiledWasmModuleStore] is specified, - /// `WebAssembly.Module` objects cannot be serialized. - pub compiled_wasm_module_store: Option, pub stdio: Stdio, - pub feature_checker: Arc, - - /// V8 code cache for module and script source code. - pub v8_code_cache: Option>, } -// todo(dsherret): this is error prone to use. We should separate -// out the WorkerOptions from the services. impl Default for WorkerOptions { fn default() -> Self { - let real_fs = Arc::new(deno_fs::RealFs); Self { create_web_worker_cb: Arc::new(|_| { unimplemented!("web workers are not supported") }), - fs: real_fs.clone(), - module_loader: Rc::new(FsModuleLoader), skip_op_registration: false, seed: None, unsafely_ignore_certificate_errors: Default::default(), should_break_on_first_statement: Default::default(), should_wait_for_inspector_session: Default::default(), strace_ops: Default::default(), - compiled_wasm_module_store: Default::default(), - shared_array_buffer_store: Default::default(), maybe_inspector_server: Default::default(), format_js_error_fn: Default::default(), get_error_class_fn: Default::default(), origin_storage_dir: Default::default(), cache_storage_dir: Default::default(), - broadcast_channel: Default::default(), - root_cert_store_provider: Default::default(), - node_services: Default::default(), - blob_store: Default::default(), extensions: Default::default(), startup_snapshot: Default::default(), create_params: Default::default(), - npm_process_state_provider: Default::default(), bootstrap: Default::default(), stdio: Default::default(), - feature_checker: Default::default(), - permission_desc_parser: Arc::new(RuntimePermissionDescriptorParser::new( - real_fs, - )), - v8_code_cache: Default::default(), } } } @@ -318,20 +303,20 @@ pub fn create_op_metrics( impl MainWorker { pub fn bootstrap_from_options( main_module: ModuleSpecifier, - permissions: PermissionsContainer, + services: WorkerServiceOptions, options: WorkerOptions, ) -> Self { - let bootstrap_options = options.bootstrap.clone(); - let mut worker = Self::from_options(main_module, permissions, options); + let (mut worker, bootstrap_options) = + Self::from_options(main_module, services, options); worker.bootstrap(bootstrap_options); worker } - pub fn from_options( + fn from_options( main_module: ModuleSpecifier, - permissions: PermissionsContainer, + services: WorkerServiceOptions, mut options: WorkerOptions, - ) -> Self { + ) -> (Self, BootstrapOptions) { deno_core::extension!(deno_permissions_worker, options = { permissions: PermissionsContainer, @@ -365,7 +350,7 @@ impl MainWorker { deno_console::deno_console::init_ops_and_esm(), deno_url::deno_url::init_ops_and_esm(), deno_web::deno_web::init_ops_and_esm::( - options.blob_store.clone(), + services.blob_store.clone(), options.bootstrap.location.clone(), ), deno_webgpu::deno_webgpu::init_ops_and_esm(), @@ -373,7 +358,7 @@ impl MainWorker { deno_fetch::deno_fetch::init_ops_and_esm::( deno_fetch::Options { user_agent: options.bootstrap.user_agent.clone(), - root_cert_store_provider: options.root_cert_store_provider.clone(), + root_cert_store_provider: services.root_cert_store_provider.clone(), unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -386,7 +371,7 @@ impl MainWorker { ), deno_websocket::deno_websocket::init_ops_and_esm::( options.bootstrap.user_agent.clone(), - options.root_cert_store_provider.clone(), + services.root_cert_store_provider.clone(), options.unsafely_ignore_certificate_errors.clone(), ), deno_webstorage::deno_webstorage::init_ops_and_esm( @@ -394,11 +379,11 @@ impl MainWorker { ), deno_crypto::deno_crypto::init_ops_and_esm(options.seed), deno_broadcast_channel::deno_broadcast_channel::init_ops_and_esm( - options.broadcast_channel.clone(), + services.broadcast_channel.clone(), ), deno_ffi::deno_ffi::init_ops_and_esm::(), deno_net::deno_net::init_ops_and_esm::( - options.root_cert_store_provider.clone(), + services.root_cert_store_provider.clone(), options.unsafely_ignore_certificate_errors.clone(), ), deno_tls::deno_tls::init_ops_and_esm(), @@ -408,7 +393,7 @@ impl MainWorker { options.seed, deno_kv::remote::HttpOptions { user_agent: options.bootstrap.user_agent.clone(), - root_cert_store_provider: options.root_cert_store_provider.clone(), + root_cert_store_provider: services.root_cert_store_provider.clone(), unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -423,11 +408,11 @@ impl MainWorker { deno_http::deno_http::init_ops_and_esm::(), deno_io::deno_io::init_ops_and_esm(Some(options.stdio)), deno_fs::deno_fs::init_ops_and_esm::( - options.fs.clone(), + services.fs.clone(), ), deno_node::deno_node::init_ops_and_esm::( - options.node_services, - options.fs, + services.node_services, + services.fs, ), // Ops from this crate ops::runtime::deno_runtime::init_ops_and_esm(main_module.clone()), @@ -438,10 +423,10 @@ impl MainWorker { ops::fs_events::deno_fs_events::init_ops_and_esm(), ops::os::deno_os::init_ops_and_esm(exit_code.clone()), ops::permissions::deno_permissions::init_ops_and_esm( - options.permission_desc_parser, + services.permission_desc_parser, ), ops::process::deno_process::init_ops_and_esm( - options.npm_process_state_provider, + services.npm_process_state_provider, ), ops::signal::deno_signal::init_ops_and_esm(), ops::tty::deno_tty::init_ops_and_esm(), @@ -454,7 +439,7 @@ impl MainWorker { }, ), deno_permissions_worker::init_ops_and_esm( - permissions, + services.permissions, enable_testing_features, ), runtime::init_ops_and_esm(), @@ -494,20 +479,20 @@ impl MainWorker { }); let mut js_runtime = JsRuntime::new(RuntimeOptions { - module_loader: Some(options.module_loader.clone()), + module_loader: Some(services.module_loader.clone()), startup_snapshot: options.startup_snapshot, create_params: options.create_params, skip_op_registration: options.skip_op_registration, get_error_class_fn: options.get_error_class_fn, - shared_array_buffer_store: options.shared_array_buffer_store.clone(), - compiled_wasm_module_store: options.compiled_wasm_module_store.clone(), + shared_array_buffer_store: services.shared_array_buffer_store.clone(), + compiled_wasm_module_store: services.compiled_wasm_module_store.clone(), extensions, extension_transpiler: Some(Rc::new(|specifier, source| { maybe_transpile_source(specifier, source) })), inspector: options.maybe_inspector_server.is_some(), is_main: true, - feature_checker: Some(options.feature_checker.clone()), + feature_checker: Some(services.feature_checker.clone()), op_metrics_factory_fn, wait_for_inspector_disconnect_callback: Some( wait_for_inspector_disconnect_callback, @@ -519,7 +504,7 @@ impl MainWorker { validate_import_attributes_callback, )), import_assertions_support: deno_core::ImportAssertionsSupport::Error, - eval_context_code_cache_cbs: options.v8_code_cache.map(|cache| { + eval_context_code_cache_cbs: services.v8_code_cache.map(|cache| { let cache_clone = cache.clone(); ( Box::new(move |specifier: &ModuleSpecifier, code: &v8::String| { @@ -666,7 +651,7 @@ impl MainWorker { ) }; - Self { + let worker = Self { js_runtime, should_break_on_first_statement: options.should_break_on_first_statement, should_wait_for_inspector_session: options @@ -678,7 +663,8 @@ impl MainWorker { dispatch_unload_event_fn_global, dispatch_process_beforeexit_event_fn_global, dispatch_process_exit_event_fn_global, - } + }; + (worker, options.bootstrap) } pub fn bootstrap(&mut self, options: BootstrapOptions) { From efb413bdaa0f5be6d9082d397aae1e20a258c85f Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Mon, 30 Sep 2024 17:10:51 +0530 Subject: [PATCH 006/302] fix: update sui to 0.4 (#25942) Properly apply offset fixup to `LC_DYLD_EXPORTS_TRIE` load commands. This should fix Node-API symbols not resolving in RC releases. Fixes https://github.com/denoland/deno/issues/25879 Fixes https://github.com/denoland/deno/issues/25940 Ref https://github.com/denoland/sui/commit/2b3a33bb6e1afbb04e2e1d345547e5686894e7f0 --- Cargo.lock | 4 ++-- cli/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bdde5d5410..b4dafacdd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4174,9 +4174,9 @@ dependencies = [ [[package]] name = "libsui" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e39af24eff8df7c8b9980ef56a1a1f4d2e77b34b2d5c0529f108c53ae96a7a" +checksum = "205eca4e7beaad637dcd38fe41292065894ee7f498077cf3c135d5f7252b9f27" dependencies = [ "editpe", "libc", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 32e0651b2f..5fb32deb7f 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -80,7 +80,7 @@ deno_semver.workspace = true deno_task_shell = "=0.17.0" deno_terminal.workspace = true eszip = "=0.78.0" -libsui = "0.3.1" +libsui = "0.4.0" napi_sym.workspace = true node_resolver.workspace = true From c8f692057b256dac57342867b7606a74309449fc Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 30 Sep 2024 09:19:24 -0400 Subject: [PATCH 007/302] refactor: bury descriptor parsing in PermissionsContainer (#25936) Closes https://github.com/denoland/deno/issues/25634 --- cli/args/flags.rs | 6 +- cli/factory.rs | 1 - cli/ops/bench.rs | 18 +- cli/ops/testing.rs | 18 +- cli/standalone/mod.rs | 3 - cli/worker.rs | 10 +- runtime/examples/extension/main.rs | 5 +- runtime/ops/permissions.rs | 198 +-------- runtime/ops/worker_host.rs | 13 +- runtime/permissions.rs | 2 +- runtime/permissions/lib.rs | 688 +++++++++++++++++++++-------- runtime/snapshot.rs | 5 +- runtime/web_worker.rs | 32 +- runtime/worker.rs | 6 +- 14 files changed, 563 insertions(+), 442 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 13c93fa831..2cbf463948 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -34,8 +34,8 @@ use deno_core::url::Url; use deno_graph::GraphKind; use deno_path_util::normalize_path; use deno_path_util::url_to_file_path; -use deno_runtime::deno_permissions::parse_sys_kind; use deno_runtime::deno_permissions::PermissionsOptions; +use deno_runtime::deno_permissions::SysDescriptor; use log::debug; use log::Level; use serde::Deserialize; @@ -3448,7 +3448,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { .require_equals(true) .value_name("API_NAME") .help("Allow access to OS information. Optionally allow specific APIs by function name") - .value_parser(|key: &str| parse_sys_kind(key).map(ToString::to_string)) + .value_parser(|key: &str| SysDescriptor::parse(key.to_string()).map(|s| s.into_string())) .hide(true) ; if let Some(requires) = requires { @@ -3466,7 +3466,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { .require_equals(true) .value_name("API_NAME") .help("Deny access to OS information. Optionally deny specific APIs by function name") - .value_parser(|key: &str| parse_sys_kind(key).map(ToString::to_string)) + .value_parser(|key: &str| SysDescriptor::parse(key.to_string()).map(|s| s.into_string())) .hide(true) ; if let Some(requires) = requires { diff --git a/cli/factory.rs b/cli/factory.rs index 770aeefa14..ffe7f8d2f7 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -824,7 +824,6 @@ impl CliFactory { )), node_resolver.clone(), npm_resolver.clone(), - self.permission_desc_parser()?.clone(), self.root_cert_store_provider().clone(), self.root_permissions_container()?.clone(), StorageKeyResolver::from_options(cli_options), diff --git a/cli/ops/bench.rs b/cli/ops/bench.rs index edd8c118ca..5d1e6e746d 100644 --- a/cli/ops/bench.rs +++ b/cli/ops/bench.rs @@ -2,7 +2,6 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use std::time; use deno_core::error::generic_error; @@ -12,9 +11,7 @@ use deno_core::op2; use deno_core::v8; use deno_core::ModuleSpecifier; use deno_core::OpState; -use deno_runtime::deno_permissions::create_child_permissions; use deno_runtime::deno_permissions::ChildPermissionsArg; -use deno_runtime::deno_permissions::PermissionDescriptorParser; use deno_runtime::deno_permissions::PermissionsContainer; use tokio::sync::mpsc::UnboundedSender; use uuid::Uuid; @@ -61,19 +58,8 @@ pub fn op_pledge_test_permissions( #[serde] args: ChildPermissionsArg, ) -> Result { let token = Uuid::new_v4(); - let permission_desc_parser = state - .borrow::>() - .clone(); let parent_permissions = state.borrow_mut::(); - let worker_permissions = { - let mut parent_permissions = parent_permissions.inner.lock(); - let perms = create_child_permissions( - permission_desc_parser.as_ref(), - &mut parent_permissions, - args, - )?; - PermissionsContainer::new(permission_desc_parser, perms) - }; + let worker_permissions = parent_permissions.create_child_permissions(args)?; let parent_permissions = parent_permissions.clone(); if state.try_take::().is_some() { @@ -83,7 +69,6 @@ pub fn op_pledge_test_permissions( state.put::(PermissionsHolder(token, parent_permissions)); // NOTE: This call overrides current permission set for the worker - state.put(worker_permissions.inner.clone()); state.put::(worker_permissions); Ok(token) @@ -100,7 +85,6 @@ pub fn op_restore_test_permissions( } let permissions = permissions_holder.1; - state.put(permissions.inner.clone()); state.put::(permissions); Ok(()) } else { diff --git a/cli/ops/testing.rs b/cli/ops/testing.rs index 6a8d310062..c3f469656c 100644 --- a/cli/ops/testing.rs +++ b/cli/ops/testing.rs @@ -16,13 +16,10 @@ use deno_core::op2; use deno_core::v8; use deno_core::ModuleSpecifier; use deno_core::OpState; -use deno_runtime::deno_permissions::create_child_permissions; use deno_runtime::deno_permissions::ChildPermissionsArg; -use deno_runtime::deno_permissions::PermissionDescriptorParser; use deno_runtime::deno_permissions::PermissionsContainer; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use uuid::Uuid; deno_core::extension!(deno_test, @@ -56,19 +53,8 @@ pub fn op_pledge_test_permissions( #[serde] args: ChildPermissionsArg, ) -> Result { let token = Uuid::new_v4(); - let permission_desc_parser = state - .borrow::>() - .clone(); let parent_permissions = state.borrow_mut::(); - let worker_permissions = { - let mut parent_permissions = parent_permissions.inner.lock(); - let perms = create_child_permissions( - permission_desc_parser.as_ref(), - &mut parent_permissions, - args, - )?; - PermissionsContainer::new(permission_desc_parser, perms) - }; + let worker_permissions = parent_permissions.create_child_permissions(args)?; let parent_permissions = parent_permissions.clone(); if state.try_take::().is_some() { @@ -77,7 +63,6 @@ pub fn op_pledge_test_permissions( state.put::(PermissionsHolder(token, parent_permissions)); // NOTE: This call overrides current permission set for the worker - state.put(worker_permissions.inner.clone()); state.put::(worker_permissions); Ok(token) @@ -94,7 +79,6 @@ pub fn op_restore_test_permissions( } let permissions = permissions_holder.1; - state.put(permissions.inner.clone()); state.put::(permissions); Ok(()) } else { diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 93ac6002bf..56f76ffb9b 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -693,8 +693,6 @@ pub async fn run( } checker }); - let permission_desc_parser = - Arc::new(RuntimePermissionDescriptorParser::new(fs.clone())); let worker_factory = CliMainWorkerFactory::new( Arc::new(BlobStore::default()), // Code cache is not supported for standalone binary yet. @@ -707,7 +705,6 @@ pub async fn run( Box::new(module_loader_factory), node_resolver, npm_resolver, - permission_desc_parser, root_cert_store_provider, permissions, StorageKeyResolver::empty(), diff --git a/cli/worker.rs b/cli/worker.rs index 71bdfd661f..cc18c0d154 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -31,7 +31,6 @@ use deno_runtime::fmt_errors::format_js_error; use deno_runtime::inspector_server::InspectorServer; use deno_runtime::ops::process::NpmProcessStateProviderRc; use deno_runtime::ops::worker_host::CreateWebWorkerCb; -use deno_runtime::permissions::RuntimePermissionDescriptorParser; use deno_runtime::web_worker::WebWorker; use deno_runtime::web_worker::WebWorkerOptions; use deno_runtime::web_worker::WebWorkerServiceOptions; @@ -136,7 +135,6 @@ struct SharedWorkerState { module_loader_factory: Box, node_resolver: Arc, npm_resolver: Arc, - permission_desc_parser: Arc, root_cert_store_provider: Arc, root_permissions: PermissionsContainer, shared_array_buffer_store: SharedArrayBufferStore, @@ -433,7 +431,6 @@ impl CliMainWorkerFactory { module_loader_factory: Box, node_resolver: Arc, npm_resolver: Arc, - permission_parser: Arc, root_cert_store_provider: Arc, root_permissions: PermissionsContainer, storage_key_resolver: StorageKeyResolver, @@ -454,7 +451,6 @@ impl CliMainWorkerFactory { module_loader_factory, node_resolver, npm_resolver, - permission_desc_parser: permission_parser, root_cert_store_provider, root_permissions, shared_array_buffer_store: Default::default(), @@ -586,7 +582,6 @@ impl CliMainWorkerFactory { ), feature_checker, permissions, - permission_desc_parser: shared.permission_desc_parser.clone(), v8_code_cache: shared.code_cache.clone(), }; let options = WorkerOptions { @@ -784,7 +779,6 @@ fn create_web_worker_callback( ), maybe_inspector_server, feature_checker, - permission_desc_parser: shared.permission_desc_parser.clone(), npm_process_state_provider: Some(shared.npm_process_state_provider()), permissions: args.permissions, }; @@ -849,6 +843,7 @@ mod tests { use deno_core::FsModuleLoader; use deno_fs::RealFs; use deno_runtime::deno_permissions::Permissions; + use deno_runtime::permissions::RuntimePermissionDescriptorParser; fn create_test_worker() -> MainWorker { let main_module = @@ -866,7 +861,7 @@ mod tests { WorkerServiceOptions { module_loader: Rc::new(FsModuleLoader), permissions: PermissionsContainer::new( - permission_desc_parser.clone(), + permission_desc_parser, Permissions::none_without_prompt(), ), blob_store: Default::default(), @@ -874,7 +869,6 @@ mod tests { feature_checker: Default::default(), node_services: Default::default(), npm_process_state_provider: Default::default(), - permission_desc_parser, root_cert_store_provider: Default::default(), shared_array_buffer_store: Default::default(), compiled_wasm_module_store: Default::default(), diff --git a/runtime/examples/extension/main.rs b/runtime/examples/extension/main.rs index 71fbdfd6a0..9889b28dcf 100644 --- a/runtime/examples/extension/main.rs +++ b/runtime/examples/extension/main.rs @@ -43,15 +43,12 @@ async fn main() -> Result<(), AnyError> { main_module.clone(), WorkerServiceOptions { module_loader: Rc::new(FsModuleLoader), - permissions: PermissionsContainer::allow_all( - permission_desc_parser.clone(), - ), + permissions: PermissionsContainer::allow_all(permission_desc_parser), blob_store: Default::default(), broadcast_channel: Default::default(), feature_checker: Default::default(), node_services: Default::default(), npm_process_state_provider: Default::default(), - permission_desc_parser, root_cert_store_provider: Default::default(), shared_array_buffer_store: Default::default(), compiled_wasm_module_store: Default::default(), diff --git a/runtime/ops/permissions.rs b/runtime/ops/permissions.rs index 9b46dd0193..1dbc852596 100644 --- a/runtime/ops/permissions.rs +++ b/runtime/ops/permissions.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use ::deno_permissions::parse_sys_kind; -use ::deno_permissions::PermissionDescriptorParser; use ::deno_permissions::PermissionState; use ::deno_permissions::PermissionsContainer; use deno_core::error::custom_error; @@ -10,7 +8,6 @@ use deno_core::op2; use deno_core::OpState; use serde::Deserialize; use serde::Serialize; -use std::sync::Arc; deno_core::extension!( deno_permissions, @@ -19,12 +16,6 @@ deno_core::extension!( op_revoke_permission, op_request_permission, ], - options = { - permission_desc_parser: Arc, - }, - state = |state, options| { - state.put(options.permission_desc_parser); - }, ); #[derive(Deserialize)] @@ -62,62 +53,15 @@ pub fn op_query_permission( state: &mut OpState, #[serde] args: PermissionArgs, ) -> Result { - let permissions_container = state.borrow::(); - // todo(dsherret): don't have this function use the properties of - // permission container - let desc_parser = &permissions_container.descriptor_parser; - let permissions = permissions_container.inner.lock(); - let path = args.path.as_deref(); + let permissions = state.borrow::(); let perm = match args.name.as_ref() { - "read" => permissions.read.query( - path - .map(|path| { - Result::<_, AnyError>::Ok( - desc_parser.parse_path_query(path)?.into_read(), - ) - }) - .transpose()? - .as_ref(), - ), - "write" => permissions.write.query( - path - .map(|path| { - Result::<_, AnyError>::Ok( - desc_parser.parse_path_query(path)?.into_write(), - ) - }) - .transpose()? - .as_ref(), - ), - "net" => permissions.net.query( - match args.host.as_deref() { - None => None, - Some(h) => Some(desc_parser.parse_net_descriptor(h)?), - } - .as_ref(), - ), - "env" => permissions.env.query(args.variable.as_deref()), - "sys" => permissions - .sys - .query(args.kind.as_deref().map(parse_sys_kind).transpose()?), - "run" => permissions.run.query( - args - .command - .as_deref() - .map(|request| desc_parser.parse_run_query(request)) - .transpose()? - .as_ref(), - ), - "ffi" => permissions.ffi.query( - path - .map(|path| { - Result::<_, AnyError>::Ok( - desc_parser.parse_path_query(path)?.into_ffi(), - ) - }) - .transpose()? - .as_ref(), - ), + "read" => permissions.query_read(args.path.as_deref())?, + "write" => permissions.query_write(args.path.as_deref())?, + "net" => permissions.query_net(args.host.as_deref())?, + "env" => permissions.query_env(args.variable.as_deref()), + "sys" => permissions.query_sys(args.kind.as_deref())?, + "run" => permissions.query_run(args.command.as_deref())?, + "ffi" => permissions.query_ffi(args.path.as_deref())?, n => { return Err(custom_error( "ReferenceError", @@ -134,62 +78,15 @@ pub fn op_revoke_permission( state: &mut OpState, #[serde] args: PermissionArgs, ) -> Result { - // todo(dsherret): don't have this function use the properties of - // permission container - let permissions_container = state.borrow_mut::(); - let desc_parser = &permissions_container.descriptor_parser; - let mut permissions = permissions_container.inner.lock(); - let path = args.path.as_deref(); + let permissions = state.borrow::(); let perm = match args.name.as_ref() { - "read" => permissions.read.revoke( - path - .map(|path| { - Result::<_, AnyError>::Ok( - desc_parser.parse_path_query(path)?.into_read(), - ) - }) - .transpose()? - .as_ref(), - ), - "write" => permissions.write.revoke( - path - .map(|path| { - Result::<_, AnyError>::Ok( - desc_parser.parse_path_query(path)?.into_write(), - ) - }) - .transpose()? - .as_ref(), - ), - "net" => permissions.net.revoke( - match args.host.as_deref() { - None => None, - Some(h) => Some(desc_parser.parse_net_descriptor(h)?), - } - .as_ref(), - ), - "env" => permissions.env.revoke(args.variable.as_deref()), - "sys" => permissions - .sys - .revoke(args.kind.as_deref().map(parse_sys_kind).transpose()?), - "run" => permissions.run.revoke( - args - .command - .as_deref() - .map(|request| desc_parser.parse_run_query(request)) - .transpose()? - .as_ref(), - ), - "ffi" => permissions.ffi.revoke( - path - .map(|path| { - Result::<_, AnyError>::Ok( - desc_parser.parse_path_query(path)?.into_ffi(), - ) - }) - .transpose()? - .as_ref(), - ), + "read" => permissions.revoke_read(args.path.as_deref())?, + "write" => permissions.revoke_write(args.path.as_deref())?, + "net" => permissions.revoke_net(args.host.as_deref())?, + "env" => permissions.revoke_env(args.variable.as_deref()), + "sys" => permissions.revoke_sys(args.kind.as_deref())?, + "run" => permissions.revoke_run(args.command.as_deref())?, + "ffi" => permissions.revoke_ffi(args.path.as_deref())?, n => { return Err(custom_error( "ReferenceError", @@ -206,62 +103,15 @@ pub fn op_request_permission( state: &mut OpState, #[serde] args: PermissionArgs, ) -> Result { - // todo(dsherret): don't have this function use the properties of - // permission container - let permissions_container = state.borrow_mut::(); - let desc_parser = &permissions_container.descriptor_parser; - let mut permissions = permissions_container.inner.lock(); - let path = args.path.as_deref(); + let permissions = state.borrow::(); let perm = match args.name.as_ref() { - "read" => permissions.read.request( - path - .map(|path| { - Result::<_, AnyError>::Ok( - desc_parser.parse_path_query(path)?.into_read(), - ) - }) - .transpose()? - .as_ref(), - ), - "write" => permissions.write.request( - path - .map(|path| { - Result::<_, AnyError>::Ok( - desc_parser.parse_path_query(path)?.into_write(), - ) - }) - .transpose()? - .as_ref(), - ), - "net" => permissions.net.request( - match args.host.as_deref() { - None => None, - Some(h) => Some(desc_parser.parse_net_descriptor(h)?), - } - .as_ref(), - ), - "env" => permissions.env.request(args.variable.as_deref()), - "sys" => permissions - .sys - .request(args.kind.as_deref().map(parse_sys_kind).transpose()?), - "run" => permissions.run.request( - args - .command - .as_deref() - .map(|request| desc_parser.parse_run_query(request)) - .transpose()? - .as_ref(), - ), - "ffi" => permissions.ffi.request( - path - .map(|path| { - Result::<_, AnyError>::Ok( - desc_parser.parse_path_query(path)?.into_ffi(), - ) - }) - .transpose()? - .as_ref(), - ), + "read" => permissions.request_read(args.path.as_deref())?, + "write" => permissions.request_write(args.path.as_deref())?, + "net" => permissions.request_net(args.host.as_deref())?, + "env" => permissions.request_env(args.variable.as_deref()), + "sys" => permissions.request_sys(args.kind.as_deref())?, + "run" => permissions.request_run(args.command.as_deref())?, + "ffi" => permissions.request_ffi(args.path.as_deref())?, n => { return Err(custom_error( "ReferenceError", diff --git a/runtime/ops/worker_host.rs b/runtime/ops/worker_host.rs index 3c00356455..b9fd066543 100644 --- a/runtime/ops/worker_host.rs +++ b/runtime/ops/worker_host.rs @@ -17,9 +17,7 @@ use deno_core::CancelFuture; use deno_core::CancelHandle; use deno_core::ModuleSpecifier; use deno_core::OpState; -use deno_permissions::create_child_permissions; use deno_permissions::ChildPermissionsArg; -use deno_permissions::PermissionDescriptorParser; use deno_permissions::PermissionsContainer; use deno_web::deserialize_js_transferables; use deno_web::JsMessageData; @@ -154,19 +152,10 @@ fn op_create_worker( "Worker.deno.permissions", ); } - let permission_desc_parser = state - .borrow::>() - .clone(); let parent_permissions = state.borrow_mut::(); let worker_permissions = if let Some(child_permissions_arg) = args.permissions { - let mut parent_permissions = parent_permissions.inner.lock(); - let perms = create_child_permissions( - permission_desc_parser.as_ref(), - &mut parent_permissions, - child_permissions_arg, - )?; - PermissionsContainer::new(permission_desc_parser, perms) + parent_permissions.create_child_permissions(child_permissions_arg)? } else { parent_permissions.clone() }; diff --git a/runtime/permissions.rs b/runtime/permissions.rs index 533319c4ef..fa62227e0d 100644 --- a/runtime/permissions.rs +++ b/runtime/permissions.rs @@ -101,7 +101,7 @@ impl deno_permissions::PermissionDescriptorParser if text.is_empty() { Err(AnyError::msg("Empty sys not allowed")) } else { - Ok(SysDescriptor(text.to_string())) + Ok(SysDescriptor::parse(text.to_string())?) } } diff --git a/runtime/permissions/lib.rs b/runtime/permissions/lib.rs index e919b81b60..77038ff2fa 100644 --- a/runtime/permissions/lib.rs +++ b/runtime/permissions/lib.rs @@ -1356,7 +1356,23 @@ fn denies_run_name(name: &str, cmd_path: &Path) -> bool { } #[derive(Clone, Eq, PartialEq, Hash, Debug)] -pub struct SysDescriptor(pub String); +pub struct SysDescriptor(String); + +impl SysDescriptor { + pub fn parse(kind: String) -> Result { + match kind.as_str() { + "hostname" | "osRelease" | "osUptime" | "loadavg" + | "networkInterfaces" | "systemMemoryInfo" | "uid" | "gid" | "cpus" + | "homedir" | "getegid" | "username" | "statfs" | "getPriority" + | "setPriority" => Ok(Self(kind)), + _ => Err(type_error(format!("unknown system info kind \"{kind}\""))), + } + } + + pub fn into_string(self) -> String { + self.0 + } +} impl QueryDescriptor for SysDescriptor { type AllowDesc = SysDescriptor; @@ -1412,15 +1428,6 @@ impl QueryDescriptor for SysDescriptor { } } -pub fn parse_sys_kind(kind: &str) -> Result<&str, AnyError> { - match kind { - "hostname" | "osRelease" | "osUptime" | "loadavg" | "networkInterfaces" - | "systemMemoryInfo" | "uid" | "gid" | "cpus" | "homedir" | "getegid" - | "username" | "statfs" | "getPriority" | "setPriority" => Ok(kind), - _ => Err(type_error(format!("unknown system info kind \"{kind}\""))), - } -} - #[derive(Clone, Eq, PartialEq, Hash, Debug)] pub struct FfiQueryDescriptor(pub PathQueryDescriptor); @@ -1664,28 +1671,25 @@ impl UnaryPermission { } impl UnaryPermission { - pub fn query(&self, kind: Option<&str>) -> PermissionState { - self.query_desc( - kind.map(|k| SysDescriptor(k.to_string())).as_ref(), - AllowPartial::TreatAsPartialGranted, - ) + pub fn query(&self, kind: Option<&SysDescriptor>) -> PermissionState { + self.query_desc(kind, AllowPartial::TreatAsPartialGranted) } - pub fn request(&mut self, kind: Option<&str>) -> PermissionState { - self.request_desc(kind.map(|k| SysDescriptor(k.to_string())).as_ref()) + pub fn request(&mut self, kind: Option<&SysDescriptor>) -> PermissionState { + self.request_desc(kind) } - pub fn revoke(&mut self, kind: Option<&str>) -> PermissionState { - self.revoke_desc(kind.map(|k| SysDescriptor(k.to_string())).as_ref()) + pub fn revoke(&mut self, kind: Option<&SysDescriptor>) -> PermissionState { + self.revoke_desc(kind) } pub fn check( &mut self, - kind: &str, + kind: &SysDescriptor, api_name: Option<&str>, ) -> Result<(), AnyError> { skip_check_if_is_permission_fully_granted!(self); - self.check_desc(Some(&SysDescriptor(kind.to_string())), false, api_name) + self.check_desc(Some(kind), false, api_name) } pub fn check_all(&mut self) -> Result<(), AnyError> { @@ -2047,12 +2051,8 @@ pub enum CheckSpecifierKind { /// to send permissions to a new thread. #[derive(Clone, Debug)] pub struct PermissionsContainer { - // todo(dsherret): make both of these private as the functionality - // can just be methods on PermissionsContainer. Additionally, a separate - // struct should be created in here that handles creating child permissions - // so that the code is not so verbose elsewhere. - pub descriptor_parser: Arc, - pub inner: Arc>, + descriptor_parser: Arc, + inner: Arc>, } impl PermissionsContainer { @@ -2072,6 +2072,96 @@ impl PermissionsContainer { Self::new(descriptor_parser, Permissions::allow_all()) } + pub fn create_child_permissions( + &self, + child_permissions_arg: ChildPermissionsArg, + ) -> Result { + fn is_granted_unary(arg: &ChildUnaryPermissionArg) -> bool { + match arg { + ChildUnaryPermissionArg::Inherit | ChildUnaryPermissionArg::Granted => { + true + } + ChildUnaryPermissionArg::NotGranted + | ChildUnaryPermissionArg::GrantedList(_) => false, + } + } + + let mut worker_perms = Permissions::none_without_prompt(); + + let mut inner = self.inner.lock(); + worker_perms.all = inner + .all + .create_child_permissions(ChildUnitPermissionArg::Inherit)?; + + // downgrade the `worker_perms.all` based on the other values + if worker_perms.all.query() == PermissionState::Granted { + let unary_perms = [ + &child_permissions_arg.read, + &child_permissions_arg.write, + &child_permissions_arg.net, + &child_permissions_arg.import, + &child_permissions_arg.env, + &child_permissions_arg.sys, + &child_permissions_arg.run, + &child_permissions_arg.ffi, + ]; + let allow_all = unary_perms.into_iter().all(is_granted_unary); + if !allow_all { + worker_perms.all.revoke(); + } + } + + // WARNING: When adding a permission here, ensure it is handled + // in the worker_perms.all block above + worker_perms.read = inner + .read + .create_child_permissions(child_permissions_arg.read, |text| { + Ok(Some(self.descriptor_parser.parse_read_descriptor(text)?)) + })?; + worker_perms.write = inner + .write + .create_child_permissions(child_permissions_arg.write, |text| { + Ok(Some(self.descriptor_parser.parse_write_descriptor(text)?)) + })?; + worker_perms.import = inner + .import + .create_child_permissions(child_permissions_arg.import, |text| { + Ok(Some(self.descriptor_parser.parse_import_descriptor(text)?)) + })?; + worker_perms.net = inner + .net + .create_child_permissions(child_permissions_arg.net, |text| { + Ok(Some(self.descriptor_parser.parse_net_descriptor(text)?)) + })?; + worker_perms.env = inner + .env + .create_child_permissions(child_permissions_arg.env, |text| { + Ok(Some(self.descriptor_parser.parse_env_descriptor(text)?)) + })?; + worker_perms.sys = inner + .sys + .create_child_permissions(child_permissions_arg.sys, |text| { + Ok(Some(self.descriptor_parser.parse_sys_descriptor(text)?)) + })?; + worker_perms.run = inner.run.create_child_permissions( + child_permissions_arg.run, + |text| match self.descriptor_parser.parse_allow_run_descriptor(text)? { + AllowRunDescriptorParseResult::Unresolved(_) => Ok(None), + AllowRunDescriptorParseResult::Descriptor(desc) => Ok(Some(desc)), + }, + )?; + worker_perms.ffi = inner + .ffi + .create_child_permissions(child_permissions_arg.ffi, |text| { + Ok(Some(self.descriptor_parser.parse_ffi_descriptor(text)?)) + })?; + + Ok(PermissionsContainer::new( + self.descriptor_parser.clone(), + worker_perms, + )) + } + #[inline(always)] pub fn check_specifier( &self, @@ -2307,7 +2397,10 @@ impl PermissionsContainer { #[inline(always)] pub fn check_sys(&self, kind: &str, api_name: &str) -> Result<(), AnyError> { - self.inner.lock().sys.check(kind, Some(api_name)) + self.inner.lock().sys.check( + &self.descriptor_parser.parse_sys_descriptor(kind)?, + Some(api_name), + ) } #[inline(always)] @@ -2510,6 +2603,336 @@ impl PermissionsContainer { Ok(desc.0.resolved) } } + + // query + + #[inline(always)] + pub fn query_read( + &self, + path: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().read.query( + path + .map(|path| { + Result::<_, AnyError>::Ok( + self.descriptor_parser.parse_path_query(path)?.into_read(), + ) + }) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn query_write( + &self, + path: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().write.query( + path + .map(|path| { + Result::<_, AnyError>::Ok( + self.descriptor_parser.parse_path_query(path)?.into_write(), + ) + }) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn query_net( + &self, + host: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().net.query( + match host { + None => None, + Some(h) => Some(self.descriptor_parser.parse_net_descriptor(h)?), + } + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn query_env(&self, var: Option<&str>) -> PermissionState { + self.inner.lock().env.query(var) + } + + #[inline(always)] + pub fn query_sys( + &self, + kind: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().sys.query( + kind + .map(|kind| self.descriptor_parser.parse_sys_descriptor(kind)) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn query_run( + &self, + cmd: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().run.query( + cmd + .map(|request| self.descriptor_parser.parse_run_query(request)) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn query_ffi( + &self, + path: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().ffi.query( + path + .map(|path| { + Result::<_, AnyError>::Ok( + self.descriptor_parser.parse_path_query(path)?.into_ffi(), + ) + }) + .transpose()? + .as_ref(), + ), + ) + } + + // revoke + + #[inline(always)] + pub fn revoke_read( + &self, + path: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().read.revoke( + path + .map(|path| { + Result::<_, AnyError>::Ok( + self.descriptor_parser.parse_path_query(path)?.into_read(), + ) + }) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn revoke_write( + &self, + path: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().write.revoke( + path + .map(|path| { + Result::<_, AnyError>::Ok( + self.descriptor_parser.parse_path_query(path)?.into_write(), + ) + }) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn revoke_net( + &self, + host: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().net.revoke( + match host { + None => None, + Some(h) => Some(self.descriptor_parser.parse_net_descriptor(h)?), + } + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn revoke_env(&self, var: Option<&str>) -> PermissionState { + self.inner.lock().env.revoke(var) + } + + #[inline(always)] + pub fn revoke_sys( + &self, + kind: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().sys.revoke( + kind + .map(|kind| self.descriptor_parser.parse_sys_descriptor(kind)) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn revoke_run( + &self, + cmd: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().run.revoke( + cmd + .map(|request| self.descriptor_parser.parse_run_query(request)) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn revoke_ffi( + &self, + path: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().ffi.revoke( + path + .map(|path| { + Result::<_, AnyError>::Ok( + self.descriptor_parser.parse_path_query(path)?.into_ffi(), + ) + }) + .transpose()? + .as_ref(), + ), + ) + } + + // request + + #[inline(always)] + pub fn request_read( + &self, + path: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().read.request( + path + .map(|path| { + Result::<_, AnyError>::Ok( + self.descriptor_parser.parse_path_query(path)?.into_read(), + ) + }) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn request_write( + &self, + path: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().write.request( + path + .map(|path| { + Result::<_, AnyError>::Ok( + self.descriptor_parser.parse_path_query(path)?.into_write(), + ) + }) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn request_net( + &self, + host: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().net.request( + match host { + None => None, + Some(h) => Some(self.descriptor_parser.parse_net_descriptor(h)?), + } + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn request_env(&self, var: Option<&str>) -> PermissionState { + self.inner.lock().env.request(var) + } + + #[inline(always)] + pub fn request_sys( + &self, + kind: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().sys.request( + kind + .map(|kind| self.descriptor_parser.parse_sys_descriptor(kind)) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn request_run( + &self, + cmd: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().run.request( + cmd + .map(|request| self.descriptor_parser.parse_run_query(request)) + .transpose()? + .as_ref(), + ), + ) + } + + #[inline(always)] + pub fn request_ffi( + &self, + path: Option<&str>, + ) -> Result { + Ok( + self.inner.lock().ffi.request( + path + .map(|path| { + Result::<_, AnyError>::Ok( + self.descriptor_parser.parse_path_query(path)?.into_ffi(), + ) + }) + .transpose()? + .as_ref(), + ), + ) + } } const fn unit_permission_from_flag_bools( @@ -2862,93 +3285,6 @@ pub trait PermissionDescriptorParser: Debug + Send + Sync { ) -> Result; } -pub fn create_child_permissions( - parser: &dyn PermissionDescriptorParser, - main_perms: &mut Permissions, - child_permissions_arg: ChildPermissionsArg, -) -> Result { - fn is_granted_unary(arg: &ChildUnaryPermissionArg) -> bool { - match arg { - ChildUnaryPermissionArg::Inherit | ChildUnaryPermissionArg::Granted => { - true - } - ChildUnaryPermissionArg::NotGranted - | ChildUnaryPermissionArg::GrantedList(_) => false, - } - } - - let mut worker_perms = Permissions::none_without_prompt(); - - worker_perms.all = main_perms - .all - .create_child_permissions(ChildUnitPermissionArg::Inherit)?; - - // downgrade the `worker_perms.all` based on the other values - if worker_perms.all.query() == PermissionState::Granted { - let unary_perms = [ - &child_permissions_arg.read, - &child_permissions_arg.write, - &child_permissions_arg.net, - &child_permissions_arg.import, - &child_permissions_arg.env, - &child_permissions_arg.sys, - &child_permissions_arg.run, - &child_permissions_arg.ffi, - ]; - let allow_all = unary_perms.into_iter().all(is_granted_unary); - if !allow_all { - worker_perms.all.revoke(); - } - } - - // WARNING: When adding a permission here, ensure it is handled - // in the worker_perms.all block above - worker_perms.read = main_perms - .read - .create_child_permissions(child_permissions_arg.read, |text| { - Ok(Some(parser.parse_read_descriptor(text)?)) - })?; - worker_perms.write = main_perms - .write - .create_child_permissions(child_permissions_arg.write, |text| { - Ok(Some(parser.parse_write_descriptor(text)?)) - })?; - worker_perms.import = main_perms - .import - .create_child_permissions(child_permissions_arg.import, |text| { - Ok(Some(parser.parse_import_descriptor(text)?)) - })?; - worker_perms.net = main_perms - .net - .create_child_permissions(child_permissions_arg.net, |text| { - Ok(Some(parser.parse_net_descriptor(text)?)) - })?; - worker_perms.env = main_perms - .env - .create_child_permissions(child_permissions_arg.env, |text| { - Ok(Some(parser.parse_env_descriptor(text)?)) - })?; - worker_perms.sys = main_perms - .sys - .create_child_permissions(child_permissions_arg.sys, |text| { - Ok(Some(parser.parse_sys_descriptor(text)?)) - })?; - worker_perms.run = main_perms.run.create_child_permissions( - child_permissions_arg.run, - |text| match parser.parse_allow_run_descriptor(text)? { - AllowRunDescriptorParseResult::Unresolved(_) => Ok(None), - AllowRunDescriptorParseResult::Descriptor(desc) => Ok(Some(desc)), - }, - )?; - worker_perms.ffi = main_perms - .ffi - .create_child_permissions(child_permissions_arg.ffi, |text| { - Ok(Some(parser.parse_ffi_descriptor(text)?)) - })?; - - Ok(worker_perms) -} - static IS_STANDALONE: AtomicFlag = AtomicFlag::lowered(); pub fn mark_standalone() { @@ -2972,7 +3308,7 @@ mod tests { ($($x:expr),*) => (vec![$($x.to_string()),*]); } - #[derive(Debug)] + #[derive(Debug, Clone)] struct TestPermissionDescriptorParser; impl TestPermissionDescriptorParser { @@ -3025,7 +3361,7 @@ mod tests { &self, text: &str, ) -> Result { - Ok(SysDescriptor(text.to_string())) + SysDescriptor::parse(text.to_string()) } fn parse_allow_run_descriptor( @@ -3534,15 +3870,16 @@ mod tests { assert_eq!(perms4.env.query(None), PermissionState::GrantedPartial); assert_eq!(perms4.env.query(Some("HOME")), PermissionState::Denied); assert_eq!(perms4.env.query(Some("AWAY")), PermissionState::Granted); + let sys_desc = |name: &str| SysDescriptor::parse(name.to_string()).unwrap(); assert_eq!(perms1.sys.query(None), PermissionState::Granted); - assert_eq!(perms1.sys.query(Some("HOME")), PermissionState::Granted); + assert_eq!(perms1.sys.query(Some(&sys_desc("osRelease"))), PermissionState::Granted); assert_eq!(perms2.sys.query(None), PermissionState::Prompt); - assert_eq!(perms2.sys.query(Some("hostname")), PermissionState::Granted); + assert_eq!(perms2.sys.query(Some(&sys_desc("hostname"))), PermissionState::Granted); assert_eq!(perms3.sys.query(None), PermissionState::Prompt); - assert_eq!(perms3.sys.query(Some("hostname")), PermissionState::Denied); + assert_eq!(perms3.sys.query(Some(&sys_desc("hostname"))), PermissionState::Denied); assert_eq!(perms4.sys.query(None), PermissionState::GrantedPartial); - assert_eq!(perms4.sys.query(Some("hostname")), PermissionState::Denied); - assert_eq!(perms4.sys.query(Some("uid")), PermissionState::Granted); + assert_eq!(perms4.sys.query(Some(&sys_desc("hostname"))), PermissionState::Denied); + assert_eq!(perms4.sys.query(Some(&sys_desc("uid"))), PermissionState::Granted); assert_eq!(perms1.run.query(None), PermissionState::Granted); let deno_run_query = RunQueryDescriptor::Path { requested: "deno".to_string(), @@ -3604,10 +3941,11 @@ mod tests { prompt_value.set(false); assert_eq!(perms.env.request(Some("HOME")), PermissionState::Granted); prompt_value.set(true); - assert_eq!(perms.sys.request(Some("hostname")), PermissionState::Granted); + let sys_desc = |name: &str| SysDescriptor::parse(name.to_string()).unwrap(); + assert_eq!(perms.sys.request(Some(&sys_desc("hostname"))), PermissionState::Granted); assert_eq!(perms.sys.query(None), PermissionState::Prompt); prompt_value.set(false); - assert_eq!(perms.sys.request(Some("hostname")), PermissionState::Granted); + assert_eq!(perms.sys.request(Some(&sys_desc("hostname"))), PermissionState::Granted); prompt_value.set(true); let run_query = RunQueryDescriptor::Path { requested: "deno".to_string(), @@ -3924,12 +4262,13 @@ mod tests { assert!(perms.env.check("PATH", None).is_ok()); prompt_value.set(false); - assert!(perms.sys.check("hostname", None).is_err()); + let sys_desc = |name: &str| SysDescriptor::parse(name.to_string()).unwrap(); + assert!(perms.sys.check(&sys_desc("hostname"), None).is_err()); prompt_value.set(true); - assert!(perms.sys.check("hostname", None).is_err()); - assert!(perms.sys.check("osRelease", None).is_ok()); + assert!(perms.sys.check(&sys_desc("hostname"), None).is_err()); + assert!(perms.sys.check(&sys_desc("osRelease"), None).is_ok()); prompt_value.set(false); - assert!(perms.sys.check("osRelease", None).is_ok()); + assert!(perms.sys.check(&sys_desc("osRelease"), None).is_ok()); } #[test] @@ -4149,7 +4488,7 @@ mod tests { fn test_create_child_permissions() { set_prompter(Box::new(TestPrompter)); let parser = TestPermissionDescriptorParser; - let mut main_perms = Permissions::from_options( + let main_perms = Permissions::from_options( &parser, &PermissionsOptions { allow_env: Some(vec![]), @@ -4158,18 +4497,19 @@ mod tests { }, ) .unwrap(); + let main_perms = PermissionsContainer::new(Arc::new(parser), main_perms); assert_eq!( - create_child_permissions( - &parser, - &mut main_perms.clone(), - ChildPermissionsArg { + main_perms + .create_child_permissions(ChildPermissionsArg { env: ChildUnaryPermissionArg::Inherit, net: ChildUnaryPermissionArg::GrantedList(svec!["foo"]), ffi: ChildUnaryPermissionArg::NotGranted, ..ChildPermissionsArg::none() - } - ) - .unwrap(), + }) + .unwrap() + .inner + .lock() + .clone(), Permissions { env: Permissions::new_unary(Some(HashSet::new()), None, false).unwrap(), net: Permissions::new_unary( @@ -4181,40 +4521,31 @@ mod tests { ..Permissions::none_without_prompt() } ); - assert!(create_child_permissions( - &parser, - &mut main_perms.clone(), - ChildPermissionsArg { + assert!(main_perms + .create_child_permissions(ChildPermissionsArg { net: ChildUnaryPermissionArg::Granted, ..ChildPermissionsArg::none() - } - ) - .is_err()); - assert!(create_child_permissions( - &parser, - &mut main_perms.clone(), - ChildPermissionsArg { + }) + .is_err()); + assert!(main_perms + .create_child_permissions(ChildPermissionsArg { net: ChildUnaryPermissionArg::GrantedList(svec!["foo", "bar", "baz"]), ..ChildPermissionsArg::none() - } - ) - .is_err()); - assert!(create_child_permissions( - &parser, - &mut main_perms, - ChildPermissionsArg { + }) + .is_err()); + assert!(main_perms + .create_child_permissions(ChildPermissionsArg { ffi: ChildUnaryPermissionArg::GrantedList(svec!["foo"]), ..ChildPermissionsArg::none() - } - ) - .is_err()); + }) + .is_err()); } #[test] fn test_create_child_permissions_with_prompt() { set_prompter(Box::new(TestPrompter)); let prompt_value = PERMISSION_PROMPT_STUB_VALUE_SETTER.lock(); - let mut main_perms = Permissions::from_options( + let main_perms = Permissions::from_options( &TestPermissionDescriptorParser, &PermissionsOptions { prompt: true, @@ -4222,20 +4553,24 @@ mod tests { }, ) .unwrap(); + let main_perms = PermissionsContainer::new( + Arc::new(TestPermissionDescriptorParser), + main_perms, + ); prompt_value.set(true); - let worker_perms = create_child_permissions( - &TestPermissionDescriptorParser, - &mut main_perms, - ChildPermissionsArg { + let worker_perms = main_perms + .create_child_permissions(ChildPermissionsArg { read: ChildUnaryPermissionArg::Granted, run: ChildUnaryPermissionArg::GrantedList(svec!["foo", "bar"]), ..ChildPermissionsArg::none() - }, - ) - .unwrap(); - assert_eq!(main_perms, worker_perms); + }) + .unwrap(); assert_eq!( - main_perms.run.granted_list, + main_perms.inner.lock().clone(), + worker_perms.inner.lock().clone() + ); + assert_eq!( + main_perms.inner.lock().run.granted_list, HashSet::from([ AllowRunDescriptor(PathBuf::from("/bar")), AllowRunDescriptor(PathBuf::from("/foo")), @@ -4248,7 +4583,7 @@ mod tests { set_prompter(Box::new(TestPrompter)); let prompt_value = PERMISSION_PROMPT_STUB_VALUE_SETTER.lock(); let parser = TestPermissionDescriptorParser; - let mut main_perms = Permissions::from_options( + let main_perms = Permissions::from_options( &parser, &PermissionsOptions { prompt: true, @@ -4256,20 +4591,21 @@ mod tests { }, ) .unwrap(); + let main_perms = + PermissionsContainer::new(Arc::new(parser.clone()), main_perms); prompt_value.set(false); assert!(main_perms + .inner + .lock() .write .check(&parser.parse_path_query("foo").unwrap().into_write(), None) .is_err()); - let worker_perms = create_child_permissions( - &TestPermissionDescriptorParser, - &mut main_perms.clone(), - ChildPermissionsArg::none(), - ) - .unwrap(); + let worker_perms = main_perms + .create_child_permissions(ChildPermissionsArg::none()) + .unwrap(); assert_eq!( - worker_perms.write.flag_denied_list, - main_perms.write.flag_denied_list + worker_perms.inner.lock().write.flag_denied_list.clone(), + main_perms.inner.lock().write.flag_denied_list ); } diff --git a/runtime/snapshot.rs b/runtime/snapshot.rs index f20c04f082..456810e6a6 100644 --- a/runtime/snapshot.rs +++ b/runtime/snapshot.rs @@ -2,7 +2,6 @@ use crate::ops; use crate::ops::bootstrap::SnapshotOptions; -use crate::permissions::RuntimePermissionDescriptorParser; use crate::shared::maybe_transpile_source; use crate::shared::runtime; use deno_cache::SqliteBackedCache; @@ -299,9 +298,7 @@ pub fn create_runtime_snapshot( ), ops::fs_events::deno_fs_events::init_ops(), ops::os::deno_os::init_ops(Default::default()), - ops::permissions::deno_permissions::init_ops(Arc::new( - RuntimePermissionDescriptorParser::new(fs), - )), + ops::permissions::deno_permissions::init_ops(), ops::process::deno_process::init_ops(None), ops::signal::deno_signal::init_ops(), ops::tty::deno_tty::init_ops(), diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index f13516268f..f560ce17ec 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -1,16 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use crate::inspector_server::InspectorServer; -use crate::ops; -use crate::ops::process::NpmProcessStateProviderRc; -use crate::ops::worker_host::WorkersTable; -use crate::shared::maybe_transpile_source; -use crate::shared::runtime; -use crate::tokio_util::create_and_run_current_thread; -use crate::worker::create_op_metrics; -use crate::worker::import_meta_resolve_callback; -use crate::worker::validate_import_attributes_callback; -use crate::worker::FormatJsErrorFn; -use crate::BootstrapOptions; + use deno_broadcast_channel::InMemoryBroadcastChannel; use deno_cache::CreateCache; use deno_cache::SqliteBackedCache; @@ -45,7 +34,6 @@ use deno_http::DefaultHttpPropertyExtractor; use deno_io::Stdio; use deno_kv::dynamic::MultiBackendDbHandler; use deno_node::NodeExtInitServices; -use deno_permissions::PermissionDescriptorParser; use deno_permissions::PermissionsContainer; use deno_terminal::colors; use deno_tls::RootCertStoreProvider; @@ -67,6 +55,19 @@ use std::sync::Arc; use std::task::Context; use std::task::Poll; +use crate::inspector_server::InspectorServer; +use crate::ops; +use crate::ops::process::NpmProcessStateProviderRc; +use crate::ops::worker_host::WorkersTable; +use crate::shared::maybe_transpile_source; +use crate::shared::runtime; +use crate::tokio_util::create_and_run_current_thread; +use crate::worker::create_op_metrics; +use crate::worker::import_meta_resolve_callback; +use crate::worker::validate_import_attributes_callback; +use crate::worker::FormatJsErrorFn; +use crate::BootstrapOptions; + pub struct WorkerMetadata { pub buffer: DetachedBuffer, pub transferables: Vec, @@ -348,7 +349,6 @@ pub struct WebWorkerServiceOptions { pub node_services: Option, pub npm_process_state_provider: Option, pub permissions: PermissionsContainer, - pub permission_desc_parser: Arc, pub root_cert_store_provider: Option>, pub shared_array_buffer_store: Option, } @@ -505,9 +505,7 @@ impl WebWorker { ), ops::fs_events::deno_fs_events::init_ops_and_esm(), ops::os::deno_os_worker::init_ops_and_esm(), - ops::permissions::deno_permissions::init_ops_and_esm( - services.permission_desc_parser, - ), + ops::permissions::deno_permissions::init_ops_and_esm(), ops::process::deno_process::init_ops_and_esm( services.npm_process_state_provider, ), diff --git a/runtime/worker.rs b/runtime/worker.rs index 3ce5562fa2..477d3b880c 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -139,8 +139,6 @@ pub struct WorkerServiceOptions { pub module_loader: Rc, pub node_services: Option, pub npm_process_state_provider: Option, - pub permission_desc_parser: - Arc, pub permissions: PermissionsContainer, pub root_cert_store_provider: Option>, @@ -422,9 +420,7 @@ impl MainWorker { ), ops::fs_events::deno_fs_events::init_ops_and_esm(), ops::os::deno_os::init_ops_and_esm(exit_code.clone()), - ops::permissions::deno_permissions::init_ops_and_esm( - services.permission_desc_parser, - ), + ops::permissions::deno_permissions::init_ops_and_esm(), ops::process::deno_process::init_ops_and_esm( services.npm_process_state_provider, ), From 69ab72002550b5797185b7651de28c700b220bb2 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 30 Sep 2024 09:33:32 -0400 Subject: [PATCH 008/302] refactor: move ByonmNpmResolver to deno_resolver (#25937) Some more slow progress on moving all the resolution code into deno_resolver. --- .dprint.json | 2 +- Cargo.lock | 10 +- Cargo.toml | 2 +- cli/factory.rs | 7 +- cli/lsp/resolver.rs | 7 +- cli/npm/byonm.rs | 351 ++-------------------------- cli/npm/managed/mod.rs | 5 +- cli/npm/managed/resolvers/common.rs | 2 +- cli/npm/managed/resolvers/global.rs | 2 +- cli/npm/managed/resolvers/local.rs | 20 +- cli/npm/managed/resolvers/mod.rs | 1 - cli/npm/mod.rs | 18 +- cli/resolver.rs | 40 ++++ cli/standalone/mod.rs | 7 +- cli/tools/task.rs | 4 +- cli/util/fs.rs | 35 +-- resolvers/deno/Cargo.toml | 5 + resolvers/deno/clippy.toml | 52 +++++ resolvers/deno/fs.rs | 27 +++ resolvers/deno/lib.rs | 2 + resolvers/deno/npm/byonm.rs | 348 +++++++++++++++++++++++++++ resolvers/deno/npm/local.rs | 27 +++ resolvers/deno/npm/mod.rs | 8 + resolvers/deno/sloppy_imports.rs | 4 +- resolvers/node/Cargo.toml | 1 + resolvers/node/analyze.rs | 40 ++-- resolvers/node/clippy.toml | 3 + resolvers/node/npm.rs | 6 +- resolvers/node/path.rs | 98 -------- resolvers/node/resolution.rs | 50 ++-- 30 files changed, 629 insertions(+), 555 deletions(-) create mode 100644 resolvers/deno/clippy.toml create mode 100644 resolvers/deno/fs.rs create mode 100644 resolvers/deno/npm/byonm.rs create mode 100644 resolvers/deno/npm/local.rs create mode 100644 resolvers/deno/npm/mod.rs diff --git a/.dprint.json b/.dprint.json index 6ee3db3c5d..082eb30058 100644 --- a/.dprint.json +++ b/.dprint.json @@ -71,7 +71,7 @@ "https://plugins.dprint.dev/typescript-0.93.0.wasm", "https://plugins.dprint.dev/json-0.19.3.wasm", "https://plugins.dprint.dev/markdown-0.17.8.wasm", - "https://plugins.dprint.dev/toml-0.6.2.wasm", + "https://plugins.dprint.dev/toml-0.6.3.wasm", "https://plugins.dprint.dev/exec-0.5.0.json@8d9972eee71fa1590e04873540421f3eda7674d0f1aae3d7c788615e7b7413d0", "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.5.0.wasm" ] diff --git a/Cargo.lock b/Cargo.lock index b4dafacdd2..2263351fc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1923,9 +1923,9 @@ dependencies = [ [[package]] name = "deno_path_util" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625b490bc4734ff778926d7034650a2ddcef2fd42b71e30bb722249156d5a144" +checksum = "4889646c1ce8437a6fde3acb057fd7e2d039e62c61f5063fc125ed1ede114dc6" dependencies = [ "percent-encoding", "thiserror", @@ -1953,8 +1953,13 @@ dependencies = [ name = "deno_resolver" version = "0.0.1" dependencies = [ + "anyhow", + "base32", "deno_media_type", + "deno_package_json", "deno_path_util", + "deno_semver", + "node_resolver", "test_server", "url", ] @@ -4557,6 +4562,7 @@ dependencies = [ "async-trait", "deno_media_type", "deno_package_json", + "deno_path_util", "futures", "lazy-regex", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 2f53c8999d..ef9a4576ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ deno_bench_util = { version = "0.162.0", path = "./bench_util" } deno_lockfile = "=0.23.1" deno_media_type = { version = "0.1.4", features = ["module_specifier"] } deno_npm = "=0.25.2" -deno_path_util = "=0.1.1" +deno_path_util = "=0.2.0" deno_permissions = { version = "0.28.0", path = "./runtime/permissions" } deno_runtime = { version = "0.177.0", path = "./runtime" } deno_semver = "=0.5.13" diff --git a/cli/factory.rs b/cli/factory.rs index ffe7f8d2f7..2cef875999 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -32,12 +32,13 @@ use crate::module_loader::ModuleLoadPreparer; use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::npm::create_cli_npm_resolver; +use crate::npm::CliByonmNpmResolverCreateOptions; use crate::npm::CliNpmResolver; -use crate::npm::CliNpmResolverByonmCreateOptions; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::resolver::CjsResolutionStore; +use crate::resolver::CliDenoResolverFs; use crate::resolver::CliGraphResolver; use crate::resolver::CliGraphResolverOptions; use crate::resolver::CliNodeResolver; @@ -361,8 +362,8 @@ impl CliFactory { let cli_options = self.cli_options()?; // For `deno install` we want to force the managed resolver so it can set up `node_modules/` directory. create_cli_npm_resolver(if cli_options.use_byonm() && !matches!(cli_options.sub_command(), DenoSubcommand::Install(_) | DenoSubcommand::Add(_) | DenoSubcommand::Remove(_)) { - CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { - fs: fs.clone(), + CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(fs.clone()), root_node_modules_dir: Some(match cli_options.node_modules_dir_path() { Some(node_modules_path) => node_modules_path.to_path_buf(), // path needs to be canonicalized for node resolution diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index f98b23a3f9..a5dfd25a64 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -42,13 +42,14 @@ use crate::lsp::config::Config; use crate::lsp::config::ConfigData; use crate::lsp::logging::lsp_warn; use crate::npm::create_cli_npm_resolver_for_lsp; +use crate::npm::CliByonmNpmResolverCreateOptions; use crate::npm::CliNpmResolver; -use crate::npm::CliNpmResolverByonmCreateOptions; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::npm::ManagedCliNpmResolver; use crate::resolver::CjsResolutionStore; +use crate::resolver::CliDenoResolverFs; use crate::resolver::CliGraphResolver; use crate::resolver::CliGraphResolverOptions; use crate::resolver::CliNodeResolver; @@ -439,8 +440,8 @@ async fn create_npm_resolver( ) -> Option> { let enable_byonm = config_data.map(|d| d.byonm).unwrap_or(false); let options = if enable_byonm { - CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { - fs: Arc::new(deno_fs::RealFs), + CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(Arc::new(deno_fs::RealFs)), root_node_modules_dir: config_data.and_then(|config_data| { config_data.node_modules_dir.clone().or_else(|| { url_to_file_path(&config_data.scope) diff --git a/cli/npm/byonm.rs b/cli/npm/byonm.rs index 83c4067655..ceef68135a 100644 --- a/cli/npm/byonm.rs +++ b/cli/npm/byonm.rs @@ -1,276 +1,36 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_core::serde_json; -use deno_package_json::PackageJsonDepValue; -use deno_path_util::url_to_file_path; -use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::DenoPkgJsonFsAdapter; +use deno_core::url::Url; +use deno_resolver::npm::ByonmNpmResolver; +use deno_resolver::npm::ByonmNpmResolverCreateOptions; use deno_runtime::deno_node::NodePermissions; use deno_runtime::deno_node::NodeRequireResolver; -use deno_runtime::deno_node::PackageJson; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageReq; -use deno_semver::Version; -use node_resolver::errors::PackageFolderResolveError; -use node_resolver::errors::PackageFolderResolveIoError; -use node_resolver::errors::PackageJsonLoadError; -use node_resolver::errors::PackageNotFoundError; -use node_resolver::load_pkg_json; use node_resolver::NpmResolver; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; +use crate::resolver::CliDenoResolverFs; -use super::managed::normalize_pkg_name_for_node_modules_deno_folder; use super::CliNpmResolver; use super::InnerCliNpmResolverRef; -pub struct CliNpmResolverByonmCreateOptions { - pub fs: Arc, - // todo(dsherret): investigate removing this - pub root_node_modules_dir: Option, -} - -pub fn create_byonm_npm_resolver( - options: CliNpmResolverByonmCreateOptions, -) -> Arc { - Arc::new(ByonmCliNpmResolver { - fs: options.fs, - root_node_modules_dir: options.root_node_modules_dir, - }) -} +pub type CliByonmNpmResolverCreateOptions = + ByonmNpmResolverCreateOptions; +pub type CliByonmNpmResolver = ByonmNpmResolver; +// todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple. #[derive(Debug)] -pub struct ByonmCliNpmResolver { - fs: Arc, - root_node_modules_dir: Option, -} +struct CliByonmWrapper(Arc); -impl ByonmCliNpmResolver { - fn load_pkg_json( - &self, - path: &Path, - ) -> Result>, PackageJsonLoadError> { - load_pkg_json(&DenoPkgJsonFsAdapter(self.fs.as_ref()), path) - } - - /// Finds the ancestor package.json that contains the specified dependency. - pub fn find_ancestor_package_json_with_dep( - &self, - dep_name: &str, - referrer: &ModuleSpecifier, - ) -> Option> { - let referrer_path = referrer.to_file_path().ok()?; - let mut current_folder = referrer_path.parent()?; - loop { - let pkg_json_path = current_folder.join("package.json"); - if let Ok(Some(pkg_json)) = self.load_pkg_json(&pkg_json_path) { - if let Some(deps) = &pkg_json.dependencies { - if deps.contains_key(dep_name) { - return Some(pkg_json); - } - } - if let Some(deps) = &pkg_json.dev_dependencies { - if deps.contains_key(dep_name) { - return Some(pkg_json); - } - } - } - - if let Some(parent) = current_folder.parent() { - current_folder = parent; - } else { - return None; - } - } - } - - fn resolve_pkg_json_and_alias_for_req( - &self, - req: &PackageReq, - referrer: &ModuleSpecifier, - ) -> Result, String)>, AnyError> { - fn resolve_alias_from_pkg_json( - req: &PackageReq, - pkg_json: &PackageJson, - ) -> Option { - let deps = pkg_json.resolve_local_package_json_deps(); - for (key, value) in deps { - if let Ok(value) = value { - match value { - PackageJsonDepValue::Req(dep_req) => { - if dep_req.name == req.name - && dep_req.version_req.intersects(&req.version_req) - { - return Some(key); - } - } - PackageJsonDepValue::Workspace(_workspace) => { - if key == req.name && req.version_req.tag() == Some("workspace") { - return Some(key); - } - } - } - } - } - None - } - - // attempt to resolve the npm specifier from the referrer's package.json, - if let Ok(file_path) = url_to_file_path(referrer) { - let mut current_path = file_path.as_path(); - while let Some(dir_path) = current_path.parent() { - let package_json_path = dir_path.join("package.json"); - if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? { - if let Some(alias) = - resolve_alias_from_pkg_json(req, pkg_json.as_ref()) - { - return Ok(Some((pkg_json, alias))); - } - } - current_path = dir_path; - } - } - - // otherwise, fall fallback to the project's package.json - if let Some(root_node_modules_dir) = &self.root_node_modules_dir { - let root_pkg_json_path = - root_node_modules_dir.parent().unwrap().join("package.json"); - if let Some(pkg_json) = self.load_pkg_json(&root_pkg_json_path)? { - if let Some(alias) = resolve_alias_from_pkg_json(req, pkg_json.as_ref()) - { - return Ok(Some((pkg_json, alias))); - } - } - } - - Ok(None) - } - - fn resolve_folder_in_root_node_modules( - &self, - req: &PackageReq, - ) -> Option { - // now check if node_modules/.deno/ matches this constraint - let root_node_modules_dir = self.root_node_modules_dir.as_ref()?; - let node_modules_deno_dir = root_node_modules_dir.join(".deno"); - let Ok(entries) = self.fs.read_dir_sync(&node_modules_deno_dir) else { - return None; - }; - let search_prefix = format!( - "{}@", - normalize_pkg_name_for_node_modules_deno_folder(&req.name) - ); - let mut best_version = None; - - // example entries: - // - @denotest+add@1.0.0 - // - @denotest+add@1.0.0_1 - for entry in entries { - if !entry.is_directory { - continue; - } - let Some(version_and_copy_idx) = entry.name.strip_prefix(&search_prefix) - else { - continue; - }; - let version = version_and_copy_idx - .rsplit_once('_') - .map(|(v, _)| v) - .unwrap_or(version_and_copy_idx); - let Ok(version) = Version::parse_from_npm(version) else { - continue; - }; - if req.version_req.matches(&version) { - if let Some((best_version_version, _)) = &best_version { - if version > *best_version_version { - best_version = Some((version, entry.name)); - } - } else { - best_version = Some((version, entry.name)); - } - } - } - - best_version.map(|(_version, entry_name)| { - join_package_name( - &node_modules_deno_dir.join(entry_name).join("node_modules"), - &req.name, - ) - }) - } -} - -impl NpmResolver for ByonmCliNpmResolver { - fn resolve_package_folder_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - ) -> Result { - fn inner( - fs: &dyn FileSystem, - name: &str, - referrer: &ModuleSpecifier, - ) -> Result { - let maybe_referrer_file = url_to_file_path(referrer).ok(); - let maybe_start_folder = - maybe_referrer_file.as_ref().and_then(|f| f.parent()); - if let Some(start_folder) = maybe_start_folder { - for current_folder in start_folder.ancestors() { - let node_modules_folder = if current_folder.ends_with("node_modules") - { - Cow::Borrowed(current_folder) - } else { - Cow::Owned(current_folder.join("node_modules")) - }; - - let sub_dir = join_package_name(&node_modules_folder, name); - if fs.is_dir_sync(&sub_dir) { - return Ok(sub_dir); - } - } - } - - Err( - PackageNotFoundError { - package_name: name.to_string(), - referrer: referrer.clone(), - referrer_extra: None, - } - .into(), - ) - } - - let path = inner(&*self.fs, name, referrer)?; - self.fs.realpath_sync(&path).map_err(|err| { - PackageFolderResolveIoError { - package_name: name.to_string(), - referrer: referrer.clone(), - source: err.into_io_error(), - } - .into() - }) - } - - fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - specifier.scheme() == "file" - && specifier - .path() - .to_ascii_lowercase() - .contains("/node_modules/") - } -} - -impl NodeRequireResolver for ByonmCliNpmResolver { +impl NodeRequireResolver for CliByonmWrapper { fn ensure_read_permission( &self, permissions: &mut dyn NodePermissions, @@ -286,110 +46,54 @@ impl NodeRequireResolver for ByonmCliNpmResolver { } } -impl NpmProcessStateProvider for ByonmCliNpmResolver { +impl NpmProcessStateProvider for CliByonmWrapper { fn get_npm_process_state(&self) -> String { serde_json::to_string(&NpmProcessState { kind: NpmProcessStateKind::Byonm, local_node_modules_path: self - .root_node_modules_dir - .as_ref() + .0 + .root_node_modules_dir() .map(|p| p.to_string_lossy().to_string()), }) .unwrap() } } -impl CliNpmResolver for ByonmCliNpmResolver { +impl CliNpmResolver for CliByonmNpmResolver { fn into_npm_resolver(self: Arc) -> Arc { self } fn into_require_resolver(self: Arc) -> Arc { - self + Arc::new(CliByonmWrapper(self)) } fn into_process_state_provider( self: Arc, ) -> Arc { - self + Arc::new(CliByonmWrapper(self)) } fn clone_snapshotted(&self) -> Arc { - Arc::new(Self { - fs: self.fs.clone(), - root_node_modules_dir: self.root_node_modules_dir.clone(), - }) + Arc::new(self.clone()) } fn as_inner(&self) -> InnerCliNpmResolverRef { InnerCliNpmResolverRef::Byonm(self) } - fn root_node_modules_path(&self) -> Option<&PathBuf> { - self.root_node_modules_dir.as_ref() + fn root_node_modules_path(&self) -> Option<&Path> { + self.root_node_modules_dir() } fn resolve_pkg_folder_from_deno_module_req( &self, req: &PackageReq, - referrer: &ModuleSpecifier, + referrer: &Url, ) -> Result { - fn node_resolve_dir( - fs: &dyn FileSystem, - alias: &str, - start_dir: &Path, - ) -> Result, AnyError> { - for ancestor in start_dir.ancestors() { - let node_modules_folder = ancestor.join("node_modules"); - let sub_dir = join_package_name(&node_modules_folder, alias); - if fs.is_dir_sync(&sub_dir) { - return Ok(Some(canonicalize_path_maybe_not_exists_with_fs( - &sub_dir, fs, - )?)); - } - } - Ok(None) - } - - // now attempt to resolve if it's found in any package.json - let maybe_pkg_json_and_alias = - self.resolve_pkg_json_and_alias_for_req(req, referrer)?; - match maybe_pkg_json_and_alias { - Some((pkg_json, alias)) => { - // now try node resolution - if let Some(resolved) = - node_resolve_dir(self.fs.as_ref(), &alias, pkg_json.dir_path())? - { - return Ok(resolved); - } - - bail!( - concat!( - "Could not find \"{}\" in a node_modules folder. ", - "Deno expects the node_modules/ directory to be up to date. ", - "Did you forget to run `deno install`?" - ), - alias, - ); - } - None => { - // now check if node_modules/.deno/ matches this constraint - if let Some(folder) = self.resolve_folder_in_root_node_modules(req) { - return Ok(folder); - } - - bail!( - concat!( - "Could not find a matching package for 'npm:{}' in the node_modules ", - "directory. Ensure you have all your JSR and npm dependencies listed ", - "in your deno.json or package.json, then run `deno install`. Alternatively, ", - r#"turn on auto-install by specifying `"nodeModulesDir": "auto"` in your "#, - "deno.json file." - ), - req, - ); - } - } + ByonmNpmResolver::resolve_pkg_folder_from_deno_module_req( + self, req, referrer, + ) } fn check_state_hash(&self) -> Option { @@ -398,12 +102,3 @@ impl CliNpmResolver for ByonmCliNpmResolver { None } } - -fn join_package_name(path: &Path, package_name: &str) -> PathBuf { - let mut path = path.to_path_buf(); - // ensure backslashes are used on windows - for part in package_name.split('/') { - path = path.join(part); - } - path -} diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index e3ac5e1af6..62af3e4aa9 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -47,7 +47,6 @@ use self::cache::NpmCache; use self::registry::CliNpmRegistryApi; use self::resolution::NpmResolution; use self::resolvers::create_npm_fs_resolver; -pub use self::resolvers::normalize_pkg_name_for_node_modules_deno_folder; use self::resolvers::NpmPackageFsResolver; use super::CliNpmResolver; @@ -575,7 +574,7 @@ impl NpmProcessStateProvider for ManagedCliNpmResolver { fn get_npm_process_state(&self) -> String { npm_process_state( self.resolution.serialized_valid_snapshot(), - self.fs_resolver.node_modules_path().map(|p| p.as_path()), + self.fs_resolver.node_modules_path(), ) } } @@ -632,7 +631,7 @@ impl CliNpmResolver for ManagedCliNpmResolver { InnerCliNpmResolverRef::Managed(self) } - fn root_node_modules_path(&self) -> Option<&PathBuf> { + fn root_node_modules_path(&self) -> Option<&Path> { self.fs_resolver.node_modules_path() } diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs index 620daf4b30..8df4debc5c 100644 --- a/cli/npm/managed/resolvers/common.rs +++ b/cli/npm/managed/resolvers/common.rs @@ -33,7 +33,7 @@ pub trait NpmPackageFsResolver: Send + Sync { fn root_dir_url(&self) -> &Url; /// The local node_modules folder if it is applicable to the implementation. - fn node_modules_path(&self) -> Option<&PathBuf>; + fn node_modules_path(&self) -> Option<&Path>; fn maybe_package_folder(&self, package_id: &NpmPackageId) -> Option; diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs index 187e6b2774..ca5722c869 100644 --- a/cli/npm/managed/resolvers/global.rs +++ b/cli/npm/managed/resolvers/global.rs @@ -73,7 +73,7 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver { self.cache.root_dir_url() } - fn node_modules_path(&self) -> Option<&PathBuf> { + fn node_modules_path(&self) -> Option<&Path> { None } diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index 297fcab230..59ba27d059 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -20,7 +20,6 @@ use crate::colors; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_cache_dir::npm::mixed_case_package_name_decode; -use deno_cache_dir::npm::mixed_case_package_name_encode; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; @@ -32,6 +31,7 @@ use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; +use deno_resolver::npm::normalize_pkg_name_for_node_modules_deno_folder; use deno_runtime::deno_fs; use deno_runtime::deno_node::NodePermissions; use deno_semver::package::PackageNv; @@ -159,8 +159,8 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver { &self.root_node_modules_url } - fn node_modules_path(&self) -> Option<&PathBuf> { - Some(&self.root_node_modules_path) + fn node_modules_path(&self) -> Option<&Path> { + Some(self.root_node_modules_path.as_ref()) } fn maybe_package_folder(&self, id: &NpmPackageId) -> Option { @@ -920,20 +920,6 @@ impl SetupCache { } } -/// Normalizes a package name for use at `node_modules/.deno/@[_]` -pub fn normalize_pkg_name_for_node_modules_deno_folder(name: &str) -> Cow { - let name = if name.to_lowercase() == name { - Cow::Borrowed(name) - } else { - Cow::Owned(format!("_{}", mixed_case_package_name_encode(name))) - }; - if name.starts_with('@') { - name.replace('/', "+").into() - } else { - name - } -} - fn get_package_folder_id_folder_name( folder_id: &NpmPackageCacheFolderId, ) -> String { diff --git a/cli/npm/managed/resolvers/mod.rs b/cli/npm/managed/resolvers/mod.rs index 234a6e4dba..36d795ee7e 100644 --- a/cli/npm/managed/resolvers/mod.rs +++ b/cli/npm/managed/resolvers/mod.rs @@ -15,7 +15,6 @@ use crate::args::NpmInstallDepsProvider; use crate::util::progress_bar::ProgressBar; pub use self::common::NpmPackageFsResolver; -pub use self::local::normalize_pkg_name_for_node_modules_deno_folder; use self::global::GlobalNpmPackageResolver; use self::local::LocalNpmPackageResolver; diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 2c9ee20bc8..1e3c752aee 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -4,6 +4,7 @@ mod byonm; mod common; mod managed; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -12,6 +13,7 @@ use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_core::serde_json; use deno_npm::registry::NpmPackageInfo; +use deno_resolver::npm::ByonmNpmResolver; use deno_runtime::deno_node::NodeRequireResolver; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; @@ -21,15 +23,15 @@ use node_resolver::NpmResolver; use crate::args::npm_registry_url; use crate::file_fetcher::FileFetcher; -pub use self::byonm::ByonmCliNpmResolver; -pub use self::byonm::CliNpmResolverByonmCreateOptions; +pub use self::byonm::CliByonmNpmResolver; +pub use self::byonm::CliByonmNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::ManagedCliNpmResolver; pub enum CliNpmResolverCreateOptions { Managed(CliNpmResolverManagedCreateOptions), - Byonm(CliNpmResolverByonmCreateOptions), + Byonm(CliByonmNpmResolverCreateOptions), } pub async fn create_cli_npm_resolver_for_lsp( @@ -40,7 +42,7 @@ pub async fn create_cli_npm_resolver_for_lsp( Managed(options) => { managed::create_managed_npm_resolver_for_lsp(options).await } - Byonm(options) => byonm::create_byonm_npm_resolver(options), + Byonm(options) => Arc::new(ByonmNpmResolver::new(options)), } } @@ -50,14 +52,14 @@ pub async fn create_cli_npm_resolver( use CliNpmResolverCreateOptions::*; match options { Managed(options) => managed::create_managed_npm_resolver(options).await, - Byonm(options) => Ok(byonm::create_byonm_npm_resolver(options)), + Byonm(options) => Ok(Arc::new(ByonmNpmResolver::new(options))), } } pub enum InnerCliNpmResolverRef<'a> { Managed(&'a ManagedCliNpmResolver), #[allow(dead_code)] - Byonm(&'a ByonmCliNpmResolver), + Byonm(&'a CliByonmNpmResolver), } pub trait CliNpmResolver: NpmResolver { @@ -78,14 +80,14 @@ pub trait CliNpmResolver: NpmResolver { } } - fn as_byonm(&self) -> Option<&ByonmCliNpmResolver> { + fn as_byonm(&self) -> Option<&CliByonmNpmResolver> { match self.as_inner() { InnerCliNpmResolverRef::Managed(_) => None, InnerCliNpmResolverRef::Byonm(inner) => Some(inner), } } - fn root_node_modules_path(&self) -> Option<&PathBuf>; + fn root_node_modules_path(&self) -> Option<&Path>; fn resolve_pkg_folder_from_deno_module_req( &self, diff --git a/cli/resolver.rs b/cli/resolver.rs index 211f8aba1c..7804261b8b 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -61,6 +61,46 @@ pub struct ModuleCodeStringSource { pub media_type: MediaType, } +#[derive(Debug, Clone)] +pub struct CliDenoResolverFs(pub Arc); + +impl deno_resolver::fs::DenoResolverFs for CliDenoResolverFs { + fn read_to_string_lossy(&self, path: &Path) -> std::io::Result { + self + .0 + .read_text_file_lossy_sync(path, None) + .map_err(|e| e.into_io_error()) + } + + fn realpath_sync(&self, path: &Path) -> std::io::Result { + self.0.realpath_sync(path).map_err(|e| e.into_io_error()) + } + + fn is_dir_sync(&self, path: &Path) -> bool { + self.0.is_dir_sync(path) + } + + fn read_dir_sync( + &self, + dir_path: &Path, + ) -> std::io::Result> { + self + .0 + .read_dir_sync(dir_path) + .map(|entries| { + entries + .into_iter() + .map(|e| deno_resolver::fs::DirEntry { + name: e.name, + is_file: e.is_file, + is_directory: e.is_directory, + }) + .collect::>() + }) + .map_err(|err| err.into_io_error()) + } +} + #[derive(Debug)] pub struct CliNodeResolver { cjs_resolutions: Arc, diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 56f76ffb9b..258de0dad3 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -60,11 +60,12 @@ use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClientProvider; use crate::node::CliCjsCodeAnalyzer; use crate::npm::create_cli_npm_resolver; -use crate::npm::CliNpmResolverByonmCreateOptions; +use crate::npm::CliByonmNpmResolverCreateOptions; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::resolver::CjsResolutionStore; +use crate::resolver::CliDenoResolverFs; use crate::resolver::CliNodeResolver; use crate::resolver::NpmModuleLoader; use crate::util::progress_bar::ProgressBar; @@ -530,8 +531,8 @@ pub async fn run( let fs = Arc::new(DenoCompileFileSystem::new(vfs)) as Arc; let npm_resolver = create_cli_npm_resolver( - CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { - fs: fs.clone(), + CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(fs.clone()), root_node_modules_dir, }), ) diff --git a/cli/tools/task.rs b/cli/tools/task.rs index a5a5027d0f..464b65d984 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -190,9 +190,7 @@ async fn run_task(opts: RunTaskOptions<'_>) -> Result { custom_commands, init_cwd: opts.cli_options.initial_cwd(), argv: cli_options.argv(), - root_node_modules_dir: npm_resolver - .root_node_modules_path() - .map(|p| p.as_path()), + root_node_modules_dir: npm_resolver.root_node_modules_path(), }) .await } diff --git a/cli/util/fs.rs b/cli/util/fs.rs index 9734d417e5..a021ec19c0 100644 --- a/cli/util/fs.rs +++ b/cli/util/fs.rs @@ -20,7 +20,6 @@ use deno_core::error::AnyError; use deno_core::unsync::spawn_blocking; use deno_core::ModuleSpecifier; use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::PathClean; use crate::util::path::get_atomic_file_path; use crate::util::progress_bar::ProgressBar; @@ -290,48 +289,18 @@ pub fn canonicalize_path(path: &Path) -> Result { pub fn canonicalize_path_maybe_not_exists( path: &Path, ) -> Result { - canonicalize_path_maybe_not_exists_with_custom_fn(path, canonicalize_path) + deno_path_util::canonicalize_path_maybe_not_exists(path, &canonicalize_path) } pub fn canonicalize_path_maybe_not_exists_with_fs( path: &Path, fs: &dyn FileSystem, ) -> Result { - canonicalize_path_maybe_not_exists_with_custom_fn(path, |path| { + deno_path_util::canonicalize_path_maybe_not_exists(path, &|path| { fs.realpath_sync(path).map_err(|err| err.into_io_error()) }) } -fn canonicalize_path_maybe_not_exists_with_custom_fn( - path: &Path, - canonicalize: impl Fn(&Path) -> Result, -) -> Result { - let path = path.to_path_buf().clean(); - let mut path = path.as_path(); - let mut names_stack = Vec::new(); - loop { - match canonicalize(path) { - Ok(mut canonicalized_path) => { - for name in names_stack.into_iter().rev() { - canonicalized_path = canonicalized_path.join(name); - } - return Ok(canonicalized_path); - } - Err(err) if err.kind() == ErrorKind::NotFound => { - names_stack.push(match path.file_name() { - Some(name) => name.to_owned(), - None => return Err(err), - }); - path = match path.parent() { - Some(parent) => parent, - None => return Err(err), - }; - } - Err(err) => return Err(err), - } - } -} - /// Collects module specifiers that satisfy the given predicate as a file path, by recursively walking `include`. /// Specifiers that start with http and https are left intact. /// Note: This ignores all .git and node_modules folders. diff --git a/resolvers/deno/Cargo.toml b/resolvers/deno/Cargo.toml index 23c43810a0..a14635c38b 100644 --- a/resolvers/deno/Cargo.toml +++ b/resolvers/deno/Cargo.toml @@ -16,8 +16,13 @@ path = "lib.rs" [features] [dependencies] +anyhow.workspace = true +base32.workspace = true deno_media_type.workspace = true +deno_package_json.workspace = true deno_path_util.workspace = true +deno_semver.workspace = true +node_resolver.workspace = true url.workspace = true [dev-dependencies] diff --git a/resolvers/deno/clippy.toml b/resolvers/deno/clippy.toml new file mode 100644 index 0000000000..733ac83da1 --- /dev/null +++ b/resolvers/deno/clippy.toml @@ -0,0 +1,52 @@ +disallowed-methods = [ + { path = "std::env::current_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::is_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::is_file", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::is_symlink", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::read_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::read_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::symlink_metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::try_exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::is_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::is_file", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::is_symlink", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::read_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::read_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::symlink_metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::try_exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::env::set_current_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::env::temp_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::copy", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::create_dir_all", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::create_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::DirBuilder::new", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::hard_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::OpenOptions::new", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read_to_string", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::remove_dir_all", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::remove_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::remove_file", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::rename", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::set_permissions", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::symlink_metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::write", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "url::Url::to_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_directory_path", reason = "Use deno_path_util instead so it works in Wasm" }, +] +disallowed-types = [ + # todo(dsherret): consider for the future + # { path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" }, +] diff --git a/resolvers/deno/fs.rs b/resolvers/deno/fs.rs new file mode 100644 index 0000000000..b08be37982 --- /dev/null +++ b/resolvers/deno/fs.rs @@ -0,0 +1,27 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::path::Path; +use std::path::PathBuf; + +pub struct DirEntry { + pub name: String, + pub is_file: bool, + pub is_directory: bool, +} + +pub trait DenoResolverFs { + fn read_to_string_lossy(&self, path: &Path) -> std::io::Result; + fn realpath_sync(&self, path: &Path) -> std::io::Result; + fn is_dir_sync(&self, path: &Path) -> bool; + fn read_dir_sync(&self, dir_path: &Path) -> std::io::Result>; +} + +pub(crate) struct DenoPkgJsonFsAdapter<'a, Fs: DenoResolverFs>(pub &'a Fs); + +impl<'a, Fs: DenoResolverFs> deno_package_json::fs::DenoPkgJsonFs + for DenoPkgJsonFsAdapter<'a, Fs> +{ + fn read_to_string_lossy(&self, path: &Path) -> std::io::Result { + self.0.read_to_string_lossy(path) + } +} diff --git a/resolvers/deno/lib.rs b/resolvers/deno/lib.rs index 7d7796d776..57fa67512a 100644 --- a/resolvers/deno/lib.rs +++ b/resolvers/deno/lib.rs @@ -1,3 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +pub mod fs; +pub mod npm; pub mod sloppy_imports; diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs new file mode 100644 index 0000000000..c847cee0f2 --- /dev/null +++ b/resolvers/deno/npm/byonm.rs @@ -0,0 +1,348 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::bail; +use anyhow::Error as AnyError; +use deno_package_json::PackageJson; +use deno_package_json::PackageJsonDepValue; +use deno_path_util::url_to_file_path; +use deno_semver::package::PackageReq; +use deno_semver::Version; +use node_resolver::errors::PackageFolderResolveError; +use node_resolver::errors::PackageFolderResolveIoError; +use node_resolver::errors::PackageJsonLoadError; +use node_resolver::errors::PackageNotFoundError; +use node_resolver::load_pkg_json; +use node_resolver::NpmResolver; +use url::Url; + +use crate::fs::DenoPkgJsonFsAdapter; +use crate::fs::DenoResolverFs; + +use super::local::normalize_pkg_name_for_node_modules_deno_folder; + +pub struct ByonmNpmResolverCreateOptions { + pub fs: Fs, + // todo(dsherret): investigate removing this + pub root_node_modules_dir: Option, +} + +#[derive(Debug)] +pub struct ByonmNpmResolver { + fs: Fs, + root_node_modules_dir: Option, +} + +impl Clone for ByonmNpmResolver { + fn clone(&self) -> Self { + Self { + fs: self.fs.clone(), + root_node_modules_dir: self.root_node_modules_dir.clone(), + } + } +} + +impl ByonmNpmResolver { + pub fn new(options: ByonmNpmResolverCreateOptions) -> Self { + Self { + fs: options.fs, + root_node_modules_dir: options.root_node_modules_dir, + } + } + + pub fn root_node_modules_dir(&self) -> Option<&Path> { + self.root_node_modules_dir.as_deref() + } + + fn load_pkg_json( + &self, + path: &Path, + ) -> Result>, PackageJsonLoadError> { + load_pkg_json(&DenoPkgJsonFsAdapter(&self.fs), path) + } + + /// Finds the ancestor package.json that contains the specified dependency. + pub fn find_ancestor_package_json_with_dep( + &self, + dep_name: &str, + referrer: &Url, + ) -> Option> { + let referrer_path = url_to_file_path(referrer).ok()?; + let mut current_folder = referrer_path.parent()?; + loop { + let pkg_json_path = current_folder.join("package.json"); + if let Ok(Some(pkg_json)) = self.load_pkg_json(&pkg_json_path) { + if let Some(deps) = &pkg_json.dependencies { + if deps.contains_key(dep_name) { + return Some(pkg_json); + } + } + if let Some(deps) = &pkg_json.dev_dependencies { + if deps.contains_key(dep_name) { + return Some(pkg_json); + } + } + } + + if let Some(parent) = current_folder.parent() { + current_folder = parent; + } else { + return None; + } + } + } + + pub fn resolve_pkg_folder_from_deno_module_req( + &self, + req: &PackageReq, + referrer: &Url, + ) -> Result { + fn node_resolve_dir( + fs: &Fs, + alias: &str, + start_dir: &Path, + ) -> Result, AnyError> { + for ancestor in start_dir.ancestors() { + let node_modules_folder = ancestor.join("node_modules"); + let sub_dir = join_package_name(&node_modules_folder, alias); + if fs.is_dir_sync(&sub_dir) { + return Ok(Some(deno_path_util::canonicalize_path_maybe_not_exists( + &sub_dir, + &|path| fs.realpath_sync(path), + )?)); + } + } + Ok(None) + } + + // now attempt to resolve if it's found in any package.json + let maybe_pkg_json_and_alias = + self.resolve_pkg_json_and_alias_for_req(req, referrer)?; + match maybe_pkg_json_and_alias { + Some((pkg_json, alias)) => { + // now try node resolution + if let Some(resolved) = + node_resolve_dir(&self.fs, &alias, pkg_json.dir_path())? + { + return Ok(resolved); + } + + bail!( + concat!( + "Could not find \"{}\" in a node_modules folder. ", + "Deno expects the node_modules/ directory to be up to date. ", + "Did you forget to run `deno install`?" + ), + alias, + ); + } + None => { + // now check if node_modules/.deno/ matches this constraint + if let Some(folder) = self.resolve_folder_in_root_node_modules(req) { + return Ok(folder); + } + + bail!( + concat!( + "Could not find a matching package for 'npm:{}' in the node_modules ", + "directory. Ensure you have all your JSR and npm dependencies listed ", + "in your deno.json or package.json, then run `deno install`. Alternatively, ", + r#"turn on auto-install by specifying `"nodeModulesDir": "auto"` in your "#, + "deno.json file." + ), + req, + ); + } + } + } + + fn resolve_pkg_json_and_alias_for_req( + &self, + req: &PackageReq, + referrer: &Url, + ) -> Result, String)>, AnyError> { + fn resolve_alias_from_pkg_json( + req: &PackageReq, + pkg_json: &PackageJson, + ) -> Option { + let deps = pkg_json.resolve_local_package_json_deps(); + for (key, value) in deps { + if let Ok(value) = value { + match value { + PackageJsonDepValue::Req(dep_req) => { + if dep_req.name == req.name + && dep_req.version_req.intersects(&req.version_req) + { + return Some(key); + } + } + PackageJsonDepValue::Workspace(_workspace) => { + if key == req.name && req.version_req.tag() == Some("workspace") { + return Some(key); + } + } + } + } + } + None + } + + // attempt to resolve the npm specifier from the referrer's package.json, + if let Ok(file_path) = url_to_file_path(referrer) { + let mut current_path = file_path.as_path(); + while let Some(dir_path) = current_path.parent() { + let package_json_path = dir_path.join("package.json"); + if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? { + if let Some(alias) = + resolve_alias_from_pkg_json(req, pkg_json.as_ref()) + { + return Ok(Some((pkg_json, alias))); + } + } + current_path = dir_path; + } + } + + // otherwise, fall fallback to the project's package.json + if let Some(root_node_modules_dir) = &self.root_node_modules_dir { + let root_pkg_json_path = + root_node_modules_dir.parent().unwrap().join("package.json"); + if let Some(pkg_json) = self.load_pkg_json(&root_pkg_json_path)? { + if let Some(alias) = resolve_alias_from_pkg_json(req, pkg_json.as_ref()) + { + return Ok(Some((pkg_json, alias))); + } + } + } + + Ok(None) + } + + fn resolve_folder_in_root_node_modules( + &self, + req: &PackageReq, + ) -> Option { + // now check if node_modules/.deno/ matches this constraint + let root_node_modules_dir = self.root_node_modules_dir.as_ref()?; + let node_modules_deno_dir = root_node_modules_dir.join(".deno"); + let Ok(entries) = self.fs.read_dir_sync(&node_modules_deno_dir) else { + return None; + }; + let search_prefix = format!( + "{}@", + normalize_pkg_name_for_node_modules_deno_folder(&req.name) + ); + let mut best_version = None; + + // example entries: + // - @denotest+add@1.0.0 + // - @denotest+add@1.0.0_1 + for entry in entries { + if !entry.is_directory { + continue; + } + let Some(version_and_copy_idx) = entry.name.strip_prefix(&search_prefix) + else { + continue; + }; + let version = version_and_copy_idx + .rsplit_once('_') + .map(|(v, _)| v) + .unwrap_or(version_and_copy_idx); + let Ok(version) = Version::parse_from_npm(version) else { + continue; + }; + if req.version_req.matches(&version) { + if let Some((best_version_version, _)) = &best_version { + if version > *best_version_version { + best_version = Some((version, entry.name)); + } + } else { + best_version = Some((version, entry.name)); + } + } + } + + best_version.map(|(_version, entry_name)| { + join_package_name( + &node_modules_deno_dir.join(entry_name).join("node_modules"), + &req.name, + ) + }) + } +} + +impl NpmResolver + for ByonmNpmResolver +{ + fn resolve_package_folder_from_package( + &self, + name: &str, + referrer: &Url, + ) -> Result { + fn inner( + fs: &Fs, + name: &str, + referrer: &Url, + ) -> Result { + let maybe_referrer_file = url_to_file_path(referrer).ok(); + let maybe_start_folder = + maybe_referrer_file.as_ref().and_then(|f| f.parent()); + if let Some(start_folder) = maybe_start_folder { + for current_folder in start_folder.ancestors() { + let node_modules_folder = if current_folder.ends_with("node_modules") + { + Cow::Borrowed(current_folder) + } else { + Cow::Owned(current_folder.join("node_modules")) + }; + + let sub_dir = join_package_name(&node_modules_folder, name); + if fs.is_dir_sync(&sub_dir) { + return Ok(sub_dir); + } + } + } + + Err( + PackageNotFoundError { + package_name: name.to_string(), + referrer: referrer.clone(), + referrer_extra: None, + } + .into(), + ) + } + + let path = inner(&self.fs, name, referrer)?; + self.fs.realpath_sync(&path).map_err(|err| { + PackageFolderResolveIoError { + package_name: name.to_string(), + referrer: referrer.clone(), + source: err, + } + .into() + }) + } + + fn in_npm_package(&self, specifier: &Url) -> bool { + specifier.scheme() == "file" + && specifier + .path() + .to_ascii_lowercase() + .contains("/node_modules/") + } +} + +fn join_package_name(path: &Path, package_name: &str) -> PathBuf { + let mut path = path.to_path_buf(); + // ensure backslashes are used on windows + for part in package_name.split('/') { + path = path.join(part); + } + path +} diff --git a/resolvers/deno/npm/local.rs b/resolvers/deno/npm/local.rs new file mode 100644 index 0000000000..aef476ad94 --- /dev/null +++ b/resolvers/deno/npm/local.rs @@ -0,0 +1,27 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; + +/// Normalizes a package name for use at `node_modules/.deno/@[_]` +pub fn normalize_pkg_name_for_node_modules_deno_folder(name: &str) -> Cow { + let name = if name.to_lowercase() == name { + Cow::Borrowed(name) + } else { + Cow::Owned(format!("_{}", mixed_case_package_name_encode(name))) + }; + if name.starts_with('@') { + name.replace('/', "+").into() + } else { + name + } +} + +fn mixed_case_package_name_encode(name: &str) -> String { + // use base32 encoding because it's reversible and the character set + // only includes the characters within 0-9 and A-Z so it can be lower cased + base32::encode( + base32::Alphabet::Rfc4648Lower { padding: false }, + name.as_bytes(), + ) + .to_lowercase() +} diff --git a/resolvers/deno/npm/mod.rs b/resolvers/deno/npm/mod.rs new file mode 100644 index 0000000000..2e24144cd6 --- /dev/null +++ b/resolvers/deno/npm/mod.rs @@ -0,0 +1,8 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +mod byonm; +mod local; + +pub use byonm::ByonmNpmResolver; +pub use byonm::ByonmNpmResolverCreateOptions; +pub use local::normalize_pkg_name_for_node_modules_deno_folder; diff --git a/resolvers/deno/sloppy_imports.rs b/resolvers/deno/sloppy_imports.rs index e4d0898e5d..e215e87686 100644 --- a/resolvers/deno/sloppy_imports.rs +++ b/resolvers/deno/sloppy_imports.rs @@ -5,6 +5,7 @@ use std::path::Path; use std::path::PathBuf; use deno_media_type::MediaType; +use deno_path_util::url_from_file_path; use deno_path_util::url_to_file_path; use url::Url; @@ -343,7 +344,7 @@ impl SloppyImportsResolver { for (probe_path, reason) in probe_paths { if self.fs.is_file(&probe_path) { - if let Ok(specifier) = Url::from_file_path(probe_path) { + if let Ok(specifier) = url_from_file_path(&probe_path) { match reason { SloppyImportsResolutionReason::JsToTs => { return Some(SloppyImportsResolution::JsToTs(specifier)); @@ -386,6 +387,7 @@ mod test { struct RealSloppyImportsResolverFs; impl SloppyImportResolverFs for RealSloppyImportsResolverFs { fn stat_sync(&self, path: &Path) -> Option { + #[allow(clippy::disallowed_methods)] let stat = std::fs::metadata(path).ok()?; if stat.is_dir() { Some(SloppyImportsFsEntry::Dir) diff --git a/resolvers/node/Cargo.toml b/resolvers/node/Cargo.toml index 1042045692..c2f2f1cc1a 100644 --- a/resolvers/node/Cargo.toml +++ b/resolvers/node/Cargo.toml @@ -21,6 +21,7 @@ anyhow.workspace = true async-trait.workspace = true deno_media_type.workspace = true deno_package_json.workspace = true +deno_path_util.workspace = true futures.workspace = true lazy-regex.workspace = true once_cell.workspace = true diff --git a/resolvers/node/analyze.rs b/resolvers/node/analyze.rs index deb56d064e..009296006a 100644 --- a/resolvers/node/analyze.rs +++ b/resolvers/node/analyze.rs @@ -6,6 +6,8 @@ use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use futures::future::LocalBoxFuture; use futures::stream::FuturesUnordered; use futures::FutureExt; @@ -18,7 +20,6 @@ use url::Url; use crate::env::NodeResolverEnv; use crate::package_json::load_pkg_json; -use crate::path::to_file_specifier; use crate::resolution::NodeResolverRc; use crate::NodeModuleKind; use crate::NodeResolutionMode; @@ -135,8 +136,7 @@ impl source.push(format!( "const mod = require(\"{}\");", - entry_specifier - .to_file_path() + url_to_file_path(entry_specifier) .unwrap() .to_str() .unwrap() @@ -297,15 +297,13 @@ impl todo!(); } - let referrer_path = referrer.to_file_path().unwrap(); + let referrer_path = url_to_file_path(referrer).unwrap(); if specifier.starts_with("./") || specifier.starts_with("../") { if let Some(parent) = referrer_path.parent() { - return Some( - self - .file_extension_probe(parent.join(specifier), &referrer_path) - .map(|p| to_file_specifier(&p)), - ) - .transpose(); + return self + .file_extension_probe(parent.join(specifier), &referrer_path) + .and_then(|p| url_from_file_path(&p).map_err(AnyError::from)) + .map(Some); } else { todo!(); } @@ -362,24 +360,22 @@ impl load_pkg_json(self.env.pkg_json_fs(), &package_json_path)?; if let Some(package_json) = maybe_package_json { if let Some(main) = package_json.main(NodeModuleKind::Cjs) { - return Ok(Some(to_file_specifier(&d.join(main).clean()))); + return Ok(Some(url_from_file_path(&d.join(main).clean())?)); } } - return Ok(Some(to_file_specifier(&d.join("index.js").clean()))); + return Ok(Some(url_from_file_path(&d.join("index.js").clean())?)); } - return Some( - self - .file_extension_probe(d, &referrer_path) - .map(|p| to_file_specifier(&p)), - ) - .transpose(); + return self + .file_extension_probe(d, &referrer_path) + .and_then(|p| url_from_file_path(&p).map_err(AnyError::from)) + .map(Some); } else if let Some(main) = package_json.main(NodeModuleKind::Cjs) { - return Ok(Some(to_file_specifier(&module_dir.join(main).clean()))); + return Ok(Some(url_from_file_path(&module_dir.join(main).clean())?)); } else { - return Ok(Some(to_file_specifier( + return Ok(Some(url_from_file_path( &module_dir.join("index.js").clean(), - ))); + )?)); } } @@ -395,7 +391,7 @@ impl parent.join("node_modules").join(specifier) }; if let Ok(path) = self.file_extension_probe(path, &referrer_path) { - return Ok(Some(to_file_specifier(&path))); + return Ok(Some(url_from_file_path(&path)?)); } last = parent; } diff --git a/resolvers/node/clippy.toml b/resolvers/node/clippy.toml index 86150781bb..90eaba3fae 100644 --- a/resolvers/node/clippy.toml +++ b/resolvers/node/clippy.toml @@ -42,6 +42,9 @@ disallowed-methods = [ { path = "std::fs::write", reason = "File system operations should be done using NodeResolverFs trait" }, { path = "std::path::Path::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, { path = "std::path::Path::exists", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "url::Url::to_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_directory_path", reason = "Use deno_path_util instead so it works in Wasm" }, ] disallowed-types = [ { path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" }, diff --git a/resolvers/node/npm.rs b/resolvers/node/npm.rs index 77df57c489..6b5f21db62 100644 --- a/resolvers/node/npm.rs +++ b/resolvers/node/npm.rs @@ -3,6 +3,8 @@ use std::path::Path; use std::path::PathBuf; +use deno_path_util::url_from_directory_path; +use deno_path_util::url_from_file_path; use url::Url; use crate::errors; @@ -24,7 +26,7 @@ pub trait NpmResolver: std::fmt::Debug + MaybeSend + MaybeSync { fn in_npm_package(&self, specifier: &Url) -> bool; fn in_npm_package_at_dir_path(&self, path: &Path) -> bool { - let specifier = match Url::from_directory_path(path.to_path_buf().clean()) { + let specifier = match url_from_directory_path(&path.to_path_buf().clean()) { Ok(p) => p, Err(_) => return false, }; @@ -32,7 +34,7 @@ pub trait NpmResolver: std::fmt::Debug + MaybeSend + MaybeSync { } fn in_npm_package_at_file_path(&self, path: &Path) -> bool { - let specifier = match Url::from_file_path(path.to_path_buf().clean()) { + let specifier = match url_from_file_path(&path.to_path_buf().clean()) { Ok(p) => p, Err(_) => return false, }; diff --git a/resolvers/node/path.rs b/resolvers/node/path.rs index ece270cd91..8c2d35fadf 100644 --- a/resolvers/node/path.rs +++ b/resolvers/node/path.rs @@ -4,8 +4,6 @@ use std::path::Component; use std::path::Path; use std::path::PathBuf; -use url::Url; - /// Extension to path_clean::PathClean pub trait PathClean { fn clean(&self) -> T; @@ -65,65 +63,6 @@ impl PathClean for PathBuf { } } -pub(crate) fn to_file_specifier(path: &Path) -> Url { - match Url::from_file_path(path) { - Ok(url) => url, - Err(_) => panic!("Invalid path: {}", path.display()), - } -} - -// todo(dsherret): we have the below code also in deno_core and it -// would be good to somehow re-use it in both places (we don't want -// to create a dependency on deno_core here) - -#[cfg(not(windows))] -#[inline] -pub fn strip_unc_prefix(path: PathBuf) -> PathBuf { - path -} - -/// Strips the unc prefix (ex. \\?\) from Windows paths. -#[cfg(windows)] -pub fn strip_unc_prefix(path: PathBuf) -> PathBuf { - use std::path::Component; - use std::path::Prefix; - - let mut components = path.components(); - match components.next() { - Some(Component::Prefix(prefix)) => { - match prefix.kind() { - // \\?\device - Prefix::Verbatim(device) => { - let mut path = PathBuf::new(); - path.push(format!(r"\\{}\", device.to_string_lossy())); - path.extend(components.filter(|c| !matches!(c, Component::RootDir))); - path - } - // \\?\c:\path - Prefix::VerbatimDisk(_) => { - let mut path = PathBuf::new(); - path.push(prefix.as_os_str().to_string_lossy().replace(r"\\?\", "")); - path.extend(components); - path - } - // \\?\UNC\hostname\share_name\path - Prefix::VerbatimUNC(hostname, share_name) => { - let mut path = PathBuf::new(); - path.push(format!( - r"\\{}\{}\", - hostname.to_string_lossy(), - share_name.to_string_lossy() - )); - path.extend(components.filter(|c| !matches!(c, Component::RootDir))); - path - } - _ => path, - } - } - _ => path, - } -} - #[cfg(test)] mod test { #[cfg(windows)] @@ -139,41 +78,4 @@ mod test { assert_eq!(PathBuf::from(input).clean(), PathBuf::from(expected)); } } - - #[cfg(windows)] - #[test] - fn test_strip_unc_prefix() { - use std::path::PathBuf; - - run_test(r"C:\", r"C:\"); - run_test(r"C:\test\file.txt", r"C:\test\file.txt"); - - run_test(r"\\?\C:\", r"C:\"); - run_test(r"\\?\C:\test\file.txt", r"C:\test\file.txt"); - - run_test(r"\\.\C:\", r"\\.\C:\"); - run_test(r"\\.\C:\Test\file.txt", r"\\.\C:\Test\file.txt"); - - run_test(r"\\?\UNC\localhost\", r"\\localhost"); - run_test(r"\\?\UNC\localhost\c$\", r"\\localhost\c$"); - run_test( - r"\\?\UNC\localhost\c$\Windows\file.txt", - r"\\localhost\c$\Windows\file.txt", - ); - run_test(r"\\?\UNC\wsl$\deno.json", r"\\wsl$\deno.json"); - - run_test(r"\\?\server1", r"\\server1"); - run_test(r"\\?\server1\e$\", r"\\server1\e$\"); - run_test( - r"\\?\server1\e$\test\file.txt", - r"\\server1\e$\test\file.txt", - ); - - fn run_test(input: &str, expected: &str) { - assert_eq!( - super::strip_unc_prefix(PathBuf::from(input)), - PathBuf::from(expected) - ); - } - } } diff --git a/resolvers/node/resolution.rs b/resolvers/node/resolution.rs index ad9dbb7100..811583a5ee 100644 --- a/resolvers/node/resolution.rs +++ b/resolvers/node/resolution.rs @@ -8,6 +8,8 @@ use anyhow::bail; use anyhow::Error as AnyError; use deno_media_type::MediaType; use deno_package_json::PackageJsonRc; +use deno_path_util::strip_unc_prefix; +use deno_path_util::url_from_file_path; use serde_json::Map; use serde_json::Value; use url::Url; @@ -47,8 +49,6 @@ use crate::errors::TypesNotFoundErrorData; use crate::errors::UnsupportedDirImportError; use crate::errors::UnsupportedEsmUrlSchemeError; use crate::errors::UrlToNodeResolutionError; -use crate::path::strip_unc_prefix; -use crate::path::to_file_specifier; use crate::NpmResolverRc; use crate::PathClean; use deno_package_json::PackageJson; @@ -394,7 +394,7 @@ impl NodeResolver { message: err.to_string(), } })?; - let url = to_file_specifier(&package_folder.join(bin_entry)); + let url = url_from_file_path(&package_folder.join(bin_entry)).unwrap(); let resolve_response = self.url_to_node_resolution(url)?; // TODO(bartlomieju): skipped checking errors for commonJS resolution and @@ -485,12 +485,12 @@ impl NodeResolver { || lowercase_path.ends_with(".d.cts") || lowercase_path.ends_with(".d.mts") { - return Ok(to_file_specifier(path)); + return Ok(url_from_file_path(path).unwrap()); } if let Some(path) = probe_extensions(&self.env, path, &lowercase_path, referrer_kind) { - return Ok(to_file_specifier(&path)); + return Ok(url_from_file_path(&path).unwrap()); } if self.env.is_dir_sync(path) { let resolution_result = self.resolve_package_dir_subpath( @@ -514,15 +514,15 @@ impl NodeResolver { &index_path.to_string_lossy().to_lowercase(), referrer_kind, ) { - return Ok(to_file_specifier(&path)); + return Ok(url_from_file_path(&path).unwrap()); } } // allow resolving .css files for types resolution if lowercase_path.ends_with(".css") { - return Ok(to_file_specifier(path)); + return Ok(url_from_file_path(path).unwrap()); } Err(TypesNotFoundError(Box::new(TypesNotFoundErrorData { - code_specifier: to_file_specifier(path), + code_specifier: url_from_file_path(path).unwrap(), maybe_referrer: maybe_referrer.cloned(), }))) } @@ -673,7 +673,8 @@ impl NodeResolver { } else { format!("{target}{subpath}") }; - let package_json_url = to_file_specifier(package_json_path); + let package_json_url = + url_from_file_path(package_json_path).unwrap(); let result = match self.package_resolve( &export_target, &package_json_url, @@ -760,7 +761,7 @@ impl NodeResolver { ); } if subpath.is_empty() { - return Ok(to_file_specifier(&resolved_path)); + return Ok(url_from_file_path(&resolved_path).unwrap()); } if invalid_segment_re.is_match(subpath) { let request = if pattern { @@ -782,9 +783,11 @@ impl NodeResolver { let resolved_path_str = resolved_path.to_string_lossy(); let replaced = pattern_re .replace(&resolved_path_str, |_caps: ®ex::Captures| subpath); - return Ok(to_file_specifier(&PathBuf::from(replaced.to_string()))); + return Ok( + url_from_file_path(&PathBuf::from(replaced.to_string())).unwrap(), + ); } - Ok(to_file_specifier(&resolved_path.join(subpath).clean())) + Ok(url_from_file_path(&resolved_path.join(subpath).clean()).unwrap()) } #[allow(clippy::too_many_arguments)] @@ -871,7 +874,7 @@ impl NodeResolver { mode, )?; if mode.is_types() && url.scheme() == "file" { - let path = url.to_file_path().unwrap(); + let path = deno_path_util::url_to_file_path(&url).unwrap(); return Ok(Some(self.path_to_declaration_url( &path, maybe_referrer, @@ -1307,7 +1310,7 @@ impl NodeResolver { if mode.is_types() { Ok(self.path_to_declaration_url(&file_path, referrer, referrer_kind)?) } else { - Ok(to_file_specifier(&file_path)) + Ok(url_from_file_path(&file_path).unwrap()) } } @@ -1338,7 +1341,7 @@ impl NodeResolver { &self, url: &Url, ) -> Result, ClosestPkgJsonError> { - let Ok(file_path) = url.to_file_path() else { + let Ok(file_path) = deno_path_util::url_to_file_path(url) else { return Ok(None); }; self.get_closest_package_json_from_path(&file_path) @@ -1433,7 +1436,7 @@ impl NodeResolver { if let Some(main) = maybe_main { let guess = package_json.path.parent().unwrap().join(main).clean(); if self.env.is_file_sync(&guess) { - return Ok(to_file_specifier(&guess)); + return Ok(url_from_file_path(&guess).unwrap()); } // todo(dsherret): investigate exactly how node and typescript handles this @@ -1463,7 +1466,7 @@ impl NodeResolver { .clean(); if self.env.is_file_sync(&guess) { // TODO(bartlomieju): emitLegacyIndexDeprecation() - return Ok(to_file_specifier(&guess)); + return Ok(url_from_file_path(&guess).unwrap()); } } } @@ -1496,14 +1499,15 @@ impl NodeResolver { let guess = directory.join(index_file_name).clean(); if self.env.is_file_sync(&guess) { // TODO(bartlomieju): emitLegacyIndexDeprecation() - return Ok(to_file_specifier(&guess)); + return Ok(url_from_file_path(&guess).unwrap()); } } if mode.is_types() { Err( TypesNotFoundError(Box::new(TypesNotFoundErrorData { - code_specifier: to_file_specifier(&directory.join("index.js")), + code_specifier: url_from_file_path(&directory.join("index.js")) + .unwrap(), maybe_referrer: maybe_referrer.cloned(), })) .into(), @@ -1511,7 +1515,7 @@ impl NodeResolver { } else { Err( ModuleNotFoundError { - specifier: to_file_specifier(&directory.join("index.js")), + specifier: url_from_file_path(&directory.join("index.js")).unwrap(), typ: "module", maybe_referrer: maybe_referrer.cloned(), } @@ -1611,9 +1615,7 @@ fn resolve_bin_entry_value<'a>( } fn to_file_path(url: &Url) -> PathBuf { - url - .to_file_path() - .unwrap_or_else(|_| panic!("Provided URL was not file:// URL: {url}")) + deno_path_util::url_to_file_path(url).unwrap() } fn to_file_path_string(url: &Url) -> String { @@ -1692,7 +1694,7 @@ fn with_known_extension(path: &Path, ext: &str) -> PathBuf { } fn to_specifier_display_string(url: &Url) -> String { - if let Ok(path) = url.to_file_path() { + if let Ok(path) = deno_path_util::url_to_file_path(url) { path.display().to_string() } else { url.to_string() From a7d0a5540fed4ecf0b5ae5f462960cce21a4adf5 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 30 Sep 2024 09:34:20 -0400 Subject: [PATCH 009/302] fix: eagerly error for specifier with empty version constraint (#25944) Eagerly errors for something like `export * from "jsr:@type/is@";` (previously it would just fail elsewhere because it would consider this as having an empty tag) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2263351fc0..92a3c5056c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2031,9 +2031,9 @@ dependencies = [ [[package]] name = "deno_semver" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6657fecb9ac6a7a71f552c95e8cc492466a75f5660224577e2226bcf30db9768" +checksum = "670fec7ef309384e23c2a90ac5d2d9d91a776d225306c75f5cdd28cf6cc8a59f" dependencies = [ "monch", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index ef9a4576ce..4ce3105399 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ deno_npm = "=0.25.2" deno_path_util = "=0.2.0" deno_permissions = { version = "0.28.0", path = "./runtime/permissions" } deno_runtime = { version = "0.177.0", path = "./runtime" } -deno_semver = "=0.5.13" +deno_semver = "=0.5.14" deno_terminal = "0.2.0" napi_sym = { version = "0.98.0", path = "./cli/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } From 05ca6994ca36a152b793654e7c631271b6280521 Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Mon, 30 Sep 2024 21:14:42 +0530 Subject: [PATCH 010/302] chore: include sha256 checksum in release (#25869) Closes https://github.com/denoland/deno/issues/7253 --- .github/workflows/ci.generate.ts | 26 ++++++++++++++++++++++++-- .github/workflows/ci.yml | 26 ++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 35020a5f8c..b23223fb48 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -757,8 +757,10 @@ const ci = { ].join("\n"), run: [ "cd target/release", + "shasum -a 256 deno > deno-${{ matrix.arch }}-unknown-linux-gnu.sha256sum", "zip -r deno-${{ matrix.arch }}-unknown-linux-gnu.zip deno", "strip denort", + "shasum -a 256 denort > denort-${{ matrix.arch }}-unknown-linux-gnu.sha256sum", "zip -r denort-${{ matrix.arch }}-unknown-linux-gnu.zip denort", "./deno types > lib.deno.d.ts", ].join("\n"), @@ -783,8 +785,10 @@ const ci = { "--p12-file=<(echo $APPLE_CODESIGN_KEY | base64 -d) " + "--entitlements-xml-file=cli/entitlements.plist", "cd target/release", + "shasum -a 256 deno > deno-${{ matrix.arch }}-apple-darwin.sha256sum", "zip -r deno-${{ matrix.arch }}-apple-darwin.zip deno", "strip denort", + "shasum -a 256 denort > denort-${{ matrix.arch }}-apple-darwin.sha256sum", "zip -r denort-${{ matrix.arch }}-apple-darwin.zip denort", ] .join("\n"), @@ -799,7 +803,9 @@ const ci = { ].join("\n"), shell: "pwsh", run: [ + "Get-FileHash target/release/deno.exe -Algorithm SHA256 | Format-List > target/release/deno-${{ matrix.arch }}-pc-windows-msvc.sha256sum", "Compress-Archive -CompressionLevel Optimal -Force -Path target/release/deno.exe -DestinationPath target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip", + "Get-FileHash target/release/denort.exe -Algorithm SHA256 | Format-List > target/release/denort-${{ matrix.arch }}-pc-windows-msvc.sha256sum", "Compress-Archive -CompressionLevel Optimal -Force -Path target/release/denort.exe -DestinationPath target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip", ].join("\n"), }, @@ -813,6 +819,7 @@ const ci = { ].join("\n"), run: [ 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/', + 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/canary/$(git rev-parse HEAD)/', "echo ${{ github.sha }} > canary-latest.txt", 'gsutil -h "Cache-Control: no-cache" cp canary-latest.txt gs://dl.deno.land/canary-$(rustc -vV | sed -n "s|host: ||p")-latest.txt', ].join("\n"), @@ -994,8 +1001,10 @@ const ci = { "github.repository == 'denoland/deno' &&", "startsWith(github.ref, 'refs/tags/')", ].join("\n"), - run: + run: [ 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/', + 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/', + ].join("\n"), }, { name: "Upload release to dl.deno.land (windows)", @@ -1009,8 +1018,10 @@ const ci = { env: { CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe", }, - run: + run: [ 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/', + 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/', + ].join("\n"), }, { name: "Create release notes", @@ -1040,15 +1051,25 @@ const ci = { with: { files: [ "target/release/deno-x86_64-pc-windows-msvc.zip", + "target/release/deno-x86_64-pc-windows-msvc.sha256sum", "target/release/denort-x86_64-pc-windows-msvc.zip", + "target/release/denort-x86_64-pc-windows-msvc.sha256sum", "target/release/deno-x86_64-unknown-linux-gnu.zip", + "target/release/deno-x86_64-unknown-linux-gnu.sha256sum", "target/release/denort-x86_64-unknown-linux-gnu.zip", + "target/release/denort-x86_64-unknown-linux-gnu.sha256sum", "target/release/deno-x86_64-apple-darwin.zip", + "target/release/deno-x86_64-apple-darwin.sha256sum", "target/release/denort-x86_64-apple-darwin.zip", + "target/release/denort-x86_64-apple-darwin.sha256sum", "target/release/deno-aarch64-unknown-linux-gnu.zip", + "target/release/deno-aarch64-unknown-linux-gnu.sha256sum", "target/release/denort-aarch64-unknown-linux-gnu.zip", + "target/release/denort-aarch64-unknown-linux-gnu.sha256sum", "target/release/deno-aarch64-apple-darwin.zip", + "target/release/deno-aarch64-apple-darwin.sha256sum", "target/release/denort-aarch64-apple-darwin.zip", + "target/release/denort-aarch64-apple-darwin.sha256sum", "target/release/deno_src.tar.gz", "target/release/lib.deno.d.ts", ].join("\n"), @@ -1067,6 +1088,7 @@ const ci = { "./target", "!./target/*/gn_out", "!./target/*/*.zip", + "!./target/*/*.sha256sum", "!./target/*/*.tar.gz", ].join("\n"), key: prCacheKeyPrefix + "${{ github.sha }}", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 280eb7cb84..df805cb235 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -448,8 +448,10 @@ jobs: github.repository == 'denoland/deno') run: |- cd target/release + shasum -a 256 deno > deno-${{ matrix.arch }}-unknown-linux-gnu.sha256sum zip -r deno-${{ matrix.arch }}-unknown-linux-gnu.zip deno strip denort + shasum -a 256 denort > denort-${{ matrix.arch }}-unknown-linux-gnu.sha256sum zip -r denort-${{ matrix.arch }}-unknown-linux-gnu.zip denort ./deno types > lib.deno.d.ts - name: Pre-release (mac) @@ -465,8 +467,10 @@ jobs: echo "Key is $(echo $APPLE_CODESIGN_KEY | base64 -d | wc -c) bytes" rcodesign sign target/release/deno --code-signature-flags=runtime --p12-password="$APPLE_CODESIGN_PASSWORD" --p12-file=<(echo $APPLE_CODESIGN_KEY | base64 -d) --entitlements-xml-file=cli/entitlements.plist cd target/release + shasum -a 256 deno > deno-${{ matrix.arch }}-apple-darwin.sha256sum zip -r deno-${{ matrix.arch }}-apple-darwin.zip deno strip denort + shasum -a 256 denort > denort-${{ matrix.arch }}-apple-darwin.sha256sum zip -r denort-${{ matrix.arch }}-apple-darwin.zip denort - name: Pre-release (windows) if: |- @@ -476,7 +480,9 @@ jobs: github.repository == 'denoland/deno') shell: pwsh run: |- + Get-FileHash target/release/deno.exe -Algorithm SHA256 | Format-List > target/release/deno-${{ matrix.arch }}-pc-windows-msvc.sha256sum Compress-Archive -CompressionLevel Optimal -Force -Path target/release/deno.exe -DestinationPath target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip + Get-FileHash target/release/denort.exe -Algorithm SHA256 | Format-List > target/release/denort-${{ matrix.arch }}-pc-windows-msvc.sha256sum Compress-Archive -CompressionLevel Optimal -Force -Path target/release/denort.exe -DestinationPath target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip - name: Upload canary to dl.deno.land if: |- @@ -486,6 +492,7 @@ jobs: github.ref == 'refs/heads/main') run: |- gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/ + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/canary/$(git rev-parse HEAD)/ echo ${{ github.sha }} > canary-latest.txt gsutil -h "Cache-Control: no-cache" cp canary-latest.txt gs://dl.deno.land/canary-$(rustc -vV | sed -n "s|host: ||p")-latest.txt - name: Autobahn testsuite @@ -615,7 +622,9 @@ jobs: matrix.profile == 'release' && github.repository == 'denoland/deno' && startsWith(github.ref, 'refs/tags/')) - run: 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/' + run: |- + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/ + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/ - name: Upload release to dl.deno.land (windows) if: |- !(matrix.skip) && (matrix.os == 'windows' && @@ -625,7 +634,9 @@ jobs: startsWith(github.ref, 'refs/tags/')) env: CLOUDSDK_PYTHON: '${{env.pythonLocation}}\python.exe' - run: 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/' + run: |- + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/ + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/ - name: Create release notes if: |- !(matrix.skip) && (matrix.job == 'test' && @@ -647,15 +658,25 @@ jobs: with: files: |- target/release/deno-x86_64-pc-windows-msvc.zip + target/release/deno-x86_64-pc-windows-msvc.sha256sum target/release/denort-x86_64-pc-windows-msvc.zip + target/release/denort-x86_64-pc-windows-msvc.sha256sum target/release/deno-x86_64-unknown-linux-gnu.zip + target/release/deno-x86_64-unknown-linux-gnu.sha256sum target/release/denort-x86_64-unknown-linux-gnu.zip + target/release/denort-x86_64-unknown-linux-gnu.sha256sum target/release/deno-x86_64-apple-darwin.zip + target/release/deno-x86_64-apple-darwin.sha256sum target/release/denort-x86_64-apple-darwin.zip + target/release/denort-x86_64-apple-darwin.sha256sum target/release/deno-aarch64-unknown-linux-gnu.zip + target/release/deno-aarch64-unknown-linux-gnu.sha256sum target/release/denort-aarch64-unknown-linux-gnu.zip + target/release/denort-aarch64-unknown-linux-gnu.sha256sum target/release/deno-aarch64-apple-darwin.zip + target/release/deno-aarch64-apple-darwin.sha256sum target/release/denort-aarch64-apple-darwin.zip + target/release/denort-aarch64-apple-darwin.sha256sum target/release/deno_src.tar.gz target/release/lib.deno.d.ts body_path: target/release/release-notes.md @@ -668,6 +689,7 @@ jobs: ./target !./target/*/gn_out !./target/*/*.zip + !./target/*/*.sha256sum !./target/*/*.tar.gz key: '15-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: From c5c18699921cd45021f8c130eecdc09099c26878 Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Mon, 30 Sep 2024 21:25:30 +0200 Subject: [PATCH 011/302] fix: precompile preserve SVG camelCase attributes (#25945) See https://github.com/denoland/deno_ast/pull/278 Fixes https://github.com/denoland/deno/issues/25810 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- tests/specs/npm/require_type_commonjs/main.out | 14 ++++---------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92a3c5056c..945e493486 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1273,9 +1273,9 @@ dependencies = [ [[package]] name = "deno_ast" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b08d11d9e4086b00d3428650e31153cf5896586411763cb88a6423ce5b18791" +checksum = "89ea2fd038c9c7e3e87e624fd708303cd33f39c33707f6c48fa9a65d65fefc47" dependencies = [ "base64 0.21.7", "deno_media_type", diff --git a/Cargo.toml b/Cargo.toml index 4ce3105399..ce333c728b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ license = "MIT" repository = "https://github.com/denoland/deno" [workspace.dependencies] -deno_ast = { version = "=0.42.0", features = ["transpiling"] } +deno_ast = { version = "=0.42.1", features = ["transpiling"] } deno_core = { version = "0.311.0" } deno_bench_util = { version = "0.162.0", path = "./bench_util" } diff --git a/tests/specs/npm/require_type_commonjs/main.out b/tests/specs/npm/require_type_commonjs/main.out index f7a81572c3..d715db8a94 100644 --- a/tests/specs/npm/require_type_commonjs/main.out +++ b/tests/specs/npm/require_type_commonjs/main.out @@ -1,10 +1,4 @@ -error: Uncaught (in promise) SyntaxError: Unexpected token 'export' - at Object.evalContext (ext:core/01_core.js:[WILDCARD]) - at wrapSafe (node:module:[WILDCARD]) - at Module._compile (node:module:[WILDCARD]) - at Object.Module._extensions..js (node:module:[WILDCARD]) - at Module.load (node:module:[WILDCARD]) - at Function.Module._load (node:module:[WILDCARD]) - at Module.require (node:module:[WILDCARD]) - at require (node:module:[WILDCARD]) - at file:///[WILDCARD]/@denotest/type-commonjs/1.0.0/index.js:3:13 +error: 'import', and 'export' cannot be used outside of module code at file://[WILDCARD]/@denotest/type-commonjs/1.0.0/index.js:1:1 + + export {}; + ~~~~~~ From d7b787792c09569bd718d9acf2d2b73ae406828c Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 30 Sep 2024 15:46:43 -0400 Subject: [PATCH 012/302] fix(info): error instead of panic for npm specifiers when using byonm (#25947) --- cli/args/mod.rs | 4 ---- cli/graph_util.rs | 7 +++++++ tests/specs/info/byonm/__test__.jsonc | 11 +++++++++++ tests/specs/info/byonm/deno.json | 6 ++++++ tests/specs/info/byonm/info.out | 1 + tests/specs/info/byonm/package.json | 5 +++++ 6 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 tests/specs/info/byonm/__test__.jsonc create mode 100644 tests/specs/info/byonm/deno.json create mode 100644 tests/specs/info/byonm/info.out create mode 100644 tests/specs/info/byonm/package.json diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 3ff2a427fb..2ae7098da5 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -1144,10 +1144,6 @@ impl CliOptions { DenoSubcommand::Run(run_flags) => { if run_flags.is_stdin() { resolve_url_or_path("./$deno$stdin.ts", self.initial_cwd())? - } else if NpmPackageReqReference::from_str(&run_flags.script) - .is_ok() - { - ModuleSpecifier::parse(&run_flags.script)? } else { resolve_url_or_path(&run_flags.script, self.initial_cwd())? } diff --git a/cli/graph_util.rs b/cli/graph_util.rs index f7194ac11b..e2f6246e74 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -21,6 +21,7 @@ use crate::tools::check::TypeChecker; use crate::util::file_watcher::WatcherCommunicator; use crate::util::fs::canonicalize_path; use deno_config::workspace::JsrPackageConfig; +use deno_core::anyhow::bail; use deno_graph::source::LoaderChecksum; use deno_graph::FillFromLockfileOptions; use deno_graph::JsrLoadError; @@ -593,6 +594,12 @@ impl ModuleGraphBuilder { let initial_package_deps_len = graph.packages.package_deps_sum(); let initial_package_mappings_len = graph.packages.mappings().len(); + if roots.iter().any(|r| r.scheme() == "npm") + && self.npm_resolver.as_byonm().is_some() + { + bail!("Resolving npm specifier entrypoints this way is currently not supported with \"nodeModules\": \"manual\". In the meantime, try with --node-modules-dir=auto instead"); + } + graph.build(roots, loader, options).await; let has_redirects_changed = graph.redirects.len() != initial_redirects_len; diff --git a/tests/specs/info/byonm/__test__.jsonc b/tests/specs/info/byonm/__test__.jsonc new file mode 100644 index 0000000000..6c9ba7dada --- /dev/null +++ b/tests/specs/info/byonm/__test__.jsonc @@ -0,0 +1,11 @@ +{ + "tempDir": true, + "steps": [{ + "args": "install", + "output": "[WILDCARD]" + }, { + "args": "info npm:@denotest/add", + "output": "info.out", + "exitCode": 1 + }] +} diff --git a/tests/specs/info/byonm/deno.json b/tests/specs/info/byonm/deno.json new file mode 100644 index 0000000000..4b88c71fbb --- /dev/null +++ b/tests/specs/info/byonm/deno.json @@ -0,0 +1,6 @@ +{ + "nodeModulesDir": "manual", + "imports": { + "chalk": "npm:@denotest/add" + } +} diff --git a/tests/specs/info/byonm/info.out b/tests/specs/info/byonm/info.out new file mode 100644 index 0000000000..4e6f11a88f --- /dev/null +++ b/tests/specs/info/byonm/info.out @@ -0,0 +1 @@ +error: Resolving npm specifier entrypoints this way is currently not supported with "nodeModules": "manual". In the meantime, try with --node-modules-dir=auto instead diff --git a/tests/specs/info/byonm/package.json b/tests/specs/info/byonm/package.json new file mode 100644 index 0000000000..15a786ad71 --- /dev/null +++ b/tests/specs/info/byonm/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "@denotest/add": "*" + } +} From 7ad14589f9e8b4a881cbbfb746c9047d4953e461 Mon Sep 17 00:00:00 2001 From: Leo Kettmeir Date: Mon, 30 Sep 2024 16:00:54 -0700 Subject: [PATCH 013/302] fix(flags): move some content from docs.deno.com into help output (#25951) --- cli/args/flags.rs | 131 +++++++++++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 53 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 2cbf463948..b4f566810c 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -1258,28 +1258,7 @@ pub fn flags_from_vec(args: Vec) -> clap::error::Result { .get_arguments() .any(|arg| arg.get_id().as_str() == "unstable") { - subcommand = subcommand - .mut_arg("unstable", |arg| { - let new_help = arg - .get_help() - .unwrap() - .to_string() - .split_once("\n") - .unwrap() - .0 - .to_string(); - arg.help_heading(UNSTABLE_HEADING).help(new_help) - }) - .mut_args(|arg| { - // long_help here is being used as a metadata, see unstable args definition - if arg.get_help_heading() == Some(UNSTABLE_HEADING) - && arg.get_long_help().is_some() - { - arg.hide(false) - } else { - arg - } - }); + subcommand = enable_unstable(subcommand); } help_parse(&mut flags, subcommand); @@ -1414,6 +1393,31 @@ pub fn flags_from_vec(args: Vec) -> clap::error::Result { Ok(flags) } +fn enable_unstable(command: Command) -> Command { + command + .mut_arg("unstable", |arg| { + let new_help = arg + .get_help() + .unwrap() + .to_string() + .split_once("\n") + .unwrap() + .0 + .to_string(); + arg.help_heading(UNSTABLE_HEADING).help(new_help) + }) + .mut_args(|arg| { + // long_help here is being used as a metadata, see unstable args definition + if arg.get_help_heading() == Some(UNSTABLE_HEADING) + && arg.get_long_help().is_some() + { + arg.hide(false) + } else { + arg + } + }) +} + macro_rules! heading { ($($name:ident = $title:expr),+; $total:literal) => { $(const $name: &str = $title;)+ @@ -1852,11 +1856,15 @@ fn compile_subcommand() -> Command { "compile", cstr!("Compiles the given script into a self contained executable. - deno compile -A jsr:@std/http/file-server + deno compile --allow-read --allow-net jsr:@std/http/file-server deno compile --output file_server jsr:@std/http/file-server Any flags specified which affect runtime behavior will be applied to the resulting binary. +This allows distribution of a Deno application to systems that do not have Deno installed. +Under the hood, it bundles a slimmed down version of the Deno runtime along with your +JavaScript or TypeScript code. + Cross-compiling to different target architectures is supported using the --target flag. On the first invocation with deno will download the proper binary and cache it in $DENO_DIR. @@ -2223,6 +2231,9 @@ Supported file types which are behind corresponding unstable flags (see formatti Format stdin and write to stdout: cat file.ts | deno fmt - +Check if the files are formatted: + deno fmt --check + Ignore formatting code by preceding it with an ignore comment: // deno-fmt-ignore @@ -2373,7 +2384,7 @@ Ignore formatting a file by adding an ignore comment at the top of the file: } fn init_subcommand() -> Command { - command("init", "Initialize a new project", UnstableArgsConfig::None).defer( + command("init", "scaffolds a basic Deno project with a script, test, and configuration file", UnstableArgsConfig::None).defer( |cmd| { cmd .arg(Arg::new("dir").value_hint(ValueHint::DirPath)) @@ -2418,7 +2429,7 @@ The following information is shown: .arg( location_arg() .conflicts_with("file") - .help("Show files used for origin bound APIs like the Web Storage API when running a script with '--location='") + .help(cstr!("Show files used for origin bound APIs like the Web Storage API when running a script with --location=<>")) ) .arg(no_check_arg().hide(true)) // TODO(lucacasonato): remove for 2.0 .arg(no_config_arg()) @@ -2460,7 +2471,7 @@ If the --global flag is set, installs a script as an executable in the deno install --global --allow-net --allow-read jsr:@std/http/file-server deno install -g https://examples.deno.land/color-logging.ts -To change the executable name, use -n/--name: +To change the executable name, use -n/--name: deno install -g --allow-net --allow-read -n serve jsr:@std/http/file-server The executable name is inferred by default: @@ -2742,7 +2753,12 @@ To ignore linting on an entire file, you can add an ignore comment at the top of } fn repl_subcommand() -> Command { - command("repl", "Read Eval Print Loop", UnstableArgsConfig::ResolutionAndRuntime) + command("repl", cstr!( + "Starts a read-eval-print-loop, which lets you interactively build up program state in the global context. +It is especially useful for quick prototyping and checking snippets of code. + +TypeScript is supported, however it is not type-checked, only transpiled." + ), UnstableArgsConfig::ResolutionAndRuntime) .defer(|cmd| runtime_args(cmd, true, true) .arg(check_arg(false)) .arg( @@ -2825,8 +2841,6 @@ fn serve_subcommand() -> Command { The serve command uses the default exports of the main module to determine which servers to start. -See https://docs.deno.com/runtime/manual/tools/serve for more detailed information. - Start a server defined in server.ts: deno serve server.ts @@ -2837,7 +2851,7 @@ Start a server defined in server.ts, watching for changes and running on port 50 .arg( Arg::new("port") .long("port") - .help("The TCP port to serve on, defaulting to 8000. Pass 0 to pick a random free port") + .help(cstr!("The TCP port to serve on. Pass 0 to pick a random free port [default: 8000]")) .value_parser(value_parser!(u16)), ) .arg( @@ -3047,11 +3061,13 @@ fn parallel_arg(descr: &str) -> Arg { fn types_subcommand() -> Command { command( "types", - "Print runtime TypeScript declarations. + cstr!( + "Print runtime TypeScript declarations. deno types > lib.deno.d.ts -The declaration file could be saved and used for typing information.", +The declaration file could be saved and used for typing information." + ), UnstableArgsConfig::None, ) } @@ -3161,7 +3177,7 @@ See the Deno 1.x to 2.x Migration Guide for migration instructions: https://docs } fn publish_subcommand() -> Command { - command("publish", "Publish the current working directory's package or workspace", UnstableArgsConfig::ResolutionOnly) + command("publish", "Publish the current working directory's package or workspace to JSR", UnstableArgsConfig::ResolutionOnly) .defer(|cmd| { cmd .arg( @@ -3550,8 +3566,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { .long("allow-hrtime") .action(ArgAction::SetTrue) .help("REMOVED in Deno 2.0") - .hide(true) - ; + .hide(true); if let Some(requires) = requires { arg = arg.requires(requires) } @@ -3564,8 +3579,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { .long("deny-hrtime") .action(ArgAction::SetTrue) .help("REMOVED in Deno 2.0") - .hide(true) - ; + .hide(true); if let Some(requires) = requires { arg = arg.requires(requires) } @@ -3801,7 +3815,9 @@ fn location_arg() -> Arg { url.set_password(None).unwrap(); Ok(url) }) - .help("Value of 'globalThis.location' used by some web APIs") + .help(cstr!( + "Value of globalThis.location used by some web APIs" + )) .value_hint(ValueHint::Url) } @@ -4655,7 +4671,7 @@ fn json_reference_parse( app.build(); fn serialize_command( - command: &mut Command, + mut command: Command, top_level: bool, ) -> deno_core::serde_json::Value { let args = command @@ -4663,7 +4679,7 @@ fn json_reference_parse( .filter(|arg| { !arg.is_hide_set() && if top_level { - true + arg.is_global_set() } else { !arg.is_global_set() } @@ -4672,40 +4688,49 @@ fn json_reference_parse( let name = arg.get_id().as_str(); let short = arg.get_short(); let long = arg.get_long(); - let aliases = arg.get_visible_aliases(); let required = arg.is_required_set(); - let help = arg.get_help().map(|help| help.to_string()); + let help = arg.get_help().map(|help| help.ansi().to_string()); + let help_heading = arg + .get_help_heading() + .map(|help_heading| help_heading.to_string()); let usage = arg.to_string(); json!({ "name": name, "short": short, "long": long, - "aliases": aliases, "required": required, "help": help, + "help_heading": help_heading, "usage": usage, }) }) .collect::>(); let name = command.get_name().to_string(); - let about = command.get_about().map(|about| about.to_string()); - let visible_aliases = command - .get_visible_aliases() - .map(|s| s.to_string()) - .collect::>(); - let usage = command.render_usage().to_string(); + let about = command.get_about().map(|about| about.ansi().to_string()); + let usage = command.render_usage().ansi().to_string(); let subcommands = command - .get_subcommands_mut() - .map(|command| serialize_command(command, false)) + .get_subcommands() + .map(|command| { + serialize_command( + if command + .get_arguments() + .any(|arg| arg.get_id().as_str() == "unstable") + { + enable_unstable(command.clone()) + } else { + command.clone() + }, + false, + ) + }) .collect::>(); json!({ "name": name, "about": about, - "visible_aliases": visible_aliases, "args": args, "subcommands": subcommands, "usage": usage, @@ -4713,7 +4738,7 @@ fn json_reference_parse( } flags.subcommand = DenoSubcommand::JSONReference(JSONReferenceFlags { - json: serialize_command(&mut app, true), + json: serialize_command(app, true), }) } From 218a9bf7ebaabacb7fc2231b15a22f6102d4cd3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 1 Oct 2024 00:19:37 +0100 Subject: [PATCH 014/302] v2.0.0-rc.8 (#25950) --- Cargo.lock | 2 +- cli/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 945e493486..14ef184180 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1146,7 +1146,7 @@ dependencies = [ [[package]] name = "deno" -version = "2.0.0-rc.7" +version = "2.0.0-rc.8" dependencies = [ "anstream", "async-trait", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 5fb32deb7f..fc786babcf 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno" -version = "2.0.0-rc.7" +version = "2.0.0-rc.8" authors.workspace = true default-run = "deno" edition.workspace = true From 0a75dc70aad9dc7cb51cef9a149a50e340c2eed0 Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Tue, 1 Oct 2024 15:05:55 +0530 Subject: [PATCH 015/302] fix: update patchver to 0.2 (#25952) Fixes https://github.com/denoland/deno/issues/25940 Forgot to update patchver scripts to latest sui. Ref #25942 --- tools/release/promote_to_release.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 tools/release/promote_to_release.ts diff --git a/tools/release/promote_to_release.ts b/tools/release/promote_to_release.ts old mode 100644 new mode 100755 index c14b590cab..046f4d33a8 --- a/tools/release/promote_to_release.ts +++ b/tools/release/promote_to_release.ts @@ -5,7 +5,7 @@ import { $ } from "jsr:@david/dax@0.41.0"; import { gray } from "jsr:@std/fmt@1/colors"; -import { patchver } from "jsr:@deno/patchver@0.1.0"; +import { patchver } from "jsr:@deno/patchver@0.2.0"; const SUPPORTED_TARGETS = [ "aarch64-apple-darwin", From aafe771b55f769d32145489c57fa33eb823716cb Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Tue, 1 Oct 2024 16:18:24 +0530 Subject: [PATCH 016/302] v2.0.0-rc.9 (#25957) --- Cargo.lock | 2 +- cli/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14ef184180..c55b9f97ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1146,7 +1146,7 @@ dependencies = [ [[package]] name = "deno" -version = "2.0.0-rc.8" +version = "2.0.0-rc.9" dependencies = [ "anstream", "async-trait", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index fc786babcf..5b08e82bc5 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno" -version = "2.0.0-rc.8" +version = "2.0.0-rc.9" authors.workspace = true default-run = "deno" edition.workspace = true From c487a86b04dc1b638d204fa4f528fb0b832859ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 1 Oct 2024 14:05:21 +0100 Subject: [PATCH 017/302] fix: Hide 'deno cache' from help output (#25960) `deno cache` was soft-deprecated in favor of `deno install`. It should not show up in the help output. --- cli/args/flags.rs | 1 - tests/integration/flags_tests.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index b4f566810c..25634f3844 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -1176,7 +1176,6 @@ static DENO_HELP: &str = cstr!( Tooling: bench Run benchmarks deno bench bench.ts - cache Cache the dependencies check Type-check the dependencies clean Remove the cache directory compile Compile the script into a self contained executable diff --git a/tests/integration/flags_tests.rs b/tests/integration/flags_tests.rs index 160de3ec1d..455507b9fa 100644 --- a/tests/integration/flags_tests.rs +++ b/tests/integration/flags_tests.rs @@ -21,7 +21,6 @@ fn help_output() { "Install script as an executable", "Uninstall a script previously installed with deno install", "Run benchmarks", - "Cache the dependencies", "Type-check the dependencies", "Compile the script into a self contained executable", "Print coverage reports", From 29104384c4cb5aab082749382802560907ebec42 Mon Sep 17 00:00:00 2001 From: MujahedSafaa <168719085+MujahedSafaa@users.noreply.github.com> Date: Tue, 1 Oct 2024 16:24:12 +0300 Subject: [PATCH 018/302] fix: remove the typo in the help message (#25962) This PR fixes: https://github.com/denoland/deno/issues/25274 Remove the extra shorthand -S that attached to the --deny-sys. --- cli/args/flags.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 25634f3844..6caef29d9f 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -3273,7 +3273,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { --deny-net | --deny-net="localhost:8080,deno.land" --deny-env[=<...] Deny access to environment variables. Optionally specify inacessible environment variables. --deny-env | --deny-env="PORT,HOME,PATH" - -S, --deny-sys[=<...] Deny access to OS information. Optionally deny specific APIs by function name. + --deny-sys[=<...] Deny access to OS information. Optionally deny specific APIs by function name. --deny-sys | --deny-sys="systemMemoryInfo,osRelease" --deny-run[=<...] Deny running subprocesses. Optionally specify denied runnable program names. --deny-run | --deny-run="whoami,ps" From 4c8d57db03bac12d2a13566b3cc7454cfbd67314 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 1 Oct 2024 14:05:40 -0400 Subject: [PATCH 019/302] BREAKING: rename "deps" remote cache folder to "remote" (#25969) Closes https://github.com/denoland/deno/issues/25967 Closes #25968 --- Cargo.lock | 5 +++-- Cargo.toml | 2 +- cli/cache/deno_dir.rs | 6 +++--- cli/cache/mod.rs | 11 ----------- cli/factory.rs | 2 +- cli/file_fetcher.rs | 12 ++++++------ cli/lsp/cache.rs | 2 +- tests/integration/jsr_tests.rs | 2 +- tests/specs/info/flag/041_info_flag.out | 2 +- .../info/flag_location/041_info_flag_location.out | 2 +- tests/specs/info/json/info_json.out | 2 +- .../specs/info/json_location/info_json_location.out | 2 +- 12 files changed, 20 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c55b9f97ca..b55e2aeb39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1348,12 +1348,13 @@ dependencies = [ [[package]] name = "deno_cache_dir" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87900cfcd07bdbf3597bc36b77da0c0e7b6c2e65213faa2ed43d9a1ec12bd31d" +checksum = "186a102b13b4512841f5f40784cd25822042d22954afe3b5b070d406d15eb4f2" dependencies = [ "base32", "deno_media_type", + "deno_path_util", "indexmap", "log", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index ce333c728b..b15c1241b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,7 @@ console_static_text = "=0.8.1" dashmap = "5.5.3" data-encoding = "2.3.3" data-url = "=0.3.0" -deno_cache_dir = "=0.12.0" +deno_cache_dir = "=0.13.0" deno_package_json = { version = "=0.1.1", default-features = false } dlopen2 = "0.6.1" ecb = "=0.1.2" diff --git a/cli/cache/deno_dir.rs b/cli/cache/deno_dir.rs index 88d8a31c04..7b7059c224 100644 --- a/cli/cache/deno_dir.rs +++ b/cli/cache/deno_dir.rs @@ -126,9 +126,9 @@ impl DenoDir { self.root.join("registries") } - /// Path to the dependencies cache folder. - pub fn deps_folder_path(&self) -> PathBuf { - self.root.join("deps") + /// Path to the remote cache folder. + pub fn remote_folder_path(&self) -> PathBuf { + self.root.join("remote") } /// Path to the origin data cache folder. diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 2296bce010..628502c506 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -87,10 +87,6 @@ impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv { std::fs::create_dir_all(path) } - fn remove_file(&self, path: &Path) -> std::io::Result<()> { - std::fs::remove_file(path) - } - fn modified(&self, path: &Path) -> std::io::Result> { match std::fs::metadata(path) { Ok(metadata) => Ok(Some( @@ -149,13 +145,6 @@ impl<'a> deno_cache_dir::DenoCacheEnv for DenoCacheEnvFsAdapter<'a> { .map_err(|e| e.into_io_error()) } - fn remove_file(&self, path: &Path) -> std::io::Result<()> { - self - .0 - .remove_sync(path, false) - .map_err(|e| e.into_io_error()) - } - fn modified(&self, path: &Path) -> std::io::Result> { self .0 diff --git a/cli/factory.rs b/cli/factory.rs index 2cef875999..b96a133e98 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -301,7 +301,7 @@ impl CliFactory { pub fn global_http_cache(&self) -> Result<&Arc, AnyError> { self.services.global_http_cache.get_or_try_init(|| { Ok(Arc::new(GlobalHttpCache::new( - self.deno_dir()?.deps_folder_path(), + self.deno_dir()?.remote_folder_path(), crate::cache::RealDenoCacheEnv, ))) }) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 1bf7635949..69daf14954 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -726,7 +726,7 @@ mod tests { maybe_temp_dir: Option, ) -> (FileFetcher, TempDir, Arc) { let temp_dir = maybe_temp_dir.unwrap_or_default(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let blob_store: Arc = Default::default(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), @@ -964,7 +964,7 @@ mod tests { // This creates a totally new instance, simulating another Deno process // invocation and indicates to "cache bust". - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, @@ -990,7 +990,7 @@ mod tests { async fn test_fetch_uses_cache() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let specifier = resolve_url("http://localhost:4545/subdir/mismatch_ext.ts").unwrap(); @@ -1156,7 +1156,7 @@ mod tests { async fn test_fetch_uses_cache_with_redirects() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let specifier = resolve_url("http://localhost:4548/subdir/mismatch_ext.ts").unwrap(); let redirected_specifier = @@ -1324,7 +1324,7 @@ mod tests { async fn test_fetch_no_remote() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, @@ -1350,7 +1350,7 @@ mod tests { async fn test_fetch_cache_only() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let file_fetcher_01 = FileFetcher::new( Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)), CacheSetting::Only, diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index db10dc9677..fbf9ea6f1b 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -94,7 +94,7 @@ impl LspCache { let deno_dir = DenoDir::new(global_cache_path) .expect("should be infallible with absolute custom root"); let global = Arc::new(GlobalHttpCache::new( - deno_dir.deps_folder_path(), + deno_dir.remote_folder_path(), crate::cache::RealDenoCacheEnv, )); Self { diff --git a/tests/integration/jsr_tests.rs b/tests/integration/jsr_tests.rs index af5b24fe41..c4812e6bfb 100644 --- a/tests/integration/jsr_tests.rs +++ b/tests/integration/jsr_tests.rs @@ -191,7 +191,7 @@ fn reload_info_not_found_cache_but_exists_remote() { Url::parse(&format!("http://127.0.0.1:4250/{}/meta.json", package)) .unwrap(); let cache = deno_cache_dir::GlobalHttpCache::new( - deno_dir.path().join("deps").to_path_buf(), + deno_dir.path().join("remote").to_path_buf(), deno_cache_dir::TestRealDenoCacheEnv, ); let entry = cache diff --git a/tests/specs/info/flag/041_info_flag.out b/tests/specs/info/flag/041_info_flag.out index 3506a29e48..72a00be303 100644 --- a/tests/specs/info/flag/041_info_flag.out +++ b/tests/specs/info/flag/041_info_flag.out @@ -1,5 +1,5 @@ DENO_DIR location: [WILDCARD] -Remote modules cache: [WILDCARD]deps +Remote modules cache: [WILDCARD]remote npm modules cache: [WILDCARD]npm Emitted modules cache: [WILDCARD]gen Language server registries cache: [WILDCARD]registries diff --git a/tests/specs/info/flag_location/041_info_flag_location.out b/tests/specs/info/flag_location/041_info_flag_location.out index b9e72f6590..684db2eec5 100644 --- a/tests/specs/info/flag_location/041_info_flag_location.out +++ b/tests/specs/info/flag_location/041_info_flag_location.out @@ -1,5 +1,5 @@ DENO_DIR location: [WILDCARD] -Remote modules cache: [WILDCARD]deps +Remote modules cache: [WILDCARD]remote npm modules cache: [WILDCARD]npm Emitted modules cache: [WILDCARD]gen Language server registries cache: [WILDCARD]registries diff --git a/tests/specs/info/json/info_json.out b/tests/specs/info/json/info_json.out index 607489ca39..551f61026d 100644 --- a/tests/specs/info/json/info_json.out +++ b/tests/specs/info/json/info_json.out @@ -1,7 +1,7 @@ { "version": 1, "denoDir": "[WILDCARD]", - "modulesCache": "[WILDCARD]deps", + "modulesCache": "[WILDCARD]remote", "npmCache": "[WILDCARD]npm", "typescriptCache": "[WILDCARD]gen", "registryCache": "[WILDCARD]registries", diff --git a/tests/specs/info/json_location/info_json_location.out b/tests/specs/info/json_location/info_json_location.out index 004bf03db1..6d01ba451d 100644 --- a/tests/specs/info/json_location/info_json_location.out +++ b/tests/specs/info/json_location/info_json_location.out @@ -1,7 +1,7 @@ { "version": 1, "denoDir": "[WILDCARD]", - "modulesCache": "[WILDCARD]deps", + "modulesCache": "[WILDCARD]remote", "npmCache": "[WILDCARD]npm", "typescriptCache": "[WILDCARD]gen", "registryCache": "[WILDCARD]registries", From 41a70898adfc61b9020dfdfec17d374aac70d935 Mon Sep 17 00:00:00 2001 From: Ian Bull Date: Tue, 1 Oct 2024 11:26:06 -0700 Subject: [PATCH 020/302] refactor(ext): align error messages (#25914) Aligns the error messages in the ext folder to be in-line with the Deno style guide. https://github.com/denoland/deno/issues/25269 --- ext/net/02_tls.js | 8 +++++--- ext/url/00_url.js | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ext/net/02_tls.js b/ext/net/02_tls.js index 11d19440f7..6dad965590 100644 --- a/ext/net/02_tls.js +++ b/ext/net/02_tls.js @@ -124,17 +124,19 @@ function loadTlsKeyPair(api, { // Check for "pem" format if (keyFormat !== undefined && keyFormat !== "pem") { - throw new TypeError('If `keyFormat` is specified, it must be "pem"'); + throw new TypeError( + `If "keyFormat" is specified, it must be "pem": received "${keyFormat}"`, + ); } if (cert !== undefined && key === undefined) { throw new TypeError( - `If \`cert\` is specified, \`key\` must be specified as well for \`${api}\`.`, + `If \`cert\` is specified, \`key\` must be specified as well for \`${api}\``, ); } if (cert === undefined && key !== undefined) { throw new TypeError( - `If \`key\` is specified, \`cert\` must be specified as well for \`${api}\`.`, + `If \`key\` is specified, \`cert\` must be specified as well for \`${api}\``, ); } diff --git a/ext/url/00_url.js b/ext/url/00_url.js index 577caba902..ec875da768 100644 --- a/ext/url/00_url.js +++ b/ext/url/00_url.js @@ -139,7 +139,7 @@ class URLSearchParams { throw new TypeError( `${prefix}: Item ${ i + 0 - } in the parameter list does have length 2 exactly.`, + } in the parameter list does have length 2 exactly`, ); } return [pair[0], pair[1]]; From f9300004152ba4b3d091beb04d74f37b3b8ec281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 1 Oct 2024 22:49:32 +0100 Subject: [PATCH 021/302] feat: Add suggestion for packages using Node-API addons (#25975) This commit adds a suggestion with information and hint how to resolve situation when user tries to run an npm package with Node-API addons using global cache (which is currently not supported). Closes https://github.com/denoland/deno/issues/25974 --- cli/main.rs | 25 +++++++++++++++++++++++++ runtime/fmt_errors.rs | 42 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/cli/main.rs b/cli/main.rs index c0eccab5df..31bebc882f 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -392,6 +392,31 @@ fn get_suggestions_for_terminal_errors(e: &JsError) -> Vec { "Run again with `--unstable-webgpu` flag to enable this API.", ), ]; + // Try to capture errors like: + // ``` + // Uncaught Error: Cannot find module '../build/Release/canvas.node' + // Require stack: + // - /.../deno/npm/registry.npmjs.org/canvas/2.11.2/lib/bindings.js + // - /.../.cache/deno/npm/registry.npmjs.org/canvas/2.11.2/lib/canvas.js + // ``` + } else if msg.contains("Cannot find module") + && msg.contains("Require stack") + && msg.contains(".node'") + { + return vec![ + FixSuggestion::info_multiline( + &[ + "Trying to execute an npm package using Node-API addons,", + "these packages require local `node_modules` directory to be present." + ] + ), + FixSuggestion::hint_multiline( + &[ + "Add `\"nodeModulesDir\": \"auto\" option to `deno.json`, and then run", + "`deno install --allow-scripts=npm: --entrypoint