From 443041c23e2e02ea59d69e1f2093c67ddfd818fd Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 14 Jun 2022 10:05:37 -0400 Subject: [PATCH] feat(vendor): support using an existing import map (#14836) --- Cargo.lock | 8 +- cli/Cargo.toml | 2 +- cli/config_file.rs | 5 +- cli/fs_util.rs | 72 ++++++ cli/lsp/completions.rs | 6 +- cli/proc_state.rs | 10 +- cli/resolver.rs | 2 +- cli/tests/integration/vendor_tests.rs | 321 +++++++++++++++++------ cli/tests/testdata/vendor/mod.ts | 1 + cli/tools/vendor/build.rs | 349 ++++++++++++++++++++++++-- cli/tools/vendor/import_map.rs | 265 ++++++++++++++----- cli/tools/vendor/mappings.rs | 42 +--- cli/tools/vendor/mod.rs | 263 ++++++++++++++++--- cli/tools/vendor/test.rs | 88 +++++-- test_util/src/temp_dir.rs | 20 ++ 15 files changed, 1198 insertions(+), 256 deletions(-) create mode 100644 cli/tests/testdata/vendor/mod.ts diff --git a/Cargo.lock b/Cargo.lock index 553f373b0b..3cbe90e45e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2069,9 +2069,9 @@ checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" [[package]] name = "import_map" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f99e0f89d56c163538ea6bf1f250049669298a26daeee15a9a18f4118cc503f1" +checksum = "5247edf057fe57036112a1fec3864baa68052b52116760dbea4909115731272f" dependencies = [ "indexmap", "log", @@ -2082,9 +2082,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" dependencies = [ "autocfg 1.1.0", "hashbrown 0.11.2", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 61063fd0b9..56cae20908 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -71,7 +71,7 @@ env_logger = "=0.9.0" eszip = "=0.20.0" fancy-regex = "=0.9.0" http = "=0.2.6" -import_map = "=0.9.0" +import_map = "=0.11.0" indexmap = "1.8.1" jsonc-parser = { version = "=0.19.0", features = ["serde"] } libc = "=0.2.126" diff --git a/cli/config_file.rs b/cli/config_file.rs index 3644bb7c1a..4b2596ba2d 100644 --- a/cli/config_file.rs +++ b/cli/config_file.rs @@ -11,6 +11,7 @@ use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::custom_error; use deno_core::error::AnyError; +use deno_core::normalize_path; use deno_core::serde::Deserialize; use deno_core::serde::Serialize; use deno_core::serde::Serializer; @@ -262,12 +263,12 @@ pub fn resolve_import_map_specifier( // file into a file path if possible and join the import map path to // the file path. if let Ok(config_file_path) = config_file.specifier.to_file_path() { - let import_map_file_path = config_file_path + let import_map_file_path = normalize_path(config_file_path .parent() .ok_or_else(|| { anyhow!("Bad config file specifier: {}", config_file.specifier) })? - .join(&import_map_path); + .join(&import_map_path)); ModuleSpecifier::from_file_path(import_map_file_path).unwrap() // otherwise if the config file is remote, we have no choice but to // use "import resolution" with the config file as the base. diff --git a/cli/fs_util.rs b/cli/fs_util.rs index fe0ef88575..578a2ec371 100644 --- a/cli/fs_util.rs +++ b/cli/fs_util.rs @@ -5,6 +5,7 @@ use deno_core::error::{uri_error, AnyError}; pub use deno_core::normalize_path; use deno_core::ModuleSpecifier; use deno_runtime::deno_crypto::rand; +use std::borrow::Cow; use std::env::current_dir; use std::fs::OpenOptions; use std::io::{Error, Write}; @@ -362,6 +363,44 @@ pub fn specifier_parent(specifier: &ModuleSpecifier) -> ModuleSpecifier { specifier } +/// `from.make_relative(to)` but with fixes. +pub fn relative_specifier( + from: &ModuleSpecifier, + to: &ModuleSpecifier, +) -> Option { + let is_dir = to.path().ends_with('/'); + + if is_dir && from == to { + return Some("./".to_string()); + } + + // workaround using parent directory until https://github.com/servo/rust-url/pull/754 is merged + let from = if !from.path().ends_with('/') { + if let Some(end_slash) = from.path().rfind('/') { + let mut new_from = from.clone(); + new_from.set_path(&from.path()[..end_slash + 1]); + Cow::Owned(new_from) + } else { + Cow::Borrowed(from) + } + } else { + Cow::Borrowed(from) + }; + + // workaround for url crate not adding a trailing slash for a directory + // it seems to be fixed once a version greater than 2.2.2 is released + let mut text = from.make_relative(to)?; + if is_dir && !text.ends_with('/') && to.query().is_none() { + text.push('/'); + } + + Some(if text.starts_with("../") || text.starts_with("./") { + text + } else { + format!("./{}", text) + }) +} + /// This function checks if input path has trailing slash or not. If input path /// has trailing slash it will return true else it will return false. pub fn path_has_trailing_slash(path: &Path) -> bool { @@ -748,6 +787,39 @@ mod tests { } } + #[test] + fn test_relative_specifier() { + run_test("file:///from", "file:///to", Some("./to")); + run_test("file:///from", "file:///from/other", Some("./from/other")); + run_test("file:///from", "file:///from/other/", Some("./from/other/")); + run_test("file:///from", "file:///other/from", Some("./other/from")); + run_test("file:///from/", "file:///other/from", Some("../other/from")); + run_test("file:///from", "file:///other/from/", Some("./other/from/")); + run_test( + "file:///from", + "file:///to/other.txt", + Some("./to/other.txt"), + ); + run_test( + "file:///from/test", + "file:///to/other.txt", + Some("../to/other.txt"), + ); + run_test( + "file:///from/other.txt", + "file:///to/other.txt", + Some("../to/other.txt"), + ); + + fn run_test(from: &str, to: &str, expected: Option<&str>) { + let result = relative_specifier( + &ModuleSpecifier::parse(from).unwrap(), + &ModuleSpecifier::parse(to).unwrap(), + ); + assert_eq!(result.as_deref(), expected); + } + } + #[test] fn test_path_has_trailing_slash() { #[cfg(not(windows))] diff --git a/cli/lsp/completions.rs b/cli/lsp/completions.rs index b3e338fafe..a41e26bf5e 100644 --- a/cli/lsp/completions.rs +++ b/cli/lsp/completions.rs @@ -208,8 +208,8 @@ fn get_base_import_map_completions( import_map: &ImportMap, ) -> Vec { import_map - .imports_keys() - .iter() + .imports() + .keys() .map(|key| { // for some strange reason, keys that start with `/` get stored in the // import map as `file:///`, and so when we pull the keys out, we need to @@ -253,7 +253,7 @@ fn get_import_map_completions( if !text.is_empty() { if let Some(import_map) = maybe_import_map { let mut items = Vec::new(); - for key in import_map.imports_keys() { + for key in import_map.imports().keys() { // for some reason, the import_map stores keys that begin with `/` as // `file:///` in its index, so we have to reverse that here let key = if key.starts_with("file://") { diff --git a/cli/proc_state.rs b/cli/proc_state.rs index d90b3f9526..2c454c0ee7 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -51,7 +51,6 @@ use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_web::BlobStore; use deno_runtime::inspector_server::InspectorServer; use deno_runtime::permissions::Permissions; -use import_map::parse_from_json; use import_map::ImportMap; use log::warn; use std::collections::HashSet; @@ -737,7 +736,12 @@ pub fn import_map_from_text( specifier: &Url, json_text: &str, ) -> Result { - let result = parse_from_json(specifier, json_text)?; + debug_assert!( + !specifier.as_str().contains("../"), + "Import map specifier incorrectly contained ../: {}", + specifier.as_str() + ); + let result = import_map::parse_from_json(specifier, json_text)?; if !result.diagnostics.is_empty() { warn!( "Import map diagnostics:\n{}", @@ -747,7 +751,7 @@ pub fn import_map_from_text( .map(|d| format!(" - {}", d)) .collect::>() .join("\n") - ) + ); } Ok(result.import_map) } diff --git a/cli/resolver.rs b/cli/resolver.rs index af0cc773c2..30149278c4 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -30,7 +30,7 @@ impl Resolver for ImportMapResolver { referrer: &ModuleSpecifier, ) -> ResolveResponse { match self.0.resolve(specifier, referrer) { - Ok(specifier) => ResolveResponse::Specifier(specifier), + Ok(resolved_specifier) => ResolveResponse::Specifier(resolved_specifier), Err(err) => ResolveResponse::Err(err.into()), } } diff --git a/cli/tests/integration/vendor_tests.rs b/cli/tests/integration/vendor_tests.rs index 5737f03655..7c106c79bd 100644 --- a/cli/tests/integration/vendor_tests.rs +++ b/cli/tests/integration/vendor_tests.rs @@ -3,20 +3,19 @@ use deno_core::serde_json; use deno_core::serde_json::json; use pretty_assertions::assert_eq; -use std::fs; use std::path::PathBuf; use std::process::Stdio; use test_util as util; use test_util::TempDir; use util::http_server; +use util::new_deno_dir; #[test] fn output_dir_exists() { let t = TempDir::new(); - let vendor_dir = t.path().join("vendor"); - fs::write(t.path().join("mod.ts"), "").unwrap(); - fs::create_dir_all(&vendor_dir).unwrap(); - fs::write(vendor_dir.join("mod.ts"), "").unwrap(); + t.write("mod.ts", ""); + t.create_dir_all("vendor"); + t.write("vendor/mod.ts", ""); let deno = util::deno_cmd() .current_dir(t.path()) @@ -76,15 +75,12 @@ fn output_dir_exists() { #[test] fn import_map_output_dir() { let t = TempDir::new(); - let vendor_dir = t.path().join("vendor"); - fs::write(t.path().join("mod.ts"), "").unwrap(); - fs::create_dir_all(&vendor_dir).unwrap(); - let import_map_path = vendor_dir.join("import_map.json"); - fs::write( - &import_map_path, + t.write("mod.ts", ""); + t.create_dir_all("vendor"); + t.write( + "vendor/import_map.json", "{ \"imports\": { \"https://localhost/\": \"./localhost/\" }}", - ) - .unwrap(); + ); let deno = util::deno_cmd() .current_dir(t.path()) @@ -92,7 +88,7 @@ fn import_map_output_dir() { .arg("vendor") .arg("--force") .arg("--import-map") - .arg(import_map_path) + .arg("vendor/import_map.json") .arg("mod.ts") .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -101,7 +97,14 @@ fn import_map_output_dir() { let output = deno.wait_with_output().unwrap(); assert_eq!( String::from_utf8_lossy(&output.stderr).trim(), - "error: Using an import map found in the output directory is not supported.", + format!( + concat!( + "error: Specifying an import map file ({}) in the deno vendor ", + "output directory is not supported. Please specify no import ", + "map or one located outside this directory.", + ), + PathBuf::from("vendor").join("import_map.json").display(), + ), ); assert!(!output.status.success()); } @@ -111,10 +114,10 @@ fn standard_test() { let _server = http_server(); let t = TempDir::new(); let vendor_dir = t.path().join("vendor2"); - fs::write( - t.path().join("my_app.ts"), + t.write( + "my_app.ts", "import {Logger} from 'http://localhost:4545/vendor/query_reexport.ts?testing'; new Logger().log('outputted');", - ).unwrap(); + ); let deno = util::deno_cmd() .current_dir(t.path()) @@ -136,7 +139,7 @@ fn standard_test() { "Download http://localhost:4545/vendor/logger.ts?test\n", "{}", ), - success_text("2 modules", "vendor2", "my_app.ts"), + success_text("2 modules", "vendor2", true), ) ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), ""); @@ -144,16 +147,14 @@ fn standard_test() { assert!(vendor_dir.exists()); assert!(!t.path().join("vendor").exists()); - let import_map: serde_json::Value = serde_json::from_str( - &fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(), - ) - .unwrap(); + let import_map: serde_json::Value = + serde_json::from_str(&t.read_to_string("vendor2/import_map.json")).unwrap(); assert_eq!( import_map, json!({ "imports": { - "http://localhost:4545/": "./localhost_4545/", "http://localhost:4545/vendor/query_reexport.ts?testing": "./localhost_4545/vendor/query_reexport.ts", + "http://localhost:4545/": "./localhost_4545/", }, "scopes": { "./localhost_4545/": { @@ -169,7 +170,8 @@ fn standard_test() { .env("NO_COLOR", "1") .arg("run") .arg("--no-remote") - .arg("--no-check") + .arg("--check") + .arg("--quiet") .arg("--import-map") .arg("vendor2/import_map.json") .arg("my_app.ts") @@ -207,7 +209,7 @@ fn remote_module_test() { "Download http://localhost:4545/vendor/logger.ts?test\n", "{}", ), - success_text("2 modules", "vendor/", "main.ts"), + success_text("2 modules", "vendor/", true), ) ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), ""); @@ -217,10 +219,8 @@ fn remote_module_test() { .join("localhost_4545/vendor/query_reexport.ts") .exists()); assert!(vendor_dir.join("localhost_4545/vendor/logger.ts").exists()); - let import_map: serde_json::Value = serde_json::from_str( - &fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(), - ) - .unwrap(); + let import_map: serde_json::Value = + serde_json::from_str(&t.read_to_string("vendor/import_map.json")).unwrap(); assert_eq!( import_map, json!({ @@ -229,7 +229,7 @@ fn remote_module_test() { }, "scopes": { "./localhost_4545/": { - "./localhost_4545/vendor/logger.ts?test": "./localhost_4545/vendor/logger.ts" + "./localhost_4545/vendor/logger.ts?test": "./localhost_4545/vendor/logger.ts", } } }), @@ -237,49 +237,155 @@ fn remote_module_test() { } #[test] -fn existing_import_map() { +fn existing_import_map_no_remote() { let _server = http_server(); let t = TempDir::new(); - let vendor_dir = t.path().join("vendor"); - fs::write( - t.path().join("mod.ts"), + t.write( + "mod.ts", "import {Logger} from 'http://localhost:4545/vendor/logger.ts';", - ) - .unwrap(); - fs::write( - t.path().join("imports.json"), - r#"{ "imports": { "http://localhost:4545/vendor/": "./logger/" } }"#, - ) - .unwrap(); - fs::create_dir(t.path().join("logger")).unwrap(); - fs::write(t.path().join("logger/logger.ts"), "export class Logger {}") - .unwrap(); + ); + let import_map_filename = "imports2.json"; + let import_map_text = + r#"{ "imports": { "http://localhost:4545/vendor/": "./logger/" } }"#; + t.write(import_map_filename, &import_map_text); + t.create_dir_all("logger"); + t.write("logger/logger.ts", "export class Logger {}"); - let status = util::deno_cmd() + let deno = util::deno_cmd() .current_dir(t.path()) + .env("NO_COLOR", "1") .arg("vendor") .arg("mod.ts") .arg("--import-map") - .arg("imports.json") + .arg(import_map_filename) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8_lossy(&output.stderr).trim(), + success_text("0 modules", "vendor/", false) + ); + assert!(output.status.success()); + // it should not have found any remote dependencies because + // the provided import map mapped it to a local directory + assert_eq!(t.read_to_string(import_map_filename), import_map_text); +} + +#[test] +fn existing_import_map_mixed_with_remote() { + let _server = http_server(); + let deno_dir = new_deno_dir(); + let t = TempDir::new(); + t.write( + "mod.ts", + "import {Logger} from 'http://localhost:4545/vendor/logger.ts';", + ); + + let status = util::deno_cmd_with_deno_dir(&deno_dir) + .current_dir(t.path()) + .arg("vendor") + .arg("mod.ts") + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); + + assert_eq!( + t.read_to_string("vendor/import_map.json"), + r#"{ + "imports": { + "http://localhost:4545/": "./localhost_4545/" + } +} +"#, + ); + + // make the import map specific to support vendoring mod.ts in the next step + t.write( + "vendor/import_map.json", + r#"{ + "imports": { + "http://localhost:4545/vendor/logger.ts": "./localhost_4545/vendor/logger.ts" + } +} +"#, + ); + + t.write( + "mod.ts", + concat!( + "import {Logger} from 'http://localhost:4545/vendor/logger.ts';\n", + "import {Logger as OtherLogger} from 'http://localhost:4545/vendor/mod.ts';\n", + ), + ); + + // now vendor with the existing import map in a separate vendor directory + let deno = util::deno_cmd_with_deno_dir(&deno_dir) + .env("NO_COLOR", "1") + .current_dir(t.path()) + .arg("vendor") + .arg("mod.ts") + .arg("--import-map") + .arg("vendor/import_map.json") + .arg("--output") + .arg("vendor2") + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8_lossy(&output.stderr).trim(), + format!( + concat!("Download http://localhost:4545/vendor/mod.ts\n", "{}",), + success_text("1 module", "vendor2", true), + ) + ); + assert!(output.status.success()); + + // tricky scenario here where the output directory now contains a mapping + // back to the previous vendor location + assert_eq!( + t.read_to_string("vendor2/import_map.json"), + r#"{ + "imports": { + "http://localhost:4545/vendor/logger.ts": "../vendor/localhost_4545/vendor/logger.ts", + "http://localhost:4545/": "./localhost_4545/" + }, + "scopes": { + "./localhost_4545/": { + "./localhost_4545/vendor/logger.ts": "../vendor/localhost_4545/vendor/logger.ts" + } + } +} +"#, + ); + + // ensure it runs + let status = util::deno_cmd() + .current_dir(t.path()) + .arg("run") + .arg("--check") + .arg("--no-remote") + .arg("--import-map") + .arg("vendor2/import_map.json") + .arg("mod.ts") .spawn() .unwrap() .wait() .unwrap(); assert!(status.success()); - // it should not have found any remote dependencies because - // the provided import map mapped it to a local directory - assert!(!vendor_dir.join("import_map.json").exists()); } #[test] fn dynamic_import() { let _server = http_server(); let t = TempDir::new(); - let vendor_dir = t.path().join("vendor"); - fs::write( - t.path().join("mod.ts"), + t.write( + "mod.ts", "import {Logger} from 'http://localhost:4545/vendor/dynamic.ts'; new Logger().log('outputted');", - ).unwrap(); + ); let status = util::deno_cmd() .current_dir(t.path()) @@ -290,10 +396,8 @@ fn dynamic_import() { .wait() .unwrap(); assert!(status.success()); - let import_map: serde_json::Value = serde_json::from_str( - &fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(), - ) - .unwrap(); + let import_map: serde_json::Value = + serde_json::from_str(&t.read_to_string("vendor/import_map.json")).unwrap(); assert_eq!( import_map, json!({ @@ -310,7 +414,8 @@ fn dynamic_import() { .arg("run") .arg("--allow-read=.") .arg("--no-remote") - .arg("--no-check") + .arg("--check") + .arg("--quiet") .arg("--import-map") .arg("vendor/import_map.json") .arg("mod.ts") @@ -328,10 +433,10 @@ fn dynamic_import() { fn dynamic_non_analyzable_import() { let _server = http_server(); let t = TempDir::new(); - fs::write( - t.path().join("mod.ts"), + t.write( + "mod.ts", "import {Logger} from 'http://localhost:4545/vendor/dynamic_non_analyzable.ts'; new Logger().log('outputted');", - ).unwrap(); + ); let deno = util::deno_cmd() .current_dir(t.path()) @@ -350,23 +455,89 @@ fn dynamic_non_analyzable_import() { String::from_utf8_lossy(&output.stderr).trim(), format!( "Download http://localhost:4545/vendor/dynamic_non_analyzable.ts\n{}", - success_text("1 module", "vendor/", "mod.ts"), + success_text("1 module", "vendor/", true), ) ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), ""); assert!(output.status.success()); } -fn success_text(module_count: &str, dir: &str, entry_point: &str) -> String { - format!( - concat!( - "Vendored {} into {} directory.\n\n", - "To use vendored modules, specify the `--import-map` flag when invoking deno subcommands:\n", - " deno run -A --import-map {} {}" - ), - module_count, - dir, - PathBuf::from(dir).join("import_map.json").display(), - entry_point, - ) +#[test] +fn update_existing_config_test() { + let _server = http_server(); + let t = TempDir::new(); + t.write( + "my_app.ts", + "import {Logger} from 'http://localhost:4545/vendor/logger.ts'; new Logger().log('outputted');", + ); + t.write("deno.json", "{\n}"); + + let deno = util::deno_cmd() + .current_dir(t.path()) + .arg("vendor") + .arg("my_app.ts") + .arg("--output") + .arg("vendor2") + .env("NO_COLOR", "1") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8_lossy(&output.stderr).trim(), + format!( + concat!( + "Download http://localhost:4545/vendor/logger.ts\n", + "Vendored 1 module into vendor2 directory.\n\n", + "Updated your local Deno configuration file with a reference to the ", + "new vendored import map at {}. Invoking Deno subcommands will ", + "now automatically resolve using the vendored modules. You may override ", + "this by providing the `--import-map ` flag or by ", + "manually editing your Deno configuration file." + ), + PathBuf::from("vendor2").join("import_map.json").display(), + ) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), ""); + assert!(output.status.success()); + + // try running the output with `--no-remote` and not specifying a `--vendor` + let deno = util::deno_cmd() + .current_dir(t.path()) + .env("NO_COLOR", "1") + .arg("run") + .arg("--no-remote") + .arg("--check") + .arg("--quiet") + .arg("my_app.ts") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), ""); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "outputted"); + assert!(output.status.success()); +} + +fn success_text(module_count: &str, dir: &str, has_import_map: bool) -> String { + let mut text = format!("Vendored {} into {} directory.", module_count, dir); + if has_import_map { + text.push_str(& + format!( + concat!( + "\n\nTo use vendored modules, specify the `--import-map {}import_map.json` flag when ", + r#"invoking Deno subcommands or add an `"importMap": ""` "#, + "entry to a deno.json file.", + ), + if dir != "vendor/" { + format!("{}{}", dir.trim_end_matches('/'), if cfg!(windows) { '\\' } else {'/'}) + } else { + dir.to_string() + } + ) + ); + } + text } diff --git a/cli/tests/testdata/vendor/mod.ts b/cli/tests/testdata/vendor/mod.ts new file mode 100644 index 0000000000..8824d1b2ae --- /dev/null +++ b/cli/tests/testdata/vendor/mod.ts @@ -0,0 +1 @@ +export * from "./logger.ts"; diff --git a/cli/tools/vendor/build.rs b/cli/tools/vendor/build.rs index dd362ebfbb..ecb7db717c 100644 --- a/cli/tools/vendor/build.rs +++ b/cli/tools/vendor/build.rs @@ -1,11 +1,17 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use std::path::Path; +use std::path::PathBuf; +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::ModuleKind; +use import_map::ImportMap; +use import_map::SpecifierMap; use super::analyze::has_default_export; use super::import_map::build_import_map; @@ -15,29 +21,53 @@ use super::specifiers::is_remote_specifier; /// Allows substituting the environment for testing purposes. pub trait VendorEnvironment { + fn cwd(&self) -> Result; fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError>; fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError>; + fn path_exists(&self, path: &Path) -> bool; } pub struct RealVendorEnvironment; impl VendorEnvironment for RealVendorEnvironment { + fn cwd(&self) -> Result { + Ok(std::env::current_dir()?) + } + fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> { Ok(std::fs::create_dir_all(dir_path)?) } fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError> { - Ok(std::fs::write(file_path, text)?) + std::fs::write(file_path, text) + .with_context(|| format!("Failed writing {}", file_path.display())) + } + + fn path_exists(&self, path: &Path) -> bool { + path.exists() } } /// Vendors remote modules and returns how many were vendored. pub fn build( - graph: &ModuleGraph, + graph: ModuleGraph, output_dir: &Path, + original_import_map: Option<&ImportMap>, environment: &impl VendorEnvironment, ) -> Result { assert!(output_dir.is_absolute()); + let output_dir_specifier = + ModuleSpecifier::from_directory_path(output_dir).unwrap(); + + if let Some(original_im) = &original_import_map { + validate_original_import_map(original_im, &output_dir_specifier)?; + } + + // build the graph + graph.lock()?; + graph.valid()?; + + // figure out how to map remote modules to local let all_modules = graph.modules(); let remote_modules = all_modules .iter() @@ -45,7 +75,7 @@ pub fn build( .copied() .collect::>(); let mappings = - Mappings::from_remote_modules(graph, &remote_modules, output_dir)?; + Mappings::from_remote_modules(&graph, &remote_modules, output_dir)?; // write out all the files for module in &remote_modules { @@ -77,16 +107,59 @@ pub fn build( environment.write_file(&proxy_path, &text)?; } - // create the import map - if !mappings.base_specifiers().is_empty() { - let import_map_text = build_import_map(graph, &all_modules, &mappings); - environment - .write_file(&output_dir.join("import_map.json"), &import_map_text)?; + // create the import map if necessary + if !remote_modules.is_empty() { + let import_map_path = output_dir.join("import_map.json"); + let import_map_text = build_import_map( + &output_dir_specifier, + &graph, + &all_modules, + &mappings, + original_import_map, + ); + environment.write_file(&import_map_path, &import_map_text)?; } Ok(remote_modules.len()) } +fn validate_original_import_map( + import_map: &ImportMap, + output_dir: &ModuleSpecifier, +) -> Result<(), AnyError> { + fn validate_imports( + imports: &SpecifierMap, + output_dir: &ModuleSpecifier, + ) -> Result<(), AnyError> { + for entry in imports.entries() { + if let Some(value) = entry.value { + if value.as_str().starts_with(output_dir.as_str()) { + bail!( + "Providing an existing import map with entries for the output directory is not supported (\"{}\": \"{}\").", + entry.raw_key, + entry.raw_value.unwrap_or(""), + ); + } + } + } + Ok(()) + } + + validate_imports(import_map.imports(), output_dir)?; + + for scope in import_map.scopes() { + if scope.key.starts_with(output_dir.as_str()) { + bail!( + "Providing an existing import map with a scope for the output directory is not supported (\"{}\").", + scope.raw_key, + ); + } + validate_imports(scope.imports, output_dir)?; + } + + Ok(()) +} + fn build_proxy_module_source( module: &Module, proxied_module: &ProxiedModule, @@ -171,9 +244,9 @@ mod test { output.import_map, Some(json!({ "imports": { - "https://localhost/": "./localhost/", "https://localhost/other.ts?test": "./localhost/other.ts", "https://localhost/redirect.ts": "./localhost/mod.ts", + "https://localhost/": "./localhost/", } })) ); @@ -241,7 +314,7 @@ mod test { "imports": { "https://localhost/": "./localhost/", "https://localhost/redirect.ts": "./localhost/other.ts", - "https://other/": "./other/" + "https://other/": "./other/", }, "scopes": { "./localhost/": { @@ -313,10 +386,10 @@ mod test { output.import_map, Some(json!({ "imports": { - "https://localhost/": "./localhost/", "https://localhost/mod.TS": "./localhost/mod_2.TS", "https://localhost/mod.ts": "./localhost/mod_3.ts", "https://localhost/mod.ts?test": "./localhost/mod_4.ts", + "https://localhost/": "./localhost/", } })) ); @@ -543,8 +616,8 @@ mod test { output.import_map, Some(json!({ "imports": { - "http://localhost:4545/": "./localhost_4545/", "http://localhost:4545/sub/logger/mod.ts?testing": "./localhost_4545/sub/logger/mod.ts", + "http://localhost:4545/": "./localhost_4545/", }, "scopes": { "./localhost_4545/": { @@ -599,9 +672,9 @@ mod test { output.import_map, Some(json!({ "imports": { + "https://localhost/std/hash/mod.ts": "./localhost/std@0.1.0/hash/mod.ts", "https://localhost/": "./localhost/", - "https://localhost/std/hash/mod.ts": "./localhost/std@0.1.0/hash/mod.ts" - } + }, })) ); assert_eq!( @@ -675,6 +748,254 @@ mod test { ); } + #[tokio::test] + async fn existing_import_map_basic() { + let mut builder = VendorTestBuilder::with_default_setup(); + let mut original_import_map = builder.new_import_map("/import_map2.json"); + original_import_map + .imports_mut() + .append( + "https://localhost/mod.ts".to_string(), + "./local_vendor/mod.ts".to_string(), + ) + .unwrap(); + let local_vendor_scope = original_import_map + .get_or_append_scope_mut("./local_vendor/") + .unwrap(); + local_vendor_scope + .append( + "https://localhost/logger.ts".to_string(), + "./local_vendor/logger.ts".to_string(), + ) + .unwrap(); + local_vendor_scope + .append( + "/console_logger.ts".to_string(), + "./local_vendor/console_logger.ts".to_string(), + ) + .unwrap(); + + let output = builder + .with_loader(|loader| { + loader.add("/mod.ts", "import 'https://localhost/mod.ts'; import 'https://localhost/other.ts';"); + loader.add("/local_vendor/mod.ts", "import 'https://localhost/logger.ts'; import '/console_logger.ts'; console.log(5);"); + loader.add("/local_vendor/logger.ts", "export class Logger {}"); + loader.add("/local_vendor/console_logger.ts", "export class ConsoleLogger {}"); + loader.add("https://localhost/mod.ts", "console.log(6);"); + loader.add("https://localhost/other.ts", "import './mod.ts';"); + }) + .set_original_import_map(original_import_map.clone()) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/mod.ts": "../local_vendor/mod.ts", + "https://localhost/": "./localhost/" + }, + "scopes": { + "../local_vendor/": { + "https://localhost/logger.ts": "../local_vendor/logger.ts", + "/console_logger.ts": "../local_vendor/console_logger.ts", + }, + "./localhost/": { + "./localhost/mod.ts": "../local_vendor/mod.ts", + }, + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[("/vendor/localhost/other.ts", "import './mod.ts';")]), + ); + } + + #[tokio::test] + async fn existing_import_map_mapped_bare_specifier() { + let mut builder = VendorTestBuilder::with_default_setup(); + let mut original_import_map = builder.new_import_map("/import_map.json"); + let imports = original_import_map.imports_mut(); + imports + .append("$fresh".to_string(), "https://localhost/fresh".to_string()) + .unwrap(); + imports + .append("std/".to_string(), "https://deno.land/std/".to_string()) + .unwrap(); + let output = builder + .with_loader(|loader| { + loader.add("/mod.ts", "import 'std/mod.ts'; import '$fresh';"); + loader.add("https://deno.land/std/mod.ts", "export function test() {}"); + loader.add_with_headers( + "https://localhost/fresh", + "export function fresh() {}", + &[("content-type", "application/typescript")], + ); + }) + .set_original_import_map(original_import_map.clone()) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://deno.land/": "./deno.land/", + "https://localhost/": "./localhost/", + "$fresh": "./localhost/fresh.ts", + "std/mod.ts": "./deno.land/std/mod.ts", + }, + })) + ); + assert_eq!( + output.files, + to_file_vec(&[ + ("/vendor/deno.land/std/mod.ts", "export function test() {}"), + ("/vendor/localhost/fresh.ts", "export function fresh() {}") + ]), + ); + } + + #[tokio::test] + async fn existing_import_map_remote_absolute_specifier_local() { + let mut builder = VendorTestBuilder::with_default_setup(); + let mut original_import_map = builder.new_import_map("/import_map.json"); + original_import_map + .imports_mut() + .append( + "https://localhost/logger.ts?test".to_string(), + "./local/logger.ts".to_string(), + ) + .unwrap(); + + let output = builder + .with_loader(|loader| { + loader.add("/mod.ts", "import 'https://localhost/mod.ts'; import 'https://localhost/logger.ts?test';"); + loader.add("/local/logger.ts", "export class Logger {}"); + // absolute specifier in a remote module that will point at ./local/logger.ts + loader.add("https://localhost/mod.ts", "import '/logger.ts?test';"); + loader.add("https://localhost/logger.ts?test", "export class Logger {}"); + }) + .set_original_import_map(original_import_map.clone()) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/logger.ts?test": "../local/logger.ts", + "https://localhost/": "./localhost/", + }, + "scopes": { + "./localhost/": { + "/logger.ts?test": "../local/logger.ts", + }, + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[("/vendor/localhost/mod.ts", "import '/logger.ts?test';")]), + ); + } + + #[tokio::test] + async fn existing_import_map_imports_output_dir() { + let mut builder = VendorTestBuilder::with_default_setup(); + let mut original_import_map = builder.new_import_map("/import_map.json"); + original_import_map + .imports_mut() + .append( + "std/mod.ts".to_string(), + "./vendor/deno.land/std/mod.ts".to_string(), + ) + .unwrap(); + let err = builder + .with_loader(|loader| { + loader.add("/mod.ts", "import 'std/mod.ts';"); + loader.add("/vendor/deno.land/std/mod.ts", "export function f() {}"); + loader.add("https://deno.land/std/mod.ts", "export function f() {}"); + }) + .set_original_import_map(original_import_map.clone()) + .build() + .await + .err() + .unwrap(); + + assert_eq!( + err.to_string(), + concat!( + "Providing an existing import map with entries for the output ", + "directory is not supported ", + "(\"std/mod.ts\": \"./vendor/deno.land/std/mod.ts\").", + ) + ); + } + + #[tokio::test] + async fn existing_import_map_scopes_entry_output_dir() { + let mut builder = VendorTestBuilder::with_default_setup(); + let mut original_import_map = builder.new_import_map("/import_map.json"); + let scopes = original_import_map + .get_or_append_scope_mut("./other/") + .unwrap(); + scopes + .append("/mod.ts".to_string(), "./vendor/mod.ts".to_string()) + .unwrap(); + let err = builder + .with_loader(|loader| { + loader.add("/mod.ts", "console.log(5);"); + }) + .set_original_import_map(original_import_map.clone()) + .build() + .await + .err() + .unwrap(); + + assert_eq!( + err.to_string(), + concat!( + "Providing an existing import map with entries for the output ", + "directory is not supported ", + "(\"/mod.ts\": \"./vendor/mod.ts\").", + ) + ); + } + + #[tokio::test] + async fn existing_import_map_scopes_key_output_dir() { + let mut builder = VendorTestBuilder::with_default_setup(); + let mut original_import_map = builder.new_import_map("/import_map.json"); + let scopes = original_import_map + .get_or_append_scope_mut("./vendor/") + .unwrap(); + scopes + .append("/mod.ts".to_string(), "./vendor/mod.ts".to_string()) + .unwrap(); + let err = builder + .with_loader(|loader| { + loader.add("/mod.ts", "console.log(5);"); + }) + .set_original_import_map(original_import_map.clone()) + .build() + .await + .err() + .unwrap(); + + assert_eq!( + err.to_string(), + concat!( + "Providing an existing import map with a scope for the output ", + "directory is not supported (\"./vendor/\").", + ) + ); + } + fn to_file_vec(items: &[(&str, &str)]) -> Vec<(String, String)> { items .iter() diff --git a/cli/tools/vendor/import_map.rs b/cli/tools/vendor/import_map.rs index 1b2a2e2634..e03260e3e3 100644 --- a/cli/tools/vendor/import_map.rs +++ b/cli/tools/vendor/import_map.rs @@ -1,44 +1,43 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use std::collections::BTreeMap; - use deno_ast::LineAndColumnIndex; use deno_ast::ModuleSpecifier; use deno_ast::SourceTextInfo; -use deno_core::serde_json; use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::Position; use deno_graph::Range; use deno_graph::Resolved; -use serde::Serialize; +use import_map::ImportMap; +use import_map::SpecifierMap; +use indexmap::IndexMap; +use log::warn; use super::mappings::Mappings; use super::specifiers::is_remote_specifier; use super::specifiers::is_remote_specifier_text; -#[derive(Serialize)] -struct SerializableImportMap { - imports: BTreeMap, - #[serde(skip_serializing_if = "BTreeMap::is_empty")] - scopes: BTreeMap>, -} - struct ImportMapBuilder<'a> { + base_dir: &'a ModuleSpecifier, mappings: &'a Mappings, imports: ImportsBuilder<'a>, - scopes: BTreeMap>, + scopes: IndexMap>, } impl<'a> ImportMapBuilder<'a> { - pub fn new(mappings: &'a Mappings) -> Self { + pub fn new(base_dir: &'a ModuleSpecifier, mappings: &'a Mappings) -> Self { ImportMapBuilder { + base_dir, mappings, - imports: ImportsBuilder::new(mappings), + imports: ImportsBuilder::new(base_dir, mappings), scopes: Default::default(), } } + pub fn base_dir(&self) -> &ModuleSpecifier { + self.base_dir + } + pub fn scope( &mut self, base_specifier: &ModuleSpecifier, @@ -48,38 +47,115 @@ impl<'a> ImportMapBuilder<'a> { .entry( self .mappings - .relative_specifier_text(self.mappings.output_dir(), base_specifier), + .relative_specifier_text(self.base_dir, base_specifier), ) - .or_insert_with(|| ImportsBuilder::new(self.mappings)) + .or_insert_with(|| ImportsBuilder::new(self.base_dir, self.mappings)) } - pub fn into_serializable(self) -> SerializableImportMap { - SerializableImportMap { - imports: self.imports.imports, - scopes: self - .scopes - .into_iter() - .map(|(key, value)| (key, value.imports)) - .collect(), + pub fn into_import_map( + self, + original_import_map: Option<&ImportMap>, + ) -> ImportMap { + fn get_local_imports( + new_relative_path: &str, + original_imports: &SpecifierMap, + ) -> Vec<(String, String)> { + let mut result = Vec::new(); + for entry in original_imports.entries() { + if let Some(raw_value) = entry.raw_value { + if raw_value.starts_with("./") || raw_value.starts_with("../") { + let sub_index = raw_value.find('/').unwrap() + 1; + result.push(( + entry.raw_key.to_string(), + format!("{}{}", new_relative_path, &raw_value[sub_index..]), + )); + } + } + } + result } - } - pub fn into_file_text(self) -> String { - let mut text = - serde_json::to_string_pretty(&self.into_serializable()).unwrap(); - text.push('\n'); - text + fn add_local_imports<'a>( + new_relative_path: &str, + original_imports: &SpecifierMap, + get_new_imports: impl FnOnce() -> &'a mut SpecifierMap, + ) { + let local_imports = + get_local_imports(new_relative_path, original_imports); + if !local_imports.is_empty() { + let new_imports = get_new_imports(); + for (key, value) in local_imports { + if let Err(warning) = new_imports.append(key, value) { + warn!("{}", warning); + } + } + } + } + + let mut import_map = ImportMap::new(self.base_dir.clone()); + + if let Some(original_im) = original_import_map { + let original_base_dir = ModuleSpecifier::from_directory_path( + original_im + .base_url() + .to_file_path() + .unwrap() + .parent() + .unwrap(), + ) + .unwrap(); + let new_relative_path = self + .mappings + .relative_specifier_text(self.base_dir, &original_base_dir); + // add the imports + add_local_imports(&new_relative_path, original_im.imports(), || { + import_map.imports_mut() + }); + + for scope in original_im.scopes() { + if scope.raw_key.starts_with("./") || scope.raw_key.starts_with("../") { + let sub_index = scope.raw_key.find('/').unwrap() + 1; + let new_key = + format!("{}{}", new_relative_path, &scope.raw_key[sub_index..]); + add_local_imports(&new_relative_path, scope.imports, || { + import_map.get_or_append_scope_mut(&new_key).unwrap() + }); + } + } + } + + let imports = import_map.imports_mut(); + for (key, value) in self.imports.imports { + if !imports.contains(&key) { + imports.append(key, value).unwrap(); + } + } + + for (scope_key, scope_value) in self.scopes { + if !scope_value.imports.is_empty() { + let imports = import_map.get_or_append_scope_mut(&scope_key).unwrap(); + for (key, value) in scope_value.imports { + if !imports.contains(&key) { + imports.append(key, value).unwrap(); + } + } + } + } + + import_map } } struct ImportsBuilder<'a> { + base_dir: &'a ModuleSpecifier, mappings: &'a Mappings, - imports: BTreeMap, + imports: IndexMap, } impl<'a> ImportsBuilder<'a> { - pub fn new(mappings: &'a Mappings) -> Self { + pub fn new(base_dir: &'a ModuleSpecifier, mappings: &'a Mappings) -> Self { Self { + base_dir, mappings, imports: Default::default(), } @@ -88,7 +164,7 @@ impl<'a> ImportsBuilder<'a> { pub fn add(&mut self, key: String, specifier: &ModuleSpecifier) { let value = self .mappings - .relative_specifier_text(self.mappings.output_dir(), specifier); + .relative_specifier_text(self.base_dir, specifier); // skip creating identity entries if key != value { @@ -98,20 +174,22 @@ impl<'a> ImportsBuilder<'a> { } pub fn build_import_map( + base_dir: &ModuleSpecifier, graph: &ModuleGraph, modules: &[&Module], mappings: &Mappings, + original_import_map: Option<&ImportMap>, ) -> String { - let mut import_map = ImportMapBuilder::new(mappings); - visit_modules(graph, modules, mappings, &mut import_map); + let mut builder = ImportMapBuilder::new(base_dir, mappings); + visit_modules(graph, modules, mappings, &mut builder); for base_specifier in mappings.base_specifiers() { - import_map + builder .imports .add(base_specifier.to_string(), base_specifier); } - import_map.into_file_text() + builder.into_import_map(original_import_map).to_json() } fn visit_modules( @@ -197,37 +275,70 @@ fn handle_dep_specifier( mappings: &Mappings, ) { let specifier = graph.resolve(unresolved_specifier); - // do not handle specifiers pointing at local modules - if !is_remote_specifier(&specifier) { - return; + // check if it's referencing a remote module + if is_remote_specifier(&specifier) { + handle_remote_dep_specifier( + text, + unresolved_specifier, + &specifier, + import_map, + referrer, + mappings, + ) + } else { + handle_local_dep_specifier( + text, + unresolved_specifier, + &specifier, + import_map, + referrer, + mappings, + ); } +} - let base_specifier = mappings.base_specifier(&specifier); +fn handle_remote_dep_specifier( + text: &str, + unresolved_specifier: &ModuleSpecifier, + specifier: &ModuleSpecifier, + import_map: &mut ImportMapBuilder, + referrer: &ModuleSpecifier, + mappings: &Mappings, +) { if is_remote_specifier_text(text) { + let base_specifier = mappings.base_specifier(specifier); if !text.starts_with(base_specifier.as_str()) { panic!("Expected {} to start with {}", text, base_specifier); } let sub_path = &text[base_specifier.as_str().len()..]; - let expected_relative_specifier_text = - mappings.relative_path(base_specifier, &specifier); - if expected_relative_specifier_text == sub_path { - return; + let relative_text = + mappings.relative_specifier_text(base_specifier, specifier); + let expected_sub_path = relative_text.trim_start_matches("./"); + if expected_sub_path != sub_path { + import_map.imports.add(text.to_string(), specifier); } - - import_map.imports.add(text.to_string(), &specifier); } else { let expected_relative_specifier_text = - mappings.relative_specifier_text(referrer, &specifier); + mappings.relative_specifier_text(referrer, specifier); if expected_relative_specifier_text == text { return; } + if !is_remote_specifier(referrer) { + // local module referencing a remote module using + // non-remote specifier text means it was something in + // the original import map, so add a mapping to it + import_map.imports.add(text.to_string(), specifier); + return; + } + + let base_specifier = mappings.base_specifier(specifier); + let base_dir = import_map.base_dir().clone(); let imports = import_map.scope(base_specifier); if text.starts_with("./") || text.starts_with("../") { // resolve relative specifier key let mut local_base_specifier = mappings.local_uri(base_specifier); - local_base_specifier.set_query(unresolved_specifier.query()); local_base_specifier = local_base_specifier // path includes "/" so make it relative .join(&format!(".{}", unresolved_specifier.path())) @@ -241,18 +352,15 @@ fn handle_dep_specifier( local_base_specifier.set_query(unresolved_specifier.query()); imports.add( - mappings.relative_specifier_text( - mappings.output_dir(), - &local_base_specifier, - ), - &specifier, + mappings.relative_specifier_text(&base_dir, &local_base_specifier), + specifier, ); // add a mapping that uses the local directory name and the remote // filename in order to support files importing this relatively imports.add( { - let local_path = mappings.local_path(&specifier); + let local_path = mappings.local_path(specifier); let mut value = ModuleSpecifier::from_directory_path(local_path.parent().unwrap()) .unwrap(); @@ -262,17 +370,58 @@ fn handle_dep_specifier( value.path(), specifier.path_segments().unwrap().last().unwrap(), )); - mappings.relative_specifier_text(mappings.output_dir(), &value) + mappings.relative_specifier_text(&base_dir, &value) }, - &specifier, + specifier, ); } else { // absolute (`/`) or bare specifier should be left as-is - imports.add(text.to_string(), &specifier); + imports.add(text.to_string(), specifier); } } } +fn handle_local_dep_specifier( + text: &str, + unresolved_specifier: &ModuleSpecifier, + specifier: &ModuleSpecifier, + import_map: &mut ImportMapBuilder, + referrer: &ModuleSpecifier, + mappings: &Mappings, +) { + if !is_remote_specifier(referrer) { + // do not handle local modules referencing local modules + return; + } + + // The remote module is referencing a local file. This could occur via an + // existing import map. In this case, we'll have to add an import map + // entry in order to map the path back to the local path once vendored. + let base_dir = import_map.base_dir().clone(); + let base_specifier = mappings.base_specifier(referrer); + let imports = import_map.scope(base_specifier); + + if text.starts_with("./") || text.starts_with("../") { + let referrer_local_uri = mappings.local_uri(referrer); + let mut specifier_local_uri = + referrer_local_uri.join(text).unwrap_or_else(|_| { + panic!( + "Error joining {} to {}", + unresolved_specifier.path(), + referrer_local_uri + ) + }); + specifier_local_uri.set_query(unresolved_specifier.query()); + + imports.add( + mappings.relative_specifier_text(&base_dir, &specifier_local_uri), + specifier, + ); + } else { + imports.add(text.to_string(), specifier); + } +} + fn text_from_range<'a>( text_info: &SourceTextInfo, text: &'a str, diff --git a/cli/tools/vendor/mappings.rs b/cli/tools/vendor/mappings.rs index 2e85445dcb..5435361283 100644 --- a/cli/tools/vendor/mappings.rs +++ b/cli/tools/vendor/mappings.rs @@ -14,6 +14,7 @@ use deno_graph::Position; use deno_graph::Resolved; use crate::fs_util::path_with_stem_suffix; +use crate::fs_util::relative_specifier; use super::specifiers::dir_name_for_root; use super::specifiers::get_unique_path; @@ -28,7 +29,6 @@ pub struct ProxiedModule { /// Constructs and holds the remote specifier to local path mappings. pub struct Mappings { - output_dir: ModuleSpecifier, mappings: HashMap, base_specifiers: Vec, proxies: HashMap, @@ -104,17 +104,12 @@ impl Mappings { } Ok(Self { - output_dir: ModuleSpecifier::from_directory_path(output_dir).unwrap(), mappings, base_specifiers, proxies, }) } - pub fn output_dir(&self) -> &ModuleSpecifier { - &self.output_dir - } - pub fn local_uri(&self, specifier: &ModuleSpecifier) -> ModuleSpecifier { if specifier.scheme() == "file" { specifier.clone() @@ -146,43 +141,14 @@ impl Mappings { } } - pub fn relative_path( - &self, - from: &ModuleSpecifier, - to: &ModuleSpecifier, - ) -> String { - let mut from = self.local_uri(from); - let to = self.local_uri(to); - - // workaround using parent directory until https://github.com/servo/rust-url/pull/754 is merged - if !from.path().ends_with('/') { - let local_path = self.local_path(&from); - from = ModuleSpecifier::from_directory_path(local_path.parent().unwrap()) - .unwrap(); - } - - // workaround for url crate not adding a trailing slash for a directory - // it seems to be fixed once a version greater than 2.2.2 is released - let is_dir = to.path().ends_with('/'); - let mut text = from.make_relative(&to).unwrap(); - if is_dir && !text.ends_with('/') && to.query().is_none() { - text.push('/'); - } - text - } - pub fn relative_specifier_text( &self, from: &ModuleSpecifier, to: &ModuleSpecifier, ) -> String { - let relative_path = self.relative_path(from, to); - - if relative_path.starts_with("../") || relative_path.starts_with("./") { - relative_path - } else { - format!("./{}", relative_path) - } + let from = self.local_uri(from); + let to = self.local_uri(to); + relative_specifier(&from, &to).unwrap() } pub fn base_specifiers(&self) -> &Vec { diff --git a/cli/tools/vendor/mod.rs b/cli/tools/vendor/mod.rs index 3a5455aae6..69c759154b 100644 --- a/cli/tools/vendor/mod.rs +++ b/cli/tools/vendor/mod.rs @@ -3,19 +3,24 @@ use std::path::Path; use std::path::PathBuf; +use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_runtime::permissions::Permissions; +use log::warn; +use crate::config_file::FmtOptionsConfig; use crate::flags::VendorFlags; use crate::fs_util; +use crate::fs_util::relative_specifier; +use crate::fs_util::specifier_to_file_path; use crate::lockfile; use crate::proc_state::ProcState; use crate::resolver::ImportMapResolver; use crate::resolver::JsxResolver; -use crate::tools::vendor::specifiers::is_remote_specifier_text; +use crate::tools::fmt::format_json; mod analyze; mod build; @@ -33,14 +38,15 @@ pub async fn vendor(ps: ProcState, flags: VendorFlags) -> Result<(), AnyError> { let output_dir = fs_util::resolve_from_cwd(&raw_output_dir)?; validate_output_dir(&output_dir, &flags, &ps)?; let graph = create_graph(&ps, &flags).await?; - let vendored_count = - build::build(&graph, &output_dir, &build::RealVendorEnvironment)?; + let vendored_count = build::build( + graph, + &output_dir, + ps.maybe_import_map.as_deref(), + &build::RealVendorEnvironment, + )?; eprintln!( - r#"Vendored {} {} into {} directory. - -To use vendored modules, specify the `--import-map` flag when invoking deno subcommands: - deno run -A --import-map {} {}"#, + concat!("Vendored {} {} into {} directory.",), vendored_count, if vendored_count == 1 { "module" @@ -48,14 +54,31 @@ To use vendored modules, specify the `--import-map` flag when invoking deno subc "modules" }, raw_output_dir.display(), - raw_output_dir.join("import_map.json").display(), - flags - .specifiers - .iter() - .map(|s| s.as_str()) - .find(|s| !is_remote_specifier_text(s)) - .unwrap_or("main.ts"), ); + if vendored_count > 0 { + let import_map_path = raw_output_dir.join("import_map.json"); + if maybe_update_config_file(&output_dir, &ps) { + eprintln!( + concat!( + "\nUpdated your local Deno configuration file with a reference to the ", + "new vendored import map at {}. Invoking Deno subcommands will now ", + "automatically resolve using the vendored modules. You may override ", + "this by providing the `--import-map ` flag or by ", + "manually editing your Deno configuration file.", + ), + import_map_path.display(), + ); + } else { + eprintln!( + concat!( + "\nTo use vendored modules, specify the `--import-map {}` flag when ", + r#"invoking Deno subcommands or add an `"importMap": ""` "#, + "entry to a deno.json file.", + ), + import_map_path.display(), + ); + } + } Ok(()) } @@ -76,7 +99,7 @@ fn validate_output_dir( if let Some(import_map_path) = ps .maybe_import_map .as_ref() - .and_then(|m| m.base_url().to_file_path().ok()) + .and_then(|m| specifier_to_file_path(m.base_url()).ok()) .and_then(|p| fs_util::canonicalize_path(&p).ok()) { // make the output directory in order to canonicalize it for the check below @@ -87,10 +110,21 @@ fn validate_output_dir( })?; if import_map_path.starts_with(&output_dir) { - // We don't allow using the output directory to help generate the new state - // of itself because supporting this scenario adds a lot of complexity. + // canonicalize to make the test for this pass on the CI + let cwd = fs_util::canonicalize_path(&std::env::current_dir()?)?; + // We don't allow using the output directory to help generate the + // new state because this may lead to cryptic error messages. bail!( - "Using an import map found in the output directory is not supported." + concat!( + "Specifying an import map file ({}) in the deno vendor output ", + "directory is not supported. Please specify no import map or one ", + "located outside this directory." + ), + import_map_path + .strip_prefix(&cwd) + .unwrap_or(&import_map_path) + .display() + .to_string(), ); } } @@ -98,6 +132,104 @@ fn validate_output_dir( Ok(()) } +fn maybe_update_config_file(output_dir: &Path, ps: &ProcState) -> bool { + assert!(output_dir.is_absolute()); + let config_file = match &ps.maybe_config_file { + Some(f) => f, + None => return false, + }; + let fmt_config = config_file + .to_fmt_config() + .unwrap_or_default() + .unwrap_or_default(); + let result = update_config_file( + &config_file.specifier, + &ModuleSpecifier::from_file_path(output_dir.join("import_map.json")) + .unwrap(), + &fmt_config.options, + ); + match result { + Ok(()) => true, + Err(err) => { + warn!("Error updating config file. {:#}", err); + false + } + } +} + +fn update_config_file( + config_specifier: &ModuleSpecifier, + import_map_specifier: &ModuleSpecifier, + fmt_options: &FmtOptionsConfig, +) -> Result<(), AnyError> { + if config_specifier.scheme() != "file" { + return Ok(()); + } + + let config_path = specifier_to_file_path(config_specifier)?; + let config_text = std::fs::read_to_string(&config_path)?; + let relative_text = + match relative_specifier(config_specifier, import_map_specifier) { + Some(text) => text, + None => return Ok(()), // ignore + }; + if let Some(new_text) = + update_config_text(&config_text, &relative_text, fmt_options) + { + std::fs::write(config_path, new_text)?; + } + + Ok(()) +} + +fn update_config_text( + text: &str, + import_map_specifier: &str, + fmt_options: &FmtOptionsConfig, +) -> Option { + use jsonc_parser::ast::ObjectProp; + use jsonc_parser::ast::Value; + let ast = jsonc_parser::parse_to_ast(text, &Default::default()).ok()?; + let obj = match ast.value { + Some(Value::Object(obj)) => obj, + _ => return None, // shouldn't happen, so ignore + }; + let import_map_specifier = import_map_specifier.replace('\"', "\\\""); + + match obj.get("importMap") { + Some(ObjectProp { + value: Value::StringLit(lit), + .. + }) => Some(format!( + "{}{}{}", + &text[..lit.range.start + 1], + import_map_specifier, + &text[lit.range.end - 1..], + )), + None => { + // insert it crudely at a position that won't cause any issues + // with comments and format after to make it look nice + let insert_position = obj.range.end - 1; + let insert_text = format!( + r#"{}"importMap": "{}""#, + if obj.properties.is_empty() { "" } else { "," }, + import_map_specifier + ); + let new_text = format!( + "{}{}{}", + &text[..insert_position], + insert_text, + &text[insert_position..], + ); + format_json(&new_text, fmt_options) + .ok() + .map(|formatted_text| formatted_text.unwrap_or(new_text)) + } + // shouldn't happen, so ignore + Some(_) => None, + } +} + fn is_dir_empty(dir_path: &Path) -> Result { match std::fs::read_dir(&dir_path) { Ok(mut dir) => Ok(dir.next().is_none()), @@ -149,20 +281,85 @@ async fn create_graph( .map(|im| im.as_resolver()) }; - let graph = deno_graph::create_graph( - entry_points, - false, - maybe_imports, - &mut cache, - maybe_resolver, - maybe_locker, - None, - None, + Ok( + deno_graph::create_graph( + entry_points, + false, + maybe_imports, + &mut cache, + maybe_resolver, + maybe_locker, + None, + None, + ) + .await, ) - .await; - - graph.lock()?; - graph.valid()?; - - Ok(graph) +} + +#[cfg(test)] +mod internal_test { + use super::*; + use pretty_assertions::assert_eq; + + #[test] + fn update_config_text_no_existing_props_add_prop() { + let text = update_config_text( + "{\n}", + "./vendor/import_map.json", + &Default::default(), + ) + .unwrap(); + assert_eq!( + text, + r#"{ + "importMap": "./vendor/import_map.json" +} +"# + ); + } + + #[test] + fn update_config_text_existing_props_add_prop() { + let text = update_config_text( + r#"{ + "tasks": { + "task1": "other" + } +} +"#, + "./vendor/import_map.json", + &Default::default(), + ) + .unwrap(); + assert_eq!( + text, + r#"{ + "tasks": { + "task1": "other" + }, + "importMap": "./vendor/import_map.json" +} +"# + ); + } + + #[test] + fn update_config_text_update_prop() { + let text = update_config_text( + r#"{ + "importMap": "./local.json" +} +"#, + "./vendor/import_map.json", + &Default::default(), + ) + .unwrap(); + assert_eq!( + text, + r#"{ + "importMap": "./vendor/import_map.json" +} +"# + ); + } } diff --git a/cli/tools/vendor/test.rs b/cli/tools/vendor/test.rs index 5060c493aa..7a8feb94bb 100644 --- a/cli/tools/vendor/test.rs +++ b/cli/tools/vendor/test.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; use deno_ast::ModuleSpecifier; use deno_core::anyhow::anyhow; @@ -16,6 +17,10 @@ use deno_graph::source::LoadFuture; use deno_graph::source::LoadResponse; use deno_graph::source::Loader; use deno_graph::ModuleGraph; +use deno_graph::ModuleKind; +use import_map::ImportMap; + +use crate::resolver::ImportMapResolver; use super::build::VendorEnvironment; @@ -120,6 +125,10 @@ struct TestVendorEnvironment { } impl VendorEnvironment for TestVendorEnvironment { + fn cwd(&self) -> Result { + Ok(make_path("/")) + } + fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> { let mut directories = self.directories.borrow_mut(); for path in dir_path.ancestors() { @@ -141,6 +150,10 @@ impl VendorEnvironment for TestVendorEnvironment { .insert(file_path.to_path_buf(), text.to_string()); Ok(()) } + + fn path_exists(&self, path: &Path) -> bool { + self.files.borrow().contains_key(&path.to_path_buf()) + } } pub struct VendorOutput { @@ -152,6 +165,8 @@ pub struct VendorOutput { pub struct VendorTestBuilder { entry_points: Vec, loader: TestLoader, + original_import_map: Option, + environment: TestVendorEnvironment, } impl VendorTestBuilder { @@ -161,6 +176,19 @@ impl VendorTestBuilder { builder } + pub fn new_import_map(&self, base_path: &str) -> ImportMap { + let base = ModuleSpecifier::from_file_path(&make_path(base_path)).unwrap(); + ImportMap::new(base) + } + + pub fn set_original_import_map( + &mut self, + import_map: ImportMap, + ) -> &mut Self { + self.original_import_map = Some(import_map); + self + } + pub fn add_entry_point(&mut self, entry_point: impl AsRef) -> &mut Self { let entry_point = make_path(entry_point.as_ref()); self @@ -170,11 +198,24 @@ impl VendorTestBuilder { } pub async fn build(&mut self) -> Result { - let graph = self.build_graph().await; let output_dir = make_path("/vendor"); - let environment = TestVendorEnvironment::default(); - super::build::build(&graph, &output_dir, &environment)?; - let mut files = environment.files.borrow_mut(); + let roots = self + .entry_points + .iter() + .map(|s| (s.to_owned(), deno_graph::ModuleKind::Esm)) + .collect(); + let loader = self.loader.clone(); + let graph = + build_test_graph(roots, self.original_import_map.clone(), loader.clone()) + .await; + super::build::build( + graph, + &output_dir, + self.original_import_map.as_ref(), + &self.environment, + )?; + + let mut files = self.environment.files.borrow_mut(); let import_map = files.remove(&output_dir.join("import_map.json")); let mut files = files .iter() @@ -193,27 +234,26 @@ impl VendorTestBuilder { action(&mut self.loader); self } +} - async fn build_graph(&mut self) -> ModuleGraph { - let graph = deno_graph::create_graph( - self - .entry_points - .iter() - .map(|s| (s.to_owned(), deno_graph::ModuleKind::Esm)) - .collect(), - false, - None, - &mut self.loader, - None, - None, - None, - None, - ) - .await; - graph.lock().unwrap(); - graph.valid().unwrap(); - graph - } +async fn build_test_graph( + roots: Vec<(ModuleSpecifier, ModuleKind)>, + original_import_map: Option, + mut loader: TestLoader, +) -> ModuleGraph { + let resolver = + original_import_map.map(|m| ImportMapResolver::new(Arc::new(m))); + deno_graph::create_graph( + roots, + false, + None, + &mut loader, + resolver.as_ref().map(|im| im.as_resolver()), + None, + None, + None, + ) + .await } fn make_path(text: &str) -> PathBuf { diff --git a/test_util/src/temp_dir.rs b/test_util/src/temp_dir.rs index da2b2a06e3..cab55cc14d 100644 --- a/test_util/src/temp_dir.rs +++ b/test_util/src/temp_dir.rs @@ -1,3 +1,4 @@ +use std::fs; use std::path::Path; use std::path::PathBuf; use std::sync::atomic::AtomicU32; @@ -84,4 +85,23 @@ impl TempDir { let inner = &self.0; inner.0.as_path() } + + pub fn create_dir_all(&self, path: impl AsRef) { + fs::create_dir_all(self.path().join(path)).unwrap(); + } + + pub fn read_to_string(&self, path: impl AsRef) -> String { + let file_path = self.path().join(path); + fs::read_to_string(&file_path) + .with_context(|| format!("Could not find file: {}", file_path.display())) + .unwrap() + } + + pub fn rename(&self, from: impl AsRef, to: impl AsRef) { + fs::rename(self.path().join(from), self.path().join(to)).unwrap(); + } + + pub fn write(&self, path: impl AsRef, text: impl AsRef) { + fs::write(self.path().join(path), text.as_ref()).unwrap(); + } }