foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-24 08:09:08 -05:00
Code Issues Packages 1 Wiki Activity

feat(vendor): support using an existing import map (#14836)

Browse source
This commit is contained in:
David Sherret 2022-06-14 10:05:37 -04:00 committed by GitHub
parent fc3a966a2d
commit 443041c23e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 1198 additions and 256 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Cargo.lock generated
View file

@ -2069,9 +2069,9 @@ checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed"
[[package]]
name = "import_map"
version = "0.9.0"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f99e0f89d56c163538ea6bf1f250049669298a26daeee15a9a18f4118cc503f1"
checksum = "5247edf057fe57036112a1fec3864baa68052b52116760dbea4909115731272f"
dependencies = [
"indexmap",
"log",
@ -2082,9 +2082,9 @@ dependencies = [
[[package]]
name = "indexmap"
version = "1.8.1"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a"
dependencies = [
"autocfg 1.1.0",
"hashbrown 0.11.2",

2
cli/Cargo.toml
View file

@ -71,7 +71,7 @@ env_logger = "=0.9.0"
eszip = "=0.20.0"
fancy-regex = "=0.9.0"
http = "=0.2.6"
import_map = "=0.9.0"
import_map = "=0.11.0"
indexmap = "1.8.1"
jsonc-parser = { version = "=0.19.0", features = ["serde"] }
libc = "=0.2.126"

5
cli/config_file.rs
View file

@ -11,6 +11,7 @@ use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::normalize_path;
use deno_core::serde::Deserialize;
use deno_core::serde::Serialize;
use deno_core::serde::Serializer;
@ -262,12 +263,12 @@ pub fn resolve_import_map_specifier(
// file into a file path if possible and join the import map path to
// the file path.
if let Ok(config_file_path) = config_file.specifier.to_file_path() {
let import_map_file_path = config_file_path
let import_map_file_path = normalize_path(config_file_path
.parent()
.ok_or_else(|| {
anyhow!("Bad config file specifier: {}", config_file.specifier)
})?
.join(&import_map_path);
.join(&import_map_path));
ModuleSpecifier::from_file_path(import_map_file_path).unwrap()
// otherwise if the config file is remote, we have no choice but to
// use "import resolution" with the config file as the base.

72
cli/fs_util.rs
View file

@ -5,6 +5,7 @@ use deno_core::error::{uri_error, AnyError};
pub use deno_core::normalize_path;
use deno_core::ModuleSpecifier;
use deno_runtime::deno_crypto::rand;
use std::borrow::Cow;
use std::env::current_dir;
use std::fs::OpenOptions;
use std::io::{Error, Write};
@ -362,6 +363,44 @@ pub fn specifier_parent(specifier: &ModuleSpecifier) -> ModuleSpecifier {
specifier
}
/// `from.make_relative(to)` but with fixes.
pub fn relative_specifier(
from: &ModuleSpecifier,
to: &ModuleSpecifier,
) -> Option<String> {
let is_dir = to.path().ends_with('/');
if is_dir && from == to {
return Some("./".to_string());
}
// workaround using parent directory until https://github.com/servo/rust-url/pull/754 is merged
let from = if !from.path().ends_with('/') {
if let Some(end_slash) = from.path().rfind('/') {
let mut new_from = from.clone();
new_from.set_path(&from.path()[..end_slash + 1]);
Cow::Owned(new_from)
} else {
Cow::Borrowed(from)
}
} else {
Cow::Borrowed(from)
};
// workaround for url crate not adding a trailing slash for a directory
// it seems to be fixed once a version greater than 2.2.2 is released
let mut text = from.make_relative(to)?;
if is_dir && !text.ends_with('/') && to.query().is_none() {
text.push('/');
}
Some(if text.starts_with("../") || text.starts_with("./") {
text
} else {
format!("./{}", text)
})
}
/// This function checks if input path has trailing slash or not. If input path
/// has trailing slash it will return true else it will return false.
pub fn path_has_trailing_slash(path: &Path) -> bool {
@ -748,6 +787,39 @@ mod tests {
}
}
#[test]
fn test_relative_specifier() {
run_test("file:///from", "file:///to", Some("./to"));
run_test("file:///from", "file:///from/other", Some("./from/other"));
run_test("file:///from", "file:///from/other/", Some("./from/other/"));
run_test("file:///from", "file:///other/from", Some("./other/from"));
run_test("file:///from/", "file:///other/from", Some("../other/from"));
run_test("file:///from", "file:///other/from/", Some("./other/from/"));
run_test(
"file:///from",
"file:///to/other.txt",
Some("./to/other.txt"),
);
run_test(
"file:///from/test",
"file:///to/other.txt",
Some("../to/other.txt"),
);
run_test(
"file:///from/other.txt",
"file:///to/other.txt",
Some("../to/other.txt"),
);
fn run_test(from: &str, to: &str, expected: Option<&str>) {
let result = relative_specifier(
&ModuleSpecifier::parse(from).unwrap(),
&ModuleSpecifier::parse(to).unwrap(),
);
assert_eq!(result.as_deref(), expected);
}
}
#[test]
fn test_path_has_trailing_slash() {
#[cfg(not(windows))]

6
cli/lsp/completions.rs
View file

@ -208,8 +208,8 @@ fn get_base_import_map_completions(
import_map: &ImportMap,
) -> Vec<lsp::CompletionItem> {
import_map
.imports_keys()
.iter()
.imports()
.keys()
.map(|key| {
// for some strange reason, keys that start with `/` get stored in the
// import map as `file:///`, and so when we pull the keys out, we need to
@ -253,7 +253,7 @@ fn get_import_map_completions(
if !text.is_empty() {
if let Some(import_map) = maybe_import_map {
let mut items = Vec::new();
for key in import_map.imports_keys() {
for key in import_map.imports().keys() {
// for some reason, the import_map stores keys that begin with `/` as
// `file:///` in its index, so we have to reverse that here
let key = if key.starts_with("file://") {

10
cli/proc_state.rs
View file

@ -51,7 +51,6 @@ use deno_runtime::deno_tls::rustls::RootCertStore;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::inspector_server::InspectorServer;
use deno_runtime::permissions::Permissions;
use import_map::parse_from_json;
use import_map::ImportMap;
use log::warn;
use std::collections::HashSet;
@ -737,7 +736,12 @@ pub fn import_map_from_text(
specifier: &Url,
json_text: &str,
) -> Result<ImportMap, AnyError> {
let result = parse_from_json(specifier, json_text)?;
debug_assert!(
!specifier.as_str().contains("../"),
"Import map specifier incorrectly contained ../: {}",
specifier.as_str()
);
let result = import_map::parse_from_json(specifier, json_text)?;
if !result.diagnostics.is_empty() {
warn!(
"Import map diagnostics:\n{}",
@ -747,7 +751,7 @@ pub fn import_map_from_text(
.map(|d| format!(" - {}", d))
.collect::<Vec<_>>()
.join("\n")
)
);
}
Ok(result.import_map)
}

2
cli/resolver.rs
View file

@ -30,7 +30,7 @@ impl Resolver for ImportMapResolver {
referrer: &ModuleSpecifier,
) -> ResolveResponse {
match self.0.resolve(specifier, referrer) {
Ok(specifier) => ResolveResponse::Specifier(specifier),
Ok(resolved_specifier) => ResolveResponse::Specifier(resolved_specifier),
Err(err) => ResolveResponse::Err(err.into()),
}
}

321
cli/tests/integration/vendor_tests.rs
View file

@ -3,20 +3,19 @@
use deno_core::serde_json;
use deno_core::serde_json::json;
use pretty_assertions::assert_eq;
use std::fs;
use std::path::PathBuf;
use std::process::Stdio;
use test_util as util;
use test_util::TempDir;
use util::http_server;
use util::new_deno_dir;
#[test]
fn output_dir_exists() {
let t = TempDir::new();
let vendor_dir = t.path().join("vendor");
fs::write(t.path().join("mod.ts"), "").unwrap();
fs::create_dir_all(&vendor_dir).unwrap();
fs::write(vendor_dir.join("mod.ts"), "").unwrap();
t.write("mod.ts", "");
t.create_dir_all("vendor");
t.write("vendor/mod.ts", "");
let deno = util::deno_cmd()
.current_dir(t.path())
@ -76,15 +75,12 @@ fn output_dir_exists() {
#[test]
fn import_map_output_dir() {
let t = TempDir::new();
let vendor_dir = t.path().join("vendor");
fs::write(t.path().join("mod.ts"), "").unwrap();
fs::create_dir_all(&vendor_dir).unwrap();
let import_map_path = vendor_dir.join("import_map.json");
fs::write(
&import_map_path,
t.write("mod.ts", "");
t.create_dir_all("vendor");
t.write(
"vendor/import_map.json",
"{ \"imports\": { \"https://localhost/\": \"./localhost/\" }}",
)
.unwrap();
);
let deno = util::deno_cmd()
.current_dir(t.path())
@ -92,7 +88,7 @@ fn import_map_output_dir() {
.arg("vendor")
.arg("--force")
.arg("--import-map")
.arg(import_map_path)
.arg("vendor/import_map.json")
.arg("mod.ts")
.stdout(Stdio::piped())
.stderr(Stdio::piped())
@ -101,7 +97,14 @@ fn import_map_output_dir() {
let output = deno.wait_with_output().unwrap();
assert_eq!(
String::from_utf8_lossy(&output.stderr).trim(),
"error: Using an import map found in the output directory is not supported.",
format!(
concat!(
"error: Specifying an import map file ({}) in the deno vendor ",
"output directory is not supported. Please specify no import ",
"map or one located outside this directory.",
),
PathBuf::from("vendor").join("import_map.json").display(),
),
);
assert!(!output.status.success());
}
@ -111,10 +114,10 @@ fn standard_test() {
let _server = http_server();
let t = TempDir::new();
let vendor_dir = t.path().join("vendor2");
fs::write(
t.path().join("my_app.ts"),
t.write(
"my_app.ts",
"import {Logger} from 'http://localhost:4545/vendor/query_reexport.ts?testing'; new Logger().log('outputted');",
).unwrap();
);
let deno = util::deno_cmd()
.current_dir(t.path())
@ -136,7 +139,7 @@ fn standard_test() {
"Download http://localhost:4545/vendor/logger.ts?test\n",
"{}",
),
success_text("2 modules", "vendor2", "my_app.ts"),
success_text("2 modules", "vendor2", true),
)
);
assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "");
@ -144,16 +147,14 @@ fn standard_test() {
assert!(vendor_dir.exists());
assert!(!t.path().join("vendor").exists());
let import_map: serde_json::Value = serde_json::from_str(
&fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(),
)
.unwrap();
let import_map: serde_json::Value =
serde_json::from_str(&t.read_to_string("vendor2/import_map.json")).unwrap();
assert_eq!(
import_map,
json!({
"imports": {
"http://localhost:4545/": "./localhost_4545/",
"http://localhost:4545/vendor/query_reexport.ts?testing": "./localhost_4545/vendor/query_reexport.ts",
"http://localhost:4545/": "./localhost_4545/",
},
"scopes": {
"./localhost_4545/": {
@ -169,7 +170,8 @@ fn standard_test() {
.env("NO_COLOR", "1")
.arg("run")
.arg("--no-remote")
.arg("--no-check")
.arg("--check")
.arg("--quiet")
.arg("--import-map")
.arg("vendor2/import_map.json")
.arg("my_app.ts")
@ -207,7 +209,7 @@ fn remote_module_test() {
"Download http://localhost:4545/vendor/logger.ts?test\n",
"{}",
),
success_text("2 modules", "vendor/", "main.ts"),
success_text("2 modules", "vendor/", true),
)
);
assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "");
@ -217,10 +219,8 @@ fn remote_module_test() {
.join("localhost_4545/vendor/query_reexport.ts")
.exists());
assert!(vendor_dir.join("localhost_4545/vendor/logger.ts").exists());
let import_map: serde_json::Value = serde_json::from_str(
&fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(),
)
.unwrap();
let import_map: serde_json::Value =
serde_json::from_str(&t.read_to_string("vendor/import_map.json")).unwrap();
assert_eq!(
import_map,
json!({
@ -229,7 +229,7 @@ fn remote_module_test() {
},
"scopes": {
"./localhost_4545/": {
"./localhost_4545/vendor/logger.ts?test": "./localhost_4545/vendor/logger.ts"
"./localhost_4545/vendor/logger.ts?test": "./localhost_4545/vendor/logger.ts",
}
}
}),
@ -237,49 +237,155 @@ fn remote_module_test() {
}
#[test]
fn existing_import_map() {
fn existing_import_map_no_remote() {
let _server = http_server();
let t = TempDir::new();
let vendor_dir = t.path().join("vendor");
fs::write(
t.path().join("mod.ts"),
t.write(
"mod.ts",
"import {Logger} from 'http://localhost:4545/vendor/logger.ts';",
)
.unwrap();
fs::write(
t.path().join("imports.json"),
r#"{ "imports": { "http://localhost:4545/vendor/": "./logger/" } }"#,
)
.unwrap();
fs::create_dir(t.path().join("logger")).unwrap();
fs::write(t.path().join("logger/logger.ts"), "export class Logger {}")
.unwrap();
);
let import_map_filename = "imports2.json";
let import_map_text =
r#"{ "imports": { "http://localhost:4545/vendor/": "./logger/" } }"#;
t.write(import_map_filename, &import_map_text);
t.create_dir_all("logger");
t.write("logger/logger.ts", "export class Logger {}");
let status = util::deno_cmd()
let deno = util::deno_cmd()
.current_dir(t.path())
.env("NO_COLOR", "1")
.arg("vendor")
.arg("mod.ts")
.arg("--import-map")
.arg("imports.json")
.arg(import_map_filename)
.stderr(Stdio::piped())
.spawn()
.unwrap();
let output = deno.wait_with_output().unwrap();
assert_eq!(
String::from_utf8_lossy(&output.stderr).trim(),
success_text("0 modules", "vendor/", false)
);
assert!(output.status.success());
// it should not have found any remote dependencies because
// the provided import map mapped it to a local directory
assert_eq!(t.read_to_string(import_map_filename), import_map_text);
}
#[test]
fn existing_import_map_mixed_with_remote() {
let _server = http_server();
let deno_dir = new_deno_dir();
let t = TempDir::new();
t.write(
"mod.ts",
"import {Logger} from 'http://localhost:4545/vendor/logger.ts';",
);
let status = util::deno_cmd_with_deno_dir(&deno_dir)
.current_dir(t.path())
.arg("vendor")
.arg("mod.ts")
.spawn()
.unwrap()
.wait()
.unwrap();
assert!(status.success());
assert_eq!(
t.read_to_string("vendor/import_map.json"),
r#"{
"imports": {
"http://localhost:4545/": "./localhost_4545/"
}
}
"#,
);
// make the import map specific to support vendoring mod.ts in the next step
t.write(
"vendor/import_map.json",
r#"{
"imports": {
"http://localhost:4545/vendor/logger.ts": "./localhost_4545/vendor/logger.ts"
}
}
"#,
);
t.write(
"mod.ts",
concat!(
"import {Logger} from 'http://localhost:4545/vendor/logger.ts';\n",
"import {Logger as OtherLogger} from 'http://localhost:4545/vendor/mod.ts';\n",
),
);
// now vendor with the existing import map in a separate vendor directory
let deno = util::deno_cmd_with_deno_dir(&deno_dir)
.env("NO_COLOR", "1")
.current_dir(t.path())
.arg("vendor")
.arg("mod.ts")
.arg("--import-map")
.arg("vendor/import_map.json")
.arg("--output")
.arg("vendor2")
.stderr(Stdio::piped())
.spawn()
.unwrap();
let output = deno.wait_with_output().unwrap();
assert_eq!(
String::from_utf8_lossy(&output.stderr).trim(),
format!(
concat!("Download http://localhost:4545/vendor/mod.ts\n", "{}",),
success_text("1 module", "vendor2", true),
)
);
assert!(output.status.success());
// tricky scenario here where the output directory now contains a mapping
// back to the previous vendor location
assert_eq!(
t.read_to_string("vendor2/import_map.json"),
r#"{
"imports": {
"http://localhost:4545/vendor/logger.ts": "../vendor/localhost_4545/vendor/logger.ts",
"http://localhost:4545/": "./localhost_4545/"
},
"scopes": {
"./localhost_4545/": {
"./localhost_4545/vendor/logger.ts": "../vendor/localhost_4545/vendor/logger.ts"
}
}
}
"#,
);
// ensure it runs
let status = util::deno_cmd()
.current_dir(t.path())
.arg("run")
.arg("--check")
.arg("--no-remote")
.arg("--import-map")
.arg("vendor2/import_map.json")
.arg("mod.ts")
.spawn()
.unwrap()
.wait()
.unwrap();
assert!(status.success());
// it should not have found any remote dependencies because
// the provided import map mapped it to a local directory
assert!(!vendor_dir.join("import_map.json").exists());
}
#[test]
fn dynamic_import() {
let _server = http_server();
let t = TempDir::new();
let vendor_dir = t.path().join("vendor");
fs::write(
t.path().join("mod.ts"),
t.write(
"mod.ts",
"import {Logger} from 'http://localhost:4545/vendor/dynamic.ts'; new Logger().log('outputted');",
).unwrap();
);
let status = util::deno_cmd()
.current_dir(t.path())
@ -290,10 +396,8 @@ fn dynamic_import() {
.wait()
.unwrap();
assert!(status.success());
let import_map: serde_json::Value = serde_json::from_str(
&fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(),
)
.unwrap();
let import_map: serde_json::Value =
serde_json::from_str(&t.read_to_string("vendor/import_map.json")).unwrap();
assert_eq!(
import_map,
json!({
@ -310,7 +414,8 @@ fn dynamic_import() {
.arg("run")
.arg("--allow-read=.")
.arg("--no-remote")
.arg("--no-check")
.arg("--check")
.arg("--quiet")
.arg("--import-map")
.arg("vendor/import_map.json")
.arg("mod.ts")
@ -328,10 +433,10 @@ fn dynamic_import() {
fn dynamic_non_analyzable_import() {
let _server = http_server();
let t = TempDir::new();
fs::write(
t.path().join("mod.ts"),
t.write(
"mod.ts",
"import {Logger} from 'http://localhost:4545/vendor/dynamic_non_analyzable.ts'; new Logger().log('outputted');",
).unwrap();
);
let deno = util::deno_cmd()
.current_dir(t.path())
@ -350,23 +455,89 @@ fn dynamic_non_analyzable_import() {
String::from_utf8_lossy(&output.stderr).trim(),
format!(
"Download http://localhost:4545/vendor/dynamic_non_analyzable.ts\n{}",
success_text("1 module", "vendor/", "mod.ts"),
success_text("1 module", "vendor/", true),
)
);
assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "");
assert!(output.status.success());
}
fn success_text(module_count: &str, dir: &str, entry_point: &str) -> String {
format!(
concat!(
"Vendored {} into {} directory.\n\n",
"To use vendored modules, specify the `--import-map` flag when invoking deno subcommands:\n",
" deno run -A --import-map {} {}"
),
module_count,
dir,
PathBuf::from(dir).join("import_map.json").display(),
entry_point,
)
#[test]
fn update_existing_config_test() {
let _server = http_server();
let t = TempDir::new();
t.write(
"my_app.ts",
"import {Logger} from 'http://localhost:4545/vendor/logger.ts'; new Logger().log('outputted');",
);
t.write("deno.json", "{\n}");
let deno = util::deno_cmd()
.current_dir(t.path())
.arg("vendor")
.arg("my_app.ts")
.arg("--output")
.arg("vendor2")
.env("NO_COLOR", "1")
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let output = deno.wait_with_output().unwrap();
assert_eq!(
String::from_utf8_lossy(&output.stderr).trim(),
format!(
concat!(
"Download http://localhost:4545/vendor/logger.ts\n",
"Vendored 1 module into vendor2 directory.\n\n",
"Updated your local Deno configuration file with a reference to the ",
"new vendored import map at {}. Invoking Deno subcommands will ",
"now automatically resolve using the vendored modules. You may override ",
"this by providing the `--import-map <other-import-map>` flag or by ",
"manually editing your Deno configuration file."
),
PathBuf::from("vendor2").join("import_map.json").display(),
)
);
assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "");
assert!(output.status.success());
// try running the output with `--no-remote` and not specifying a `--vendor`
let deno = util::deno_cmd()
.current_dir(t.path())
.env("NO_COLOR", "1")
.arg("run")
.arg("--no-remote")
.arg("--check")
.arg("--quiet")
.arg("my_app.ts")
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let output = deno.wait_with_output().unwrap();
assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), "");
assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "outputted");
assert!(output.status.success());
}
fn success_text(module_count: &str, dir: &str, has_import_map: bool) -> String {
let mut text = format!("Vendored {} into {} directory.", module_count, dir);
if has_import_map {
text.push_str(&
format!(
concat!(
"\n\nTo use vendored modules, specify the `--import-map {}import_map.json` flag when ",
r#"invoking Deno subcommands or add an `"importMap": "<path_to_vendored_import_map>"` "#,
"entry to a deno.json file.",
),
if dir != "vendor/" {
format!("{}{}", dir.trim_end_matches('/'), if cfg!(windows) { '\\' } else {'/'})
} else {
dir.to_string()
}
)
);
}
text
}

1
cli/tests/testdata/vendor/mod.ts vendored Normal file
View file

@ -0,0 +1 @@
export * from "./logger.ts";

349
cli/tools/vendor/build.rs vendored
View file

@ -1,11 +1,17 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use std::path::Path;
use std::path::PathBuf;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_graph::Module;
use deno_graph::ModuleGraph;
use deno_graph::ModuleKind;
use import_map::ImportMap;
use import_map::SpecifierMap;
use super::analyze::has_default_export;
use super::import_map::build_import_map;
@ -15,29 +21,53 @@ use super::specifiers::is_remote_specifier;
/// Allows substituting the environment for testing purposes.
pub trait VendorEnvironment {
fn cwd(&self) -> Result<PathBuf, AnyError>;
fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError>;
fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError>;
fn path_exists(&self, path: &Path) -> bool;
}
pub struct RealVendorEnvironment;
impl VendorEnvironment for RealVendorEnvironment {
fn cwd(&self) -> Result<PathBuf, AnyError> {
Ok(std::env::current_dir()?)
}
fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> {
Ok(std::fs::create_dir_all(dir_path)?)
}
fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError> {
Ok(std::fs::write(file_path, text)?)
std::fs::write(file_path, text)
.with_context(|| format!("Failed writing {}", file_path.display()))
}
fn path_exists(&self, path: &Path) -> bool {
path.exists()
}
}
/// Vendors remote modules and returns how many were vendored.
pub fn build(
graph: &ModuleGraph,
graph: ModuleGraph,
output_dir: &Path,
original_import_map: Option<&ImportMap>,
environment: &impl VendorEnvironment,
) -> Result<usize, AnyError> {
assert!(output_dir.is_absolute());
let output_dir_specifier =
ModuleSpecifier::from_directory_path(output_dir).unwrap();
if let Some(original_im) = &original_import_map {
validate_original_import_map(original_im, &output_dir_specifier)?;
}
// build the graph
graph.lock()?;
graph.valid()?;
// figure out how to map remote modules to local
let all_modules = graph.modules();
let remote_modules = all_modules
.iter()
@ -45,7 +75,7 @@ pub fn build(
.copied()
.collect::<Vec<_>>();
let mappings =
Mappings::from_remote_modules(graph, &remote_modules, output_dir)?;
Mappings::from_remote_modules(&graph, &remote_modules, output_dir)?;
// write out all the files
for module in &remote_modules {
@ -77,16 +107,59 @@ pub fn build(
environment.write_file(&proxy_path, &text)?;
}
// create the import map
if !mappings.base_specifiers().is_empty() {
let import_map_text = build_import_map(graph, &all_modules, &mappings);
environment
.write_file(&output_dir.join("import_map.json"), &import_map_text)?;
// create the import map if necessary
if !remote_modules.is_empty() {
let import_map_path = output_dir.join("import_map.json");
let import_map_text = build_import_map(
&output_dir_specifier,
&graph,
&all_modules,
&mappings,
original_import_map,
);
environment.write_file(&import_map_path, &import_map_text)?;
}
Ok(remote_modules.len())
}
fn validate_original_import_map(
import_map: &ImportMap,
output_dir: &ModuleSpecifier,
) -> Result<(), AnyError> {
fn validate_imports(
imports: &SpecifierMap,
output_dir: &ModuleSpecifier,
) -> Result<(), AnyError> {
for entry in imports.entries() {
if let Some(value) = entry.value {
if value.as_str().starts_with(output_dir.as_str()) {
bail!(
"Providing an existing import map with entries for the output directory is not supported (\"{}\": \"{}\").",
entry.raw_key,
entry.raw_value.unwrap_or("<INVALID>"),
);
}
}
}
Ok(())
}
validate_imports(import_map.imports(), output_dir)?;
for scope in import_map.scopes() {
if scope.key.starts_with(output_dir.as_str()) {
bail!(
"Providing an existing import map with a scope for the output directory is not supported (\"{}\").",
scope.raw_key,
);
}
validate_imports(scope.imports, output_dir)?;
}
Ok(())
}
fn build_proxy_module_source(
module: &Module,
proxied_module: &ProxiedModule,
@ -171,9 +244,9 @@ mod test {
output.import_map,
Some(json!({
"imports": {
"https://localhost/": "./localhost/",
"https://localhost/other.ts?test": "./localhost/other.ts",
"https://localhost/redirect.ts": "./localhost/mod.ts",
"https://localhost/": "./localhost/",
}
}))
);
@ -241,7 +314,7 @@ mod test {
"imports": {
"https://localhost/": "./localhost/",
"https://localhost/redirect.ts": "./localhost/other.ts",
"https://other/": "./other/"
"https://other/": "./other/",
},
"scopes": {
"./localhost/": {
@ -313,10 +386,10 @@ mod test {
output.import_map,
Some(json!({
"imports": {
"https://localhost/": "./localhost/",
"https://localhost/mod.TS": "./localhost/mod_2.TS",
"https://localhost/mod.ts": "./localhost/mod_3.ts",
"https://localhost/mod.ts?test": "./localhost/mod_4.ts",
"https://localhost/": "./localhost/",
}
}))
);
@ -543,8 +616,8 @@ mod test {
output.import_map,
Some(json!({
"imports": {
"http://localhost:4545/": "./localhost_4545/",
"http://localhost:4545/sub/logger/mod.ts?testing": "./localhost_4545/sub/logger/mod.ts",
"http://localhost:4545/": "./localhost_4545/",
},
"scopes": {
"./localhost_4545/": {
@ -599,9 +672,9 @@ mod test {
output.import_map,
Some(json!({
"imports": {
"https://localhost/std/hash/mod.ts": "./localhost/std@0.1.0/hash/mod.ts",
"https://localhost/": "./localhost/",
"https://localhost/std/hash/mod.ts": "./localhost/std@0.1.0/hash/mod.ts"
}
},
}))
);
assert_eq!(
@ -675,6 +748,254 @@ mod test {
);
}
#[tokio::test]
async fn existing_import_map_basic() {
let mut builder = VendorTestBuilder::with_default_setup();
let mut original_import_map = builder.new_import_map("/import_map2.json");
original_import_map
.imports_mut()
.append(
"https://localhost/mod.ts".to_string(),
"./local_vendor/mod.ts".to_string(),
)
.unwrap();
let local_vendor_scope = original_import_map
.get_or_append_scope_mut("./local_vendor/")
.unwrap();
local_vendor_scope
.append(
"https://localhost/logger.ts".to_string(),
"./local_vendor/logger.ts".to_string(),
)
.unwrap();
local_vendor_scope
.append(
"/console_logger.ts".to_string(),
"./local_vendor/console_logger.ts".to_string(),
)
.unwrap();
let output = builder
.with_loader(|loader| {
loader.add("/mod.ts", "import 'https://localhost/mod.ts'; import 'https://localhost/other.ts';");
loader.add("/local_vendor/mod.ts", "import 'https://localhost/logger.ts'; import '/console_logger.ts'; console.log(5);");
loader.add("/local_vendor/logger.ts", "export class Logger {}");
loader.add("/local_vendor/console_logger.ts", "export class ConsoleLogger {}");
loader.add("https://localhost/mod.ts", "console.log(6);");
loader.add("https://localhost/other.ts", "import './mod.ts';");
})
.set_original_import_map(original_import_map.clone())
.build()
.await
.unwrap();
assert_eq!(
output.import_map,
Some(json!({
"imports": {
"https://localhost/mod.ts": "../local_vendor/mod.ts",
"https://localhost/": "./localhost/"
},
"scopes": {
"../local_vendor/": {
"https://localhost/logger.ts": "../local_vendor/logger.ts",
"/console_logger.ts": "../local_vendor/console_logger.ts",
},
"./localhost/": {
"./localhost/mod.ts": "../local_vendor/mod.ts",
},
}
}))
);
assert_eq!(
output.files,
to_file_vec(&[("/vendor/localhost/other.ts", "import './mod.ts';")]),
);
}
#[tokio::test]
async fn existing_import_map_mapped_bare_specifier() {
let mut builder = VendorTestBuilder::with_default_setup();
let mut original_import_map = builder.new_import_map("/import_map.json");
let imports = original_import_map.imports_mut();
imports
.append("$fresh".to_string(), "https://localhost/fresh".to_string())
.unwrap();
imports
.append("std/".to_string(), "https://deno.land/std/".to_string())
.unwrap();
let output = builder
.with_loader(|loader| {
loader.add("/mod.ts", "import 'std/mod.ts'; import '$fresh';");
loader.add("https://deno.land/std/mod.ts", "export function test() {}");
loader.add_with_headers(
"https://localhost/fresh",
"export function fresh() {}",
&[("content-type", "application/typescript")],
);
})
.set_original_import_map(original_import_map.clone())
.build()
.await
.unwrap();
assert_eq!(
output.import_map,
Some(json!({
"imports": {
"https://deno.land/": "./deno.land/",
"https://localhost/": "./localhost/",
"$fresh": "./localhost/fresh.ts",
"std/mod.ts": "./deno.land/std/mod.ts",
},
}))
);
assert_eq!(
output.files,
to_file_vec(&[
("/vendor/deno.land/std/mod.ts", "export function test() {}"),
("/vendor/localhost/fresh.ts", "export function fresh() {}")
]),
);
}
#[tokio::test]
async fn existing_import_map_remote_absolute_specifier_local() {
let mut builder = VendorTestBuilder::with_default_setup();
let mut original_import_map = builder.new_import_map("/import_map.json");
original_import_map
.imports_mut()
.append(
"https://localhost/logger.ts?test".to_string(),
"./local/logger.ts".to_string(),
)
.unwrap();
let output = builder
.with_loader(|loader| {
loader.add("/mod.ts", "import 'https://localhost/mod.ts'; import 'https://localhost/logger.ts?test';");
loader.add("/local/logger.ts", "export class Logger {}");
// absolute specifier in a remote module that will point at ./local/logger.ts
loader.add("https://localhost/mod.ts", "import '/logger.ts?test';");
loader.add("https://localhost/logger.ts?test", "export class Logger {}");
})
.set_original_import_map(original_import_map.clone())
.build()
.await
.unwrap();
assert_eq!(
output.import_map,
Some(json!({
"imports": {
"https://localhost/logger.ts?test": "../local/logger.ts",
"https://localhost/": "./localhost/",
},
"scopes": {
"./localhost/": {
"/logger.ts?test": "../local/logger.ts",
},
}
}))
);
assert_eq!(
output.files,
to_file_vec(&[("/vendor/localhost/mod.ts", "import '/logger.ts?test';")]),
);
}
#[tokio::test]
async fn existing_import_map_imports_output_dir() {
let mut builder = VendorTestBuilder::with_default_setup();
let mut original_import_map = builder.new_import_map("/import_map.json");
original_import_map
.imports_mut()
.append(
"std/mod.ts".to_string(),
"./vendor/deno.land/std/mod.ts".to_string(),
)
.unwrap();
let err = builder
.with_loader(|loader| {
loader.add("/mod.ts", "import 'std/mod.ts';");
loader.add("/vendor/deno.land/std/mod.ts", "export function f() {}");
loader.add("https://deno.land/std/mod.ts", "export function f() {}");
})
.set_original_import_map(original_import_map.clone())
.build()
.await
.err()
.unwrap();
assert_eq!(
err.to_string(),
concat!(
"Providing an existing import map with entries for the output ",
"directory is not supported ",
"(\"std/mod.ts\": \"./vendor/deno.land/std/mod.ts\").",
)
);
}
#[tokio::test]
async fn existing_import_map_scopes_entry_output_dir() {
let mut builder = VendorTestBuilder::with_default_setup();
let mut original_import_map = builder.new_import_map("/import_map.json");
let scopes = original_import_map
.get_or_append_scope_mut("./other/")
.unwrap();
scopes
.append("/mod.ts".to_string(), "./vendor/mod.ts".to_string())
.unwrap();
let err = builder
.with_loader(|loader| {
loader.add("/mod.ts", "console.log(5);");
})
.set_original_import_map(original_import_map.clone())
.build()
.await
.err()
.unwrap();
assert_eq!(
err.to_string(),
concat!(
"Providing an existing import map with entries for the output ",
"directory is not supported ",
"(\"/mod.ts\": \"./vendor/mod.ts\").",
)
);
}
#[tokio::test]
async fn existing_import_map_scopes_key_output_dir() {
let mut builder = VendorTestBuilder::with_default_setup();
let mut original_import_map = builder.new_import_map("/import_map.json");
let scopes = original_import_map
.get_or_append_scope_mut("./vendor/")
.unwrap();
scopes
.append("/mod.ts".to_string(), "./vendor/mod.ts".to_string())
.unwrap();
let err = builder
.with_loader(|loader| {
loader.add("/mod.ts", "console.log(5);");
})
.set_original_import_map(original_import_map.clone())
.build()
.await
.err()
.unwrap();
assert_eq!(
err.to_string(),
concat!(
"Providing an existing import map with a scope for the output ",
"directory is not supported (\"./vendor/\").",
)
);
}
fn to_file_vec(items: &[(&str, &str)]) -> Vec<(String, String)> {
items
.iter()

265
cli/tools/vendor/import_map.rs vendored
View file

@ -1,44 +1,43 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use std::collections::BTreeMap;
use deno_ast::LineAndColumnIndex;
use deno_ast::ModuleSpecifier;
use deno_ast::SourceTextInfo;
use deno_core::serde_json;
use deno_graph::Module;
use deno_graph::ModuleGraph;
use deno_graph::Position;
use deno_graph::Range;
use deno_graph::Resolved;
use serde::Serialize;
use import_map::ImportMap;
use import_map::SpecifierMap;
use indexmap::IndexMap;
use log::warn;
use super::mappings::Mappings;
use super::specifiers::is_remote_specifier;
use super::specifiers::is_remote_specifier_text;
#[derive(Serialize)]
struct SerializableImportMap {
imports: BTreeMap<String, String>,
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
scopes: BTreeMap<String, BTreeMap<String, String>>,
}
struct ImportMapBuilder<'a> {
base_dir: &'a ModuleSpecifier,
mappings: &'a Mappings,
imports: ImportsBuilder<'a>,
scopes: BTreeMap<String, ImportsBuilder<'a>>,
scopes: IndexMap<String, ImportsBuilder<'a>>,
}
impl<'a> ImportMapBuilder<'a> {
pub fn new(mappings: &'a Mappings) -> Self {
pub fn new(base_dir: &'a ModuleSpecifier, mappings: &'a Mappings) -> Self {
ImportMapBuilder {
base_dir,
mappings,
imports: ImportsBuilder::new(mappings),
imports: ImportsBuilder::new(base_dir, mappings),
scopes: Default::default(),
}
}
pub fn base_dir(&self) -> &ModuleSpecifier {
self.base_dir
}
pub fn scope(
&mut self,
base_specifier: &ModuleSpecifier,
@ -48,38 +47,115 @@ impl<'a> ImportMapBuilder<'a> {
.entry(
self
.mappings
.relative_specifier_text(self.mappings.output_dir(), base_specifier),
.relative_specifier_text(self.base_dir, base_specifier),
)
.or_insert_with(|| ImportsBuilder::new(self.mappings))
.or_insert_with(|| ImportsBuilder::new(self.base_dir, self.mappings))
}
pub fn into_serializable(self) -> SerializableImportMap {
SerializableImportMap {
imports: self.imports.imports,
scopes: self
.scopes
.into_iter()
.map(|(key, value)| (key, value.imports))
.collect(),
pub fn into_import_map(
self,
original_import_map: Option<&ImportMap>,
) -> ImportMap {
fn get_local_imports(
new_relative_path: &str,
original_imports: &SpecifierMap,
) -> Vec<(String, String)> {
let mut result = Vec::new();
for entry in original_imports.entries() {
if let Some(raw_value) = entry.raw_value {
if raw_value.starts_with("./") || raw_value.starts_with("../") {
let sub_index = raw_value.find('/').unwrap() + 1;
result.push((
entry.raw_key.to_string(),
format!("{}{}", new_relative_path, &raw_value[sub_index..]),
));
}
}
}
result
}
}
pub fn into_file_text(self) -> String {
let mut text =
serde_json::to_string_pretty(&self.into_serializable()).unwrap();
text.push('\n');
text
fn add_local_imports<'a>(
new_relative_path: &str,
original_imports: &SpecifierMap,
get_new_imports: impl FnOnce() -> &'a mut SpecifierMap,
) {
let local_imports =
get_local_imports(new_relative_path, original_imports);
if !local_imports.is_empty() {
let new_imports = get_new_imports();
for (key, value) in local_imports {
if let Err(warning) = new_imports.append(key, value) {
warn!("{}", warning);
}
}
}
}
let mut import_map = ImportMap::new(self.base_dir.clone());
if let Some(original_im) = original_import_map {
let original_base_dir = ModuleSpecifier::from_directory_path(
original_im
.base_url()
.to_file_path()
.unwrap()
.parent()
.unwrap(),
)
.unwrap();
let new_relative_path = self
.mappings
.relative_specifier_text(self.base_dir, &original_base_dir);
// add the imports
add_local_imports(&new_relative_path, original_im.imports(), || {
import_map.imports_mut()
});
for scope in original_im.scopes() {
if scope.raw_key.starts_with("./") || scope.raw_key.starts_with("../") {
let sub_index = scope.raw_key.find('/').unwrap() + 1;
let new_key =
format!("{}{}", new_relative_path, &scope.raw_key[sub_index..]);
add_local_imports(&new_relative_path, scope.imports, || {
import_map.get_or_append_scope_mut(&new_key).unwrap()
});
}
}
}
let imports = import_map.imports_mut();
for (key, value) in self.imports.imports {
if !imports.contains(&key) {
imports.append(key, value).unwrap();
}
}
for (scope_key, scope_value) in self.scopes {
if !scope_value.imports.is_empty() {
let imports = import_map.get_or_append_scope_mut(&scope_key).unwrap();
for (key, value) in scope_value.imports {
if !imports.contains(&key) {
imports.append(key, value).unwrap();
}
}
}
}
import_map
}
}
struct ImportsBuilder<'a> {
base_dir: &'a ModuleSpecifier,
mappings: &'a Mappings,
imports: BTreeMap<String, String>,
imports: IndexMap<String, String>,
}
impl<'a> ImportsBuilder<'a> {
pub fn new(mappings: &'a Mappings) -> Self {
pub fn new(base_dir: &'a ModuleSpecifier, mappings: &'a Mappings) -> Self {
Self {
base_dir,
mappings,
imports: Default::default(),
}
@ -88,7 +164,7 @@ impl<'a> ImportsBuilder<'a> {
pub fn add(&mut self, key: String, specifier: &ModuleSpecifier) {
let value = self
.mappings
.relative_specifier_text(self.mappings.output_dir(), specifier);
.relative_specifier_text(self.base_dir, specifier);
// skip creating identity entries
if key != value {
@ -98,20 +174,22 @@ impl<'a> ImportsBuilder<'a> {
}
pub fn build_import_map(
base_dir: &ModuleSpecifier,
graph: &ModuleGraph,
modules: &[&Module],
mappings: &Mappings,
original_import_map: Option<&ImportMap>,
) -> String {
let mut import_map = ImportMapBuilder::new(mappings);
visit_modules(graph, modules, mappings, &mut import_map);
let mut builder = ImportMapBuilder::new(base_dir, mappings);
visit_modules(graph, modules, mappings, &mut builder);
for base_specifier in mappings.base_specifiers() {
import_map
builder
.imports
.add(base_specifier.to_string(), base_specifier);
}
import_map.into_file_text()
builder.into_import_map(original_import_map).to_json()
}
fn visit_modules(
@ -197,37 +275,70 @@ fn handle_dep_specifier(
mappings: &Mappings,
) {
let specifier = graph.resolve(unresolved_specifier);
// do not handle specifiers pointing at local modules
if !is_remote_specifier(&specifier) {
return;
// check if it's referencing a remote module
if is_remote_specifier(&specifier) {
handle_remote_dep_specifier(
text,
unresolved_specifier,
&specifier,
import_map,
referrer,
mappings,
)
} else {
handle_local_dep_specifier(
text,
unresolved_specifier,
&specifier,
import_map,
referrer,
mappings,
);
}
}
let base_specifier = mappings.base_specifier(&specifier);
fn handle_remote_dep_specifier(
text: &str,
unresolved_specifier: &ModuleSpecifier,
specifier: &ModuleSpecifier,
import_map: &mut ImportMapBuilder,
referrer: &ModuleSpecifier,
mappings: &Mappings,
) {
if is_remote_specifier_text(text) {
let base_specifier = mappings.base_specifier(specifier);
if !text.starts_with(base_specifier.as_str()) {
panic!("Expected {} to start with {}", text, base_specifier);
}
let sub_path = &text[base_specifier.as_str().len()..];
let expected_relative_specifier_text =
mappings.relative_path(base_specifier, &specifier);
if expected_relative_specifier_text == sub_path {
return;
let relative_text =
mappings.relative_specifier_text(base_specifier, specifier);
let expected_sub_path = relative_text.trim_start_matches("./");
if expected_sub_path != sub_path {
import_map.imports.add(text.to_string(), specifier);
}
import_map.imports.add(text.to_string(), &specifier);
} else {
let expected_relative_specifier_text =
mappings.relative_specifier_text(referrer, &specifier);
mappings.relative_specifier_text(referrer, specifier);
if expected_relative_specifier_text == text {
return;
}
if !is_remote_specifier(referrer) {
// local module referencing a remote module using
// non-remote specifier text means it was something in
// the original import map, so add a mapping to it
import_map.imports.add(text.to_string(), specifier);
return;
}
let base_specifier = mappings.base_specifier(specifier);
let base_dir = import_map.base_dir().clone();
let imports = import_map.scope(base_specifier);
if text.starts_with("./") || text.starts_with("../") {
// resolve relative specifier key
let mut local_base_specifier = mappings.local_uri(base_specifier);
local_base_specifier.set_query(unresolved_specifier.query());
local_base_specifier = local_base_specifier
// path includes "/" so make it relative
.join(&format!(".{}", unresolved_specifier.path()))
@ -241,18 +352,15 @@ fn handle_dep_specifier(
local_base_specifier.set_query(unresolved_specifier.query());
imports.add(
mappings.relative_specifier_text(
mappings.output_dir(),
&local_base_specifier,
),
&specifier,
mappings.relative_specifier_text(&base_dir, &local_base_specifier),
specifier,
);
// add a mapping that uses the local directory name and the remote
// filename in order to support files importing this relatively
imports.add(
{
let local_path = mappings.local_path(&specifier);
let local_path = mappings.local_path(specifier);
let mut value =
ModuleSpecifier::from_directory_path(local_path.parent().unwrap())
.unwrap();
@ -262,17 +370,58 @@ fn handle_dep_specifier(
value.path(),
specifier.path_segments().unwrap().last().unwrap(),
));
mappings.relative_specifier_text(mappings.output_dir(), &value)
mappings.relative_specifier_text(&base_dir, &value)
},
&specifier,
specifier,
);
} else {
// absolute (`/`) or bare specifier should be left as-is
imports.add(text.to_string(), &specifier);
imports.add(text.to_string(), specifier);
}
}
}
fn handle_local_dep_specifier(
text: &str,
unresolved_specifier: &ModuleSpecifier,
specifier: &ModuleSpecifier,
import_map: &mut ImportMapBuilder,
referrer: &ModuleSpecifier,
mappings: &Mappings,
) {
if !is_remote_specifier(referrer) {
// do not handle local modules referencing local modules
return;
}
// The remote module is referencing a local file. This could occur via an
// existing import map. In this case, we'll have to add an import map
// entry in order to map the path back to the local path once vendored.
let base_dir = import_map.base_dir().clone();
let base_specifier = mappings.base_specifier(referrer);
let imports = import_map.scope(base_specifier);
if text.starts_with("./") || text.starts_with("../") {
let referrer_local_uri = mappings.local_uri(referrer);
let mut specifier_local_uri =
referrer_local_uri.join(text).unwrap_or_else(|_| {
panic!(
"Error joining {} to {}",
unresolved_specifier.path(),
referrer_local_uri
)
});
specifier_local_uri.set_query(unresolved_specifier.query());
imports.add(
mappings.relative_specifier_text(&base_dir, &specifier_local_uri),
specifier,
);
} else {
imports.add(text.to_string(), specifier);
}
}
fn text_from_range<'a>(
text_info: &SourceTextInfo,
text: &'a str,

42
cli/tools/vendor/mappings.rs vendored
View file

@ -14,6 +14,7 @@ use deno_graph::Position;
use deno_graph::Resolved;
use crate::fs_util::path_with_stem_suffix;
use crate::fs_util::relative_specifier;
use super::specifiers::dir_name_for_root;
use super::specifiers::get_unique_path;
@ -28,7 +29,6 @@ pub struct ProxiedModule {
/// Constructs and holds the remote specifier to local path mappings.
pub struct Mappings {
output_dir: ModuleSpecifier,
mappings: HashMap<ModuleSpecifier, PathBuf>,
base_specifiers: Vec<ModuleSpecifier>,
proxies: HashMap<ModuleSpecifier, ProxiedModule>,
@ -104,17 +104,12 @@ impl Mappings {
}
Ok(Self {
output_dir: ModuleSpecifier::from_directory_path(output_dir).unwrap(),
mappings,
base_specifiers,
proxies,
})
}
pub fn output_dir(&self) -> &ModuleSpecifier {
&self.output_dir
}
pub fn local_uri(&self, specifier: &ModuleSpecifier) -> ModuleSpecifier {
if specifier.scheme() == "file" {
specifier.clone()
@ -146,43 +141,14 @@ impl Mappings {
}
}
pub fn relative_path(
&self,
from: &ModuleSpecifier,
to: &ModuleSpecifier,
) -> String {
let mut from = self.local_uri(from);
let to = self.local_uri(to);
// workaround using parent directory until https://github.com/servo/rust-url/pull/754 is merged
if !from.path().ends_with('/') {
let local_path = self.local_path(&from);
from = ModuleSpecifier::from_directory_path(local_path.parent().unwrap())
.unwrap();
}
// workaround for url crate not adding a trailing slash for a directory
// it seems to be fixed once a version greater than 2.2.2 is released
let is_dir = to.path().ends_with('/');
let mut text = from.make_relative(&to).unwrap();
if is_dir && !text.ends_with('/') && to.query().is_none() {
text.push('/');
}
text
}
pub fn relative_specifier_text(
&self,
from: &ModuleSpecifier,
to: &ModuleSpecifier,
) -> String {
let relative_path = self.relative_path(from, to);
if relative_path.starts_with("../") || relative_path.starts_with("./") {
relative_path
} else {
format!("./{}", relative_path)
}
let from = self.local_uri(from);
let to = self.local_uri(to);
relative_specifier(&from, &to).unwrap()
}
pub fn base_specifiers(&self) -> &Vec<ModuleSpecifier> {

263
cli/tools/vendor/mod.rs vendored
View file

@ -3,19 +3,24 @@
use std::path::Path;
use std::path::PathBuf;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::resolve_url_or_path;
use deno_runtime::permissions::Permissions;
use log::warn;
use crate::config_file::FmtOptionsConfig;
use crate::flags::VendorFlags;
use crate::fs_util;
use crate::fs_util::relative_specifier;
use crate::fs_util::specifier_to_file_path;
use crate::lockfile;
use crate::proc_state::ProcState;
use crate::resolver::ImportMapResolver;
use crate::resolver::JsxResolver;
use crate::tools::vendor::specifiers::is_remote_specifier_text;
use crate::tools::fmt::format_json;
mod analyze;
mod build;
@ -33,14 +38,15 @@ pub async fn vendor(ps: ProcState, flags: VendorFlags) -> Result<(), AnyError> {
let output_dir = fs_util::resolve_from_cwd(&raw_output_dir)?;
validate_output_dir(&output_dir, &flags, &ps)?;
let graph = create_graph(&ps, &flags).await?;
let vendored_count =
build::build(&graph, &output_dir, &build::RealVendorEnvironment)?;
let vendored_count = build::build(
graph,
&output_dir,
ps.maybe_import_map.as_deref(),
&build::RealVendorEnvironment,
)?;
eprintln!(
r#"Vendored {} {} into {} directory.
To use vendored modules, specify the `--import-map` flag when invoking deno subcommands:
deno run -A --import-map {} {}"#,
concat!("Vendored {} {} into {} directory.",),
vendored_count,
if vendored_count == 1 {
"module"
@ -48,14 +54,31 @@ To use vendored modules, specify the `--import-map` flag when invoking deno subc
"modules"
},
raw_output_dir.display(),
raw_output_dir.join("import_map.json").display(),
flags
.specifiers
.iter()
.map(|s| s.as_str())
.find(|s| !is_remote_specifier_text(s))
.unwrap_or("main.ts"),
);
if vendored_count > 0 {
let import_map_path = raw_output_dir.join("import_map.json");
if maybe_update_config_file(&output_dir, &ps) {
eprintln!(
concat!(
"\nUpdated your local Deno configuration file with a reference to the ",
"new vendored import map at {}. Invoking Deno subcommands will now ",
"automatically resolve using the vendored modules. You may override ",
"this by providing the `--import-map <other-import-map>` flag or by ",
"manually editing your Deno configuration file.",
),
import_map_path.display(),
);
} else {
eprintln!(
concat!(
"\nTo use vendored modules, specify the `--import-map {}` flag when ",
r#"invoking Deno subcommands or add an `"importMap": "<path_to_vendored_import_map>"` "#,
"entry to a deno.json file.",
),
import_map_path.display(),
);
}
}
Ok(())
}
@ -76,7 +99,7 @@ fn validate_output_dir(
if let Some(import_map_path) = ps
.maybe_import_map
.as_ref()
.and_then(|m| m.base_url().to_file_path().ok())
.and_then(|m| specifier_to_file_path(m.base_url()).ok())
.and_then(|p| fs_util::canonicalize_path(&p).ok())
{
// make the output directory in order to canonicalize it for the check below
@ -87,10 +110,21 @@ fn validate_output_dir(
})?;
if import_map_path.starts_with(&output_dir) {
// We don't allow using the output directory to help generate the new state
// of itself because supporting this scenario adds a lot of complexity.
// canonicalize to make the test for this pass on the CI
let cwd = fs_util::canonicalize_path(&std::env::current_dir()?)?;
// We don't allow using the output directory to help generate the
// new state because this may lead to cryptic error messages.
bail!(
"Using an import map found in the output directory is not supported."
concat!(
"Specifying an import map file ({}) in the deno vendor output ",
"directory is not supported. Please specify no import map or one ",
"located outside this directory."
),
import_map_path
.strip_prefix(&cwd)
.unwrap_or(&import_map_path)
.display()
.to_string(),
);
}
}
@ -98,6 +132,104 @@ fn validate_output_dir(
Ok(())
}
fn maybe_update_config_file(output_dir: &Path, ps: &ProcState) -> bool {
assert!(output_dir.is_absolute());
let config_file = match &ps.maybe_config_file {
Some(f) => f,
None => return false,
};
let fmt_config = config_file
.to_fmt_config()
.unwrap_or_default()
.unwrap_or_default();
let result = update_config_file(
&config_file.specifier,
&ModuleSpecifier::from_file_path(output_dir.join("import_map.json"))
.unwrap(),
&fmt_config.options,
);
match result {
Ok(()) => true,
Err(err) => {
warn!("Error updating config file. {:#}", err);
false
}
}
}
fn update_config_file(
config_specifier: &ModuleSpecifier,
import_map_specifier: &ModuleSpecifier,
fmt_options: &FmtOptionsConfig,
) -> Result<(), AnyError> {
if config_specifier.scheme() != "file" {
return Ok(());
}
let config_path = specifier_to_file_path(config_specifier)?;
let config_text = std::fs::read_to_string(&config_path)?;
let relative_text =
match relative_specifier(config_specifier, import_map_specifier) {
Some(text) => text,
None => return Ok(()), // ignore
};
if let Some(new_text) =
update_config_text(&config_text, &relative_text, fmt_options)
{
std::fs::write(config_path, new_text)?;
}
Ok(())
}
fn update_config_text(
text: &str,
import_map_specifier: &str,
fmt_options: &FmtOptionsConfig,
) -> Option<String> {
use jsonc_parser::ast::ObjectProp;
use jsonc_parser::ast::Value;
let ast = jsonc_parser::parse_to_ast(text, &Default::default()).ok()?;
let obj = match ast.value {
Some(Value::Object(obj)) => obj,
_ => return None, // shouldn't happen, so ignore
};
let import_map_specifier = import_map_specifier.replace('\"', "\\\"");
match obj.get("importMap") {
Some(ObjectProp {
value: Value::StringLit(lit),
..
}) => Some(format!(
"{}{}{}",
&text[..lit.range.start + 1],
import_map_specifier,
&text[lit.range.end - 1..],
)),
None => {
// insert it crudely at a position that won't cause any issues
// with comments and format after to make it look nice
let insert_position = obj.range.end - 1;
let insert_text = format!(
r#"{}"importMap": "{}""#,
if obj.properties.is_empty() { "" } else { "," },
import_map_specifier
);
let new_text = format!(
"{}{}{}",
&text[..insert_position],
insert_text,
&text[insert_position..],
);
format_json(&new_text, fmt_options)
.ok()
.map(|formatted_text| formatted_text.unwrap_or(new_text))
}
// shouldn't happen, so ignore
Some(_) => None,
}
}
fn is_dir_empty(dir_path: &Path) -> Result<bool, AnyError> {
match std::fs::read_dir(&dir_path) {
Ok(mut dir) => Ok(dir.next().is_none()),
@ -149,20 +281,85 @@ async fn create_graph(
.map(|im| im.as_resolver())
};
let graph = deno_graph::create_graph(
entry_points,
false,
maybe_imports,
&mut cache,
maybe_resolver,
maybe_locker,
None,
None,
Ok(
deno_graph::create_graph(
entry_points,
false,
maybe_imports,
&mut cache,
maybe_resolver,
maybe_locker,
None,
None,
)
.await,
)
.await;
graph.lock()?;
graph.valid()?;
Ok(graph)
}
#[cfg(test)]
mod internal_test {
use super::*;
use pretty_assertions::assert_eq;
#[test]
fn update_config_text_no_existing_props_add_prop() {
let text = update_config_text(
"{\n}",
"./vendor/import_map.json",
&Default::default(),
)
.unwrap();
assert_eq!(
text,
r#"{
"importMap": "./vendor/import_map.json"
}
"#
);
}
#[test]
fn update_config_text_existing_props_add_prop() {
let text = update_config_text(
r#"{
"tasks": {
"task1": "other"
}
}
"#,
"./vendor/import_map.json",
&Default::default(),
)
.unwrap();
assert_eq!(
text,
r#"{
"tasks": {
"task1": "other"
},
"importMap": "./vendor/import_map.json"
}
"#
);
}
#[test]
fn update_config_text_update_prop() {
let text = update_config_text(
r#"{
"importMap": "./local.json"
}
"#,
"./vendor/import_map.json",
&Default::default(),
)
.unwrap();
assert_eq!(
text,
r#"{
"importMap": "./vendor/import_map.json"
}
"#
);
}
}

88
cli/tools/vendor/test.rs vendored
View file

@ -5,6 +5,7 @@ use std::collections::HashMap;
use std::collections::HashSet;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::anyhow;
@ -16,6 +17,10 @@ use deno_graph::source::LoadFuture;
use deno_graph::source::LoadResponse;
use deno_graph::source::Loader;
use deno_graph::ModuleGraph;
use deno_graph::ModuleKind;
use import_map::ImportMap;
use crate::resolver::ImportMapResolver;
use super::build::VendorEnvironment;
@ -120,6 +125,10 @@ struct TestVendorEnvironment {
}
impl VendorEnvironment for TestVendorEnvironment {
fn cwd(&self) -> Result<PathBuf, AnyError> {
Ok(make_path("/"))
}
fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> {
let mut directories = self.directories.borrow_mut();
for path in dir_path.ancestors() {
@ -141,6 +150,10 @@ impl VendorEnvironment for TestVendorEnvironment {
.insert(file_path.to_path_buf(), text.to_string());
Ok(())
}
fn path_exists(&self, path: &Path) -> bool {
self.files.borrow().contains_key(&path.to_path_buf())
}
}
pub struct VendorOutput {
@ -152,6 +165,8 @@ pub struct VendorOutput {
pub struct VendorTestBuilder {
entry_points: Vec<ModuleSpecifier>,
loader: TestLoader,
original_import_map: Option<ImportMap>,
environment: TestVendorEnvironment,
}
impl VendorTestBuilder {
@ -161,6 +176,19 @@ impl VendorTestBuilder {
builder
}
pub fn new_import_map(&self, base_path: &str) -> ImportMap {
let base = ModuleSpecifier::from_file_path(&make_path(base_path)).unwrap();
ImportMap::new(base)
}
pub fn set_original_import_map(
&mut self,
import_map: ImportMap,
) -> &mut Self {
self.original_import_map = Some(import_map);
self
}
pub fn add_entry_point(&mut self, entry_point: impl AsRef<str>) -> &mut Self {
let entry_point = make_path(entry_point.as_ref());
self
@ -170,11 +198,24 @@ impl VendorTestBuilder {
}
pub async fn build(&mut self) -> Result<VendorOutput, AnyError> {
let graph = self.build_graph().await;
let output_dir = make_path("/vendor");
let environment = TestVendorEnvironment::default();
super::build::build(&graph, &output_dir, &environment)?;
let mut files = environment.files.borrow_mut();
let roots = self
.entry_points
.iter()
.map(|s| (s.to_owned(), deno_graph::ModuleKind::Esm))
.collect();
let loader = self.loader.clone();
let graph =
build_test_graph(roots, self.original_import_map.clone(), loader.clone())
.await;
super::build::build(
graph,
&output_dir,
self.original_import_map.as_ref(),
&self.environment,
)?;
let mut files = self.environment.files.borrow_mut();
let import_map = files.remove(&output_dir.join("import_map.json"));
let mut files = files
.iter()
@ -193,27 +234,26 @@ impl VendorTestBuilder {
action(&mut self.loader);
self
}
}
async fn build_graph(&mut self) -> ModuleGraph {
let graph = deno_graph::create_graph(
self
.entry_points
.iter()
.map(|s| (s.to_owned(), deno_graph::ModuleKind::Esm))
.collect(),
false,
None,
&mut self.loader,
None,
None,
None,
None,
)
.await;
graph.lock().unwrap();
graph.valid().unwrap();
graph
}
async fn build_test_graph(
roots: Vec<(ModuleSpecifier, ModuleKind)>,
original_import_map: Option<ImportMap>,
mut loader: TestLoader,
) -> ModuleGraph {
let resolver =
original_import_map.map(|m| ImportMapResolver::new(Arc::new(m)));
deno_graph::create_graph(
roots,
false,
None,
&mut loader,
resolver.as_ref().map(|im| im.as_resolver()),
None,
None,
None,
)
.await
}
fn make_path(text: &str) -> PathBuf {

20
test_util/src/temp_dir.rs
View file

@ -1,3 +1,4 @@
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::sync::atomic::AtomicU32;
@ -84,4 +85,23 @@ impl TempDir {
let inner = &self.0;
inner.0.as_path()
}
pub fn create_dir_all(&self, path: impl AsRef<Path>) {
fs::create_dir_all(self.path().join(path)).unwrap();
}
pub fn read_to_string(&self, path: impl AsRef<Path>) -> String {
let file_path = self.path().join(path);
fs::read_to_string(&file_path)
.with_context(|| format!("Could not find file: {}", file_path.display()))
.unwrap()
}
pub fn rename(&self, from: impl AsRef<Path>, to: impl AsRef<Path>) {
fs::rename(self.path().join(from), self.path().join(to)).unwrap();
}
pub fn write(&self, path: impl AsRef<Path>, text: impl AsRef<str>) {
fs::write(self.path().join(path), text.as_ref()).unwrap();
}
}