From 69d5f136badfd7cfa9b979ff2fee7caf397098ca Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 22 Jan 2024 16:31:12 -0500 Subject: [PATCH] feat(lockfile): track JSR and npm dependencies in config file (#22004) See overview in https://github.com/denoland/deno_lockfile/pull/13 --- Cargo.lock | 34 ++--- Cargo.toml | 2 +- cli/Cargo.toml | 6 +- cli/args/mod.rs | 4 + cli/args/package_json.rs | 34 +---- cli/factory.rs | 88 ++++++++++- cli/graph_util.rs | 4 + cli/main.rs | 10 +- cli/npm/managed/installer.rs | 2 +- cli/npm/managed/mod.rs | 18 ++- cli/npm/managed/resolution.rs | 18 +-- cli/standalone/binary.rs | 10 -- cli/tests/integration/npm_tests.rs | 44 +++--- cli/tests/integration/run_tests.rs | 164 +++++++++++++++++++++ cli/tests/testdata/npm/lock_file/main.out | 3 + cli/tests/testdata/run/lock_check_ok2.json | 19 ++- cli/util/import_map.rs | 59 ++++++++ test_util/src/fs.rs | 13 ++ 18 files changed, 409 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3da8b82000..49168feb82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1004,7 +1004,7 @@ dependencies = [ "libz-sys", "log", "lsp-types", - "monch 0.5.0", + "monch", "napi_sym", "nix 0.26.2", "notify", @@ -1352,9 +1352,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.63.3" +version = "0.63.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7cf16929b6a267be4bc535877303a3a36b3595b6b27543baa07d3b77b679d92" +checksum = "be751db3da720bcd35d42f9c0967a04cafd52d554d79262426d1c00179b0c085" dependencies = [ "anyhow", "async-trait", @@ -1365,7 +1365,7 @@ dependencies = [ "import_map", "indexmap", "log", - "monch 0.4.3", + "monch", "once_cell", "parking_lot 0.12.1", "regex", @@ -1478,9 +1478,9 @@ dependencies = [ [[package]] name = "deno_lockfile" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd29f62e6dec60e585f579df3e9c2fc562aadf881319152974bc442a9042077" +checksum = "dfe06eda519ed05b69da567bcba1d728c482fd553ddaa2ffe008468158da6de0" dependencies = [ "ring", "serde", @@ -1606,9 +1606,9 @@ dependencies = [ [[package]] name = "deno_npm" -version = "0.15.3" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718b0b55031643de7808f8b426661b22a685820f1f459e028776bcc49e07b881" +checksum = "376262760b173ff01f8f5d05d58a64f6d863472396afb5582590fa0949342854" dependencies = [ "anyhow", "async-trait", @@ -1616,7 +1616,7 @@ dependencies = [ "deno_semver", "futures", "log", - "monch 0.4.3", + "monch", "serde", "thiserror", ] @@ -1706,7 +1706,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b49e14effd9df8ed261f7a1a34ac19bbaf0fa940c59bd19a6d8313cf41525e1c" dependencies = [ - "monch 0.5.0", + "monch", "once_cell", "serde", "thiserror", @@ -1722,7 +1722,7 @@ dependencies = [ "anyhow", "futures", "glob", - "monch 0.5.0", + "monch", "os_pipe", "path-dedot", "tokio", @@ -2349,9 +2349,9 @@ dependencies = [ [[package]] name = "eszip" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e081e749cb42d7e52b9a066564b86c70e586614d5de684080a2be861f35a0721" +checksum = "214f3a524473bb2c76385c02fdada6c2791ba5e07e9ac72a6c1e373d665b1680" dependencies = [ "anyhow", "base64", @@ -3775,12 +3775,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "monch" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4519a88847ba2d5ead3dc53f1060ec6a571de93f325d9c5c4968147382b1cbc3" - [[package]] name = "monch" version = "0.5.0" @@ -6173,7 +6167,7 @@ dependencies = [ "lazy-regex", "libc", "lsp-types", - "monch 0.5.0", + "monch", "nix 0.26.2", "once_cell", "os_pipe", diff --git a/Cargo.toml b/Cargo.toml index dc626f090e..c2030b89fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ deno_runtime = { version = "0.140.0", path = "./runtime" } napi_sym = { version = "0.62.0", path = "./cli/napi/sym" } deno_bench_util = { version = "0.126.0", path = "./bench_util" } test_util = { path = "./test_util" } -deno_lockfile = "0.17.2" +deno_lockfile = "0.18.0" deno_media_type = { version = "0.1.1", features = ["module_specifier"] } denokv_proto = "0.5.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 16b561c05d..fc715f8fba 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -59,14 +59,14 @@ deno_config = "=0.8.1" deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_doc = { version = "=0.93.0", features = ["html"] } deno_emit = "=0.33.0" -deno_graph = "=0.63.3" +deno_graph = "=0.63.4" deno_lint = { version = "=0.53.0", features = ["docs"] } deno_lockfile.workspace = true -deno_npm = "=0.15.3" +deno_npm = "=0.16.0" deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_semver = "=0.5.4" deno_task_shell = "=0.14.3" -eszip = "=0.57.0" +eszip = "=0.58.0" napi_sym.workspace = true async-trait.workspace = true diff --git a/cli/args/mod.rs b/cli/args/mod.rs index c6bc712f85..6cf051dca3 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -1344,6 +1344,10 @@ impl CliOptions { self.flags.no_npm } + pub fn no_config(&self) -> bool { + self.flags.config_flag == deno_config::ConfigFlag::Disabled + } + pub fn permissions_options(&self) -> PermissionsOptions { PermissionsOptions { allow_env: self.flags.allow_env.clone(), diff --git a/cli/args/package_json.rs b/cli/args/package_json.rs index c304d0715c..67481fd07a 100644 --- a/cli/args/package_json.rs +++ b/cli/args/package_json.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_npm::registry::parse_dep_entry_name_and_raw_version; -use deno_npm::registry::PackageDepNpmSchemeValueParseError; use deno_runtime::deno_node::PackageJson; use deno_semver::package::PackageReq; use deno_semver::VersionReq; @@ -16,8 +15,6 @@ use thiserror::Error; #[derive(Debug, Error, Clone)] pub enum PackageJsonDepValueParseError { - #[error(transparent)] - SchemeValue(#[from] PackageDepNpmSchemeValueParseError), #[error(transparent)] Specifier(#[from] VersionReqSpecifierParseError), #[error("Not implemented scheme '{scheme}'")] @@ -39,7 +36,7 @@ impl PackageJsonDepsProvider { self.0.as_ref() } - pub fn reqs(&self) -> Vec<&PackageReq> { + pub fn reqs(&self) -> Option> { match &self.0 { Some(deps) => { let mut package_reqs = deps @@ -47,9 +44,9 @@ impl PackageJsonDepsProvider { .filter_map(|r| r.as_ref().ok()) .collect::>(); package_reqs.sort(); // deterministic resolution - package_reqs + Some(package_reqs) } - None => Vec::new(), + None => None, } } } @@ -77,9 +74,7 @@ pub fn get_local_package_json_version_reqs( scheme: value.split(':').next().unwrap().to_string(), }); } - let (name, version_req) = parse_dep_entry_name_and_raw_version(key, value) - .map_err(PackageJsonDepValueParseError::SchemeValue)?; - + let (name, version_req) = parse_dep_entry_name_and_raw_version(key, value); let result = VersionReq::parse_from_specifier(version_req); match result { Ok(version_req) => Ok(PackageReq { @@ -159,27 +154,6 @@ mod test { use super::*; - #[test] - fn test_parse_dep_entry_name_and_raw_version() { - let cases = [ - ("test", "^1.2", Ok(("test", "^1.2"))), - ("test", "1.x - 2.6", Ok(("test", "1.x - 2.6"))), - ("test", "npm:package@^1.2", Ok(("package", "^1.2"))), - ( - "test", - "npm:package", - Err("Could not find @ symbol in npm url 'npm:package'"), - ), - ]; - for (key, value, expected_result) in cases { - let result = parse_dep_entry_name_and_raw_version(key, value); - match result { - Ok(result) => assert_eq!(result, expected_result.unwrap()), - Err(err) => assert_eq!(err.to_string(), expected_result.err().unwrap()), - } - } - } - fn get_local_package_json_version_reqs_for_tests( package_json: &PackageJson, ) -> IndexMap> { diff --git a/cli/factory.rs b/cli/factory.rs index 1b084fc28b..bee805215d 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -44,6 +44,8 @@ use crate::standalone::DenoCompileBinaryWriter; use crate::tools::check::TypeChecker; use crate::util::file_watcher::WatcherCommunicator; use crate::util::fs::canonicalize_path_maybe_not_exists; +use crate::util::import_map::deno_json_deps; +use crate::util::import_map::import_map_deps; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; use crate::worker::CliMainWorkerFactory; @@ -54,12 +56,14 @@ use deno_core::parking_lot::Mutex; use deno_core::FeatureChecker; use deno_graph::GraphKind; +use deno_lockfile::WorkspaceMemberConfig; use deno_runtime::deno_fs; use deno_runtime::deno_node::analyze::NodeCodeTranslator; use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_tls::RootCertStoreProvider; use deno_runtime::deno_web::BlobStore; use deno_runtime::inspector_server::InspectorServer; +use deno_semver::package::PackageNv; use import_map::ImportMap; use log::warn; use std::future::Future; @@ -289,10 +293,84 @@ impl CliFactory { } pub fn maybe_lockfile(&self) -> &Option>> { - self - .services - .lockfile - .get_or_init(|| self.options.maybe_lockfile()) + self.services.lockfile.get_or_init(|| { + let maybe_lockfile = self.options.maybe_lockfile(); + + // initialize the lockfile with the workspace's configuration + if let Some(lockfile) = &maybe_lockfile { + let package_json_deps = self + .package_json_deps_provider() + .reqs() + .map(|reqs| reqs.into_iter().map(|s| format!("npm:{}", s)).collect()) + .unwrap_or_default(); + let mut lockfile = lockfile.lock(); + let config = match self.options.maybe_workspace_config() { + Some(workspace_config) => deno_lockfile::WorkspaceConfig { + root: WorkspaceMemberConfig { + package_json_deps, + dependencies: import_map_deps( + &workspace_config.base_import_map_value, + ) + .into_iter() + .map(|req| req.to_string()) + .collect(), + }, + members: workspace_config + .members + .iter() + .map(|member| { + ( + member.package_name.clone(), + WorkspaceMemberConfig { + package_json_deps: Default::default(), + dependencies: deno_json_deps(&member.config_file) + .into_iter() + .map(|req| req.to_string()) + .collect(), + }, + ) + }) + .collect(), + }, + None => deno_lockfile::WorkspaceConfig { + root: WorkspaceMemberConfig { + package_json_deps, + dependencies: self + .options + .maybe_config_file() + .as_ref() + .map(|config| { + deno_json_deps(config) + .into_iter() + .map(|req| req.to_string()) + .collect() + }) + .unwrap_or_default(), + }, + members: Default::default(), + }, + }; + lockfile.set_workspace_config( + deno_lockfile::SetWorkspaceConfigOptions { + no_npm: self.options.no_npm(), + no_config: self.options.no_config(), + config, + nv_to_jsr_url: |nv| { + let nv = PackageNv::from_str(nv).ok()?; + Some( + deno_graph::source::recommended_registry_package_url( + crate::args::deno_registry_url(), + &nv, + ) + .to_string(), + ) + }, + }, + ); + } + + maybe_lockfile + }) } pub async fn npm_resolver( @@ -320,7 +398,7 @@ impl CliFactory { Some(snapshot) => { CliNpmResolverManagedSnapshotOption::Specified(Some(snapshot)) } - None => match self.maybe_lockfile() { + None => match self.maybe_lockfile().as_ref() { Some(lockfile) => { CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( lockfile.clone(), diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 53861415c6..342013e6cb 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -504,6 +504,10 @@ impl ModuleGraphBuilder { format!("jsr:{}", to), ); } + for (name, deps) in graph.packages.package_deps() { + lockfile + .insert_package_deps(name.to_string(), deps.map(|s| s.to_string())); + } } } diff --git a/cli/main.rs b/cli/main.rs index cd065f81e9..fb88ad137a 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -38,6 +38,7 @@ use deno_core::error::AnyError; use deno_core::error::JsError; use deno_core::futures::FutureExt; use deno_core::unsync::JoinHandle; +use deno_npm::resolution::SnapshotFromLockfileError; use deno_runtime::colors; use deno_runtime::fmt_errors::format_js_error; use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics; @@ -261,7 +262,14 @@ fn unwrap_or_exit(result: Result) -> T { if let Some(e) = error.downcast_ref::() { error_string = format_js_error(e); - } else if let Some(e) = error.downcast_ref::() { + } else if let Some(args::LockfileError::IntegrityCheckFailed(e)) = + error.downcast_ref::() + { + error_string = e.to_string(); + error_code = 10; + } else if let Some(SnapshotFromLockfileError::IntegrityCheckFailed(e)) = + error.downcast_ref::() + { error_string = e.to_string(); error_code = 10; } diff --git a/cli/npm/managed/installer.rs b/cli/npm/managed/installer.rs index c836ff7d87..f762be70ec 100644 --- a/cli/npm/managed/installer.rs +++ b/cli/npm/managed/installer.rs @@ -82,7 +82,7 @@ impl PackageJsonDepsInstaller { return Ok(()); // already installed by something else } - let package_reqs = inner.deps_provider.reqs(); + let package_reqs = inner.deps_provider.reqs().unwrap_or_default(); // check if something needs resolving before bothering to load all // the package information (which is slow) diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index b3bd77b259..829750c0a9 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -225,13 +225,21 @@ async fn snapshot_from_lockfile( lockfile: Arc>, api: &dyn NpmRegistryApi, ) -> Result { - let incomplete_snapshot = { + let (incomplete_snapshot, skip_integrity_check) = { let lock = lockfile.lock(); - deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)? + ( + deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)?, + lock.overwrite, + ) }; - let snapshot = - deno_npm::resolution::snapshot_from_lockfile(incomplete_snapshot, api) - .await?; + let snapshot = deno_npm::resolution::snapshot_from_lockfile( + deno_npm::resolution::SnapshotFromLockfileParams { + incomplete_snapshot, + api, + skip_integrity_check, + }, + ) + .await?; Ok(snapshot) } diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs index b020cec038..4d9c4c3e90 100644 --- a/cli/npm/managed/resolution.rs +++ b/cli/npm/managed/resolution.rs @@ -10,7 +10,6 @@ use deno_core::parking_lot::RwLock; use deno_lockfile::NpmPackageDependencyLockfileInfo; use deno_lockfile::NpmPackageLockfileInfo; use deno_npm::registry::NpmPackageInfo; -use deno_npm::registry::NpmPackageVersionDistInfoIntegrity; use deno_npm::registry::NpmRegistryApi; use deno_npm::resolution::NpmPackageVersionResolutionError; use deno_npm::resolution::NpmPackagesPartitioned; @@ -388,21 +387,6 @@ fn populate_lockfile_from_snapshot( fn npm_package_to_lockfile_info( pkg: &NpmResolutionPackage, ) -> NpmPackageLockfileInfo { - fn integrity_for_lockfile( - integrity: NpmPackageVersionDistInfoIntegrity, - ) -> String { - match integrity { - NpmPackageVersionDistInfoIntegrity::Integrity { - algorithm, - base64_hash, - } => format!("{}-{}", algorithm, base64_hash), - NpmPackageVersionDistInfoIntegrity::UnknownIntegrity(integrity) => { - integrity.to_string() - } - NpmPackageVersionDistInfoIntegrity::LegacySha1Hex(hex) => hex.to_string(), - } - } - let dependencies = pkg .dependencies .iter() @@ -415,7 +399,7 @@ fn npm_package_to_lockfile_info( NpmPackageLockfileInfo { display_id: pkg.id.nv.to_string(), serialized_id: pkg.id.as_serialized(), - integrity: integrity_for_lockfile(pkg.dist.integrity()), + integrity: pkg.dist.integrity().for_lockfile(), dependencies, } } diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 3204ca397a..f8bb1e21c7 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -18,7 +18,6 @@ use deno_core::futures::AsyncReadExt; use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; -use deno_npm::registry::PackageDepNpmSchemeValueParseError; use deno_npm::NpmSystemInfo; use deno_runtime::permissions::PermissionsOptions; use deno_semver::package::PackageReq; @@ -51,7 +50,6 @@ const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd"; #[derive(Serialize, Deserialize)] enum SerializablePackageJsonDepValueParseError { - SchemeValue(String), Specifier(String), Unsupported { scheme: String }, } @@ -59,9 +57,6 @@ enum SerializablePackageJsonDepValueParseError { impl SerializablePackageJsonDepValueParseError { pub fn from_err(err: PackageJsonDepValueParseError) -> Self { match err { - PackageJsonDepValueParseError::SchemeValue(err) => { - Self::SchemeValue(err.value) - } PackageJsonDepValueParseError::Specifier(err) => { Self::Specifier(err.source.to_string()) } @@ -73,11 +68,6 @@ impl SerializablePackageJsonDepValueParseError { pub fn into_err(self) -> PackageJsonDepValueParseError { match self { - SerializablePackageJsonDepValueParseError::SchemeValue(value) => { - PackageJsonDepValueParseError::SchemeValue( - PackageDepNpmSchemeValueParseError { value }, - ) - } SerializablePackageJsonDepValueParseError::Specifier(source) => { PackageJsonDepValueParseError::Specifier( VersionReqSpecifierParseError { diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index cfb1861b78..9840f27713 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -256,7 +256,7 @@ itest!(import_map { http_server: true, }); -itest!(lock_file { +itest!(lock_file_integrity_failure { args: "run --allow-read --allow-env --lock npm/lock_file/lock.json npm/lock_file/main.js", output: "npm/lock_file/main.out", envs: env_vars_for_npm_tests(), @@ -1517,10 +1517,9 @@ fn lock_file_lock_write() { #[test] fn auto_discover_lock_file() { - let _server = http_server(); + let context = TestContextBuilder::for_npm().use_temp_cwd().build(); - let deno_dir = util::new_deno_dir(); - let temp_dir = util::TempDir::new(); + let temp_dir = context.temp_dir(); // write empty config file temp_dir.write("deno.json", "{}"); @@ -1541,25 +1540,26 @@ fn auto_discover_lock_file() { }"#; temp_dir.write("deno.lock", lock_file_content); - let deno = util::deno_cmd_with_deno_dir(&deno_dir) - .current_dir(temp_dir.path()) - .arg("run") - .arg("--unstable") - .arg("-A") - .arg("npm:@denotest/bin/cli-esm") - .arg("test") - .envs(env_vars_for_npm_tests()) - .piped_output() - .spawn() - .unwrap(); - let output = deno.wait_with_output().unwrap(); - assert!(!output.status.success()); - assert_eq!(output.status.code(), Some(10)); + let output = context + .new_command() + .args("run --unstable -A npm:@denotest/bin/cli-esm test") + .run(); + output + .assert_matches_text( +r#"Download http://localhost:4545/npm/registry/@denotest/bin +error: Integrity check failed for npm package: "@denotest/bin@1.0.0". Unable to verify that the package +is the same as when the lockfile was generated. - let stderr = String::from_utf8(output.stderr).unwrap(); - assert!(stderr.contains( - "Integrity check failed for npm package: \"@denotest/bin@1.0.0\"" - )); +Actual: sha512-[WILDCARD] +Expected: sha512-foobar + +This could be caused by: + * the lock file may be corrupt + * the source itself may be corrupt + +Use "--lock-write" flag to regenerate the lockfile at "[WILDCARD]deno.lock". +"#) + .assert_exit_code(10); } #[test] diff --git a/cli/tests/integration/run_tests.rs b/cli/tests/integration/run_tests.rs index d7d65726ea..466972c92e 100644 --- a/cli/tests/integration/run_tests.rs +++ b/cli/tests/integration/run_tests.rs @@ -1017,6 +1017,170 @@ fn lock_redirects() { ); } +#[test] +fn lock_deno_json_package_json_deps() { + let context = TestContextBuilder::new() + .use_temp_cwd() + .use_http_server() + .add_npm_env_vars() + .add_jsr_env_vars() + .build(); + let temp_dir = context.temp_dir().path(); + let deno_json = temp_dir.join("deno.json"); + let package_json = temp_dir.join("package.json"); + + // add a jsr and npm dependency + deno_json.write_json(&json!({ + "imports": { + "esm-basic": "npm:@denotest/esm-basic", + "module_graph": "jsr:@denotest/module_graph@1.4", + } + })); + let main_ts = temp_dir.join("main.ts"); + main_ts.write("import 'esm-basic'; import 'module_graph';"); + context + .new_command() + .args("cache main.ts") + .run() + .skip_output_check(); + let lockfile = temp_dir.join("deno.lock"); + // todo(dsherret): it would be nice if the test server didn't produce + // different hashes depending on what operating system it's running on + let esm_basic_integrity = lockfile + .read_json_value() + .get("packages") + .unwrap() + .get("npm") + .unwrap() + .get("@denotest/esm-basic@1.0.0") + .unwrap() + .get("integrity") + .unwrap() + .as_str() + .unwrap() + .to_string(); + lockfile.assert_matches_json(json!({ + "version": "3", + "packages": { + "specifiers": { + "jsr:@denotest/module_graph@1.4": "jsr:@denotest/module_graph@1.4.0", + "npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" + }, + "npm": { + "@denotest/esm-basic@1.0.0": { + "integrity": esm_basic_integrity, + "dependencies": {} + } + } + }, + "remote": { + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/module_graph@1.4", + "npm:@denotest/esm-basic" + ] + } + })); + + // now remove the npm dependency from the deno.json and move + // it to a package.json that uses an alias + deno_json.write_json(&json!({ + "imports": { + "module_graph": "jsr:@denotest/module_graph@1.4", + } + })); + package_json.write_json(&json!({ + "dependencies": { + "esm-basic": "npm:@denotest/esm-basic" + } + })); + context + .new_command() + .args("cache main.ts") + .run() + .skip_output_check(); + main_ts.write("import 'module_graph';"); + context + .new_command() + // ensure this doesn't clear out packageJson below + .args("cache --no-npm main.ts") + .run() + .skip_output_check(); + lockfile.assert_matches_json(json!({ + "version": "3", + "packages": { + "specifiers": { + "jsr:@denotest/module_graph@1.4": "jsr:@denotest/module_graph@1.4.0", + "npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" + }, + "npm": { + "@denotest/esm-basic@1.0.0": { + "integrity": esm_basic_integrity, + "dependencies": {} + } + } + }, + "remote": { + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/module_graph@1.4" + ], + "packageJson": { + "dependencies": [ + "npm:@denotest/esm-basic" + ] + } + } + })); + + // now remove the package.json + package_json.remove_file(); + + // cache and it will remove the package.json + context + .new_command() + .args("cache main.ts") + .run() + .skip_output_check(); + lockfile.assert_matches_json(json!({ + "version": "3", + "packages": { + "specifiers": { + "jsr:@denotest/module_graph@1.4": "jsr:@denotest/module_graph@1.4.0", + } + }, + "remote": { + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", + "http://localhost:4545/jsr/registry/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/module_graph@1.4" + ] + } + })); + + // now remove the deps from the deno.json + deno_json.write("{}"); + main_ts.write(""); + context + .new_command() + .args("cache main.ts") + .run() + .skip_output_check(); + + lockfile.assert_matches_json(json!({ + "version": "3", + "remote": {} + })); +} + itest!(mts_dmts_mjs { args: "run subdir/import.mts", output: "run/mts_dmts_mjs.out", diff --git a/cli/tests/testdata/npm/lock_file/main.out b/cli/tests/testdata/npm/lock_file/main.out index b8447bee63..65e881be6a 100644 --- a/cli/tests/testdata/npm/lock_file/main.out +++ b/cli/tests/testdata/npm/lock_file/main.out @@ -2,6 +2,9 @@ Download [WILDCARD] error: Integrity check failed for npm package: "@babel/parser@7.19.0". Unable to verify that the package is the same as when the lockfile was generated. +Actual: sha512-74bEXKX2h+8rrfQUfsBfuZZHzsEs6Eql4pqy/T4Nn6Y9wNPggQOqD6z6pn5Bl8ZfysKouFZT/UXEH94ummEeQw== +Expected: sha512-foobar! + This could be caused by: * the lock file may be corrupt * the source itself may be corrupt diff --git a/cli/tests/testdata/run/lock_check_ok2.json b/cli/tests/testdata/run/lock_check_ok2.json index 162c755e2c..14d8b71171 100644 --- a/cli/tests/testdata/run/lock_check_ok2.json +++ b/cli/tests/testdata/run/lock_check_ok2.json @@ -1,10 +1,13 @@ { - "http://localhost:4545/subdir/mt_application_ecmascript.j2.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_application_x_javascript.j4.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_application_x_typescript.t4.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_text_ecmascript.j3.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_text_javascript.j1.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_text_typescript.t1.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_video_mp2t.t3.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", - "http://localhost:4545/subdir/mt_video_vdn.t2.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18" + "version": "3", + "remote": { + "http://localhost:4545/subdir/mt_application_ecmascript.j2.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_application_x_javascript.j4.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_application_x_typescript.t4.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_text_ecmascript.j3.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_text_javascript.j1.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_text_typescript.t1.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_video_mp2t.t3.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/subdir/mt_video_vdn.t2.ts": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18" + } } diff --git a/cli/util/import_map.rs b/cli/util/import_map.rs index 0b78a133c3..10c5dc3f42 100644 --- a/cli/util/import_map.rs +++ b/cli/util/import_map.rs @@ -1,17 +1,76 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::collections::HashSet; + use deno_ast::ParsedSource; use deno_core::error::AnyError; +use deno_core::serde_json; use deno_core::ModuleSpecifier; use deno_graph::DefaultModuleAnalyzer; use deno_graph::DependencyDescriptor; use deno_graph::DynamicTemplatePart; use deno_graph::MediaType; use deno_graph::TypeScriptReference; +use deno_semver::jsr::JsrDepPackageReq; +use deno_semver::jsr::JsrPackageReqReference; +use deno_semver::npm::NpmPackageReqReference; use import_map::ImportMap; use crate::graph_util::format_range_with_colors; +pub fn import_map_deps(value: &serde_json::Value) -> HashSet { + let Some(obj) = value.as_object() else { + return Default::default(); + }; + let values = imports_values(obj.get("imports")) + .into_iter() + .chain(scope_values(obj.get("scopes"))); + values_to_set(values) +} + +pub fn deno_json_deps( + config: &deno_config::ConfigFile, +) -> HashSet { + let values = imports_values(config.json.imports.as_ref()) + .into_iter() + .chain(scope_values(config.json.scopes.as_ref())); + values_to_set(values) +} + +fn imports_values(value: Option<&serde_json::Value>) -> Vec<&String> { + let Some(obj) = value.and_then(|v| v.as_object()) else { + return Vec::new(); + }; + let mut items = Vec::with_capacity(obj.len()); + for value in obj.values() { + if let serde_json::Value::String(value) = value { + items.push(value); + } + } + items +} + +fn scope_values(value: Option<&serde_json::Value>) -> Vec<&String> { + let Some(obj) = value.and_then(|v| v.as_object()) else { + return Vec::new(); + }; + obj.values().flat_map(|v| imports_values(Some(v))).collect() +} + +fn values_to_set<'a>( + values: impl Iterator, +) -> HashSet { + let mut entries = HashSet::new(); + for value in values { + if let Ok(req_ref) = JsrPackageReqReference::from_str(value) { + entries.insert(JsrDepPackageReq::jsr(req_ref.into_inner().req)); + } else if let Ok(req_ref) = NpmPackageReqReference::from_str(value) { + entries.insert(JsrDepPackageReq::npm(req_ref.into_inner().req)); + } + } + entries +} + pub struct ImportMapUnfurler<'a> { import_map: &'a ImportMap, } diff --git a/test_util/src/fs.rs b/test_util/src/fs.rs index 9cfbcaaa70..17620276bc 100644 --- a/test_util/src/fs.rs +++ b/test_util/src/fs.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use pretty_assertions::assert_eq; use std::borrow::Cow; use std::ffi::OsStr; use std::fs; @@ -218,6 +219,7 @@ impl PathRef { } } + #[track_caller] pub fn assert_matches_file(&self, wildcard_file: impl AsRef) -> &Self { let wildcard_file = testdata_path().join(wildcard_file); println!("output path {}", wildcard_file); @@ -225,11 +227,22 @@ impl PathRef { self.assert_matches_text(&expected_text) } + #[track_caller] pub fn assert_matches_text(&self, wildcard_text: impl AsRef) -> &Self { let actual = self.read_to_string(); assert_wildcard_match(&actual, wildcard_text.as_ref()); self } + + #[track_caller] + pub fn assert_matches_json(&self, expected: serde_json::Value) { + let actual_json = self.read_json_value(); + if actual_json != expected { + let actual_text = serde_json::to_string_pretty(&actual_json).unwrap(); + let expected_text = serde_json::to_string_pretty(&expected).unwrap(); + assert_eq!(actual_text, expected_text); + } + } } #[cfg(not(windows))]