diff --git a/cli/node/mod.rs b/cli/node/mod.rs index c265afdf6a..3ea263b049 100644 --- a/cli/node/mod.rs +++ b/cli/node/mod.rs @@ -578,7 +578,7 @@ fn package_config_resolve( Ok(package_dir.join(package_subpath)) } -fn url_to_node_resolution( +pub fn url_to_node_resolution( url: ModuleSpecifier, npm_resolver: &dyn RequireNpmResolver, ) -> Result { diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index a0ffd05447..ccbe18c7f8 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -92,7 +92,7 @@ impl NpmPackageVersionInfo { } } -#[derive(Debug, Default, Deserialize, Serialize, Clone)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct NpmPackageVersionDistInfo { /// URL to the tarball. pub tarball: String, diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index 87fa9922f5..15fffdf04e 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -11,6 +11,8 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures; use deno_core::parking_lot::RwLock; +use serde::Deserialize; +use serde::Serialize; use super::registry::NpmPackageInfo; use super::registry::NpmPackageVersionDistInfo; @@ -91,7 +93,9 @@ impl std::fmt::Display for NpmPackageReference { } } -#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +#[derive( + Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize, +)] pub struct NpmPackageReq { pub name: String, pub version_req: Option, @@ -127,7 +131,9 @@ impl NpmVersionMatcher for NpmPackageReq { } } -#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[derive( + Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize, +)] pub struct NpmPackageId { pub name: String, pub version: NpmVersion, @@ -150,7 +156,7 @@ impl std::fmt::Display for NpmPackageId { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct NpmResolutionPackage { pub id: NpmPackageId, pub dist: NpmPackageVersionDistInfo, @@ -159,13 +165,54 @@ pub struct NpmResolutionPackage { pub dependencies: HashMap, } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct NpmResolutionSnapshot { + #[serde(with = "map_to_vec")] package_reqs: HashMap, packages_by_name: HashMap>, + #[serde(with = "map_to_vec")] packages: HashMap, } +// This is done so the maps with non-string keys get serialized and deserialized as vectors. +// Adapted from: https://github.com/serde-rs/serde/issues/936#issuecomment-302281792 +mod map_to_vec { + use std::collections::HashMap; + + use serde::de::Deserialize; + use serde::de::Deserializer; + use serde::ser::Serializer; + use serde::Serialize; + + pub fn serialize( + map: &HashMap, + serializer: S, + ) -> Result + where + S: Serializer, + { + serializer.collect_seq(map.iter()) + } + + pub fn deserialize< + 'de, + D, + K: Deserialize<'de> + Eq + std::hash::Hash, + V: Deserialize<'de>, + >( + deserializer: D, + ) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let mut map = HashMap::new(); + for (key, value) in Vec::<(K, V)>::deserialize(deserializer)? { + map.insert(key, value); + } + Ok(map) + } +} + impl NpmResolutionSnapshot { /// Resolve a node package from a deno module. pub fn resolve_package_from_deno_module( @@ -292,10 +339,13 @@ impl std::fmt::Debug for NpmResolution { } impl NpmResolution { - pub fn new(api: NpmRegistryApi) -> Self { + pub fn new( + api: NpmRegistryApi, + initial_snapshot: Option, + ) -> Self { Self { api, - snapshot: Default::default(), + snapshot: RwLock::new(initial_snapshot.unwrap_or_default()), update_sempahore: tokio::sync::Semaphore::new(1), } } diff --git a/cli/npm/resolvers/common.rs b/cli/npm/resolvers/common.rs index 508b783c95..7769d2322f 100644 --- a/cli/npm/resolvers/common.rs +++ b/cli/npm/resolvers/common.rs @@ -10,6 +10,7 @@ use deno_core::futures; use deno_core::futures::future::BoxFuture; use deno_core::url::Url; +use crate::npm::resolution::NpmResolutionSnapshot; use crate::npm::NpmCache; use crate::npm::NpmPackageReq; use crate::npm::NpmResolutionPackage; @@ -39,6 +40,8 @@ pub trait InnerNpmPackageResolver: Send + Sync { ) -> BoxFuture<'static, Result<(), AnyError>>; fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError>; + + fn snapshot(&self) -> NpmResolutionSnapshot; } /// Caches all the packages in parallel. diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs index 94b963898a..c1b6818fde 100644 --- a/cli/npm/resolvers/global.rs +++ b/cli/npm/resolvers/global.rs @@ -13,6 +13,7 @@ use deno_core::futures::FutureExt; use deno_core::url::Url; use crate::npm::resolution::NpmResolution; +use crate::npm::resolution::NpmResolutionSnapshot; use crate::npm::resolvers::common::cache_packages; use crate::npm::NpmCache; use crate::npm::NpmPackageId; @@ -31,9 +32,13 @@ pub struct GlobalNpmPackageResolver { } impl GlobalNpmPackageResolver { - pub fn new(cache: NpmCache, api: NpmRegistryApi) -> Self { + pub fn new( + cache: NpmCache, + api: NpmRegistryApi, + initial_snapshot: Option, + ) -> Self { let registry_url = api.base_url().to_owned(); - let resolution = Arc::new(NpmResolution::new(api)); + let resolution = Arc::new(NpmResolution::new(api, initial_snapshot)); Self { cache, @@ -105,4 +110,8 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { let registry_path = self.cache.registry_folder(&self.registry_url); ensure_registry_read_permission(®istry_path, path) } + + fn snapshot(&self) -> NpmResolutionSnapshot { + self.resolution.snapshot() + } } diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index fa2ad4275d..10ac8abfae 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -47,9 +47,10 @@ impl LocalNpmPackageResolver { cache: NpmCache, api: NpmRegistryApi, node_modules_folder: PathBuf, + initial_snapshot: Option, ) -> Self { let registry_url = api.base_url().to_owned(); - let resolution = Arc::new(NpmResolution::new(api)); + let resolution = Arc::new(NpmResolution::new(api, initial_snapshot)); Self { cache, @@ -180,6 +181,10 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { ensure_registry_read_permission(&self.root_node_modules_path, path) } + + fn snapshot(&self) -> NpmResolutionSnapshot { + self.resolution.snapshot() + } } /// Creates a pnpm style folder structure. diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs index 3a40340f0b..4b4ec4723e 100644 --- a/cli/npm/resolvers/mod.rs +++ b/cli/npm/resolvers/mod.rs @@ -8,9 +8,13 @@ use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; use deno_core::error::custom_error; use deno_core::error::AnyError; +use deno_core::serde_json; use deno_runtime::deno_node::PathClean; use deno_runtime::deno_node::RequireNpmResolver; use global::GlobalNpmPackageResolver; +use once_cell::sync::Lazy; +use serde::Deserialize; +use serde::Serialize; use std::path::Path; use std::path::PathBuf; @@ -20,15 +24,50 @@ use crate::fs_util; use self::common::InnerNpmPackageResolver; use self::local::LocalNpmPackageResolver; +use super::resolution::NpmResolutionSnapshot; use super::NpmCache; use super::NpmPackageReq; use super::NpmRegistryApi; +const RESOLUTION_STATE_ENV_VAR_NAME: &str = + "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE"; + +static IS_NPM_MAIN: Lazy = + Lazy::new(|| std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).is_ok()); + +/// State provided to the process via an environment variable. +#[derive(Debug, Serialize, Deserialize)] +struct NpmProcessState { + snapshot: NpmResolutionSnapshot, + local_node_modules_path: Option, +} + +impl NpmProcessState { + pub fn was_set() -> bool { + *IS_NPM_MAIN + } + + pub fn take() -> Option { + // initialize the lazy before we remove the env var below + if !Self::was_set() { + return None; + } + + let state = std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).ok()?; + let state = serde_json::from_str(&state).ok()?; + // remove the environment variable so that sub processes + // that are spawned do not also use this. + std::env::remove_var(RESOLUTION_STATE_ENV_VAR_NAME); + Some(state) + } +} + #[derive(Clone)] pub struct NpmPackageResolver { unstable: bool, no_npm: bool, inner: Arc, + local_node_modules_path: Option, } impl NpmPackageResolver { @@ -39,19 +78,30 @@ impl NpmPackageResolver { no_npm: bool, local_node_modules_path: Option, ) -> Self { - let inner: Arc = match local_node_modules_path + let process_npm_state = NpmProcessState::take(); + let local_node_modules_path = local_node_modules_path.or_else(|| { + process_npm_state + .as_ref() + .and_then(|s| s.local_node_modules_path.as_ref().map(PathBuf::from)) + }); + let maybe_snapshot = process_npm_state.map(|s| s.snapshot); + let inner: Arc = match &local_node_modules_path { Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new( cache, api, - node_modules_folder, + node_modules_folder.clone(), + maybe_snapshot, )), - None => Arc::new(GlobalNpmPackageResolver::new(cache, api)), + None => { + Arc::new(GlobalNpmPackageResolver::new(cache, api, maybe_snapshot)) + } }; Self { unstable, no_npm, inner, + local_node_modules_path, } } @@ -137,6 +187,26 @@ impl NpmPackageResolver { self.inner.add_package_reqs(packages).await } + + // If the main module should be treated as being in an npm package. + // This is triggered via a secret environment variable which is used + // for functionality like child_process.fork. Users should NOT depend + // on this functionality. + pub fn is_npm_main(&self) -> bool { + NpmProcessState::was_set() + } + + /// Gets the state of npm for the process. + pub fn get_npm_process_state(&self) -> String { + serde_json::to_string(&NpmProcessState { + snapshot: self.inner.snapshot(), + local_node_modules_path: self + .local_node_modules_path + .as_ref() + .map(|p| p.to_string_lossy().to_string()), + }) + .unwrap() + } } impl RequireNpmResolver for NpmPackageResolver { diff --git a/cli/npm/semver/mod.rs b/cli/npm/semver/mod.rs index 53f0f199fe..dd6ca03dbc 100644 --- a/cli/npm/semver/mod.rs +++ b/cli/npm/semver/mod.rs @@ -6,6 +6,8 @@ use std::fmt; use deno_core::anyhow::Context; use deno_core::error::AnyError; use monch::*; +use serde::Deserialize; +use serde::Serialize; use crate::npm::resolution::NpmVersionMatcher; @@ -25,7 +27,9 @@ mod specifier; // A lot of the below is a re-implementation of parts of https://github.com/npm/node-semver // which is Copyright (c) Isaac Z. Schlueter and Contributors (ISC License) -#[derive(Clone, Debug, PartialEq, Eq, Default, Hash)] +#[derive( + Clone, Debug, PartialEq, Eq, Default, Hash, Serialize, Deserialize, +)] pub struct NpmVersion { pub major: u64, pub minor: u64, diff --git a/cli/npm/semver/range.rs b/cli/npm/semver/range.rs index faf11580b7..901b852c0e 100644 --- a/cli/npm/semver/range.rs +++ b/cli/npm/semver/range.rs @@ -2,6 +2,9 @@ use std::cmp::Ordering; +use serde::Deserialize; +use serde::Serialize; + use super::NpmVersion; /// Collection of ranges. @@ -14,7 +17,7 @@ impl VersionRangeSet { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum RangeBound { Version(VersionBound), Unbounded, // matches everything @@ -91,13 +94,13 @@ impl RangeBound { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum VersionBoundKind { Inclusive, Exclusive, } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct VersionBound { pub kind: VersionBoundKind, pub version: NpmVersion, @@ -109,7 +112,7 @@ impl VersionBound { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct VersionRange { pub start: RangeBound, pub end: RangeBound, diff --git a/cli/npm/semver/specifier.rs b/cli/npm/semver/specifier.rs index 64e3c4f9be..220e0a6018 100644 --- a/cli/npm/semver/specifier.rs +++ b/cli/npm/semver/specifier.rs @@ -3,6 +3,8 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use monch::*; +use serde::Deserialize; +use serde::Serialize; use super::errors::with_failure_handling; use super::range::Partial; @@ -11,7 +13,7 @@ use super::range::XRange; use super::NpmVersion; /// Version requirement found in npm specifiers. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct SpecifierVersionReq { raw_text: String, range: VersionRange, diff --git a/cli/ops/mod.rs b/cli/ops/mod.rs index 98029ab44f..df33313534 100644 --- a/cli/ops/mod.rs +++ b/cli/ops/mod.rs @@ -1,7 +1,11 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use crate::proc_state::ProcState; +use deno_core::anyhow::bail; +use deno_core::error::AnyError; +use deno_core::op; use deno_core::Extension; +use deno_core::OpState; pub mod bench; pub mod testing; @@ -12,9 +16,21 @@ pub fn cli_exts(ps: ProcState) -> Vec { fn init_proc_state(ps: ProcState) -> Extension { Extension::builder() + .ops(vec![op_npm_process_state::decl()]) .state(move |state| { state.put(ps.clone()); Ok(()) }) .build() } + +#[op] +fn op_npm_process_state(state: &mut OpState) -> Result { + let proc_state = state.borrow_mut::(); + if !proc_state.options.unstable() { + bail!( + "Unstable use of npm process state. The --unstable flag must be provided." + ) + } + Ok(proc_state.npm_resolver.get_npm_process_state()) +} diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index 6f77cda84c..db53305455 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -104,6 +104,13 @@ itest!(dual_cjs_esm { http_server: true, }); +itest!(child_process_fork_test { + args: "run --unstable -A --quiet npm/child_process_fork_test/main.ts", + output: "npm/child_process_fork_test/main.out", + envs: env_vars(), + http_server: true, +}); + // FIXME(bartlomieju): npm: specifiers are not handled in dynamic imports // at the moment // itest!(dynamic_import { diff --git a/cli/tests/testdata/npm/child_process_fork_test/main.out b/cli/tests/testdata/npm/child_process_fork_test/main.out new file mode 100644 index 0000000000..d5bc57741c --- /dev/null +++ b/cli/tests/testdata/npm/child_process_fork_test/main.out @@ -0,0 +1,2 @@ +function +Done. diff --git a/cli/tests/testdata/npm/child_process_fork_test/main.ts b/cli/tests/testdata/npm/child_process_fork_test/main.ts new file mode 100644 index 0000000000..e560edb7e0 --- /dev/null +++ b/cli/tests/testdata/npm/child_process_fork_test/main.ts @@ -0,0 +1,4 @@ +import "npm:chalk@4"; +import { run } from "npm:@denotest/child-process-fork"; + +run(); diff --git a/cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/forked_path.js b/cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/forked_path.js new file mode 100644 index 0000000000..aaa106315c --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/forked_path.js @@ -0,0 +1,3 @@ +const chalk = require("chalk"); + +console.log(typeof chalk.green); diff --git a/cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/index.js b/cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/index.js new file mode 100644 index 0000000000..674268e6f2 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/index.js @@ -0,0 +1,19 @@ +const path = require("path"); + +function childProcessFork(path) { + const p = Deno.run({ + cmd: [Deno.execPath(), "run", "--unstable", "-A", path], + env: { + "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE": Deno.core.ops.op_npm_process_state(), + } + }); + p.status().then(() => { + console.log("Done."); + }); +} + +module.exports = { + run() { + childProcessFork(path.join(__dirname, "forked_path.js")); + } +}; diff --git a/cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/package.json b/cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/package.json new file mode 100644 index 0000000000..9ab14e3f70 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "@denotest/child-process-fork", + "version": "1.0.0" +} diff --git a/cli/worker.rs b/cli/worker.rs index 0454069fa1..581d4f90f0 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -354,7 +354,6 @@ pub async fn create_main_worker( ps.npm_resolver .add_package_reqs(vec![package_ref.req.clone()]) .await?; - ps.prepare_node_std_graph().await?; let node_resolution = node::node_resolve_binary_export( &package_ref.req, package_ref.sub_path.as_deref(), @@ -363,9 +362,20 @@ pub async fn create_main_worker( let is_main_cjs = matches!(node_resolution, node::NodeResolution::CommonJs(_)); (node_resolution.into_url(), is_main_cjs) + } else if ps.npm_resolver.is_npm_main() { + let node_resolution = + node::url_to_node_resolution(main_module, &ps.npm_resolver)?; + let is_main_cjs = + matches!(node_resolution, node::NodeResolution::CommonJs(_)); + (node_resolution.into_url(), is_main_cjs) } else { (main_module, false) }; + + if ps.npm_resolver.has_packages() { + ps.prepare_node_std_graph().await?; + } + let module_loader = CliModuleLoader::new(ps.clone()); let maybe_inspector_server = ps.maybe_inspector_server.clone();