feat(npm): functionality to support child_process.fork (#15891)

cli/node/mod.rs

@@ -578,7 +578,7 @@ fn package_config_resolve(
   Ok(package_dir.join(package_subpath))
 }
 
-fn url_to_node_resolution(
+pub fn url_to_node_resolution(
   url: ModuleSpecifier,
   npm_resolver: &dyn RequireNpmResolver,
 ) -> Result<NodeResolution, AnyError> {

cli/npm/registry.rs

@@ -92,7 +92,7 @@ impl NpmPackageVersionInfo {
   }
 }
 
-#[derive(Debug, Default, Deserialize, Serialize, Clone)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct NpmPackageVersionDistInfo {
   /// URL to the tarball.
   pub tarball: String,

cli/npm/resolution.rs

@@ -11,6 +11,8 @@ use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use deno_core::futures;
 use deno_core::parking_lot::RwLock;
+use serde::Deserialize;
+use serde::Serialize;
 
 use super::registry::NpmPackageInfo;
 use super::registry::NpmPackageVersionDistInfo;
@@ -91,7 +93,9 @@ impl std::fmt::Display for NpmPackageReference {
   }
 }
 
-#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+#[derive(
+  Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize,
+)]
 pub struct NpmPackageReq {
   pub name: String,
   pub version_req: Option<SpecifierVersionReq>,
@@ -127,7 +131,9 @@ impl NpmVersionMatcher for NpmPackageReq {
   }
 }
 
-#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)]
+#[derive(
+  Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize,
+)]
 pub struct NpmPackageId {
   pub name: String,
   pub version: NpmVersion,
@@ -150,7 +156,7 @@ impl std::fmt::Display for NpmPackageId {
   }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct NpmResolutionPackage {
   pub id: NpmPackageId,
   pub dist: NpmPackageVersionDistInfo,
@@ -159,13 +165,54 @@ pub struct NpmResolutionPackage {
   pub dependencies: HashMap<String, NpmPackageId>,
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct NpmResolutionSnapshot {
+  #[serde(with = "map_to_vec")]
   package_reqs: HashMap<NpmPackageReq, NpmVersion>,
   packages_by_name: HashMap<String, Vec<NpmVersion>>,
+  #[serde(with = "map_to_vec")]
   packages: HashMap<NpmPackageId, NpmResolutionPackage>,
 }
 
+// This is done so the maps with non-string keys get serialized and deserialized as vectors.
+// Adapted from: https://github.com/serde-rs/serde/issues/936#issuecomment-302281792
+mod map_to_vec {
+  use std::collections::HashMap;
+
+  use serde::de::Deserialize;
+  use serde::de::Deserializer;
+  use serde::ser::Serializer;
+  use serde::Serialize;
+
+  pub fn serialize<S, K: Serialize, V: Serialize>(
+    map: &HashMap<K, V>,
+    serializer: S,
+  ) -> Result<S::Ok, S::Error>
+  where
+    S: Serializer,
+  {
+    serializer.collect_seq(map.iter())
+  }
+
+  pub fn deserialize<
+    'de,
+    D,
+    K: Deserialize<'de> + Eq + std::hash::Hash,
+    V: Deserialize<'de>,
+  >(
+    deserializer: D,
+  ) -> Result<HashMap<K, V>, D::Error>
+  where
+    D: Deserializer<'de>,
+  {
+    let mut map = HashMap::new();
+    for (key, value) in Vec::<(K, V)>::deserialize(deserializer)? {
+      map.insert(key, value);
+    }
+    Ok(map)
+  }
+}
+
 impl NpmResolutionSnapshot {
   /// Resolve a node package from a deno module.
   pub fn resolve_package_from_deno_module(
@@ -292,10 +339,13 @@ impl std::fmt::Debug for NpmResolution {
 }
 
 impl NpmResolution {
-  pub fn new(api: NpmRegistryApi) -> Self {
+  pub fn new(
+    api: NpmRegistryApi,
+    initial_snapshot: Option<NpmResolutionSnapshot>,
+  ) -> Self {
     Self {
       api,
-      snapshot: Default::default(),
+      snapshot: RwLock::new(initial_snapshot.unwrap_or_default()),
       update_sempahore: tokio::sync::Semaphore::new(1),
     }
   }

cli/npm/resolvers/common.rs

@@ -10,6 +10,7 @@ use deno_core::futures;
 use deno_core::futures::future::BoxFuture;
 use deno_core::url::Url;
 
+use crate::npm::resolution::NpmResolutionSnapshot;
 use crate::npm::NpmCache;
 use crate::npm::NpmPackageReq;
 use crate::npm::NpmResolutionPackage;
@@ -39,6 +40,8 @@ pub trait InnerNpmPackageResolver: Send + Sync {
   ) -> BoxFuture<'static, Result<(), AnyError>>;
 
   fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError>;
+
+  fn snapshot(&self) -> NpmResolutionSnapshot;
 }
 
 /// Caches all the packages in parallel.

cli/npm/resolvers/global.rs

@@ -13,6 +13,7 @@ use deno_core::futures::FutureExt;
 use deno_core::url::Url;
 
 use crate::npm::resolution::NpmResolution;
+use crate::npm::resolution::NpmResolutionSnapshot;
 use crate::npm::resolvers::common::cache_packages;
 use crate::npm::NpmCache;
 use crate::npm::NpmPackageId;
@@ -31,9 +32,13 @@ pub struct GlobalNpmPackageResolver {
 }
 
 impl GlobalNpmPackageResolver {
-  pub fn new(cache: NpmCache, api: NpmRegistryApi) -> Self {
+  pub fn new(
+    cache: NpmCache,
+    api: NpmRegistryApi,
+    initial_snapshot: Option<NpmResolutionSnapshot>,
+  ) -> Self {
     let registry_url = api.base_url().to_owned();
-    let resolution = Arc::new(NpmResolution::new(api));
+    let resolution = Arc::new(NpmResolution::new(api, initial_snapshot));
 
     Self {
       cache,
@@ -105,4 +110,8 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver {
     let registry_path = self.cache.registry_folder(&self.registry_url);
     ensure_registry_read_permission(&registry_path, path)
   }
+
+  fn snapshot(&self) -> NpmResolutionSnapshot {
+    self.resolution.snapshot()
+  }
 }

cli/npm/resolvers/local.rs

@@ -47,9 +47,10 @@ impl LocalNpmPackageResolver {
     cache: NpmCache,
     api: NpmRegistryApi,
     node_modules_folder: PathBuf,
+    initial_snapshot: Option<NpmResolutionSnapshot>,
   ) -> Self {
     let registry_url = api.base_url().to_owned();
-    let resolution = Arc::new(NpmResolution::new(api));
+    let resolution = Arc::new(NpmResolution::new(api, initial_snapshot));
 
     Self {
       cache,
@@ -180,6 +181,10 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver {
   fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
     ensure_registry_read_permission(&self.root_node_modules_path, path)
   }
+
+  fn snapshot(&self) -> NpmResolutionSnapshot {
+    self.resolution.snapshot()
+  }
 }
 
 /// Creates a pnpm style folder structure.

cli/npm/resolvers/mod.rs

@@ -8,9 +8,13 @@ use deno_ast::ModuleSpecifier;
 use deno_core::anyhow::bail;
 use deno_core::error::custom_error;
 use deno_core::error::AnyError;
+use deno_core::serde_json;
 use deno_runtime::deno_node::PathClean;
 use deno_runtime::deno_node::RequireNpmResolver;
 use global::GlobalNpmPackageResolver;
+use once_cell::sync::Lazy;
+use serde::Deserialize;
+use serde::Serialize;
 
 use std::path::Path;
 use std::path::PathBuf;
@@ -20,15 +24,50 @@ use crate::fs_util;
 
 use self::common::InnerNpmPackageResolver;
 use self::local::LocalNpmPackageResolver;
+use super::resolution::NpmResolutionSnapshot;
 use super::NpmCache;
 use super::NpmPackageReq;
 use super::NpmRegistryApi;
 
+const RESOLUTION_STATE_ENV_VAR_NAME: &str =
+  "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE";
+
+static IS_NPM_MAIN: Lazy<bool> =
+  Lazy::new(|| std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).is_ok());
+
+/// State provided to the process via an environment variable.
+#[derive(Debug, Serialize, Deserialize)]
+struct NpmProcessState {
+  snapshot: NpmResolutionSnapshot,
+  local_node_modules_path: Option<String>,
+}
+
+impl NpmProcessState {
+  pub fn was_set() -> bool {
+    *IS_NPM_MAIN
+  }
+
+  pub fn take() -> Option<NpmProcessState> {
+    // initialize the lazy before we remove the env var below
+    if !Self::was_set() {
+      return None;
+    }
+
+    let state = std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).ok()?;
+    let state = serde_json::from_str(&state).ok()?;
+    // remove the environment variable so that sub processes
+    // that are spawned do not also use this.
+    std::env::remove_var(RESOLUTION_STATE_ENV_VAR_NAME);
+    Some(state)
+  }
+}
+
 #[derive(Clone)]
 pub struct NpmPackageResolver {
   unstable: bool,
   no_npm: bool,
   inner: Arc<dyn InnerNpmPackageResolver>,
+  local_node_modules_path: Option<PathBuf>,
 }
 
 impl NpmPackageResolver {
@@ -39,19 +78,30 @@ impl NpmPackageResolver {
     no_npm: bool,
     local_node_modules_path: Option<PathBuf>,
   ) -> Self {
-    let inner: Arc<dyn InnerNpmPackageResolver> = match local_node_modules_path
+    let process_npm_state = NpmProcessState::take();
+    let local_node_modules_path = local_node_modules_path.or_else(|| {
+      process_npm_state
+        .as_ref()
+        .and_then(|s| s.local_node_modules_path.as_ref().map(PathBuf::from))
+    });
+    let maybe_snapshot = process_npm_state.map(|s| s.snapshot);
+    let inner: Arc<dyn InnerNpmPackageResolver> = match &local_node_modules_path
     {
       Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
         cache,
         api,
-        node_modules_folder,
+        node_modules_folder.clone(),
+        maybe_snapshot,
       )),
-      None => Arc::new(GlobalNpmPackageResolver::new(cache, api)),
+      None => {
+        Arc::new(GlobalNpmPackageResolver::new(cache, api, maybe_snapshot))
+      }
     };
     Self {
       unstable,
       no_npm,
       inner,
+      local_node_modules_path,
     }
   }
 
@@ -137,6 +187,26 @@ impl NpmPackageResolver {
 
     self.inner.add_package_reqs(packages).await
   }
+
+  // If the main module should be treated as being in an npm package.
+  // This is triggered via a secret environment variable which is used
+  // for functionality like child_process.fork. Users should NOT depend
+  // on this functionality.
+  pub fn is_npm_main(&self) -> bool {
+    NpmProcessState::was_set()
+  }
+
+  /// Gets the state of npm for the process.
+  pub fn get_npm_process_state(&self) -> String {
+    serde_json::to_string(&NpmProcessState {
+      snapshot: self.inner.snapshot(),
+      local_node_modules_path: self
+        .local_node_modules_path
+        .as_ref()
+        .map(|p| p.to_string_lossy().to_string()),
+    })
+    .unwrap()
+  }
 }
 
 impl RequireNpmResolver for NpmPackageResolver {

cli/npm/semver/mod.rs

@@ -6,6 +6,8 @@ use std::fmt;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use monch::*;
+use serde::Deserialize;
+use serde::Serialize;
 
 use crate::npm::resolution::NpmVersionMatcher;
 
@@ -25,7 +27,9 @@ mod specifier;
 // A lot of the below is a re-implementation of parts of https://github.com/npm/node-semver
 // which is Copyright (c) Isaac Z. Schlueter and Contributors (ISC License)
 
-#[derive(Clone, Debug, PartialEq, Eq, Default, Hash)]
+#[derive(
+  Clone, Debug, PartialEq, Eq, Default, Hash, Serialize, Deserialize,
+)]
 pub struct NpmVersion {
   pub major: u64,
   pub minor: u64,

cli/npm/semver/range.rs

@@ -2,6 +2,9 @@
 
 use std::cmp::Ordering;
 
+use serde::Deserialize;
+use serde::Serialize;
+
 use super::NpmVersion;
 
 /// Collection of ranges.
@@ -14,7 +17,7 @@ impl VersionRangeSet {
   }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub enum RangeBound {
   Version(VersionBound),
   Unbounded, // matches everything
@@ -91,13 +94,13 @@ impl RangeBound {
   }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub enum VersionBoundKind {
   Inclusive,
   Exclusive,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct VersionBound {
   pub kind: VersionBoundKind,
   pub version: NpmVersion,
@@ -109,7 +112,7 @@ impl VersionBound {
   }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct VersionRange {
   pub start: RangeBound,
   pub end: RangeBound,

cli/npm/semver/specifier.rs

@@ -3,6 +3,8 @@
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use monch::*;
+use serde::Deserialize;
+use serde::Serialize;
 
 use super::errors::with_failure_handling;
 use super::range::Partial;
@@ -11,7 +13,7 @@ use super::range::XRange;
 use super::NpmVersion;
 
 /// Version requirement found in npm specifiers.
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct SpecifierVersionReq {
   raw_text: String,
   range: VersionRange,

cli/ops/mod.rs

@@ -1,7 +1,11 @@
 // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
 
 use crate::proc_state::ProcState;
+use deno_core::anyhow::bail;
+use deno_core::error::AnyError;
+use deno_core::op;
 use deno_core::Extension;
+use deno_core::OpState;
 
 pub mod bench;
 pub mod testing;
@@ -12,9 +16,21 @@ pub fn cli_exts(ps: ProcState) -> Vec<Extension> {
 
 fn init_proc_state(ps: ProcState) -> Extension {
   Extension::builder()
+    .ops(vec![op_npm_process_state::decl()])
     .state(move |state| {
       state.put(ps.clone());
       Ok(())
     })
     .build()
 }
+
+#[op]
+fn op_npm_process_state(state: &mut OpState) -> Result<String, AnyError> {
+  let proc_state = state.borrow_mut::<ProcState>();
+  if !proc_state.options.unstable() {
+    bail!(
+      "Unstable use of npm process state. The --unstable flag must be provided."
+    )
+  }
+  Ok(proc_state.npm_resolver.get_npm_process_state())
+}

cli/tests/integration/npm_tests.rs

@@ -104,6 +104,13 @@ itest!(dual_cjs_esm {
   http_server: true,
 });
 
+itest!(child_process_fork_test {
+  args: "run --unstable -A --quiet npm/child_process_fork_test/main.ts",
+  output: "npm/child_process_fork_test/main.out",
+  envs: env_vars(),
+  http_server: true,
+});
+
 // FIXME(bartlomieju): npm: specifiers are not handled in dynamic imports
 // at the moment
 // itest!(dynamic_import {

cli/tests/testdata/npm/child_process_fork_test/main.out

@@ -0,0 +1,2 @@
+function
+Done.

cli/tests/testdata/npm/child_process_fork_test/main.ts

@@ -0,0 +1,4 @@
+import "npm:chalk@4";
+import { run } from "npm:@denotest/child-process-fork";
+
+run();

cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/forked_path.js

@@ -0,0 +1,3 @@
+const chalk = require("chalk");
+
+console.log(typeof chalk.green);

cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/index.js

@@ -0,0 +1,19 @@
+const path = require("path");
+
+function childProcessFork(path) {
+  const p = Deno.run({
+    cmd: [Deno.execPath(), "run", "--unstable", "-A", path],
+    env: {
+      "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE": Deno.core.ops.op_npm_process_state(),
+    }
+  });
+  p.status().then(() => {
+    console.log("Done.");
+  });
+}
+
+module.exports = {
+  run() {
+    childProcessFork(path.join(__dirname, "forked_path.js"));
+  }
+};

cli/tests/testdata/npm/registry/@denotest/child-process-fork/1.0.0/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@denotest/child-process-fork",
+  "version": "1.0.0"
+}

cli/worker.rs

@@ -354,7 +354,6 @@ pub async fn create_main_worker(
     ps.npm_resolver
       .add_package_reqs(vec![package_ref.req.clone()])
       .await?;
-    ps.prepare_node_std_graph().await?;
     let node_resolution = node::node_resolve_binary_export(
       &package_ref.req,
       package_ref.sub_path.as_deref(),
@@ -363,9 +362,20 @@ pub async fn create_main_worker(
     let is_main_cjs =
       matches!(node_resolution, node::NodeResolution::CommonJs(_));
     (node_resolution.into_url(), is_main_cjs)
+  } else if ps.npm_resolver.is_npm_main() {
+    let node_resolution =
+      node::url_to_node_resolution(main_module, &ps.npm_resolver)?;
+    let is_main_cjs =
+      matches!(node_resolution, node::NodeResolution::CommonJs(_));
+    (node_resolution.into_url(), is_main_cjs)
   } else {
     (main_module, false)
   };
+
+  if ps.npm_resolver.has_packages() {
+    ps.prepare_node_std_graph().await?;
+  }
+
   let module_loader = CliModuleLoader::new(ps.clone());
 
   let maybe_inspector_server = ps.maybe_inspector_server.clone();