feat: wire up ext/node to the Node compatibility layer (#17785)

This PR changes Node.js/npm compatibility layer to use polyfills for
built-in Node.js
embedded in the snapshot (that are coming from "ext/node" extension).

As a result loading `std/node`, either from
"https://deno.land/std@<latest>/" or
from "DENO_NODE_COMPAT_URL" env variable were removed. All code that is
imported via "npm:" specifiers now uses code embedded in the snapshot.

Several fixes were applied to various modules in "ext/node" to make
tests pass.

---------

Co-authored-by: Yoshiya Hinosawa <stibium121@gmail.com>
Co-authored-by: Divy Srivastava <dj.srivastava23@gmail.com>

cli/cache/mod.rs

@@ -114,18 +114,14 @@ impl Loader for FetchCacher {
 
     let specifier =
       if let Some(module_name) = specifier.as_str().strip_prefix("node:") {
-        if module_name == "module" {
+        // Built-in Node modules are embedded in the Deno binary (in V8 snapshot)
-          // the source code for "node:module" is built-in rather than
+        // so we don't want them to be loaded by the "deno graph".
-          // being from deno_std like the other modules
-          return Box::pin(futures::future::ready(Ok(Some(
-            deno_graph::source::LoadResponse::External {
-              specifier: specifier.clone(),
-            },
-          ))));
-        }
-
         match crate::node::resolve_builtin_node_module(module_name) {
-          Ok(specifier) => specifier,
+          Ok(specifier) => {
+            return Box::pin(futures::future::ready(Ok(Some(
+              deno_graph::source::LoadResponse::External { specifier },
+            ))))
+          }
           Err(err) => return Box::pin(futures::future::ready(Err(err))),
         }
       } else {

cli/module_loader.rs

@@ -77,11 +77,10 @@ impl CliModuleLoader {
     specifier: &ModuleSpecifier,
     maybe_referrer: Option<ModuleSpecifier>,
   ) -> Result<ModuleCodeSource, AnyError> {
-    // TODO(bartlomieju): uncomment, when all `node:` module have been
+    if specifier.scheme() == "node" {
-    // snapshotted
+      unreachable!("Node built-in modules should be handled internally.");
-    // if specifier.scheme() == "node" {
+    }
-    //   unreachable!("Node built-in modules should be handled internally.");
+
-    // }
     let graph = self.ps.graph();
     match graph.get(specifier) {
       Some(deno_graph::Module {

cli/node/mod.rs

@@ -27,7 +27,6 @@ use deno_runtime::deno_node::package_imports_resolve;
 use deno_runtime::deno_node::package_resolve;
 use deno_runtime::deno_node::path_to_declaration_path;
 use deno_runtime::deno_node::NodeModuleKind;
-use deno_runtime::deno_node::NodeModulePolyfillSpecifier;
 use deno_runtime::deno_node::NodePermissions;
 use deno_runtime::deno_node::NodeResolutionMode;
 use deno_runtime::deno_node::PackageJson;
@@ -39,7 +38,6 @@ use once_cell::sync::Lazy;
 use regex::Regex;
 
 use crate::cache::NodeAnalysisCache;
-use crate::deno_std::CURRENT_STD_URL;
 use crate::file_fetcher::FileFetcher;
 use crate::npm::NpmPackageResolver;
 
@@ -106,33 +104,10 @@ impl NodeResolution {
   }
 }
 
-static NODE_COMPAT_URL: Lazy<Url> = Lazy::new(|| {
+// TODO(bartlomieju): seems super wasteful to parse specified each time
-  if let Ok(url_str) = std::env::var("DENO_NODE_COMPAT_URL") {
-    let url = Url::parse(&url_str).expect(
-      "Malformed DENO_NODE_COMPAT_URL value, make sure it's a file URL ending with a slash"
-    );
-    return url;
-  }
-
-  CURRENT_STD_URL.clone()
-});
-
-pub static MODULE_ALL_URL: Lazy<Url> =
-  Lazy::new(|| NODE_COMPAT_URL.join("node/module_all.ts").unwrap());
-
 pub fn resolve_builtin_node_module(specifier: &str) -> Result<Url, AnyError> {
   if let Some(module) = find_builtin_node_module(specifier) {
-    match module.specifier {
+    return Ok(ModuleSpecifier::parse(module.specifier).unwrap());
-      // We will load the source code from the `std/node` polyfill.
-      NodeModulePolyfillSpecifier::StdNode(specifier) => {
-        let module_url = NODE_COMPAT_URL.join(specifier).unwrap();
-        return Ok(module_url);
-      }
-      // The module has already been snapshotted and is present in the binary.
-      NodeModulePolyfillSpecifier::Embedded(specifier) => {
-        return Ok(ModuleSpecifier::parse(specifier).unwrap());
-      }
-    }
   }
 
   Err(generic_error(format!(

cli/proc_state.rs

@@ -63,8 +63,6 @@ use std::collections::HashMap;
 use std::collections::HashSet;
 use std::ops::Deref;
 use std::path::PathBuf;
-use std::sync::atomic::AtomicBool;
-use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
 /// This structure represents state of single "deno" program.
@@ -98,7 +96,6 @@ pub struct Inner {
   pub npm_resolver: NpmPackageResolver,
   pub cjs_resolutions: Mutex<HashSet<ModuleSpecifier>>,
   progress_bar: ProgressBar,
-  node_std_graph_prepared: AtomicBool,
 }
 
 impl Deref for ProcState {
@@ -160,7 +157,6 @@ impl ProcState {
       npm_resolver: self.npm_resolver.clone(),
       cjs_resolutions: Default::default(),
       progress_bar: self.progress_bar.clone(),
-      node_std_graph_prepared: AtomicBool::new(false),
     });
     self.init_watcher();
   }
@@ -293,7 +289,6 @@ impl ProcState {
       npm_resolver,
       cjs_resolutions: Default::default(),
       progress_bar,
-      node_std_graph_prepared: AtomicBool::new(false),
     })))
   }
 
@@ -369,7 +364,6 @@ impl ProcState {
 
     if !npm_package_reqs.is_empty() {
       self.npm_resolver.add_package_reqs(npm_package_reqs).await?;
-      self.prepare_node_std_graph().await?;
     }
 
     if has_node_builtin_specifier
@@ -384,9 +378,7 @@ impl ProcState {
     drop(_pb_clear_guard);
 
     // type check if necessary
-    let is_std_node = roots.len() == 1 && roots[0] == *node::MODULE_ALL_URL;
     if self.options.type_check_mode() != TypeCheckMode::None
-      && !is_std_node
       && !self.graph_data.read().is_type_checked(&roots, lib)
     {
       log::debug!("Type checking.");
@@ -456,31 +448,6 @@ impl ProcState {
       .await
   }
 
-  /// Add the builtin node modules to the graph data.
-  pub async fn prepare_node_std_graph(&self) -> Result<(), AnyError> {
-    if self.node_std_graph_prepared.load(Ordering::Relaxed) {
-      return Ok(());
-    }
-
-    let mut graph = self.graph_data.read().graph_inner_clone();
-    let mut loader = self.create_graph_loader();
-    let analyzer = self.parsed_source_cache.as_analyzer();
-    graph
-      .build(
-        vec![node::MODULE_ALL_URL.clone()],
-        &mut loader,
-        deno_graph::BuildOptions {
-          module_analyzer: Some(&*analyzer),
-          ..Default::default()
-        },
-      )
-      .await;
-
-    self.graph_data.write().update_graph(Arc::new(graph));
-    self.node_std_graph_prepared.store(true, Ordering::Relaxed);
-    Ok(())
-  }
-
   fn handle_node_resolve_result(
     &self,
     result: Result<Option<node::NodeResolution>, AnyError>,
@@ -712,6 +679,10 @@ impl ProcState {
   pub fn graph(&self) -> Arc<ModuleGraph> {
     self.graph_data.read().graph.clone()
   }
+
+  pub fn has_node_builtin_specifier(&self) -> bool {
+    self.graph_data.read().has_node_builtin_specifier
+  }
 }
 
 #[derive(Clone, Debug)]

cli/tests/integration/run_tests.rs

@@ -3830,14 +3830,14 @@ fn stdio_streams_are_locked_in_permission_prompt() {
 }
 
 itest!(node_builtin_modules_ts {
-  args: "run --quiet run/node_builtin_modules/mod.ts",
+  args: "run --quiet --allow-read run/node_builtin_modules/mod.ts hello there",
   output: "run/node_builtin_modules/mod.ts.out",
   envs: env_vars_for_npm_tests_no_sync_download(),
   exit_code: 0,
 });
 
 itest!(node_builtin_modules_js {
-  args: "run --quiet run/node_builtin_modules/mod.js",
+  args: "run --quiet --allow-read run/node_builtin_modules/mod.js hello there",
   output: "run/node_builtin_modules/mod.js.out",
   envs: env_vars_for_npm_tests_no_sync_download(),
   exit_code: 0,

cli/tests/testdata/run/node_builtin_modules/mod.js

@@ -2,3 +2,4 @@ import { createRequire } from "node:module";
 console.log(createRequire);
 import process from "node:process";
 console.log(process.version);
+console.log(process.argv);

cli/tests/testdata/run/node_builtin_modules/mod.js.out

@@ -1,2 +1,8 @@
 [Function: createRequire]
 v[WILDCARD].[WILDCARD].[WILDCARD]
+[
+  "[WILDCARD]",
+  "[WILDCARD]mod.js",
+  "hello",
+  "there"
+]

cli/tests/testdata/run/node_builtin_modules/mod.ts

@@ -2,3 +2,4 @@ import { createRequire } from "node:module";
 console.log(createRequire);
 import process from "node:process";
 console.log(process.version);
+console.log(process.argv);

cli/tests/testdata/run/node_builtin_modules/mod.ts.out

@@ -1,2 +1,8 @@
 [Function: createRequire]
 v[WILDCARD].[WILDCARD].[WILDCARD]
+[
+  "[WILDCARD]",
+  "[WILDCARD]mod.ts",
+  "hello",
+  "there"
+]

cli/tools/repl/session.rs

@@ -461,10 +461,8 @@ impl ReplSession {
       resolved_imports.iter().any(|url| url.scheme() == "node");
     if !npm_imports.is_empty() || has_node_specifier {
       if !self.has_initialized_node_runtime {
-        self.proc_state.prepare_node_std_graph().await?;
         deno_node::initialize_runtime(
           &mut self.worker.js_runtime,
-          crate::node::MODULE_ALL_URL.as_str(),
           self.proc_state.options.node_modules_dir(),
         )
         .await?;

cli/worker.rs

@@ -67,7 +67,6 @@ impl CliMainWorker {
     log::debug!("main_module {}", self.main_module);
 
     if self.is_main_cjs {
-      self.ps.prepare_node_std_graph().await?;
       self.initialize_main_module_for_node().await?;
       deno_node::load_cjs_module(
         &mut self.worker.js_runtime,
@@ -279,9 +278,6 @@ impl CliMainWorker {
   async fn execute_main_module_possibly_with_npm(
     &mut self,
   ) -> Result<(), AnyError> {
-    if self.ps.npm_resolver.has_packages() {
-      self.ps.prepare_node_std_graph().await?;
-    }
     let id = self.worker.preload_main_module(&self.main_module).await?;
     self.evaluate_module_possibly_with_npm(id).await
   }
@@ -297,17 +293,17 @@ impl CliMainWorker {
     &mut self,
     id: ModuleId,
   ) -> Result<(), AnyError> {
-    if self.ps.npm_resolver.has_packages() {
+    if self.ps.npm_resolver.has_packages()
+      || self.ps.has_node_builtin_specifier()
+    {
       self.initialize_main_module_for_node().await?;
     }
     self.worker.evaluate_module(id).await
   }
 
   async fn initialize_main_module_for_node(&mut self) -> Result<(), AnyError> {
-    self.ps.prepare_node_std_graph().await?;
     deno_node::initialize_runtime(
       &mut self.worker.js_runtime,
-      node::MODULE_ALL_URL.as_str(),
       self.ps.options.node_modules_dir(),
     )
     .await?;
@@ -630,7 +626,6 @@ fn create_web_worker_pre_execute_module_callback(
       if ps.npm_resolver.has_packages() {
         deno_node::initialize_runtime(
           &mut worker.js_runtime,
-          node::MODULE_ALL_URL.as_str(),
           ps.options.node_modules_dir(),
         )
         .await?;

core/runtime.rs

@@ -841,7 +841,14 @@ impl JsRuntime {
           )
           .await?;
         let receiver = runtime.mod_evaluate(id);
-        runtime.run_event_loop(false).await?;
+        poll_fn(|cx| {
+          let r = runtime.poll_event_loop(cx, false);
+          // TODO(bartlomieju): some code in readable-stream polyfill in `ext/node`
+          // is calling `nextTick()` during snapshotting, which causes infinite loop
+          runtime.state.borrow_mut().has_tick_scheduled = false;
+          r
+        })
+        .await?;
         receiver.await?
       })
       .with_context(|| format!("Couldn't execute '{}'", file_source.specifier))
@@ -2532,7 +2539,6 @@ impl JsRuntime {
       let tc_scope = &mut v8::TryCatch::new(scope);
       let this = v8::undefined(tc_scope).into();
       js_nexttick_cb.call(tc_scope, this, &[]);
-
       if let Some(exception) = tc_scope.exception() {
         return exception_to_err_result(tc_scope, exception, false);
       }

ext/node/01_node.js

@@ -109,6 +109,8 @@ function initialize(nodeModules, nodeGlobalThisName) {
     writable: false,
     value: nodeGlobalThis,
   });
+  // FIXME(bartlomieju): not nice to depend on `Deno` namespace here
+  internals.__bootstrapNodeProcess(Deno.args);
 }
 
 internals.node = {

ext/node/lib.rs

@@ -26,7 +26,6 @@ pub use path::PathClean;
 pub use polyfill::find_builtin_node_module;
 pub use polyfill::is_builtin_node_module;
 pub use polyfill::NodeModulePolyfill;
-pub use polyfill::NodeModulePolyfillSpecifier;
 pub use polyfill::SUPPORTED_BUILTIN_NODE_MODULES;
 pub use resolution::get_closest_package_json;
 pub use resolution::get_package_scope_config;
@@ -462,18 +461,15 @@ pub fn init<P: NodePermissions + 'static>(
 
 pub async fn initialize_runtime(
   js_runtime: &mut JsRuntime,
-  module_all_url: &str,
   uses_local_node_modules_dir: bool,
 ) -> Result<(), AnyError> {
   let source_code = &format!(
-    r#"(async function loadBuiltinNodeModules(moduleAllUrl, nodeGlobalThisName, usesLocalNodeModulesDir) {{
+    r#"(async function loadBuiltinNodeModules(nodeGlobalThisName, usesLocalNodeModulesDir) {{
-      const moduleAll = await import(moduleAllUrl);
+      Deno[Deno.internal].node.initialize(Deno[Deno.internal].nodeModuleAll, nodeGlobalThisName);
-      Deno[Deno.internal].node.initialize(moduleAll.default, nodeGlobalThisName);
       if (usesLocalNodeModulesDir) {{
         Deno[Deno.internal].require.setUsesLocalNodeModulesDir();
       }}
-    }})('{}', '{}', {});"#,
+    }})('{}', {});"#,
-    module_all_url,
     NODE_GLOBAL_THIS_NAME.as_str(),
     uses_local_node_modules_dir,
   );

ext/node/polyfill.rs

@@ -12,204 +12,191 @@ pub fn is_builtin_node_module(specifier: &str) -> bool {
   find_builtin_node_module(specifier).is_some()
 }
 
-pub enum NodeModulePolyfillSpecifier {
-  /// An internal module specifier, like "internal:deno_node/assert.ts". The
-  /// module must be either embedded in the binary or snapshotted.
-  Embedded(&'static str),
-
-  /// Specifier relative to the root of `deno_std` repo, like "node/assert.ts"
-  StdNode(&'static str),
-}
-
 pub struct NodeModulePolyfill {
   /// Name of the module like "assert" or "timers/promises"
   pub name: &'static str,
-  pub specifier: NodeModulePolyfillSpecifier,
+  pub specifier: &'static str,
 }
 
 pub static SUPPORTED_BUILTIN_NODE_MODULES: &[NodeModulePolyfill] = &[
   NodeModulePolyfill {
     name: "assert",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/assert.ts"),
+    specifier: "internal:deno_node/polyfills/assert.ts",
   },
   NodeModulePolyfill {
     name: "assert/strict",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/assert/strict.ts"),
+    specifier: "internal:deno_node/polyfills/assert/strict.ts",
   },
   NodeModulePolyfill {
     name: "async_hooks",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/async_hooks.ts"),
+    specifier: "internal:deno_node/polyfills/async_hooks.ts",
   },
   NodeModulePolyfill {
     name: "buffer",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/buffer.ts"),
+    specifier: "internal:deno_node/polyfills/buffer.ts",
   },
   NodeModulePolyfill {
     name: "child_process",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/child_process.ts"),
+    specifier: "internal:deno_node/polyfills/child_process.ts",
   },
   NodeModulePolyfill {
     name: "cluster",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/cluster.ts"),
+    specifier: "internal:deno_node/polyfills/cluster.ts",
   },
   NodeModulePolyfill {
     name: "console",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/console.ts"),
+    specifier: "internal:deno_node/polyfills/console.ts",
   },
   NodeModulePolyfill {
     name: "constants",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/constants.ts"),
+    specifier: "internal:deno_node/polyfills/constants.ts",
   },
   NodeModulePolyfill {
     name: "crypto",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/crypto.ts"),
+    specifier: "internal:deno_node/polyfills/crypto.ts",
   },
   NodeModulePolyfill {
     name: "dgram",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/dgram.ts"),
+    specifier: "internal:deno_node/polyfills/dgram.ts",
   },
   NodeModulePolyfill {
     name: "dns",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/dns.ts"),
+    specifier: "internal:deno_node/polyfills/dns.ts",
   },
   NodeModulePolyfill {
     name: "dns/promises",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/dns/promises.ts"),
+    specifier: "internal:deno_node/polyfills/dns/promises.ts",
   },
   NodeModulePolyfill {
     name: "domain",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/domain.ts"),
+    specifier: "internal:deno_node/polyfills/domain.ts",
   },
   NodeModulePolyfill {
     name: "events",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/events.ts"),
+    specifier: "internal:deno_node/polyfills/events.ts",
   },
   NodeModulePolyfill {
     name: "fs",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/fs.ts"),
+    specifier: "internal:deno_node/polyfills/fs.ts",
   },
   NodeModulePolyfill {
     name: "fs/promises",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/fs/promises.ts"),
+    specifier: "internal:deno_node/polyfills/fs/promises.ts",
   },
   NodeModulePolyfill {
     name: "http",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/http.ts"),
+    specifier: "internal:deno_node/polyfills/http.ts",
   },
   NodeModulePolyfill {
     name: "https",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/https.ts"),
+    specifier: "internal:deno_node/polyfills/https.ts",
   },
   NodeModulePolyfill {
     name: "module",
-    specifier: NodeModulePolyfillSpecifier::Embedded(
+    specifier: "internal:deno_node_loading/module_es_shim.js",
-      "internal:deno_node_loading/module_es_shim.js",
-    ),
   },
   NodeModulePolyfill {
     name: "net",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/net.ts"),
+    specifier: "internal:deno_node/polyfills/net.ts",
   },
   NodeModulePolyfill {
     name: "os",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/os.ts"),
+    specifier: "internal:deno_node/polyfills/os.ts",
   },
   NodeModulePolyfill {
     name: "path",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/path.ts"),
+    specifier: "internal:deno_node/polyfills/path.ts",
   },
   NodeModulePolyfill {
     name: "path/posix",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/path/posix.ts"),
+    specifier: "internal:deno_node/polyfills/path/posix.ts",
   },
   NodeModulePolyfill {
     name: "path/win32",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/path/win32.ts"),
+    specifier: "internal:deno_node/polyfills/path/win32.ts",
   },
   NodeModulePolyfill {
     name: "perf_hooks",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/perf_hooks.ts"),
+    specifier: "internal:deno_node/polyfills/perf_hooks.ts",
   },
   NodeModulePolyfill {
     name: "process",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/process.ts"),
+    specifier: "internal:deno_node/polyfills/process.ts",
   },
   NodeModulePolyfill {
     name: "querystring",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/querystring.ts"),
+    specifier: "internal:deno_node/polyfills/querystring.ts",
   },
   NodeModulePolyfill {
     name: "readline",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/readline.ts"),
+    specifier: "internal:deno_node/polyfills/readline.ts",
   },
   NodeModulePolyfill {
     name: "stream",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/stream.ts"),
+    specifier: "internal:deno_node/polyfills/stream.ts",
   },
   NodeModulePolyfill {
     name: "stream/consumers",
-    specifier: NodeModulePolyfillSpecifier::StdNode(
+    specifier: "internal:deno_node/polyfills/stream/consumers.mjs",
-      "node/stream/consumers.mjs",
-    ),
   },
   NodeModulePolyfill {
     name: "stream/promises",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/stream/promises.mjs"),
+    specifier: "internal:deno_node/polyfills/stream/promises.mjs",
   },
   NodeModulePolyfill {
     name: "stream/web",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/stream/web.ts"),
+    specifier: "internal:deno_node/polyfills/stream/web.ts",
   },
   NodeModulePolyfill {
     name: "string_decoder",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/string_decoder.ts"),
+    specifier: "internal:deno_node/polyfills/string_decoder.ts",
   },
   NodeModulePolyfill {
     name: "sys",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/sys.ts"),
+    specifier: "internal:deno_node/polyfills/sys.ts",
   },
   NodeModulePolyfill {
     name: "timers",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/timers.ts"),
+    specifier: "internal:deno_node/polyfills/timers.ts",
   },
   NodeModulePolyfill {
     name: "timers/promises",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/timers/promises.ts"),
+    specifier: "internal:deno_node/polyfills/timers/promises.ts",
   },
   NodeModulePolyfill {
     name: "tls",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/tls.ts"),
+    specifier: "internal:deno_node/polyfills/tls.ts",
   },
   NodeModulePolyfill {
     name: "tty",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/tty.ts"),
+    specifier: "internal:deno_node/polyfills/tty.ts",
   },
   NodeModulePolyfill {
     name: "url",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/url.ts"),
+    specifier: "internal:deno_node/polyfills/url.ts",
   },
   NodeModulePolyfill {
     name: "util",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/util.ts"),
+    specifier: "internal:deno_node/polyfills/util.ts",
   },
   NodeModulePolyfill {
     name: "util/types",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/util/types.ts"),
+    specifier: "internal:deno_node/polyfills/util/types.ts",
   },
   NodeModulePolyfill {
     name: "v8",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/v8.ts"),
+    specifier: "internal:deno_node/polyfills/v8.ts",
   },
   NodeModulePolyfill {
     name: "vm",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/vm.ts"),
+    specifier: "internal:deno_node/polyfills/vm.ts",
   },
   NodeModulePolyfill {
     name: "worker_threads",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/worker_threads.ts"),
+    specifier: "internal:deno_node/polyfills/worker_threads.ts",
   },
   NodeModulePolyfill {
     name: "zlib",
-    specifier: NodeModulePolyfillSpecifier::StdNode("node/zlib.ts"),
+    specifier: "internal:deno_node/polyfills/zlib.ts",
   },
 ];

ext/node/polyfills/_next_tick.ts

@@ -1,8 +1,6 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 // Copyright Joyent, Inc. and other Node contributors.
 
-// deno-lint-ignore-file no-inner-declarations
-
 import { core } from "internal:deno_node/polyfills/_core.ts";
 import { validateFunction } from "internal:deno_node/polyfills/internal/validators.mjs";
 import { _exiting } from "internal:deno_node/polyfills/_process/exiting.ts";
@@ -15,9 +13,6 @@ interface Tock {
 
 const queue = new FixedQueue();
 
-// deno-lint-ignore no-explicit-any
-let _nextTick: any;
-
 export function processTicksAndRejections() {
   let tock;
   do {
@@ -68,92 +63,21 @@ export function processTicksAndRejections() {
   // setHasRejectionToWarn(false);
 }
 
-if (typeof core.setNextTickCallback !== "undefined") {
+export function runNextTicks() {
-  function runNextTicks() {
+  // FIXME(bartlomieju): Deno currently doesn't unhandled rejections
-    // FIXME(bartlomieju): Deno currently doesn't unhandled rejections
+  // if (!hasTickScheduled() && !hasRejectionToWarn())
-    // if (!hasTickScheduled() && !hasRejectionToWarn())
+  //   runMicrotasks();
-    //   runMicrotasks();
+  // if (!hasTickScheduled() && !hasRejectionToWarn())
-    // if (!hasTickScheduled() && !hasRejectionToWarn())
+  //   return;
-    //   return;
+  if (!core.hasTickScheduled()) {
-    if (!core.hasTickScheduled()) {
+    core.runMicrotasks();
-      core.runMicrotasks();
+  }
-    }
+  if (!core.hasTickScheduled()) {
-    if (!core.hasTickScheduled()) {
-      return true;
-    }
-
-    processTicksAndRejections();
     return true;
   }
 
-  core.setNextTickCallback(processTicksAndRejections);
+  processTicksAndRejections();
-  core.setMacrotaskCallback(runNextTicks);
+  return true;
-
-  function __nextTickNative<T extends Array<unknown>>(
-    this: unknown,
-    callback: (...args: T) => void,
-    ...args: T
-  ) {
-    validateFunction(callback, "callback");
-
-    if (_exiting) {
-      return;
-    }
-
-    // TODO(bartlomieju): seems superfluous if we don't depend on `arguments`
-    let args_;
-    switch (args.length) {
-      case 0:
-        break;
-      case 1:
-        args_ = [args[0]];
-        break;
-      case 2:
-        args_ = [args[0], args[1]];
-        break;
-      case 3:
-        args_ = [args[0], args[1], args[2]];
-        break;
-      default:
-        args_ = new Array(args.length);
-        for (let i = 0; i < args.length; i++) {
-          args_[i] = args[i];
-        }
-    }
-
-    if (queue.isEmpty()) {
-      core.setHasTickScheduled(true);
-    }
-    // FIXME(bartlomieju): Deno currently doesn't support async hooks
-    // const asyncId = newAsyncId();
-    // const triggerAsyncId = getDefaultTriggerAsyncId();
-    const tickObject = {
-      // FIXME(bartlomieju): Deno currently doesn't support async hooks
-      // [async_id_symbol]: asyncId,
-      // [trigger_async_id_symbol]: triggerAsyncId,
-      callback,
-      args: args_,
-    };
-    // FIXME(bartlomieju): Deno currently doesn't support async hooks
-    // if (initHooksExist())
-    //   emitInit(asyncId, 'TickObject', triggerAsyncId, tickObject);
-    queue.push(tickObject);
-  }
-  _nextTick = __nextTickNative;
-} else {
-  function __nextTickQueueMicrotask<T extends Array<unknown>>(
-    this: unknown,
-    callback: (...args: T) => void,
-    ...args: T
-  ) {
-    if (args) {
-      queueMicrotask(() => callback.call(this, ...args));
-    } else {
-      queueMicrotask(callback);
-    }
-  }
-
-  _nextTick = __nextTickQueueMicrotask;
 }
 
 // `nextTick()` will not enqueue any callback when the process is about to
@@ -169,5 +93,48 @@ export function nextTick<T extends Array<unknown>>(
   callback: (...args: T) => void,
   ...args: T
 ) {
-  _nextTick(callback, ...args);
+  validateFunction(callback, "callback");
+
+  if (_exiting) {
+    return;
+  }
+
+  // TODO(bartlomieju): seems superfluous if we don't depend on `arguments`
+  let args_;
+  switch (args.length) {
+    case 0:
+      break;
+    case 1:
+      args_ = [args[0]];
+      break;
+    case 2:
+      args_ = [args[0], args[1]];
+      break;
+    case 3:
+      args_ = [args[0], args[1], args[2]];
+      break;
+    default:
+      args_ = new Array(args.length);
+      for (let i = 0; i < args.length; i++) {
+        args_[i] = args[i];
+      }
+  }
+
+  if (queue.isEmpty()) {
+    core.setHasTickScheduled(true);
+  }
+  // FIXME(bartlomieju): Deno currently doesn't support async hooks
+  // const asyncId = newAsyncId();
+  // const triggerAsyncId = getDefaultTriggerAsyncId();
+  const tickObject = {
+    // FIXME(bartlomieju): Deno currently doesn't support async hooks
+    // [async_id_symbol]: asyncId,
+    // [trigger_async_id_symbol]: triggerAsyncId,
+    callback,
+    args: args_,
+  };
+  // FIXME(bartlomieju): Deno currently doesn't support async hooks
+  // if (initHooksExist())
+  //   emitInit(asyncId, 'TickObject', triggerAsyncId, tickObject);
+  queue.push(tickObject);
 }

ext/node/polyfills/_process/process.ts

@@ -7,6 +7,7 @@
 import { build } from "internal:runtime/js/01_build.js";
 import { nextTick as _nextTick } from "internal:deno_node/polyfills/_next_tick.ts";
 import { _exiting } from "internal:deno_node/polyfills/_process/exiting.ts";
+import * as fs from "internal:runtime/js/30_fs.js";
 
 /** Returns the operating system CPU architecture for which the Deno binary was compiled */
 export function arch(): string {
@@ -20,10 +21,10 @@ export function arch(): string {
 }
 
 /** https://nodejs.org/api/process.html#process_process_chdir_directory */
-export const chdir = Deno.chdir;
+export const chdir = fs.chdir;
 
 /** https://nodejs.org/api/process.html#process_process_cwd */
-export const cwd = Deno.cwd;
+export const cwd = fs.cwd;
 
 /** https://nodejs.org/api/process.html#process_process_nexttick_callback_args */
 export const nextTick = _nextTick;

ext/node/polyfills/_process/streams.mjs

@@ -10,7 +10,9 @@ import {
 } from "internal:deno_node/polyfills/internal/readline/callbacks.mjs";
 import { Duplex, Readable, Writable } from "internal:deno_node/polyfills/stream.ts";
 import { stdio } from "internal:deno_node/polyfills/_process/stdio.mjs";
+import { isWindows } from "internal:deno_node/polyfills/_util/os.ts";
 import { fs as fsConstants } from "internal:deno_node/polyfills/internal_binding/constants.ts";
+import * as files from "internal:runtime/js/40_files.js";
 
 // https://github.com/nodejs/node/blob/00738314828074243c9a52a228ab4c68b04259ef/lib/internal/bootstrap/switches/is_main_thread.js#L41
 function createWritableStdioStream(writer, name) {
@@ -92,13 +94,13 @@ function createWritableStdioStream(writer, name) {
 
 /** https://nodejs.org/api/process.html#process_process_stderr */
 export const stderr = stdio.stderr = createWritableStdioStream(
-  Deno.stderr,
+  files.stderr,
   "stderr",
 );
 
 /** https://nodejs.org/api/process.html#process_process_stdout */
 export const stdout = stdio.stdout = createWritableStdioStream(
-  Deno.stdout,
+  files.stdout,
   "stdout",
 );
 
@@ -113,7 +115,7 @@ function _guessStdinType(fd) {
     const fileInfo = Deno.fstatSync?.(fd);
 
     // https://github.com/nodejs/node/blob/v18.12.1/deps/uv/src/unix/tty.c#L333
-    if (Deno.build.os !== "windows") {
+    if (!isWindows) {
       switch (fileInfo.mode & fsConstants.S_IFMT) {
         case fsConstants.S_IFREG:
         case fsConstants.S_IFCHR:
@@ -143,7 +145,7 @@ function _guessStdinType(fd) {
     // TODO(PolarETech): Need a better way to identify a character file on Windows.
     // "EISDIR" error occurs when stdin is "null" on Windows,
     // so use the error as a workaround.
-    if (Deno.build.os === "windows" && e.code === "EISDIR") return "FILE";
+    if (isWindows && e.code === "EISDIR") return "FILE";
   }
 
   return "UNKNOWN";
@@ -151,7 +153,7 @@ function _guessStdinType(fd) {
 
 const _read = function (size) {
   const p = Buffer.alloc(size || 16 * 1024);
-  Deno.stdin?.read(p).then((length) => {
+  files.stdin?.read(p).then((length) => {
     this.push(length === null ? null : p.slice(0, length));
   }, (error) => {
     this.destroy(error);
@@ -161,7 +163,7 @@ const _read = function (size) {
 /** https://nodejs.org/api/process.html#process_process_stdin */
 // https://github.com/nodejs/node/blob/v18.12.1/lib/internal/bootstrap/switches/is_main_thread.js#L189
 export const stdin = stdio.stdin = (() => {
-  const fd = Deno.stdin?.rid;
+  const fd = files.stdin?.rid;
   let _stdin;
   const stdinType = _guessStdinType(fd);
 
@@ -222,8 +224,8 @@ export const stdin = stdio.stdin = (() => {
 
   return _stdin;
 })();
-stdin.on("close", () => Deno.stdin?.close());
+stdin.on("close", () => files.stdin?.close());
-stdin.fd = Deno.stdin?.rid ?? -1;
+stdin.fd = files.stdin?.rid ?? -1;
 Object.defineProperty(stdin, "isTTY", {
   enumerable: true,
   configurable: true,
@@ -233,7 +235,7 @@ Object.defineProperty(stdin, "isTTY", {
 });
 stdin._isRawMode = false;
 stdin.setRawMode = (enable) => {
-  Deno.stdin?.setRaw?.(enable);
+  files.stdin?.setRaw?.(enable);
   stdin._isRawMode = enable;
   return stdin;
 };

ext/node/polyfills/_util/os.ts

@@ -1,22 +1,10 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
+const { ops } = globalThis.__bootstrap.core;
+
 export type OSType = "windows" | "linux" | "darwin" | "freebsd";
 
-export const osType: OSType = (() => {
+export const osType: OSType = ops.op_node_build_os();
-  // deno-lint-ignore no-explicit-any
-  const { Deno } = globalThis as any;
-  if (typeof Deno?.build?.os === "string") {
-    return Deno.build.os;
-  }
-
-  // deno-lint-ignore no-explicit-any
-  const { navigator } = globalThis as any;
-  if (navigator?.appVersion?.includes?.("Win")) {
-    return "windows";
-  }
-
-  return "linux";
-})();
 
 export const isWindows = osType === "windows";
 export const isLinux = osType === "linux";

ext/node/polyfills/console.ts

@@ -1,6 +1,10 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
 import { Console } from "internal:deno_node/polyfills/internal/console/constructor.mjs";
+import { windowOrWorkerGlobalScope } from "internal:runtime/js/98_global_scope.js";
+// Don't rely on global `console` because during bootstrapping, it is pointing
+// to native `console` object provided by V8.
+const console = windowOrWorkerGlobalScope.console.value;
 
 export default Object.assign({}, console, { Console });
 

ext/node/polyfills/internal/child_process.ts

@@ -67,10 +67,6 @@ export function mapValues<T, O>(
 type NodeStdio = "pipe" | "overlapped" | "ignore" | "inherit" | "ipc";
 type DenoStdio = "inherit" | "piped" | "null";
 
-// @ts-ignore Deno[Deno.internal] is used on purpose here
-const DenoCommand = Deno[Deno.internal]?.nodeUnstable?.Command ||
-  Deno.Command;
-
 export function stdioStringToArray(
   stdio: NodeStdio,
   channel: NodeStdio | number,
@@ -183,9 +179,8 @@ export class ChildProcess extends EventEmitter {
     this.spawnargs = [cmd, ...cmdArgs];
 
     const stringEnv = mapValues(env, (value) => value.toString());
-
     try {
-      this.#process = new DenoCommand(cmd, {
+      this.#process = new Deno.Command(cmd, {
         args: cmdArgs,
         cwd,
         env: stringEnv,
@@ -804,7 +799,7 @@ export function spawnSync(
 
   const result: SpawnSyncResult = {};
   try {
-    const output = new DenoCommand(command, {
+    const output = new Deno.Command(command, {
       args,
       cwd,
       env,

ext/node/polyfills/internal/crypto/hash.ts

@@ -107,7 +107,7 @@ export class Hash extends Transform {
    * Supported encodings are currently 'hex', 'binary', 'base64', 'base64url'.
    */
   digest(encoding?: string): Buffer | string {
-    const digest = this.#context.digest(undefined);
+    const digest = ops.op_node_hash_digest(this.#context);
     if (encoding === undefined) {
       return Buffer.from(digest);
     }

ext/node/polyfills/internal/timers.mjs

@@ -5,10 +5,11 @@ import { inspect } from "internal:deno_node/polyfills/internal/util/inspect.mjs"
 import { validateFunction, validateNumber } from "internal:deno_node/polyfills/internal/validators.mjs";
 import { ERR_OUT_OF_RANGE } from "internal:deno_node/polyfills/internal/errors.ts";
 import { emitWarning } from "internal:deno_node/polyfills/process.ts";
-
+import {
-const setTimeout_ = globalThis.setTimeout;
+  setTimeout as setTimeout_,
-const clearTimeout_ = globalThis.clearTimeout;
+  clearTimeout as clearTimeout_,
-const setInterval_ = globalThis.setInterval;
+  setInterval as setInterval_,
+} from "internal:deno_web/02_timers.js";
 
 // Timeout values > TIMEOUT_MAX are set to 1.
 export const TIMEOUT_MAX = 2 ** 31 - 1;

ext/node/polyfills/module_all.ts

@@ -1,4 +1,5 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
+const internals = globalThis.__bootstrap.internals;
 import _httpAgent from "internal:deno_node/polyfills/_http_agent.mjs";
 import _httpOutgoing from "internal:deno_node/polyfills/_http_outgoing.ts";
 import _streamDuplex from "internal:deno_node/polyfills/internal/streams/duplex.mjs";
@@ -88,7 +89,7 @@ import wasi from "internal:deno_node/polyfills/wasi.ts";
 import zlib from "internal:deno_node/polyfills/zlib.ts";
 
 // Canonical mapping of supported modules
-export default {
+const moduleAll = {
   "_http_agent": _httpAgent,
   "_http_outgoing": _httpOutgoing,
   "_stream_duplex": _streamDuplex,
@@ -185,3 +186,6 @@ export default {
   worker_threads: workerThreads,
   zlib,
 } as Record<string, unknown>;
+
+internals.nodeModuleAll = moduleAll;
+export default moduleAll;

ext/node/polyfills/process.ts

@@ -2,6 +2,7 @@
 // Copyright Joyent, Inc. and Node.js contributors. All rights reserved. MIT license.
 
 const internals = globalThis.__bootstrap.internals;
+import { core } from "internal:deno_node/polyfills/_core.ts";
 import {
   notImplemented,
   warnNotImplemented,
@@ -32,8 +33,11 @@ import {
   stdin as stdin_,
   stdout as stdout_,
 } from "internal:deno_node/polyfills/_process/streams.mjs";
-import { core } from "internal:deno_node/polyfills/_core.ts";
+import {
-import { processTicksAndRejections } from "internal:deno_node/polyfills/_next_tick.ts";
+  processTicksAndRejections,
+  runNextTicks,
+} from "internal:deno_node/polyfills/_next_tick.ts";
+import { isWindows } from "internal:deno_node/polyfills/_util/os.ts";
 
 // TODO(kt3k): This should be set at start up time
 export let arch = "";
@@ -71,10 +75,19 @@ const notImplementedEvents = [
   "worker",
 ];
 
-export const argv = [];
+export const argv: string[] = [];
 
 // Overwrites the 1st item with getter.
-Object.defineProperty(argv, "0", { get: Deno.execPath });
+// TODO(bartlomieju): added "configurable: true" to make this work for binary
+// commands, but that is probably a wrong solution
+// TODO(bartlomieju): move the configuration for all "argv" to
+// "internals.__bootstrapNodeProcess"
+Object.defineProperty(argv, "0", {
+  get: () => {
+    return Deno.execPath();
+  },
+  configurable: true,
+});
 // Overwrites the 2st item with getter.
 Object.defineProperty(argv, "1", {
   get: () => {
@@ -86,13 +99,6 @@ Object.defineProperty(argv, "1", {
   },
 });
 
-// TODO(kt3k): Set the rest of args at start up time instead of defining
-// random number of getters.
-for (let i = 0; i < 30; i++) {
-  const j = i;
-  Object.defineProperty(argv, j + 2, { get: () => Deno.args[j] });
-}
-
 /** https://nodejs.org/api/process.html#process_process_exit_code */
 export const exit = (code?: number | string) => {
   if (code || code === 0) {
@@ -681,9 +687,18 @@ addReadOnlyProcessAlias("throwDeprecation", "--throw-deprecation");
 export const removeListener = process.removeListener;
 export const removeAllListeners = process.removeAllListeners;
 
-// FIXME(bartlomieju): currently it's not called
+// Should be called only once, in `runtime/js/99_main.js` when the runtime is
-// only call this from runtime's main.js
+// bootstrapped.
-internals.__bootstrapNodeProcess = function () {
+internals.__bootstrapNodeProcess = function (args: string[]) {
+  for (let i = 0; i < args.length; i++) {
+    argv[i + 2] = args[i];
+  }
+
+  core.setNextTickCallback(processTicksAndRejections);
+  core.setMacrotaskCallback(runNextTicks);
+
+  // TODO(bartlomieju): this is buggy, see https://github.com/denoland/deno/issues/16928
+  // We should use a specialized API in 99_main.js instead
   globalThis.addEventListener("unhandledrejection", (event) => {
     if (process.listenerCount("unhandledRejection") === 0) {
       // The Node.js default behavior is to raise an uncaught exception if
@@ -724,6 +739,8 @@ internals.__bootstrapNodeProcess = function () {
       process.emit("exit", process.exitCode || 0);
     }
   });
+
+  delete internals.__bootstrapNodeProcess;
 };
 
 export default process;

test_util/src/lib.rs

@@ -96,7 +96,6 @@ lazy_static! {
 
 pub fn env_vars_for_npm_tests_no_sync_download() -> Vec<(String, String)> {
   vec![
-    ("DENO_NODE_COMPAT_URL".to_string(), std_file_url()),
     ("NPM_CONFIG_REGISTRY".to_string(), npm_registry_url()),
     ("NO_COLOR".to_string(), "1".to_string()),
   ]

test_util/src/lsp.rs

@@ -1,7 +1,6 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
 use crate::npm_registry_url;
-use crate::std_file_url;
 
 use super::new_deno_dir;
 use super::TempDir;
@@ -234,7 +233,6 @@ impl LspClient {
     let mut command = Command::new(deno_exe);
     command
       .env("DENO_DIR", deno_dir.path())
-      .env("DENO_NODE_COMPAT_URL", std_file_url())
       .env("NPM_CONFIG_REGISTRY", npm_registry_url())
       .arg("lsp")
       .stdin(Stdio::piped())