feat(unstable): Node CJS and ESM resolvers for compat mode (#12424)

This commit adds CJS and ESM Node resolvers to the "--compat" mode.

The functionality is spread across "cli/compat" module and Node compatibility
layer in "deno_std/node"; this stems from the fact that ES module resolution
can only be implemented in Rust as it needs to directly integrated with 
"deno_core"; however "deno_std/node" already provided CJS module resolution.

Currently this resolution is only active when running a files using 
"deno run --compat --unstable <filename>", and is not available in other
subcommands, which will be changed in follow up commits.

cli/compat.rs

@@ -1,88 +0,0 @@
-// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
-
-use deno_core::url::Url;
-use std::collections::HashMap;
-
-// TODO(bartlomieju): this needs to be bumped manually for
-// each release, a better mechanism is preferable, but it's a quick and dirty
-// solution to avoid printing `X-Deno-Warning` headers when the compat layer is
-// downloaded
-static STD_URL: &str = "https://deno.land/std@0.111.0/";
-static GLOBAL_MODULE: &str = "global.ts";
-
-static SUPPORTED_MODULES: &[&str] = &[
-  "assert",
-  "assert/strict",
-  "async_hooks",
-  "buffer",
-  "child_process",
-  "cluster",
-  "console",
-  "constants",
-  "crypto",
-  "dgram",
-  "dns",
-  "domain",
-  "events",
-  "fs",
-  "fs/promises",
-  "http",
-  "https",
-  "module",
-  "net",
-  "os",
-  "path",
-  "path/posix",
-  "path/win32",
-  "perf_hooks",
-  "process",
-  "querystring",
-  "readline",
-  "stream",
-  "stream/promises",
-  "stream/web",
-  "string_decoder",
-  "sys",
-  "timers",
-  "timers/promises",
-  "tls",
-  "tty",
-  "url",
-  "util",
-  "util/types",
-  "v8",
-  "vm",
-  "zlib",
-];
-
-lazy_static::lazy_static! {
-  static ref GLOBAL_URL_STR: String = format!("{}node/{}", STD_URL, GLOBAL_MODULE);
-  pub(crate) static ref GLOBAL_URL: Url = Url::parse(&GLOBAL_URL_STR).unwrap();
-  static ref COMPAT_IMPORT_URL: Url = Url::parse("flags:compat").unwrap();
-}
-
-/// Provide imports into a module graph when the compat flag is true.
-pub(crate) fn get_node_imports() -> Vec<(Url, Vec<String>)> {
-  vec![(COMPAT_IMPORT_URL.clone(), vec![GLOBAL_URL_STR.clone()])]
-}
-
-/// Create a map that can be used to update import map.
-///
-/// Keys are built-in Node modules (and built-ins prefixed with "node:"), while
-/// values are URLs pointing to relevant files in deno.land/std/node/ directory.
-pub fn get_mapped_node_builtins() -> HashMap<String, String> {
-  let mut mappings = HashMap::new();
-
-  for module in SUPPORTED_MODULES {
-    // TODO(bartlomieju): this is unversioned, and should be fixed to use latest stable?
-    let module_url = format!("{}node/{}.ts", STD_URL, module);
-    mappings.insert(module.to_string(), module_url.clone());
-
-    // Support for `node:<module_name>`
-    // https://nodejs.org/api/esm.html#esm_node_imports
-    let node_prefixed = format!("node:{}", module);
-    mappings.insert(node_prefixed, module_url);
-  }
-
-  mappings
-}

cli/compat/errors.rs

@@ -0,0 +1,145 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+use deno_core::error::generic_error;
+use deno_core::error::type_error;
+use deno_core::error::AnyError;
+use deno_core::url::Url;
+
+pub(crate) fn err_invalid_module_specifier(
+  request: &str,
+  reason: &str,
+  maybe_base: Option<String>,
+) -> AnyError {
+  let mut msg = format!(
+    "[ERR_INVALID_MODULE_SPECIFIER] Invalid module \"{}\" {}",
+    request, reason
+  );
+
+  if let Some(base) = maybe_base {
+    msg = format!("{} imported from {}", msg, base);
+  }
+
+  type_error(msg)
+}
+
+pub(crate) fn err_invalid_package_config(
+  path: &str,
+  maybe_base: Option<String>,
+  maybe_message: Option<String>,
+) -> AnyError {
+  let mut msg = format!(
+    "[ERR_INVALID_PACKAGE_CONFIG] Invalid package config {}",
+    path
+  );
+
+  if let Some(base) = maybe_base {
+    msg = format!("{} while importing {}", msg, base);
+  }
+
+  if let Some(message) = maybe_message {
+    msg = format!("{}. {}", msg, message);
+  }
+
+  generic_error(msg)
+}
+
+pub(crate) fn err_module_not_found(
+  path: &str,
+  base: &str,
+  typ: &str,
+) -> AnyError {
+  generic_error(format!(
+    "[ERR_MODULE_NOT_FOUND] Cannot find {} '{}' imported from {}",
+    typ, path, base
+  ))
+}
+
+pub(crate) fn err_unsupported_dir_import(path: &str, base: &str) -> AnyError {
+  generic_error(format!("[ERR_UNSUPPORTED_DIR_IMPORT] Directory import '{}' is not supported resolving ES modules imported from {}", path, base))
+}
+
+pub(crate) fn err_unsupported_esm_url_scheme(url: &Url) -> AnyError {
+  let mut msg =
+    "[ERR_UNSUPPORTED_ESM_URL_SCHEME] Only file and data URLS are supported by the default ESM loader"
+      .to_string();
+
+  if cfg!(window) && url.scheme().len() == 2 {
+    msg = format!(
+      "{}. On Windows, absolute path must be valid file:// URLs",
+      msg
+    );
+  }
+
+  msg = format!("{}. Received protocol '{}'", msg, url.scheme());
+  generic_error(msg)
+}
+
+pub(crate) fn err_invalid_package_target(
+  pkg_path: String,
+  key: String,
+  target: String,
+  is_import: bool,
+  maybe_base: Option<String>,
+) -> AnyError {
+  let rel_error = !is_import && !target.is_empty() && !target.starts_with("./");
+  let mut msg = "[ERR_INVALID_PACKAGE_TARGET]".to_string();
+
+  if key == "." {
+    assert!(!is_import);
+    msg = format!("{} Invalid \"exports\" main target {} defined in the package config {}package.json", msg, target, pkg_path)
+  } else {
+    let ie = if is_import { "imports" } else { "exports" };
+    msg = format!("{} Invalid \"{}\" target {} defined for '{}' in the package config {}package.json", msg, ie, target, key, pkg_path)
+  };
+
+  if let Some(base) = maybe_base {
+    msg = format!("{} imported from {}", msg, base);
+  };
+  if rel_error {
+    msg = format!("{}; target must start with \"./\"", msg);
+  }
+
+  generic_error(msg)
+}
+
+pub(crate) fn err_package_path_not_exported(
+  pkg_path: String,
+  subpath: String,
+  maybe_base: Option<String>,
+) -> AnyError {
+  let mut msg = "[ERR_PACKAGE_PATH_NOT_EXPORTED]".to_string();
+
+  if subpath == "." {
+    msg = format!(
+      "{} No \"exports\" main defined in {}package.json",
+      msg, pkg_path
+    );
+  } else {
+    msg = format!("{} Package subpath \'{}\' is not defined by \"exports\" in {}package.json", msg, subpath, pkg_path);
+  };
+
+  if let Some(base) = maybe_base {
+    msg = format!("{} imported from {}", msg, base);
+  }
+
+  generic_error(msg)
+}
+
+pub(crate) fn err_package_import_not_defined(
+  specifier: &str,
+  package_path: Option<String>,
+  base: &str,
+) -> AnyError {
+  let mut msg = format!(
+    "[ERR_PACKAGE_IMPORT_NOT_DEFINED] Package import specifier \"{}\" is not defined in",
+    specifier
+  );
+
+  if let Some(package_path) = package_path {
+    msg = format!("{} in package {}package.json", msg, package_path);
+  }
+
+  msg = format!("{} imported from {}", msg, base);
+
+  type_error(msg)
+}

cli/compat/mod.rs

@@ -0,0 +1,132 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+mod errors;
+mod esm_resolver;
+
+use deno_core::error::AnyError;
+use deno_core::located_script_name;
+use deno_core::url::Url;
+use deno_core::JsRuntime;
+
+pub use esm_resolver::NodeEsmResolver;
+
+// TODO(bartlomieju): this needs to be bumped manually for
+// each release, a better mechanism is preferable, but it's a quick and dirty
+// solution to avoid printing `X-Deno-Warning` headers when the compat layer is
+// downloaded
+static STD_URL_STR: &str = "https://deno.land/std@0.112.0/";
+
+static SUPPORTED_MODULES: &[&str] = &[
+  "assert",
+  "assert/strict",
+  "async_hooks",
+  "buffer",
+  "child_process",
+  "cluster",
+  "console",
+  "constants",
+  "crypto",
+  "dgram",
+  "dns",
+  "domain",
+  "events",
+  "fs",
+  "fs/promises",
+  "http",
+  "https",
+  "module",
+  "net",
+  "os",
+  "path",
+  "path/posix",
+  "path/win32",
+  "perf_hooks",
+  "process",
+  "querystring",
+  "readline",
+  "stream",
+  "stream/promises",
+  "stream/web",
+  "string_decoder",
+  "sys",
+  "timers",
+  "timers/promises",
+  "tls",
+  "tty",
+  "url",
+  "util",
+  "util/types",
+  "v8",
+  "vm",
+  "zlib",
+];
+
+lazy_static::lazy_static! {
+  static ref GLOBAL_URL_STR: String = format!("{}node/global.ts", STD_URL_STR);
+  pub(crate) static ref GLOBAL_URL: Url = Url::parse(&GLOBAL_URL_STR).unwrap();
+  static ref MODULE_URL_STR: String = format!("{}node/module.ts", STD_URL_STR);
+  pub(crate) static ref MODULE_URL: Url = Url::parse(&MODULE_URL_STR).unwrap();
+  static ref COMPAT_IMPORT_URL: Url = Url::parse("flags:compat").unwrap();
+}
+
+/// Provide imports into a module graph when the compat flag is true.
+pub(crate) fn get_node_imports() -> Vec<(Url, Vec<String>)> {
+  vec![(COMPAT_IMPORT_URL.clone(), vec![GLOBAL_URL_STR.clone()])]
+}
+
+fn try_resolve_builtin_module(specifier: &str) -> Option<Url> {
+  if SUPPORTED_MODULES.contains(&specifier) {
+    let module_url = format!("{}node/{}.ts", STD_URL_STR, specifier);
+    Some(Url::parse(&module_url).unwrap())
+  } else {
+    None
+  }
+}
+
+pub async fn check_if_should_use_esm_loader(
+  js_runtime: &mut JsRuntime,
+  main_module: &str,
+) -> Result<bool, AnyError> {
+  // Decide if we're running with Node ESM loader or CJS loader.
+  let source_code = &format!(
+    r#"(async function checkIfEsm(main) {{
+      const {{ resolveMainPath, shouldUseESMLoader }} = await import("{}");
+      const resolvedMain = resolveMainPath(main);
+      const useESMLoader = shouldUseESMLoader(resolvedMain);
+      return useESMLoader;
+    }})('{}');"#,
+    MODULE_URL_STR.as_str(),
+    escape_for_single_quote_string(main_module),
+  );
+  let result =
+    js_runtime.execute_script(&located_script_name!(), source_code)?;
+  let use_esm_loader_global = js_runtime.resolve_value(result).await?;
+  let use_esm_loader = {
+    let scope = &mut js_runtime.handle_scope();
+    let use_esm_loader_local = use_esm_loader_global.get(scope);
+    use_esm_loader_local.boolean_value(scope)
+  };
+
+  Ok(use_esm_loader)
+}
+
+pub fn load_cjs_module(
+  js_runtime: &mut JsRuntime,
+  main_module: &str,
+) -> Result<(), AnyError> {
+  let source_code = &format!(
+    r#"(async function loadCjsModule(main) {{
+      const Module = await import("{}");
+      Module.default._load(main, null, true);
+    }})('{}');"#,
+    MODULE_URL_STR.as_str(),
+    escape_for_single_quote_string(main_module),
+  );
+
+  js_runtime.execute_script(&located_script_name!(), source_code)?;
+  Ok(())
+}
+
+fn escape_for_single_quote_string(text: &str) -> String {
+  text.replace(r"\", r"\\").replace("'", r"\'")
+}

cli/compat/testdata/basic/main.js

@@ -0,0 +1 @@
+import "foo";

cli/compat/testdata/basic/node_modules/foo/package.json

@@ -0,0 +1,5 @@
+{
+    "name": "foo",
+    "type": "module",
+    "exports": "./index.js"
+}

cli/compat/testdata/basic/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "bar",
+  "type": "module",
+  "dependencies": {
+    "foo": "1.0.0"
+  }
+}

cli/compat/testdata/basic_deps/main.js

@@ -0,0 +1 @@
+import "foo";

cli/compat/testdata/basic_deps/node_modules/bar/bar.js

@@ -0,0 +1 @@
+export const BAR = 123;

cli/compat/testdata/basic_deps/node_modules/bar/package.json

@@ -0,0 +1,6 @@
+{
+    "name": "bar",
+    "version": "0.1.2",
+    "type": "module",
+    "exports": "./bar.js"
+}

cli/compat/testdata/basic_deps/node_modules/foo/foo.js

@@ -0,0 +1 @@
+import "bar";

cli/compat/testdata/basic_deps/node_modules/foo/package.json

@@ -0,0 +1,8 @@
+{
+    "name": "foo",
+    "type": "module",
+    "exports": "./foo.js",
+    "dependencies": {
+      "bar": "0.1.2"
+    }
+}

cli/compat/testdata/basic_deps/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "main_program",
+  "type": "module",
+  "dependencies": {
+    "foo": "1.0.0"
+  }
+}

cli/compat/testdata/conditions/main.js

@@ -0,0 +1 @@
+import "imports_exports";

cli/compat/testdata/conditions/node_modules/imports_exports/import_export.js

@@ -0,0 +1,6 @@
+import dep from "#dep";
+
+export default {
+  bar: "bar",
+  dep,
+};

cli/compat/testdata/conditions/node_modules/imports_exports/import_polyfill.js

@@ -0,0 +1,3 @@
+export default {
+  polyfill: "import",
+};

cli/compat/testdata/conditions/node_modules/imports_exports/package.json

@@ -0,0 +1,17 @@
+{
+  "version": "1.0.0",
+  "name": "imports_exports",
+  "main": "./require_export.cjs",
+  "imports": {
+    "#dep": {
+      "import": "./import_polyfill.js",
+      "require": "./require_polyfill.js"
+    }
+  },
+  "exports": {
+    ".": {
+      "import": "./import_export.js",
+      "require": "./require_export.cjs"
+    }
+  }
+}

cli/compat/testdata/conditions/node_modules/imports_exports/require_export.cjs

@@ -0,0 +1,6 @@
+const dep = require("#dep");
+
+module.exports = {
+    foo: "foo",
+    dep,
+};

cli/compat/testdata/conditions/node_modules/imports_exports/require_polyfill.js

@@ -0,0 +1,3 @@
+module.exports = {
+  polyfill: "require",
+};

cli/compat/testdata/conditions/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "conditions",
+  "type": "module",
+  "dependencies": {
+    "imports_exports": "1.0.0"
+  }
+}

cli/compat/testdata/deep/a/b/c/d/main.js

@@ -0,0 +1 @@
+import "foo";

cli/compat/testdata/deep/node_modules/foo/package.json

@@ -0,0 +1,5 @@
+{
+    "name": "foo",
+    "type": "module",
+    "exports": "./index.js"
+}

cli/main.rs

@@ -1092,6 +1092,11 @@ async fn run_command(
     return run_with_watch(flags, run_flags.script).await;
   }
 
+  // TODO(bartlomieju): it should not be resolved here if we're in compat mode
+  // because it might be a bare specifier
+  // TODO(bartlomieju): actually I think it will also fail if there's an import
+  // map specified and bare specifier is used on the command line - this should
+  // probably call `ProcState::resolve` instead
   let main_module = resolve_url_or_path(&run_flags.script)?;
   let ps = ProcState::build(flags.clone()).await?;
   let permissions = Permissions::from_options(&flags.clone().into());
@@ -1114,10 +1119,41 @@ async fn run_command(
     };
 
   debug!("main_module {}", main_module);
+
   if flags.compat {
+    // TODO(bartlomieju): fix me
+    assert_eq!(main_module.scheme(), "file");
+
+    // Set up Node globals
     worker.execute_side_module(&compat::GLOBAL_URL).await?;
+    // And `module` module that we'll use for checking which
+    // loader to use and potentially load CJS module with.
+    // This allows to skip permission check for `--allow-net`
+    // which would otherwise be requested by dynamically importing
+    // this file.
+    worker.execute_side_module(&compat::MODULE_URL).await?;
+
+    let use_esm_loader = compat::check_if_should_use_esm_loader(
+      &mut worker.js_runtime,
+      &main_module.to_file_path().unwrap().display().to_string(),
+    )
+    .await?;
+
+    if use_esm_loader {
+      // ES module execution in Node compatiblity mode
+      worker.execute_main_module(&main_module).await?;
+    } else {
+      // CJS module execution in Node compatiblity mode
+      compat::load_cjs_module(
+        &mut worker.js_runtime,
+        &main_module.to_file_path().unwrap().display().to_string(),
+      )?;
+    }
+  } else {
+    // Regular ES module execution
+    worker.execute_main_module(&main_module).await?;
   }
-  worker.execute_main_module(&main_module).await?;
+
   worker.execute_script(
     &located_script_name!(),
     "window.dispatchEvent(new Event('load'))",

cli/proc_state.rs

@@ -3,6 +3,7 @@
 use crate::cache;
 use crate::colors;
 use crate::compat;
+use crate::compat::NodeEsmResolver;
 use crate::config_file::ConfigFile;
 use crate::deno_dir;
 use crate::emit;
@@ -195,7 +196,7 @@ impl ProcState {
         None
       };
 
-    let mut maybe_import_map: Option<ImportMap> =
+    let maybe_import_map: Option<ImportMap> =
       match flags.import_map_path.as_ref() {
         None => None,
         Some(import_map_url) => {
@@ -217,32 +218,6 @@ impl ProcState {
         }
       };
 
-    if flags.compat {
-      let mut import_map = match maybe_import_map {
-        Some(import_map) => import_map,
-        None => {
-          // INFO: we're creating an empty import map, with its specifier pointing
-          // to `CWD/node_import_map.json` to make sure the map still works as expected.
-          let import_map_specifier =
-            std::env::current_dir()?.join("node_import_map.json");
-          ImportMap::from_json(import_map_specifier.to_str().unwrap(), "{}")
-            .unwrap()
-        }
-      };
-      let node_builtins = compat::get_mapped_node_builtins();
-      let diagnostics = import_map.update_imports(node_builtins)?;
-
-      if !diagnostics.is_empty() {
-        log::info!("Some Node built-ins were not added to the import map:");
-        for diagnostic in diagnostics {
-          log::info!("  - {}", diagnostic);
-        }
-        log::info!("If you want to use Node built-ins provided by Deno remove listed specifiers from \"imports\" mapping in the import map file.");
-      }
-
-      maybe_import_map = Some(import_map);
-    }
-
     let maybe_inspect_host = flags.inspect.or(flags.inspect_brk);
     let maybe_inspector_server = maybe_inspect_host.map(|host| {
       Arc::new(InspectorServer::new(host, version::get_user_agent()))
@@ -316,14 +291,29 @@ impl ProcState {
     );
     let maybe_locker = as_maybe_locker(self.lockfile.clone());
     let maybe_imports = self.get_maybe_imports();
-    let maybe_resolver =
+    let node_resolver = NodeEsmResolver;
+    let import_map_resolver =
       self.maybe_import_map.as_ref().map(ImportMapResolver::new);
+    let maybe_resolver = if self.flags.compat {
+      Some(node_resolver.as_resolver())
+    } else {
+      import_map_resolver.as_ref().map(|im| im.as_resolver())
+    };
+    // TODO(bartlomieju): this is very make-shift, is there an existing API
+    // that we could include it like with "maybe_imports"?
+    let roots = if self.flags.compat {
+      let mut r = vec![compat::GLOBAL_URL.clone()];
+      r.extend(roots);
+      r
+    } else {
+      roots
+    };
     let graph = deno_graph::create_graph(
       roots,
       is_dynamic,
       maybe_imports,
       &mut cache,
-      maybe_resolver.as_ref().map(|im| im.as_resolver()),
+      maybe_resolver,
       maybe_locker,
       None,
     )

cli/tests/integration/compat_tests.rs

@@ -9,21 +9,15 @@ itest!(globals {
 });
 
 itest!(fs_promises {
-  args: "run --compat --unstable -A compat/fs_promises.js",
+  args: "run --compat --unstable -A compat/fs_promises.mjs",
   output: "compat/fs_promises.out",
 });
 
 itest!(node_prefix_fs_promises {
-  args: "run --compat --unstable -A compat/node_fs_promises.js",
+  args: "run --compat --unstable -A compat/node_fs_promises.mjs",
   output: "compat/fs_promises.out",
 });
 
-itest!(existing_import_map {
-  args: "run --compat --unstable --import-map compat/existing_import_map.json compat/fs_promises.js",
-  output: "compat/existing_import_map.out",
-  exit_code: 1,
-});
-
 #[test]
 fn globals_in_repl() {
   let (out, _err) = util::run_and_collect_output_with_args(

cli/tests/testdata/compat/existing_import_map.json

@@ -1,5 +0,0 @@
-{
-  "imports": {
-    "fs/promises": "./non_existent_file.js"
-  }
-}

cli/tests/testdata/compat/existing_import_map.out

@@ -1,7 +0,0 @@
-[WILDCARD]
-Some Node built-ins were not added to the import map:
-  - "fs/promises" already exists and is mapped to "file://[WILDCARD]/non_existent_file.js"
-If you want to use Node built-ins provided by Deno remove listed specifiers from "imports" mapping in the import map file.
-[WILDCARD]
-error: Cannot load module "file://[WILDCARD]/non_existent_file.js".
-    at file://[WILDCARD]/fs_promises.js:1:16

cli/tests/testdata/compat/globals.out

@@ -2,6 +2,8 @@
 process {
 [WILDCARD]
 }
-[Function: Buffer]
+[Function: Buffer] {
+[WILDCARD]
+}
 [Function: setImmediate]
 [Function: clearTimeout]