fix: avoid global declaration collisions in cjs (#15608)

* Use a default stack size * 2 in debug for Windows because swc using so much stack size. We should look into this more later though.

.cargo/config.toml

@@ -7,7 +7,7 @@ rustflags = [
   "target-feature=+crt-static",
   "-C",
   # increase the stack size to prevent swc overflowing the stack in debug
-  "link-arg=/STACK:1572864",
+  "link-arg=/STACK:2097152",
 ]
 
 [target.aarch64-apple-darwin]

Cargo.lock

@@ -4603,6 +4603,7 @@ dependencies = [
  "async-stream",
  "atty",
  "base64 0.13.0",
+ "flate2",
  "futures",
  "hyper",
  "lazy_static",
@@ -4613,9 +4614,11 @@ dependencies = [
  "pty",
  "regex",
  "reqwest",
+ "ring",
  "rustls-pemfile 1.0.0",
  "serde",
  "serde_json",
+ "tar",
  "tokio",
  "tokio-rustls",
  "tokio-tungstenite",

cli/npm/cache.rs

@@ -210,7 +210,16 @@ impl NpmCache {
     if response.status() == 404 {
       bail!("Could not find npm package tarball at: {}", dist.tarball);
     } else if !response.status().is_success() {
-      bail!("Bad response: {:?}", response.status());
+      let status = response.status();
+      let maybe_response_text = response.text().await.ok();
+      bail!(
+        "Bad response: {:?}{}",
+        status,
+        match maybe_response_text {
+          Some(text) => format!("\n\n{}", text),
+          None => String::new(),
+        }
+      );
     } else {
       let bytes = response.bytes().await?;
 

cli/npm/registry.rs

@@ -302,7 +302,16 @@ impl NpmRegistryApi {
     if response.status() == 404 {
       Ok(None)
     } else if !response.status().is_success() {
-      bail!("Bad response: {:?}", response.status());
+      let status = response.status();
+      let maybe_response_text = response.text().await.ok();
+      bail!(
+        "Bad response: {:?}{}",
+        status,
+        match maybe_response_text {
+          Some(text) => format!("\n\n{}", text),
+          None => String::new(),
+        }
+      );
     } else {
       let bytes = response.bytes().await?;
       let package_info = serde_json::from_slice(&bytes)?;

cli/tests/integration/npm_tests.rs

@@ -6,8 +6,7 @@ use test_util as util;
 use util::assert_contains;
 use util::http_server;
 
-// NOTE: It's possible to automatically update the npm registry data in the test server
-// by setting the DENO_TEST_UTIL_UPDATE_NPM=1 environment variable.
+// NOTE: See how to make test npm packages at ../testdata/npm/README.md
 
 itest!(esm_module {
   args: "run --allow-read --unstable npm/esm/main.js",
@@ -48,6 +47,13 @@ itest!(cjs_sub_path {
   http_server: true,
 });
 
+itest!(cjs_local_global_decls {
+  args: "run --allow-read --unstable npm/cjs_local_global_decls/main.ts",
+  output: "npm/cjs_local_global_decls/main.out",
+  envs: env_vars(),
+  http_server: true,
+});
+
 itest!(dynamic_import {
   args: "run --allow-read --unstable npm/dynamic_import/main.ts",
   output: "npm/dynamic_import/main.out",
@@ -238,12 +244,14 @@ fn ensure_registry_files_local() {
       let registry_json_path = registry_dir_path
         .join(entry.file_name())
         .join("registry.json");
-      let file_text = std::fs::read_to_string(&registry_json_path).unwrap();
-      if file_text.contains("https://registry.npmjs.org/") {
-        panic!(
-          "file {} contained a reference to the npm registry",
-          registry_json_path.display(),
-        );
+      if registry_json_path.exists() {
+        let file_text = std::fs::read_to_string(&registry_json_path).unwrap();
+        if file_text.contains("https://registry.npmjs.org/") {
+          panic!(
+            "file {} contained a reference to the npm registry",
+            registry_json_path.display(),
+          );
+        }
       }
     }
   }

cli/tests/testdata/npm/README.md

@@ -0,0 +1,18 @@
+# npm test data
+
+This folder contains test data for npm specifiers.
+
+## Registry
+
+The registry is served by the test server (server in test_util) at
+http://localhost:4545/npm/registry/ via the `./registry` folder.
+
+### Updating with real npm packages
+
+1. Set the `DENO_TEST_UTIL_UPDATE_NPM=1` environment variable
+2. Run the test and it should download the packages.
+
+### Using a custom npm package
+
+1. Add the custom package to `./registry/@denotest`
+2. Reference `npm:@denotest/<your-package-name>` in the tests.

cli/tests/testdata/npm/cjs_local_global_decls/main.out

@@ -0,0 +1,3 @@
+Download http://localhost:4545/npm/registry/@denotest/cjs-local-global-decls
+Download http://localhost:4545/npm/registry/@denotest/cjs-local-global-decls/1.0.0.tgz
+Loaded.

cli/tests/testdata/npm/cjs_local_global_decls/main.ts

@@ -0,0 +1 @@
+import "npm:@denotest/cjs-local-global-decls@1.0.0";

cli/tests/testdata/npm/registry/@denotest/cjs-local-global-decls/1.0.0/index.js

@@ -0,0 +1,4 @@
+// package that has all the locals defined
+const Buffer = 1, clearImmediate = 1, clearInterval = 1, clearTimeout = 1, global = 1, process = 1, setImmediate = 1, setInterval = 1, setTimeout = 1, globalThis = 1;
+const exports = 2;
+console.log("Loaded.");

cli/tests/testdata/npm/registry/@denotest/cjs-local-global-decls/1.0.0/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@deno/cjs-local-global-decls",
+  "version": "1.0.0"
+}

ext/node/02_require.js

@@ -656,11 +656,10 @@
   };
 
   Module.wrapper = [
-    // TODO:
-    // We provide non standard timer APIs in the CommonJS wrapper
+    // We provide the non-standard APIs in the CommonJS wrapper
     // to avoid exposing them in global namespace.
-    "(function (exports, require, module, __filename, __dirname, globalThis) { (function (exports, require, module, __filename, __dirname, globalThis, Buffer, clearImmediate, clearInterval, clearTimeout, global, process, setImmediate, setInterval, setTimeout) {",
-    "\n}).call(this, exports, require, module, __filename, __dirname, globalThis, globalThis.Buffer, globalThis.clearImmediate, globalThis.clearInterval, globalThis.clearTimeout, globalThis.global, globalThis.process, globalThis.setImmediate, globalThis.setInterval, globalThis.setTimeout); })",
+    "(function (exports, require, module, __filename, __dirname, globalThis) { const { Buffer, clearImmediate, clearInterval, clearTimeout, global, process, setImmediate, setInterval, setTimeout} = globalThis; (function () {",
+    "\n}).call(this); })",
   ];
   Module.wrap = function (script) {
     script = script.replace(/^#!.*?\n/, "");

test_util/Cargo.toml

@@ -16,6 +16,7 @@ anyhow = "1.0.57"
 async-stream = "0.3.3"
 atty = "0.2.14"
 base64 = "0.13.0"
+flate2 = "1.0.24"
 futures = "0.3.21"
 hyper = { version = "0.14.18", features = ["server", "http1", "http2", "runtime"] }
 lazy_static = "1.4.0"
@@ -25,9 +26,11 @@ parking_lot = "0.12.0"
 pretty_assertions = "=1.2.1"
 regex = "1.6.0"
 reqwest = { version = "0.11.11", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks"] }
+ring = "0.16.20"
 rustls-pemfile = "1.0.0"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.79"
+tar = "0.4.38"
 tokio = { version = "1.19", features = ["full"] }
 tokio-rustls = "0.23"
 tokio-tungstenite = "0.16"

test_util/src/lib.rs

@@ -14,6 +14,7 @@ use hyper::Request;
 use hyper::Response;
 use hyper::StatusCode;
 use lazy_static::lazy_static;
+use npm::custom_npm_cache;
 use os_pipe::pipe;
 use pretty_assertions::assert_eq;
 use regex::Regex;
@@ -51,6 +52,7 @@ use tokio_tungstenite::accept_async;
 
 pub mod assertions;
 pub mod lsp;
+mod npm;
 pub mod pty;
 mod temp_dir;
 
@@ -953,7 +955,22 @@ async fn main_server(
       }
 
       // serve npm registry files
-      if req.uri().path().starts_with("/npm/registry/") {
+      if let Some(suffix) =
+        req.uri().path().strip_prefix("/npm/registry/@denotest/")
+      {
+        // serve all requests to /npm/registry/@deno using the file system
+        // at that path
+        match handle_custom_npm_registry_path(suffix) {
+          Ok(Some(response)) => return Ok(response),
+          Ok(None) => {} // ignore, not found
+          Err(err) => {
+            return Response::builder()
+              .status(StatusCode::INTERNAL_SERVER_ERROR)
+              .body(format!("{:#}", err).into());
+          }
+        }
+      } else if req.uri().path().starts_with("/npm/registry/") {
+        // otherwise, serve based on registry.json and tgz files
         let is_tarball = req.uri().path().ends_with(".tgz");
         if !is_tarball {
           file_path.push("registry.json");
@@ -985,6 +1002,33 @@ async fn main_server(
   };
 }
 
+fn handle_custom_npm_registry_path(
+  path: &str,
+) -> Result<Option<Response<Body>>, anyhow::Error> {
+  let parts = path
+    .split('/')
+    .filter(|p| !p.is_empty())
+    .collect::<Vec<_>>();
+  let cache = custom_npm_cache()?;
+  let package_name = format!("@denotest/{}", parts[0]);
+  if parts.len() == 2 {
+    if let Some(file_bytes) =
+      cache.tarball_bytes(&package_name, parts[1].trim_end_matches(".tgz"))
+    {
+      let file_resp = custom_headers("file.tgz", file_bytes.to_owned());
+      return Ok(Some(file_resp));
+    }
+  } else if parts.len() == 1 {
+    if let Some(registry_file) = cache.registry_file(&package_name) {
+      let file_resp =
+        custom_headers("registry.json", registry_file.as_bytes().to_vec());
+      return Ok(Some(file_resp));
+    }
+  }
+
+  Ok(None)
+}
+
 fn should_download_npm_packages() -> bool {
   // when this env var is set, it will download and save npm packages
   // to the testdata/npm/registry directory
@@ -1489,6 +1533,8 @@ fn custom_headers(p: &str, body: Vec<u8>) -> Response<Body> {
     Some("application/json")
   } else if p.ends_with(".wasm") {
     Some("application/wasm")
+  } else if p.ends_with(".tgz") {
+    Some("application/gzip")
   } else {
     None
   };

test_util/src/npm.rs

@@ -0,0 +1,143 @@
+use std::collections::HashMap;
+use std::fs;
+
+use anyhow::Context;
+use flate2::write::GzEncoder;
+use flate2::Compression;
+use once_cell::sync::Lazy;
+use tar::Builder;
+
+use crate::testdata_path;
+
+static CUSTOM_NPM_PACKAGE_CACHE: Lazy<Result<CustomNpmPackageCache, String>> =
+  Lazy::new(|| CustomNpmPackageCache::load().map_err(|e| e.to_string()));
+
+/// Get a reference to the custom npm cache which is lazily created.
+pub fn custom_npm_cache(
+) -> Result<&'static CustomNpmPackageCache, anyhow::Error> {
+  match &*CUSTOM_NPM_PACKAGE_CACHE {
+    Ok(cache) => Ok(cache),
+    Err(err) => Err(anyhow::anyhow!("{}", err)),
+  }
+}
+
+struct CustomNpmPackage {
+  pub registry_file: String,
+  pub tarballs: HashMap<String, Vec<u8>>,
+}
+
+/// Creates tarballs and a registry json file for npm packages
+/// in the `testdata/npm/registry/@denotest` directory.
+pub struct CustomNpmPackageCache(HashMap<String, CustomNpmPackage>);
+
+impl CustomNpmPackageCache {
+  pub fn load() -> Result<Self, anyhow::Error> {
+    use ring::digest::Context;
+    use ring::digest::SHA512;
+
+    // read all the packages in the @denotest folder
+    let custom_packages_path = testdata_path().join("npm/registry/@denotest");
+    let mut packages = HashMap::new();
+    for entry in fs::read_dir(&custom_packages_path)? {
+      let entry = entry?;
+      let file_type = entry.file_type()?;
+      if !file_type.is_dir() {
+        continue;
+      }
+
+      // read all the package's versions
+      let mut tarballs = HashMap::new();
+      let package_folder_name = entry.file_name().to_string_lossy().to_string();
+      let package_name = format!("@denotest/{}", package_folder_name);
+      let package_folder = custom_packages_path.join(&package_folder_name);
+      let mut versions = serde_json::Map::new();
+      for entry in fs::read_dir(&package_folder)? {
+        let entry = entry?;
+        let file_type = entry.file_type()?;
+        if !file_type.is_dir() {
+          continue;
+        }
+        let version = entry.file_name().to_string_lossy().to_string();
+        let version_folder = package_folder.join(&version);
+
+        // create the tarball
+        let mut tarball_bytes = Vec::new();
+        {
+          let mut encoder =
+            GzEncoder::new(&mut tarball_bytes, Compression::default());
+          {
+            let mut builder = Builder::new(&mut encoder);
+            builder
+              .append_dir_all("package", &version_folder)
+              .with_context(|| {
+                format!(
+                  "Error adding tarball for directory: {}",
+                  version_folder.display()
+                )
+              })?;
+            builder.finish()?;
+          }
+          encoder.finish()?;
+        }
+
+        // get tarball hash
+        let mut hash_ctx = Context::new(&SHA512);
+        hash_ctx.update(&tarball_bytes);
+        let digest = hash_ctx.finish();
+        let tarball_checksum = base64::encode(digest.as_ref()).to_lowercase();
+
+        // create the registry file JSON for this version
+        let mut dist = serde_json::Map::new();
+        dist.insert(
+          "integrity".to_string(),
+          format!("sha512-{}", tarball_checksum).into(),
+        );
+        dist.insert("shasum".to_string(), "dummy-value".into());
+        dist.insert(
+          "tarball".to_string(),
+          format!(
+            "http://localhost:4545/npm/registry/{}/{}.tgz",
+            package_name, version
+          )
+          .into(),
+        );
+
+        tarballs.insert(version.clone(), tarball_bytes);
+        let package_json_path = version_folder.join("package.json");
+        let package_json_text = fs::read_to_string(&package_json_path)
+          .with_context(|| {
+            format!(
+              "Error reading package.json at {}",
+              package_json_path.display()
+            )
+          })?;
+        let mut version_info: serde_json::Map<String, serde_json::Value> =
+          serde_json::from_str(&package_json_text)?;
+        version_info.insert("dist".to_string(), dist.into());
+        versions.insert(version, version_info.into());
+      }
+
+      // create the registry file for this package
+      let mut registry_file = serde_json::Map::new();
+      registry_file.insert("name".to_string(), package_name.clone().into());
+      registry_file.insert("versions".to_string(), versions.into());
+      packages.insert(
+        package_name,
+        CustomNpmPackage {
+          registry_file: serde_json::to_string(&registry_file).unwrap(),
+          tarballs,
+        },
+      );
+    }
+
+    Ok(Self(packages))
+  }
+
+  pub fn tarball_bytes(&self, name: &str, version: &str) -> Option<&Vec<u8>> {
+    self.0.get(name).and_then(|p| p.tarballs.get(version))
+  }
+
+  pub fn registry_file(&self, name: &str) -> Option<&String> {
+    self.0.get(name).map(|p| &p.registry_file)
+  }
+}

tools/lint.js

@@ -29,6 +29,7 @@ async function dlint() {
     ":!:cli/tests/testdata/encoding/**",
     ":!:cli/tests/testdata/error_syntax.js",
     ":!:cli/tests/testdata/fmt/**",
+    ":!:cli/tests/testdata/npm/**",
     ":!:cli/tests/testdata/lint/**",
     ":!:cli/tests/testdata/tsc/**",
     ":!:cli/tsc/*typescript.js",