fix(compile): make output more deterministic (#25092)

Closes https://github.com/denoland/deno/issues/25084

Cargo.lock

@@ -1869,9 +1869,9 @@ dependencies = [
 
 [[package]]
 name = "deno_npm"
-version = "0.22.0"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01fd279be9f522ebfaabce7ff0f4b069c2c1d737946ba57456b579f9246669e8"
+checksum = "ceef28152643a021fc3487a3ec93ae83996972c087547d97f425c83e15dca932"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -2778,9 +2778,9 @@ dependencies = [
 
 [[package]]
 name = "eszip"
-version = "0.74.0"
+version = "0.75.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a74ab660df48e00a4582f6ca506b3cfc5683ce80e73158c9e4dbf84341bb3aea"
+checksum = "67ab3253e0e6a99b79ffc6d7a6243ad541f54543768cecd2198b64476a5971f8"
 dependencies = [
  "anyhow",
  "async-trait",

cli/Cargo.toml

@@ -72,13 +72,13 @@ deno_emit = "=0.44.0"
 deno_graph = { version = "=0.81.2" }
 deno_lint = { version = "=0.63.1", features = ["docs"] }
 deno_lockfile.workspace = true
-deno_npm = "=0.22.0"
+deno_npm = "=0.23.0"
 deno_package_json.workspace = true
 deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] }
 deno_semver = "=0.5.10"
 deno_task_shell = "=0.17.0"
 deno_terminal.workspace = true
-eszip = "=0.74.0"
+eszip = "=0.75.0"
 libsui = "0.3.0"
 napi_sym.workspace = true
 node_resolver.workspace = true

cli/mainrt.rs

@@ -31,6 +31,7 @@ use deno_runtime::fmt_errors::format_js_error;
 use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics;
 pub use deno_runtime::UNSTABLE_GRANULAR_FLAGS;
 use deno_terminal::colors;
+use indexmap::IndexMap;
 
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -73,7 +74,7 @@ fn unwrap_or_exit<T>(result: Result<T, AnyError>) -> T {
   }
 }
 
-fn load_env_vars(env_vars: &HashMap<String, String>) {
+fn load_env_vars(env_vars: &IndexMap<String, String>) {
   env_vars.iter().for_each(|env_var| {
     if env::var(env_var.0).is_err() {
       std::env::set_var(env_var.0, env_var.1);

cli/standalone/binary.rs

@@ -100,6 +100,8 @@ pub struct SerializedWorkspaceResolver {
   pub pkg_json_resolution: PackageJsonDepResolution,
 }
 
+// Note: Don't use hashmaps/hashsets. Ensure the serialization
+// is deterministic.
 #[derive(Deserialize, Serialize)]
 pub struct Metadata {
   pub argv: Vec<String>,
@@ -111,7 +113,7 @@ pub struct Metadata {
   pub ca_stores: Option<Vec<String>>,
   pub ca_data: Option<Vec<u8>>,
   pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
-  pub env_vars_from_env_file: HashMap<String, String>,
+  pub env_vars_from_env_file: IndexMap<String, String>,
   pub workspace_resolver: SerializedWorkspaceResolver,
   pub entrypoint_key: String,
   pub node_modules: Option<NodeModules>,
@@ -667,8 +669,10 @@ impl<'a> DenoCompileBinaryWriter<'a> {
           // but also don't make this dependent on the registry url
           let root_path = npm_resolver.global_cache_root_folder();
           let mut builder = VfsBuilder::new(root_path)?;
-          for package in npm_resolver.all_system_packages(&self.npm_system_info)
-          {
+          let mut packages =
+            npm_resolver.all_system_packages(&self.npm_system_info);
+          packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism
+          for package in packages {
             let folder =
               npm_resolver.resolve_pkg_folder_from_pkg_id(&package.id)?;
             builder.add_dir_recursive(&folder)?;
@@ -729,11 +733,13 @@ impl<'a> DenoCompileBinaryWriter<'a> {
           cli_options.workspace().root_dir().to_file_path().unwrap(),
         );
         while let Some(pending_dir) = pending_dirs.pop_front() {
-          let entries = fs::read_dir(&pending_dir).with_context(|| {
-            format!("Failed reading: {}", pending_dir.display())
-          })?;
+          let mut entries = fs::read_dir(&pending_dir)
+            .with_context(|| {
+              format!("Failed reading: {}", pending_dir.display())
+            })?
+            .collect::<Result<Vec<_>, _>>()?;
+          entries.sort_by_cached_key(|entry| entry.file_name()); // determinism
           for entry in entries {
-            let entry = entry?;
             let path = entry.path();
             if !path.is_dir() {
               continue;
@@ -755,8 +761,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
 /// in the passed environment file.
 fn get_file_env_vars(
   filename: String,
-) -> Result<HashMap<String, String>, dotenvy::Error> {
-  let mut file_env_vars = HashMap::new();
+) -> Result<IndexMap<String, String>, dotenvy::Error> {
+  let mut file_env_vars = IndexMap::new();
   for item in dotenvy::from_filename_iter(filename)? {
     let Ok((key, val)) = item else {
       continue; // this failure will be warned about on load

cli/standalone/virtual_fs.rs

@@ -90,8 +90,11 @@ impl VfsBuilder {
     let read_dir = std::fs::read_dir(path)
       .with_context(|| format!("Reading {}", path.display()))?;
 
-    for entry in read_dir {
-      let entry = entry?;
+    let mut dir_entries =
+      read_dir.into_iter().collect::<Result<Vec<_>, _>>()?;
+    dir_entries.sort_by_cached_key(|entry| entry.file_name()); // determinism
+
+    for entry in dir_entries {
       let file_type = entry.file_type()?;
       let path = entry.path();
 

tests/specs/compile/determinism/__test__.jsonc

@@ -0,0 +1,28 @@
+{
+  "tempDir": true,
+  "steps": [{
+    "if": "unix",
+    "args": "compile --output main1 main.ts",
+    "output": "[WILDCARD]"
+  }, {
+    "if": "unix",
+    "args": "compile --output main2 main.ts",
+    "output": "[WILDCARD]"
+  }, {
+    "if": "unix",
+    "args": "run --allow-read=. assert_equal.ts main1 main2",
+    "output": "Same\n"
+  }, {
+    "if": "windows",
+    "args": "compile --output main1.exe main.ts",
+    "output": "[WILDCARD]"
+  }, {
+    "if": "windows",
+    "args": "compile --output main2.exe main.ts",
+    "output": "[WILDCARD]"
+  }, {
+    "if": "windows",
+    "args": "run --allow-read=. assert_equal.ts main1.exe main2.exe",
+    "output": "Same\n"
+  }]
+}

tests/specs/compile/determinism/assert_equal.ts

@@ -0,0 +1,15 @@
+const file1 = Deno.readFileSync(Deno.args[0]);
+const file2 = Deno.readFileSync(Deno.args[1]);
+
+if (file1.length !== file2.length) {
+  console.error("File lengths are different");
+  Deno.exit(1);
+}
+for (let i = 0; i < file1.length; i++) {
+  if (file1[i] !== file2[i]) {
+    console.error("Files are different");
+    Deno.exit(1);
+  }
+}
+
+console.error("Same");

tests/specs/compile/determinism/main.ts

@@ -0,0 +1,4 @@
+import "npm:@denotest/esm-basic";
+import "npm:@denotest/add";
+import "npm:@denotest/subtract";
+import "npm:preact";