chore: test builders for integration tests (#17965)

Start of adding test builders to simplify integration tests.

I only updated a few test files. We can complete upgrading over time.

cli/tests/integration/bench_tests.rs

@@ -2,7 +2,11 @@
 
 use deno_core::url::Url;
 use test_util as util;
+use util::assert_contains;
+use util::assert_exit_code;
+use util::assert_output_file;
 use util::env_vars_for_npm_tests;
+use util::TestContext;
 
 itest!(overloads {
   args: "bench bench/overloads.ts",
@@ -187,19 +191,16 @@ itest!(json_output {
 
 #[test]
 fn recursive_permissions_pledge() {
-  let output = util::deno_cmd()
+  let context = TestContext::default();
-    .current_dir(util::testdata_path())
+  let output = context
-    .arg("bench")
+    .new_command()
-    .arg("bench/recursive_permissions_pledge.js")
+    .args("bench bench/recursive_permissions_pledge.js")
-    .stderr(std::process::Stdio::piped())
+    .run();
-    .spawn()
+  assert_exit_code!(output, 1);
-    .unwrap()
+  assert_contains!(
-    .wait_with_output()
+    output.text(),
-    .unwrap();
-  assert!(!output.status.success());
-  assert!(String::from_utf8(output.stderr).unwrap().contains(
     "pledge test permissions called before restoring previous pledge"
-  ));
+  );
 }
 
 #[test]
@@ -208,14 +209,12 @@ fn file_protocol() {
     Url::from_file_path(util::testdata_path().join("bench/file_protocol.ts"))
       .unwrap()
       .to_string();
-
+  let context = TestContext::default();
-  (util::CheckOutputIntegrationTest {
+  let output = context
-    args_vec: vec!["bench", &file_url],
+    .new_command()
-    exit_code: 0,
+    .args(format!("bench bench/file_protocol.ts {file_url}"))
-    output: "bench/file_protocol.out",
+    .run();
-    ..Default::default()
+  assert_output_file!(output, "bench/file_protocol.out");
-  })
-  .run();
 }
 
 itest!(package_json_basic {

cli/tests/integration/cert_tests.rs

@@ -3,6 +3,7 @@
 use deno_runtime::deno_net::ops_tls::TlsStream;
 use deno_runtime::deno_tls::rustls;
 use deno_runtime::deno_tls::rustls_pemfile;
+use lsp_types::Url;
 use std::io::BufReader;
 use std::io::Cursor;
 use std::io::Read;
@@ -11,6 +12,9 @@ use std::sync::Arc;
 use test_util as util;
 use test_util::TempDir;
 use tokio::task::LocalSet;
+use util::assert_exit_code;
+use util::assert_output_text;
+use util::TestContext;
 
 itest_flaky!(cafile_url_imports {
   args: "run --quiet --reload --cert tls/RootCA.pem cert/cafile_url_imports.ts",
@@ -19,132 +23,113 @@ itest_flaky!(cafile_url_imports {
 });
 
 itest_flaky!(cafile_ts_fetch {
-    args:
+  args:
-      "run --quiet --reload --allow-net --cert tls/RootCA.pem cert/cafile_ts_fetch.ts",
+    "run --quiet --reload --allow-net --cert tls/RootCA.pem cert/cafile_ts_fetch.ts",
-    output: "cert/cafile_ts_fetch.ts.out",
+  output: "cert/cafile_ts_fetch.ts.out",
-    http_server: true,
+  http_server: true,
-  });
+});
 
 itest_flaky!(cafile_eval {
-    args: "eval --cert tls/RootCA.pem fetch('https://localhost:5545/cert/cafile_ts_fetch.ts.out').then(r=>r.text()).then(t=>console.log(t.trimEnd()))",
+  args: "eval --cert tls/RootCA.pem fetch('https://localhost:5545/cert/cafile_ts_fetch.ts.out').then(r=>r.text()).then(t=>console.log(t.trimEnd()))",
-    output: "cert/cafile_ts_fetch.ts.out",
+  output: "cert/cafile_ts_fetch.ts.out",
-    http_server: true,
+  http_server: true,
-  });
+});
 
 itest_flaky!(cafile_info {
-    args:
+  args:
-      "info --quiet --cert tls/RootCA.pem https://localhost:5545/cert/cafile_info.ts",
+    "info --quiet --cert tls/RootCA.pem https://localhost:5545/cert/cafile_info.ts",
-    output: "cert/cafile_info.ts.out",
+  output: "cert/cafile_info.ts.out",
-    http_server: true,
+  http_server: true,
-  });
+});
 
 itest_flaky!(cafile_url_imports_unsafe_ssl {
-    args: "run --quiet --reload --unsafely-ignore-certificate-errors=localhost cert/cafile_url_imports.ts",
+  args: "run --quiet --reload --unsafely-ignore-certificate-errors=localhost cert/cafile_url_imports.ts",
-    output: "cert/cafile_url_imports_unsafe_ssl.ts.out",
+  output: "cert/cafile_url_imports_unsafe_ssl.ts.out",
-    http_server: true,
+  http_server: true,
-  });
+});
 
 itest_flaky!(cafile_ts_fetch_unsafe_ssl {
-    args:
+  args:
-      "run --quiet --reload --allow-net --unsafely-ignore-certificate-errors cert/cafile_ts_fetch.ts",
+    "run --quiet --reload --allow-net --unsafely-ignore-certificate-errors cert/cafile_ts_fetch.ts",
-    output: "cert/cafile_ts_fetch_unsafe_ssl.ts.out",
+  output: "cert/cafile_ts_fetch_unsafe_ssl.ts.out",
-    http_server: true,
+  http_server: true,
-  });
+});
 
 // TODO(bartlomieju): reenable, this test was flaky on macOS CI during 1.30.3 release
 // itest!(deno_land_unsafe_ssl {
-//     args:
+//   args:
-//       "run --quiet --reload --allow-net --unsafely-ignore-certificate-errors=deno.land cert/deno_land_unsafe_ssl.ts",
+//     "run --quiet --reload --allow-net --unsafely-ignore-certificate-errors=deno.land cert/deno_land_unsafe_ssl.ts",
-//     output: "cert/deno_land_unsafe_ssl.ts.out",
+//   output: "cert/deno_land_unsafe_ssl.ts.out",
-//   });
+// });
 
 itest!(ip_address_unsafe_ssl {
-    args:
+  args:
-      "run --quiet --reload --allow-net --unsafely-ignore-certificate-errors=1.1.1.1 cert/ip_address_unsafe_ssl.ts",
+    "run --quiet --reload --allow-net --unsafely-ignore-certificate-errors=1.1.1.1 cert/ip_address_unsafe_ssl.ts",
-    output: "cert/ip_address_unsafe_ssl.ts.out",
+  output: "cert/ip_address_unsafe_ssl.ts.out",
-  });
+});
 
 itest!(localhost_unsafe_ssl {
-    args:
+  args: "run --quiet --reload --allow-net --unsafely-ignore-certificate-errors=deno.land cert/cafile_url_imports.ts",
-      "run --quiet --reload --allow-net --unsafely-ignore-certificate-errors=deno.land cert/cafile_url_imports.ts",
+  output: "cert/localhost_unsafe_ssl.ts.out",
-    output: "cert/localhost_unsafe_ssl.ts.out",
+  http_server: true,
-    http_server: true,
+  exit_code: 1,
-    exit_code: 1,
+});
-  });
 
 #[flaky_test::flaky_test]
 fn cafile_env_fetch() {
-  use deno_core::url::Url;
-  let _g = util::http_server();
-  let deno_dir = TempDir::new();
   let module_url =
     Url::parse("https://localhost:5545/cert/cafile_url_imports.ts").unwrap();
-  let cafile = util::testdata_path().join("tls/RootCA.pem");
+  let context = TestContext::with_http_server();
-  let output = Command::new(util::deno_exe_path())
+  let cafile = context.testdata_path().join("tls/RootCA.pem");
-    .env("DENO_DIR", deno_dir.path())
+  let output = context
-    .env("DENO_CERT", cafile)
+    .new_command()
-    .current_dir(util::testdata_path())
+    .args(format!("cache {module_url}"))
-    .arg("cache")
+    .env("DENO_CERT", cafile.to_string_lossy())
-    .arg(module_url.to_string())
+    .run();
-    .output()
+
-    .expect("Failed to spawn script");
+  assert_exit_code!(output, 0);
-  assert!(output.status.success());
+  output.skip_output_check();
 }
 
 #[flaky_test::flaky_test]
 fn cafile_fetch() {
-  use deno_core::url::Url;
-  let _g = util::http_server();
-  let deno_dir = TempDir::new();
   let module_url =
     Url::parse("http://localhost:4545/cert/cafile_url_imports.ts").unwrap();
-  let cafile = util::testdata_path().join("tls/RootCA.pem");
+  let context = TestContext::with_http_server();
-  let output = Command::new(util::deno_exe_path())
+  let cafile = context.testdata_path().join("tls/RootCA.pem");
-    .env("DENO_DIR", deno_dir.path())
+  let output = context
-    .current_dir(util::testdata_path())
+    .new_command()
-    .arg("cache")
+    .args(format!(
-    .arg("--cert")
+      "cache --quiet --cert {} {}",
-    .arg(cafile)
+      cafile.to_string_lossy(),
-    .arg(module_url.to_string())
+      module_url,
-    .output()
+    ))
-    .expect("Failed to spawn script");
+    .run();
-  assert!(output.status.success());
+
-  let out = std::str::from_utf8(&output.stdout).unwrap();
+  assert_exit_code!(output, 0);
-  assert_eq!(out, "");
+  assert_output_text!(output, "");
 }
 
 #[test]
 fn cafile_compile() {
-  let _g = util::http_server();
+  let context = TestContext::with_http_server();
-  let dir = TempDir::new();
+  let temp_dir = context.deno_dir().path();
-  let exe = if cfg!(windows) {
+  let output_exe = if cfg!(windows) {
-    dir.path().join("cert.exe")
+    temp_dir.join("cert.exe")
   } else {
-    dir.path().join("cert")
+    temp_dir.join("cert")
   };
-  let output = util::deno_cmd()
+  let output = context.new_command()
-    .current_dir(util::testdata_path())
+    .args(format!("compile --quiet --cert ./tls/RootCA.pem --allow-net --output {} ./cert/cafile_ts_fetch.ts", output_exe.to_string_lossy()))
-    .arg("compile")
+    .run();
-    .arg("--cert")
+  output.skip_output_check();
-    .arg("./tls/RootCA.pem")
+
-    .arg("--allow-net")
+  let exe_output = context
-    .arg("--output")
+    .new_command()
-    .arg(&exe)
+    .command_name(output_exe.to_string_lossy())
-    .arg("./cert/cafile_ts_fetch.ts")
+    .run();
-    .stdout(std::process::Stdio::piped())
+
-    .spawn()
+  assert_output_text!(exe_output, "[WILDCARD]\nHello\n");
-    .unwrap()
-    .wait_with_output()
-    .unwrap();
-  assert!(output.status.success());
-  let output = Command::new(exe)
-    .stdout(std::process::Stdio::piped())
-    .spawn()
-    .unwrap()
-    .wait_with_output()
-    .unwrap();
-  assert!(output.status.success());
-  assert_eq!(output.stdout, b"[WILDCARD]\nHello\n")
 }
 
 #[flaky_test::flaky_test]

cli/tests/integration/lint_tests.rs

@@ -1,26 +1,10 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
-use test_util as util;
+itest!(ignore_unexplicit_files {
-
+  args: "lint --unstable --ignore=./",
-#[test]
+  output_str: Some("error: No target files found.\n"),
-fn ignore_unexplicit_files() {
+  exit_code: 1,
-  let output = util::deno_cmd()
+});
-    .current_dir(util::root_path())
-    .env("NO_COLOR", "1")
-    .arg("lint")
-    .arg("--unstable")
-    .arg("--ignore=./")
-    .stderr(std::process::Stdio::piped())
-    .spawn()
-    .unwrap()
-    .wait_with_output()
-    .unwrap();
-  assert!(!output.status.success());
-  assert_eq!(
-    String::from_utf8_lossy(&output.stderr),
-    "error: No target files found.\n"
-  );
-}
 
 itest!(all {
   args: "lint lint/without_config/file1.js lint/without_config/file2.ts lint/without_config/ignored_file.ts",

cli/tests/integration/mod.rs

@@ -5,7 +5,65 @@ macro_rules! itest(
 ($name:ident {$( $key:ident: $value:expr,)*})  => {
   #[test]
   fn $name() {
-    (test_util::CheckOutputIntegrationTest {
+    let test = test_util::CheckOutputIntegrationTest {
+      $(
+        $key: $value,
+       )*
+      .. Default::default()
+    };
+    let output = test.output();
+    test_util::assert_exit_code!(output, test.exit_code);
+    if !test.output.is_empty() {
+      assert!(test.output_str.is_none());
+      test_util::assert_output_file!(output, test.output);
+    } else {
+      test_util::assert_output_text!(output, test.output_str.unwrap_or(""));
+    }
+  }
+}
+);
+
+#[macro_export]
+macro_rules! itest_flaky(
+($name:ident {$( $key:ident: $value:expr,)*})  => {
+  #[flaky_test::flaky_test]
+  fn $name() {
+    let test = test_util::CheckOutputIntegrationTest {
+      $(
+        $key: $value,
+       )*
+      .. Default::default()
+    };
+    let output = test.output();
+    test_util::assert_exit_code!(output, test.exit_code);
+    if !test.output.is_empty() {
+      assert!(test.output_str.is_none());
+      test_util::assert_output_file!(output, test.output);
+    } else {
+      test_util::assert_output_text!(output, test.output_str.unwrap_or(""));
+    }
+  }
+}
+);
+
+#[macro_export]
+macro_rules! context(
+({$( $key:ident: $value:expr,)*})  => {
+  test_util::TestContext::create(test_util::TestContextOptions {
+    $(
+      $key: $value,
+      )*
+    .. Default::default()
+  })
+}
+);
+
+#[macro_export]
+macro_rules! itest_steps(
+($name:ident {$( $key:ident: $value:expr,)*})  => {
+  #[test]
+  fn $name() {
+    (test_util::CheckOutputIntegrationTestSteps {
       $(
         $key: $value,
        )*
@@ -16,16 +74,13 @@ macro_rules! itest(
 );
 
 #[macro_export]
-macro_rules! itest_flaky(
+macro_rules! command_step(
-($name:ident {$( $key:ident: $value:expr,)*})  => {
+({$( $key:ident: $value:expr,)*})  => {
-  #[flaky_test::flaky_test]
+  test_util::CheckOutputIntegrationTestCommandStep {
-  fn $name() {
+    $(
-    (test_util::CheckOutputIntegrationTest {
+      $key: $value,
-      $(
+      )*
-        $key: $value,
+    .. Default::default()
-       )*
-      .. Default::default()
-    }).run()
   }
 }
 );

cli/tests/integration/test_tests.rs

@@ -2,7 +2,9 @@
 
 use deno_core::url::Url;
 use test_util as util;
+use util::assert_output_file;
 use util::env_vars_for_npm_tests;
+use util::TestContext;
 
 #[test]
 fn no_color() {
@@ -414,13 +416,9 @@ fn file_protocol() {
       .unwrap()
       .to_string();
 
-  (util::CheckOutputIntegrationTest {
+  let context = TestContext::default();
-    args_vec: vec!["test", &file_url],
+  let output = context.new_command().args(format!("test {file_url}")).run();
-    exit_code: 0,
+  assert_output_file!(output, "test/file_protocol.out");
-    output: "test/file_protocol.out",
-    ..Default::default()
-  })
-  .run();
 }
 
 itest!(uncaught_errors {

test_util/src/assertions.rs

@@ -39,3 +39,63 @@ macro_rules! assert_not_contains {
     }
   }
 }
+
+#[macro_export]
+macro_rules! assert_output_text {
+  ($output:expr, $expected:expr) => {
+    let expected_text = $expected;
+    let actual = $output.text();
+
+    if !expected_text.contains("[WILDCARD]") {
+      assert_eq!(actual, expected_text)
+    } else if !test_util::wildcard_match(&expected_text, actual) {
+      println!("OUTPUT START\n{}\nOUTPUT END", actual);
+      println!("EXPECTED START\n{expected_text}\nEXPECTED END");
+      panic!("pattern match failed");
+    }
+  };
+}
+
+#[macro_export]
+macro_rules! assert_output_file {
+  ($output:expr, $file_path:expr) => {
+    let output = &$output;
+    let output_path = output.testdata_dir().join($file_path);
+    println!("output path {}", output_path.display());
+    let expected_text =
+      std::fs::read_to_string(&output_path).unwrap_or_else(|err| {
+        panic!("failed loading {}\n\n{err:#}", output_path.display())
+      });
+    test_util::assert_output_text!(output, expected_text);
+  };
+}
+
+#[macro_export]
+macro_rules! assert_exit_code {
+  ($output:expr, $exit_code:expr) => {
+    let output = &$output;
+    let actual = output.text();
+    let expected_exit_code = $exit_code;
+    let actual_exit_code = output.exit_code();
+
+    if let Some(exit_code) = &actual_exit_code {
+      if *exit_code != expected_exit_code {
+        println!("OUTPUT\n{actual}\nOUTPUT");
+        panic!(
+          "bad exit code, expected: {:?}, actual: {:?}",
+          expected_exit_code, exit_code
+        );
+      }
+    } else {
+      println!("OUTPUT\n{actual}\nOUTPUT");
+      if let Some(signal) = output.signal() {
+        panic!(
+          "process terminated by signal, expected exit code: {:?}, actual signal: {:?}",
+          actual_exit_code, signal,
+        );
+      } else {
+        panic!("process terminated without status code on non unix platform, expected exit code: {:?}", actual_exit_code);
+      }
+    }
+  };
+}

test_util/src/builders.rs

@@ -0,0 +1,369 @@
+// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
+
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::io::Read;
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::Command;
+use std::process::Stdio;
+use std::rc::Rc;
+
+use os_pipe::pipe;
+
+use crate::copy_dir_recursive;
+use crate::deno_exe_path;
+use crate::http_server;
+use crate::new_deno_dir;
+use crate::strip_ansi_codes;
+use crate::testdata_path;
+use crate::HttpServerGuard;
+use crate::TempDir;
+
+#[derive(Default)]
+pub struct TestContextBuilder {
+  use_http_server: bool,
+  use_temp_cwd: bool,
+  /// Copies the files at the specified directory in the "testdata" directory
+  /// to the temp folder and runs the test from there. This is useful when
+  /// the test creates files in the testdata directory (ex. a node_modules folder)
+  copy_temp_dir: Option<String>,
+  cwd: Option<String>,
+  envs: HashMap<String, String>,
+}
+
+impl TestContextBuilder {
+  pub fn new() -> Self {
+    Self::default()
+  }
+
+  pub fn use_http_server(&mut self) -> &mut Self {
+    self.use_http_server = true;
+    self
+  }
+
+  pub fn use_temp_cwd(&mut self) -> &mut Self {
+    self.use_temp_cwd = true;
+    self
+  }
+
+  /// Copies the files at the specified directory in the "testdata" directory
+  /// to the temp folder and runs the test from there. This is useful when
+  /// the test creates files in the testdata directory (ex. a node_modules folder)
+  pub fn use_copy_temp_dir(&mut self, dir: impl AsRef<str>) {
+    self.copy_temp_dir = Some(dir.as_ref().to_string());
+  }
+
+  pub fn set_cwd(&mut self, cwd: impl AsRef<str>) -> &mut Self {
+    self.cwd = Some(cwd.as_ref().to_string());
+    self
+  }
+
+  pub fn env(
+    &mut self,
+    key: impl AsRef<str>,
+    value: impl AsRef<str>,
+  ) -> &mut Self {
+    self
+      .envs
+      .insert(key.as_ref().to_string(), value.as_ref().to_string());
+    self
+  }
+
+  pub fn build(&self) -> TestContext {
+    let deno_dir = new_deno_dir(); // keep this alive for the test
+    let testdata_dir = if let Some(temp_copy_dir) = &self.copy_temp_dir {
+      let test_data_path = testdata_path().join(temp_copy_dir);
+      let temp_copy_dir = deno_dir.path().join(temp_copy_dir);
+      std::fs::create_dir_all(&temp_copy_dir).unwrap();
+      copy_dir_recursive(&test_data_path, &temp_copy_dir).unwrap();
+      deno_dir.path().to_owned()
+    } else {
+      testdata_path()
+    };
+
+    let deno_exe = deno_exe_path();
+    println!("deno_exe path {}", deno_exe.display());
+
+    let http_server_guard = if self.use_http_server {
+      Some(Rc::new(http_server()))
+    } else {
+      None
+    };
+
+    TestContext {
+      cwd: self.cwd.clone(),
+      envs: self.envs.clone(),
+      use_temp_cwd: self.use_temp_cwd,
+      _http_server_guard: http_server_guard,
+      deno_dir,
+      testdata_dir,
+    }
+  }
+}
+
+#[derive(Clone)]
+pub struct TestContext {
+  envs: HashMap<String, String>,
+  use_temp_cwd: bool,
+  cwd: Option<String>,
+  _http_server_guard: Option<Rc<HttpServerGuard>>,
+  deno_dir: TempDir,
+  testdata_dir: PathBuf,
+}
+
+impl Default for TestContext {
+  fn default() -> Self {
+    TestContextBuilder::default().build()
+  }
+}
+
+impl TestContext {
+  pub fn with_http_server() -> Self {
+    TestContextBuilder::default().use_http_server().build()
+  }
+
+  pub fn testdata_path(&self) -> &PathBuf {
+    &self.testdata_dir
+  }
+
+  pub fn deno_dir(&self) -> &TempDir {
+    &self.deno_dir
+  }
+
+  pub fn new_command(&self) -> TestCommandBuilder {
+    TestCommandBuilder {
+      command_name: Default::default(),
+      args: Default::default(),
+      args_vec: Default::default(),
+      stdin: Default::default(),
+      envs: Default::default(),
+      env_clear: Default::default(),
+      cwd: Default::default(),
+      context: self.clone(),
+    }
+  }
+}
+
+pub struct TestCommandBuilder {
+  command_name: Option<String>,
+  args: String,
+  args_vec: Vec<String>,
+  stdin: Option<String>,
+  envs: HashMap<String, String>,
+  env_clear: bool,
+  cwd: Option<String>,
+  context: TestContext,
+}
+
+impl TestCommandBuilder {
+  pub fn command_name(&mut self, name: impl AsRef<str>) -> &mut Self {
+    self.command_name = Some(name.as_ref().to_string());
+    self
+  }
+
+  pub fn args(&mut self, text: impl AsRef<str>) -> &mut Self {
+    self.args = text.as_ref().to_string();
+    self
+  }
+
+  pub fn args_vec(&mut self, args: Vec<String>) -> &mut Self {
+    self.args_vec = args;
+    self
+  }
+
+  pub fn stdin(&mut self, text: impl AsRef<str>) -> &mut Self {
+    self.stdin = Some(text.as_ref().to_string());
+    self
+  }
+
+  pub fn env(
+    &mut self,
+    key: impl AsRef<str>,
+    value: impl AsRef<str>,
+  ) -> &mut Self {
+    self
+      .envs
+      .insert(key.as_ref().to_string(), value.as_ref().to_string());
+    self
+  }
+
+  pub fn env_clear(&mut self) -> &mut Self {
+    self.env_clear = true;
+    self
+  }
+
+  pub fn cwd(&mut self, cwd: impl AsRef<str>) -> &mut Self {
+    self.cwd = Some(cwd.as_ref().to_string());
+    self
+  }
+
+  pub fn run(&self) -> TestCommandOutput {
+    let cwd = self.cwd.as_ref().or(self.context.cwd.as_ref());
+    let cwd = if self.context.use_temp_cwd {
+      assert!(cwd.is_none());
+      self.context.deno_dir.path().to_owned()
+    } else if let Some(cwd_) = cwd {
+      self.context.testdata_dir.join(cwd_)
+    } else {
+      self.context.testdata_dir.clone()
+    };
+    let args = if self.args_vec.is_empty() {
+      std::borrow::Cow::Owned(
+        self
+          .args
+          .split_whitespace()
+          .map(|s| s.to_string())
+          .collect::<Vec<_>>(),
+      )
+    } else {
+      assert!(
+        self.args.is_empty(),
+        "Do not provide args when providing args_vec."
+      );
+      std::borrow::Cow::Borrowed(&self.args_vec)
+    }
+    .iter()
+    .map(|arg| {
+      arg.replace("$TESTDATA", &self.context.testdata_dir.to_string_lossy())
+    })
+    .collect::<Vec<_>>();
+    let (mut reader, writer) = pipe().unwrap();
+    let command_name = self
+      .command_name
+      .as_ref()
+      .cloned()
+      .unwrap_or("deno".to_string());
+    let mut command = if command_name == "deno" {
+      Command::new(deno_exe_path())
+    } else {
+      Command::new(&command_name)
+    };
+    command.env("DENO_DIR", self.context.deno_dir.path());
+
+    println!("command {} {}", command_name, args.join(" "));
+    println!("command cwd {:?}", &cwd);
+    command.args(args.iter());
+    if self.env_clear {
+      command.env_clear();
+    }
+    command.envs({
+      let mut envs = self.context.envs.clone();
+      for (key, value) in &self.envs {
+        envs.insert(key.to_string(), value.to_string());
+      }
+      envs
+    });
+    command.current_dir(cwd);
+    command.stdin(Stdio::piped());
+    let writer_clone = writer.try_clone().unwrap();
+    command.stderr(writer_clone);
+    command.stdout(writer);
+
+    let mut process = command.spawn().unwrap();
+
+    if let Some(input) = &self.stdin {
+      let mut p_stdin = process.stdin.take().unwrap();
+      write!(p_stdin, "{input}").unwrap();
+    }
+
+    // This parent process is still holding its copies of the write ends,
+    // and we have to close them before we read, otherwise the read end
+    // will never report EOF. The Command object owns the writers now,
+    // and dropping it closes them.
+    drop(command);
+
+    let mut actual = String::new();
+    reader.read_to_string(&mut actual).unwrap();
+
+    let status = process.wait().expect("failed to finish process");
+    let exit_code = status.code();
+    #[cfg(unix)]
+    let signal = {
+      use std::os::unix::process::ExitStatusExt;
+      status.signal()
+    };
+    #[cfg(not(unix))]
+    let signal = None;
+
+    actual = strip_ansi_codes(&actual).to_string();
+
+    // deno test's output capturing flushes with a zero-width space in order to
+    // synchronize the output pipes. Occassionally this zero width space
+    // might end up in the output so strip it from the output comparison here.
+    if args.first().map(|s| s.as_str()) == Some("test") {
+      actual = actual.replace('\u{200B}', "");
+    }
+
+    TestCommandOutput {
+      exit_code,
+      signal,
+      text: actual,
+      testdata_dir: self.context.testdata_dir.clone(),
+      asserted_exit_code: RefCell::new(false),
+      asserted_text: RefCell::new(false),
+      _test_context: self.context.clone(),
+    }
+  }
+}
+
+pub struct TestCommandOutput {
+  text: String,
+  exit_code: Option<i32>,
+  signal: Option<i32>,
+  testdata_dir: PathBuf,
+  asserted_text: RefCell<bool>,
+  asserted_exit_code: RefCell<bool>,
+  // keep alive for the duration of the output reference
+  _test_context: TestContext,
+}
+
+impl Drop for TestCommandOutput {
+  fn drop(&mut self) {
+    if std::thread::panicking() {
+      return;
+    }
+    // force the caller to assert these
+    if !*self.asserted_exit_code.borrow() && self.exit_code != Some(0) {
+      panic!(
+        "The non-zero exit code of the command was not asserted: {:?}.",
+        self.exit_code
+      )
+    }
+    if !*self.asserted_text.borrow() && !self.text.is_empty() {
+      println!("OUTPUT\n{}\nOUTPUT", self.text);
+      panic!(concat!(
+        "The non-empty text of the command was not asserted. ",
+        "Call `output.skip_output_check()` to skip if necessary.",
+      ));
+    }
+  }
+}
+
+impl TestCommandOutput {
+  pub fn testdata_dir(&self) -> &PathBuf {
+    &self.testdata_dir
+  }
+
+  pub fn skip_output_check(&self) {
+    *self.asserted_text.borrow_mut() = true;
+  }
+
+  pub fn skip_exit_code_check(&self) {
+    *self.asserted_exit_code.borrow_mut() = true;
+  }
+
+  pub fn exit_code(&self) -> Option<i32> {
+    self.skip_exit_code_check();
+    self.exit_code
+  }
+
+  pub fn signal(&self) -> Option<i32> {
+    self.signal
+  }
+
+  pub fn text(&self) -> &str {
+    self.skip_output_check();
+    &self.text
+  }
+}

test_util/src/lib.rs

@@ -15,7 +15,6 @@ use hyper::Response;
 use hyper::StatusCode;
 use lazy_static::lazy_static;
 use npm::CUSTOM_NPM_PACKAGE_CACHE;
-use os_pipe::pipe;
 use pretty_assertions::assert_eq;
 use regex::Regex;
 use rustls::Certificate;
@@ -54,11 +53,16 @@ use tokio_tungstenite::accept_async;
 use url::Url;
 
 pub mod assertions;
+mod builders;
 pub mod lsp;
 mod npm;
 pub mod pty;
 mod temp_dir;
 
+pub use builders::TestCommandBuilder;
+pub use builders::TestCommandOutput;
+pub use builders::TestContext;
+pub use builders::TestContextBuilder;
 pub use temp_dir::TempDir;
 
 const PORT: u16 = 4545;
@@ -1948,131 +1952,43 @@ pub struct CheckOutputIntegrationTest<'a> {
 }
 
 impl<'a> CheckOutputIntegrationTest<'a> {
-  pub fn run(&self) {
+  pub fn output(&self) -> TestCommandOutput {
-    let deno_dir = new_deno_dir(); // keep this alive for the test
+    let mut context_builder = TestContextBuilder::default();
-    let args = if self.args_vec.is_empty() {
+    if self.temp_cwd {
-      std::borrow::Cow::Owned(self.args.split_whitespace().collect::<Vec<_>>())
+      context_builder.use_temp_cwd();
-    } else {
+    }
-      assert!(
+    if let Some(dir) = &self.copy_temp_dir {
-        self.args.is_empty(),
+      context_builder.use_copy_temp_dir(dir);
-        "Do not provide args when providing args_vec."
+    }
-      );
+    if self.http_server {
-      std::borrow::Cow::Borrowed(&self.args_vec)
+      context_builder.use_http_server();
-    };
+    }
-    let testdata_dir = if let Some(temp_copy_dir) = &self.copy_temp_dir {
-      let test_data_path = testdata_path().join(temp_copy_dir);
-      let temp_copy_dir = deno_dir.path().join(temp_copy_dir);
-      std::fs::create_dir_all(&temp_copy_dir).unwrap();
-      copy_dir_recursive(&test_data_path, &temp_copy_dir).unwrap();
-      deno_dir.path().to_owned()
-    } else {
-      testdata_path()
-    };
-    let args = args
-      .iter()
-      .map(|arg| arg.replace("$TESTDATA", &testdata_dir.to_string_lossy()))
-      .collect::<Vec<_>>();
-    let deno_exe = deno_exe_path();
-    println!("deno_exe path {}", deno_exe.display());
 
-    let _http_server_guard = if self.http_server {
+    let context = context_builder.build();
-      Some(http_server())
-    } else {
-      None
-    };
 
-    let (mut reader, writer) = pipe().unwrap();
+    let mut command_builder = context.new_command();
-    let mut command = deno_cmd_with_deno_dir(&deno_dir);
+
-    let cwd = if self.temp_cwd {
+    if !self.args.is_empty() {
-      deno_dir.path().to_owned()
+      command_builder.args(self.args);
-    } else if let Some(cwd_) = &self.cwd {
+    }
-      testdata_dir.join(cwd_)
+    if !self.args_vec.is_empty() {
-    } else {
+      command_builder
-      testdata_dir.clone()
+        .args_vec(self.args_vec.iter().map(|a| a.to_string()).collect());
-    };
+    }
-    println!("deno_exe args {}", args.join(" "));
+    if let Some(input) = &self.input {
-    println!("deno_exe cwd {:?}", &cwd);
+      command_builder.stdin(input);
-    command.args(args.iter());
+    }
+    for (key, value) in &self.envs {
+      command_builder.env(key, value);
+    }
     if self.env_clear {
-      command.env_clear();
+      command_builder.env_clear();
     }
-    command.envs(self.envs.clone());
+    if let Some(cwd) = &self.cwd {
-    command.current_dir(cwd);
+      command_builder.cwd(cwd);
-    command.stdin(Stdio::piped());
-    let writer_clone = writer.try_clone().unwrap();
-    command.stderr(writer_clone);
-    command.stdout(writer);
-
-    let mut process = command.spawn().expect("failed to execute process");
-
-    if let Some(input) = self.input {
-      let mut p_stdin = process.stdin.take().unwrap();
-      write!(p_stdin, "{input}").unwrap();
     }
 
-    // Very important when using pipes: This parent process is still
+    command_builder.run()
-    // holding its copies of the write ends, and we have to close them
-    // before we read, otherwise the read end will never report EOF. The
-    // Command object owns the writers now, and dropping it closes them.
-    drop(command);
-
-    let mut actual = String::new();
-    reader.read_to_string(&mut actual).unwrap();
-
-    let status = process.wait().expect("failed to finish process");
-
-    if let Some(exit_code) = status.code() {
-      if self.exit_code != exit_code {
-        println!("OUTPUT\n{actual}\nOUTPUT");
-        panic!(
-          "bad exit code, expected: {:?}, actual: {:?}",
-          self.exit_code, exit_code
-        );
-      }
-    } else {
-      #[cfg(unix)]
-      {
-        use std::os::unix::process::ExitStatusExt;
-        let signal = status.signal().unwrap();
-        println!("OUTPUT\n{actual}\nOUTPUT");
-        panic!(
-          "process terminated by signal, expected exit code: {:?}, actual signal: {:?}",
-          self.exit_code, signal,
-        );
-      }
-      #[cfg(not(unix))]
-      {
-        println!("OUTPUT\n{actual}\nOUTPUT");
-        panic!("process terminated without status code on non unix platform, expected exit code: {:?}", self.exit_code);
-      }
-    }
-
-    actual = strip_ansi_codes(&actual).to_string();
-
-    // deno test's output capturing flushes with a zero-width space in order to
-    // synchronize the output pipes. Occassionally this zero width space
-    // might end up in the output so strip it from the output comparison here.
-    if args.first().map(|s| s.as_str()) == Some("test") {
-      actual = actual.replace('\u{200B}', "");
-    }
-
-    let expected = if let Some(s) = self.output_str {
-      s.to_owned()
-    } else if self.output.is_empty() {
-      String::new()
-    } else {
-      let output_path = testdata_dir.join(self.output);
-      println!("output path {}", output_path.display());
-      std::fs::read_to_string(output_path).expect("cannot read output")
-    };
-
-    if !expected.contains("[WILDCARD]") {
-      assert_eq!(actual, expected)
-    } else if !wildcard_match(&expected, &actual) {
-      println!("OUTPUT\n{actual}\nOUTPUT");
-      println!("EXPECTED\n{expected}\nEXPECTED");
-      panic!("pattern match failed");
-    }
   }
 }