mark cafile tests as flaky (#11649)

Cargo.lock

@@ -596,6 +596,7 @@ dependencies = [
  "exec",
  "fancy-regex",
  "filetime",
+ "flaky_test",
  "fwdansi",
  "http",
  "indexmap",
@@ -1257,6 +1258,17 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
 
+[[package]]
+name = "flaky_test"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "479cde5eb168cf5a056dd98f311cbfab7494c216394e4fb9eba0336827a8db93"
+dependencies = [
+ "proc-macro2 1.0.27",
+ "quote 1.0.9",
+ "syn 1.0.65",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.20"

cli/Cargo.toml

@@ -101,6 +101,7 @@ winapi = { version = "0.3.9", features = ["knownfolders", "mswsock", "objbase",
 [dev-dependencies]
 # Used in benchmark
 chrono = "0.4.19"
+flaky_test = "0.1.0"
 os_pipe = "0.9.2"
 test_util = { path = "../test_util" }
 tower-test = "0.4.0"

cli/tests/integration/mod.rs

@@ -30,6 +30,21 @@ macro_rules! itest(
 }
 );
 
+#[macro_export]
+macro_rules! itest_flaky(
+($name:ident {$( $key:ident: $value:expr,)*})  => {
+  #[flaky_test::flaky_test]
+  fn $name() {
+    (test_util::CheckOutputIntegrationTest {
+      $(
+        $key: $value,
+       )*
+      .. Default::default()
+    }).run()
+  }
+}
+);
+
 // These files have `_tests.rs` suffix to make it easier to tell which file is
 // the test (ex. `lint_tests.rs`) and which is the implementation (ex. `lint.rs`)
 // when both are open, especially for two tabs in VS Code
@@ -445,39 +460,39 @@ fn broken_stdout() {
   assert!(!stderr.contains("panic"));
 }
 
-itest!(cafile_url_imports {
+itest_flaky!(cafile_url_imports {
   args: "run --quiet --reload --cert tls/RootCA.pem cafile_url_imports.ts",
   output: "cafile_url_imports.ts.out",
   http_server: true,
 });
 
-itest!(cafile_ts_fetch {
+itest_flaky!(cafile_ts_fetch {
   args:
     "run --quiet --reload --allow-net --cert tls/RootCA.pem cafile_ts_fetch.ts",
   output: "cafile_ts_fetch.ts.out",
   http_server: true,
 });
 
-itest!(cafile_eval {
+itest_flaky!(cafile_eval {
   args: "eval --cert tls/RootCA.pem fetch('https://localhost:5545/cafile_ts_fetch.ts.out').then(r=>r.text()).then(t=>console.log(t.trimEnd()))",
   output: "cafile_ts_fetch.ts.out",
   http_server: true,
 });
 
-itest!(cafile_info {
+itest_flaky!(cafile_info {
   args:
     "info --quiet --cert tls/RootCA.pem https://localhost:5545/cafile_info.ts",
   output: "cafile_info.ts.out",
   http_server: true,
 });
 
-itest!(cafile_url_imports_unsafe_ssl {
+itest_flaky!(cafile_url_imports_unsafe_ssl {
   args: "run --quiet --reload --unsafely-ignore-certificate-errors=localhost cafile_url_imports.ts",
   output: "cafile_url_imports_unsafe_ssl.ts.out",
   http_server: true,
 });
 
-itest!(cafile_ts_fetch_unsafe_ssl {
+itest_flaky!(cafile_ts_fetch_unsafe_ssl {
   args:
     "run --quiet --reload --allow-net --unsafely-ignore-certificate-errors cafile_ts_fetch.ts",
   output: "cafile_ts_fetch_unsafe_ssl.ts.out",
@@ -498,7 +513,7 @@ itest!(localhost_unsafe_ssl {
   exit_code: 1,
 });
 
-#[test]
+#[flaky_test::flaky_test]
 fn cafile_env_fetch() {
   use deno_core::url::Url;
   let _g = util::http_server();
@@ -517,7 +532,7 @@ fn cafile_env_fetch() {
   assert!(output.status.success());
 }
 
-#[test]
+#[flaky_test::flaky_test]
 fn cafile_fetch() {
   use deno_core::url::Url;
   let _g = util::http_server();
@@ -539,7 +554,7 @@ fn cafile_fetch() {
   assert_eq!(out, "");
 }
 
-#[test]
+#[flaky_test::flaky_test]
 fn cafile_install_remote_module() {
   let _g = util::http_server();
   let temp_dir = TempDir::new().expect("tempdir fail");
@@ -581,7 +596,7 @@ fn cafile_install_remote_module() {
   assert!(stdout.ends_with("foo"));
 }
 
-#[test]
+#[flaky_test::flaky_test]
 fn cafile_bundle_remote_exports() {
   let _g = util::http_server();