feat(cli): evaluate code snippets in JSDoc and markdown (#25220)

This commit lets `deno test --doc` command actually evaluate code snippets in
JSDoc and markdown files.

## How it works

1. Extract code snippets from JSDoc or code fences
2. Convert them into pseudo files by wrapping them in `Deno.test(...)`
3. Register the pseudo files as in-memory files
4. Run type-check and evaluation

We apply some magic at the step 2 - let's say we have the following file named
`mod.ts` as an input:

````ts
/**
 * ```ts
 * import { assertEquals } from "jsr:@std/assert/equals";
 *
 * assertEquals(add(1, 2), 3);
 * ```
 */
export function add(a: number, b: number) {
  return a + b;
}
````

This is virtually transformed into:

```ts
import { assertEquals } from "jsr:@std/assert/equals";
import { add } from "files:///path/to/mod.ts";

Deno.test("mod.ts$2-7.ts", async () => {
  assertEquals(add(1, 2), 3);
});
```

Note that a new import statement is inserted here to make `add` function
available. In a nutshell, all items exported from `mod.ts` become available in
the generated pseudo file with this automatic import insertion.

The intention behind this design is that, from library user's standpoint, it
should be very obvious that this `add` function is what this example code is
attached to. Also, if there is an explicit import statement like
`import { add } from "./mod.ts"`, this import path `./mod.ts` is not helpful for
doc readers because they will need to import it in a different way.

The automatic import insertion has some edge cases, in particular where there is
a local variable in a snippet with the same name as one of the exported items.
This case is addressed by employing swc's scope analysis (see test cases for
more details).

## "type-checking only" mode stays around

This change will likely impact a lot of existing doc tests in the ecosystem
because some doc tests rely on the fact that they are not evaluated - some cause
side effects if executed, some throw errors at runtime although they do pass the
type check, etc. To help those tests gradually transition to the ones runnable
with the new `deno test --doc`, we will keep providing the ability to run
type-checking only via `deno check --doc`. Additionally there is a `--doc-only`
option added to the `check` subcommand too, which is useful when you want to
type-check on code snippets in markdown files, as normal `deno check` command
doesn't accept markdown.

## Demo

https://github.com/user-attachments/assets/47e9af73-d16e-472d-b09e-1853b9e8f5ce

---

Closes #4716

cli/args/flags.rs

@@ -109,6 +109,8 @@ pub struct CacheFlags {
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct CheckFlags {
   pub files: Vec<String>,
+  pub doc: bool,
+  pub doc_only: bool,
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -1694,6 +1696,19 @@ Unless --reload is specified, this command will not re-download already cached d
             .conflicts_with("no-remote")
             .hide(true)
         )
+        .arg(
+          Arg::new("doc")
+            .long("doc")
+            .help("Type-check code blocks in JSDoc as well as actual code")
+            .action(ArgAction::SetTrue)
+        )
+        .arg(
+          Arg::new("doc-only")
+          .long("doc-only")
+          .help("Type-check code blocks in JSDoc and Markdown only")
+            .action(ArgAction::SetTrue)
+            .conflicts_with("doc")
+        )
         .arg(
           Arg::new("file")
             .num_args(1..)
@@ -2789,7 +2804,7 @@ or <c>**/__tests__/**</>:
       .arg(
         Arg::new("doc")
           .long("doc")
-          .help("Type-check code blocks in JSDoc and Markdown")
+          .help("Evaluate code blocks in JSDoc and Markdown")
           .action(ArgAction::SetTrue)
           .help_heading(TEST_HEADING),
       )
@@ -4121,7 +4136,11 @@ fn check_parse(
   if matches.get_flag("all") || matches.get_flag("remote") {
     flags.type_check_mode = TypeCheckMode::All;
   }
-  flags.subcommand = DenoSubcommand::Check(CheckFlags { files });
+  flags.subcommand = DenoSubcommand::Check(CheckFlags {
+    files,
+    doc: matches.get_flag("doc"),
+    doc_only: matches.get_flag("doc-only"),
+  });
   Ok(())
 }
 
@@ -6862,12 +6881,55 @@ mod tests {
       Flags {
         subcommand: DenoSubcommand::Check(CheckFlags {
           files: svec!["script.ts"],
+          doc: false,
+          doc_only: false,
         }),
         type_check_mode: TypeCheckMode::Local,
         ..Flags::default()
       }
     );
 
+    let r = flags_from_vec(svec!["deno", "check", "--doc", "script.ts"]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Check(CheckFlags {
+          files: svec!["script.ts"],
+          doc: true,
+          doc_only: false,
+        }),
+        type_check_mode: TypeCheckMode::Local,
+        ..Flags::default()
+      }
+    );
+
+    let r = flags_from_vec(svec!["deno", "check", "--doc-only", "markdown.md"]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Check(CheckFlags {
+          files: svec!["markdown.md"],
+          doc: false,
+          doc_only: true,
+        }),
+        type_check_mode: TypeCheckMode::Local,
+        ..Flags::default()
+      }
+    );
+
+    // `--doc` and `--doc-only` are mutually exclusive
+    let r = flags_from_vec(svec![
+      "deno",
+      "check",
+      "--doc",
+      "--doc-only",
+      "script.ts"
+    ]);
+    assert_eq!(
+      r.unwrap_err().kind(),
+      clap::error::ErrorKind::ArgumentConflict
+    );
+
     for all_flag in ["--remote", "--all"] {
       let r = flags_from_vec(svec!["deno", "check", all_flag, "script.ts"]);
       assert_eq!(
@@ -6875,6 +6937,8 @@ mod tests {
         Flags {
           subcommand: DenoSubcommand::Check(CheckFlags {
             files: svec!["script.ts"],
+            doc: false,
+            doc_only: false,
           }),
           type_check_mode: TypeCheckMode::All,
           ..Flags::default()

cli/lsp/testing/execution.rs

@@ -234,15 +234,8 @@ impl TestRun {
       &cli_options.permissions_options(),
     )?;
     let main_graph_container = factory.main_module_graph_container().await?;
-    test::check_specifiers(
-      factory.file_fetcher()?,
-      main_graph_container,
-      self
-        .queue
-        .iter()
-        .map(|s| (s.clone(), test::TestMode::Executable))
-        .collect(),
-    )
+    main_graph_container
+      .check_specifiers(&self.queue.iter().cloned().collect::<Vec<_>>())
       .await?;
 
     let (concurrent_jobs, fail_fast) =

cli/main.rs

@@ -121,12 +121,7 @@ async fn run_subcommand(flags: Arc<Flags>) -> Result<i32, AnyError> {
       tools::installer::install_from_entrypoints(flags, &cache_flags.files).await
     }),
     DenoSubcommand::Check(check_flags) => spawn_subcommand(async move {
-      let factory = CliFactory::from_flags(flags);
-      let main_graph_container =
-        factory.main_module_graph_container().await?;
-      main_graph_container
-        .load_and_type_check_files(&check_flags.files)
-        .await
+      tools::check::check(flags, check_flags).await
     }),
     DenoSubcommand::Clean => spawn_subcommand(async move {
       tools::clean::clean()

cli/tools/check.rs

@@ -15,7 +15,9 @@ use once_cell::sync::Lazy;
 use regex::Regex;
 
 use crate::args::check_warn_tsconfig;
+use crate::args::CheckFlags;
 use crate::args::CliOptions;
+use crate::args::Flags;
 use crate::args::TsConfig;
 use crate::args::TsConfigType;
 use crate::args::TsTypeLib;
@@ -24,13 +26,57 @@ use crate::cache::CacheDBHash;
 use crate::cache::Caches;
 use crate::cache::FastInsecureHasher;
 use crate::cache::TypeCheckCache;
+use crate::factory::CliFactory;
 use crate::graph_util::BuildFastCheckGraphOptions;
 use crate::graph_util::ModuleGraphBuilder;
 use crate::npm::CliNpmResolver;
 use crate::tsc;
 use crate::tsc::Diagnostics;
+use crate::util::extract;
 use crate::util::path::to_percent_decoded_str;
 
+pub async fn check(
+  flags: Arc<Flags>,
+  check_flags: CheckFlags,
+) -> Result<(), AnyError> {
+  let factory = CliFactory::from_flags(flags);
+
+  let main_graph_container = factory.main_module_graph_container().await?;
+
+  let specifiers =
+    main_graph_container.collect_specifiers(&check_flags.files)?;
+  if specifiers.is_empty() {
+    log::warn!("{} No matching files found.", colors::yellow("Warning"));
+  }
+
+  let specifiers_for_typecheck = if check_flags.doc || check_flags.doc_only {
+    let file_fetcher = factory.file_fetcher()?;
+
+    let mut specifiers_for_typecheck = if check_flags.doc {
+      specifiers.clone()
+    } else {
+      vec![]
+    };
+
+    for s in specifiers {
+      let file = file_fetcher.fetch_bypass_permissions(&s).await?;
+      let snippet_files = extract::extract_snippet_files(file)?;
+      for snippet_file in snippet_files {
+        specifiers_for_typecheck.push(snippet_file.specifier.clone());
+        file_fetcher.insert_memory_files(snippet_file);
+      }
+    }
+
+    specifiers_for_typecheck
+  } else {
+    specifiers
+  };
+
+  main_graph_container
+    .check_specifiers(&specifiers_for_typecheck)
+    .await
+}
+
 /// Options for performing a check of a module graph. Note that the decision to
 /// emit or not is determined by the `ts_config` settings.
 pub struct CheckOptions {

cli/tools/test/mod.rs

@@ -9,21 +9,18 @@ use crate::display;
 use crate::factory::CliFactory;
 use crate::file_fetcher::File;
 use crate::file_fetcher::FileFetcher;
-use crate::graph_container::MainModuleGraphContainer;
 use crate::graph_util::has_graph_root_local_dependent_changed;
 use crate::ops;
+use crate::util::extract::extract_doc_tests;
 use crate::util::file_watcher;
 use crate::util::fs::collect_specifiers;
 use crate::util::path::get_extension;
 use crate::util::path::is_script_ext;
-use crate::util::path::mapped_specifier_for_tsc;
 use crate::util::path::matches_pattern_or_exact_path;
 use crate::worker::CliMainWorkerFactory;
 use crate::worker::CoverageCollector;
 
-use deno_ast::swc::common::comments::CommentKind;
 use deno_ast::MediaType;
-use deno_ast::SourceRangedForSpanned;
 use deno_config::glob::FilePatterns;
 use deno_config::glob::WalkEntry;
 use deno_core::anyhow;
@@ -151,6 +148,20 @@ pub enum TestMode {
   Both,
 }
 
+impl TestMode {
+  /// Returns `true` if the test mode indicates that code snippet extraction is
+  /// needed.
+  fn needs_test_extraction(&self) -> bool {
+    matches!(self, Self::Documentation | Self::Both)
+  }
+
+  /// Returns `true` if the test mode indicates that the test should be
+  /// type-checked and run.
+  fn needs_test_run(&self) -> bool {
+    matches!(self, Self::Executable | Self::Both)
+  }
+}
+
 #[derive(Clone, Debug, Default)]
 pub struct TestFilter {
   pub substring: Option<String>,
@@ -1174,233 +1185,6 @@ async fn wait_for_activity_to_stabilize(
   })
 }
 
-fn extract_files_from_regex_blocks(
-  specifier: &ModuleSpecifier,
-  source: &str,
-  media_type: MediaType,
-  file_line_index: usize,
-  blocks_regex: &Regex,
-  lines_regex: &Regex,
-) -> Result<Vec<File>, AnyError> {
-  let files = blocks_regex
-    .captures_iter(source)
-    .filter_map(|block| {
-      block.get(1)?;
-
-      let maybe_attributes: Option<Vec<_>> = block
-        .get(1)
-        .map(|attributes| attributes.as_str().split(' ').collect());
-
-      let file_media_type = if let Some(attributes) = maybe_attributes {
-        if attributes.contains(&"ignore") {
-          return None;
-        }
-
-        match attributes.first() {
-          Some(&"js") => MediaType::JavaScript,
-          Some(&"javascript") => MediaType::JavaScript,
-          Some(&"mjs") => MediaType::Mjs,
-          Some(&"cjs") => MediaType::Cjs,
-          Some(&"jsx") => MediaType::Jsx,
-          Some(&"ts") => MediaType::TypeScript,
-          Some(&"typescript") => MediaType::TypeScript,
-          Some(&"mts") => MediaType::Mts,
-          Some(&"cts") => MediaType::Cts,
-          Some(&"tsx") => MediaType::Tsx,
-          _ => MediaType::Unknown,
-        }
-      } else {
-        media_type
-      };
-
-      if file_media_type == MediaType::Unknown {
-        return None;
-      }
-
-      let line_offset = source[0..block.get(0).unwrap().start()]
-        .chars()
-        .filter(|c| *c == '\n')
-        .count();
-
-      let line_count = block.get(0).unwrap().as_str().split('\n').count();
-
-      let body = block.get(2).unwrap();
-      let text = body.as_str();
-
-      // TODO(caspervonb) generate an inline source map
-      let mut file_source = String::new();
-      for line in lines_regex.captures_iter(text) {
-        let text = line.get(1).unwrap();
-        writeln!(file_source, "{}", text.as_str()).unwrap();
-      }
-
-      let file_specifier = ModuleSpecifier::parse(&format!(
-        "{}${}-{}",
-        specifier,
-        file_line_index + line_offset + 1,
-        file_line_index + line_offset + line_count + 1,
-      ))
-      .unwrap();
-      let file_specifier =
-        mapped_specifier_for_tsc(&file_specifier, file_media_type)
-          .map(|s| ModuleSpecifier::parse(&s).unwrap())
-          .unwrap_or(file_specifier);
-
-      Some(File {
-        specifier: file_specifier,
-        maybe_headers: None,
-        source: file_source.into_bytes().into(),
-      })
-    })
-    .collect();
-
-  Ok(files)
-}
-
-fn extract_files_from_source_comments(
-  specifier: &ModuleSpecifier,
-  source: Arc<str>,
-  media_type: MediaType,
-) -> Result<Vec<File>, AnyError> {
-  let parsed_source = deno_ast::parse_module(deno_ast::ParseParams {
-    specifier: specifier.clone(),
-    text: source,
-    media_type,
-    capture_tokens: false,
-    maybe_syntax: None,
-    scope_analysis: false,
-  })?;
-  let comments = parsed_source.comments().get_vec();
-  let blocks_regex = lazy_regex::regex!(r"```([^\r\n]*)\r?\n([\S\s]*?)```");
-  let lines_regex = lazy_regex::regex!(r"(?:\* ?)(?:\# ?)?(.*)");
-
-  let files = comments
-    .iter()
-    .filter(|comment| {
-      if comment.kind != CommentKind::Block || !comment.text.starts_with('*') {
-        return false;
-      }
-
-      true
-    })
-    .flat_map(|comment| {
-      extract_files_from_regex_blocks(
-        specifier,
-        &comment.text,
-        media_type,
-        parsed_source.text_info_lazy().line_index(comment.start()),
-        blocks_regex,
-        lines_regex,
-      )
-    })
-    .flatten()
-    .collect();
-
-  Ok(files)
-}
-
-fn extract_files_from_fenced_blocks(
-  specifier: &ModuleSpecifier,
-  source: &str,
-  media_type: MediaType,
-) -> Result<Vec<File>, AnyError> {
-  // The pattern matches code blocks as well as anything in HTML comment syntax,
-  // but it stores the latter without any capturing groups. This way, a simple
-  // check can be done to see if a block is inside a comment (and skip typechecking)
-  // or not by checking for the presence of capturing groups in the matches.
-  let blocks_regex =
-    lazy_regex::regex!(r"(?s)<!--.*?-->|```([^\r\n]*)\r?\n([\S\s]*?)```");
-  let lines_regex = lazy_regex::regex!(r"(?:\# ?)?(.*)");
-
-  extract_files_from_regex_blocks(
-    specifier,
-    source,
-    media_type,
-    /* file line index */ 0,
-    blocks_regex,
-    lines_regex,
-  )
-}
-
-async fn fetch_inline_files(
-  file_fetcher: &FileFetcher,
-  specifiers: Vec<ModuleSpecifier>,
-) -> Result<Vec<File>, AnyError> {
-  let mut files = Vec::new();
-  for specifier in specifiers {
-    let file = file_fetcher
-      .fetch_bypass_permissions(&specifier)
-      .await?
-      .into_text_decoded()?;
-
-    let inline_files = if file.media_type == MediaType::Unknown {
-      extract_files_from_fenced_blocks(
-        &file.specifier,
-        &file.source,
-        file.media_type,
-      )
-    } else {
-      extract_files_from_source_comments(
-        &file.specifier,
-        file.source,
-        file.media_type,
-      )
-    };
-
-    files.extend(inline_files?);
-  }
-
-  Ok(files)
-}
-
-/// Type check a collection of module and document specifiers.
-pub async fn check_specifiers(
-  file_fetcher: &FileFetcher,
-  main_graph_container: &Arc<MainModuleGraphContainer>,
-  specifiers: Vec<(ModuleSpecifier, TestMode)>,
-) -> Result<(), AnyError> {
-  let inline_files = fetch_inline_files(
-    file_fetcher,
-    specifiers
-      .iter()
-      .filter_map(|(specifier, mode)| {
-        if *mode != TestMode::Executable {
-          Some(specifier.clone())
-        } else {
-          None
-        }
-      })
-      .collect(),
-  )
-  .await?;
-
-  let mut module_specifiers = specifiers
-    .into_iter()
-    .filter_map(|(specifier, mode)| {
-      if mode != TestMode::Documentation {
-        Some(specifier)
-      } else {
-        None
-      }
-    })
-    .collect::<Vec<_>>();
-
-  if !inline_files.is_empty() {
-    module_specifiers
-      .extend(inline_files.iter().map(|file| file.specifier.clone()));
-
-    for file in inline_files {
-      file_fetcher.insert_memory_files(file);
-    }
-  }
-
-  main_graph_container
-    .check_specifiers(&module_specifiers)
-    .await?;
-
-  Ok(())
-}
-
 static HAS_TEST_RUN_SIGINT_HANDLER: AtomicBool = AtomicBool::new(false);
 
 /// Test a collection of specifiers with test modes concurrently.
@@ -1788,13 +1572,18 @@ pub async fn run_tests(
     return Err(generic_error("No test modules found"));
   }
 
+  let doc_tests = get_doc_tests(&specifiers_with_mode, file_fetcher).await?;
+  let specifiers_for_typecheck_and_test =
+    get_target_specifiers(specifiers_with_mode, &doc_tests);
+  for doc_test in doc_tests {
+    file_fetcher.insert_memory_files(doc_test);
+  }
+
   let main_graph_container = factory.main_module_graph_container().await?;
 
-  check_specifiers(
-    file_fetcher,
-    main_graph_container,
-    specifiers_with_mode.clone(),
-  )
+  // Typecheck
+  main_graph_container
+    .check_specifiers(&specifiers_for_typecheck_and_test)
     .await?;
 
   if workspace_test_options.no_run {
@@ -1804,17 +1593,12 @@ pub async fn run_tests(
   let worker_factory =
     Arc::new(factory.create_cli_main_worker_factory().await?);
 
+  // Run tests
   test_specifiers(
     worker_factory,
     &permissions,
     permission_desc_parser,
-    specifiers_with_mode
-      .into_iter()
-      .filter_map(|(s, m)| match m {
-        TestMode::Documentation => None,
-        _ => Some(s),
-      })
-      .collect(),
+    specifiers_for_typecheck_and_test,
     TestSpecifiersOptions {
       cwd: Url::from_directory_path(cli_options.initial_cwd()).map_err(
         |_| {
@@ -1949,8 +1733,6 @@ pub async fn run_tests_with_watch(
           test_modules.clone()
         };
 
-        let worker_factory =
-          Arc::new(factory.create_cli_main_worker_factory().await?);
         let specifiers_with_mode = fetch_specifiers_with_test_mode(
           &cli_options,
           file_fetcher,
@@ -1962,30 +1744,34 @@ pub async fn run_tests_with_watch(
         .filter(|(specifier, _)| test_modules_to_reload.contains(specifier))
         .collect::<Vec<(ModuleSpecifier, TestMode)>>();
 
+        let doc_tests =
+          get_doc_tests(&specifiers_with_mode, file_fetcher).await?;
+        let specifiers_for_typecheck_and_test =
+          get_target_specifiers(specifiers_with_mode, &doc_tests);
+        for doc_test in doc_tests {
+          file_fetcher.insert_memory_files(doc_test);
+        }
+
         let main_graph_container =
           factory.main_module_graph_container().await?;
-        check_specifiers(
-          file_fetcher,
-          main_graph_container,
-          specifiers_with_mode.clone(),
-        )
+
+        // Typecheck
+        main_graph_container
+          .check_specifiers(&specifiers_for_typecheck_and_test)
           .await?;
 
         if workspace_test_options.no_run {
           return Ok(());
         }
 
+        let worker_factory =
+          Arc::new(factory.create_cli_main_worker_factory().await?);
+
         test_specifiers(
           worker_factory,
           &permissions,
           permission_desc_parser,
-          specifiers_with_mode
-            .into_iter()
-            .filter_map(|(s, m)| match m {
-              TestMode::Documentation => None,
-              _ => Some(s),
-            })
-            .collect(),
+          specifiers_for_typecheck_and_test,
           TestSpecifiersOptions {
             cwd: Url::from_directory_path(cli_options.initial_cwd()).map_err(
               |_| {
@@ -2020,6 +1806,38 @@ pub async fn run_tests_with_watch(
   Ok(())
 }
 
+/// Extracts doc tests from files specified by the given specifiers.
+async fn get_doc_tests(
+  specifiers_with_mode: &[(Url, TestMode)],
+  file_fetcher: &FileFetcher,
+) -> Result<Vec<File>, AnyError> {
+  let specifiers_needing_extraction = specifiers_with_mode
+    .iter()
+    .filter(|(_, mode)| mode.needs_test_extraction())
+    .map(|(s, _)| s);
+
+  let mut doc_tests = Vec::new();
+  for s in specifiers_needing_extraction {
+    let file = file_fetcher.fetch_bypass_permissions(s).await?;
+    doc_tests.extend(extract_doc_tests(file)?);
+  }
+
+  Ok(doc_tests)
+}
+
+/// Get a list of specifiers that we need to perform typecheck and run tests on.
+/// The result includes "pseudo specifiers" for doc tests.
+fn get_target_specifiers(
+  specifiers_with_mode: Vec<(Url, TestMode)>,
+  doc_tests: &[File],
+) -> Vec<Url> {
+  specifiers_with_mode
+    .into_iter()
+    .filter_map(|(s, mode)| mode.needs_test_run().then_some(s))
+    .chain(doc_tests.iter().map(|d| d.specifier.clone()))
+    .collect()
+}
+
 /// Tracks failures for the `--fail-fast` argument in
 /// order to tell when to stop running tests.
 #[derive(Clone, Default)]

cli/util/mod.rs

@@ -7,6 +7,7 @@ pub mod console;
 pub mod diff;
 pub mod display;
 pub mod draw_thread;
+pub mod extract;
 pub mod file_watcher;
 pub mod fs;
 pub mod logger;

tests/integration/check_tests.rs

@@ -185,8 +185,8 @@ fn reload_flag() {
 fn typecheck_declarations_ns() {
   let context = TestContextBuilder::for_jsr().build();
   let args = vec![
-    "test".to_string(),
-    "--doc".to_string(),
+    "check".to_string(),
+    "--doc-only".to_string(),
     util::root_path()
       .join("cli/tsc/dts/lib.deno.ns.d.ts")
       .to_string_lossy()
@@ -208,8 +208,8 @@ fn typecheck_declarations_ns() {
 fn typecheck_declarations_unstable() {
   let context = TestContext::default();
   let args = vec![
-    "test".to_string(),
-    "--doc".to_string(),
+    "check".to_string(),
+    "--doc-only".to_string(),
     util::root_path()
       .join("cli/tsc/dts/lib.deno.unstable.d.ts")
       .to_string_lossy()

tests/integration/watcher_tests.rs

@@ -1022,6 +1022,8 @@ async fn test_watch_doc() {
   let mut child = util::deno_cmd()
     .current_dir(t.path())
     .arg("test")
+    .arg("--config")
+    .arg(util::deno_config_path())
     .arg("--watch")
     .arg("--doc")
     .arg(t.path())
@@ -1039,26 +1041,110 @@ async fn test_watch_doc() {
   wait_contains("Test finished", &mut stderr_lines).await;
 
   let foo_file = t.path().join("foo.ts");
+  let foo_file_url = foo_file.url_file();
   foo_file.write(
     r#"
-    export default function foo() {}
+    export function add(a: number, b: number) {
+      return a + b;
+    }
   "#,
   );
 
+  wait_contains("ok | 0 passed | 0 failed", &mut stdout_lines).await;
+  wait_contains("Test finished", &mut stderr_lines).await;
+
+  // Trigger a type error
   foo_file.write(
     r#"
     /**
      * ```ts
-     * import foo from "./foo.ts";
+     * const sum: string = add(1, 2);
      * ```
      */
-    export default function foo() {}
+    export function add(a: number, b: number) {
+      return a + b;
+    }
   "#,
   );
 
-  // We only need to scan for a Check file://.../foo.ts$3-6 line that
-  // corresponds to the documentation block being type-checked.
-  assert_contains!(skip_restarting_line(&mut stderr_lines).await, "foo.ts$3-6");
+  assert_eq!(
+    skip_restarting_line(&mut stderr_lines).await,
+    format!("Check {foo_file_url}$3-6.ts")
+  );
+  assert_eq!(
+    next_line(&mut stderr_lines).await.unwrap(),
+    "error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'."
+  );
+  assert_eq!(
+    next_line(&mut stderr_lines).await.unwrap(),
+    "    const sum: string = add(1, 2);"
+  );
+  assert_eq!(next_line(&mut stderr_lines).await.unwrap(), "          ~~~");
+  assert_eq!(
+    next_line(&mut stderr_lines).await.unwrap(),
+    format!("    at {foo_file_url}$3-6.ts:3:11")
+  );
+  wait_contains("Test failed", &mut stderr_lines).await;
+
+  // Trigger a runtime error
+  foo_file.write(
+    r#"
+    /**
+     * ```ts
+     * import { assertEquals } from "@std/assert/equals";
+     *
+     * assertEquals(add(1, 2), 4);
+     * ```
+     */
+    export function add(a: number, b: number) {
+      return a + b;
+    }
+  "#,
+  );
+
+  wait_contains("running 1 test from", &mut stdout_lines).await;
+  assert_contains!(
+    next_line(&mut stdout_lines).await.unwrap(),
+    &format!("{foo_file_url}$3-8.ts ... FAILED")
+  );
+  wait_contains("ERRORS", &mut stdout_lines).await;
+  wait_contains(
+    "error: AssertionError: Values are not equal.",
+    &mut stdout_lines,
+  )
+  .await;
+  wait_contains("-   3", &mut stdout_lines).await;
+  wait_contains("+   4", &mut stdout_lines).await;
+  wait_contains("FAILURES", &mut stdout_lines).await;
+  wait_contains("FAILED | 0 passed | 1 failed", &mut stdout_lines).await;
+
+  wait_contains("Test failed", &mut stderr_lines).await;
+
+  // Fix the runtime error
+  foo_file.write(
+    r#"
+    /**
+     * ```ts
+     * import { assertEquals } from "@std/assert/equals";
+     *
+     * assertEquals(add(1, 2), 3);
+     * ```
+     */
+    export function add(a: number, b: number) {
+      return a + b;
+    }
+  "#,
+  );
+
+  wait_contains("running 1 test from", &mut stdout_lines).await;
+  assert_contains!(
+    next_line(&mut stdout_lines).await.unwrap(),
+    &format!("{foo_file_url}$3-8.ts ... ok")
+  );
+  wait_contains("ok | 1 passed | 0 failed", &mut stdout_lines).await;
+
+  wait_contains("Test finished", &mut stderr_lines).await;
+
   check_alive_then_kill(child);
 }
 

tests/specs/check/typecheck_doc_duplicate_identifiers/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "check --doc --config ../../../config/deno.json mod.ts",
+  "exitCode": 0,
+  "output": "mod.out"
+}

tests/specs/check/typecheck_doc_duplicate_identifiers/mod.out

@@ -0,0 +1,2 @@
+Check [WILDCARD]/mod.ts
+Check [WILDCARD]/mod.ts$2-8.ts

tests/specs/check/typecheck_doc_duplicate_identifiers/mod.ts

@@ -0,0 +1,13 @@
+/**
+ * ```ts
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * const foo = createFoo(3);
+ * assertEquals(foo, 9);
+ * ```
+ */
+export function createFoo(x: number): number {
+  return x * x;
+}
+
+export const foo = 42;

tests/specs/check/typecheck_doc_failure/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "check --doc mod.ts",
+  "exitCode": 1,
+  "output": "mod.out"
+}

tests/specs/check/typecheck_doc_failure/mod.out

@@ -0,0 +1,6 @@
+Check [WILDCARD]/mod.ts
+Check [WILDCARD]/mod.ts$2-5.ts
+error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
+const sum: string = add(1, 2);
+      ~~~
+    at [WILDCARD]/mod.ts$2-5.ts:2:7

tests/specs/check/typecheck_doc_failure/mod.ts

@@ -0,0 +1,8 @@
+/**
+ * ```ts
+ * const sum: string = add(1, 2);
+ * ```
+ */
+export function add(a: number, b: number): number {
+  return a + b;
+}

tests/specs/check/typecheck_doc_in_markdown/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "check --doc-only markdown.md",
+  "exitCode": 1,
+  "output": "markdown.out"
+}

tests/specs/check/typecheck_doc_in_markdown/markdown.md

@@ -0,0 +1,31 @@
+# Documentation
+
+The following block does not have a language attribute and should be ignored:
+
+```
+This is a fenced block without attributes, it's invalid and it should be ignored.
+```
+
+The following block should be given a js extension on extraction:
+
+```js
+console.log("js");
+```
+
+The following block should be given a ts extension on extraction:
+
+```ts
+console.log("ts");
+```
+
+The following example contains the ignore attribute and will be ignored:
+
+```ts ignore
+const value: Invalid = "ignored";
+```
+
+The following example will trigger the type-checker to fail:
+
+```ts
+const a: string = 42;
+```

tests/specs/check/typecheck_doc_in_markdown/markdown.out

@@ -0,0 +1,7 @@
+Check [WILDCARD]/markdown.md$11-14.js
+Check [WILDCARD]/markdown.md$17-20.ts
+Check [WILDCARD]/markdown.md$29-32.ts
+error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
+const a: string = 42;
+      ^
+    at [WILDCARD]/markdown.md$29-32.ts:1:7

tests/specs/check/typecheck_doc_success/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "check --doc mod.ts",
+  "exitCode": 0,
+  "output": "mod.out"
+}

tests/specs/check/typecheck_doc_success/mod.out

@@ -0,0 +1,2 @@
+Check [WILDCARD]/tests/specs/check/typecheck_doc_success/mod.ts
+Check [WILDCARD]/tests/specs/check/typecheck_doc_success/mod.ts$2-5.ts

tests/specs/check/typecheck_doc_success/mod.ts

@@ -0,0 +1,12 @@
+/**
+ * ```ts
+ * const sum: number = add(1, 2);
+ * ```
+ *
+ * ```mts ignore
+ * const sum: boolean = add(3, 4);
+ * ```
+ */
+export function add(a: number, b: number): number {
+  return a + b;
+}

tests/specs/test/doc/main.out

@@ -6,4 +6,4 @@ Check [WILDCARD]/main.ts$30-35.ts
 error: TS2367 [ERROR]: This comparison appears to be unintentional because the types 'string' and 'number' have no overlap.
     console.assert(check() == 42);
                    ~~~~~~~~~~~~~
-    at [WILDCARD]/main.ts$30-35.ts:3:16
+    at [WILDCARD]/main.ts$30-35.ts:3:20

tests/specs/test/doc_duplicate_identifier/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "test --doc --config ../../../config/deno.json main.ts",
+  "exitCode": 0,
+  "output": "main.out"
+}

tests/specs/test/doc_duplicate_identifier/main.out

@@ -0,0 +1,11 @@
+Check [WILDCARD]/main.ts
+Check [WILDCARD]/main.ts$11-19.ts
+Check [WILDCARD]/main.ts$25-30.ts
+running 0 tests from ./main.ts
+running 1 test from ./main.ts$11-19.ts
+[WILDCARD]/main.ts$11-19.ts ... ok ([WILDCARD]ms)
+running 1 test from ./main.ts$25-30.ts
+[WILDCARD]/main.ts$25-30.ts ... ok ([WILDCARD]ms)
+
+ok | 2 passed | 0 failed ([WILDCARD]ms)
+

tests/specs/test/doc_duplicate_identifier/main.ts

@@ -0,0 +1,33 @@
+// `deno test --doc` tries to convert the example code snippets into pseudo
+// test files in a way that all the exported items are available without
+// explicit import statements. Therefore, in the test code, you don't have to
+// write like `import { add } from "./main.ts";`.
+// However, this automatic import resolution might conflict with other
+// explicitly declared identifiers in the test code you write. This spec test
+// makes sure that such cases will not cause any issues - explicit identifiers
+// take precedence.
+
+/**
+ * ```ts
+ * import { assertEquals } from "@std/assert/equals";
+ * import { getModuleName, createFoo } from "./mod.ts";
+ *
+ * const foo = createFoo();
+ * assertEquals(getModuleName(), "mod.ts");
+ * assertEquals(add(1, 2), foo());
+ * ```
+ */
+export function add(a: number, b: number): number {
+  return a + b;
+}
+
+/**
+ * ```ts
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(getModuleName(), "main.ts");
+ * ```
+ */
+export const getModuleName = () => "main.ts";
+
+export let foo = 1234;

tests/specs/test/doc_duplicate_identifier/mod.ts

@@ -0,0 +1,7 @@
+export function getModuleName() {
+  return "mod.ts";
+}
+
+export const createFoo = () => {
+  return () => 3;
+};

tests/specs/test/doc_failure/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "test --doc --config ../../../config/deno.json main.ts",
+  "exitCode": 1,
+  "output": "main.out"
+}

tests/specs/test/doc_failure/main.out

@@ -0,0 +1,60 @@
+Check [WILDCARD]/main.ts
+Check [WILDCARD]/main.ts$2-9.ts
+Check [WILDCARD]/main.ts$13-18.ts
+Check [WILDCARD]/main.ts$24-29.ts
+running 0 tests from ./main.ts
+running 1 test from ./main.ts$2-9.ts
+[WILDCARD]/main.ts$2-9.ts ... FAILED ([WILDCARD]ms)
+running 1 test from ./main.ts$13-18.ts
+[WILDCARD]/main.ts$13-18.ts ... FAILED ([WILDCARD]ms)
+running 1 test from ./main.ts$24-29.ts
+[WILDCARD]/main.ts$24-29.ts ... FAILED ([WILDCARD]ms)
+
+ ERRORS 
+
+[WILDCARD]/main.ts$13-18.ts => ./main.ts$13-18.ts:3:6
+error: AssertionError: Values are not equal.
+
+
+    [Diff] Actual / Expected
+
+
+-   3
++   4
+
+  throw new AssertionError(message);
+        ^
+    at assertEquals ([WILDCARD]/std/assert/equals.ts:[WILDCARD])
+    at [WILDCARD]/main.ts$13-18.ts:4:5
+
+[WILDCARD]/main.ts$2-9.ts => ./main.ts$2-9.ts:3:6
+error: AssertionError: Expected actual: "2.5e+0" to be close to "2": delta "5e-1" is greater than "2e-7".
+  throw new AssertionError(
+        ^
+    at assertAlmostEquals ([WILDCARD]/std/assert/almost_equals.ts:[WILDCARD])
+    at [WILDCARD]/main.ts$2-9.ts:6:5
+
+[WILDCARD]/main.ts$24-29.ts => ./main.ts$24-29.ts:3:6
+error: AssertionError: Values are not equal.
+
+
+    [Diff] Actual / Expected
+
+
+-   4
++   3
+
+  throw new AssertionError(message);
+        ^
+    at assertEquals ([WILDCARD]/std/assert/equals.ts:[WILDCARD])
+    at [WILDCARD]/main.ts$24-29.ts:4:5
+
+ FAILURES 
+
+[WILDCARD]/main.ts$13-18.ts => ./main.ts$13-18.ts:3:6
+[WILDCARD]/main.ts$2-9.ts => ./main.ts$2-9.ts:3:6
+[WILDCARD]/main.ts$24-29.ts => ./main.ts$24-29.ts:3:6
+
+FAILED | 0 passed | 3 failed ([WILDCARD]ms)
+
+error: Test failed

tests/specs/test/doc_failure/main.ts

@@ -0,0 +1,32 @@
+/**
+ * ```ts
+ * import { assertAlmostEquals } from "@std/assert/almost-equals";
+ *
+ * const x = sub(3, 1);
+ * const y = div(5, x);
+ * assertAlmostEquals(y, 2.0); // throws
+ * ```
+ * @module doc
+ */
+
+/**
+ * ```ts
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(div(6, 2), 4); // throws
+ * ```
+ */
+export function div(a: number, b: number): number {
+  return a / b;
+}
+
+/**
+ * ```ts
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(sub(6, 2), 3); // throws
+ * ```
+ */
+const sub = (a: number, b: number): number => a - b;
+
+export { sub };

tests/specs/test/doc_only/__test__.jsonc

@@ -1,5 +1,5 @@
 {
-  "args": "test --doc --allow-all doc_only",
+  "args": "test --doc --config ../../../config/deno.json doc_only",
   "exitCode": 0,
   "output": "main.out"
 }

tests/specs/test/doc_only/doc_only/mod.ts

@@ -1,6 +1,8 @@
 /**
  * ```ts
- * import "./mod.ts";
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(42, 40 + 2);
  * ```
  */
 Deno.test("unreachable", function () {

tests/specs/test/doc_only/main.out

@@ -1,4 +1,6 @@
-Check [WILDCARD]/doc_only/mod.ts$2-5.ts
+Check [WILDCARD]/doc_only/mod.ts$2-7.ts
+running 1 test from ./doc_only/mod.ts$2-7.ts
+[WILDCARD]/doc_only/mod.ts$2-7.ts ... ok ([WILDCARD]ms)
 
-ok | 0 passed | 0 failed ([WILDCARD])
+ok | 1 passed | 0 failed ([WILDCARD]ms)
 

tests/specs/test/doc_permission_respected/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "test --doc --allow-env=PATH --reload main.ts",
+  "exitCode": 1,
+  "output": "main.out"
+}

tests/specs/test/doc_permission_respected/main.out

@@ -0,0 +1,25 @@
+Check [WILDCARD]/main.ts
+Check [WILDCARD]/main.ts$3-6.ts
+Check [WILDCARD]/main.ts$8-11.ts
+running 0 tests from ./main.ts
+running 1 test from ./main.ts$3-6.ts
+[WILDCARD]/main.ts$3-6.ts ... ok ([WILDCARD]ms)
+running 1 test from ./main.ts$8-11.ts
+[WILDCARD]/main.ts$8-11.ts ... FAILED ([WILDCARD]ms)
+
+ ERRORS 
+
+[WILDCARD]/main.ts$8-11.ts => ./main.ts$8-11.ts:1:6
+error: NotCapable: Requires env access to "USER", run again with the --allow-env flag
+    const _user = Deno.env.get("USER");
+                           ^
+    at Object.getEnv [as get] ([WILDCARD])
+    at [WILDCARD]/main.ts$8-11.ts:2:28
+
+ FAILURES 
+
+[WILDCARD]/main.ts$8-11.ts => ./main.ts$8-11.ts:1:6
+
+FAILED | 1 passed | 1 failed ([WILDCARD]ms)
+
+error: Test failed

tests/specs/test/doc_permission_respected/main.ts

@@ -0,0 +1,12 @@
+/**
+ * This should succeed because we pass `--allow-env=PATH`
+ * ```ts
+ * const _path = Deno.env.get("PATH");
+ * ```
+ *
+ * This should fail because we don't allow for env access to `USER`
+ * ```ts
+ * const _user = Deno.env.get("USER");
+ * ```
+ * @module doc
+ */

tests/specs/test/doc_success/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "test --doc --config ../../../config/deno.json main.ts",
+  "exitCode": 0,
+  "output": "main.out"
+}

tests/specs/test/doc_success/main.out

@@ -0,0 +1,19 @@
+Check [WILDCARD]/main.ts$8-13.js
+Check [WILDCARD]/main.ts$14-19.jsx
+Check [WILDCARD]/main.ts$20-25.ts
+Check [WILDCARD]/main.ts$26-31.tsx
+Check [WILDCARD]/main.ts$42-47.ts
+running 0 tests from ./main.ts
+running 1 test from ./main.ts$8-13.js
+[WILDCARD]/main.ts$8-13.js ... ok ([WILDCARD]ms)
+running 1 test from ./main.ts$14-19.jsx
+[WILDCARD]/main.ts$14-19.jsx ... ok ([WILDCARD]ms)
+running 1 test from ./main.ts$20-25.ts
+[WILDCARD]/main.ts$20-25.ts ... ok ([WILDCARD]ms)
+running 1 test from ./main.ts$26-31.tsx
+[WILDCARD]/main.ts$26-31.tsx ... ok ([WILDCARD]ms)
+running 1 test from ./main.ts$42-47.ts
+[WILDCARD]/main.ts$42-47.ts ... ok ([WILDCARD]ms)
+
+ok | 5 passed | 0 failed ([WILDCARD]ms)
+

tests/specs/test/doc_success/main.ts

@@ -0,0 +1,50 @@
+/**
+ * ```
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(add(1, 2), 3);
+ * ```
+ *
+ * ```js
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(add(1, 2), 3);
+ * ```
+ *
+ * ```jsx
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(add(1, 2), 3);
+ * ```
+ *
+ * ```ts
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(add(1, 2), 3);
+ * ```
+ *
+ * ```tsx
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(add(1, 2), 3);
+ * ```
+ *
+ * ```text
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(add(1, 2), 3);
+ * ```
+ *
+ * @module doc
+ */
+
+/**
+ * ```ts
+ * import { assertEquals } from "@std/assert/equals";
+ *
+ * assertEquals(add(1, 2), 3);
+ * ```
+ */
+export function add(a: number, b: number): number {
+  return a + b;
+}

tests/specs/test/doc_ts_declare_global/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "test --doc --config ../../../config/deno.json lib.d.ts",
+  "exitCode": 0,
+  "output": "lib.d.ts.out"
+}

tests/specs/test/doc_ts_declare_global/lib.d.ts

@@ -0,0 +1,13 @@
+export {};
+
+declare global {
+  /**
+   * ```ts
+   * import { assertEquals } from "@std/assert/equals";
+   * import "./mod.js";
+   *
+   * assertEquals(myFunction(1, 2), 3);
+   * ```
+   */
+  export function myFunction(a: number, b: number): number;
+}

tests/specs/test/doc_ts_declare_global/lib.d.ts.out

@@ -0,0 +1,6 @@
+Check [WILDCARD]/lib$d$ts$5-11.ts
+running 1 test from ./lib$d$ts$5-11.ts
+[WILDCARD]/lib$d$ts$5-11.ts ... ok ([WILDCARD]ms)
+
+ok | 1 passed | 0 failed ([WILDCARD]ms)
+

tests/specs/test/doc_ts_declare_global/mod.js

@@ -0,0 +1 @@
+globalThis.myFunction = (a, b) => a + b;

tests/specs/test/doc_ts_namespace_decl/__test__.jsonc

@@ -0,0 +1,5 @@
+{
+  "args": "test --doc --config ../../../config/deno.json lib.d.ts",
+  "exitCode": 0,
+  "output": "lib.d.ts.out"
+}

tests/specs/test/doc_ts_namespace_decl/lib.d.ts

@@ -0,0 +1,11 @@
+declare namespace MyNamespace {
+  /**
+   * ```ts
+   * import { assertEquals } from "@std/assert/equals";
+   * import "./mod.js";
+   *
+   * assertEquals(MyNamespace.add(1, 2), 3);
+   * ```
+   */
+  export function add(a: number, b: number): number;
+}

tests/specs/test/doc_ts_namespace_decl/lib.d.ts.out

@@ -0,0 +1,6 @@
+Check [WILDCARD]/lib$d$ts$3-9.ts
+running 1 test from ./lib$d$ts$3-9.ts
+[WILDCARD]/lib$d$ts$3-9.ts ... ok ([WILDCARD]ms)
+
+ok | 1 passed | 0 failed ([WILDCARD]ms)
+

tests/specs/test/doc_ts_namespace_decl/mod.js

@@ -0,0 +1,5 @@
+globalThis.MyNamespace = {
+  add(a, b) {
+    return a + b;
+  },
+};

tests/specs/test/markdown/main.out

@@ -4,4 +4,4 @@ Check [WILDCARD]/main.md$29-32.ts
 error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
     const a: string = 42;
           ^
-    at [WILDCARD]/main.md$29-32.ts:1:7
+    at [WILDCARD]/main.md$29-32.ts:2:11

tests/specs/test/markdown_full_block_names/main.out

@@ -3,4 +3,4 @@ Check [WILDCARD]/main.md$17-20.ts
 error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
     const a: string = 42;
           ^
-    at [WILDCARD]/main.md$17-20.ts:1:7
+    at [WILDCARD]/main.md$17-20.ts:2:11

tests/specs/test/markdown_ignore_html_comment/main.out

@@ -2,4 +2,4 @@ Check [WILDCARD]/main.md$34-37.ts
 error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
     const a: string = 42;
           ^
-    at [WILDCARD]/main.md$34-37.ts:1:7
+    at [WILDCARD]/main.md$34-37.ts:2:11

tests/specs/test/markdown_windows/main.out

@@ -4,4 +4,4 @@ Check [WILDCARD]/main.md$29-32.ts
 error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
     const a: string = 42;
           ^
-    at [WILDCARD]/main.md$29-32.ts:1:7
+    at [WILDCARD]/main.md$29-32.ts:2:11

tests/specs/test/type_check_with_doc/main.out

@@ -8,6 +8,6 @@ const a: string = 1;
 TS2322 [ERROR]: Type 'string' is not assignable to type 'number'.
     const b: number = "1";
           ^
-    at file://[WILDCARD]/main.ts$2-5.ts:1:7
+    at file://[WILDCARD]/main.ts$2-5.ts:2:11
 
 Found 2 errors.