feat(test): add support for type checking documentation (#10521)

This commit adds support for type checking codeblocks in the JS doc 
comments.

cli/ast.rs

@@ -304,6 +304,22 @@ impl ParsedModule {
     self.leading_comments.clone()
   }
 
+  /// Get the module's comments.
+  pub fn get_comments(&self) -> Vec<Comment> {
+    let mut comments = Vec::new();
+    let (leading_comments, trailing_comments) = self.comments.borrow_all();
+
+    for value in leading_comments.values() {
+      comments.append(&mut value.clone());
+    }
+
+    for value in trailing_comments.values() {
+      comments.append(&mut value.clone());
+    }
+
+    comments
+  }
+
   /// Get a location for a given span within the module.
   pub fn get_location(&self, span: &Span) -> Location {
     self.source_map.lookup_char_pos(span.lo).into()

cli/flags.rs

@@ -96,6 +96,7 @@ pub enum DenoSubcommand {
     script: String,
   },
   Test {
+    doc: bool,
     no_run: bool,
     fail_fast: bool,
     quiet: bool,
@@ -984,6 +985,12 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> {
         .help("Cache test modules, but don't run tests")
         .takes_value(false),
     )
+    .arg(
+      Arg::with_name("doc")
+        .long("doc")
+        .help("UNSTABLE: type check code blocks")
+        .takes_value(false),
+    )
     .arg(
       Arg::with_name("fail-fast")
         .long("fail-fast")
@@ -1667,6 +1674,7 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   runtime_args_parse(flags, matches, true, true);
 
   let no_run = matches.is_present("no-run");
+  let doc = matches.is_present("doc");
   let fail_fast = matches.is_present("fail-fast");
   let allow_none = matches.is_present("allow-none");
   let quiet = matches.is_present("quiet");
@@ -1711,6 +1719,7 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   flags.coverage_dir = matches.value_of("coverage").map(String::from);
   flags.subcommand = DenoSubcommand::Test {
     no_run,
+    doc,
     fail_fast,
     quiet,
     include,
@@ -3357,6 +3366,7 @@ mod tests {
       Flags {
         subcommand: DenoSubcommand::Test {
           no_run: true,
+          doc: false,
           fail_fast: false,
           filter: Some("- foo".to_string()),
           allow_none: true,

cli/main.rs

@@ -914,6 +914,7 @@ async fn test_command(
   flags: Flags,
   include: Option<Vec<String>>,
   no_run: bool,
+  doc: bool,
   fail_fast: bool,
   quiet: bool,
   allow_none: bool,
@@ -924,6 +925,8 @@ async fn test_command(
     env::set_var("DENO_UNSTABLE_COVERAGE_DIR", coverage_dir);
   }
 
+  // TODO(caspervonb) move this chunk into tools::test_runner.
+
   let program_state = ProgramState::build(flags.clone()).await?;
 
   let include = include.unwrap_or_else(|| vec![".".to_string()]);
@@ -944,9 +947,20 @@ async fn test_command(
 
     let paths_to_watch: Vec<_> = include.iter().map(PathBuf::from).collect();
 
+    // TODO(caspervonb) clean this up.
     let resolver = |changed: Option<Vec<PathBuf>>| {
-      let test_modules_result =
-        test_runner::collect_test_module_specifiers(include.clone(), &cwd);
+      let doc_modules_result = test_runner::collect_test_module_specifiers(
+        include.clone(),
+        &cwd,
+        fs_util::is_supported_ext,
+      );
+
+      let test_modules_result = test_runner::collect_test_module_specifiers(
+        include.clone(),
+        &cwd,
+        tools::test_runner::is_supported,
+      );
+
       let paths_to_watch = paths_to_watch.clone();
       let paths_to_watch_clone = paths_to_watch.clone();
 
@@ -954,6 +968,8 @@ async fn test_command(
       let program_state = program_state.clone();
       let files_changed = changed.is_some();
       async move {
+        let doc_modules = if doc { doc_modules_result? } else { Vec::new() };
+
         let test_modules = test_modules_result?;
 
         let mut paths_to_watch = paths_to_watch_clone;
@@ -976,6 +992,12 @@ async fn test_command(
         }
         let graph = builder.get_graph();
 
+        for specifier in doc_modules {
+          if let Ok(path) = specifier.to_file_path() {
+            paths_to_watch.push(path);
+          }
+        }
+
         for specifier in test_modules {
           fn get_dependencies<'a>(
             graph: &'a module_graph::Graph,
@@ -1070,6 +1092,7 @@ async fn test_command(
           program_state.clone(),
           permissions.clone(),
           lib.clone(),
+          modules_to_reload.clone(),
           modules_to_reload,
           no_run,
           fail_fast,
@@ -1084,13 +1107,27 @@ async fn test_command(
     )
     .await?;
   } else {
-    let test_modules =
-      test_runner::collect_test_module_specifiers(include, &cwd)?;
+    let doc_modules = if doc {
+      test_runner::collect_test_module_specifiers(
+        include.clone(),
+        &cwd,
+        fs_util::is_supported_ext,
+      )?
+    } else {
+      Vec::new()
+    };
+
+    let test_modules = test_runner::collect_test_module_specifiers(
+      include.clone(),
+      &cwd,
+      tools::test_runner::is_supported,
+    )?;
 
     let failed = test_runner::run_tests(
       program_state.clone(),
       permissions,
       lib,
+      doc_modules,
       test_modules,
       no_run,
       fail_fast,
@@ -1235,6 +1272,7 @@ fn get_subcommand(
     DenoSubcommand::Run { script } => run_command(flags, script).boxed_local(),
     DenoSubcommand::Test {
       no_run,
+      doc,
       fail_fast,
       quiet,
       include,
@@ -1245,6 +1283,7 @@ fn get_subcommand(
       flags,
       include,
       no_run,
+      doc,
       fail_fast,
       quiet,
       allow_none,

cli/tests/integration_tests.rs

@@ -2580,6 +2580,12 @@ mod integration {
       output: "test/deno_test.out",
     });
 
+    itest!(doc {
+      args: "test --doc --allow-all test/doc.ts",
+      exit_code: 1,
+      output: "test/doc.out",
+    });
+
     itest!(allow_all {
       args: "test --unstable --allow-all test/allow_all.ts",
       exit_code: 0,

cli/tests/test/doc.out

@@ -0,0 +1,5 @@
+Check [WILDCARD]/doc.ts:2-7
+error: TS2367 [ERROR]: This condition will always return 'false' since the types 'string' and 'number' have no overlap.
+console.assert(example() == 42);
+               ~~~~~~~~~~~~~~~
+    at [WILDCARD]/doc.ts:2-7.ts:3:16

cli/tests/test/doc.ts

@@ -0,0 +1,10 @@
+/**
+ * ```
+ * import { example } from "./doc.ts";
+ *
+ * console.assert(example() == 42);
+ * ```
+ */
+export function example(): string {
+  return "example";
+}

cli/tools/test_runner.rs

@@ -1,9 +1,11 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
+use crate::ast;
 use crate::colors;
 use crate::create_main_worker;
 use crate::file_fetcher::File;
-use crate::fs_util;
+use crate::fs_util::collect_files;
+use crate::fs_util::normalize_path;
 use crate::media_type::MediaType;
 use crate::module_graph;
 use crate::program_state::ProgramState;
@@ -18,6 +20,7 @@ use deno_core::serde_json::json;
 use deno_core::url::Url;
 use deno_core::ModuleSpecifier;
 use deno_runtime::permissions::Permissions;
+use regex::Regex;
 use serde::Deserialize;
 use std::path::Path;
 use std::path::PathBuf;
@@ -25,6 +28,7 @@ use std::sync::mpsc::channel;
 use std::sync::mpsc::Sender;
 use std::sync::Arc;
 use std::time::Instant;
+use swc_common::comments::CommentKind;
 
 #[derive(Debug, Clone, PartialEq, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -198,7 +202,7 @@ fn create_reporter(concurrent: bool) -> Box<dyn TestReporter + Send> {
   Box::new(PrettyTestReporter::new(concurrent))
 }
 
-fn is_supported(p: &Path) -> bool {
+pub(crate) fn is_supported(p: &Path) -> bool {
   use std::path::Component;
   if let Some(Component::Normal(basename_os_str)) = p.components().next_back() {
     let basename = basename_os_str.to_string_lossy();
@@ -222,19 +226,22 @@ fn is_supported(p: &Path) -> bool {
   }
 }
 
-pub fn collect_test_module_specifiers(
+pub fn collect_test_module_specifiers<P>(
   include: Vec<String>,
   root_path: &Path,
-) -> Result<Vec<Url>, AnyError> {
+  predicate: P,
+) -> Result<Vec<Url>, AnyError>
+where
+  P: Fn(&Path) -> bool,
+{
   let (include_paths, include_urls): (Vec<String>, Vec<String>) =
     include.into_iter().partition(|n| !is_remote_url(n));
-
   let mut prepared = vec![];
 
   for path in include_paths {
-    let p = fs_util::normalize_path(&root_path.join(path));
+    let p = normalize_path(&root_path.join(path));
     if p.is_dir() {
-      let test_files = fs_util::collect_files(&[p], &[], is_supported).unwrap();
+      let test_files = collect_files(&[p], &[], &predicate).unwrap();
       let test_files_as_urls = test_files
         .iter()
         .map(|f| Url::from_file_path(f).unwrap())
@@ -311,6 +318,7 @@ pub async fn run_tests(
   program_state: Arc<ProgramState>,
   permissions: Permissions,
   lib: module_graph::TypeLib,
+  doc_modules: Vec<ModuleSpecifier>,
   test_modules: Vec<ModuleSpecifier>,
   no_run: bool,
   fail_fast: bool,
@@ -327,6 +335,80 @@ pub async fn run_tests(
     return Ok(false);
   }
 
+  if !doc_modules.is_empty() {
+    let mut test_programs = Vec::new();
+
+    let blocks_regex = Regex::new(r"```([^\n]*)\n([\S\s]*?)```")?;
+    let lines_regex = Regex::new(r"(?:\* ?)(?:\# ?)?(.*)")?;
+
+    for specifier in &doc_modules {
+      let file = program_state.file_fetcher.get_source(&specifier).unwrap();
+
+      let parsed_module =
+        ast::parse(&file.specifier.as_str(), &file.source, &file.media_type)?;
+
+      let mut comments = parsed_module.get_comments();
+      comments.sort_by_key(|comment| {
+        let location = parsed_module.get_location(&comment.span);
+        location.line
+      });
+
+      for comment in comments {
+        if comment.kind != CommentKind::Block || !comment.text.starts_with('*')
+        {
+          continue;
+        }
+
+        for block in blocks_regex.captures_iter(&comment.text) {
+          let body = block.get(2).unwrap();
+          let text = body.as_str();
+
+          // TODO(caspervonb) generate an inline source map
+          let mut source = String::new();
+          for line in lines_regex.captures_iter(&text) {
+            let text = line.get(1).unwrap();
+            source.push_str(&format!("{}\n", text.as_str()));
+          }
+
+          source.push_str("export {};");
+
+          let element = block.get(0).unwrap();
+          let span = comment
+            .span
+            .from_inner_byte_pos(element.start(), element.end());
+          let location = parsed_module.get_location(&span);
+
+          let specifier = deno_core::resolve_url_or_path(&format!(
+            "{}:{}-{}",
+            location.filename,
+            location.line,
+            location.line + element.as_str().split('\n').count(),
+          ))?;
+
+          let file = File {
+            local: specifier.to_file_path().unwrap(),
+            maybe_types: None,
+            media_type: MediaType::TypeScript, // media_type.clone(),
+            source: source.clone(),
+            specifier: specifier.clone(),
+          };
+
+          program_state.file_fetcher.insert_cached(file.clone());
+          test_programs.push(file.specifier.clone());
+        }
+      }
+    }
+
+    program_state
+      .prepare_module_graph(
+        test_programs.clone(),
+        lib.clone(),
+        permissions.clone(),
+        program_state.maybe_import_map.clone(),
+      )
+      .await?;
+  }
+
   program_state
     .prepare_module_graph(
       test_modules.clone(),
@@ -343,8 +425,8 @@ pub async fn run_tests(
   // Because scripts, and therefore worker.execute cannot detect unresolved promises at the moment
   // we generate a module for the actual test execution.
   let test_options = json!({
-    "disableLog": quiet,
-    "filter": filter,
+      "disableLog": quiet,
+      "filter": filter,
   });
 
   let test_module = deno_core::resolve_path("$deno$test.js")?;
@@ -487,6 +569,7 @@ mod tests {
         "http://example.com/printf_test.ts".to_string(),
       ],
       &test_data_path,
+      is_supported,
     )
     .unwrap();
     let test_data_url =
@@ -535,8 +618,12 @@ mod tests {
       .join("std")
       .join("http");
     println!("root {:?}", root);
-    let mut matched_urls =
-      collect_test_module_specifiers(vec![".".to_string()], &root).unwrap();
+    let mut matched_urls = collect_test_module_specifiers(
+      vec![".".to_string()],
+      &root,
+      is_supported,
+    )
+    .unwrap();
     matched_urls.sort();
     let root_url = Url::from_file_path(root).unwrap().to_string();
     println!("root_url {}", root_url);