denoland-deno/cli/tools/coverage.rs

// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.

use crate::colors;
use crate::flags::Flags;
use crate::fs_util::collect_files;
use crate::module_graph::TypeLib;
use crate::proc_state::ProcState;
use crate::source_maps::SourceMapGetter;
use deno_ast::swc::common::Span;
use deno_ast::MediaType;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_core::LocalInspectorSession;
use deno_runtime::permissions::Permissions;
use regex::Regex;
use serde::Deserialize;
use serde::Serialize;
use sourcemap::SourceMap;
use std::fs;
use std::fs::File;
use std::io::BufWriter;
use std::io::Write;
use std::path::PathBuf;
use std::sync::Arc;
use uuid::Uuid;

// TODO(caspervonb) all of these structs can and should be made private, possibly moved to
// inspector::protocol.
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CoverageRange {
  pub start_offset: usize,
  pub end_offset: usize,
  pub count: usize,
}

#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")]
pub struct FunctionCoverage {
  pub function_name: String,
  pub ranges: Vec<CoverageRange>,
  pub is_block_coverage: bool,
}

#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScriptCoverage {
  pub script_id: String,
  pub url: String,
  pub functions: Vec<FunctionCoverage>,
}

#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct StartPreciseCoverageParameters {
  pub call_count: bool,
  pub detailed: bool,
  pub allow_triggered_updates: bool,
}

#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct StartPreciseCoverageReturnObject {
  pub timestamp: f64,
}

#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TakePreciseCoverageReturnObject {
  pub result: Vec<ScriptCoverage>,
  pub timestamp: f64,
}

pub struct CoverageCollector {
  pub dir: PathBuf,
  session: LocalInspectorSession,
}

impl CoverageCollector {
  pub fn new(dir: PathBuf, session: LocalInspectorSession) -> Self {
    Self { dir, session }
  }

  async fn enable_debugger(&mut self) -> Result<(), AnyError> {
    self.session.post_message("Debugger.enable", None).await?;

    Ok(())
  }

  async fn enable_profiler(&mut self) -> Result<(), AnyError> {
    self.session.post_message("Profiler.enable", None).await?;

    Ok(())
  }

  async fn disable_debugger(&mut self) -> Result<(), AnyError> {
    self.session.post_message("Debugger.disable", None).await?;

    Ok(())
  }

  async fn disable_profiler(&mut self) -> Result<(), AnyError> {
    self.session.post_message("Profiler.disable", None).await?;

    Ok(())
  }

  async fn start_precise_coverage(
    &mut self,
    parameters: StartPreciseCoverageParameters,
  ) -> Result<StartPreciseCoverageReturnObject, AnyError> {
    let parameters_value = serde_json::to_value(parameters)?;
    let return_value = self
      .session
      .post_message("Profiler.startPreciseCoverage", Some(parameters_value))
      .await?;

    let return_object = serde_json::from_value(return_value)?;

    Ok(return_object)
  }

  async fn take_precise_coverage(
    &mut self,
  ) -> Result<TakePreciseCoverageReturnObject, AnyError> {
    let return_value = self
      .session
      .post_message("Profiler.takePreciseCoverage", None)
      .await?;

    let return_object = serde_json::from_value(return_value)?;

    Ok(return_object)
  }

  pub async fn start_collecting(&mut self) -> Result<(), AnyError> {
    self.enable_debugger().await?;
    self.enable_profiler().await?;
    self
      .start_precise_coverage(StartPreciseCoverageParameters {
        call_count: true,
        detailed: true,
        allow_triggered_updates: false,
      })
      .await?;

    Ok(())
  }

  pub async fn stop_collecting(&mut self) -> Result<(), AnyError> {
    fs::create_dir_all(&self.dir)?;

    let script_coverages = self.take_precise_coverage().await?.result;
    for script_coverage in script_coverages {
      let filename = format!("{}.json", Uuid::new_v4());
      let filepath = self.dir.join(filename);

      let mut out = BufWriter::new(File::create(filepath)?);
      serde_json::to_writer_pretty(&mut out, &script_coverage)?;
      out.write_all(b"\n")?;
      out.flush()?;
    }

    self.disable_debugger().await?;
    self.disable_profiler().await?;

    Ok(())
  }
}

pub enum CoverageReporterKind {
  Pretty,
  Lcov,
}

fn create_reporter(
  kind: CoverageReporterKind,
) -> Box<dyn CoverageReporter + Send> {
  match kind {
    CoverageReporterKind::Lcov => Box::new(LcovCoverageReporter::new()),
    CoverageReporterKind::Pretty => Box::new(PrettyCoverageReporter::new()),
  }
}

pub trait CoverageReporter {
  fn visit_coverage(
    &mut self,
    script_coverage: &ScriptCoverage,
    script_source: &str,
    maybe_source_map: Option<Vec<u8>>,
    maybe_original_source: Option<Arc<String>>,
  );

  fn done(&mut self);
}

pub struct LcovCoverageReporter {}

impl LcovCoverageReporter {
  pub fn new() -> LcovCoverageReporter {
    LcovCoverageReporter {}
  }
}

impl CoverageReporter for LcovCoverageReporter {
  fn visit_coverage(
    &mut self,
    script_coverage: &ScriptCoverage,
    script_source: &str,
    maybe_source_map: Option<Vec<u8>>,
    _maybe_original_source: Option<Arc<String>>,
  ) {
    // TODO(caspervonb) cleanup and reduce duplication between reporters, pre-compute line coverage
    // elsewhere.
    let maybe_source_map = maybe_source_map
      .map(|source_map| SourceMap::from_slice(&source_map).unwrap());

    let url = Url::parse(&script_coverage.url).unwrap();
    let file_path = url.to_file_path().unwrap();
    println!("SF:{}", file_path.to_str().unwrap());

    let mut functions_found = 0;
    for function in &script_coverage.functions {
      if function.function_name.is_empty() {
        continue;
      }

      let source_line = script_source[0..function.ranges[0].start_offset]
        .split('\n')
        .count();

      let line_index = if let Some(source_map) = maybe_source_map.as_ref() {
        source_map
          .tokens()
          .find(|token| token.get_dst_line() as usize == source_line)
          .map(|token| token.get_src_line() as usize)
          .unwrap_or(0)
      } else {
        source_line
      };

      let function_name = &function.function_name;

      println!("FN:{},{}", line_index + 1, function_name);

      functions_found += 1;
    }

    let mut functions_hit = 0;
    for function in &script_coverage.functions {
      if function.function_name.is_empty() {
        continue;
      }

      let execution_count = function.ranges[0].count;
      let function_name = &function.function_name;

      println!("FNDA:{},{}", execution_count, function_name);

      if execution_count != 0 {
        functions_hit += 1;
      }
    }

    println!("FNF:{}", functions_found);
    println!("FNH:{}", functions_hit);

    let mut branches_found = 0;
    let mut branches_hit = 0;
    for (block_number, function) in script_coverage.functions.iter().enumerate()
    {
      let block_hits = function.ranges[0].count;
      for (branch_number, range) in function.ranges[1..].iter().enumerate() {
        let source_line =
          script_source[0..range.start_offset].split('\n').count();

        let line_index = if let Some(source_map) = maybe_source_map.as_ref() {
          source_map
            .tokens()
            .find(|token| token.get_dst_line() as usize == source_line)
            .map(|token| token.get_src_line() as usize)
            .unwrap_or(0)
        } else {
          source_line
        };

        // From https://manpages.debian.org/unstable/lcov/geninfo.1.en.html:
        //
        // Block number and branch number are gcc internal IDs for the branch. Taken is either '-'
        // if the basic block containing the branch was never executed or a number indicating how
        // often that branch was taken.
        //
        // However with the data we get from v8 coverage profiles it seems we can't actually hit
        // this as appears it won't consider any nested branches it hasn't seen but its here for
        // the sake of accuracy.
        let taken = if block_hits > 0 {
          range.count.to_string()
        } else {
          "-".to_string()
        };

        println!(
          "BRDA:{},{},{},{}",
          line_index + 1,
          block_number,
          branch_number,
          taken
        );

        branches_found += 1;
        if range.count > 0 {
          branches_hit += 1;
        }
      }
    }

    println!("BRF:{}", branches_found);
    println!("BRH:{}", branches_hit);

    let lines = script_source.split('\n').collect::<Vec<_>>();
    let line_offsets = {
      let mut offsets: Vec<(usize, usize)> = Vec::new();
      let mut index = 0;

      for line in &lines {
        offsets.push((index, index + line.len() + 1));
        index += line.len() + 1;
      }

      offsets
    };

    let line_counts = line_offsets
      .iter()
      .map(|(line_start_offset, line_end_offset)| {
        let mut count = 0;

        // Count the hits of ranges that include the entire line which will always be at-least one
        // as long as the code has been evaluated.
        for function in &script_coverage.functions {
          for range in &function.ranges {
            if range.start_offset <= *line_start_offset
              && range.end_offset >= *line_end_offset
            {
              count += range.count;
            }
          }
        }

        // We reset the count if any block with a zero count overlaps with the line range.
        for function in &script_coverage.functions {
          for range in &function.ranges {
            if range.count > 0 {
              continue;
            }

            let overlaps = std::cmp::max(line_end_offset, &range.end_offset)
              - std::cmp::min(line_start_offset, &range.start_offset)
              < (line_end_offset - line_start_offset)
                + (range.end_offset - range.start_offset);

            if overlaps {
              count = 0;
            }
          }
        }

        count
      })
      .collect::<Vec<usize>>();

    let found_lines = if let Some(source_map) = maybe_source_map.as_ref() {
      let mut found_lines = line_counts
        .iter()
        .enumerate()
        .map(|(index, count)| {
          source_map
            .tokens()
            .filter(move |token| token.get_dst_line() as usize == index)
            .map(move |token| (token.get_src_line() as usize, *count))
        })
        .flatten()
        .collect::<Vec<(usize, usize)>>();

      found_lines.sort_unstable_by_key(|(index, _)| *index);
      found_lines.dedup_by_key(|(index, _)| *index);
      found_lines
    } else {
      line_counts
        .iter()
        .enumerate()
        .map(|(index, count)| (index, *count))
        .collect::<Vec<(usize, usize)>>()
    };

    for (index, count) in &found_lines {
      println!("DA:{},{}", index + 1, count);
    }

    let lines_hit = found_lines.iter().filter(|(_, count)| *count != 0).count();

    println!("LH:{}", lines_hit);

    let lines_found = found_lines.len();
    println!("LF:{}", lines_found);

    println!("end_of_record");
  }

  fn done(&mut self) {}
}

pub struct PrettyCoverageReporter {}

impl PrettyCoverageReporter {
  pub fn new() -> PrettyCoverageReporter {
    PrettyCoverageReporter {}
  }
}

impl CoverageReporter for PrettyCoverageReporter {
  fn visit_coverage(
    &mut self,
    script_coverage: &ScriptCoverage,
    script_source: &str,
    maybe_source_map: Option<Vec<u8>>,
    maybe_original_source: Option<Arc<String>>,
  ) {
    let maybe_source_map = maybe_source_map
      .map(|source_map| SourceMap::from_slice(&source_map).unwrap());

    let mut ignored_spans: Vec<Span> = Vec::new();
    for item in deno_ast::lex(script_source, MediaType::JavaScript) {
      if let deno_ast::TokenOrComment::Token(_) = item.inner {
        continue;
      }

      ignored_spans.push(item.span);
    }

    let lines = script_source.split('\n').collect::<Vec<_>>();

    let line_offsets = {
      let mut offsets: Vec<(usize, usize)> = Vec::new();
      let mut index = 0;

      for line in &lines {
        offsets.push((index, index + line.len() + 1));
        index += line.len() + 1;
      }

      offsets
    };

    // TODO(caspervonb): collect uncovered ranges on the lines so that we can highlight specific
    // parts of a line in color (word diff style) instead of the entire line.
    let line_counts = line_offsets
      .iter()
      .enumerate()
      .map(|(index, (line_start_offset, line_end_offset))| {
        let ignore = ignored_spans.iter().any(|span| {
          (span.lo.0 as usize) <= *line_start_offset
            && (span.hi.0 as usize) >= *line_end_offset
        });

        if ignore {
          return (index, 1);
        }

        let mut count = 0;

        // Count the hits of ranges that include the entire line which will always be at-least one
        // as long as the code has been evaluated.
        for function in &script_coverage.functions {
          for range in &function.ranges {
            if range.start_offset <= *line_start_offset
              && range.end_offset >= *line_end_offset
            {
              count += range.count;
            }
          }
        }

        // We reset the count if any block with a zero count overlaps with the line range.
        for function in &script_coverage.functions {
          for range in &function.ranges {
            if range.count > 0 {
              continue;
            }

            let overlaps = std::cmp::max(line_end_offset, &range.end_offset)
              - std::cmp::min(line_start_offset, &range.start_offset)
              < (line_end_offset - line_start_offset)
                + (range.end_offset - range.start_offset);

            if overlaps {
              count = 0;
            }
          }
        }

        (index, count)
      })
      .collect::<Vec<(usize, usize)>>();

    let lines = if let Some(original_source) = maybe_original_source.as_ref() {
      original_source.split('\n').collect::<Vec<_>>()
    } else {
      lines
    };

    let line_counts = if let Some(source_map) = maybe_source_map.as_ref() {
      let mut line_counts = line_counts
        .iter()
        .map(|(index, count)| {
          source_map
            .tokens()
            .filter(move |token| token.get_dst_line() as usize == *index)
            .map(move |token| (token.get_src_line() as usize, *count))
        })
        .flatten()
        .collect::<Vec<(usize, usize)>>();

      line_counts.sort_unstable_by_key(|(index, _)| *index);
      line_counts.dedup_by_key(|(index, _)| *index);

      line_counts
    } else {
      line_counts
    };

    print!("cover {} ... ", script_coverage.url);

    let hit_lines = line_counts
      .iter()
      .filter(|(_, count)| *count != 0)
      .map(|(index, _)| *index);

    let missed_lines = line_counts
      .iter()
      .filter(|(_, count)| *count == 0)
      .map(|(index, _)| *index);

    let lines_found = line_counts.len();
    let lines_hit = hit_lines.count();
    let line_ratio = lines_hit as f32 / lines_found as f32;

    let line_coverage =
      format!("{:.3}% ({}/{})", line_ratio * 100.0, lines_hit, lines_found,);

    if line_ratio >= 0.9 {
      println!("{}", colors::green(&line_coverage));
    } else if line_ratio >= 0.75 {
      println!("{}", colors::yellow(&line_coverage));
    } else {
      println!("{}", colors::red(&line_coverage));
    }

    let mut last_line = None;
    for line_index in missed_lines {
      const WIDTH: usize = 4;
      const SEPERATOR: &str = "|";

      // Put a horizontal separator between disjoint runs of lines
      if let Some(last_line) = last_line {
        if last_line + 1 != line_index {
          let dash = colors::gray("-".repeat(WIDTH + 1));
          println!("{}{}{}", dash, colors::gray(SEPERATOR), dash);
        }
      }

      println!(
        "{:width$} {} {}",
        line_index + 1,
        colors::gray(SEPERATOR),
        colors::red(&lines[line_index]),
        width = WIDTH
      );

      last_line = Some(line_index);
    }
  }

  fn done(&mut self) {}
}

fn collect_coverages(
  files: Vec<PathBuf>,
  ignore: Vec<PathBuf>,
) -> Result<Vec<ScriptCoverage>, AnyError> {
  let mut coverages: Vec<ScriptCoverage> = Vec::new();
  let file_paths = collect_files(&files, &ignore, |file_path| {
    file_path.extension().map_or(false, |ext| ext == "json")
  })?;

  for file_path in file_paths {
    let json = fs::read_to_string(file_path.as_path())?;
    let new_coverage: ScriptCoverage = serde_json::from_str(&json)?;

    let existing_coverage =
      coverages.iter_mut().find(|x| x.url == new_coverage.url);

    if let Some(existing_coverage) = existing_coverage {
      for new_function in new_coverage.functions {
        let existing_function = existing_coverage
          .functions
          .iter_mut()
          .find(|x| x.function_name == new_function.function_name);

        if let Some(existing_function) = existing_function {
          for new_range in new_function.ranges {
            let existing_range =
              existing_function.ranges.iter_mut().find(|x| {
                x.start_offset == new_range.start_offset
                  && x.end_offset == new_range.end_offset
              });

            if let Some(existing_range) = existing_range {
              existing_range.count += new_range.count;
            } else {
              existing_function.ranges.push(new_range);
            }
          }
        } else {
          existing_coverage.functions.push(new_function);
        }
      }
    } else {
      coverages.push(new_coverage);
    }
  }

  coverages.sort_by_key(|k| k.url.clone());

  Ok(coverages)
}

fn filter_coverages(
  coverages: Vec<ScriptCoverage>,
  include: Vec<String>,
  exclude: Vec<String>,
) -> Vec<ScriptCoverage> {
  let include: Vec<Regex> =
    include.iter().map(|e| Regex::new(e).unwrap()).collect();

  let exclude: Vec<Regex> =
    exclude.iter().map(|e| Regex::new(e).unwrap()).collect();

  coverages
    .into_iter()
    .filter(|e| {
      let is_internal = e.url.starts_with("deno:")
        || e.url.ends_with("__anonymous__")
        || e.url.ends_with("$deno$test.js");

      let is_included = include.iter().any(|p| p.is_match(&e.url));
      let is_excluded = exclude.iter().any(|p| p.is_match(&e.url));

      (include.is_empty() || is_included) && !is_excluded && !is_internal
    })
    .collect::<Vec<ScriptCoverage>>()
}

pub async fn cover_files(
  flags: Flags,
  files: Vec<PathBuf>,
  ignore: Vec<PathBuf>,
  include: Vec<String>,
  exclude: Vec<String>,
  lcov: bool,
) -> Result<(), AnyError> {
  let ps = ProcState::build(flags).await?;

  let script_coverages = collect_coverages(files, ignore)?;
  let script_coverages = filter_coverages(script_coverages, include, exclude);

  let reporter_kind = if lcov {
    CoverageReporterKind::Lcov
  } else {
    CoverageReporterKind::Pretty
  };

  let mut reporter = create_reporter(reporter_kind);

  for script_coverage in script_coverages {
    let module_specifier =
      deno_core::resolve_url_or_path(&script_coverage.url)?;
    ps.prepare_module_load(
      module_specifier.clone(),
      TypeLib::UnstableDenoWindow,
      Permissions::allow_all(),
      Permissions::allow_all(),
      false,
      ps.maybe_import_map.clone(),
    )
    .await?;

    let module_source = ps.load(module_specifier.clone(), None)?;
    let script_source = &module_source.code;

    let maybe_source_map = ps.get_source_map(&script_coverage.url);
    let maybe_cached_source = ps
      .file_fetcher
      .get_source(&module_specifier)
      .map(|f| f.source);

    reporter.visit_coverage(
      &script_coverage,
      script_source,
      maybe_source_map,
      maybe_cached_source,
    );
  }

  reporter.done();

  Ok(())
}