chore: migrate to new deno_core and metrics (#21057)

- Uses the new OpMetrics system for sync and async calls
- Partial revert of #21048 as we moved Array.fromAsync upstream to
deno_core

Cargo.lock

@@ -1113,9 +1113,9 @@ dependencies = [
 
 [[package]]
 name = "deno_core"
-version = "0.224.0"
+version = "0.225.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d23e2333ee4155236c3c7be8ec5ad9586eed854a47fe3804924627ca5a476e8"
+checksum = "68a284bcdd664d5dbbb8efce93b7822b9a0f125a1b53f0bc8edb876e4cee47ed"
 dependencies = [
  "anyhow",
  "bytes",
@@ -1539,9 +1539,9 @@ dependencies = [
 
 [[package]]
 name = "deno_ops"
-version = "0.100.0"
+version = "0.101.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "466fad92ea239f85cad8de032660c9b8d6fe361260ce3169b5893592902fe2f4"
+checksum = "e1a813d4ea049601db9e5330c307b177c4f38e4baddf931c3c00c1a625b3dc1f"
 dependencies = [
  "deno-proc-macro-rules",
  "lazy-regex",
@@ -4783,9 +4783,9 @@ dependencies = [
 
 [[package]]
 name = "serde_v8"
-version = "0.133.0"
+version = "0.134.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8ea791b0c83c4f4c6684b6d2e04aa8f936f3abbdff613624553fe5b80ea7c0c"
+checksum = "b43265d540cbeb168d730b5df2069f8dfb9de318201f4e8f4f956876266432af"
 dependencies = [
  "bytes",
  "derive_more",

Cargo.toml

@@ -40,7 +40,7 @@ repository = "https://github.com/denoland/deno"
 
 [workspace.dependencies]
 deno_ast = { version = "0.31.2", features = ["transpiling"] }
-deno_core = { version = "0.224.0" }
+deno_core = { version = "0.225.0" }
 
 deno_runtime = { version = "0.130.0", path = "./runtime" }
 napi_sym = { version = "0.52.0", path = "./cli/napi/sym" }

cli/args/flags.rs

@@ -398,6 +398,7 @@ pub struct Flags {
   pub config_flag: ConfigFlag,
   pub node_modules_dir: Option<bool>,
   pub vendor: Option<bool>,
+  pub enable_op_summary_metrics: bool,
   pub enable_testing_features: bool,
   pub ext: Option<String>,
   pub ignore: Vec<PathBuf>,

cli/args/mod.rs

@@ -1137,6 +1137,14 @@ impl CliOptions {
     }
   }
 
+  pub fn enable_op_summary_metrics(&self) -> bool {
+    self.flags.enable_op_summary_metrics
+      || matches!(
+        self.flags.subcommand,
+        DenoSubcommand::Test(_) | DenoSubcommand::Repl(_)
+      )
+  }
+
   pub fn enable_testing_features(&self) -> bool {
     self.flags.enable_testing_features
   }

cli/factory.rs

@@ -670,6 +670,7 @@ impl CliFactory {
       argv: self.options.argv().clone(),
       log_level: self.options.log_level().unwrap_or(log::Level::Info).into(),
       coverage_dir: self.options.coverage_dir(),
+      enable_op_summary_metrics: self.options.enable_op_summary_metrics(),
       enable_testing_features: self.options.enable_testing_features(),
       has_node_modules_dir: self.options.has_node_modules_dir(),
       hmr: self.options.has_hmr(),

cli/main.rs

@@ -367,8 +367,7 @@ pub fn main() {
       // Using same default as VSCode:
       // https://github.com/microsoft/vscode/blob/48d4ba271686e8072fc6674137415bc80d936bc7/extensions/typescript-language-features/src/configuration/configuration.ts#L213-L214
       DenoSubcommand::Lsp => vec!["--max-old-space-size=3072".to_string()],
-      // TODO(bartlomieju): upstream this to `deno_core` crate
-      _ => vec!["--harmony-array-from-async".to_string()],
+      _ => vec![],
     };
     init_v8_flags(&default_v8_flags, &flags.v8_flags, get_v8_flags_from_env());
     deno_core::JsRuntime::init_platform(None);

cli/ops/testing.rs

@@ -14,16 +14,17 @@ use deno_core::error::AnyError;
 use deno_core::op2;
 use deno_core::v8;
 use deno_core::ModuleSpecifier;
-use deno_core::OpMetrics;
+use deno_core::OpMetricsSummary;
+use deno_core::OpMetricsSummaryTracker;
 use deno_core::OpState;
 use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::permissions::create_child_permissions;
 use deno_runtime::permissions::ChildPermissionsArg;
 use deno_runtime::permissions::PermissionsContainer;
 use serde::Serialize;
-use std::cell::Ref;
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
+use std::rc::Rc;
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering;
 use uuid::Uuid;
@@ -243,38 +244,28 @@ fn op_test_event_step_result_failed(
 struct TestOpSanitizers(HashMap<u32, TestOpSanitizerState>);
 
 enum TestOpSanitizerState {
-  Collecting { metrics: Vec<OpMetrics> },
+  Collecting { metrics: Vec<OpMetricsSummary> },
   Finished { report: Vec<TestOpSanitizerReport> },
 }
 
 fn try_collect_metrics(
-  state: &OpState,
+  metrics: &OpMetricsSummaryTracker,
   force: bool,
   op_id_host_recv_msg: usize,
   op_id_host_recv_ctrl: usize,
-) -> Result<Ref<Vec<OpMetrics>>, bool> {
-  let metrics = state.tracker.per_op();
-  for op_metric in &*metrics {
-    let has_pending_ops = op_metric.ops_dispatched_async
-      + op_metric.ops_dispatched_async_unref
-      > op_metric.ops_completed_async + op_metric.ops_completed_async_unref;
-    if has_pending_ops && !force {
-      let host_recv_msg = metrics
-        .get(op_id_host_recv_msg)
-        .map(|op_metric| {
-          op_metric.ops_dispatched_async + op_metric.ops_dispatched_async_unref
-            > op_metric.ops_completed_async
-              + op_metric.ops_completed_async_unref
-        })
-        .unwrap_or(false);
-      let host_recv_ctrl = metrics
-        .get(op_id_host_recv_ctrl)
-        .map(|op_metric| {
-          op_metric.ops_dispatched_async + op_metric.ops_dispatched_async_unref
-            > op_metric.ops_completed_async
-              + op_metric.ops_completed_async_unref
-        })
-        .unwrap_or(false);
+) -> Result<std::cell::Ref<Vec<OpMetricsSummary>>, bool> {
+  let metrics = metrics.per_op();
+  let host_recv_msg = metrics
+    .get(op_id_host_recv_msg)
+    .map(OpMetricsSummary::has_outstanding_ops)
+    .unwrap_or(false);
+  let host_recv_ctrl = metrics
+    .get(op_id_host_recv_ctrl)
+    .map(OpMetricsSummary::has_outstanding_ops)
+    .unwrap_or(false);
+
+  for op_metric in metrics.iter() {
+    if op_metric.has_outstanding_ops() && !force {
       return Err(host_recv_msg || host_recv_ctrl);
     }
   }
@@ -294,23 +285,23 @@ fn op_test_op_sanitizer_collect(
   #[smi] op_id_host_recv_msg: usize,
   #[smi] op_id_host_recv_ctrl: usize,
 ) -> Result<u8, AnyError> {
-  let metrics = {
-    let metrics = match try_collect_metrics(
-      state,
-      force,
-      op_id_host_recv_msg,
-      op_id_host_recv_ctrl,
-    ) {
-      Ok(metrics) => metrics,
-      Err(false) => {
-        return Ok(1);
-      }
-      Err(true) => {
-        return Ok(2);
-      }
-    };
-    metrics.clone()
-  };
+  let metrics = state.borrow::<Rc<OpMetricsSummaryTracker>>();
+  let metrics = match try_collect_metrics(
+    metrics,
+    force,
+    op_id_host_recv_msg,
+    op_id_host_recv_ctrl,
+  ) {
+    Ok(metrics) => metrics,
+    Err(false) => {
+      return Ok(1);
+    }
+    Err(true) => {
+      return Ok(2);
+    }
+  }
+  .clone();
+
   let op_sanitizers = state.borrow_mut::<TestOpSanitizers>();
   match op_sanitizers.0.entry(id) {
     Entry::Vacant(entry) => {
@@ -348,11 +339,12 @@ fn op_test_op_sanitizer_finish(
 ) -> Result<u8, AnyError> {
   // Drop `fetch` connection pool at the end of a test
   state.try_take::<reqwest::Client>();
+  let metrics = state.borrow::<Rc<OpMetricsSummaryTracker>>();
 
   // Generate a report of pending ops
   let report = {
     let after_metrics = match try_collect_metrics(
-      state,
+      metrics,
       force,
       op_id_host_recv_msg,
       op_id_host_recv_ctrl,
@@ -380,14 +372,10 @@ fn op_test_op_sanitizer_finish(
     for (id, (before, after)) in
       before_metrics.iter().zip(after_metrics.iter()).enumerate()
     {
-      let async_pending_before = before.ops_dispatched_async
-        + before.ops_dispatched_async_unref
-        - before.ops_completed_async
-        - before.ops_completed_async_unref;
-      let async_pending_after = after.ops_dispatched_async
-        + after.ops_dispatched_async_unref
-        - after.ops_completed_async
-        - after.ops_completed_async_unref;
+      let async_pending_before =
+        before.ops_dispatched_async - before.ops_completed_async;
+      let async_pending_after =
+        after.ops_dispatched_async - after.ops_completed_async;
       let diff = async_pending_after as i64 - async_pending_before as i64;
       if diff != 0 {
         report.push(TestOpSanitizerReport { id, diff });

cli/standalone/mod.rs

@@ -453,6 +453,7 @@ pub async fn run(
       argv: metadata.argv,
       log_level: WorkerLogLevel::Info,
       coverage_dir: None,
+      enable_op_summary_metrics: false,
       enable_testing_features: false,
       has_node_modules_dir,
       hmr: false,

cli/tests/integration/js_unit_tests.rs

@@ -53,7 +53,6 @@ util::unit_test_factory!(
     link_test,
     make_temp_test,
     message_channel_test,
-    metrics_test,
     mkdir_test,
     navigator_test,
     net_test,

cli/tests/integration/run_tests.rs

@@ -3425,11 +3425,6 @@ itest!(unstable_ffi_19 {
   exit_code: 70,
 });
 
-itest!(future_check2 {
-  args: "run --check run/future_check.ts",
-  output: "run/future_check2.out",
-});
-
 itest!(event_listener_error {
   args: "run --quiet run/event_listener_error.ts",
   output: "run/event_listener_error.ts.out",

cli/tests/testdata/run/future_check.ts

@@ -1 +0,0 @@
-Deno.metrics();

cli/tests/testdata/run/future_check2.out

@@ -1 +0,0 @@
-Check [WILDCARD]/future_check.ts

cli/tests/unit/metrics_test.ts

@@ -1,93 +0,0 @@
-// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
-import { assert, assertEquals } from "./test_util.ts";
-
-Deno.test(async function metrics() {
-  // Write to stdout to ensure a "data" message gets sent instead of just
-  // control messages.
-  const dataMsg = new Uint8Array([13, 13, 13]); // "\r\r\r",
-  await Deno.stdout.write(dataMsg);
-
-  // WARNING: bytesReceived & bytesSentControl are now always zero
-  // following https://github.com/denoland/deno/pull/9843
-
-  const m1 = Deno.metrics();
-  assert(m1.opsDispatched > 0);
-  assert(m1.opsCompleted > 0);
-  assert(m1.bytesSentControl === 0);
-  assert(m1.bytesSentData === 0);
-  assert(m1.bytesReceived === 0);
-  const m1OpWrite = m1.ops["op_write"];
-  assert(m1OpWrite.opsDispatchedAsync > 0);
-  assert(m1OpWrite.opsCompletedAsync > 0);
-  assert(m1OpWrite.bytesSentControl === 0);
-  assert(m1OpWrite.bytesSentData >= 0);
-  assert(m1OpWrite.bytesReceived === 0);
-
-  await Deno.stdout.write(dataMsg);
-
-  const m2 = Deno.metrics();
-  assert(m2.opsDispatchedAsync > m1.opsDispatchedAsync);
-  assert(m2.opsCompletedAsync > m1.opsCompletedAsync);
-  assert(m2.bytesSentControl === m1.bytesSentControl);
-  assert(m2.bytesSentData === 0);
-  assert(m2.bytesReceived === m1.bytesReceived);
-  const m2OpWrite = m2.ops["op_write"];
-  assert(m2OpWrite.opsDispatchedAsync > m1OpWrite.opsDispatchedAsync);
-  assert(m2OpWrite.opsCompletedAsync > m1OpWrite.opsCompletedAsync);
-  assert(m2OpWrite.bytesSentControl === m1OpWrite.bytesSentControl);
-  assert(m2OpWrite.bytesSentData === 0);
-  assert(m2OpWrite.bytesReceived === m1OpWrite.bytesReceived);
-});
-
-Deno.test(
-  { permissions: { write: true } },
-  function metricsUpdatedIfNoResponseSync() {
-    const filename = Deno.makeTempDirSync() + "/test.txt";
-
-    const data = new Uint8Array([41, 42, 43]);
-    Deno.writeFileSync(filename, data, { mode: 0o666 });
-
-    const metrics = Deno.metrics();
-    assert(metrics.opsDispatched === metrics.opsCompleted);
-    assert(metrics.opsDispatchedSync === metrics.opsCompletedSync);
-  },
-);
-
-Deno.test(
-  { permissions: { write: true } },
-  async function metricsUpdatedIfNoResponseAsync() {
-    const filename = Deno.makeTempDirSync() + "/test.txt";
-
-    const data = new Uint8Array([41, 42, 43]);
-    await Deno.writeFile(filename, data, { mode: 0o666 });
-
-    const metrics = Deno.metrics();
-    assert(metrics.opsDispatched === metrics.opsCompleted);
-    assert(metrics.opsDispatchedSync === metrics.opsCompletedSync);
-    assert(metrics.opsDispatchedAsync === metrics.opsCompletedAsync);
-  },
-);
-
-// Test that ops from extensions have metrics (via OpMiddleware)
-Deno.test.ignore(function metricsForOpCrates() {
-  const _ = new URL("https://deno.land");
-
-  const m1 = Deno.metrics().ops["op_url_parse"];
-  assert(m1.opsDispatched > 0);
-  assert(m1.opsCompleted > 0);
-});
-
-// Test that op_names == Objects.keys(Deno[Deno.internal].core.ops)
-// since building the per-op metrics depends on op_names being complete
-Deno.test(function opNamesMatch() {
-  // @ts-ignore: Deno[Deno.internal].core allowed
-  const ops = Object.keys(Deno[Deno.internal].core.ops);
-  // @ts-ignore: Deno[Deno.internal].core allowed
-  ops.concat(Object.keys(Deno[Deno.internal].core.asyncOps));
-
-  assertEquals(
-    // @ts-ignore: Deno[Deno.internal].core allowed
-    Deno[Deno.internal].core.opNames().sort(),
-    ops.sort().filter((name) => name !== "asyncOpsInfo"),
-  );
-});

cli/tests/unit/timers_test.ts

@@ -412,19 +412,6 @@ Deno.test(function clearTimeoutAndClearIntervalNotBeEquals() {
   assertNotEquals(clearTimeout, clearInterval);
 });
 
-Deno.test(async function timerMaxCpuBug() {
-  // There was a bug where clearing a timeout would cause Deno to use 100% CPU.
-  clearTimeout(setTimeout(() => {}, 1000));
-  // We can check this by counting how many ops have triggered in the interim.
-  // Certainly less than 10 ops should have been dispatched in next 100 ms.
-  const { ops: pre } = Deno.metrics();
-  await delay(100);
-  const { ops: post } = Deno.metrics();
-  const before = pre.op_sleep.opsDispatched;
-  const after = post.op_sleep.opsDispatched;
-  assert(after - before < 10);
-});
-
 Deno.test(async function timerOrdering() {
   const array: number[] = [];
   const donePromise = deferred();

cli/tsc/mod.rs

@@ -948,7 +948,7 @@ mod tests {
         .context("Unable to get CWD")
         .unwrap(),
     );
-    let mut op_state = OpState::new(1, None);
+    let mut op_state = OpState::new(None);
     op_state.put(state);
     op_state
   }

cli/worker.rs

@@ -86,6 +86,7 @@ pub struct CliMainWorkerOptions {
   pub argv: Vec<String>,
   pub log_level: WorkerLogLevel,
   pub coverage_dir: Option<String>,
+  pub enable_op_summary_metrics: bool,
   pub enable_testing_features: bool,
   pub has_node_modules_dir: bool,
   pub hmr: bool,
@@ -546,6 +547,7 @@ impl CliMainWorkerFactory {
           .map(|p| p.get())
           .unwrap_or(1),
         log_level: shared.options.log_level,
+        enable_op_summary_metrics: shared.options.enable_op_summary_metrics,
         enable_testing_features: shared.options.enable_testing_features,
         locale: deno_core::v8::icu::get_language_tag(),
         location: shared.options.location.clone(),
@@ -732,6 +734,7 @@ fn create_web_worker_callback(
           .map(|p| p.get())
           .unwrap_or(1),
         log_level: shared.options.log_level,
+        enable_op_summary_metrics: shared.options.enable_op_summary_metrics,
         enable_testing_features: shared.options.enable_testing_features,
         locale: deno_core::v8::icu::get_language_tag(),
         location: Some(args.main_module.clone()),

runtime/web_worker.rs

@@ -32,6 +32,7 @@ use deno_core::ModuleCode;
 use deno_core::ModuleId;
 use deno_core::ModuleLoader;
 use deno_core::ModuleSpecifier;
+use deno_core::OpMetricsSummaryTracker;
 use deno_core::RuntimeOptions;
 use deno_core::SharedArrayBufferStore;
 use deno_core::Snapshot;
@@ -507,6 +508,18 @@ impl WebWorker {
     #[cfg(all(feature = "include_js_files_for_snapshotting", feature = "dont_create_runtime_snapshot", not(feature = "__runtime_js_sources")))]
     options.startup_snapshot.as_ref().expect("Sources are not embedded, snapshotting was disabled and a user snapshot was not provided.");
 
+    // Hook up the summary metrics if the user or subcommand requested them
+    let (op_summary_metrics, op_metrics_factory_fn) =
+      if options.bootstrap.enable_op_summary_metrics {
+        let op_summary_metrics = Rc::new(OpMetricsSummaryTracker::default());
+        (
+          Some(op_summary_metrics.clone()),
+          Some(op_summary_metrics.op_metrics_factory_fn(|_| true)),
+        )
+      } else {
+        (None, None)
+      };
+
     // Clear extension modules from the module map, except preserve `node:*`
     // modules as `node:` specifiers.
     let preserve_snapshotted_modules =
@@ -525,9 +538,14 @@ impl WebWorker {
       inspector: options.maybe_inspector_server.is_some(),
       preserve_snapshotted_modules,
       feature_checker: Some(options.feature_checker.clone()),
+      op_metrics_factory_fn,
       ..Default::default()
     });
 
+    if let Some(op_summary_metrics) = op_summary_metrics {
+      js_runtime.op_state().borrow_mut().put(op_summary_metrics);
+    }
+
     if let Some(server) = options.maybe_inspector_server.clone() {
       server.register_inspector(
         main_module.to_string(),

runtime/worker.rs

@@ -27,6 +27,7 @@ use deno_core::ModuleCode;
 use deno_core::ModuleId;
 use deno_core::ModuleLoader;
 use deno_core::ModuleSpecifier;
+use deno_core::OpMetricsSummaryTracker;
 use deno_core::RuntimeOptions;
 use deno_core::SharedArrayBufferStore;
 use deno_core::Snapshot;
@@ -324,6 +325,18 @@ impl MainWorker {
       }
     }
 
+    // Hook up the summary metrics if the user or subcommand requested them
+    let (op_summary_metrics, op_metrics_factory_fn) =
+      if options.bootstrap.enable_op_summary_metrics {
+        let op_summary_metrics = Rc::new(OpMetricsSummaryTracker::default());
+        (
+          Some(op_summary_metrics.clone()),
+          Some(op_summary_metrics.op_metrics_factory_fn(|_| true)),
+        )
+      } else {
+        (None, None)
+      };
+
     extensions.extend(std::mem::take(&mut options.extensions));
 
     #[cfg(all(feature = "include_js_files_for_snapshotting", feature = "dont_create_runtime_snapshot", not(feature = "__runtime_js_sources")))]
@@ -349,9 +362,14 @@ impl MainWorker {
       inspector: options.maybe_inspector_server.is_some(),
       is_main: true,
       feature_checker: Some(options.feature_checker.clone()),
+      op_metrics_factory_fn,
       ..Default::default()
     });
 
+    if let Some(op_summary_metrics) = op_summary_metrics {
+      js_runtime.op_state().borrow_mut().put(op_summary_metrics);
+    }
+
     if let Some(server) = options.maybe_inspector_server.clone() {
       server.register_inspector(
         main_module.to_string(),

runtime/worker_bootstrap.rs

@@ -44,6 +44,7 @@ pub struct BootstrapOptions {
   pub args: Vec<String>,
   pub cpu_count: usize,
   pub log_level: WorkerLogLevel,
+  pub enable_op_summary_metrics: bool,
   pub enable_testing_features: bool,
   pub locale: String,
   pub location: Option<ModuleSpecifier>,
@@ -79,6 +80,7 @@ impl Default for BootstrapOptions {
       cpu_count,
       no_color: !colors::use_color(),
       is_tty: colors::is_tty(),
+      enable_op_summary_metrics: Default::default(),
       enable_testing_features: Default::default(),
       log_level: Default::default(),
       ts_version: Default::default(),