1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-23 15:49:44 -05:00

fix(test): capture worker stdout and stderr in test output (#14410)

This commit is contained in:
David Sherret 2022-04-26 19:00:04 -04:00 committed by GitHub
parent 2c33293f66
commit 58eab0e2b3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 224 additions and 119 deletions

View file

@ -236,7 +236,7 @@ jobs:
~/.cargo/registry/index
~/.cargo/registry/cache
~/.cargo/git/db
key: 8-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
key: 9-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
# In main branch, always creates fresh cache
- name: Cache build output (main)
@ -252,7 +252,7 @@ jobs:
!./target/*/*.zip
!./target/*/*.tar.gz
key: |
8-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-${{ github.sha }}
9-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-${{ github.sha }}
# Restore cache from the latest 'main' branch build.
- name: Cache build output (PR)
@ -268,7 +268,7 @@ jobs:
!./target/*/*.tar.gz
key: never_saved
restore-keys: |
8-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-
9-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-
# Don't save cache after building PRs or branches other than 'main'.
- name: Skip save cache (PR)

View file

@ -14,7 +14,6 @@ use crate::lsp::client::TestingNotification;
use crate::lsp::config;
use crate::lsp::logging::lsp_log;
use crate::ops;
use crate::ops::testing::create_stdout_stderr_pipes;
use crate::proc_state;
use crate::tools::test;
@ -27,6 +26,8 @@ use deno_core::parking_lot::Mutex;
use deno_core::serde_json::json;
use deno_core::serde_json::Value;
use deno_core::ModuleSpecifier;
use deno_runtime::ops::io::Stdio;
use deno_runtime::ops::io::StdioPipe;
use deno_runtime::permissions::Permissions;
use deno_runtime::tokio_util::run_basic;
use std::collections::HashMap;
@ -184,17 +185,17 @@ async fn test_specifier(
options: Option<Value>,
) -> Result<(), AnyError> {
if !token.is_cancelled() {
let (stdout_writer, stderr_writer) =
create_stdout_stderr_pipes(channel.clone());
let (stdout, stderr) = test::create_stdout_stderr_pipes(channel.clone());
let mut worker = create_main_worker(
&ps,
specifier.clone(),
permissions,
vec![ops::testing::init(
channel.clone(),
stdout_writer,
stderr_writer,
)],
vec![ops::testing::init(channel.clone())],
Stdio {
stdin: StdioPipe::Inherit,
stdout: StdioPipe::File(stdout),
stderr: StdioPipe::File(stderr),
},
);
worker

View file

@ -123,7 +123,10 @@ fn create_web_worker_preload_module_callback(
})
}
fn create_web_worker_callback(ps: ProcState) -> Arc<CreateWebWorkerCb> {
fn create_web_worker_callback(
ps: ProcState,
stdio: deno_runtime::ops::io::Stdio,
) -> Arc<CreateWebWorkerCb> {
Arc::new(move |args| {
let maybe_inspector_server = ps.maybe_inspector_server.clone();
@ -131,7 +134,8 @@ fn create_web_worker_callback(ps: ProcState) -> Arc<CreateWebWorkerCb> {
ps.clone(),
args.parent_permissions.clone(),
);
let create_web_worker_cb = create_web_worker_callback(ps.clone());
let create_web_worker_cb =
create_web_worker_callback(ps.clone(), stdio.clone());
let preload_module_cb =
create_web_worker_preload_module_callback(ps.clone());
@ -177,6 +181,7 @@ fn create_web_worker_callback(ps: ProcState) -> Arc<CreateWebWorkerCb> {
shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()),
compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()),
maybe_exit_code: args.maybe_exit_code,
stdio: stdio.clone(),
};
WebWorker::bootstrap_from_options(
@ -194,13 +199,15 @@ pub fn create_main_worker(
main_module: ModuleSpecifier,
permissions: Permissions,
mut custom_extensions: Vec<Extension>,
stdio: deno_runtime::ops::io::Stdio,
) -> MainWorker {
let module_loader = CliModuleLoader::new(ps.clone());
let maybe_inspector_server = ps.maybe_inspector_server.clone();
let should_break_on_first_statement = ps.flags.inspect_brk.is_some();
let create_web_worker_cb = create_web_worker_callback(ps.clone());
let create_web_worker_cb =
create_web_worker_callback(ps.clone(), stdio.clone());
let web_worker_preload_module_cb =
create_web_worker_preload_module_callback(ps.clone());
@ -269,6 +276,7 @@ pub fn create_main_worker(
broadcast_channel: ps.broadcast_channel.clone(),
shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()),
compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()),
stdio,
};
MainWorker::bootstrap_from_options(main_module, permissions, options)
@ -510,8 +518,13 @@ async fn install_command(
Permissions::from_options(&preload_flags.permissions_options());
let ps = ProcState::build(Arc::new(preload_flags)).await?;
let main_module = resolve_url_or_path(&install_flags.module_url)?;
let mut worker =
create_main_worker(&ps, main_module.clone(), permissions, vec![]);
let mut worker = create_main_worker(
&ps,
main_module.clone(),
permissions,
vec![],
Default::default(),
);
// First, fetch and compile the module; this step ensures that the module exists.
worker.preload_module(&main_module, true).await?;
tools::installer::install(flags, install_flags)?;
@ -605,8 +618,13 @@ async fn eval_command(
resolve_url_or_path(&format!("./$deno$eval.{}", eval_flags.ext)).unwrap();
let permissions = Permissions::from_options(&flags.permissions_options());
let ps = ProcState::build(Arc::new(flags)).await?;
let mut worker =
create_main_worker(&ps, main_module.clone(), permissions, vec![]);
let mut worker = create_main_worker(
&ps,
main_module.clone(),
permissions,
vec![],
Default::default(),
);
// Create a dummy source file.
let source_code = if eval_flags.print {
format!("console.log({})", eval_flags.code)
@ -920,8 +938,13 @@ async fn repl_command(
let main_module = resolve_url_or_path("./$deno$repl.ts").unwrap();
let permissions = Permissions::from_options(&flags.permissions_options());
let ps = ProcState::build(Arc::new(flags)).await?;
let mut worker =
create_main_worker(&ps, main_module.clone(), permissions, vec![]);
let mut worker = create_main_worker(
&ps,
main_module.clone(),
permissions,
vec![],
Default::default(),
);
if ps.flags.compat {
worker.execute_side_module(&compat::GLOBAL_URL).await?;
compat::add_global_require(&mut worker.js_runtime, main_module.as_str())?;
@ -937,8 +960,13 @@ async fn run_from_stdin(flags: Flags) -> Result<i32, AnyError> {
let ps = ProcState::build(Arc::new(flags)).await?;
let permissions = Permissions::from_options(&ps.flags.permissions_options());
let main_module = resolve_url_or_path("./$deno$stdin.ts").unwrap();
let mut worker =
create_main_worker(&ps.clone(), main_module.clone(), permissions, vec![]);
let mut worker = create_main_worker(
&ps.clone(),
main_module.clone(),
permissions,
vec![],
Default::default(),
);
let mut source = Vec::new();
std::io::stdin().read_to_end(&mut source)?;
@ -1125,7 +1153,13 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<i32, AnyError> {
// We make use an module executor guard to ensure that unload is always fired when an
// operation is called.
let mut executor = FileWatcherModuleExecutor::new(
create_main_worker(&ps, main_module.clone(), permissions, vec![]),
create_main_worker(
&ps,
main_module.clone(),
permissions,
vec![],
Default::default(),
),
flags.compat,
);
@ -1168,8 +1202,13 @@ async fn run_command(
let main_module = resolve_url_or_path(&run_flags.script)?;
let ps = ProcState::build(Arc::new(flags)).await?;
let permissions = Permissions::from_options(&ps.flags.permissions_options());
let mut worker =
create_main_worker(&ps, main_module.clone(), permissions, vec![]);
let mut worker = create_main_worker(
&ps,
main_module.clone(),
permissions,
vec![],
Default::default(),
);
let mut maybe_coverage_collector =
if let Some(ref coverage_dir) = ps.coverage_dir {

View file

@ -1,31 +1,21 @@
use std::cell::RefCell;
use std::io::Read;
use std::rc::Rc;
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use crate::tools::test::TestEvent;
use crate::tools::test::TestOutput;
use deno_core::error::generic_error;
use deno_core::error::AnyError;
use deno_core::op;
use deno_core::Extension;
use deno_core::ModuleSpecifier;
use deno_core::OpState;
use deno_runtime::ops::io::StdFileResource;
use deno_runtime::permissions::create_child_permissions;
use deno_runtime::permissions::ChildPermissionsArg;
use deno_runtime::permissions::Permissions;
use tokio::sync::mpsc::UnboundedSender;
use uuid::Uuid;
pub fn init(
sender: UnboundedSender<TestEvent>,
stdout_writer: os_pipe::PipeWriter,
stderr_writer: os_pipe::PipeWriter,
) -> Extension {
// todo(dsheret): don't do this? Taking out the writers was necessary to prevent invalid handle panics
let stdout_writer = Rc::new(RefCell::new(Some(stdout_writer)));
let stderr_writer = Rc::new(RefCell::new(Some(stderr_writer)));
pub fn init(sender: UnboundedSender<TestEvent>) -> Extension {
Extension::builder()
.ops(vec![
op_pledge_test_permissions::decl(),
@ -38,74 +28,12 @@ pub fn init(
_ => op,
})
.state(move |state| {
state.resource_table.replace(
1,
StdFileResource::stdio(
&pipe_writer_to_file(&stdout_writer.borrow_mut().take().unwrap()),
"stdout",
),
);
state.resource_table.replace(
2,
StdFileResource::stdio(
&pipe_writer_to_file(&stderr_writer.borrow_mut().take().unwrap()),
"stderr",
),
);
state.put(sender.clone());
Ok(())
})
.build()
}
#[cfg(windows)]
fn pipe_writer_to_file(writer: &os_pipe::PipeWriter) -> std::fs::File {
use std::os::windows::prelude::AsRawHandle;
use std::os::windows::prelude::FromRawHandle;
unsafe { std::fs::File::from_raw_handle(writer.as_raw_handle()) }
}
#[cfg(unix)]
fn pipe_writer_to_file(writer: &os_pipe::PipeWriter) -> std::fs::File {
use std::os::unix::io::AsRawFd;
use std::os::unix::io::FromRawFd;
unsafe { std::fs::File::from_raw_fd(writer.as_raw_fd()) }
}
/// Creates the stdout and stderr pipes and returns the writers for stdout and stderr.
pub fn create_stdout_stderr_pipes(
sender: UnboundedSender<TestEvent>,
) -> (os_pipe::PipeWriter, os_pipe::PipeWriter) {
let (stdout_reader, stdout_writer) = os_pipe::pipe().unwrap();
let (stderr_reader, stderr_writer) = os_pipe::pipe().unwrap();
start_output_redirect_thread(stdout_reader, sender.clone());
start_output_redirect_thread(stderr_reader, sender);
(stdout_writer, stderr_writer)
}
fn start_output_redirect_thread(
mut pipe_reader: os_pipe::PipeReader,
sender: UnboundedSender<TestEvent>,
) {
tokio::task::spawn_blocking(move || loop {
let mut buffer = [0; 512];
let size = match pipe_reader.read(&mut buffer) {
Ok(0) | Err(_) => break,
Ok(size) => size,
};
if sender
.send(TestEvent::Output(TestOutput::Bytes(
buffer[0..size].to_vec(),
)))
.is_err()
{
break;
}
});
}
#[derive(Clone)]
struct PermissionsHolder(Uuid, Permissions);

View file

@ -305,6 +305,7 @@ pub async fn run(
broadcast_channel,
shared_array_buffer_store: None,
compiled_wasm_module_store: None,
stdio: Default::default(),
};
let mut worker = MainWorker::bootstrap_from_options(
main_module.clone(),

View file

@ -302,10 +302,10 @@ itest!(no_prompt_with_denied_perms {
output: "test/no_prompt_with_denied_perms.out",
});
itest!(captured_subprocess_output {
args: "test --allow-run --allow-read --unstable test/captured_subprocess_output.ts",
itest!(captured_output {
args: "test --allow-run --allow-read --unstable test/captured_output.ts",
exit_code: 0,
output: "test/captured_subprocess_output.out",
output: "test/captured_output.out",
});
#[test]

View file

@ -1,5 +1,5 @@
[WILDCARD]
running 1 test from [WILDCARD]/captured_subprocess_output.ts
running 1 test from [WILDCARD]/captured_output.ts
output ...
------- output -------
1
@ -10,6 +10,8 @@ output ...
6
7
8
9
10
----- output end -----
ok ([WILDCARD]s)

View file

@ -20,4 +20,16 @@ Deno.test("output", async () => {
stderr: "inherit",
});
await c.status;
const worker = new Worker(
new URL("./captured_output.worker.js", import.meta.url).href,
{ type: "module" },
);
// ensure worker output is captured
const response = new Promise<void>((resolve) =>
worker.onmessage = () => resolve()
);
worker.postMessage({});
await response;
worker.terminate();
});

View file

@ -0,0 +1,6 @@
self.onmessage = () => {
console.log(9);
console.error(10);
self.postMessage({});
self.close();
};

View file

@ -385,6 +385,7 @@ async fn bench_specifier(
specifier.clone(),
permissions,
vec![ops::bench::init(channel.clone(), ps.flags.unstable)],
Default::default(),
);
if options.compat_mode {

View file

@ -22,7 +22,6 @@ use crate::graph_util::graph_valid;
use crate::located_script_name;
use crate::lockfile;
use crate::ops;
use crate::ops::testing::create_stdout_stderr_pipes;
use crate::proc_state::ProcState;
use crate::resolver::ImportMapResolver;
use crate::resolver::JsxResolver;
@ -41,6 +40,8 @@ use deno_core::serde_json::json;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_graph::ModuleKind;
use deno_runtime::ops::io::Stdio;
use deno_runtime::ops::io::StdioPipe;
use deno_runtime::permissions::Permissions;
use deno_runtime::tokio_util::run_basic;
use log::Level;
@ -52,6 +53,7 @@ use serde::Deserialize;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
use std::io::Read;
use std::io::Write;
use std::num::NonZeroUsize;
use std::path::PathBuf;
@ -588,17 +590,17 @@ async fn test_specifier(
channel: UnboundedSender<TestEvent>,
options: TestSpecifierOptions,
) -> Result<(), AnyError> {
let (stdout_writer, stderr_writer) =
create_stdout_stderr_pipes(channel.clone());
let (stdout, stderr) = create_stdout_stderr_pipes(channel.clone());
let mut worker = create_main_worker(
&ps,
specifier.clone(),
permissions,
vec![ops::testing::init(
channel.clone(),
stdout_writer,
stderr_writer,
)],
vec![ops::testing::init(channel.clone())],
Stdio {
stdin: StdioPipe::Inherit,
stdout: StdioPipe::File(stdout),
stderr: StdioPipe::File(stderr),
},
);
let mut maybe_coverage_collector = if let Some(ref coverage_dir) =
@ -1452,3 +1454,56 @@ pub async fn run_tests_with_watch(
Ok(())
}
/// Creates the stdout and stderr pipes and returns the writers for stdout and stderr.
pub fn create_stdout_stderr_pipes(
sender: UnboundedSender<TestEvent>,
) -> (std::fs::File, std::fs::File) {
let (stdout_reader, stdout_writer) = os_pipe::pipe().unwrap();
let (stderr_reader, stderr_writer) = os_pipe::pipe().unwrap();
start_output_redirect_thread(stdout_reader, sender.clone());
start_output_redirect_thread(stderr_reader, sender);
(
pipe_writer_to_file(stdout_writer),
pipe_writer_to_file(stderr_writer),
)
}
#[cfg(windows)]
fn pipe_writer_to_file(writer: os_pipe::PipeWriter) -> std::fs::File {
use std::os::windows::prelude::FromRawHandle;
use std::os::windows::prelude::IntoRawHandle;
// SAFETY: Requires consuming ownership of the provided handle
unsafe { std::fs::File::from_raw_handle(writer.into_raw_handle()) }
}
#[cfg(unix)]
fn pipe_writer_to_file(writer: os_pipe::PipeWriter) -> std::fs::File {
use std::os::unix::io::FromRawFd;
use std::os::unix::io::IntoRawFd;
// SAFETY: Requires consuming ownership of the provided handle
unsafe { std::fs::File::from_raw_fd(writer.into_raw_fd()) }
}
fn start_output_redirect_thread(
mut pipe_reader: os_pipe::PipeReader,
sender: UnboundedSender<TestEvent>,
) {
tokio::task::spawn_blocking(move || loop {
let mut buffer = [0; 512];
let size = match pipe_reader.read(&mut buffer) {
Ok(0) | Err(_) => break,
Ok(size) => size,
};
if sender
.send(TestEvent::Output(TestOutput::Bytes(
buffer[0..size].to_vec(),
)))
.is_err()
{
break;
}
});
}

View file

@ -57,6 +57,7 @@ async fn main() -> Result<(), AnyError> {
broadcast_channel: InMemoryBroadcastChannel::default(),
shared_array_buffer_store: None,
compiled_wasm_module_store: None,
stdio: Default::default(),
};
let js_path =

View file

@ -77,13 +77,67 @@ pub fn init() -> Extension {
.build()
}
pub fn init_stdio() -> Extension {
pub enum StdioPipe {
Inherit,
File(StdFile),
}
impl Default for StdioPipe {
fn default() -> Self {
Self::Inherit
}
}
impl Clone for StdioPipe {
fn clone(&self) -> Self {
match self {
StdioPipe::Inherit => StdioPipe::Inherit,
StdioPipe::File(pipe) => StdioPipe::File(pipe.try_clone().unwrap()),
}
}
}
/// Specify how stdin, stdout, and stderr are piped.
/// By default, inherits from the process.
#[derive(Clone, Default)]
pub struct Stdio {
pub stdin: StdioPipe,
pub stdout: StdioPipe,
pub stderr: StdioPipe,
}
pub fn init_stdio(stdio: Stdio) -> Extension {
// todo(dsheret): don't do this? Taking out the writers was necessary to prevent invalid handle panics
let stdio = Rc::new(RefCell::new(Some(stdio)));
Extension::builder()
.state(|state| {
.state(move |state| {
let stdio = stdio
.borrow_mut()
.take()
.expect("Extension only supports being used once.");
let t = &mut state.resource_table;
t.add(StdFileResource::stdio(&STDIN_HANDLE, "stdin"));
t.add(StdFileResource::stdio(&STDOUT_HANDLE, "stdout"));
t.add(StdFileResource::stdio(&STDERR_HANDLE, "stderr"));
t.add(StdFileResource::stdio(
match &stdio.stdin {
StdioPipe::Inherit => &STDIN_HANDLE,
StdioPipe::File(pipe) => pipe,
},
"stdin",
));
t.add(StdFileResource::stdio(
match &stdio.stdout {
StdioPipe::Inherit => &STDOUT_HANDLE,
StdioPipe::File(pipe) => pipe,
},
"stdout",
));
t.add(StdFileResource::stdio(
match &stdio.stderr {
StdioPipe::Inherit => &STDERR_HANDLE,
StdioPipe::File(pipe) => pipe,
},
"stderr",
));
Ok(())
})
.build()

View file

@ -3,6 +3,7 @@ use crate::colors;
use crate::inspector_server::InspectorServer;
use crate::js;
use crate::ops;
use crate::ops::io::Stdio;
use crate::permissions::Permissions;
use crate::tokio_util::run_basic;
use crate::BootstrapOptions;
@ -335,6 +336,7 @@ pub struct WebWorkerOptions {
pub shared_array_buffer_store: Option<SharedArrayBufferStore>,
pub compiled_wasm_module_store: Option<CompiledWasmModuleStore>,
pub maybe_exit_code: Option<Arc<AtomicI32>>,
pub stdio: Stdio,
}
impl WebWorker {
@ -411,7 +413,7 @@ impl WebWorker {
ops::fs_events::init().enabled(options.use_deno_namespace),
ops::fs::init().enabled(options.use_deno_namespace),
ops::io::init(),
ops::io::init_stdio().enabled(options.use_deno_namespace),
ops::io::init_stdio(options.stdio).enabled(options.use_deno_namespace),
deno_tls::init().enabled(options.use_deno_namespace),
deno_net::init::<Permissions>(
options.root_cert_store.clone(),

View file

@ -3,6 +3,7 @@
use crate::inspector_server::InspectorServer;
use crate::js;
use crate::ops;
use crate::ops::io::Stdio;
use crate::permissions::Permissions;
use crate::BootstrapOptions;
use deno_broadcast_channel::InMemoryBroadcastChannel;
@ -65,6 +66,7 @@ pub struct WorkerOptions {
pub broadcast_channel: InMemoryBroadcastChannel,
pub shared_array_buffer_store: Option<SharedArrayBufferStore>,
pub compiled_wasm_module_store: Option<CompiledWasmModuleStore>,
pub stdio: Stdio,
}
impl MainWorker {
@ -136,7 +138,7 @@ impl MainWorker {
ops::fs_events::init(),
ops::fs::init(),
ops::io::init(),
ops::io::init_stdio(),
ops::io::init_stdio(options.stdio),
deno_tls::init(),
deno_net::init::<Permissions>(
options.root_cert_store.clone(),
@ -390,6 +392,7 @@ mod tests {
broadcast_channel: InMemoryBroadcastChannel::default(),
shared_array_buffer_store: None,
compiled_wasm_module_store: None,
stdio: Default::default(),
};
MainWorker::bootstrap_from_options(main_module, permissions, options)