1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-21 15:04:11 -05:00

fix(ext/web): add stream tests to detect v8slice split bug (#20253)

Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>
This commit is contained in:
Matt Mastracci 2023-08-23 17:03:05 -06:00
parent 658e1e126b
commit 09a1fda8e4
42 changed files with 189 additions and 70 deletions

26
Cargo.lock generated
View file

@ -1006,13 +1006,14 @@ dependencies = [
[[package]]
name = "deno_core"
version = "0.202.0"
version = "0.204.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9d4f3ad9c2861e0bb8745e1f228aaee04782a9ab6a3c3bbb887e60d7faf087a"
checksum = "b4ddf51deb9a3bb60a4ab74784414b3f2f89de83a77d6d90a64c6447f7765d68"
dependencies = [
"anyhow",
"bytes",
"deno_ops",
"deno_unsync",
"futures",
"indexmap 1.9.3",
"libc",
@ -1380,9 +1381,9 @@ dependencies = [
[[package]]
name = "deno_ops"
version = "0.80.0"
version = "0.82.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb6a1ceabfbab1c29b32872e68ec994d393b58ccdf12a835d150199555496f3"
checksum = "1b660872f9a9737d3424470483dd6730d2129481af5055449a2a37ab5bc2145e"
dependencies = [
"deno-proc-macro-rules",
"lazy-regex",
@ -1497,6 +1498,15 @@ dependencies = [
"webpki-roots",
]
[[package]]
name = "deno_unsync"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac0984205f25e71ddd1be603d76e70255953c12ff864707359ab195d26dfc7b3"
dependencies = [
"tokio",
]
[[package]]
name = "deno_url"
version = "0.116.0"
@ -4412,9 +4422,9 @@ dependencies = [
[[package]]
name = "serde_v8"
version = "0.113.0"
version = "0.115.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fa7b3ecd650d790ff8781402d0704d35a2f51c3bec87fe92d43eea6d371f05d"
checksum = "36f6cc041512391aabdae4dd11d51e370824ea35bfe896fb2585b6792e28c9bf"
dependencies = [
"bytes",
"derive_more",
@ -5836,9 +5846,9 @@ dependencies = [
[[package]]
name = "v8"
version = "0.75.0"
version = "0.74.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9be435abe79a8427b0969f1ac0a3c0e91644235f68a3de5da4a27ec69666985"
checksum = "2eedac634b8dd39b889c5b62349cbc55913780226239166435c5cf66771792ea"
dependencies = [
"bitflags 1.3.2",
"fslock",

View file

@ -40,7 +40,7 @@ repository = "https://github.com/denoland/deno"
[workspace.dependencies]
deno_ast = { version = "0.28.0", features = ["transpiling"] }
deno_core = "0.202.0"
deno_core = "0.204.0"
deno_runtime = { version = "0.124.0", path = "./runtime" }
napi_sym = { version = "0.46.0", path = "./cli/napi/sym" }

View file

@ -3,7 +3,7 @@
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::parking_lot::MutexGuard;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_runtime::deno_webstorage::rusqlite;
use deno_runtime::deno_webstorage::rusqlite::Connection;
use deno_runtime::deno_webstorage::rusqlite::OptionalExtension;

View file

@ -7,8 +7,8 @@ use std::path::PathBuf;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::serde_json;
use deno_core::task::spawn;
use deno_core::task::JoinHandle;
use deno_core::unsync::spawn;
use deno_core::unsync::JoinHandle;
use deno_runtime::deno_webstorage::rusqlite::params;
use serde::Serialize;

View file

@ -8,7 +8,7 @@ use deno_core::anyhow::bail;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::serde_json::Value;
use deno_core::task::spawn;
use deno_core::unsync::spawn;
use tower_lsp::lsp_types as lsp;
use tower_lsp::lsp_types::ConfigurationItem;

View file

@ -28,9 +28,9 @@ use deno_core::resolve_url;
use deno_core::serde::Deserialize;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::task::spawn;
use deno_core::task::spawn_blocking;
use deno_core::task::JoinHandle;
use deno_core::unsync::spawn;
use deno_core::unsync::spawn_blocking;
use deno_core::unsync::JoinHandle;
use deno_core::ModuleSpecifier;
use deno_graph::Resolution;
use deno_graph::ResolutionError;

View file

@ -8,7 +8,7 @@ use deno_core::resolve_url;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::serde_json::Value;
use deno_core::task::spawn;
use deno_core::unsync::spawn;
use deno_core::ModuleSpecifier;
use deno_graph::GraphKind;
use deno_lockfile::Lockfile;
@ -1705,7 +1705,7 @@ impl Inner {
}
// spawn a blocking task to allow doing other work while this is occurring
let format_result = deno_core::task::spawn_blocking({
let format_result = deno_core::unsync::spawn_blocking({
let fmt_options = self.fmt_options.options.clone();
let document = document.clone();
move || {

View file

@ -1,6 +1,6 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use deno_core::task::spawn;
use deno_core::unsync::spawn;
use tokio::time::sleep;
use tokio::time::Duration;

View file

@ -24,8 +24,8 @@ use deno_core::futures::stream;
use deno_core::futures::StreamExt;
use deno_core::parking_lot::Mutex;
use deno_core::parking_lot::RwLock;
use deno_core::task::spawn;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn;
use deno_core::unsync::spawn_blocking;
use deno_core::ModuleSpecifier;
use deno_runtime::permissions::Permissions;
use deno_runtime::tokio_util::create_and_run_current_thread;

View file

@ -36,7 +36,7 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::error::JsError;
use deno_core::futures::FutureExt;
use deno_core::task::JoinHandle;
use deno_core::unsync::JoinHandle;
use deno_runtime::colors;
use deno_runtime::fmt_errors::format_js_error;
use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics;
@ -76,7 +76,7 @@ impl SubcommandOutput for Result<(), std::io::Error> {
fn spawn_subcommand<F: Future<Output = T> + 'static, T: SubcommandOutput>(
f: F,
) -> JoinHandle<Result<i32, AnyError>> {
deno_core::task::spawn(f.map(|r| r.output()))
deno_core::unsync::spawn(f.map(|r| r.output()))
}
async fn run_subcommand(flags: Flags) -> Result<i32, AnyError> {

View file

@ -11,7 +11,7 @@ use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::task::spawn;
use deno_core::unsync::spawn;
use deno_core::url::Url;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;

View file

@ -24,8 +24,8 @@ use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::task::spawn;
use deno_core::task::JoinHandle;
use deno_core::unsync::spawn;
use deno_core::unsync::JoinHandle;
use deno_core::url::Url;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::NpmPackageCacheFolderId;

View file

@ -34,7 +34,7 @@ where
Fut::Output: Send + 'static,
{
fn execute(&self, fut: Fut) {
deno_core::task::spawn(fut);
deno_core::unsync::spawn(fut);
}
}

View file

@ -4220,7 +4220,7 @@ where
Fut::Output: Send + 'static,
{
fn execute(&self, fut: Fut) {
deno_core::task::spawn(fut);
deno_core::unsync::spawn(fut);
}
}

View file

@ -1,2 +1,2 @@
Allocated: 4MB
Freed: -4MB
Allocated: 8MB
Freed: -8MB

View file

@ -1483,6 +1483,48 @@ Deno.test(
},
);
// Make sure that the chunks of a large response aren't repeated or corrupted in some other way by
// scatterning sentinels throughout.
// https://github.com/denoland/fresh/issues/1699
Deno.test(
{ permissions: { net: true } },
async function httpLargeReadableStreamChunk() {
const ac = new AbortController();
const server = Deno.serve({
handler() {
return new Response(
new ReadableStream({
start(controller) {
const buffer = new Uint8Array(1024 * 1024);
// Mark the buffer with sentinels
for (let i = 0; i < 256; i++) {
buffer[i * 4096] = i;
}
controller.enqueue(buffer);
controller.close();
},
}),
);
},
port: servePort,
signal: ac.signal,
});
const response = await fetch(`http://localhost:${servePort}/`);
const body = await response.arrayBuffer();
assertEquals(1024 * 1024, body.byteLength);
const buffer = new Uint8Array(body);
for (let i = 0; i < 256; i++) {
assertEquals(
i,
buffer[i * 4096],
`sentinel mismatch at index ${i * 4096}`,
);
}
ac.abort();
await server.finished;
},
);
Deno.test(
{ permissions: { net: true } },
async function httpRequestLatin1Headers() {

View file

@ -77,6 +77,22 @@ function emptyStream(onPull: boolean) {
}).pipeThrough(new TextEncoderStream());
}
function largePacketStream(packetSize: number, count: number) {
return new ReadableStream({
pull(controller) {
if (count-- > 0) {
const buffer = new Uint8Array(packetSize);
for (let i = 0; i < 256; i++) {
buffer[i * (packetSize / 256)] = i;
}
controller.enqueue(buffer);
} else {
controller.close();
}
},
});
}
// Include an empty chunk
function emptyChunkStream() {
return new ReadableStream({
@ -260,6 +276,61 @@ Deno.test(async function readableStreamWithEmptyChunkOneByOne() {
core.ops.op_close(rid);
});
// Ensure that we correctly transmit all the sub-chunks of the larger chunks.
Deno.test(async function readableStreamReadSmallerChunks() {
const packetSize = 16 * 1024;
const rid = resourceForReadableStream(largePacketStream(packetSize, 1));
const buffer = new Uint8Array(packetSize);
for (let i = 0; i < packetSize / 1024; i++) {
await core.ops.op_read(rid, buffer.subarray(i * 1024, i * 1024 + 1024));
}
for (let i = 0; i < 256; i++) {
assertEquals(
i,
buffer[i * (packetSize / 256)],
`at index ${i * (packetSize / 256)}`,
);
}
core.ops.op_close(rid);
});
Deno.test(async function readableStreamLargePackets() {
const packetSize = 128 * 1024;
const rid = resourceForReadableStream(largePacketStream(packetSize, 1024));
for (let i = 0; i < 1024; i++) {
const buffer = new Uint8Array(packetSize);
assertEquals(packetSize, await core.ops.op_read(rid, buffer));
for (let i = 0; i < 256; i++) {
assertEquals(
i,
buffer[i * (packetSize / 256)],
`at index ${i * (packetSize / 256)}`,
);
}
}
assertEquals(0, await core.ops.op_read(rid, new Uint8Array(1)));
core.ops.op_close(rid);
});
Deno.test(async function readableStreamVeryLargePackets() {
// 1024 packets of 1MB
const rid = resourceForReadableStream(largePacketStream(1024 * 1024, 1024));
let total = 0;
// Read 96kB up to 12,288 times (96kB is not an even multiple of the 1MB packet size to test this)
const readCounts: Record<number, number> = {};
for (let i = 0; i < 12 * 1024; i++) {
const nread = await core.ops.op_read(rid, new Uint8Array(96 * 1024));
total += nread;
readCounts[nread] = (readCounts[nread] || 0) + 1;
if (nread == 0) {
break;
}
}
assertEquals({ 0: 1, 65536: 1024, 98304: 10 * 1024 }, readCounts);
assertEquals(total, 1024 * 1024 * 1024);
core.ops.op_close(rid);
});
for (const count of [0, 1, 2, 3]) {
for (const delay of [0, 1, 10]) {
// Creating a stream that errors in start will throw

View file

@ -27,8 +27,8 @@ use deno_core::futures::stream;
use deno_core::futures::StreamExt;
use deno_core::located_script_name;
use deno_core::serde_v8;
use deno_core::task::spawn;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn;
use deno_core::unsync::spawn_blocking;
use deno_core::v8;
use deno_core::ModuleSpecifier;
use deno_runtime::permissions::Permissions;

View file

@ -29,7 +29,7 @@ use deno_core::error::generic_error;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::parking_lot::Mutex;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use log::debug;
use log::info;
use log::warn;

View file

@ -8,7 +8,7 @@ use crate::factory::CliFactory;
use crate::file_fetcher::FileFetcher;
use deno_core::error::AnyError;
use deno_core::futures::StreamExt;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_runtime::permissions::Permissions;
use deno_runtime::permissions::PermissionsContainer;
use rustyline::error::ReadlineError;

View file

@ -40,8 +40,8 @@ use deno_core::futures::StreamExt;
use deno_core::located_script_name;
use deno_core::parking_lot::Mutex;
use deno_core::serde_v8;
use deno_core::task::spawn;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn;
use deno_core::unsync::spawn_blocking;
use deno_core::url::Url;
use deno_core::v8;
use deno_core::ModuleSpecifier;

View file

@ -17,7 +17,7 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures::future::BoxFuture;
use deno_core::futures::FutureExt;
use deno_core::task::spawn;
use deno_core::unsync::spawn;
use deno_semver::Version;
use once_cell::sync::Lazy;
use std::borrow::Cow;

View file

@ -2,7 +2,7 @@
use console_static_text::ConsoleStaticText;
use deno_core::parking_lot::Mutex;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_runtime::ops::tty::ConsoleSize;
use once_cell::sync::Lazy;
use std::sync::Arc;

View file

@ -3,7 +3,7 @@
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
pub use deno_core::normalize_path;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_core::ModuleSpecifier;
use deno_runtime::deno_crypto::rand;
use deno_runtime::deno_node::PathClean;

2
ext/cache/sqlite.rs vendored
View file

@ -10,7 +10,7 @@ use std::time::UNIX_EPOCH;
use async_trait::async_trait;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_core::AsyncRefCell;
use deno_core::AsyncResult;
use deno_core::ByteString;

View file

@ -20,7 +20,7 @@ use deno_core::error::custom_error;
use deno_core::error::type_error;
use deno_core::error::AnyError;
use deno_core::op;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_core::JsBuffer;
use deno_core::ToJsBuffer;
use rsa::pkcs1::DecodeRsaPrivateKey;

View file

@ -19,7 +19,7 @@ use ctr::Ctr64BE;
use deno_core::error::type_error;
use deno_core::error::AnyError;
use deno_core::op;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_core::JsBuffer;
use deno_core::ToJsBuffer;
use rand::rngs::OsRng;

View file

@ -2,7 +2,7 @@
use deno_core::error::AnyError;
use deno_core::op;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_core::ToJsBuffer;
use elliptic_curve::rand_core::OsRng;
use num_traits::FromPrimitive;

View file

@ -11,7 +11,7 @@ use deno_core::error::AnyError;
use deno_core::op;
use deno_core::ToJsBuffer;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_core::JsBuffer;
use deno_core::OpState;
use serde::Deserialize;

View file

@ -24,7 +24,7 @@ use deno_core::op;
use deno_core::BufView;
use deno_core::WriteOutcome;
use deno_core::task::spawn;
use deno_core::unsync::spawn;
use deno_core::url::Url;
use deno_core::AsyncRefCell;
use deno_core::AsyncResult;

View file

@ -15,7 +15,7 @@ use deno_core::op;
use deno_core::serde_json::Value;
use deno_core::serde_v8;
use deno_core::serde_v8::ExternalPointer;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_core::v8;
use deno_core::OpState;
use deno_core::ResourceId;

View file

@ -9,7 +9,7 @@ use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_io::fs::File;
use deno_io::fs::FsResult;
use deno_io::fs::FsStat;

View file

@ -25,8 +25,8 @@ use deno_core::op;
use deno_core::op2;
use deno_core::serde_v8;
use deno_core::serde_v8::from_v8;
use deno_core::task::spawn;
use deno_core::task::JoinHandle;
use deno_core::unsync::spawn;
use deno_core::unsync::JoinHandle;
use deno_core::v8;
use deno_core::AsyncRefCell;
use deno_core::AsyncResult;

View file

@ -21,7 +21,7 @@ use deno_core::futures::FutureExt;
use deno_core::futures::StreamExt;
use deno_core::futures::TryFutureExt;
use deno_core::op;
use deno_core::task::spawn;
use deno_core::unsync::spawn;
use deno_core::AsyncRefCell;
use deno_core::AsyncResult;
use deno_core::BufView;
@ -1021,7 +1021,7 @@ where
Fut::Output: 'static,
{
fn execute(&self, fut: Fut) {
deno_core::task::spawn(fut);
deno_core::unsync::spawn(fut);
}
}
@ -1031,7 +1031,7 @@ where
Fut::Output: 'static,
{
fn execute(&self, fut: Fut) {
deno_core::task::spawn(fut);
deno_core::unsync::spawn(fut);
}
}

View file

@ -2,7 +2,7 @@
use deno_core::error::AnyError;
use deno_core::op;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_core::AsyncMutFuture;
use deno_core::AsyncRefCell;
use deno_core::AsyncResult;

View file

@ -19,8 +19,8 @@ use deno_core::error::type_error;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::futures::FutureExt;
use deno_core::task::spawn;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn;
use deno_core::unsync::spawn_blocking;
use deno_core::AsyncRefCell;
use deno_core::OpState;
use rand::Rng;
@ -285,7 +285,7 @@ impl<P: SqliteDbHandlerPermissions> DatabaseHandler for SqliteDbHandler<P> {
pub struct SqliteDb {
conn: ProtectedConn,
queue: OnceCell<SqliteQueue>,
expiration_watcher: deno_core::task::JoinHandle<()>,
expiration_watcher: deno_core::unsync::JoinHandle<()>,
}
impl Drop for SqliteDb {

View file

@ -26,7 +26,7 @@ use deno_core::futures::task::Waker;
use deno_core::op;
use deno_core::parking_lot::Mutex;
use deno_core::task::spawn;
use deno_core::unsync::spawn;
use deno_core::AsyncRefCell;
use deno_core::AsyncResult;
use deno_core::ByteString;

View file

@ -4,7 +4,7 @@ use deno_core::error::type_error;
use deno_core::error::AnyError;
use deno_core::op;
use deno_core::serde_v8;
use deno_core::task::spawn_blocking;
use deno_core::unsync::spawn_blocking;
use deno_core::JsBuffer;
use deno_core::OpState;
use deno_core::ResourceId;

View file

@ -412,7 +412,7 @@ pub fn op_ws_send_binary(state: &mut OpState, rid: ResourceId, data: &[u8]) {
let len = data.len();
resource.buffered.set(resource.buffered.get() + len);
let lock = resource.reserve_lock();
deno_core::task::spawn(async move {
deno_core::unsync::spawn(async move {
if let Err(err) = resource
.write_frame(lock, Frame::new(true, OpCode::Binary, None, data.into()))
.await
@ -430,7 +430,7 @@ pub fn op_ws_send_text(state: &mut OpState, rid: ResourceId, data: String) {
let len = data.len();
resource.buffered.set(resource.buffered.get() + len);
let lock = resource.reserve_lock();
deno_core::task::spawn(async move {
deno_core::unsync::spawn(async move {
if let Err(err) = resource
.write_frame(
lock,
@ -719,6 +719,6 @@ where
Fut::Output: 'static,
{
fn execute(&self, fut: Fut) {
deno_core::task::spawn(fut);
deno_core::unsync::spawn(fut);
}
}

View file

@ -15,7 +15,7 @@ use deno_core::futures::task::Poll;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::serde_json::Value;
use deno_core::task::spawn;
use deno_core::unsync::spawn;
use deno_core::url::Url;
use deno_core::InspectorMsg;
use deno_core::InspectorSessionProxy;
@ -111,7 +111,7 @@ where
Fut::Output: 'static,
{
fn execute(&self, fut: Fut) {
deno_core::task::spawn(fut);
deno_core::unsync::spawn(fut);
}
}

View file

@ -2,7 +2,7 @@
use std::fmt::Debug;
use std::str::FromStr;
use deno_core::task::MaskFutureAsSend;
use deno_core::unsync::MaskFutureAsSend;
#[cfg(tokio_unstable)]
use tokio_metrics::RuntimeMonitor;

View file

@ -2807,12 +2807,8 @@
},
"class-string-interface.any.html": true,
"class-string-interface.any.worker.html": true,
"class-string-iterator-prototype-object.any.html": [
"Object.prototype.toString applied after deleting @@toStringTag"
],
"class-string-iterator-prototype-object.any.worker.html": [
"Object.prototype.toString applied after deleting @@toStringTag"
],
"class-string-iterator-prototype-object.any.html": true,
"class-string-iterator-prototype-object.any.worker.html": true,
"class-string-named-properties-object.window.html": false,
"global-immutable-prototype.any.html": [
"Setting to a different prototype"