Merge branch 'main' into will/test-buildjet

.github/workflows/ci.generate.ts

@@ -24,12 +24,14 @@ const prCacheKeyPrefix =
   `${cacheVersion}-cargo-target-\${{ matrix.os }}-\${{ matrix.profile }}-\${{ matrix.job }}-`;
 
 const installPkgsCommand =
-  "sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15";
+  "sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15 clang-tools-15 clang-format-15 clang-tidy-15";
 const sysRootStep = {
   name: "Set up incremental LTO and sysroot build",
   run: `# Avoid running man-db triggers, which sometimes takes several minutes
 # to complete.
 sudo apt-get remove --purge -y man-db
+# Remove older clang before we install
+sudo apt-get remove 'clang-12*' 'clang-13*' 'clang-14*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'lld-12*' 'lld-13*' 'lld-14*'
 
 # Install clang-15, lld-15, and debootstrap.
 echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" |
@@ -40,6 +42,8 @@ sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg
 sudo apt-get update
 # this was unreliable sometimes, so try again if it fails
 ${installPkgsCommand} || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && ${installPkgsCommand}
+# Fix alternatives
+(yes '' | sudo update-alternatives --force --all) || true
 
 # Create ubuntu-16.04 sysroot environment, which is used to avoid
 # depending on a very recent version of glibc.
@@ -461,6 +465,7 @@ const ci = {
             "python --version",
             "rustc --version",
             "cargo --version",
+            "which dpkg && dpkg -l",
             // Deno is installed when linting.
             'if [ "${{ matrix.job }}" == "lint" ]',
             "then",
@@ -575,7 +580,12 @@ const ci = {
             "(github.ref == 'refs/heads/main' ||",
             "startsWith(github.ref, 'refs/tags/'))))",
           ].join("\n"),
-          run: "cargo build --release --locked --all-targets",
+          run: [
+            // output fs space before and after building
+            "df -h",
+            "cargo build --release --locked --all-targets",
+            "df -h",
+          ].join("\n"),
         },
         {
           name: "Upload PR artifact (linux)",

.github/workflows/ci.yml

@@ -209,6 +209,8 @@ jobs:
           # Avoid running man-db triggers, which sometimes takes several minutes
           # to complete.
           sudo apt-get remove --purge -y man-db
+          # Remove older clang before we install
+          sudo apt-get remove 'clang-12*' 'clang-13*' 'clang-14*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'lld-12*' 'lld-13*' 'lld-14*'
 
           # Install clang-15, lld-15, and debootstrap.
           echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" |
@@ -218,7 +220,9 @@ jobs:
           sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg
           sudo apt-get update
           # this was unreliable sometimes, so try again if it fails
-          sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15
+          sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15 clang-tools-15 clang-format-15 clang-tidy-15 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15 clang-tools-15 clang-format-15 clang-tidy-15
+          # Fix alternatives
+          (yes '' | sudo update-alternatives --force --all) || true
 
           # Create ubuntu-16.04 sysroot environment, which is used to avoid
           # depending on a very recent version of glibc.
@@ -277,6 +281,7 @@ jobs:
           python --version
           rustc --version
           cargo --version
+          which dpkg && dpkg -l
           if [ "${{ matrix.job }}" == "lint" ]
           then
             deno --version
@@ -349,7 +354,10 @@ jobs:
           (github.repository == 'denoland/deno' &&
           (github.ref == 'refs/heads/main' ||
           startsWith(github.ref, 'refs/tags/')))))
-        run: cargo build --release --locked --all-targets
+        run: |-
+          df -h
+          cargo build --release --locked --all-targets
+          df -h
       - name: Upload PR artifact (linux)
         if: |-
           !(github.event_name == 'pull_request' && matrix.skip_pr) && (matrix.job == 'test' &&

Cargo.lock

@@ -1412,6 +1412,7 @@ dependencies = [
  "fastwebsockets",
  "http",
  "hyper 0.14.26",
+ "once_cell",
  "serde",
  "tokio",
  "tokio-rustls",
@@ -2788,9 +2789,9 @@ checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c"
 
 [[package]]
 name = "libffi"
-version = "3.1.0"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6cb06d5b4c428f3cd682943741c39ed4157ae989fffe1094a08eaf7c4014cf60"
+checksum = "ce826c243048e3d5cec441799724de52e2d42f820468431fc3fceee2341871e2"
 dependencies = [
  "libc",
  "libffi-sys",
@@ -2798,9 +2799,9 @@ dependencies = [
 
 [[package]]
 name = "libffi-sys"
-version = "2.1.0"
+version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11c6f11e063a27ffe040a9d15f0b661bf41edc2383b7ae0e0ad5a7e7d53d9da3"
+checksum = "dc65067b78c0fc069771e8b9a9e02df71e08858bec92c1f101377c67b9dca7c7"
 dependencies = [
  "cc",
 ]

cli/tests/unit_node/_fs/_fs_handle_test.ts

@@ -1,5 +1,7 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 import * as path from "../../../../test_util/std/path/mod.ts";
+import { Buffer } from "node:buffer";
+import * as fs from "node:fs/promises";
 import {
   assert,
   assertEquals,
@@ -7,14 +9,83 @@ import {
 
 const moduleDir = path.dirname(path.fromFileUrl(import.meta.url));
 const testData = path.resolve(moduleDir, "testdata", "hello.txt");
+const decoder = new TextDecoder();
 
 Deno.test("readFileSuccess", async function () {
-  const fs = await import("node:fs/promises");
   const fileHandle = await fs.open(testData);
   const data = await fileHandle.readFile();
 
   assert(data instanceof Uint8Array);
-  assertEquals(new TextDecoder().decode(data as Uint8Array), "hello world");
+  assertEquals(decoder.decode(data as Uint8Array), "hello world");
 
   await fileHandle.close();
 });
+
+Deno.test("read", async function () {
+  const fileHandle = await fs.open(testData);
+  const byteLength = "hello world".length;
+
+  const buf = new Buffer(byteLength);
+  await fileHandle.read(buf, 0, byteLength, 0);
+
+  assertEquals(decoder.decode(buf as Uint8Array), "hello world");
+
+  await fileHandle.close();
+});
+
+Deno.test("read specify opt", async function () {
+  const fileHandle = await fs.open(testData);
+  const byteLength = "hello world".length;
+
+  const opt = {
+    buffer: new Buffer(byteLength),
+    offset: 6,
+    length: 5,
+  };
+  let res = await fileHandle.read(opt);
+
+  assertEquals(res.bytesRead, byteLength);
+  assertEquals(new TextDecoder().decode(res.buffer as Uint8Array), "world");
+
+  const opt2 = {
+    buffer: new Buffer(byteLength),
+    length: 5,
+    position: 0,
+  };
+  res = await fileHandle.read(opt2);
+
+  assertEquals(res.bytesRead, byteLength);
+  assertEquals(decoder.decode(res.buffer as Uint8Array), "hello");
+
+  await fileHandle.close();
+});
+
+Deno.test("[node/fs filehandle.write] Write from Buffer", async function () {
+  const tempFile: string = await Deno.makeTempFile();
+  const fileHandle = await fs.open(tempFile, "a+");
+
+  const buffer = Buffer.from("hello world");
+  const res = await fileHandle.write(buffer, 0, 5, 0);
+
+  const data = Deno.readFileSync(tempFile);
+  await Deno.remove(tempFile);
+  await fileHandle.close();
+
+  assertEquals(res.bytesWritten, 5);
+  assertEquals(decoder.decode(data), "hello");
+});
+
+Deno.test("[node/fs filehandle.write] Write from string", async function () {
+  const tempFile: string = await Deno.makeTempFile();
+  const fileHandle = await fs.open(tempFile, "a+");
+
+  const str = "hello world";
+  const res = await fileHandle.write(str);
+
+  const data = Deno.readFileSync(tempFile);
+  await Deno.remove(tempFile);
+  await fileHandle.close();
+
+  assertEquals(res.bytesWritten, 11);
+  assertEquals(decoder.decode(data), "hello world");
+});

ext/ffi/Cargo.toml

@@ -17,8 +17,8 @@ path = "lib.rs"
 deno_core.workspace = true
 dlopen.workspace = true
 dynasmrt = "1.2.3"
-libffi = "=3.1.0"
-libffi-sys = "=2.1.0" # temporary pin for downgrade to Rust 1.69
+libffi = "=3.2.0"
+libffi-sys = "=2.2.1"
 serde.workspace = true
 serde-value = "0.7"
 serde_json = "1.0"

ext/http/http_next.rs

@@ -73,13 +73,13 @@ type Request = hyper1::Request<Incoming>;
 type Response = hyper1::Response<ResponseBytes>;
 
 static USE_WRITEV: Lazy<bool> = Lazy::new(|| {
-  let disable_writev = std::env::var("DENO_HYPER_USE_WRITEV").ok();
+  let enable = std::env::var("DENO_USE_WRITEV").ok();
 
-  if let Some(val) = disable_writev {
-    return val != "0";
+  if let Some(val) = enable {
+    return !val.is_empty();
   }
 
-  true
+  false
 });
 
 /// All HTTP/2 connections start with this byte string.

ext/napi/lib.rs

@@ -647,12 +647,13 @@ where
   // SAFETY: we are going blind, calling the register function on the other side.
 
   let maybe_exports = unsafe {
-    let init = library
+    let Ok(init) = library
       .get::<unsafe extern "C" fn(
         env: napi_env,
         exports: napi_value,
-      ) -> napi_value>(b"napi_register_module_v1")
-      .expect("napi_register_module_v1 not found");
+      ) -> napi_value>(b"napi_register_module_v1") else {
+        return Err(type_error(format!("Unable to find napi_register_module_v1 symbol in {}", path)));
+      };
     init(
       env_ptr,
       std::mem::transmute::<v8::Local<v8::Value>, napi_value>(exports.into()),

ext/node/polyfills/_fs/_fs_common.ts

@@ -33,6 +33,13 @@ export type BinaryOptionsArgument =
   | ({ encoding: BinaryEncodings } & FileOptions);
 export type FileOptionsArgument = Encodings | FileOptions;
 
+export type ReadOptions = {
+  buffer: Buffer | Uint8Array;
+  offset: number;
+  length: number;
+  position: number | null;
+};
+
 export interface WriteFileOptions extends FileOptions {
   mode?: number;
 }

ext/node/polyfills/_fs/_fs_read.ts

@@ -3,6 +3,7 @@ import { Buffer } from "ext:deno_node/buffer.ts";
 import { ERR_INVALID_ARG_TYPE } from "ext:deno_node/internal/errors.ts";
 import * as io from "ext:deno_io/12_io.js";
 import * as fs from "ext:deno_fs/30_fs.js";
+import { ReadOptions } from "ext:deno_node/_fs/_fs_common.ts";
 import {
   validateOffsetLengthRead,
   validatePosition,
@@ -12,13 +13,6 @@ import {
   validateInteger,
 } from "ext:deno_node/internal/validators.mjs";
 
-type readOptions = {
-  buffer: Buffer | Uint8Array;
-  offset: number;
-  length: number;
-  position: number | null;
-};
-
 type readSyncOptions = {
   offset: number;
   length: number;
@@ -35,7 +29,7 @@ type Callback = BinaryCallback;
 export function read(fd: number, callback: Callback): void;
 export function read(
   fd: number,
-  options: readOptions,
+  options: ReadOptions,
   callback: Callback,
 ): void;
 export function read(
@@ -48,7 +42,7 @@ export function read(
 ): void;
 export function read(
   fd: number,
-  optOrBufferOrCb?: Buffer | Uint8Array | readOptions | Callback,
+  optOrBufferOrCb?: Buffer | Uint8Array | ReadOptions | Callback,
   offsetOrCallback?: number | Callback,
   length?: number,
   position?: number | null,
@@ -86,7 +80,7 @@ export function read(
     length = buffer.byteLength;
     position = null;
   } else {
-    const opt = optOrBufferOrCb as readOptions;
+    const opt = optOrBufferOrCb as ReadOptions;
     if (
       !(opt.buffer instanceof Buffer) && !(opt.buffer instanceof Uint8Array)
     ) {

ext/node/polyfills/internal/fs/handle.ts

@@ -1,13 +1,24 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 import { EventEmitter } from "ext:deno_node/events.ts";
 import { Buffer } from "ext:deno_node/buffer.ts";
-import { promises } from "ext:deno_node/fs.ts";
+import { promises, read, write } from "ext:deno_node/fs.ts";
 import {
   BinaryOptionsArgument,
   FileOptionsArgument,
+  ReadOptions,
   TextOptionsArgument,
 } from "ext:deno_node/_fs/_fs_common.ts";
 
+interface WriteResult {
+  bytesWritten: number;
+  buffer: Buffer | string;
+}
+
+interface ReadResult {
+  bytesRead: number;
+  buffer: Buffer;
+}
+
 export class FileHandle extends EventEmitter {
   #rid: number;
   constructor(rid: number) {
@@ -19,12 +30,104 @@ export class FileHandle extends EventEmitter {
     return this.rid;
   }
 
+  read(
+    buffer: Buffer,
+    offset?: number,
+    length?: number,
+    position?: number | null,
+  ): Promise<ReadResult>;
+  read(options?: ReadOptions): Promise<ReadResult>;
+  read(
+    bufferOrOpt: Buffer | ReadOptions,
+    offset?: number,
+    length?: number,
+    position?: number | null,
+  ): Promise<ReadResult> {
+    if (bufferOrOpt instanceof Buffer) {
+      return new Promise((resolve, reject) => {
+        read(
+          this.fd,
+          bufferOrOpt,
+          offset,
+          length,
+          position,
+          (err, bytesRead, buffer) => {
+            if (err) reject(err);
+            else resolve({ buffer: buffer, bytesRead: bytesRead });
+          },
+        );
+      });
+    } else {
+      return new Promise((resolve, reject) => {
+        read(this.fd, bufferOrOpt, (err, bytesRead, buffer) => {
+          if (err) reject(err);
+          else resolve({ buffer: buffer, bytesRead: bytesRead });
+        });
+      });
+    }
+  }
+
   readFile(
     opt?: TextOptionsArgument | BinaryOptionsArgument | FileOptionsArgument,
   ): Promise<string | Buffer> {
     return promises.readFile(this, opt);
   }
 
+  write(
+    buffer: Buffer,
+    offset: number,
+    length: number,
+    position: number,
+  ): Promise<WriteResult>;
+  write(
+    str: string,
+    position: number,
+    encoding: string,
+  ): Promise<WriteResult>;
+  write(
+    bufferOrStr: Buffer | string,
+    offsetOrPotition: number,
+    lengthOrEncoding: number | string,
+    position?: number,
+  ): Promise<WriteResult> {
+    if (bufferOrStr instanceof Buffer) {
+      const buffer = bufferOrStr;
+      const offset = offsetOrPotition;
+      const length = lengthOrEncoding;
+
+      return new Promise((resolve, reject) => {
+        write(
+          this.fd,
+          buffer,
+          offset,
+          length,
+          position,
+          (err, bytesWritten, buffer) => {
+            if (err) reject(err);
+            else resolve({ buffer, bytesWritten });
+          },
+        );
+      });
+    } else {
+      const str = bufferOrStr;
+      const position = offsetOrPotition;
+      const encoding = lengthOrEncoding;
+
+      return new Promise((resolve, reject) => {
+        write(
+          this.fd,
+          str,
+          position,
+          encoding,
+          (err, bytesWritten, buffer) => {
+            if (err) reject(err);
+            else resolve({ buffer, bytesWritten });
+          },
+        );
+      });
+    }
+  }
+
   close(): Promise<void> {
     // Note that Deno.close is not async
     return Promise.resolve(Deno.close(this.fd));

ext/websocket/Cargo.toml

@@ -21,6 +21,7 @@ deno_tls.workspace = true
 fastwebsockets = { workspace = true, features = ["upgrade"] }
 http.workspace = true
 hyper = { workspace = true, features = ["backports"] }
+once_cell.workspace = true
 serde.workspace = true
 tokio.workspace = true
 tokio-rustls.workspace = true

ext/websocket/lib.rs

@@ -27,6 +27,7 @@ use http::Method;
 use http::Request;
 use http::Uri;
 use hyper::Body;
+use once_cell::sync::Lazy;
 use serde::Serialize;
 use std::borrow::Cow;
 use std::cell::Cell;
@@ -50,9 +51,18 @@ use fastwebsockets::Frame;
 use fastwebsockets::OpCode;
 use fastwebsockets::Role;
 use fastwebsockets::WebSocket;
-
 mod stream;
 
+static USE_WRITEV: Lazy<bool> = Lazy::new(|| {
+  let enable = std::env::var("DENO_USE_WRITEV").ok();
+
+  if let Some(val) = enable {
+    return !val.is_empty();
+  }
+
+  false
+});
+
 #[derive(Clone)]
 pub struct WsRootStoreProvider(Option<Arc<dyn RootCertStoreProvider>>);
 
@@ -360,7 +370,7 @@ pub fn ws_create_server_stream(
     ),
     Role::Server,
   );
-  ws.set_writev(true);
+  ws.set_writev(*USE_WRITEV);
   ws.set_auto_close(true);
   ws.set_auto_pong(true);
 

rust-toolchain.toml

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "1.68.0"
+channel = "1.68.2"
 components = ["rustfmt", "clippy"]