From 92ed4d38dbef98b9353d6dd6d96abb400be56f9f Mon Sep 17 00:00:00 2001 From: Satya Rohith Date: Wed, 23 Oct 2024 13:17:43 +0530 Subject: [PATCH 01/63] fix(node:tls): set TLSSocket.alpnProtocol for client connections (#26476) Towards https://github.com/denoland/deno/issues/26127 --- ext/node/polyfills/_tls_wrap.ts | 16 +++++++++++++++- tests/unit_node/tls_test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/ext/node/polyfills/_tls_wrap.ts b/ext/node/polyfills/_tls_wrap.ts index a614b45df0..e36fc637e7 100644 --- a/ext/node/polyfills/_tls_wrap.ts +++ b/ext/node/polyfills/_tls_wrap.ts @@ -68,6 +68,7 @@ export class TLSSocket extends net.Socket { secureConnecting: boolean; _SNICallback: any; servername: string | null; + alpnProtocol: string | boolean | null; alpnProtocols: string[] | null; authorized: boolean; authorizationError: any; @@ -114,6 +115,7 @@ export class TLSSocket extends net.Socket { this.secureConnecting = true; this._SNICallback = null; this.servername = null; + this.alpnProtocol = null; this.alpnProtocols = tlsOptions.ALPNProtocols; this.authorized = false; this.authorizationError = null; @@ -151,10 +153,21 @@ export class TLSSocket extends net.Socket { handle.afterConnect = async (req: any, status: number) => { try { const conn = await Deno.startTls(handle[kStreamBaseField], options); + try { + const hs = await conn.handshake(); + if (hs.alpnProtocol) { + tlssock.alpnProtocol = hs.alpnProtocol; + } else { + tlssock.alpnProtocol = false; + } + } catch { + // Don't interrupt "secure" event to let the first read/write + // operation emit the error. + } handle[kStreamBaseField] = conn; tlssock.emit("secure"); tlssock.removeListener("end", onConnectEnd); - } catch { + } catch (_) { // TODO(kt3k): Handle this } return afterConnect.call(handle, req, status); @@ -269,6 +282,7 @@ export class ServerImpl extends EventEmitter { // Creates TCP handle and socket directly from Deno.TlsConn. // This works as TLS socket. We don't use TLSSocket class for doing // this because Deno.startTls only supports client side tcp connection. + // TODO(@satyarohith): set TLSSocket.alpnProtocol when we use TLSSocket class. const handle = new TCP(TCPConstants.SOCKET, await listener.accept()); const socket = new net.Socket({ handle }); this.emit("secureConnection", socket); diff --git a/tests/unit_node/tls_test.ts b/tests/unit_node/tls_test.ts index 847ec2dde3..236dab2086 100644 --- a/tests/unit_node/tls_test.ts +++ b/tests/unit_node/tls_test.ts @@ -235,3 +235,31 @@ Deno.test("tls.rootCertificates is not empty", () => { (tls.rootCertificates as string[]).push("new cert"); }, TypeError); }); + +Deno.test("TLSSocket.alpnProtocol is set for client", async () => { + const listener = Deno.listenTls({ + hostname: "localhost", + port: 0, + key, + cert, + alpnProtocols: ["a"], + }); + const outgoing = tls.connect({ + host: "::1", + servername: "localhost", + port: listener.addr.port, + ALPNProtocols: ["a"], + secureContext: { + ca: rootCaCert, + // deno-lint-ignore no-explicit-any + } as any, + }); + + const conn = await listener.accept(); + const handshake = await conn.handshake(); + assertEquals(handshake.alpnProtocol, "a"); + conn.close(); + outgoing.destroy(); + listener.close(); + await new Promise((resolve) => outgoing.on("close", resolve)); +}); From 69e1d7a4edb45c8b4cb15f64ee89cec8a693be5e Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Wed, 23 Oct 2024 15:01:45 -0700 Subject: [PATCH 02/63] fix(install): cache type only module deps in `deno install` (#26497) Fixes https://github.com/denoland/deno/issues/26180. --- cli/args/mod.rs | 3 +++ .../jsr/@denotest/type-only-import/1.0.0/foo.ts | 3 +++ .../jsr/@denotest/type-only-import/1.0.0/mod.ts | 3 +++ .../jsr/@denotest/type-only-import/1.0.0_meta.json | 5 +++++ .../registry/jsr/@denotest/type-only-import/meta.json | 5 +++++ tests/specs/install/type_only_import/__test__.jsonc | 10 ++++++++++ tests/specs/install/type_only_import/install.out | 6 ++++++ tests/specs/install/type_only_import/main.ts | 3 +++ tests/specs/install/type_only_import/run.out | 2 ++ 9 files changed, 40 insertions(+) create mode 100644 tests/registry/jsr/@denotest/type-only-import/1.0.0/foo.ts create mode 100644 tests/registry/jsr/@denotest/type-only-import/1.0.0/mod.ts create mode 100644 tests/registry/jsr/@denotest/type-only-import/1.0.0_meta.json create mode 100644 tests/registry/jsr/@denotest/type-only-import/meta.json create mode 100644 tests/specs/install/type_only_import/__test__.jsonc create mode 100644 tests/specs/install/type_only_import/install.out create mode 100644 tests/specs/install/type_only_import/main.ts create mode 100644 tests/specs/install/type_only_import/run.out diff --git a/cli/args/mod.rs b/cli/args/mod.rs index f905e186ba..d8f5531015 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -963,6 +963,9 @@ impl CliOptions { match self.sub_command() { DenoSubcommand::Cache(_) => GraphKind::All, DenoSubcommand::Check(_) => GraphKind::TypesOnly, + DenoSubcommand::Install(InstallFlags { + kind: InstallKind::Local(_), + }) => GraphKind::All, _ => self.type_check_mode().as_graph_kind(), } } diff --git a/tests/registry/jsr/@denotest/type-only-import/1.0.0/foo.ts b/tests/registry/jsr/@denotest/type-only-import/1.0.0/foo.ts new file mode 100644 index 0000000000..4e09512c03 --- /dev/null +++ b/tests/registry/jsr/@denotest/type-only-import/1.0.0/foo.ts @@ -0,0 +1,3 @@ +export interface Foo { + bar: string; +} \ No newline at end of file diff --git a/tests/registry/jsr/@denotest/type-only-import/1.0.0/mod.ts b/tests/registry/jsr/@denotest/type-only-import/1.0.0/mod.ts new file mode 100644 index 0000000000..093a57d8a1 --- /dev/null +++ b/tests/registry/jsr/@denotest/type-only-import/1.0.0/mod.ts @@ -0,0 +1,3 @@ +import type { Foo } from "./foo.ts"; + +export const foo: Foo = { bar: "foo" }; \ No newline at end of file diff --git a/tests/registry/jsr/@denotest/type-only-import/1.0.0_meta.json b/tests/registry/jsr/@denotest/type-only-import/1.0.0_meta.json new file mode 100644 index 0000000000..631a18d0e5 --- /dev/null +++ b/tests/registry/jsr/@denotest/type-only-import/1.0.0_meta.json @@ -0,0 +1,5 @@ +{ + "exports": { + ".": "./mod.ts" + } +} diff --git a/tests/registry/jsr/@denotest/type-only-import/meta.json b/tests/registry/jsr/@denotest/type-only-import/meta.json new file mode 100644 index 0000000000..02601e4d0d --- /dev/null +++ b/tests/registry/jsr/@denotest/type-only-import/meta.json @@ -0,0 +1,5 @@ +{ + "versions": { + "1.0.0": {} + } +} diff --git a/tests/specs/install/type_only_import/__test__.jsonc b/tests/specs/install/type_only_import/__test__.jsonc new file mode 100644 index 0000000000..22b8134ac3 --- /dev/null +++ b/tests/specs/install/type_only_import/__test__.jsonc @@ -0,0 +1,10 @@ +{ + "tempDir": true, + "steps": [ + { "args": "install -e main.ts", "output": "install.out" }, + { + "args": "run --check --cached-only main.ts", + "output": "run.out" + } + ] +} diff --git a/tests/specs/install/type_only_import/install.out b/tests/specs/install/type_only_import/install.out new file mode 100644 index 0000000000..1bdbec4e58 --- /dev/null +++ b/tests/specs/install/type_only_import/install.out @@ -0,0 +1,6 @@ +Download http://127.0.0.1:4250/@denotest/type-only-import/meta.json +Download http://127.0.0.1:4250/@denotest/type-only-import/1.0.0_meta.json +[UNORDERED_START] +Download http://127.0.0.1:4250/@denotest/type-only-import/1.0.0/mod.ts +Download http://127.0.0.1:4250/@denotest/type-only-import/1.0.0/foo.ts +[UNORDERED_END] diff --git a/tests/specs/install/type_only_import/main.ts b/tests/specs/install/type_only_import/main.ts new file mode 100644 index 0000000000..bf22935a2d --- /dev/null +++ b/tests/specs/install/type_only_import/main.ts @@ -0,0 +1,3 @@ +import { foo } from "jsr:@denotest/type-only-import"; + +console.log(foo.bar); diff --git a/tests/specs/install/type_only_import/run.out b/tests/specs/install/type_only_import/run.out new file mode 100644 index 0000000000..d36ac8aa37 --- /dev/null +++ b/tests/specs/install/type_only_import/run.out @@ -0,0 +1,2 @@ +Check [WILDCARD]main.ts +foo From 6d587cbfc800a627efb32d377b260954ccd7a1ed Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Wed, 23 Oct 2024 20:12:52 -0700 Subject: [PATCH 03/63] fix(install): cache all exports of JSR packages listed in `deno.json` (#26501) Fixes #26498. This was a sort of intentional decision originally, as I wanted to avoid caching extra files that may not be needed. It seems like that behavior is unintuitive, so I propose we cache all of the exports of listed jsr packages when you run a bare `deno install`. --- cli/tools/registry/pm/cache_deps.rs | 4 ---- tests/specs/install/jsr_exports/__test__.jsonc | 6 ++++++ tests/specs/install/jsr_exports/deno.json | 5 +++++ tests/specs/install/jsr_exports/install.out | 12 ++++++++++++ 4 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 tests/specs/install/jsr_exports/__test__.jsonc create mode 100644 tests/specs/install/jsr_exports/deno.json create mode 100644 tests/specs/install/jsr_exports/install.out diff --git a/cli/tools/registry/pm/cache_deps.rs b/cli/tools/registry/pm/cache_deps.rs index b4cd1c2532..365622d11f 100644 --- a/cli/tools/registry/pm/cache_deps.rs +++ b/cli/tools/registry/pm/cache_deps.rs @@ -89,10 +89,6 @@ pub async fn cache_top_level_deps( while let Some(info_future) = info_futures.next().await { if let Some((specifier, info)) = info_future { - if info.export(".").is_some() { - roots.push(specifier.clone()); - continue; - } let exports = info.exports(); for (k, _) in exports { if let Ok(spec) = specifier.join(k) { diff --git a/tests/specs/install/jsr_exports/__test__.jsonc b/tests/specs/install/jsr_exports/__test__.jsonc new file mode 100644 index 0000000000..934f3c5885 --- /dev/null +++ b/tests/specs/install/jsr_exports/__test__.jsonc @@ -0,0 +1,6 @@ +{ + "tempDir": true, + "steps": [ + { "args": "install", "output": "install.out" } + ] +} diff --git a/tests/specs/install/jsr_exports/deno.json b/tests/specs/install/jsr_exports/deno.json new file mode 100644 index 0000000000..21212f5794 --- /dev/null +++ b/tests/specs/install/jsr_exports/deno.json @@ -0,0 +1,5 @@ +{ + "imports": { + "@denotest/different-deps-per-export": "jsr:@denotest/different-deps-per-export@^1.0.0" + } +} diff --git a/tests/specs/install/jsr_exports/install.out b/tests/specs/install/jsr_exports/install.out new file mode 100644 index 0000000000..8f64001636 --- /dev/null +++ b/tests/specs/install/jsr_exports/install.out @@ -0,0 +1,12 @@ +[UNORDERED_START] +Download http://127.0.0.1:4250/@denotest/different-deps-per-export/meta.json +Download http://127.0.0.1:4250/@denotest/different-deps-per-export/1.0.0_meta.json +Download http://127.0.0.1:4250/@denotest/different-deps-per-export/1.0.0/add.ts +Download http://127.0.0.1:4250/@denotest/different-deps-per-export/1.0.0/subtract.ts +Download http://127.0.0.1:4250/@denotest/add/meta.json +Download http://127.0.0.1:4250/@denotest/subtract/meta.json +Download http://127.0.0.1:4250/@denotest/add/1.0.0_meta.json +Download http://127.0.0.1:4250/@denotest/subtract/1.0.0_meta.json +Download http://127.0.0.1:4250/@denotest/add/1.0.0/mod.ts +Download http://127.0.0.1:4250/@denotest/subtract/1.0.0/mod.ts +[UNORDERED_END] From fa49fd404be82daf8ac7228bf54e780135f67b17 Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Thu, 24 Oct 2024 09:41:38 +0530 Subject: [PATCH 04/63] fix(ext/ffi): return u64/i64 as bigints from nonblocking ffi calls (#26486) Fixes https://github.com/denoland/deno/issues/25194 --- Cargo.lock | 1 + ext/ffi/Cargo.toml | 1 + ext/ffi/call.rs | 27 +++++++++++++++------------ tests/ffi/tests/integration_tests.rs | 8 ++++---- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d4f5e252a..d550baee66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1577,6 +1577,7 @@ dependencies = [ "libffi", "libffi-sys", "log", + "num-bigint", "serde", "serde-value", "serde_json", diff --git a/ext/ffi/Cargo.toml b/ext/ffi/Cargo.toml index 80b0180d0b..59dc13ed96 100644 --- a/ext/ffi/Cargo.toml +++ b/ext/ffi/Cargo.toml @@ -21,6 +21,7 @@ dynasmrt = "1.2.3" libffi = "=3.2.0" libffi-sys = "=2.3.0" log.workspace = true +num-bigint.workspace = true serde.workspace = true serde-value = "0.7" serde_json = "1.0" diff --git a/ext/ffi/call.rs b/ext/ffi/call.rs index d337b29b00..ef61dc383e 100644 --- a/ext/ffi/call.rs +++ b/ext/ffi/call.rs @@ -9,12 +9,14 @@ use crate::FfiPermissions; use crate::ForeignFunction; use deno_core::op2; use deno_core::serde_json::Value; +use deno_core::serde_v8::BigInt as V8BigInt; use deno_core::serde_v8::ExternalPointer; use deno_core::unsync::spawn_blocking; use deno_core::v8; use deno_core::OpState; use deno_core::ResourceId; use libffi::middle::Arg; +use num_bigint::BigInt; use serde::Serialize; use std::cell::RefCell; use std::ffi::c_void; @@ -202,6 +204,7 @@ where #[serde(untagged)] pub enum FfiValue { Value(Value), + BigInt(V8BigInt), External(ExternalPointer), } @@ -251,18 +254,18 @@ fn ffi_call( NativeType::I32 => { FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) } - NativeType::U64 => { - FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) - } - NativeType::I64 => { - FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) - } - NativeType::USize => { - FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) - } - NativeType::ISize => { - FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) - } + NativeType::U64 => FfiValue::BigInt(V8BigInt::from(BigInt::from( + cif.call::(fun_ptr, &call_args), + ))), + NativeType::I64 => FfiValue::BigInt(V8BigInt::from(BigInt::from( + cif.call::(fun_ptr, &call_args), + ))), + NativeType::USize => FfiValue::BigInt(V8BigInt::from(BigInt::from( + cif.call::(fun_ptr, &call_args), + ))), + NativeType::ISize => FfiValue::BigInt(V8BigInt::from(BigInt::from( + cif.call::(fun_ptr, &call_args), + ))), NativeType::F32 => { FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) } diff --git a/tests/ffi/tests/integration_tests.rs b/tests/ffi/tests/integration_tests.rs index c84a1b820d..dbc0036bc2 100644 --- a/tests/ffi/tests/integration_tests.rs +++ b/tests/ffi/tests/integration_tests.rs @@ -95,10 +95,10 @@ fn basic() { 579.912\n\ 579.912\n\ 579\n\ - 8589934590\n\ - -8589934590\n\ - 8589934590\n\ - -8589934590\n\ + 8589934590n\n\ + -8589934590n\n\ + 8589934590n\n\ + -8589934590n\n\ 9007199254740992n\n\ 9007199254740992n\n\ -9007199254740992n\n\ From 7c57105cc41cf16c5f48f85d85c7fd9bd3bb4d1f Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:13:30 -0700 Subject: [PATCH 05/63] fix(ext/node): only set our end of child process pipe to nonblocking mode (#26495) Fixes playwright on linux, as reported in https://github.com/denoland/deno/issues/16899#issuecomment-2378268454. The issue was that we were opening the socket in nonblocking mode, which meant that subprocesses trying to use it would get a `EWOULDBLOCK` error (unexpectedly). The fix here is to only set nonblocking mode on our end (which we need to use asynchronously) --- ext/io/bi_pipe.rs | 21 +++++++--------- .../node/child_process_extra_pipes/main.out | 2 +- .../node/child_process_extra_pipes/main.ts | 20 ++++++++++++--- .../test-pipe/src/main.rs | 25 ++++++++++++++++--- 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/ext/io/bi_pipe.rs b/ext/io/bi_pipe.rs index b6fc70ca24..3492e2f441 100644 --- a/ext/io/bi_pipe.rs +++ b/ext/io/bi_pipe.rs @@ -183,9 +183,10 @@ fn from_raw( ) -> Result<(BiPipeRead, BiPipeWrite), std::io::Error> { use std::os::fd::FromRawFd; // Safety: The fd is part of a pair of connected sockets - let unix_stream = tokio::net::UnixStream::from_std(unsafe { - std::os::unix::net::UnixStream::from_raw_fd(stream) - })?; + let unix_stream = + unsafe { std::os::unix::net::UnixStream::from_raw_fd(stream) }; + unix_stream.set_nonblocking(true)?; + let unix_stream = tokio::net::UnixStream::from_std(unix_stream)?; let (read, write) = unix_stream.into_split(); Ok((BiPipeRead { inner: read }, BiPipeWrite { inner: write })) } @@ -280,7 +281,7 @@ pub fn bi_pipe_pair_raw( // https://github.com/nix-rust/nix/issues/861 let mut fds = [-1, -1]; #[cfg(not(target_os = "macos"))] - let flags = libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK; + let flags = libc::SOCK_CLOEXEC; #[cfg(target_os = "macos")] let flags = 0; @@ -301,13 +302,13 @@ pub fn bi_pipe_pair_raw( if cfg!(target_os = "macos") { let fcntl = |fd: i32, flag: libc::c_int| -> Result<(), std::io::Error> { // SAFETY: libc call, fd is valid - let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + let flags = unsafe { libc::fcntl(fd, libc::F_GETFD) }; if flags == -1 { return Err(fail(fds)); } // SAFETY: libc call, fd is valid - let ret = unsafe { libc::fcntl(fd, libc::F_SETFL, flags | flag) }; + let ret = unsafe { libc::fcntl(fd, libc::F_SETFD, flags | flag) }; if ret == -1 { return Err(fail(fds)); } @@ -323,13 +324,9 @@ pub fn bi_pipe_pair_raw( std::io::Error::last_os_error() } - // SOCK_NONBLOCK is not supported on macOS. - (fcntl)(fds[0], libc::O_NONBLOCK)?; - (fcntl)(fds[1], libc::O_NONBLOCK)?; - // SOCK_CLOEXEC is not supported on macOS. - (fcntl)(fds[0], libc::FD_CLOEXEC)?; - (fcntl)(fds[1], libc::FD_CLOEXEC)?; + fcntl(fds[0], libc::FD_CLOEXEC)?; + fcntl(fds[1], libc::FD_CLOEXEC)?; } let fd1 = fds[0]; diff --git a/tests/specs/node/child_process_extra_pipes/main.out b/tests/specs/node/child_process_extra_pipes/main.out index 694126b924..436afd2f75 100644 --- a/tests/specs/node/child_process_extra_pipes/main.out +++ b/tests/specs/node/child_process_extra_pipes/main.out @@ -1,5 +1,5 @@ -data: hello world [UNORDERED_START] child closed +got: hello world pipe closed [UNORDERED_END] diff --git a/tests/specs/node/child_process_extra_pipes/main.ts b/tests/specs/node/child_process_extra_pipes/main.ts index a3683fe9ee..2837cdc53c 100644 --- a/tests/specs/node/child_process_extra_pipes/main.ts +++ b/tests/specs/node/child_process_extra_pipes/main.ts @@ -1,5 +1,4 @@ import child_process from "node:child_process"; -import { Buffer } from "node:buffer"; import console from "node:console"; const child = child_process.spawn("./test-pipe/target/debug/test-pipe", [], { @@ -8,19 +7,32 @@ const child = child_process.spawn("./test-pipe/target/debug/test-pipe", [], { const extra = child.stdio[4]; -const p = Promise.withResolvers(); +if (!extra) { + throw new Error("no extra pipe"); +} + +const p = Promise.withResolvers(); + +let got = ""; child.on("close", () => { console.log("child closed"); - p.resolve(); + console.log("got:", got); + if (got === "hello world") { + p.resolve(); + } else { + p.reject(new Error(`wanted "hello world", got "${got}"`)); + } }); extra.on("data", (d) => { - console.log("data:", d.toString().trim()); + got += d.toString(); }); extra.on("close", () => { console.log("pipe closed"); }); +extra.write("start"); + await p.promise; diff --git a/tests/specs/node/child_process_extra_pipes/test-pipe/src/main.rs b/tests/specs/node/child_process_extra_pipes/test-pipe/src/main.rs index 192f827313..acc034830f 100644 --- a/tests/specs/node/child_process_extra_pipes/test-pipe/src/main.rs +++ b/tests/specs/node/child_process_extra_pipes/test-pipe/src/main.rs @@ -1,12 +1,31 @@ +use std::fs::File; use std::io::prelude::*; use std::os::fd::FromRawFd; -use std::os::unix::net::UnixStream; fn main() { #[cfg(unix)] { - let mut stream = unsafe { UnixStream::from_raw_fd(4) }; + let mut pipe = unsafe { File::from_raw_fd(4) }; - stream.write_all(b"hello world\n").unwrap(); + let mut read = 0; + let mut buf = [0u8; 1024]; + loop { + if read > 4 { + assert_eq!(&buf[..5], b"start"); + break; + } + match pipe.read(&mut buf) { + Ok(n) => { + read += n; + } + Ok(0) => { + return; + } + Err(e) => { + eprintln!("GOT ERROR: {e:?}"); + } + } + } + pipe.write_all(b"hello world").unwrap(); } } From 27df42f659ae7b77968d31363ade89addb516ea1 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:50:35 -0700 Subject: [PATCH 06/63] fix(ext/node): cancel pending ipc writes on channel close (#26504) Fixes the issue described in https://github.com/denoland/deno/issues/23882#issuecomment-2423316362. The parent was starting to send a message right before the process would exit, and the channel closed in the middle of the write. Unlike with reads, we weren't cancelling the pending writes, which resulted in a `Broken pipe` error surfacing to the user. --- ext/node/ops/ipc.rs | 9 ++++++++- ext/node/polyfills/internal/child_process.ts | 19 +++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/ext/node/ops/ipc.rs b/ext/node/ops/ipc.rs index 59b6fece14..c5e0f875d5 100644 --- a/ext/node/ops/ipc.rs +++ b/ext/node/ops/ipc.rs @@ -216,10 +216,17 @@ mod impl_ { queue_ok.set_index(scope, 0, v); } Ok(async move { - stream.clone().write_msg_bytes(&serialized).await?; + let cancel = stream.cancel.clone(); + let result = stream + .clone() + .write_msg_bytes(&serialized) + .or_cancel(cancel) + .await; + // adjust count even on error stream .queued_bytes .fetch_sub(serialized.len(), std::sync::atomic::Ordering::Relaxed); + result??; Ok(()) }) } diff --git a/ext/node/polyfills/internal/child_process.ts b/ext/node/polyfills/internal/child_process.ts index 65d825fd25..cfff1079ff 100644 --- a/ext/node/polyfills/internal/child_process.ts +++ b/ext/node/polyfills/internal/child_process.ts @@ -1339,7 +1339,7 @@ export function setupChannel(target: any, ipc: number) { } } - process.nextTick(handleMessage, msg); + nextTick(handleMessage, msg); } } catch (err) { if ( @@ -1400,7 +1400,7 @@ export function setupChannel(target: any, ipc: number) { if (!target.connected) { const err = new ERR_IPC_CHANNEL_CLOSED(); if (typeof callback === "function") { - process.nextTick(callback, err); + nextTick(callback, err); } else { nextTick(() => target.emit("error", err)); } @@ -1416,7 +1416,18 @@ export function setupChannel(target: any, ipc: number) { .then(() => { control.unrefCounted(); if (callback) { - process.nextTick(callback, null); + nextTick(callback, null); + } + }, (err: Error) => { + control.unrefCounted(); + if (err instanceof Deno.errors.Interrupted) { + // Channel closed on us mid-write. + } else { + if (typeof callback === "function") { + nextTick(callback, err); + } else { + nextTick(() => target.emit("error", err)); + } } }); return queueOk[0]; @@ -1433,7 +1444,7 @@ export function setupChannel(target: any, ipc: number) { target.connected = false; target[kCanDisconnect] = false; control[kControlDisconnect](); - process.nextTick(() => { + nextTick(() => { target.channel = null; core.close(ipc); target.emit("disconnect"); From 79a3ad2b950009f560641cea359d7deb6f7a61ac Mon Sep 17 00:00:00 2001 From: snek Date: Thu, 24 Oct 2024 09:13:54 +0200 Subject: [PATCH 07/63] feat: support node-api in denort (#26389) exposes node-api symbols in denort so that `deno compile` can run native addons. --- Cargo.lock | 8 +- Cargo.toml | 4 +- cli/Cargo.toml | 3 - cli/build.rs | 55 +-------- cli/main.rs | 5 +- cli/napi/README.md | 114 ------------------ cli/napi/mod.rs | 21 ---- ext/napi/Cargo.toml | 9 ++ ext/napi/README.md | 114 ++++++++++++++++++ ext/napi/build.rs | 22 ++++ .../generated_symbol_exports_list_linux.def | 0 .../generated_symbol_exports_list_macos.def | 0 .../generated_symbol_exports_list_windows.def | 0 {cli => ext}/napi/js_native_api.rs | 12 +- ext/napi/lib.rs | 43 +++++++ {cli => ext}/napi/node_api.rs | 16 ++- {cli => ext}/napi/sym/Cargo.toml | 0 {cli => ext}/napi/sym/README.md | 3 +- {cli => ext}/napi/sym/lib.rs | 0 {cli => ext}/napi/sym/symbol_exports.json | 0 {cli => ext}/napi/util.rs | 34 +++--- {cli => ext}/napi/uv.rs | 10 +- tools/napi/generate_symbols_lists.js | 4 +- 23 files changed, 242 insertions(+), 235 deletions(-) delete mode 100644 cli/napi/README.md delete mode 100644 cli/napi/mod.rs create mode 100644 ext/napi/build.rs rename {cli => ext}/napi/generated_symbol_exports_list_linux.def (100%) rename {cli => ext}/napi/generated_symbol_exports_list_macos.def (100%) rename {cli => ext}/napi/generated_symbol_exports_list_windows.def (100%) rename {cli => ext}/napi/js_native_api.rs (99%) rename {cli => ext}/napi/node_api.rs (98%) rename {cli => ext}/napi/sym/Cargo.toml (100%) rename {cli => ext}/napi/sym/README.md (92%) rename {cli => ext}/napi/sym/lib.rs (100%) rename {cli => ext}/napi/sym/symbol_exports.json (100%) rename {cli => ext}/napi/util.rs (84%) rename {cli => ext}/napi/uv.rs (95%) diff --git a/Cargo.lock b/Cargo.lock index d550baee66..3549f0e422 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1214,7 +1214,6 @@ dependencies = [ "lazy-regex", "libc", "libsui", - "libuv-sys-lite", "libz-sys", "log", "lsp-types", @@ -1222,7 +1221,6 @@ dependencies = [ "markup_fmt", "memmem", "monch", - "napi_sym", "nix", "node_resolver", "notify", @@ -1263,7 +1261,6 @@ dependencies = [ "walkdir", "which 4.4.2", "winapi", - "windows-sys 0.52.0", "winres", "zeromq", "zip", @@ -1775,8 +1772,13 @@ version = "0.104.0" dependencies = [ "deno_core", "deno_permissions", + "libc", "libloading 0.7.4", + "libuv-sys-lite", + "log", + "napi_sym", "thiserror", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 61d6ca37ae..a420a9968e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ resolver = "2" members = [ "bench_util", "cli", - "cli/napi/sym", "ext/broadcast_channel", "ext/cache", "ext/canvas", @@ -19,6 +18,7 @@ members = [ "ext/io", "ext/kv", "ext/napi", + "ext/napi/sym", "ext/net", "ext/node", "ext/url", @@ -57,7 +57,7 @@ deno_permissions = { version = "0.33.0", path = "./runtime/permissions" } deno_runtime = { version = "0.182.0", path = "./runtime" } deno_semver = "=0.5.16" deno_terminal = "0.2.0" -napi_sym = { version = "0.103.0", path = "./cli/napi/sym" } +napi_sym = { version = "0.103.0", path = "./ext/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } denokv_proto = "0.8.1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index d6d8982b12..ded6551cd7 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -86,7 +86,6 @@ deno_task_shell = "=0.18.1" deno_terminal.workspace = true eszip = "=0.79.1" libsui = "0.4.0" -napi_sym.workspace = true node_resolver.workspace = true anstream = "0.6.14" @@ -175,14 +174,12 @@ zstd.workspace = true [target.'cfg(windows)'.dependencies] junction.workspace = true winapi = { workspace = true, features = ["knownfolders", "mswsock", "objbase", "shlobj", "tlhelp32", "winbase", "winerror", "winsock2"] } -windows-sys.workspace = true [target.'cfg(unix)'.dependencies] nix.workspace = true [dev-dependencies] deno_bench_util.workspace = true -libuv-sys-lite = "=1.48.2" pretty_assertions.workspace = true test_util.workspace = true diff --git a/cli/build.rs b/cli/build.rs index aa5d3d18c6..2678a8dbb0 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -365,6 +365,9 @@ fn main() { return; } + deno_napi::print_linker_flags("deno"); + deno_napi::print_linker_flags("denort"); + // Host snapshots won't work when cross compiling. let target = env::var("TARGET").unwrap(); let host = env::var("HOST").unwrap(); @@ -374,58 +377,6 @@ fn main() { panic!("Cross compiling with snapshot is not supported."); } - let symbols_file_name = match env::consts::OS { - "android" | "freebsd" | "openbsd" => { - "generated_symbol_exports_list_linux.def".to_string() - } - os => format!("generated_symbol_exports_list_{}.def", os), - }; - let symbols_path = std::path::Path::new("napi") - .join(symbols_file_name) - .canonicalize() - .expect( - "Missing symbols list! Generate using tools/napi/generate_symbols_lists.js", - ); - - println!("cargo:rustc-rerun-if-changed={}", symbols_path.display()); - - #[cfg(target_os = "windows")] - println!( - "cargo:rustc-link-arg-bin=deno=/DEF:{}", - symbols_path.display() - ); - - #[cfg(target_os = "macos")] - println!( - "cargo:rustc-link-arg-bin=deno=-Wl,-exported_symbols_list,{}", - symbols_path.display() - ); - - #[cfg(target_os = "linux")] - { - // If a custom compiler is set, the glibc version is not reliable. - // Here, we assume that if a custom compiler is used, that it will be modern enough to support a dynamic symbol list. - if env::var("CC").is_err() - && glibc_version::get_version() - .map(|ver| ver.major <= 2 && ver.minor < 35) - .unwrap_or(false) - { - println!("cargo:warning=Compiling with all symbols exported, this will result in a larger binary. Please use glibc 2.35 or later for an optimised build."); - println!("cargo:rustc-link-arg-bin=deno=-rdynamic"); - } else { - println!( - "cargo:rustc-link-arg-bin=deno=-Wl,--export-dynamic-symbol-list={}", - symbols_path.display() - ); - } - } - - #[cfg(target_os = "android")] - println!( - "cargo:rustc-link-arg-bin=deno=-Wl,--export-dynamic-symbol-list={}", - symbols_path.display() - ); - // To debug snapshot issues uncomment: // op_fetch_asset::trace_serializer(); diff --git a/cli/main.rs b/cli/main.rs index d99a4c402a..c2639c9088 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -15,7 +15,6 @@ mod js; mod jsr; mod lsp; mod module_loader; -mod napi; mod node; mod npm; mod ops; @@ -169,10 +168,10 @@ async fn run_subcommand(flags: Arc) -> Result { if std::io::stderr().is_terminal() { log::warn!( "{} command is intended to be run by text editors and IDEs and shouldn't be run manually. - + Visit https://docs.deno.com/runtime/getting_started/setup_your_environment/ for instruction how to setup your favorite text editor. - + Press Ctrl+C to exit. ", colors::cyan("deno lsp")); } diff --git a/cli/napi/README.md b/cli/napi/README.md deleted file mode 100644 index 7b359ac6ec..0000000000 --- a/cli/napi/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# napi - -This directory contains source for Deno's Node-API implementation. It depends on -`napi_sym` and `deno_napi`. - -Files are generally organized the same as in Node.js's implementation to ease in -ensuring compatibility. - -## Adding a new function - -Add the symbol name to -[`cli/napi_sym/symbol_exports.json`](../napi_sym/symbol_exports.json). - -```diff -{ - "symbols": [ - ... - "napi_get_undefined", -- "napi_get_null" -+ "napi_get_null", -+ "napi_get_boolean" - ] -} -``` - -Determine where to place the implementation. `napi_get_boolean` is related to JS -values so we will place it in `js_native_api.rs`. If something is not clear, -just create a new file module. - -See [`napi_sym`](../napi_sym/) for writing the implementation: - -```rust -#[napi_sym::napi_sym] -pub fn napi_get_boolean( - env: *mut Env, - value: bool, - result: *mut napi_value, -) -> Result { - // ... - Ok(()) -} -``` - -Update the generated symbol lists using the script: - -``` -deno run --allow-write tools/napi/generate_symbols_lists.js -``` - -Add a test in [`/tests/napi`](../../tests/napi/). You can also refer to Node.js -test suite for Node-API. - -```js -// tests/napi/boolean_test.js -import { assertEquals, loadTestLibrary } from "./common.js"; -const lib = loadTestLibrary(); -Deno.test("napi get boolean", function () { - assertEquals(lib.test_get_boolean(true), true); - assertEquals(lib.test_get_boolean(false), false); -}); -``` - -```rust -// tests/napi/src/boolean.rs - -use napi_sys::Status::napi_ok; -use napi_sys::ValueType::napi_boolean; -use napi_sys::*; - -extern "C" fn test_boolean( - env: napi_env, - info: napi_callback_info, -) -> napi_value { - let (args, argc, _) = crate::get_callback_info!(env, info, 1); - assert_eq!(argc, 1); - - let mut ty = -1; - assert!(unsafe { napi_typeof(env, args[0], &mut ty) } == napi_ok); - assert_eq!(ty, napi_boolean); - - // Use napi_get_boolean here... - - value -} - -pub fn init(env: napi_env, exports: napi_value) { - let properties = &[crate::new_property!(env, "test_boolean\0", test_boolean)]; - - unsafe { - napi_define_properties(env, exports, properties.len(), properties.as_ptr()) - }; -} -``` - -```diff -// tests/napi/src/lib.rs - -+ mod boolean; - -... - -#[no_mangle] -unsafe extern "C" fn napi_register_module_v1( - env: napi_env, - exports: napi_value, -) -> napi_value { - ... -+ boolean::init(env, exports); - - exports -} -``` - -Run the test using `cargo test -p tests/napi`. diff --git a/cli/napi/mod.rs b/cli/napi/mod.rs deleted file mode 100644 index 811efb1ecc..0000000000 --- a/cli/napi/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -#![allow(unused_mut)] -#![allow(non_camel_case_types)] -#![allow(clippy::undocumented_unsafe_blocks)] - -//! Symbols to be exported are now defined in this JSON file. -//! The `#[napi_sym]` macro checks for missing entries and panics. -//! -//! `./tools/napi/generate_symbols_list.js` is used to generate the LINK `cli/exports.def` on Windows, -//! which is also checked into git. -//! -//! To add a new napi function: -//! 1. Place `#[napi_sym]` on top of your implementation. -//! 2. Add the function's identifier to this JSON list. -//! 3. Finally, run `tools/napi/generate_symbols_list.js` to update `cli/napi/generated_symbol_exports_list_*.def`. - -pub mod js_native_api; -pub mod node_api; -pub mod util; -pub mod uv; diff --git a/ext/napi/Cargo.toml b/ext/napi/Cargo.toml index ef2e41d57e..ed39d96df3 100644 --- a/ext/napi/Cargo.toml +++ b/ext/napi/Cargo.toml @@ -16,5 +16,14 @@ path = "lib.rs" [dependencies] deno_core.workspace = true deno_permissions.workspace = true +libc.workspace = true libloading = { version = "0.7" } +log.workspace = true +napi_sym.workspace = true thiserror.workspace = true + +[target.'cfg(windows)'.dependencies] +windows-sys.workspace = true + +[dev-dependencies] +libuv-sys-lite = "=1.48.2" diff --git a/ext/napi/README.md b/ext/napi/README.md index e69de29bb2..b479295244 100644 --- a/ext/napi/README.md +++ b/ext/napi/README.md @@ -0,0 +1,114 @@ +# napi + +This directory contains source for Deno's Node-API implementation. It depends on +`napi_sym` and `deno_napi`. + +Files are generally organized the same as in Node.js's implementation to ease in +ensuring compatibility. + +## Adding a new function + +Add the symbol name to +[`cli/napi_sym/symbol_exports.json`](../napi_sym/symbol_exports.json). + +```diff +{ + "symbols": [ + ... + "napi_get_undefined", +- "napi_get_null" ++ "napi_get_null", ++ "napi_get_boolean" + ] +} +``` + +Determine where to place the implementation. `napi_get_boolean` is related to JS +values so we will place it in `js_native_api.rs`. If something is not clear, +just create a new file module. + +See [`napi_sym`](../napi_sym/) for writing the implementation: + +```rust +#[napi_sym::napi_sym] +fn napi_get_boolean( + env: *mut Env, + value: bool, + result: *mut napi_value, +) -> Result { + // ... + Ok(()) +} +``` + +Update the generated symbol lists using the script: + +``` +deno run --allow-write tools/napi/generate_symbols_lists.js +``` + +Add a test in [`/tests/napi`](../../tests/napi/). You can also refer to Node.js +test suite for Node-API. + +```js +// tests/napi/boolean_test.js +import { assertEquals, loadTestLibrary } from "./common.js"; +const lib = loadTestLibrary(); +Deno.test("napi get boolean", function () { + assertEquals(lib.test_get_boolean(true), true); + assertEquals(lib.test_get_boolean(false), false); +}); +``` + +```rust +// tests/napi/src/boolean.rs + +use napi_sys::Status::napi_ok; +use napi_sys::ValueType::napi_boolean; +use napi_sys::*; + +extern "C" fn test_boolean( + env: napi_env, + info: napi_callback_info, +) -> napi_value { + let (args, argc, _) = crate::get_callback_info!(env, info, 1); + assert_eq!(argc, 1); + + let mut ty = -1; + assert!(unsafe { napi_typeof(env, args[0], &mut ty) } == napi_ok); + assert_eq!(ty, napi_boolean); + + // Use napi_get_boolean here... + + value +} + +pub fn init(env: napi_env, exports: napi_value) { + let properties = &[crate::new_property!(env, "test_boolean\0", test_boolean)]; + + unsafe { + napi_define_properties(env, exports, properties.len(), properties.as_ptr()) + }; +} +``` + +```diff +// tests/napi/src/lib.rs + ++ mod boolean; + +... + +#[no_mangle] +unsafe extern "C" fn napi_register_module_v1( + env: napi_env, + exports: napi_value, +) -> napi_value { + ... ++ boolean::init(env, exports); + + exports +} +``` + +Run the test using `cargo test -p tests/napi`. diff --git a/ext/napi/build.rs b/ext/napi/build.rs new file mode 100644 index 0000000000..8705830a95 --- /dev/null +++ b/ext/napi/build.rs @@ -0,0 +1,22 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +fn main() { + let symbols_file_name = match std::env::consts::OS { + "android" | "freebsd" | "openbsd" => { + "generated_symbol_exports_list_linux.def".to_string() + } + os => format!("generated_symbol_exports_list_{}.def", os), + }; + let symbols_path = std::path::Path::new(".") + .join(symbols_file_name) + .canonicalize() + .expect( + "Missing symbols list! Generate using tools/napi/generate_symbols_lists.js", + ); + + println!("cargo:rustc-rerun-if-changed={}", symbols_path.display()); + + let path = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap()) + .join("napi_symbol_path.txt"); + std::fs::write(path, symbols_path.as_os_str().as_encoded_bytes()).unwrap(); +} diff --git a/cli/napi/generated_symbol_exports_list_linux.def b/ext/napi/generated_symbol_exports_list_linux.def similarity index 100% rename from cli/napi/generated_symbol_exports_list_linux.def rename to ext/napi/generated_symbol_exports_list_linux.def diff --git a/cli/napi/generated_symbol_exports_list_macos.def b/ext/napi/generated_symbol_exports_list_macos.def similarity index 100% rename from cli/napi/generated_symbol_exports_list_macos.def rename to ext/napi/generated_symbol_exports_list_macos.def diff --git a/cli/napi/generated_symbol_exports_list_windows.def b/ext/napi/generated_symbol_exports_list_windows.def similarity index 100% rename from cli/napi/generated_symbol_exports_list_windows.def rename to ext/napi/generated_symbol_exports_list_windows.def diff --git a/cli/napi/js_native_api.rs b/ext/napi/js_native_api.rs similarity index 99% rename from cli/napi/js_native_api.rs rename to ext/napi/js_native_api.rs index 35e7690c3c..53a12d6eba 100644 --- a/cli/napi/js_native_api.rs +++ b/ext/napi/js_native_api.rs @@ -5,7 +5,7 @@ const NAPI_VERSION: u32 = 9; -use deno_runtime::deno_napi::*; +use crate::*; use libc::INT_MAX; use super::util::check_new_from_utf8; @@ -17,9 +17,9 @@ use super::util::napi_set_last_error; use super::util::v8_name_from_property_descriptor; use crate::check_arg; use crate::check_env; -use deno_runtime::deno_napi::function::create_function; -use deno_runtime::deno_napi::function::create_function_template; -use deno_runtime::deno_napi::function::CallbackInfo; +use crate::function::create_function; +use crate::function::create_function_template; +use crate::function::CallbackInfo; use napi_sym::napi_sym; use std::ptr::NonNull; @@ -1083,7 +1083,7 @@ fn napi_create_string_latin1( } #[napi_sym] -fn napi_create_string_utf8( +pub(crate) fn napi_create_string_utf8( env_ptr: *mut Env, string: *const c_char, length: usize, @@ -1647,7 +1647,7 @@ fn napi_get_cb_info( check_arg!(env, argc); let argc = unsafe { *argc as usize }; for i in 0..argc { - let mut arg = args.get(i as _); + let arg = args.get(i as _); unsafe { *argv.add(i) = arg.into(); } diff --git a/ext/napi/lib.rs b/ext/napi/lib.rs index 0b2b3eb5e7..0d41bb40fe 100644 --- a/ext/napi/lib.rs +++ b/ext/napi/lib.rs @@ -5,6 +5,22 @@ #![allow(clippy::undocumented_unsafe_blocks)] #![deny(clippy::missing_safety_doc)] +//! Symbols to be exported are now defined in this JSON file. +//! The `#[napi_sym]` macro checks for missing entries and panics. +//! +//! `./tools/napi/generate_symbols_list.js` is used to generate the LINK `cli/exports.def` on Windows, +//! which is also checked into git. +//! +//! To add a new napi function: +//! 1. Place `#[napi_sym]` on top of your implementation. +//! 2. Add the function's identifier to this JSON list. +//! 3. Finally, run `tools/napi/generate_symbols_list.js` to update `ext/napi/generated_symbol_exports_list_*.def`. + +pub mod js_native_api; +pub mod node_api; +pub mod util; +pub mod uv; + use core::ptr::NonNull; use deno_core::op2; use deno_core::parking_lot::RwLock; @@ -631,3 +647,30 @@ where Ok(exports) } + +#[allow(clippy::print_stdout)] +pub fn print_linker_flags(name: &str) { + let symbols_path = + include_str!(concat!(env!("OUT_DIR"), "/napi_symbol_path.txt")); + + #[cfg(target_os = "windows")] + println!("cargo:rustc-link-arg-bin={name}=/DEF:{}", symbols_path); + + #[cfg(target_os = "macos")] + println!( + "cargo:rustc-link-arg-bin={name}=-Wl,-exported_symbols_list,{}", + symbols_path, + ); + + #[cfg(target_os = "linux")] + println!( + "cargo:rustc-link-arg-bin={name}=-Wl,--export-dynamic-symbol-list={}", + symbols_path, + ); + + #[cfg(target_os = "android")] + println!( + "cargo:rustc-link-arg-bin={name}=-Wl,--export-dynamic-symbol-list={}", + symbols_path, + ); +} diff --git a/cli/napi/node_api.rs b/ext/napi/node_api.rs similarity index 98% rename from cli/napi/node_api.rs rename to ext/napi/node_api.rs index 4497a4695a..186ae42c48 100644 --- a/cli/napi/node_api.rs +++ b/ext/napi/node_api.rs @@ -9,10 +9,10 @@ use super::util::napi_set_last_error; use super::util::SendPtr; use crate::check_arg; use crate::check_env; +use crate::*; use deno_core::parking_lot::Condvar; use deno_core::parking_lot::Mutex; use deno_core::V8CrossThreadTaskSpawner; -use deno_runtime::deno_napi::*; use napi_sym::napi_sym; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU8; @@ -488,7 +488,7 @@ impl AsyncWork { } #[napi_sym] -fn napi_create_async_work( +pub(crate) fn napi_create_async_work( env: *mut Env, async_resource: napi_value, async_resource_name: napi_value, @@ -537,7 +537,10 @@ fn napi_create_async_work( } #[napi_sym] -fn napi_delete_async_work(env: *mut Env, work: napi_async_work) -> napi_status { +pub(crate) fn napi_delete_async_work( + env: *mut Env, + work: napi_async_work, +) -> napi_status { let env = check_env!(env); check_arg!(env, work); @@ -560,7 +563,10 @@ fn napi_get_uv_event_loop( } #[napi_sym] -fn napi_queue_async_work(env: *mut Env, work: napi_async_work) -> napi_status { +pub(crate) fn napi_queue_async_work( + env: *mut Env, + work: napi_async_work, +) -> napi_status { let env = check_env!(env); check_arg!(env, work); @@ -897,7 +903,7 @@ fn napi_create_threadsafe_function( }; let resource_name = resource_name.to_rust_string_lossy(&mut env.scope()); - let mut tsfn = Box::new(TsFn { + let tsfn = Box::new(TsFn { env, func, max_queue_size, diff --git a/cli/napi/sym/Cargo.toml b/ext/napi/sym/Cargo.toml similarity index 100% rename from cli/napi/sym/Cargo.toml rename to ext/napi/sym/Cargo.toml diff --git a/cli/napi/sym/README.md b/ext/napi/sym/README.md similarity index 92% rename from cli/napi/sym/README.md rename to ext/napi/sym/README.md index de08a8e174..66eb4bff26 100644 --- a/cli/napi/sym/README.md +++ b/ext/napi/sym/README.md @@ -2,7 +2,8 @@ A proc_macro for Deno's Node-API implementation. It does the following things: -- Marks the symbol as `#[no_mangle]` and rewrites it as `pub extern "C" $name`. +- Marks the symbol as `#[no_mangle]` and rewrites it as + `unsafe extern "C" $name`. - Asserts that the function symbol is present in [`symbol_exports.json`](./symbol_exports.json). - Maps `deno_napi::Result` to raw `napi_result`. diff --git a/cli/napi/sym/lib.rs b/ext/napi/sym/lib.rs similarity index 100% rename from cli/napi/sym/lib.rs rename to ext/napi/sym/lib.rs diff --git a/cli/napi/sym/symbol_exports.json b/ext/napi/sym/symbol_exports.json similarity index 100% rename from cli/napi/sym/symbol_exports.json rename to ext/napi/sym/symbol_exports.json diff --git a/cli/napi/util.rs b/ext/napi/util.rs similarity index 84% rename from cli/napi/util.rs rename to ext/napi/util.rs index 63d8effbf2..21e9d433aa 100644 --- a/cli/napi/util.rs +++ b/ext/napi/util.rs @@ -1,9 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_runtime::deno_napi::*; +use crate::*; use libc::INT_MAX; #[repr(transparent)] -pub struct SendPtr(pub *const T); +pub(crate) struct SendPtr(pub *const T); impl SendPtr { // silly function to get around `clippy::redundant_locals` @@ -37,7 +37,7 @@ impl Drop for BufferFinalizer { } } -pub extern "C" fn backing_store_deleter_callback( +pub(crate) extern "C" fn backing_store_deleter_callback( data: *mut c_void, _byte_length: usize, deleter_data: *mut c_void, @@ -50,7 +50,7 @@ pub extern "C" fn backing_store_deleter_callback( drop(finalizer); } -pub fn make_external_backing_store( +pub(crate) fn make_external_backing_store( env: *mut Env, data: *mut c_void, byte_length: usize, @@ -90,9 +90,7 @@ macro_rules! check_env { macro_rules! return_error_status_if_false { ($env: expr, $condition: expr, $status: ident) => { if !$condition { - return Err( - $crate::napi::util::napi_set_last_error($env, $status).into(), - ); + return Err($crate::util::napi_set_last_error($env, $status).into()); } }; } @@ -101,7 +99,7 @@ macro_rules! return_error_status_if_false { macro_rules! return_status_if_false { ($env: expr, $condition: expr, $status: ident) => { if !$condition { - return $crate::napi::util::napi_set_last_error($env, $status); + return $crate::util::napi_set_last_error($env, $status); } }; } @@ -222,7 +220,7 @@ macro_rules! check_arg { ($env: expr, $ptr: expr) => { $crate::return_status_if_false!( $env, - !$crate::napi::util::Nullable::is_null(&$ptr), + !$crate::util::Nullable::is_null(&$ptr), napi_invalid_arg ); }; @@ -230,17 +228,17 @@ macro_rules! check_arg { #[macro_export] macro_rules! napi_wrap { - ( $( # $attr:tt )* fn $name:ident $( < $( $x:lifetime ),* > )? ( $env:ident : & $( $lt:lifetime )? mut Env $( , $ident:ident : $ty:ty )* $(,)? ) -> napi_status $body:block ) => { - $( # $attr )* + ( $( # [ $attr:meta ] )* $vis:vis fn $name:ident $( < $( $x:lifetime ),* > )? ( $env:ident : & $( $lt:lifetime )? mut Env $( , $ident:ident : $ty:ty )* $(,)? ) -> napi_status $body:block ) => { + $( # [ $attr ] )* #[no_mangle] - pub unsafe extern "C" fn $name $( < $( $x ),* > )? ( env_ptr : *mut Env , $( $ident : $ty ),* ) -> napi_status { + $vis unsafe extern "C" fn $name $( < $( $x ),* > )? ( env_ptr : *mut Env , $( $ident : $ty ),* ) -> napi_status { let env: & $( $lt )? mut Env = $crate::check_env!(env_ptr); if env.last_exception.is_some() { return napi_pending_exception; } - $crate::napi::util::napi_clear_last_error(env); + $crate::util::napi_clear_last_error(env); let scope_env = unsafe { &mut *env_ptr }; let scope = &mut scope_env.scope(); @@ -259,21 +257,21 @@ macro_rules! napi_wrap { let env = unsafe { &mut *env_ptr }; let global = v8::Global::new(env.isolate(), exception); env.last_exception = Some(global); - return $crate::napi::util::napi_set_last_error(env_ptr, napi_pending_exception); + return $crate::util::napi_set_last_error(env_ptr, napi_pending_exception); } if result != napi_ok { - return $crate::napi::util::napi_set_last_error(env_ptr, result); + return $crate::util::napi_set_last_error(env_ptr, result); } return result; } }; - ( $( # $attr:tt )* fn $name:ident $( < $( $x:lifetime ),* > )? ( $( $ident:ident : $ty:ty ),* $(,)? ) -> napi_status $body:block ) => { - $( # $attr )* + ( $( # [ $attr:meta ] )* $vis:vis fn $name:ident $( < $( $x:lifetime ),* > )? ( $( $ident:ident : $ty:ty ),* $(,)? ) -> napi_status $body:block ) => { + $( # [ $attr ] )* #[no_mangle] - pub unsafe extern "C" fn $name $( < $( $x ),* > )? ( $( $ident : $ty ),* ) -> napi_status { + $vis unsafe extern "C" fn $name $( < $( $x ),* > )? ( $( $ident : $ty ),* ) -> napi_status { #[inline(always)] fn inner $( < $( $x ),* > )? ( $( $ident : $ty ),* ) -> napi_status $body diff --git a/cli/napi/uv.rs b/ext/napi/uv.rs similarity index 95% rename from cli/napi/uv.rs rename to ext/napi/uv.rs index d4cb5c0b36..6f728a92b3 100644 --- a/cli/napi/uv.rs +++ b/ext/napi/uv.rs @@ -1,7 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use crate::*; use deno_core::parking_lot::Mutex; -use deno_runtime::deno_napi::*; use std::mem::MaybeUninit; use std::ptr::addr_of_mut; @@ -16,10 +16,10 @@ fn assert_ok(res: c_int) -> c_int { res } -use crate::napi::js_native_api::napi_create_string_utf8; -use crate::napi::node_api::napi_create_async_work; -use crate::napi::node_api::napi_delete_async_work; -use crate::napi::node_api::napi_queue_async_work; +use js_native_api::napi_create_string_utf8; +use node_api::napi_create_async_work; +use node_api::napi_delete_async_work; +use node_api::napi_queue_async_work; use std::ffi::c_int; const UV_MUTEX_SIZE: usize = { diff --git a/tools/napi/generate_symbols_lists.js b/tools/napi/generate_symbols_lists.js index 11cf1c434a..efb0edc043 100755 --- a/tools/napi/generate_symbols_lists.js +++ b/tools/napi/generate_symbols_lists.js @@ -1,7 +1,7 @@ #!/usr/bin/env -S deno run --allow-read --allow-write // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import exports from "../../cli/napi/sym/symbol_exports.json" with { +import exports from "../../ext/napi/sym/symbol_exports.json" with { type: "json", }; @@ -17,7 +17,7 @@ const symbolExportLists = { for await (const [os, def] of Object.entries(symbolExportLists)) { const defUrl = new URL( - `../../cli/napi/generated_symbol_exports_list_${os}.def`, + `../../ext/napi/generated_symbol_exports_list_${os}.def`, import.meta.url, ); await Deno.writeTextFile(defUrl.pathname, def, { create: true }); From ef53ce3ac40e2c545397669e61e4e92423555d94 Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Thu, 24 Oct 2024 11:46:51 +0200 Subject: [PATCH 08/63] fix(node/util): support array formats in `styleText` (#26507) We missed adding support for an array of formats being passed to `util.styleText`. Fixes https://github.com/denoland/deno/issues/26496 --- ext/node/polyfills/internal/util/inspect.mjs | 13 +++++++++++++ tests/unit_node/util_test.ts | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/ext/node/polyfills/internal/util/inspect.mjs b/ext/node/polyfills/internal/util/inspect.mjs index 3a61c387c0..ae797449bf 100644 --- a/ext/node/polyfills/internal/util/inspect.mjs +++ b/ext/node/polyfills/internal/util/inspect.mjs @@ -565,6 +565,19 @@ export function stripVTControlCharacters(str) { export function styleText(format, text) { validateString(text, "text"); + + if (Array.isArray(format)) { + for (let i = 0; i < format.length; i++) { + const item = format[i]; + const formatCodes = inspect.colors[item]; + if (formatCodes == null) { + validateOneOf(item, "format", Object.keys(inspect.colors)); + } + text = `\u001b[${formatCodes[0]}m${text}\u001b[${formatCodes[1]}m`; + } + return text; + } + const formatCodes = inspect.colors[format]; if (formatCodes == null) { validateOneOf(format, "format", Object.keys(inspect.colors)); diff --git a/tests/unit_node/util_test.ts b/tests/unit_node/util_test.ts index a47c231a18..6267018b12 100644 --- a/tests/unit_node/util_test.ts +++ b/tests/unit_node/util_test.ts @@ -353,3 +353,8 @@ Deno.test("[util] styleText()", () => { const redText = util.styleText("red", "error"); assertEquals(redText, "\x1B[31merror\x1B[39m"); }); + +Deno.test("[util] styleText() with array of formats", () => { + const colored = util.styleText(["red", "green"], "error"); + assertEquals(colored, "\x1b[32m\x1b[31merror\x1b[39m\x1b[39m"); +}); From 9c80ddaf931db79ab4cbdf2262fa5b3f4da1e67f Mon Sep 17 00:00:00 2001 From: Nayeem Rahman Date: Thu, 24 Oct 2024 16:03:46 +0100 Subject: [PATCH 09/63] fix(config): schemas for lint rule and tag autocompletion (#26515) --- cli/schemas/config-file.v1.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/schemas/config-file.v1.json b/cli/schemas/config-file.v1.json index 4a6239f0d4..db40e5062b 100644 --- a/cli/schemas/config-file.v1.json +++ b/cli/schemas/config-file.v1.json @@ -291,7 +291,7 @@ "type": "array", "description": "List of tag names that will be run. Empty list disables all tags and will only use rules from `include`.", "items": { - "type": "string" + "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/tags.v1.json" }, "minItems": 0, "uniqueItems": true @@ -300,7 +300,7 @@ "type": "array", "description": "List of rule names that will be excluded from configured tag sets. If the same rule is in `include` it will be run.", "items": { - "type": "string" + "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json" }, "minItems": 0, "uniqueItems": true @@ -309,7 +309,7 @@ "type": "array", "description": "List of rule names that will be run. Even if the same rule is in `exclude` it will be run.", "items": { - "type": "string" + "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json" }, "minItems": 0, "uniqueItems": true From b063cfecfe0479b1de14b8e9e06f1921ce830127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 24 Oct 2024 18:43:24 +0100 Subject: [PATCH 10/63] =?UTF-8?q?Revert=20"fix(runtime):=20send=20ws=20pin?= =?UTF-8?q?g=20frames=20from=20inspector=20server=20(#263=E2=80=A6=20(#265?= =?UTF-8?q?13)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- runtime/inspector_server.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/runtime/inspector_server.rs b/runtime/inspector_server.rs index 33b2ab8727..1f8cd5e714 100644 --- a/runtime/inspector_server.rs +++ b/runtime/inspector_server.rs @@ -23,7 +23,6 @@ use deno_core::InspectorSessionProxy; use deno_core::JsRuntime; use fastwebsockets::Frame; use fastwebsockets::OpCode; -use fastwebsockets::Payload; use fastwebsockets::WebSocket; use hyper::body::Bytes; use hyper_util::rt::TokioIo; @@ -34,7 +33,6 @@ use std::pin::pin; use std::process; use std::rc::Rc; use std::thread; -use std::time::Duration; use tokio::net::TcpListener; use tokio::sync::broadcast; use uuid::Uuid; @@ -395,13 +393,8 @@ async fn pump_websocket_messages( inbound_tx: UnboundedSender, mut outbound_rx: UnboundedReceiver, ) { - let mut ticker = tokio::time::interval(Duration::from_secs(30)); - 'pump: loop { tokio::select! { - _ = ticker.tick() => { - let _ = websocket.write_frame(Frame::new(true, OpCode::Ping, None, Payload::Borrowed(&[]))).await; - } Some(msg) = outbound_rx.next() => { let msg = Frame::text(msg.content.into_bytes().into()); let _ = websocket.write_frame(msg).await; From c71e020668b40666aecfdffb1dbf979abcb41958 Mon Sep 17 00:00:00 2001 From: Leo Kettmeir Date: Thu, 24 Oct 2024 10:45:17 -0700 Subject: [PATCH 11/63] refactor(ext/node): use concrete error types (#26419) --- ext/io/fs.rs | 30 ++--- ext/node/lib.rs | 2 +- ext/node/ops/blocklist.rs | 53 ++++++--- ext/node/ops/fs.rs | 76 +++++++----- ext/node/ops/http2.rs | 73 ++++++++---- ext/node/ops/idna.rs | 47 ++++---- ext/node/ops/ipc.rs | 50 +++++--- ext/node/ops/os/mod.rs | 54 ++++++--- ext/node/ops/os/priority.rs | 30 +++-- ext/node/ops/process.rs | 3 +- ext/node/ops/require.rs | 93 +++++++++------ ext/node/ops/util.rs | 3 +- ext/node/ops/v8.rs | 25 ++-- ext/node/ops/worker_threads.rs | 61 +++++++--- ext/node/ops/zlib/brotli.rs | 77 +++++++++---- ext/node/ops/zlib/mod.rs | 76 ++++++------ ext/node/ops/zlib/mode.rs | 21 +--- runtime/errors.rs | 205 +++++++++++++++++++++++++++++++-- 18 files changed, 674 insertions(+), 305 deletions(-) diff --git a/ext/io/fs.rs b/ext/io/fs.rs index 06fc3da098..8854265209 100644 --- a/ext/io/fs.rs +++ b/ext/io/fs.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::borrow::Cow; +use std::fmt::Formatter; use std::io; use std::rc::Rc; use std::time::SystemTime; @@ -21,6 +22,21 @@ pub enum FsError { NotCapable(&'static str), } +impl std::fmt::Display for FsError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + FsError::Io(err) => std::fmt::Display::fmt(err, f), + FsError::FileBusy => f.write_str("file busy"), + FsError::NotSupported => f.write_str("not supported"), + FsError::NotCapable(err) => { + f.write_str(&format!("requires {err} access")) + } + } + } +} + +impl std::error::Error for FsError {} + impl FsError { pub fn kind(&self) -> io::ErrorKind { match self { @@ -55,20 +71,6 @@ impl From for FsError { } } -impl From for deno_core::error::AnyError { - fn from(err: FsError) -> Self { - match err { - FsError::Io(err) => err.into(), - FsError::FileBusy => deno_core::error::resource_unavailable(), - FsError::NotSupported => deno_core::error::not_supported(), - FsError::NotCapable(err) => deno_core::error::custom_error( - "NotCapable", - format!("permission denied: {err}"), - ), - } - } -} - impl From for FsError { fn from(err: JoinError) -> Self { if err.is_cancelled() { diff --git a/ext/node/lib.rs b/ext/node/lib.rs index 9b22add453..32624f38b7 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -24,7 +24,7 @@ use once_cell::sync::Lazy; extern crate libz_sys as zlib; mod global; -mod ops; +pub mod ops; mod polyfill; pub use deno_package_json::PackageJson; diff --git a/ext/node/ops/blocklist.rs b/ext/node/ops/blocklist.rs index 332cdda8f9..6c64d68eca 100644 --- a/ext/node/ops/blocklist.rs +++ b/ext/node/ops/blocklist.rs @@ -7,9 +7,6 @@ use std::net::Ipv4Addr; use std::net::Ipv6Addr; use std::net::SocketAddr; -use deno_core::anyhow::anyhow; -use deno_core::anyhow::bail; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; @@ -27,13 +24,25 @@ impl deno_core::GarbageCollected for BlockListResource {} #[derive(Serialize)] struct SocketAddressSerialization(String, String); +#[derive(Debug, thiserror::Error)] +pub enum BlocklistError { + #[error("{0}")] + AddrParse(#[from] std::net::AddrParseError), + #[error("{0}")] + IpNetwork(#[from] ipnetwork::IpNetworkError), + #[error("Invalid address")] + InvalidAddress, + #[error("IP version mismatch between start and end addresses")] + IpVersionMismatch, +} + #[op2(fast)] pub fn op_socket_address_parse( state: &mut OpState, #[string] addr: &str, #[smi] port: u16, #[string] family: &str, -) -> Result { +) -> Result { let ip = addr.parse::()?; let parsed: SocketAddr = SocketAddr::new(ip, port); let parsed_ip_str = parsed.ip().to_string(); @@ -52,7 +61,7 @@ pub fn op_socket_address_parse( Ok(false) } } else { - Err(anyhow!("Invalid address")) + Err(BlocklistError::InvalidAddress) } } @@ -60,8 +69,8 @@ pub fn op_socket_address_parse( #[serde] pub fn op_socket_address_get_serialization( state: &mut OpState, -) -> Result { - Ok(state.take::()) +) -> SocketAddressSerialization { + state.take::() } #[op2] @@ -77,7 +86,7 @@ pub fn op_blocklist_new() -> BlockListResource { pub fn op_blocklist_add_address( #[cppgc] wrap: &BlockListResource, #[string] addr: &str, -) -> Result<(), AnyError> { +) -> Result<(), BlocklistError> { wrap.blocklist.borrow_mut().add_address(addr) } @@ -86,7 +95,7 @@ pub fn op_blocklist_add_range( #[cppgc] wrap: &BlockListResource, #[string] start: &str, #[string] end: &str, -) -> Result { +) -> Result { wrap.blocklist.borrow_mut().add_range(start, end) } @@ -95,7 +104,7 @@ pub fn op_blocklist_add_subnet( #[cppgc] wrap: &BlockListResource, #[string] addr: &str, #[smi] prefix: u8, -) -> Result<(), AnyError> { +) -> Result<(), BlocklistError> { wrap.blocklist.borrow_mut().add_subnet(addr, prefix) } @@ -104,7 +113,7 @@ pub fn op_blocklist_check( #[cppgc] wrap: &BlockListResource, #[string] addr: &str, #[string] r#type: &str, -) -> Result { +) -> Result { wrap.blocklist.borrow().check(addr, r#type) } @@ -123,7 +132,7 @@ impl BlockList { &mut self, addr: IpAddr, prefix: Option, - ) -> Result<(), AnyError> { + ) -> Result<(), BlocklistError> { match addr { IpAddr::V4(addr) => { let ipv4_prefix = prefix.unwrap_or(32); @@ -154,7 +163,7 @@ impl BlockList { Ok(()) } - pub fn add_address(&mut self, address: &str) -> Result<(), AnyError> { + pub fn add_address(&mut self, address: &str) -> Result<(), BlocklistError> { let ip: IpAddr = address.parse()?; self.map_addr_add_network(ip, None)?; Ok(()) @@ -164,7 +173,7 @@ impl BlockList { &mut self, start: &str, end: &str, - ) -> Result { + ) -> Result { let start_ip: IpAddr = start.parse()?; let end_ip: IpAddr = end.parse()?; @@ -193,25 +202,33 @@ impl BlockList { self.map_addr_add_network(IpAddr::V6(addr), None)?; } } - _ => bail!("IP version mismatch between start and end addresses"), + _ => return Err(BlocklistError::IpVersionMismatch), } Ok(true) } - pub fn add_subnet(&mut self, addr: &str, prefix: u8) -> Result<(), AnyError> { + pub fn add_subnet( + &mut self, + addr: &str, + prefix: u8, + ) -> Result<(), BlocklistError> { let ip: IpAddr = addr.parse()?; self.map_addr_add_network(ip, Some(prefix))?; Ok(()) } - pub fn check(&self, addr: &str, r#type: &str) -> Result { + pub fn check( + &self, + addr: &str, + r#type: &str, + ) -> Result { let addr: IpAddr = addr.parse()?; let family = r#type.to_lowercase(); if family == "ipv4" && addr.is_ipv4() || family == "ipv6" && addr.is_ipv6() { Ok(self.rules.iter().any(|net| net.contains(addr))) } else { - Err(anyhow!("Invalid address")) + Err(BlocklistError::InvalidAddress) } } } diff --git a/ext/node/ops/fs.rs b/ext/node/ops/fs.rs index 6253f32d05..6645792e74 100644 --- a/ext/node/ops/fs.rs +++ b/ext/node/ops/fs.rs @@ -3,7 +3,6 @@ use std::cell::RefCell; use std::rc::Rc; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_fs::FileSystemRc; @@ -11,11 +10,27 @@ use serde::Serialize; use crate::NodePermissions; +#[derive(Debug, thiserror::Error)] +pub enum FsError { + #[error(transparent)] + Permission(deno_core::error::AnyError), + #[error("{0}")] + Io(#[from] std::io::Error), + #[cfg(windows)] + #[error("Path has no root.")] + PathHasNoRoot, + #[cfg(not(any(unix, windows)))] + #[error("Unsupported platform.")] + UnsupportedPlatform, + #[error(transparent)] + Fs(#[from] deno_io::fs::FsError), +} + #[op2(fast)] pub fn op_node_fs_exists_sync

( state: &mut OpState, #[string] path: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -30,7 +45,7 @@ where pub async fn op_node_fs_exists

( state: Rc>, #[string] path: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -38,7 +53,8 @@ where let mut state = state.borrow_mut(); let path = state .borrow_mut::

() - .check_read_with_api_name(&path, Some("node:fs.exists()"))?; + .check_read_with_api_name(&path, Some("node:fs.exists()")) + .map_err(FsError::Permission)?; (state.borrow::().clone(), path) }; @@ -50,16 +66,18 @@ pub fn op_node_cp_sync

( state: &mut OpState, #[string] path: &str, #[string] new_path: &str, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { let path = state .borrow_mut::

() - .check_read_with_api_name(path, Some("node:fs.cpSync"))?; + .check_read_with_api_name(path, Some("node:fs.cpSync")) + .map_err(FsError::Permission)?; let new_path = state .borrow_mut::

() - .check_write_with_api_name(new_path, Some("node:fs.cpSync"))?; + .check_write_with_api_name(new_path, Some("node:fs.cpSync")) + .map_err(FsError::Permission)?; let fs = state.borrow::(); fs.cp_sync(&path, &new_path)?; @@ -71,7 +89,7 @@ pub async fn op_node_cp

( state: Rc>, #[string] path: String, #[string] new_path: String, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { @@ -79,10 +97,12 @@ where let mut state = state.borrow_mut(); let path = state .borrow_mut::

() - .check_read_with_api_name(&path, Some("node:fs.cpSync"))?; + .check_read_with_api_name(&path, Some("node:fs.cpSync")) + .map_err(FsError::Permission)?; let new_path = state .borrow_mut::

() - .check_write_with_api_name(&new_path, Some("node:fs.cpSync"))?; + .check_write_with_api_name(&new_path, Some("node:fs.cpSync")) + .map_err(FsError::Permission)?; (state.borrow::().clone(), path, new_path) }; @@ -108,7 +128,7 @@ pub fn op_node_statfs

( state: Rc>, #[string] path: String, bigint: bool, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -116,10 +136,12 @@ where let mut state = state.borrow_mut(); let path = state .borrow_mut::

() - .check_read_with_api_name(&path, Some("node:fs.statfs"))?; + .check_read_with_api_name(&path, Some("node:fs.statfs")) + .map_err(FsError::Permission)?; state .borrow_mut::

() - .check_sys("statfs", "node:fs.statfs")?; + .check_sys("statfs", "node:fs.statfs") + .map_err(FsError::Permission)?; path }; #[cfg(unix)] @@ -176,7 +198,6 @@ where } #[cfg(windows)] { - use deno_core::anyhow::anyhow; use std::ffi::OsStr; use std::os::windows::ffi::OsStrExt; use windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceW; @@ -186,10 +207,7 @@ where // call below. #[allow(clippy::disallowed_methods)] let path = path.canonicalize()?; - let root = path - .ancestors() - .last() - .ok_or(anyhow!("Path has no root."))?; + let root = path.ancestors().last().ok_or(FsError::PathHasNoRoot)?; let mut root = OsStr::new(root).encode_wide().collect::>(); root.push(0); let mut sectors_per_cluster = 0; @@ -229,7 +247,7 @@ where { let _ = path; let _ = bigint; - Err(anyhow!("Unsupported platform.")) + Err(FsError::UnsupportedPlatform) } } @@ -241,13 +259,14 @@ pub fn op_node_lutimes_sync

( #[smi] atime_nanos: u32, #[number] mtime_secs: i64, #[smi] mtime_nanos: u32, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { let path = state .borrow_mut::

() - .check_write_with_api_name(path, Some("node:fs.lutimes"))?; + .check_write_with_api_name(path, Some("node:fs.lutimes")) + .map_err(FsError::Permission)?; let fs = state.borrow::(); fs.lutime_sync(&path, atime_secs, atime_nanos, mtime_secs, mtime_nanos)?; @@ -262,7 +281,7 @@ pub async fn op_node_lutimes

( #[smi] atime_nanos: u32, #[number] mtime_secs: i64, #[smi] mtime_nanos: u32, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { @@ -270,7 +289,8 @@ where let mut state = state.borrow_mut(); let path = state .borrow_mut::

() - .check_write_with_api_name(&path, Some("node:fs.lutimesSync"))?; + .check_write_with_api_name(&path, Some("node:fs.lutimesSync")) + .map_err(FsError::Permission)?; (state.borrow::().clone(), path) }; @@ -286,13 +306,14 @@ pub fn op_node_lchown_sync

( #[string] path: String, uid: Option, gid: Option, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { let path = state .borrow_mut::

() - .check_write_with_api_name(&path, Some("node:fs.lchownSync"))?; + .check_write_with_api_name(&path, Some("node:fs.lchownSync")) + .map_err(FsError::Permission)?; let fs = state.borrow::(); fs.lchown_sync(&path, uid, gid)?; Ok(()) @@ -304,7 +325,7 @@ pub async fn op_node_lchown

( #[string] path: String, uid: Option, gid: Option, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { @@ -312,7 +333,8 @@ where let mut state = state.borrow_mut(); let path = state .borrow_mut::

() - .check_write_with_api_name(&path, Some("node:fs.lchown"))?; + .check_write_with_api_name(&path, Some("node:fs.lchown")) + .map_err(FsError::Permission)?; (state.borrow::().clone(), path) }; fs.lchown_async(path, uid, gid).await?; diff --git a/ext/node/ops/http2.rs b/ext/node/ops/http2.rs index 705a8ecdcb..53dada9f41 100644 --- a/ext/node/ops/http2.rs +++ b/ext/node/ops/http2.rs @@ -7,7 +7,6 @@ use std::rc::Rc; use std::task::Poll; use bytes::Bytes; -use deno_core::error::AnyError; use deno_core::futures::future::poll_fn; use deno_core::op2; use deno_core::serde::Serialize; @@ -110,17 +109,28 @@ impl Resource for Http2ServerSendResponse { } } +#[derive(Debug, thiserror::Error)] +pub enum Http2Error { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + UrlParse(#[from] url::ParseError), + #[error(transparent)] + H2(#[from] h2::Error), +} + #[op2(async)] #[serde] pub async fn op_http2_connect( state: Rc>, #[smi] rid: ResourceId, #[string] url: String, -) -> Result<(ResourceId, ResourceId), AnyError> { +) -> Result<(ResourceId, ResourceId), Http2Error> { // No permission check necessary because we're using an existing connection let network_stream = { let mut state = state.borrow_mut(); - take_network_stream_resource(&mut state.resource_table, rid)? + take_network_stream_resource(&mut state.resource_table, rid) + .map_err(Http2Error::Resource)? }; let url = Url::parse(&url)?; @@ -144,9 +154,10 @@ pub async fn op_http2_connect( pub async fn op_http2_listen( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let stream = - take_network_stream_resource(&mut state.borrow_mut().resource_table, rid)?; + take_network_stream_resource(&mut state.borrow_mut().resource_table, rid) + .map_err(Http2Error::Resource)?; let conn = h2::server::Builder::new().handshake(stream).await?; Ok( @@ -166,12 +177,13 @@ pub async fn op_http2_accept( #[smi] rid: ResourceId, ) -> Result< Option<(Vec<(ByteString, ByteString)>, ResourceId, ResourceId)>, - AnyError, + Http2Error, > { let resource = state .borrow() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(Http2Error::Resource)?; let mut conn = RcRef::map(&resource, |r| &r.conn).borrow_mut().await; if let Some(res) = conn.accept().await { let (req, resp) = res?; @@ -233,11 +245,12 @@ pub async fn op_http2_send_response( #[smi] rid: ResourceId, #[smi] status: u16, #[serde] headers: Vec<(ByteString, ByteString)>, -) -> Result<(ResourceId, u32), AnyError> { +) -> Result<(ResourceId, u32), Http2Error> { let resource = state .borrow() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(Http2Error::Resource)?; let mut send_response = RcRef::map(resource, |r| &r.send_response) .borrow_mut() .await; @@ -262,8 +275,12 @@ pub async fn op_http2_send_response( pub async fn op_http2_poll_client_connection( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { - let resource = state.borrow().resource_table.get::(rid)?; +) -> Result<(), Http2Error> { + let resource = state + .borrow() + .resource_table + .get::(rid) + .map_err(Http2Error::Resource)?; let cancel_handle = RcRef::map(resource.clone(), |this| &this.cancel_handle); let mut conn = RcRef::map(resource, |this| &this.conn).borrow_mut().await; @@ -289,11 +306,12 @@ pub async fn op_http2_client_request( // 4 strings of keys? #[serde] mut pseudo_headers: HashMap, #[serde] headers: Vec<(ByteString, ByteString)>, -) -> Result<(ResourceId, u32), AnyError> { +) -> Result<(ResourceId, u32), Http2Error> { let resource = state .borrow() .resource_table - .get::(client_rid)?; + .get::(client_rid) + .map_err(Http2Error::Resource)?; let url = resource.url.clone(); @@ -326,7 +344,10 @@ pub async fn op_http2_client_request( let resource = { let state = state.borrow(); - state.resource_table.get::(client_rid)? + state + .resource_table + .get::(client_rid) + .map_err(Http2Error::Resource)? }; let mut client = RcRef::map(&resource, |r| &r.client).borrow_mut().await; poll_fn(|cx| client.poll_ready(cx)).await?; @@ -345,11 +366,12 @@ pub async fn op_http2_client_send_data( #[smi] stream_rid: ResourceId, #[buffer] data: JsBuffer, end_of_stream: bool, -) -> Result<(), AnyError> { +) -> Result<(), Http2Error> { let resource = state .borrow() .resource_table - .get::(stream_rid)?; + .get::(stream_rid) + .map_err(Http2Error::Resource)?; let mut stream = RcRef::map(&resource, |r| &r.stream).borrow_mut().await; stream.send_data(data.to_vec().into(), end_of_stream)?; @@ -361,7 +383,7 @@ pub async fn op_http2_client_reset_stream( state: Rc>, #[smi] stream_rid: ResourceId, #[smi] code: u32, -) -> Result<(), AnyError> { +) -> Result<(), deno_core::error::AnyError> { let resource = state .borrow() .resource_table @@ -376,11 +398,12 @@ pub async fn op_http2_client_send_trailers( state: Rc>, #[smi] stream_rid: ResourceId, #[serde] trailers: Vec<(ByteString, ByteString)>, -) -> Result<(), AnyError> { +) -> Result<(), Http2Error> { let resource = state .borrow() .resource_table - .get::(stream_rid)?; + .get::(stream_rid) + .map_err(Http2Error::Resource)?; let mut stream = RcRef::map(&resource, |r| &r.stream).borrow_mut().await; let mut trailers_map = http::HeaderMap::new(); @@ -408,11 +431,12 @@ pub struct Http2ClientResponse { pub async fn op_http2_client_get_response( state: Rc>, #[smi] stream_rid: ResourceId, -) -> Result<(Http2ClientResponse, bool), AnyError> { +) -> Result<(Http2ClientResponse, bool), Http2Error> { let resource = state .borrow() .resource_table - .get::(stream_rid)?; + .get::(stream_rid) + .map_err(Http2Error::Resource)?; let mut response_future = RcRef::map(&resource, |r| &r.response).borrow_mut().await; @@ -478,11 +502,12 @@ fn poll_data_or_trailers( pub async fn op_http2_client_get_response_body_chunk( state: Rc>, #[smi] body_rid: ResourceId, -) -> Result<(Option>, bool, bool), AnyError> { +) -> Result<(Option>, bool, bool), Http2Error> { let resource = state .borrow() .resource_table - .get::(body_rid)?; + .get::(body_rid) + .map_err(Http2Error::Resource)?; let mut body = RcRef::map(&resource, |r| &r.body).borrow_mut().await; loop { @@ -525,7 +550,7 @@ pub async fn op_http2_client_get_response_body_chunk( pub async fn op_http2_client_get_response_trailers( state: Rc>, #[smi] body_rid: ResourceId, -) -> Result>, AnyError> { +) -> Result>, deno_core::error::AnyError> { let resource = state .borrow() .resource_table diff --git a/ext/node/ops/idna.rs b/ext/node/ops/idna.rs index 9c9450c70f..a3d85e77c2 100644 --- a/ext/node/ops/idna.rs +++ b/ext/node/ops/idna.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::anyhow::Error; -use deno_core::error::range_error; use deno_core::op2; use std::borrow::Cow; @@ -11,19 +9,21 @@ use std::borrow::Cow; const PUNY_PREFIX: &str = "xn--"; -fn invalid_input_err() -> Error { - range_error("Invalid input") -} - -fn not_basic_err() -> Error { - range_error("Illegal input >= 0x80 (not a basic code point)") +#[derive(Debug, thiserror::Error)] +pub enum IdnaError { + #[error("Invalid input")] + InvalidInput, + #[error("Input would take more than 63 characters to encode")] + InputTooLong, + #[error("Illegal input >= 0x80 (not a basic code point)")] + IllegalInput, } /// map a domain by mapping each label with the given function -fn map_domain( +fn map_domain( domain: &str, - f: impl Fn(&str) -> Result, E>, -) -> Result { + f: impl Fn(&str) -> Result, IdnaError>, +) -> Result { let mut result = String::with_capacity(domain.len()); let mut domain = domain; @@ -48,7 +48,7 @@ fn map_domain( /// Maps a unicode domain to ascii by punycode encoding each label /// /// Note this is not IDNA2003 or IDNA2008 compliant, rather it matches node.js's punycode implementation -fn to_ascii(input: &str) -> Result { +fn to_ascii(input: &str) -> Result { if input.is_ascii() { return Ok(input.into()); } @@ -61,9 +61,7 @@ fn to_ascii(input: &str) -> Result { } else { idna::punycode::encode_str(label) .map(|encoded| [PUNY_PREFIX, &encoded].join("").into()) // add the prefix - .ok_or_else(|| { - Error::msg("Input would take more than 63 characters to encode") // only error possible per the docs - }) + .ok_or(IdnaError::InputTooLong) // only error possible per the docs } })?; @@ -74,13 +72,13 @@ fn to_ascii(input: &str) -> Result { /// Maps an ascii domain to unicode by punycode decoding each label /// /// Note this is not IDNA2003 or IDNA2008 compliant, rather it matches node.js's punycode implementation -fn to_unicode(input: &str) -> Result { +fn to_unicode(input: &str) -> Result { map_domain(input, |s| { if let Some(puny) = s.strip_prefix(PUNY_PREFIX) { // it's a punycode encoded label Ok( idna::punycode::decode_to_string(&puny.to_lowercase()) - .ok_or_else(invalid_input_err)? + .ok_or(IdnaError::InvalidInput)? .into(), ) } else { @@ -95,7 +93,7 @@ fn to_unicode(input: &str) -> Result { #[string] pub fn op_node_idna_punycode_to_ascii( #[string] domain: String, -) -> Result { +) -> Result { to_ascii(&domain) } @@ -105,7 +103,7 @@ pub fn op_node_idna_punycode_to_ascii( #[string] pub fn op_node_idna_punycode_to_unicode( #[string] domain: String, -) -> Result { +) -> Result { to_unicode(&domain) } @@ -115,8 +113,8 @@ pub fn op_node_idna_punycode_to_unicode( #[string] pub fn op_node_idna_domain_to_ascii( #[string] domain: String, -) -> Result { - idna::domain_to_ascii(&domain).map_err(|e| e.into()) +) -> Result { + idna::domain_to_ascii(&domain) } /// Converts a domain to Unicode as per the IDNA spec @@ -131,7 +129,7 @@ pub fn op_node_idna_domain_to_unicode(#[string] domain: String) -> String { #[string] pub fn op_node_idna_punycode_decode( #[string] domain: String, -) -> Result { +) -> Result { if domain.is_empty() { return Ok(domain); } @@ -147,11 +145,10 @@ pub fn op_node_idna_punycode_decode( .unwrap_or(domain.len() - 1); if !domain[..last_dash].is_ascii() { - return Err(not_basic_err()); + return Err(IdnaError::IllegalInput); } - idna::punycode::decode_to_string(&domain) - .ok_or_else(|| deno_core::error::range_error("Invalid input")) + idna::punycode::decode_to_string(&domain).ok_or(IdnaError::InvalidInput) } #[op2] diff --git a/ext/node/ops/ipc.rs b/ext/node/ops/ipc.rs index c5e0f875d5..672cf0d707 100644 --- a/ext/node/ops/ipc.rs +++ b/ext/node/ops/ipc.rs @@ -17,8 +17,6 @@ mod impl_ { use std::task::Context; use std::task::Poll; - use deno_core::error::bad_resource_id; - use deno_core::error::AnyError; use deno_core::op2; use deno_core::serde; use deno_core::serde::Serializer; @@ -167,7 +165,7 @@ mod impl_ { #[smi] pub fn op_node_child_ipc_pipe( state: &mut OpState, - ) -> Result, AnyError> { + ) -> Result, io::Error> { let fd = match state.try_borrow_mut::() { Some(child_pipe_fd) => child_pipe_fd.0, None => return Ok(None), @@ -180,6 +178,18 @@ mod impl_ { )) } + #[derive(Debug, thiserror::Error)] + pub enum IpcError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + IpcJsonStream(#[from] IpcJsonStreamError), + #[error(transparent)] + Canceled(#[from] deno_core::Canceled), + #[error("failed to serialize json value: {0}")] + SerdeJson(serde_json::Error), + } + #[op2(async)] pub fn op_node_ipc_write<'a>( scope: &mut v8::HandleScope<'a>, @@ -192,27 +202,23 @@ mod impl_ { // ideally we would just return `Result<(impl Future, bool), ..>`, but that's not // supported by `op2` currently. queue_ok: v8::Local<'a, v8::Array>, - ) -> Result>, AnyError> { + ) -> Result>, IpcError> { let mut serialized = Vec::with_capacity(64); let mut ser = serde_json::Serializer::new(&mut serialized); - serialize_v8_value(scope, value, &mut ser).map_err(|e| { - deno_core::error::type_error(format!( - "failed to serialize json value: {e}" - )) - })?; + serialize_v8_value(scope, value, &mut ser).map_err(IpcError::SerdeJson)?; serialized.push(b'\n'); let stream = state .borrow() .resource_table .get::(rid) - .map_err(|_| bad_resource_id())?; + .map_err(IpcError::Resource)?; let old = stream .queued_bytes .fetch_add(serialized.len(), std::sync::atomic::Ordering::Relaxed); if old + serialized.len() > 2 * INITIAL_CAPACITY { // sending messages too fast - let v = false.to_v8(scope)?; + let v = false.to_v8(scope).unwrap(); // Infallible queue_ok.set_index(scope, 0, v); } Ok(async move { @@ -246,12 +252,12 @@ mod impl_ { pub async fn op_node_ipc_read( state: Rc>, #[smi] rid: ResourceId, - ) -> Result { + ) -> Result { let stream = state .borrow() .resource_table .get::(rid) - .map_err(|_| bad_resource_id())?; + .map_err(IpcError::Resource)?; let cancel = stream.cancel.clone(); let mut stream = RcRef::map(stream, |r| &r.read_half).borrow_mut().await; @@ -407,7 +413,7 @@ mod impl_ { async fn write_msg_bytes( self: Rc, msg: &[u8], - ) -> Result<(), AnyError> { + ) -> Result<(), io::Error> { let mut write_half = RcRef::map(self, |r| &r.write_half).borrow_mut().await; write_half.write_all(msg).await?; @@ -462,6 +468,14 @@ mod impl_ { } } + #[derive(Debug, thiserror::Error)] + pub enum IpcJsonStreamError { + #[error("{0}")] + Io(#[source] std::io::Error), + #[error("{0}")] + SimdJson(#[source] simd_json::Error), + } + // JSON serialization stream over IPC pipe. // // `\n` is used as a delimiter between messages. @@ -482,7 +496,7 @@ mod impl_ { async fn read_msg( &mut self, - ) -> Result, AnyError> { + ) -> Result, IpcJsonStreamError> { let mut json = None; let nread = read_msg_inner( &mut self.pipe, @@ -490,7 +504,8 @@ mod impl_ { &mut json, &mut self.read_buffer, ) - .await?; + .await + .map_err(IpcJsonStreamError::Io)?; if nread == 0 { // EOF. return Ok(None); @@ -500,7 +515,8 @@ mod impl_ { Some(v) => v, None => { // Took more than a single read and some buffering. - simd_json::from_slice(&mut self.buffer[..nread])? + simd_json::from_slice(&mut self.buffer[..nread]) + .map_err(IpcJsonStreamError::SimdJson)? } }; diff --git a/ext/node/ops/os/mod.rs b/ext/node/ops/os/mod.rs index ca91895f22..b4c9eaa8ca 100644 --- a/ext/node/ops/os/mod.rs +++ b/ext/node/ops/os/mod.rs @@ -1,28 +1,38 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::NodePermissions; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; mod cpus; -mod priority; +pub mod priority; + +#[derive(Debug, thiserror::Error)] +pub enum OsError { + #[error(transparent)] + Priority(priority::PriorityError), + #[error(transparent)] + Permission(deno_core::error::AnyError), + #[error("Failed to get cpu info")] + FailedToGetCpuInfo, +} #[op2(fast)] pub fn op_node_os_get_priority

( state: &mut OpState, pid: u32, -) -> Result +) -> Result where P: NodePermissions + 'static, { { let permissions = state.borrow_mut::

(); - permissions.check_sys("getPriority", "node:os.getPriority()")?; + permissions + .check_sys("getPriority", "node:os.getPriority()") + .map_err(OsError::Permission)?; } - priority::get_priority(pid) + priority::get_priority(pid).map_err(OsError::Priority) } #[op2(fast)] @@ -30,21 +40,25 @@ pub fn op_node_os_set_priority

( state: &mut OpState, pid: u32, priority: i32, -) -> Result<(), AnyError> +) -> Result<(), OsError> where P: NodePermissions + 'static, { { let permissions = state.borrow_mut::

(); - permissions.check_sys("setPriority", "node:os.setPriority()")?; + permissions + .check_sys("setPriority", "node:os.setPriority()") + .map_err(OsError::Permission)?; } - priority::set_priority(pid, priority) + priority::set_priority(pid, priority).map_err(OsError::Priority) } #[op2] #[string] -pub fn op_node_os_username

(state: &mut OpState) -> Result +pub fn op_node_os_username

( + state: &mut OpState, +) -> Result where P: NodePermissions + 'static, { @@ -57,7 +71,9 @@ where } #[op2(fast)] -pub fn op_geteuid

(state: &mut OpState) -> Result +pub fn op_geteuid

( + state: &mut OpState, +) -> Result where P: NodePermissions + 'static, { @@ -76,7 +92,9 @@ where } #[op2(fast)] -pub fn op_getegid

(state: &mut OpState) -> Result +pub fn op_getegid

( + state: &mut OpState, +) -> Result where P: NodePermissions + 'static, { @@ -96,21 +114,25 @@ where #[op2] #[serde] -pub fn op_cpus

(state: &mut OpState) -> Result, AnyError> +pub fn op_cpus

(state: &mut OpState) -> Result, OsError> where P: NodePermissions + 'static, { { let permissions = state.borrow_mut::

(); - permissions.check_sys("cpus", "node:os.cpus()")?; + permissions + .check_sys("cpus", "node:os.cpus()") + .map_err(OsError::Permission)?; } - cpus::cpu_info().ok_or_else(|| type_error("Failed to get cpu info")) + cpus::cpu_info().ok_or(OsError::FailedToGetCpuInfo) } #[op2] #[string] -pub fn op_homedir

(state: &mut OpState) -> Result, AnyError> +pub fn op_homedir

( + state: &mut OpState, +) -> Result, deno_core::error::AnyError> where P: NodePermissions + 'static, { diff --git a/ext/node/ops/os/priority.rs b/ext/node/ops/os/priority.rs index 043928e2a6..9a1ebcca70 100644 --- a/ext/node/ops/os/priority.rs +++ b/ext/node/ops/os/priority.rs @@ -1,12 +1,18 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; - pub use impl_::*; +#[derive(Debug, thiserror::Error)] +pub enum PriorityError { + #[error("{0}")] + Io(#[from] std::io::Error), + #[cfg(windows)] + #[error("Invalid priority")] + InvalidPriority, +} + #[cfg(unix)] mod impl_ { - use super::*; use errno::errno; use errno::set_errno; use errno::Errno; @@ -16,7 +22,7 @@ mod impl_ { const PRIORITY_HIGH: i32 = -14; // Ref: https://github.com/libuv/libuv/blob/55376b044b74db40772e8a6e24d67a8673998e02/src/unix/core.c#L1533-L1547 - pub fn get_priority(pid: u32) -> Result { + pub fn get_priority(pid: u32) -> Result { set_errno(Errno(0)); match ( // SAFETY: libc::getpriority is unsafe @@ -29,7 +35,10 @@ mod impl_ { } } - pub fn set_priority(pid: u32, priority: i32) -> Result<(), AnyError> { + pub fn set_priority( + pid: u32, + priority: i32, + ) -> Result<(), super::PriorityError> { // SAFETY: libc::setpriority is unsafe match unsafe { libc::setpriority(PRIO_PROCESS, pid as id_t, priority) } { -1 => Err(std::io::Error::last_os_error().into()), @@ -40,8 +49,6 @@ mod impl_ { #[cfg(windows)] mod impl_ { - use super::*; - use deno_core::error::type_error; use winapi::shared::minwindef::DWORD; use winapi::shared::minwindef::FALSE; use winapi::shared::ntdef::NULL; @@ -67,7 +74,7 @@ mod impl_ { const PRIORITY_HIGHEST: i32 = -20; // Ported from: https://github.com/libuv/libuv/blob/a877ca2435134ef86315326ef4ef0c16bdbabf17/src/win/util.c#L1649-L1685 - pub fn get_priority(pid: u32) -> Result { + pub fn get_priority(pid: u32) -> Result { // SAFETY: Windows API calls unsafe { let handle = if pid == 0 { @@ -95,7 +102,10 @@ mod impl_ { } // Ported from: https://github.com/libuv/libuv/blob/a877ca2435134ef86315326ef4ef0c16bdbabf17/src/win/util.c#L1688-L1719 - pub fn set_priority(pid: u32, priority: i32) -> Result<(), AnyError> { + pub fn set_priority( + pid: u32, + priority: i32, + ) -> Result<(), super::PriorityError> { // SAFETY: Windows API calls unsafe { let handle = if pid == 0 { @@ -109,7 +119,7 @@ mod impl_ { #[allow(clippy::manual_range_contains)] let priority_class = if priority < PRIORITY_HIGHEST || priority > PRIORITY_LOW { - return Err(type_error("Invalid priority")); + return Err(super::PriorityError::InvalidPriority); } else if priority < PRIORITY_HIGH { REALTIME_PRIORITY_CLASS } else if priority < PRIORITY_ABOVE_NORMAL { diff --git a/ext/node/ops/process.rs b/ext/node/ops/process.rs index 0992c46c62..282567226e 100644 --- a/ext/node/ops/process.rs +++ b/ext/node/ops/process.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_permissions::PermissionsContainer; @@ -51,7 +50,7 @@ pub fn op_node_process_kill( state: &mut OpState, #[smi] pid: i32, #[smi] sig: i32, -) -> Result { +) -> Result { state .borrow_mut::() .check_run_all("process.kill")?; diff --git a/ext/node/ops/require.rs b/ext/node/ops/require.rs index 7d85ee8532..f4607f4e8a 100644 --- a/ext/node/ops/require.rs +++ b/ext/node/ops/require.rs @@ -1,8 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::anyhow::Context; -use deno_core::error::generic_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::url::Url; use deno_core::v8; @@ -30,7 +27,7 @@ use crate::NpmResolverRc; fn ensure_read_permission<'a, P>( state: &mut OpState, file_path: &'a Path, -) -> Result, AnyError> +) -> Result, deno_core::error::AnyError> where P: NodePermissions + 'static, { @@ -39,6 +36,32 @@ where resolver.ensure_read_permission(permissions, file_path) } +#[derive(Debug, thiserror::Error)] +pub enum RequireError { + #[error(transparent)] + UrlParse(#[from] url::ParseError), + #[error(transparent)] + Permission(deno_core::error::AnyError), + #[error(transparent)] + PackageExportsResolve( + #[from] node_resolver::errors::PackageExportsResolveError, + ), + #[error(transparent)] + PackageJsonLoad(#[from] node_resolver::errors::PackageJsonLoadError), + #[error(transparent)] + ClosestPkgJson(#[from] node_resolver::errors::ClosestPkgJsonError), + #[error(transparent)] + PackageImportsResolve( + #[from] node_resolver::errors::PackageImportsResolveError, + ), + #[error("failed to convert '{0}' to file path")] + FilePathConversion(Url), + #[error(transparent)] + Fs(#[from] deno_io::fs::FsError), + #[error("Unable to get CWD: {0}")] + UnableToGetCwd(deno_io::fs::FsError), +} + #[op2] #[serde] pub fn op_require_init_paths() -> Vec { @@ -95,7 +118,7 @@ pub fn op_require_init_paths() -> Vec { pub fn op_require_node_module_paths

( state: &mut OpState, #[string] from: String, -) -> Result, AnyError> +) -> Result, RequireError> where P: NodePermissions + 'static, { @@ -104,12 +127,12 @@ where let from = if from.starts_with("file:///") { url_to_file_path(&Url::parse(&from)?)? } else { - let current_dir = - &(fs.cwd().map_err(AnyError::from)).context("Unable to get CWD")?; - deno_path_util::normalize_path(current_dir.join(from)) + let current_dir = &fs.cwd().map_err(RequireError::UnableToGetCwd)?; + normalize_path(current_dir.join(from)) }; - let from = ensure_read_permission::

(state, &from)?; + let from = ensure_read_permission::

(state, &from) + .map_err(RequireError::Permission)?; if cfg!(windows) { // return root node_modules when path is 'D:\\'. @@ -264,7 +287,7 @@ pub fn op_require_path_is_absolute(#[string] p: String) -> bool { pub fn op_require_stat

( state: &mut OpState, #[string] path: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -287,12 +310,13 @@ where pub fn op_require_real_path

( state: &mut OpState, #[string] request: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { let path = PathBuf::from(request); - let path = ensure_read_permission::

(state, &path)?; + let path = ensure_read_permission::

(state, &path) + .map_err(RequireError::Permission)?; let fs = state.borrow::(); let canonicalized_path = deno_path_util::strip_unc_prefix(fs.realpath_sync(&path)?); @@ -319,12 +343,14 @@ pub fn op_require_path_resolve(#[serde] parts: Vec) -> String { #[string] pub fn op_require_path_dirname( #[string] request: String, -) -> Result { +) -> Result { let p = PathBuf::from(request); if let Some(parent) = p.parent() { Ok(parent.to_string_lossy().into_owned()) } else { - Err(generic_error("Path doesn't have a parent")) + Err(deno_core::error::generic_error( + "Path doesn't have a parent", + )) } } @@ -332,12 +358,14 @@ pub fn op_require_path_dirname( #[string] pub fn op_require_path_basename( #[string] request: String, -) -> Result { +) -> Result { let p = PathBuf::from(request); if let Some(path) = p.file_name() { Ok(path.to_string_lossy().into_owned()) } else { - Err(generic_error("Path doesn't have a file name")) + Err(deno_core::error::generic_error( + "Path doesn't have a file name", + )) } } @@ -348,7 +376,7 @@ pub fn op_require_try_self_parent_path

( has_parent: bool, #[string] maybe_parent_filename: Option, #[string] maybe_parent_id: Option, -) -> Result, AnyError> +) -> Result, deno_core::error::AnyError> where P: NodePermissions + 'static, { @@ -378,7 +406,7 @@ pub fn op_require_try_self

( state: &mut OpState, #[string] parent_path: Option, #[string] request: String, -) -> Result, AnyError> +) -> Result, RequireError> where P: NodePermissions + 'static, { @@ -440,12 +468,13 @@ where pub fn op_require_read_file

( state: &mut OpState, #[string] file_path: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { let file_path = PathBuf::from(file_path); - let file_path = ensure_read_permission::

(state, &file_path)?; + let file_path = ensure_read_permission::

(state, &file_path) + .map_err(RequireError::Permission)?; let fs = state.borrow::(); Ok(fs.read_text_file_lossy_sync(&file_path, None)?) } @@ -472,7 +501,7 @@ pub fn op_require_resolve_exports

( #[string] name: String, #[string] expansion: String, #[string] parent_path: String, -) -> Result, AnyError> +) -> Result, RequireError> where P: NodePermissions + 'static, { @@ -525,16 +554,14 @@ where pub fn op_require_read_closest_package_json

( state: &mut OpState, #[string] filename: String, -) -> Result, AnyError> +) -> Result, node_resolver::errors::ClosestPkgJsonError> where P: NodePermissions + 'static, { let filename = PathBuf::from(filename); // permissions: allow reading the closest package.json files let node_resolver = state.borrow::().clone(); - node_resolver - .get_closest_package_json_from_path(&filename) - .map_err(AnyError::from) + node_resolver.get_closest_package_json_from_path(&filename) } #[op2] @@ -564,12 +591,13 @@ pub fn op_require_package_imports_resolve

( state: &mut OpState, #[string] referrer_filename: String, #[string] request: String, -) -> Result, AnyError> +) -> Result, RequireError> where P: NodePermissions + 'static, { let referrer_path = PathBuf::from(&referrer_filename); - let referrer_path = ensure_read_permission::

(state, &referrer_path)?; + let referrer_path = ensure_read_permission::

(state, &referrer_path) + .map_err(RequireError::Permission)?; let node_resolver = state.borrow::(); let Some(pkg) = node_resolver.get_closest_package_json_from_path(&referrer_path)? @@ -578,8 +606,7 @@ where }; if pkg.imports.is_some() { - let referrer_url = - deno_core::url::Url::from_file_path(&referrer_filename).unwrap(); + let referrer_url = Url::from_file_path(&referrer_filename).unwrap(); let url = node_resolver.package_imports_resolve( &request, Some(&referrer_url), @@ -604,17 +631,15 @@ pub fn op_require_break_on_next_statement(state: Rc>) { inspector.wait_for_session_and_break_on_next_statement() } -fn url_to_file_path_string(url: &Url) -> Result { +fn url_to_file_path_string(url: &Url) -> Result { let file_path = url_to_file_path(url)?; Ok(file_path.to_string_lossy().into_owned()) } -fn url_to_file_path(url: &Url) -> Result { +fn url_to_file_path(url: &Url) -> Result { match url.to_file_path() { Ok(file_path) => Ok(file_path), - Err(()) => { - deno_core::anyhow::bail!("failed to convert '{}' to file path", url) - } + Err(()) => Err(RequireError::FilePathConversion(url.clone())), } } diff --git a/ext/node/ops/util.rs b/ext/node/ops/util.rs index 533d51c92c..1c177ac043 100644 --- a/ext/node/ops/util.rs +++ b/ext/node/ops/util.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::ResourceHandle; @@ -22,7 +21,7 @@ enum HandleType { pub fn op_node_guess_handle_type( state: &mut OpState, rid: u32, -) -> Result { +) -> Result { let handle = state.resource_table.get_handle(rid)?; let handle_type = match handle { diff --git a/ext/node/ops/v8.rs b/ext/node/ops/v8.rs index 8813d2e18e..61f67f11f7 100644 --- a/ext/node/ops/v8.rs +++ b/ext/node/ops/v8.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::generic_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; + use deno_core::op2; use deno_core::v8; use deno_core::FastString; @@ -206,10 +204,9 @@ pub fn op_v8_write_value( scope: &mut v8::HandleScope, #[cppgc] ser: &Serializer, value: v8::Local, -) -> Result<(), AnyError> { +) { let context = scope.get_current_context(); ser.inner.write_value(context, value); - Ok(()) } struct DeserBuffer { @@ -271,11 +268,13 @@ pub fn op_v8_new_deserializer( scope: &mut v8::HandleScope, obj: v8::Local, buffer: v8::Local, -) -> Result, AnyError> { +) -> Result, deno_core::error::AnyError> { let offset = buffer.byte_offset(); let len = buffer.byte_length(); let backing_store = buffer.get_backing_store().ok_or_else(|| { - generic_error("deserialization buffer has no backing store") + deno_core::error::generic_error( + "deserialization buffer has no backing store", + ) })?; let (buf_slice, buf_ptr) = if let Some(data) = backing_store.data() { // SAFETY: the offset is valid for the underlying buffer because we're getting it directly from v8 @@ -317,10 +316,10 @@ pub fn op_v8_transfer_array_buffer_de( #[op2(fast)] pub fn op_v8_read_double( #[cppgc] deser: &Deserializer, -) -> Result { +) -> Result { let mut double = 0f64; if !deser.inner.read_double(&mut double) { - return Err(type_error("ReadDouble() failed")); + return Err(deno_core::error::type_error("ReadDouble() failed")); } Ok(double) } @@ -355,10 +354,10 @@ pub fn op_v8_read_raw_bytes( #[op2(fast)] pub fn op_v8_read_uint32( #[cppgc] deser: &Deserializer, -) -> Result { +) -> Result { let mut value = 0; if !deser.inner.read_uint32(&mut value) { - return Err(type_error("ReadUint32() failed")); + return Err(deno_core::error::type_error("ReadUint32() failed")); } Ok(value) @@ -368,10 +367,10 @@ pub fn op_v8_read_uint32( #[serde] pub fn op_v8_read_uint64( #[cppgc] deser: &Deserializer, -) -> Result<(u32, u32), AnyError> { +) -> Result<(u32, u32), deno_core::error::AnyError> { let mut val = 0; if !deser.inner.read_uint64(&mut val) { - return Err(type_error("ReadUint64() failed")); + return Err(deno_core::error::type_error("ReadUint64() failed")); } Ok(((val >> 32) as u32, val as u32)) diff --git a/ext/node/ops/worker_threads.rs b/ext/node/ops/worker_threads.rs index 5f139c5dc8..e33cf91574 100644 --- a/ext/node/ops/worker_threads.rs +++ b/ext/node/ops/worker_threads.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::generic_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::url::Url; use deno_core::OpState; @@ -19,7 +17,7 @@ use crate::NodeResolverRc; fn ensure_read_permission<'a, P>( state: &mut OpState, file_path: &'a Path, -) -> Result, AnyError> +) -> Result, deno_core::error::AnyError> where P: NodePermissions + 'static, { @@ -28,12 +26,36 @@ where resolver.ensure_read_permission(permissions, file_path) } +#[derive(Debug, thiserror::Error)] +pub enum WorkerThreadsFilenameError { + #[error(transparent)] + Permission(deno_core::error::AnyError), + #[error("{0}")] + UrlParse(#[from] url::ParseError), + #[error("Relative path entries must start with '.' or '..'")] + InvalidRelativeUrl, + #[error("URL from Path-String")] + UrlFromPathString, + #[error("URL to Path-String")] + UrlToPathString, + #[error("URL to Path")] + UrlToPath, + #[error("File not found [{0:?}]")] + FileNotFound(PathBuf), + #[error("Neither ESM nor CJS")] + NeitherEsmNorCjs, + #[error("{0}")] + UrlToNodeResolution(node_resolver::errors::UrlToNodeResolutionError), + #[error(transparent)] + Fs(#[from] deno_io::fs::FsError), +} + #[op2] #[string] pub fn op_worker_threads_filename

( state: &mut OpState, #[string] specifier: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -45,40 +67,47 @@ where } else { let path = PathBuf::from(&specifier); if path.is_relative() && !specifier.starts_with('.') { - return Err(generic_error( - "Relative path entries must start with '.' or '..'", - )); + return Err(WorkerThreadsFilenameError::InvalidRelativeUrl); } - let path = ensure_read_permission::

(state, &path)?; + let path = ensure_read_permission::

(state, &path) + .map_err(WorkerThreadsFilenameError::Permission)?; let fs = state.borrow::(); let canonicalized_path = deno_path_util::strip_unc_prefix(fs.realpath_sync(&path)?); Url::from_file_path(canonicalized_path) - .map_err(|e| generic_error(format!("URL from Path-String: {:#?}", e)))? + .map_err(|_| WorkerThreadsFilenameError::UrlFromPathString)? }; let url_path = url .to_file_path() - .map_err(|e| generic_error(format!("URL to Path-String: {:#?}", e)))?; - let url_path = ensure_read_permission::

(state, &url_path)?; + .map_err(|_| WorkerThreadsFilenameError::UrlToPathString)?; + let url_path = ensure_read_permission::

(state, &url_path) + .map_err(WorkerThreadsFilenameError::Permission)?; let fs = state.borrow::(); if !fs.exists_sync(&url_path) { - return Err(generic_error(format!("File not found [{:?}]", url_path))); + return Err(WorkerThreadsFilenameError::FileNotFound( + url_path.to_path_buf(), + )); } let node_resolver = state.borrow::(); - match node_resolver.url_to_node_resolution(url)? { + match node_resolver + .url_to_node_resolution(url) + .map_err(WorkerThreadsFilenameError::UrlToNodeResolution)? + { NodeResolution::Esm(u) => Ok(u.to_string()), NodeResolution::CommonJs(u) => wrap_cjs(u), - NodeResolution::BuiltIn(_) => Err(generic_error("Neither ESM nor CJS")), + NodeResolution::BuiltIn(_) => { + Err(WorkerThreadsFilenameError::NeitherEsmNorCjs) + } } } /// /// Wrap a CJS file-URL and the required setup in a stringified `data:`-URL /// -fn wrap_cjs(url: Url) -> Result { +fn wrap_cjs(url: Url) -> Result { let path = url .to_file_path() - .map_err(|e| generic_error(format!("URL to Path: {:#?}", e)))?; + .map_err(|_| WorkerThreadsFilenameError::UrlToPath)?; let filename = path.file_name().unwrap().to_string_lossy(); Ok(format!( "data:text/javascript,import {{ createRequire }} from \"node:module\";\ diff --git a/ext/node/ops/zlib/brotli.rs b/ext/node/ops/zlib/brotli.rs index 3e3905fc3d..1a681ff7f7 100644 --- a/ext/node/ops/zlib/brotli.rs +++ b/ext/node/ops/zlib/brotli.rs @@ -9,8 +9,6 @@ use brotli::BrotliDecompressStream; use brotli::BrotliResult; use brotli::BrotliState; use brotli::Decompressor; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::JsBuffer; use deno_core::OpState; @@ -19,7 +17,23 @@ use deno_core::ToJsBuffer; use std::cell::RefCell; use std::io::Read; -fn encoder_mode(mode: u32) -> Result { +#[derive(Debug, thiserror::Error)] +pub enum BrotliError { + #[error("Invalid encoder mode")] + InvalidEncoderMode, + #[error("Failed to compress")] + CompressFailed, + #[error("Failed to decompress")] + DecompressFailed, + #[error(transparent)] + Join(#[from] tokio::task::JoinError), + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error("{0}")] + Io(std::io::Error), +} + +fn encoder_mode(mode: u32) -> Result { Ok(match mode { 0 => BrotliEncoderMode::BROTLI_MODE_GENERIC, 1 => BrotliEncoderMode::BROTLI_MODE_TEXT, @@ -28,7 +42,7 @@ fn encoder_mode(mode: u32) -> Result { 4 => BrotliEncoderMode::BROTLI_FORCE_MSB_PRIOR, 5 => BrotliEncoderMode::BROTLI_FORCE_UTF8_PRIOR, 6 => BrotliEncoderMode::BROTLI_FORCE_SIGNED_PRIOR, - _ => return Err(type_error("Invalid encoder mode")), + _ => return Err(BrotliError::InvalidEncoderMode), }) } @@ -40,7 +54,7 @@ pub fn op_brotli_compress( #[smi] quality: i32, #[smi] lgwin: i32, #[smi] mode: u32, -) -> Result { +) -> Result { let mode = encoder_mode(mode)?; let mut out_size = out.len(); @@ -57,7 +71,7 @@ pub fn op_brotli_compress( &mut |_, _, _, _| (), ); if result != 1 { - return Err(type_error("Failed to compress")); + return Err(BrotliError::CompressFailed); } Ok(out_size) @@ -87,7 +101,7 @@ pub async fn op_brotli_compress_async( #[smi] quality: i32, #[smi] lgwin: i32, #[smi] mode: u32, -) -> Result { +) -> Result { let mode = encoder_mode(mode)?; tokio::task::spawn_blocking(move || { let input = &*input; @@ -107,7 +121,7 @@ pub async fn op_brotli_compress_async( &mut |_, _, _, _| (), ); if result != 1 { - return Err(type_error("Failed to compress")); + return Err(BrotliError::CompressFailed); } out.truncate(out_size); @@ -151,8 +165,11 @@ pub fn op_brotli_compress_stream( #[smi] rid: u32, #[buffer] input: &[u8], #[buffer] output: &mut [u8], -) -> Result { - let ctx = state.resource_table.get::(rid)?; +) -> Result { + let ctx = state + .resource_table + .get::(rid) + .map_err(BrotliError::Resource)?; let mut inst = ctx.inst.borrow_mut(); let mut output_offset = 0; @@ -168,7 +185,7 @@ pub fn op_brotli_compress_stream( &mut |_, _, _, _| (), ); if !result { - return Err(type_error("Failed to compress")); + return Err(BrotliError::CompressFailed); } Ok(output_offset) @@ -180,8 +197,11 @@ pub fn op_brotli_compress_stream_end( state: &mut OpState, #[smi] rid: u32, #[buffer] output: &mut [u8], -) -> Result { - let ctx = state.resource_table.get::(rid)?; +) -> Result { + let ctx = state + .resource_table + .get::(rid) + .map_err(BrotliError::Resource)?; let mut inst = ctx.inst.borrow_mut(); let mut output_offset = 0; @@ -197,13 +217,13 @@ pub fn op_brotli_compress_stream_end( &mut |_, _, _, _| (), ); if !result { - return Err(type_error("Failed to compress")); + return Err(BrotliError::CompressFailed); } Ok(output_offset) } -fn brotli_decompress(buffer: &[u8]) -> Result { +fn brotli_decompress(buffer: &[u8]) -> Result { let mut output = Vec::with_capacity(4096); let mut decompressor = Decompressor::new(buffer, buffer.len()); decompressor.read_to_end(&mut output)?; @@ -214,7 +234,7 @@ fn brotli_decompress(buffer: &[u8]) -> Result { #[serde] pub fn op_brotli_decompress( #[buffer] buffer: &[u8], -) -> Result { +) -> Result { brotli_decompress(buffer) } @@ -222,8 +242,11 @@ pub fn op_brotli_decompress( #[serde] pub async fn op_brotli_decompress_async( #[buffer] buffer: JsBuffer, -) -> Result { - tokio::task::spawn_blocking(move || brotli_decompress(&buffer)).await? +) -> Result { + tokio::task::spawn_blocking(move || { + brotli_decompress(&buffer).map_err(BrotliError::Io) + }) + .await? } struct BrotliDecompressCtx { @@ -252,8 +275,11 @@ pub fn op_brotli_decompress_stream( #[smi] rid: u32, #[buffer] input: &[u8], #[buffer] output: &mut [u8], -) -> Result { - let ctx = state.resource_table.get::(rid)?; +) -> Result { + let ctx = state + .resource_table + .get::(rid) + .map_err(BrotliError::Resource)?; let mut inst = ctx.inst.borrow_mut(); let mut output_offset = 0; @@ -268,7 +294,7 @@ pub fn op_brotli_decompress_stream( &mut inst, ); if matches!(result, BrotliResult::ResultFailure) { - return Err(type_error("Failed to decompress")); + return Err(BrotliError::DecompressFailed); } Ok(output_offset) @@ -280,8 +306,11 @@ pub fn op_brotli_decompress_stream_end( state: &mut OpState, #[smi] rid: u32, #[buffer] output: &mut [u8], -) -> Result { - let ctx = state.resource_table.get::(rid)?; +) -> Result { + let ctx = state + .resource_table + .get::(rid) + .map_err(BrotliError::Resource)?; let mut inst = ctx.inst.borrow_mut(); let mut output_offset = 0; @@ -296,7 +325,7 @@ pub fn op_brotli_decompress_stream_end( &mut inst, ); if matches!(result, BrotliResult::ResultFailure) { - return Err(type_error("Failed to decompress")); + return Err(BrotliError::DecompressFailed); } Ok(output_offset) diff --git a/ext/node/ops/zlib/mod.rs b/ext/node/ops/zlib/mod.rs index b1d6d21d22..e75ef050d2 100644 --- a/ext/node/ops/zlib/mod.rs +++ b/ext/node/ops/zlib/mod.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; -use deno_core::error::AnyError; + use deno_core::op2; use std::borrow::Cow; use std::cell::RefCell; @@ -8,7 +7,7 @@ use zlib::*; mod alloc; pub mod brotli; -mod mode; +pub mod mode; mod stream; use mode::Flush; @@ -17,11 +16,11 @@ use mode::Mode; use self::stream::StreamWrapper; #[inline] -fn check(condition: bool, msg: &str) -> Result<(), AnyError> { +fn check(condition: bool, msg: &str) -> Result<(), deno_core::error::AnyError> { if condition { Ok(()) } else { - Err(type_error(msg.to_string())) + Err(deno_core::error::type_error(msg.to_string())) } } @@ -56,7 +55,7 @@ impl ZlibInner { out_off: u32, out_len: u32, flush: Flush, - ) -> Result<(), AnyError> { + ) -> Result<(), deno_core::error::AnyError> { check(self.init_done, "write before init")?; check(!self.write_in_progress, "write already in progress")?; check(!self.pending_close, "close already in progress")?; @@ -65,11 +64,11 @@ impl ZlibInner { let next_in = input .get(in_off as usize..in_off as usize + in_len as usize) - .ok_or_else(|| type_error("invalid input range"))? + .ok_or_else(|| deno_core::error::type_error("invalid input range"))? .as_ptr() as *mut _; let next_out = out .get_mut(out_off as usize..out_off as usize + out_len as usize) - .ok_or_else(|| type_error("invalid output range"))? + .ok_or_else(|| deno_core::error::type_error("invalid output range"))? .as_mut_ptr(); self.strm.avail_in = in_len; @@ -81,7 +80,10 @@ impl ZlibInner { Ok(()) } - fn do_write(&mut self, flush: Flush) -> Result<(), AnyError> { + fn do_write( + &mut self, + flush: Flush, + ) -> Result<(), deno_core::error::AnyError> { self.flush = flush; match self.mode { Mode::Deflate | Mode::Gzip | Mode::DeflateRaw => { @@ -127,7 +129,7 @@ impl ZlibInner { self.mode = Mode::Inflate; } } else if next_expected_header_byte.is_some() { - return Err(type_error( + return Err(deno_core::error::type_error( "invalid number of gzip magic number bytes read", )); } @@ -181,7 +183,7 @@ impl ZlibInner { Ok(()) } - fn init_stream(&mut self) -> Result<(), AnyError> { + fn init_stream(&mut self) -> Result<(), deno_core::error::AnyError> { match self.mode { Mode::Gzip | Mode::Gunzip => self.window_bits += 16, Mode::Unzip => self.window_bits += 32, @@ -199,7 +201,7 @@ impl ZlibInner { Mode::Inflate | Mode::Gunzip | Mode::InflateRaw | Mode::Unzip => { self.strm.inflate_init(self.window_bits) } - Mode::None => return Err(type_error("Unknown mode")), + Mode::None => return Err(deno_core::error::type_error("Unknown mode")), }; self.write_in_progress = false; @@ -208,7 +210,7 @@ impl ZlibInner { Ok(()) } - fn close(&mut self) -> Result { + fn close(&mut self) -> Result { if self.write_in_progress { self.pending_close = true; return Ok(false); @@ -222,10 +224,8 @@ impl ZlibInner { Ok(true) } - fn reset_stream(&mut self) -> Result<(), AnyError> { + fn reset_stream(&mut self) { self.err = self.strm.reset(self.mode); - - Ok(()) } } @@ -243,7 +243,7 @@ impl deno_core::Resource for Zlib { #[op2] #[cppgc] -pub fn op_zlib_new(#[smi] mode: i32) -> Result { +pub fn op_zlib_new(#[smi] mode: i32) -> Result { let mode = Mode::try_from(mode)?; let inner = ZlibInner { @@ -256,12 +256,20 @@ pub fn op_zlib_new(#[smi] mode: i32) -> Result { }) } +#[derive(Debug, thiserror::Error)] +pub enum ZlibError { + #[error("zlib not initialized")] + NotInitialized, + #[error(transparent)] + Mode(#[from] mode::ModeError), + #[error(transparent)] + Other(#[from] deno_core::error::AnyError), +} + #[op2(fast)] -pub fn op_zlib_close(#[cppgc] resource: &Zlib) -> Result<(), AnyError> { +pub fn op_zlib_close(#[cppgc] resource: &Zlib) -> Result<(), ZlibError> { let mut resource = resource.inner.borrow_mut(); - let zlib = resource - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = resource.as_mut().ok_or(ZlibError::NotInitialized)?; // If there is a pending write, defer the close until the write is done. zlib.close()?; @@ -282,11 +290,9 @@ pub fn op_zlib_write( #[smi] out_off: u32, #[smi] out_len: u32, #[buffer] result: &mut [u32], -) -> Result { +) -> Result { let mut zlib = resource.inner.borrow_mut(); - let zlib = zlib - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = zlib.as_mut().ok_or(ZlibError::NotInitialized)?; let flush = Flush::try_from(flush)?; zlib.start_write(input, in_off, in_len, out, out_off, out_len, flush)?; @@ -307,11 +313,9 @@ pub fn op_zlib_init( #[smi] mem_level: i32, #[smi] strategy: i32, #[buffer] dictionary: &[u8], -) -> Result { +) -> Result { let mut zlib = resource.inner.borrow_mut(); - let zlib = zlib - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = zlib.as_mut().ok_or(ZlibError::NotInitialized)?; check((8..=15).contains(&window_bits), "invalid windowBits")?; check((-1..=9).contains(&level), "invalid level")?; @@ -348,13 +352,11 @@ pub fn op_zlib_init( #[op2(fast)] #[smi] -pub fn op_zlib_reset(#[cppgc] resource: &Zlib) -> Result { +pub fn op_zlib_reset(#[cppgc] resource: &Zlib) -> Result { let mut zlib = resource.inner.borrow_mut(); - let zlib = zlib - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = zlib.as_mut().ok_or(ZlibError::NotInitialized)?; - zlib.reset_stream()?; + zlib.reset_stream(); Ok(zlib.err) } @@ -362,12 +364,10 @@ pub fn op_zlib_reset(#[cppgc] resource: &Zlib) -> Result { #[op2(fast)] pub fn op_zlib_close_if_pending( #[cppgc] resource: &Zlib, -) -> Result<(), AnyError> { +) -> Result<(), ZlibError> { let pending_close = { let mut zlib = resource.inner.borrow_mut(); - let zlib = zlib - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = zlib.as_mut().ok_or(ZlibError::NotInitialized)?; zlib.write_in_progress = false; zlib.pending_close diff --git a/ext/node/ops/zlib/mode.rs b/ext/node/ops/zlib/mode.rs index 753300cc49..41565f9b11 100644 --- a/ext/node/ops/zlib/mode.rs +++ b/ext/node/ops/zlib/mode.rs @@ -1,19 +1,8 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -#[derive(Debug)] -pub enum Error { - BadArgument, -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Error::BadArgument => write!(f, "bad argument"), - } - } -} - -impl std::error::Error for Error {} +#[derive(Debug, thiserror::Error)] +#[error("bad argument")] +pub struct ModeError; macro_rules! repr_i32 { ($(#[$meta:meta])* $vis:vis enum $name:ident { @@ -25,12 +14,12 @@ macro_rules! repr_i32 { } impl core::convert::TryFrom for $name { - type Error = Error; + type Error = ModeError; fn try_from(v: i32) -> Result { match v { $(x if x == $name::$vname as i32 => Ok($name::$vname),)* - _ => Err(Error::BadArgument), + _ => Err(ModeError), } } } diff --git a/runtime/errors.rs b/runtime/errors.rs index 869f97968e..07bf694dc1 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -704,15 +704,10 @@ fn get_websocket_handshake_error(error: &HandshakeError) -> &'static str { } } -fn get_fs_error(error: &FsOpsError) -> &'static str { +fn get_fs_ops_error(error: &FsOpsError) -> &'static str { match error { FsOpsError::Io(e) => get_io_error_class(e), - FsOpsError::OperationError(e) => match &e.err { - FsError::Io(e) => get_io_error_class(e), - FsError::FileBusy => "Busy", - FsError::NotSupported => "NotSupported", - FsError::NotCapable(_) => "NotCapable", - }, + FsOpsError::OperationError(e) => get_fs_error(&e.err), FsOpsError::Permission(e) | FsOpsError::Resource(e) | FsOpsError::Other(e) => get_error_class_name(e).unwrap_or("Error"), @@ -964,6 +959,160 @@ fn get_websocket_upgrade_error(error: &WebSocketUpgradeError) -> &'static str { } } +fn get_fs_error(e: &FsError) -> &'static str { + match &e { + FsError::Io(e) => get_io_error_class(e), + FsError::FileBusy => "Busy", + FsError::NotSupported => "NotSupported", + FsError::NotCapable(_) => "NotCapable", + } +} + +mod node { + use super::get_error_class_name; + use super::get_io_error_class; + use super::get_serde_json_error_class; + use super::get_url_parse_error_class; + pub use deno_node::ops::blocklist::BlocklistError; + pub use deno_node::ops::fs::FsError; + pub use deno_node::ops::http2::Http2Error; + pub use deno_node::ops::idna::IdnaError; + pub use deno_node::ops::ipc::IpcError; + pub use deno_node::ops::ipc::IpcJsonStreamError; + use deno_node::ops::os::priority::PriorityError; + pub use deno_node::ops::os::OsError; + pub use deno_node::ops::require::RequireError; + pub use deno_node::ops::worker_threads::WorkerThreadsFilenameError; + pub use deno_node::ops::zlib::brotli::BrotliError; + pub use deno_node::ops::zlib::mode::ModeError; + pub use deno_node::ops::zlib::ZlibError; + + pub fn get_blocklist_error(error: &BlocklistError) -> &'static str { + match error { + BlocklistError::AddrParse(_) => "Error", + BlocklistError::IpNetwork(_) => "Error", + BlocklistError::InvalidAddress => "Error", + BlocklistError::IpVersionMismatch => "Error", + } + } + + pub fn get_fs_error(error: &FsError) -> &'static str { + match error { + FsError::Permission(e) => get_error_class_name(e).unwrap_or("Error"), + FsError::Io(e) => get_io_error_class(e), + #[cfg(windows)] + FsError::PathHasNoRoot => "Error", + #[cfg(not(any(unix, windows)))] + FsError::UnsupportedPlatform => "Error", + FsError::Fs(e) => super::get_fs_error(e), + } + } + + pub fn get_idna_error(error: &IdnaError) -> &'static str { + match error { + IdnaError::InvalidInput => "RangeError", + IdnaError::InputTooLong => "Error", + IdnaError::IllegalInput => "RangeError", + } + } + + pub fn get_ipc_json_stream_error(error: &IpcJsonStreamError) -> &'static str { + match error { + IpcJsonStreamError::Io(e) => get_io_error_class(e), + IpcJsonStreamError::SimdJson(_) => "Error", + } + } + + pub fn get_ipc_error(error: &IpcError) -> &'static str { + match error { + IpcError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + IpcError::IpcJsonStream(e) => get_ipc_json_stream_error(e), + IpcError::Canceled(e) => { + let io_err: std::io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + IpcError::SerdeJson(e) => get_serde_json_error_class(e), + } + } + + pub fn get_worker_threads_filename_error( + error: &WorkerThreadsFilenameError, + ) -> &'static str { + match error { + WorkerThreadsFilenameError::Permission(e) => { + get_error_class_name(e).unwrap_or("Error") + } + WorkerThreadsFilenameError::UrlParse(e) => get_url_parse_error_class(e), + WorkerThreadsFilenameError::InvalidRelativeUrl => "Error", + WorkerThreadsFilenameError::UrlFromPathString => "Error", + WorkerThreadsFilenameError::UrlToPathString => "Error", + WorkerThreadsFilenameError::UrlToPath => "Error", + WorkerThreadsFilenameError::FileNotFound(_) => "Error", + WorkerThreadsFilenameError::NeitherEsmNorCjs => "Error", + WorkerThreadsFilenameError::UrlToNodeResolution(_) => "Error", + WorkerThreadsFilenameError::Fs(e) => super::get_fs_error(e), + } + } + + pub fn get_require_error(error: &RequireError) -> &'static str { + match error { + RequireError::UrlParse(e) => get_url_parse_error_class(e), + RequireError::Permission(e) => get_error_class_name(e).unwrap_or("Error"), + RequireError::PackageExportsResolve(_) => "Error", + RequireError::PackageJsonLoad(_) => "Error", + RequireError::ClosestPkgJson(_) => "Error", + RequireError::FilePathConversion(_) => "Error", + RequireError::PackageImportsResolve(_) => "Error", + RequireError::Fs(e) | RequireError::UnableToGetCwd(e) => { + super::get_fs_error(e) + } + } + } + + pub fn get_http2_error(error: &Http2Error) -> &'static str { + match error { + Http2Error::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + Http2Error::UrlParse(e) => get_url_parse_error_class(e), + Http2Error::H2(_) => "Error", + } + } + + pub fn get_os_error(error: &OsError) -> &'static str { + match error { + OsError::Priority(e) => match e { + PriorityError::Io(e) => get_io_error_class(e), + #[cfg(windows)] + PriorityError::InvalidPriority => "TypeError", + }, + OsError::Permission(e) => get_error_class_name(e).unwrap_or("Error"), + OsError::FailedToGetCpuInfo => "TypeError", + } + } + + pub fn get_brotli_error(error: &BrotliError) -> &'static str { + match error { + BrotliError::InvalidEncoderMode => "TypeError", + BrotliError::CompressFailed => "TypeError", + BrotliError::DecompressFailed => "TypeError", + BrotliError::Join(_) => "Error", + BrotliError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + BrotliError::Io(e) => get_io_error_class(e), + } + } + + pub fn get_mode_error(_: &ModeError) -> &'static str { + "Error" + } + + pub fn get_zlib_error(e: &ZlibError) -> &'static str { + match e { + ZlibError::NotInitialized => "TypeError", + ZlibError::Mode(e) => get_mode_error(e), + ZlibError::Other(e) => get_error_class_name(e).unwrap_or("Error"), + } + } +} + fn get_os_error(error: &OsError) -> &'static str { match error { OsError::Permission(e) => get_error_class_name(e).unwrap_or("Error"), @@ -994,6 +1143,46 @@ fn get_sync_fetch_error(error: &SyncFetchError) -> &'static str { pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { deno_core::error::get_custom_error_class(e) + .or_else(|| e.downcast_ref::().map(get_fs_error)) + .or_else(|| { + e.downcast_ref::() + .map(node::get_blocklist_error) + }) + .or_else(|| e.downcast_ref::().map(node::get_fs_error)) + .or_else(|| { + e.downcast_ref::() + .map(node::get_idna_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_ipc_json_stream_error) + }) + .or_else(|| e.downcast_ref::().map(node::get_ipc_error)) + .or_else(|| { + e.downcast_ref::() + .map(node::get_worker_threads_filename_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_require_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_http2_error) + }) + .or_else(|| e.downcast_ref::().map(node::get_os_error)) + .or_else(|| { + e.downcast_ref::() + .map(node::get_brotli_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_mode_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_zlib_error) + }) .or_else(|| e.downcast_ref::().map(get_napi_error_class)) .or_else(|| e.downcast_ref::().map(get_web_error_class)) .or_else(|| { @@ -1029,7 +1218,7 @@ pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { e.downcast_ref::() .map(get_websocket_upgrade_error) }) - .or_else(|| e.downcast_ref::().map(get_fs_error)) + .or_else(|| e.downcast_ref::().map(get_fs_ops_error)) .or_else(|| { e.downcast_ref::() .map(get_ffi_dlfcn_error_class) From 5f0bb3c6f4328003012e98ba70ce18e4e2e842de Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Thu, 24 Oct 2024 20:03:56 +0200 Subject: [PATCH 12/63] fix: `.npmrc` settings not being passed to install/add command (#26473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We weren't passing the resolved npmrc settings to the install commands. This lead us to always fall back to the default registry url instead of using the one from npmrc. Fixes https://github.com/denoland/deno/issues/26139 Fixes https://github.com/denoland/deno/issues/26033 Fixes https://github.com/denoland/deno/issues/25924 Fixes https://github.com/denoland/deno/issues/25822 Fixes https://github.com/denoland/deno/issues/26152 --------- Co-authored-by: Bartek Iwańczuk --- cli/args/mod.rs | 1 + cli/cache/mod.rs | 1 + cli/file_fetcher.rs | 30 +++++++- cli/http_util.rs | 21 ++++++ cli/lsp/npm.rs | 6 +- cli/lsp/registries.rs | 1 + cli/npm/managed/cache/mod.rs | 2 +- cli/npm/managed/cache/registry_info.rs | 68 +++++++++---------- cli/npm/managed/mod.rs | 2 +- cli/npm/mod.rs | 25 +++++-- cli/tools/registry/pm.rs | 6 +- .../@denotest3/basic/1.0.0/main.d.mts | 3 + .../@denotest3/basic/1.0.0/main.mjs | 11 +++ .../@denotest3/basic/1.0.0/other.mjs | 3 + .../@denotest3/basic/1.0.0/package.json | 7 ++ tests/specs/npm/npmrc_install_arg/.npmrc | 2 + .../npm/npmrc_install_arg/__test__.jsonc | 23 +++++++ tests/specs/npm/npmrc_install_arg/add.out | 4 ++ tests/specs/npm/npmrc_install_arg/install.out | 4 ++ tests/specs/npm/npmrc_install_arg/main.js | 5 ++ tests/specs/npm/npmrc_install_arg/main.out | 2 + .../specs/npm/npmrc_install_arg/package.json | 5 ++ tests/specs/npm/npmrc_no_auth/.npmrc | 1 + tests/specs/npm/npmrc_no_auth/__test__.jsonc | 23 +++++++ tests/specs/npm/npmrc_no_auth/add.out | 4 ++ tests/specs/npm/npmrc_no_auth/install.out | 4 ++ tests/specs/npm/npmrc_no_auth/main.js | 5 ++ tests/specs/npm/npmrc_no_auth/main.out | 2 + tests/specs/npm/npmrc_no_auth/package.json | 6 ++ tests/specs/npm/npmrc_no_auth_install/.npmrc | 1 + .../npm/npmrc_no_auth_install/__test__.jsonc | 14 ++++ .../npm/npmrc_no_auth_install/install.out | 3 + tests/specs/npm/npmrc_no_auth_install/main.js | 5 ++ .../specs/npm/npmrc_no_auth_install/main.out | 2 + .../npm/npmrc_no_auth_install/package.json | 7 ++ tests/util/server/src/lib.rs | 3 +- tests/util/server/src/npm.rs | 32 +++++++++ tests/util/server/src/servers/mod.rs | 4 ++ tests/util/server/src/servers/npm_registry.rs | 15 ++++ 39 files changed, 319 insertions(+), 44 deletions(-) create mode 100644 tests/registry/npm-private3/@denotest3/basic/1.0.0/main.d.mts create mode 100644 tests/registry/npm-private3/@denotest3/basic/1.0.0/main.mjs create mode 100644 tests/registry/npm-private3/@denotest3/basic/1.0.0/other.mjs create mode 100644 tests/registry/npm-private3/@denotest3/basic/1.0.0/package.json create mode 100644 tests/specs/npm/npmrc_install_arg/.npmrc create mode 100644 tests/specs/npm/npmrc_install_arg/__test__.jsonc create mode 100644 tests/specs/npm/npmrc_install_arg/add.out create mode 100644 tests/specs/npm/npmrc_install_arg/install.out create mode 100644 tests/specs/npm/npmrc_install_arg/main.js create mode 100644 tests/specs/npm/npmrc_install_arg/main.out create mode 100644 tests/specs/npm/npmrc_install_arg/package.json create mode 100644 tests/specs/npm/npmrc_no_auth/.npmrc create mode 100644 tests/specs/npm/npmrc_no_auth/__test__.jsonc create mode 100644 tests/specs/npm/npmrc_no_auth/add.out create mode 100644 tests/specs/npm/npmrc_no_auth/install.out create mode 100644 tests/specs/npm/npmrc_no_auth/main.js create mode 100644 tests/specs/npm/npmrc_no_auth/main.out create mode 100644 tests/specs/npm/npmrc_no_auth/package.json create mode 100644 tests/specs/npm/npmrc_no_auth_install/.npmrc create mode 100644 tests/specs/npm/npmrc_no_auth_install/__test__.jsonc create mode 100644 tests/specs/npm/npmrc_no_auth_install/install.out create mode 100644 tests/specs/npm/npmrc_no_auth_install/main.js create mode 100644 tests/specs/npm/npmrc_no_auth_install/main.out create mode 100644 tests/specs/npm/npmrc_no_auth_install/package.json diff --git a/cli/args/mod.rs b/cli/args/mod.rs index d8f5531015..927f43e85a 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -578,6 +578,7 @@ fn discover_npmrc( let resolved = npmrc .as_resolved(npm_registry_url()) .context("Failed to resolve .npmrc options")?; + log::debug!(".npmrc found at: '{}'", path.display()); Ok(Arc::new(resolved)) } diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index ded163b4e4..bf8f1b1f0b 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -378,6 +378,7 @@ impl Loader for FetchCacher { } else { FetchPermissionsOptionRef::DynamicContainer(&permissions) }, + maybe_auth: None, maybe_accept: None, maybe_cache_setting: maybe_cache_setting.as_ref(), }, diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index e92aca5420..95d778f0bb 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -24,6 +24,7 @@ use deno_graph::source::LoaderChecksum; use deno_path_util::url_to_file_path; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_web::BlobStore; +use http::header; use log::debug; use std::borrow::Cow; use std::collections::HashMap; @@ -181,6 +182,7 @@ pub enum FetchPermissionsOptionRef<'a> { pub struct FetchOptions<'a> { pub specifier: &'a ModuleSpecifier, pub permissions: FetchPermissionsOptionRef<'a>, + pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, pub maybe_accept: Option<&'a str>, pub maybe_cache_setting: Option<&'a CacheSetting>, } @@ -350,6 +352,7 @@ impl FileFetcher { maybe_accept: Option<&str>, cache_setting: &CacheSetting, maybe_checksum: Option<&LoaderChecksum>, + maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, ) -> Result { debug!( "FileFetcher::fetch_remote_no_follow - specifier: {}", @@ -442,6 +445,7 @@ impl FileFetcher { .as_ref() .map(|(_, etag)| etag.clone()), maybe_auth_token: maybe_auth_token.clone(), + maybe_auth: maybe_auth.clone(), maybe_progress_guard: maybe_progress_guard.as_ref(), }) .await? @@ -538,7 +542,18 @@ impl FileFetcher { specifier: &ModuleSpecifier, ) -> Result { self - .fetch_inner(specifier, FetchPermissionsOptionRef::AllowAll) + .fetch_inner(specifier, None, FetchPermissionsOptionRef::AllowAll) + .await + } + + #[inline(always)] + pub async fn fetch_bypass_permissions_with_maybe_auth( + &self, + specifier: &ModuleSpecifier, + maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, + ) -> Result { + self + .fetch_inner(specifier, maybe_auth, FetchPermissionsOptionRef::AllowAll) .await } @@ -552,6 +567,7 @@ impl FileFetcher { self .fetch_inner( specifier, + None, FetchPermissionsOptionRef::StaticContainer(permissions), ) .await @@ -560,12 +576,14 @@ impl FileFetcher { async fn fetch_inner( &self, specifier: &ModuleSpecifier, + maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, permissions: FetchPermissionsOptionRef<'_>, ) -> Result { self .fetch_with_options(FetchOptions { specifier, permissions, + maybe_auth, maybe_accept: None, maybe_cache_setting: None, }) @@ -585,12 +603,14 @@ impl FileFetcher { max_redirect: usize, ) -> Result { let mut specifier = Cow::Borrowed(options.specifier); + let mut maybe_auth = options.maybe_auth.clone(); for _ in 0..=max_redirect { match self .fetch_no_follow_with_options(FetchNoFollowOptions { fetch_options: FetchOptions { specifier: &specifier, permissions: options.permissions, + maybe_auth: maybe_auth.clone(), maybe_accept: options.maybe_accept, maybe_cache_setting: options.maybe_cache_setting, }, @@ -602,6 +622,10 @@ impl FileFetcher { return Ok(file); } FileOrRedirect::Redirect(redirect_specifier) => { + // If we were redirected to another origin, don't send the auth header anymore. + if redirect_specifier.origin() != specifier.origin() { + maybe_auth = None; + } specifier = Cow::Owned(redirect_specifier); } } @@ -666,6 +690,7 @@ impl FileFetcher { options.maybe_accept, options.maybe_cache_setting.unwrap_or(&self.cache_setting), maybe_checksum, + options.maybe_auth, ) .await } @@ -756,6 +781,7 @@ mod tests { FetchOptions { specifier, permissions: FetchPermissionsOptionRef::AllowAll, + maybe_auth: None, maybe_accept: None, maybe_cache_setting: Some(&file_fetcher.cache_setting), }, @@ -1255,6 +1281,7 @@ mod tests { FetchOptions { specifier: &specifier, permissions: FetchPermissionsOptionRef::AllowAll, + maybe_auth: None, maybe_accept: None, maybe_cache_setting: Some(&file_fetcher.cache_setting), }, @@ -1268,6 +1295,7 @@ mod tests { FetchOptions { specifier: &specifier, permissions: FetchPermissionsOptionRef::AllowAll, + maybe_auth: None, maybe_accept: None, maybe_cache_setting: Some(&file_fetcher.cache_setting), }, diff --git a/cli/http_util.rs b/cli/http_util.rs index 9c9ae9e413..4b17936d68 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -19,6 +19,7 @@ use deno_runtime::deno_fetch; use deno_runtime::deno_fetch::create_http_client; use deno_runtime::deno_fetch::CreateHttpClientOptions; use deno_runtime::deno_tls::RootCertStoreProvider; +use http::header; use http::header::HeaderName; use http::header::HeaderValue; use http::header::ACCEPT; @@ -204,6 +205,7 @@ pub struct FetchOnceArgs<'a> { pub maybe_accept: Option, pub maybe_etag: Option, pub maybe_auth_token: Option, + pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, pub maybe_progress_guard: Option<&'a UpdateGuard>, } @@ -382,6 +384,8 @@ impl HttpClient { request .headers_mut() .insert(AUTHORIZATION, authorization_val); + } else if let Some((header, value)) = args.maybe_auth { + request.headers_mut().insert(header, value); } if let Some(accept) = args.maybe_accept { let accepts_val = HeaderValue::from_str(&accept)?; @@ -792,6 +796,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -818,6 +823,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -845,6 +851,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -866,6 +873,7 @@ mod test { maybe_etag: Some("33a64df551425fcc55e".to_string()), maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; assert_eq!(res.unwrap(), FetchOnceResult::NotModified); @@ -885,6 +893,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -914,6 +923,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, _)) = result { @@ -939,6 +949,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Redirect(url, _)) = result { @@ -974,6 +985,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -1021,6 +1033,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; @@ -1083,6 +1096,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; @@ -1136,6 +1150,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -1177,6 +1192,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -1199,6 +1215,7 @@ mod test { maybe_etag: Some("33a64df551425fcc55e".to_string()), maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; assert_eq!(res.unwrap(), FetchOnceResult::NotModified); @@ -1233,6 +1250,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -1262,6 +1280,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; assert!(result.is_err()); @@ -1283,6 +1302,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; @@ -1306,6 +1326,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; diff --git a/cli/lsp/npm.rs b/cli/lsp/npm.rs index 8bdeb7e7d8..2decfc3429 100644 --- a/cli/lsp/npm.rs +++ b/cli/lsp/npm.rs @@ -4,6 +4,7 @@ use dashmap::DashMap; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; use deno_core::serde_json; +use deno_npm::npm_rc::NpmRc; use deno_semver::package::PackageNv; use deno_semver::Version; use serde::Deserialize; @@ -25,7 +26,10 @@ pub struct CliNpmSearchApi { impl CliNpmSearchApi { pub fn new(file_fetcher: Arc) -> Self { - let resolver = NpmFetchResolver::new(file_fetcher.clone()); + let resolver = NpmFetchResolver::new( + file_fetcher.clone(), + Arc::new(NpmRc::default().as_resolved(npm_registry_url()).unwrap()), + ); Self { file_fetcher, resolver, diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index 5f7ce00823..ade353e683 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -482,6 +482,7 @@ impl ModuleRegistry { .fetch_with_options(FetchOptions { specifier: &specifier, permissions: FetchPermissionsOptionRef::AllowAll, + maybe_auth: None, maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"), maybe_cache_setting: None, }) diff --git a/cli/npm/managed/cache/mod.rs b/cli/npm/managed/cache/mod.rs index fa0e8c8a59..aaec486681 100644 --- a/cli/npm/managed/cache/mod.rs +++ b/cli/npm/managed/cache/mod.rs @@ -26,7 +26,7 @@ use crate::cache::CACHE_PERM; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::hard_link_dir_recursive; -mod registry_info; +pub mod registry_info; mod tarball; mod tarball_extract; diff --git a/cli/npm/managed/cache/registry_info.rs b/cli/npm/managed/cache/registry_info.rs index 6c4a7503b5..6d39d3c13f 100644 --- a/cli/npm/managed/cache/registry_info.rs +++ b/cli/npm/managed/cache/registry_info.rs @@ -84,7 +84,7 @@ impl RegistryInfoDownloader { self.load_package_info_inner(name).await.with_context(|| { format!( "Error getting response at {} for package \"{}\"", - self.get_package_url(name), + get_package_url(&self.npmrc, name), name ) }) @@ -190,7 +190,7 @@ impl RegistryInfoDownloader { fn create_load_future(self: &Arc, name: &str) -> LoadFuture { let downloader = self.clone(); - let package_url = self.get_package_url(name); + let package_url = get_package_url(&self.npmrc, name); let registry_config = self.npmrc.get_registry_config(name); let maybe_auth_header = match maybe_auth_header_for_npm_registry(registry_config) { @@ -239,36 +239,36 @@ impl RegistryInfoDownloader { .map(|r| r.map_err(Arc::new)) .boxed_local() } - - fn get_package_url(&self, name: &str) -> Url { - let registry_url = self.npmrc.get_registry_url(name); - // The '/' character in scoped package names "@scope/name" must be - // encoded for older third party registries. Newer registries and - // npm itself support both ways - // - encoded: https://registry.npmjs.org/@rollup%2fplugin-json - // - non-ecoded: https://registry.npmjs.org/@rollup/plugin-json - // To support as many third party registries as possible we'll - // always encode the '/' character. - - // list of all characters used in npm packages: - // !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~ - const ASCII_SET: percent_encoding::AsciiSet = - percent_encoding::NON_ALPHANUMERIC - .remove(b'!') - .remove(b'\'') - .remove(b'(') - .remove(b')') - .remove(b'*') - .remove(b'-') - .remove(b'.') - .remove(b'@') - .remove(b'_') - .remove(b'~'); - let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET); - registry_url - // Ensure that scoped package name percent encoding is lower cased - // to match npm. - .join(&name.to_string().replace("%2F", "%2f")) - .unwrap() - } +} + +pub fn get_package_url(npmrc: &ResolvedNpmRc, name: &str) -> Url { + let registry_url = npmrc.get_registry_url(name); + // The '/' character in scoped package names "@scope/name" must be + // encoded for older third party registries. Newer registries and + // npm itself support both ways + // - encoded: https://registry.npmjs.org/@rollup%2fplugin-json + // - non-ecoded: https://registry.npmjs.org/@rollup/plugin-json + // To support as many third party registries as possible we'll + // always encode the '/' character. + + // list of all characters used in npm packages: + // !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~ + const ASCII_SET: percent_encoding::AsciiSet = + percent_encoding::NON_ALPHANUMERIC + .remove(b'!') + .remove(b'\'') + .remove(b'(') + .remove(b')') + .remove(b'*') + .remove(b'-') + .remove(b'.') + .remove(b'@') + .remove(b'_') + .remove(b'~'); + let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET); + registry_url + // Ensure that scoped package name percent encoding is lower cased + // to match npm. + .join(&name.to_string().replace("%2F", "%2f")) + .unwrap() } diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index ec50a9c65a..d0880557fe 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -55,7 +55,7 @@ use super::CliNpmResolver; use super::InnerCliNpmResolverRef; use super::ResolvePkgFolderFromDenoReqError; -mod cache; +pub mod cache; mod registry; mod resolution; mod resolvers; diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 53baaf77b4..f48f7a7405 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -8,10 +8,12 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use common::maybe_auth_header_for_npm_registry; use dashmap::DashMap; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_core::serde_json; +use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; use deno_resolver::npm::ByonmNpmResolver; use deno_resolver::npm::ByonmResolvePkgFolderFromDenoReqError; @@ -19,10 +21,10 @@ use deno_runtime::deno_node::NodeRequireResolver; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; +use managed::cache::registry_info::get_package_url; use node_resolver::NpmResolver; use thiserror::Error; -use crate::args::npm_registry_url; use crate::file_fetcher::FileFetcher; pub use self::byonm::CliByonmNpmResolver; @@ -115,14 +117,19 @@ pub struct NpmFetchResolver { nv_by_req: DashMap>, info_by_name: DashMap>>, file_fetcher: Arc, + npmrc: Arc, } impl NpmFetchResolver { - pub fn new(file_fetcher: Arc) -> Self { + pub fn new( + file_fetcher: Arc, + npmrc: Arc, + ) -> Self { Self { nv_by_req: Default::default(), info_by_name: Default::default(), file_fetcher, + npmrc, } } @@ -157,11 +164,21 @@ impl NpmFetchResolver { return info.value().clone(); } let fetch_package_info = || async { - let info_url = npm_registry_url().join(name).ok()?; + let info_url = get_package_url(&self.npmrc, name); let file_fetcher = self.file_fetcher.clone(); + let registry_config = self.npmrc.get_registry_config(name); + // TODO(bartlomieju): this should error out, not use `.ok()`. + let maybe_auth_header = + maybe_auth_header_for_npm_registry(registry_config).ok()?; // spawn due to the lsp's `Send` requirement let file = deno_core::unsync::spawn(async move { - file_fetcher.fetch_bypass_permissions(&info_url).await.ok() + file_fetcher + .fetch_bypass_permissions_with_maybe_auth( + &info_url, + maybe_auth_header, + ) + .await + .ok() }) .await .ok()??; diff --git a/cli/tools/registry/pm.rs b/cli/tools/registry/pm.rs index 2060b9a13f..d1be901d67 100644 --- a/cli/tools/registry/pm.rs +++ b/cli/tools/registry/pm.rs @@ -367,10 +367,14 @@ pub async fn add( Default::default(), None, ); + + let npmrc = cli_factory.cli_options().unwrap().npmrc(); + deps_file_fetcher.set_download_log_level(log::Level::Trace); let deps_file_fetcher = Arc::new(deps_file_fetcher); let jsr_resolver = Arc::new(JsrFetchResolver::new(deps_file_fetcher.clone())); - let npm_resolver = Arc::new(NpmFetchResolver::new(deps_file_fetcher)); + let npm_resolver = + Arc::new(NpmFetchResolver::new(deps_file_fetcher, npmrc.clone())); let mut selected_packages = Vec::with_capacity(add_flags.packages.len()); let mut package_reqs = Vec::with_capacity(add_flags.packages.len()); diff --git a/tests/registry/npm-private3/@denotest3/basic/1.0.0/main.d.mts b/tests/registry/npm-private3/@denotest3/basic/1.0.0/main.d.mts new file mode 100644 index 0000000000..29da1e6d7b --- /dev/null +++ b/tests/registry/npm-private3/@denotest3/basic/1.0.0/main.d.mts @@ -0,0 +1,3 @@ +export declare function setValue(val: number): void; +export declare function getValue(): number; +export declare const url: string; diff --git a/tests/registry/npm-private3/@denotest3/basic/1.0.0/main.mjs b/tests/registry/npm-private3/@denotest3/basic/1.0.0/main.mjs new file mode 100644 index 0000000000..0a44f75859 --- /dev/null +++ b/tests/registry/npm-private3/@denotest3/basic/1.0.0/main.mjs @@ -0,0 +1,11 @@ +let value = 0; + +export function setValue(newValue) { + value = newValue; +} + +export function getValue() { + return value; +} + +export const url = import.meta.url; diff --git a/tests/registry/npm-private3/@denotest3/basic/1.0.0/other.mjs b/tests/registry/npm-private3/@denotest3/basic/1.0.0/other.mjs new file mode 100644 index 0000000000..00ed99da45 --- /dev/null +++ b/tests/registry/npm-private3/@denotest3/basic/1.0.0/other.mjs @@ -0,0 +1,3 @@ +export function hello() { + return "hello, world!"; +} \ No newline at end of file diff --git a/tests/registry/npm-private3/@denotest3/basic/1.0.0/package.json b/tests/registry/npm-private3/@denotest3/basic/1.0.0/package.json new file mode 100644 index 0000000000..ce6ea33830 --- /dev/null +++ b/tests/registry/npm-private3/@denotest3/basic/1.0.0/package.json @@ -0,0 +1,7 @@ +{ + "name": "@denotest3/basic", + "version": "1.0.0", + "type": "module", + "main": "main.mjs", + "types": "main.d.mts" +} diff --git a/tests/specs/npm/npmrc_install_arg/.npmrc b/tests/specs/npm/npmrc_install_arg/.npmrc new file mode 100644 index 0000000000..de3704b924 --- /dev/null +++ b/tests/specs/npm/npmrc_install_arg/.npmrc @@ -0,0 +1,2 @@ +@denotest:registry=http://localhost:4261/ +//localhost:4261/:_authToken=private-reg-token diff --git a/tests/specs/npm/npmrc_install_arg/__test__.jsonc b/tests/specs/npm/npmrc_install_arg/__test__.jsonc new file mode 100644 index 0000000000..f34cfe116a --- /dev/null +++ b/tests/specs/npm/npmrc_install_arg/__test__.jsonc @@ -0,0 +1,23 @@ +{ + "tempDir": true, + "tests": { + "deno_install_add": { + "steps": [{ + "args": "install npm:@denotest/basic", + "output": "install.out" + }, { + "args": "run -A main.js", + "output": "main.out" + }] + }, + "deno_add": { + "steps": [{ + "args": "add npm:@denotest/basic", + "output": "add.out" + }, { + "args": "run -A main.js", + "output": "main.out" + }] + } + } +} diff --git a/tests/specs/npm/npmrc_install_arg/add.out b/tests/specs/npm/npmrc_install_arg/add.out new file mode 100644 index 0000000000..11948e54d5 --- /dev/null +++ b/tests/specs/npm/npmrc_install_arg/add.out @@ -0,0 +1,4 @@ +Add npm:@denotest/basic@1.0.0 +Download http://localhost:4261/@denotest%2fbasic +Download http://localhost:4261/@denotest/basic/1.0.0.tgz +Initialize @denotest/basic@1.0.0 diff --git a/tests/specs/npm/npmrc_install_arg/install.out b/tests/specs/npm/npmrc_install_arg/install.out new file mode 100644 index 0000000000..11948e54d5 --- /dev/null +++ b/tests/specs/npm/npmrc_install_arg/install.out @@ -0,0 +1,4 @@ +Add npm:@denotest/basic@1.0.0 +Download http://localhost:4261/@denotest%2fbasic +Download http://localhost:4261/@denotest/basic/1.0.0.tgz +Initialize @denotest/basic@1.0.0 diff --git a/tests/specs/npm/npmrc_install_arg/main.js b/tests/specs/npm/npmrc_install_arg/main.js new file mode 100644 index 0000000000..e8ccf7611c --- /dev/null +++ b/tests/specs/npm/npmrc_install_arg/main.js @@ -0,0 +1,5 @@ +import { getValue, setValue } from "@denotest/basic"; + +console.log(getValue()); +setValue(42); +console.log(getValue()); diff --git a/tests/specs/npm/npmrc_install_arg/main.out b/tests/specs/npm/npmrc_install_arg/main.out new file mode 100644 index 0000000000..dcd912c89f --- /dev/null +++ b/tests/specs/npm/npmrc_install_arg/main.out @@ -0,0 +1,2 @@ +0 +42 diff --git a/tests/specs/npm/npmrc_install_arg/package.json b/tests/specs/npm/npmrc_install_arg/package.json new file mode 100644 index 0000000000..c1318b361a --- /dev/null +++ b/tests/specs/npm/npmrc_install_arg/package.json @@ -0,0 +1,5 @@ +{ + "name": "npmrc_test", + "version": "0.0.1", + "dependencies": {} +} diff --git a/tests/specs/npm/npmrc_no_auth/.npmrc b/tests/specs/npm/npmrc_no_auth/.npmrc new file mode 100644 index 0000000000..860a7c3d9a --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth/.npmrc @@ -0,0 +1 @@ +@denotest3:registry=http://localhost:4263/ diff --git a/tests/specs/npm/npmrc_no_auth/__test__.jsonc b/tests/specs/npm/npmrc_no_auth/__test__.jsonc new file mode 100644 index 0000000000..efd04f0e46 --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth/__test__.jsonc @@ -0,0 +1,23 @@ +{ + "tempDir": true, + "tests": { + "deno_install_arg": { + "steps": [{ + "args": "install npm:@denotest3/basic", + "output": "install.out" + }, { + "args": "run -A main.js", + "output": "main.out" + }] + }, + "deno_add_arg": { + "steps": [{ + "args": "add npm:@denotest3/basic", + "output": "add.out" + }, { + "args": "run -A main.js", + "output": "main.out" + }] + } + } +} diff --git a/tests/specs/npm/npmrc_no_auth/add.out b/tests/specs/npm/npmrc_no_auth/add.out new file mode 100644 index 0000000000..31b34da13f --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth/add.out @@ -0,0 +1,4 @@ +Add npm:@denotest3/basic@1.0.0 +Download http://localhost:4263/@denotest3%2fbasic +Download http://localhost:4263/@denotest3/basic/1.0.0.tgz +Initialize @denotest3/basic@1.0.0 diff --git a/tests/specs/npm/npmrc_no_auth/install.out b/tests/specs/npm/npmrc_no_auth/install.out new file mode 100644 index 0000000000..31b34da13f --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth/install.out @@ -0,0 +1,4 @@ +Add npm:@denotest3/basic@1.0.0 +Download http://localhost:4263/@denotest3%2fbasic +Download http://localhost:4263/@denotest3/basic/1.0.0.tgz +Initialize @denotest3/basic@1.0.0 diff --git a/tests/specs/npm/npmrc_no_auth/main.js b/tests/specs/npm/npmrc_no_auth/main.js new file mode 100644 index 0000000000..22713bb0da --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth/main.js @@ -0,0 +1,5 @@ +import { getValue, setValue } from "@denotest3/basic"; + +console.log(getValue()); +setValue(42); +console.log(getValue()); diff --git a/tests/specs/npm/npmrc_no_auth/main.out b/tests/specs/npm/npmrc_no_auth/main.out new file mode 100644 index 0000000000..dcd912c89f --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth/main.out @@ -0,0 +1,2 @@ +0 +42 diff --git a/tests/specs/npm/npmrc_no_auth/package.json b/tests/specs/npm/npmrc_no_auth/package.json new file mode 100644 index 0000000000..dddaaac464 --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth/package.json @@ -0,0 +1,6 @@ +{ + "name": "npmrc_test", + "version": "0.0.1", + "dependencies": { + } +} diff --git a/tests/specs/npm/npmrc_no_auth_install/.npmrc b/tests/specs/npm/npmrc_no_auth_install/.npmrc new file mode 100644 index 0000000000..860a7c3d9a --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth_install/.npmrc @@ -0,0 +1 @@ +@denotest3:registry=http://localhost:4263/ diff --git a/tests/specs/npm/npmrc_no_auth_install/__test__.jsonc b/tests/specs/npm/npmrc_no_auth_install/__test__.jsonc new file mode 100644 index 0000000000..56a598e3b9 --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth_install/__test__.jsonc @@ -0,0 +1,14 @@ +{ + "tempDir": true, + "tests": { + "deno_install_arg": { + "steps": [{ + "args": "install", + "output": "install.out" + }, { + "args": "run -A main.js", + "output": "main.out" + }] + } + } +} diff --git a/tests/specs/npm/npmrc_no_auth_install/install.out b/tests/specs/npm/npmrc_no_auth_install/install.out new file mode 100644 index 0000000000..b82747af79 --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth_install/install.out @@ -0,0 +1,3 @@ +Download http://localhost:4263/@denotest3%2fbasic +Download http://localhost:4263/@denotest3/basic/1.0.0.tgz +Initialize @denotest3/basic@1.0.0 diff --git a/tests/specs/npm/npmrc_no_auth_install/main.js b/tests/specs/npm/npmrc_no_auth_install/main.js new file mode 100644 index 0000000000..22713bb0da --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth_install/main.js @@ -0,0 +1,5 @@ +import { getValue, setValue } from "@denotest3/basic"; + +console.log(getValue()); +setValue(42); +console.log(getValue()); diff --git a/tests/specs/npm/npmrc_no_auth_install/main.out b/tests/specs/npm/npmrc_no_auth_install/main.out new file mode 100644 index 0000000000..dcd912c89f --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth_install/main.out @@ -0,0 +1,2 @@ +0 +42 diff --git a/tests/specs/npm/npmrc_no_auth_install/package.json b/tests/specs/npm/npmrc_no_auth_install/package.json new file mode 100644 index 0000000000..2d4fb15bae --- /dev/null +++ b/tests/specs/npm/npmrc_no_auth_install/package.json @@ -0,0 +1,7 @@ +{ + "name": "npmrc_test", + "version": "0.0.1", + "dependencies": { + "@denotest3/basic": "1.0.0" + } +} diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index 88e8287e08..e4a2cc02e3 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -307,7 +307,7 @@ async fn get_tcp_listener_stream( futures::stream::select_all(listeners) } -pub const TEST_SERVERS_COUNT: usize = 30; +pub const TEST_SERVERS_COUNT: usize = 32; #[derive(Default)] struct HttpServerCount { @@ -360,6 +360,7 @@ impl Default for HttpServerStarter { let mut ready_count = 0; for maybe_line in lines { if let Ok(line) = maybe_line { + eprintln!("LINE: {}", line); if line.starts_with("ready:") { ready_count += 1; } diff --git a/tests/util/server/src/npm.rs b/tests/util/server/src/npm.rs index f1c341738a..4b17b95f72 100644 --- a/tests/util/server/src/npm.rs +++ b/tests/util/server/src/npm.rs @@ -18,6 +18,7 @@ use crate::PathRef; pub const DENOTEST_SCOPE_NAME: &str = "@denotest"; pub const DENOTEST2_SCOPE_NAME: &str = "@denotest2"; +pub const DENOTEST3_SCOPE_NAME: &str = "@denotest3"; pub static PUBLIC_TEST_NPM_REGISTRY: Lazy = Lazy::new(|| { TestNpmRegistry::new( @@ -54,6 +55,18 @@ pub static PRIVATE_TEST_NPM_REGISTRY_2: Lazy = ) }); +pub static PRIVATE_TEST_NPM_REGISTRY_3: Lazy = + Lazy::new(|| { + TestNpmRegistry::new( + NpmRegistryKind::Private, + &format!( + "http://localhost:{}", + crate::servers::PRIVATE_NPM_REGISTRY_3_PORT + ), + "npm-private3", + ) + }); + pub enum NpmRegistryKind { Public, Private, @@ -90,6 +103,7 @@ impl TestNpmRegistry { } pub fn root_dir(&self) -> PathRef { + eprintln!("root {}", self.local_path); tests_path().join("registry").join(&self.local_path) } @@ -106,6 +120,7 @@ impl TestNpmRegistry { } pub fn registry_file(&self, name: &str) -> Result>> { + eprintln!("registry file {}", name); self.get_package_property(name, |p| p.registry_file.as_bytes().to_vec()) } @@ -123,6 +138,7 @@ impl TestNpmRegistry { package_name: &str, func: impl FnOnce(&CustomNpmPackage) -> TResult, ) -> Result> { + eprintln!("get package property {}", package_name); // it's ok if multiple threads race here as they will do the same work twice if !self.cache.lock().contains_key(package_name) { match get_npm_package(&self.hostname, &self.local_path, package_name)? { @@ -139,6 +155,7 @@ impl TestNpmRegistry { &self, uri_path: &'s str, ) -> Option<(&'s str, &'s str)> { + eprintln!("GEETT {}", uri_path); let prefix1 = format!("/{}/", DENOTEST_SCOPE_NAME); let prefix2 = format!("/{}%2f", DENOTEST_SCOPE_NAME); @@ -161,6 +178,17 @@ impl TestNpmRegistry { return Some((DENOTEST2_SCOPE_NAME, package_name_with_path)); } + let prefix1 = format!("/{}/", DENOTEST3_SCOPE_NAME); + let prefix2 = format!("/{}%2f", DENOTEST3_SCOPE_NAME); + + let maybe_package_name_with_path = uri_path + .strip_prefix(&prefix1) + .or_else(|| uri_path.strip_prefix(&prefix2)); + + if let Some(package_name_with_path) = maybe_package_name_with_path { + return Some((DENOTEST3_SCOPE_NAME, package_name_with_path)); + } + None } } @@ -170,6 +198,10 @@ fn get_npm_package( local_path: &str, package_name: &str, ) -> Result> { + eprintln!( + "get npm package {} {} {}", + registry_hostname, local_path, package_name + ); let registry_hostname = if package_name == "@denotest/tarballs-privateserver2" { "http://localhost:4262" diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs index 3e18aafce4..3a9c440104 100644 --- a/tests/util/server/src/servers/mod.rs +++ b/tests/util/server/src/servers/mod.rs @@ -91,6 +91,7 @@ const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251; pub(crate) const PUBLIC_NPM_REGISTRY_PORT: u16 = 4260; pub(crate) const PRIVATE_NPM_REGISTRY_1_PORT: u16 = 4261; pub(crate) const PRIVATE_NPM_REGISTRY_2_PORT: u16 = 4262; +pub(crate) const PRIVATE_NPM_REGISTRY_3_PORT: u16 = 4263; // Use the single-threaded scheduler. The hyper server is used as a point of // comparison for the (single-threaded!) benchmarks in cli/bench. We're not @@ -143,6 +144,8 @@ pub async fn run_all_servers() { npm_registry::private_npm_registry1(PRIVATE_NPM_REGISTRY_1_PORT); let private_npm_registry_2_server_futs = npm_registry::private_npm_registry2(PRIVATE_NPM_REGISTRY_2_PORT); + let private_npm_registry_3_server_futs = + npm_registry::private_npm_registry3(PRIVATE_NPM_REGISTRY_3_PORT); let mut futures = vec![ redirect_server_fut.boxed_local(), @@ -173,6 +176,7 @@ pub async fn run_all_servers() { futures.extend(npm_registry_server_futs); futures.extend(private_npm_registry_1_server_futs); futures.extend(private_npm_registry_2_server_futs); + futures.extend(private_npm_registry_3_server_futs); assert_eq!(futures.len(), TEST_SERVERS_COUNT); diff --git a/tests/util/server/src/servers/npm_registry.rs b/tests/util/server/src/servers/npm_registry.rs index acbd9cab48..4ada468fac 100644 --- a/tests/util/server/src/servers/npm_registry.rs +++ b/tests/util/server/src/servers/npm_registry.rs @@ -56,6 +56,14 @@ pub fn private_npm_registry2(port: u16) -> Vec> { ) } +pub fn private_npm_registry3(port: u16) -> Vec> { + run_npm_server( + port, + "npm private registry server error", + private_npm_registry3_handler, + ) +} + fn run_npm_server( port: u16, error_msg: &'static str, @@ -141,6 +149,13 @@ async fn private_npm_registry2_handler( handle_req_for_registry(req, &npm::PRIVATE_TEST_NPM_REGISTRY_2).await } +async fn private_npm_registry3_handler( + req: Request, +) -> Result>, anyhow::Error> { + // No auth for this registry + handle_req_for_registry(req, &npm::PRIVATE_TEST_NPM_REGISTRY_3).await +} + async fn handle_req_for_registry( req: Request, test_npm_registry: &npm::TestNpmRegistry, From ea641897c92e3975dd102b31c1419720df358d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 24 Oct 2024 20:22:36 +0100 Subject: [PATCH 13/63] fix(fmt): --ext flag requires to pass files (#26525) To avoid situations like described in https://github.com/denoland/deno/issues/26402 using `deno fmt` with `--ext` flag now requires to explicitly specify list of files (or globs) to format. Closes https://github.com/denoland/deno/issues/26402 --- cli/args/flags.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 5c2f83561f..235dd53a42 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -2274,7 +2274,7 @@ Ignore formatting a file by adding an ignore comment at the top of the file: "sass", "less", "html", "svelte", "vue", "astro", "yml", "yaml", "ipynb", ]) - .help_heading(FMT_HEADING), + .help_heading(FMT_HEADING).requires("files"), ) .arg( Arg::new("ignore") @@ -6802,6 +6802,32 @@ mod tests { ..Flags::default() } ); + + let r = flags_from_vec(svec!["deno", "fmt", "--ext", "html"]); + assert!(r.is_err()); + let r = flags_from_vec(svec!["deno", "fmt", "--ext", "html", "./**"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Fmt(FmtFlags { + check: false, + files: FileFlags { + include: vec!["./**".to_string()], + ignore: vec![], + }, + use_tabs: None, + line_width: None, + indent_width: None, + single_quote: None, + prose_wrap: None, + no_semicolons: None, + unstable_component: false, + watch: Default::default(), + }), + ext: Some("html".to_string()), + ..Flags::default() + } + ); } #[test] From eedf243b5ea98d22649bb0445444719a2fc12c59 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 24 Oct 2024 15:48:48 -0400 Subject: [PATCH 14/63] perf(compile): pass module source data from binary directly to v8 (#26494) This changes denort to pass a static reference of the moude source bytes found in the binary to v8 instead of copying it. --- Cargo.lock | 35 +- cli/Cargo.toml | 1 - cli/emit.rs | 1 + cli/factory.rs | 1 + cli/mainrt.rs | 9 +- cli/standalone/binary.rs | 463 +++++++------ cli/standalone/file_system.rs | 4 +- cli/standalone/mod.rs | 308 +++------ cli/standalone/serialization.rs | 642 ++++++++++++++++++ cli/standalone/virtual_fs.rs | 115 +++- cli/tools/compile.rs | 30 +- cli/util/text_encoding.rs | 15 + .../compile/byonm_main_sub_dir/deno.json | 2 +- tests/specs/compile/detect_cjs/output.out | 3 +- 14 files changed, 1148 insertions(+), 481 deletions(-) create mode 100644 cli/standalone/serialization.rs diff --git a/Cargo.lock b/Cargo.lock index 3549f0e422..f1209b32ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1196,7 +1196,6 @@ dependencies = [ "dprint-plugin-markdown", "dprint-plugin-typescript", "env_logger", - "eszip", "fancy-regex", "faster-hex", "flate2", @@ -2895,29 +2894,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31ae425815400e5ed474178a7a22e275a9687086a12ca63ec793ff292d8fdae8" -[[package]] -name = "eszip" -version = "0.79.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb55c89bdde75a3826a79d49c9d847623ae7fbdb2695b542982982da990d33e" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.21.7", - "deno_ast", - "deno_graph", - "deno_npm", - "deno_semver", - "futures", - "hashlink 0.8.4", - "indexmap", - "serde", - "serde_json", - "sha2", - "thiserror", - "url", -] - [[package]] name = "fallible-iterator" version = "0.3.0" @@ -3529,15 +3505,6 @@ dependencies = [ "allocator-api2", ] -[[package]] -name = "hashlink" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" -dependencies = [ - "hashbrown", -] - [[package]] name = "hashlink" version = "0.9.1" @@ -5815,7 +5782,7 @@ dependencies = [ "bitflags 2.6.0", "fallible-iterator", "fallible-streaming-iterator", - "hashlink 0.9.1", + "hashlink", "libsqlite3-sys", "smallvec", ] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ded6551cd7..14eda5427e 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -84,7 +84,6 @@ deno_runtime = { workspace = true, features = ["include_js_files_for_snapshottin deno_semver.workspace = true deno_task_shell = "=0.18.1" deno_terminal.workspace = true -eszip = "=0.79.1" libsui = "0.4.0" node_resolver.workspace = true diff --git a/cli/emit.rs b/cli/emit.rs index b3f4a4477a..ad200af050 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -60,6 +60,7 @@ impl Emitter { continue; }; + // todo(https://github.com/denoland/deno_media_type/pull/12): use is_emittable() let is_emittable = matches!( module.media_type, MediaType::TypeScript diff --git a/cli/factory.rs b/cli/factory.rs index 25f3551102..d5ef4fd8b3 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -762,6 +762,7 @@ impl CliFactory { let cli_options = self.cli_options()?; Ok(DenoCompileBinaryWriter::new( self.deno_dir()?, + self.emitter()?, self.file_fetcher()?, self.http_client_provider(), self.npm_resolver().await?.as_ref(), diff --git a/cli/mainrt.rs b/cli/mainrt.rs index 02d58fcee1..f5b798f817 100644 --- a/cli/mainrt.rs +++ b/cli/mainrt.rs @@ -88,11 +88,10 @@ fn main() { let standalone = standalone::extract_standalone(Cow::Owned(args)); let future = async move { match standalone { - Ok(Some(future)) => { - let (metadata, eszip) = future.await?; - util::logger::init(metadata.log_level); - load_env_vars(&metadata.env_vars_from_env_file); - let exit_code = standalone::run(eszip, metadata).await?; + Ok(Some(data)) => { + util::logger::init(data.metadata.log_level); + load_env_vars(&data.metadata.env_vars_from_env_file); + let exit_code = standalone::run(data).await?; std::process::exit(exit_code); } Ok(None) => Ok(()), diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 52ee4eeb28..0f8b0b49d4 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -9,14 +9,18 @@ use std::ffi::OsString; use std::fs; use std::fs::File; use std::future::Future; +use std::io::ErrorKind; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; use std::io::Write; +use std::ops::Range; use std::path::Path; use std::path::PathBuf; use std::process::Command; +use std::sync::Arc; +use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::ResolverWorkspaceJsrPackage; @@ -30,13 +34,22 @@ use deno_core::futures::AsyncReadExt; use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; +use deno_graph::source::RealFileSystem; +use deno_graph::ModuleGraph; +use deno_npm::resolution::SerializedNpmResolutionSnapshot; +use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_npm::NpmPackageId; use deno_npm::NpmSystemInfo; +use deno_runtime::deno_fs; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::RealFs; +use deno_runtime::deno_io::fs::FsError; use deno_runtime::deno_node::PackageJson; use deno_semver::npm::NpmVersionReqParseError; use deno_semver::package::PackageReq; use deno_semver::Version; use deno_semver::VersionReqSpecifierParseError; -use eszip::EszipRelativeFileBaseUrl; use indexmap::IndexMap; use log::Level; use serde::Deserialize; @@ -49,6 +62,7 @@ use crate::args::NpmInstallDepsProvider; use crate::args::PermissionFlags; use crate::args::UnstableConfig; use crate::cache::DenoDir; +use crate::emit::Emitter; use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClientProvider; use crate::npm::CliNpmResolver; @@ -60,12 +74,63 @@ use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; +use super::file_system::DenoCompileFileSystem; +use super::serialization::deserialize_binary_data_section; +use super::serialization::serialize_binary_data_section; +use super::serialization::DenoCompileModuleData; +use super::serialization::DeserializedDataSection; +use super::serialization::RemoteModulesStore; +use super::serialization::RemoteModulesStoreBuilder; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; -const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd"; +/// A URL that can be designated as the base for relative URLs. +/// +/// After creation, this URL may be used to get the key for a +/// module in the binary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url); + +impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> { + fn from(url: &'a Url) -> Self { + Self(url) + } +} + +impl<'a> StandaloneRelativeFileBaseUrl<'a> { + pub fn new(url: &'a Url) -> Self { + debug_assert_eq!(url.scheme(), "file"); + Self(url) + } + + /// Gets the module map key of the provided specifier. + /// + /// * Descendant file specifiers will be made relative to the base. + /// * Non-descendant file specifiers will stay as-is (absolute). + /// * Non-file specifiers will stay as-is. + pub fn specifier_key<'b>(&self, target: &'b Url) -> Cow<'b, str> { + if target.scheme() != "file" { + return Cow::Borrowed(target.as_str()); + } + + match self.0.make_relative(target) { + Some(relative) => { + if relative.starts_with("../") { + Cow::Borrowed(target.as_str()) + } else { + Cow::Owned(relative) + } + } + None => Cow::Borrowed(target.as_str()), + } + } + + pub fn inner(&self) -> &Url { + self.0 + } +} #[derive(Deserialize, Serialize)] pub enum NodeModules { @@ -120,78 +185,23 @@ pub struct Metadata { pub unstable_config: UnstableConfig, } -pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result { - let data = libsui::find_section("d3n0l4nd").unwrap(); - - // We do the first part sync so it can complete quickly - let trailer: [u8; TRAILER_SIZE] = data[0..TRAILER_SIZE].try_into().unwrap(); - let trailer = match Trailer::parse(&trailer)? { - None => panic!("Could not find trailer"), - Some(trailer) => trailer, - }; - let data = &data[TRAILER_SIZE..]; - - let vfs_data = - &data[trailer.npm_vfs_pos as usize..trailer.npm_files_pos as usize]; - let mut dir: VirtualDirectory = serde_json::from_slice(vfs_data)?; - - // align the name of the directory with the root dir - dir.name = root_dir_path - .file_name() - .unwrap() - .to_string_lossy() - .to_string(); - - let fs_root = VfsRoot { - dir, - root_path: root_dir_path, - start_file_offset: trailer.npm_files_pos, - }; - Ok(FileBackedVfs::new(data.to_vec(), fs_root)) -} - fn write_binary_bytes( mut file_writer: File, original_bin: Vec, metadata: &Metadata, - eszip: eszip::EszipV2, - npm_vfs: Option<&VirtualDirectory>, - npm_files: &Vec>, + npm_snapshot: Option, + remote_modules: &RemoteModulesStoreBuilder, + vfs: VfsBuilder, compile_flags: &CompileFlags, ) -> Result<(), AnyError> { - let metadata = serde_json::to_string(metadata)?.as_bytes().to_vec(); - let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec(); - let eszip_archive = eszip.into_bytes(); - - let mut writer = Vec::new(); - - // write the trailer, which includes the positions - // of the data blocks in the file - writer.write_all(&{ - let metadata_pos = eszip_archive.len() as u64; - let npm_vfs_pos = metadata_pos + (metadata.len() as u64); - let npm_files_pos = npm_vfs_pos + (npm_vfs.len() as u64); - Trailer { - eszip_pos: 0, - metadata_pos, - npm_vfs_pos, - npm_files_pos, - } - .as_bytes() - })?; - - writer.write_all(&eszip_archive)?; - writer.write_all(&metadata)?; - writer.write_all(&npm_vfs)?; - for file in npm_files { - writer.write_all(file)?; - } + let data_section_bytes = + serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs)?; let target = compile_flags.resolve_target(); if target.contains("linux") { libsui::Elf::new(&original_bin).append( "d3n0l4nd", - &writer, + &data_section_bytes, &mut file_writer, )?; } else if target.contains("windows") { @@ -201,11 +211,11 @@ fn write_binary_bytes( pe = pe.set_icon(&icon)?; } - pe.write_resource("d3n0l4nd", writer)? + pe.write_resource("d3n0l4nd", data_section_bytes)? .build(&mut file_writer)?; } else if target.contains("darwin") { libsui::Macho::from(original_bin)? - .write_section("d3n0l4nd", writer)? + .write_section("d3n0l4nd", data_section_bytes)? .build_and_sign(&mut file_writer)?; } Ok(()) @@ -221,6 +231,63 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool { || libsui::utils::is_macho(&data) } +pub struct StandaloneData { + pub fs: Arc, + pub metadata: Metadata, + pub modules: StandaloneModules, + pub npm_snapshot: Option, + pub root_path: PathBuf, + pub vfs: Arc, +} + +pub struct StandaloneModules { + remote_modules: RemoteModulesStore, + vfs: Arc, +} + +impl StandaloneModules { + pub fn resolve_specifier<'a>( + &'a self, + specifier: &'a ModuleSpecifier, + ) -> Result, AnyError> { + if specifier.scheme() == "file" { + Ok(Some(specifier)) + } else { + self.remote_modules.resolve_specifier(specifier) + } + } + + pub fn read<'a>( + &'a self, + specifier: &'a ModuleSpecifier, + ) -> Result>, AnyError> { + if specifier.scheme() == "file" { + let path = deno_path_util::url_to_file_path(specifier)?; + let bytes = match self.vfs.file_entry(&path) { + Ok(entry) => self.vfs.read_file_all(entry)?, + Err(err) if err.kind() == ErrorKind::NotFound => { + let bytes = match RealFs.read_file_sync(&path, None) { + Ok(bytes) => bytes, + Err(FsError::Io(err)) if err.kind() == ErrorKind::NotFound => { + return Ok(None) + } + Err(err) => return Err(err.into()), + }; + Cow::Owned(bytes) + } + Err(err) => return Err(err.into()), + }; + Ok(Some(DenoCompileModuleData { + media_type: MediaType::from_specifier(specifier), + specifier, + data: bytes, + })) + } else { + self.remote_modules.read(specifier) + } + } +} + /// This function will try to run this binary as a standalone binary /// produced by `deno compile`. It determines if this is a standalone /// binary by skipping over the trailer width at the end of the file, @@ -228,110 +295,66 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool { /// the bundle is executed. If not, this function exits with `Ok(None)`. pub fn extract_standalone( cli_args: Cow>, -) -> Result< - Option>>, - AnyError, -> { +) -> Result, AnyError> { let Some(data) = libsui::find_section("d3n0l4nd") else { return Ok(None); }; - // We do the first part sync so it can complete quickly - let trailer = match Trailer::parse(&data[0..TRAILER_SIZE])? { + let DeserializedDataSection { + mut metadata, + npm_snapshot, + remote_modules, + mut vfs_dir, + vfs_files_data, + } = match deserialize_binary_data_section(data)? { + Some(data_section) => data_section, None => return Ok(None), - Some(trailer) => trailer, }; + let root_path = { + let maybe_current_exe = std::env::current_exe().ok(); + let current_exe_name = maybe_current_exe + .as_ref() + .and_then(|p| p.file_name()) + .map(|p| p.to_string_lossy()) + // should never happen + .unwrap_or_else(|| Cow::Borrowed("binary")); + std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name)) + }; let cli_args = cli_args.into_owned(); - // If we have an eszip, read it out - Ok(Some(async move { - let bufreader = - deno_core::futures::io::BufReader::new(&data[TRAILER_SIZE..]); + metadata.argv.reserve(cli_args.len() - 1); + for arg in cli_args.into_iter().skip(1) { + metadata.argv.push(arg.into_string().unwrap()); + } + let vfs = { + // align the name of the directory with the root dir + vfs_dir.name = root_path.file_name().unwrap().to_string_lossy().to_string(); - let (eszip, loader) = eszip::EszipV2::parse(bufreader) - .await - .context("Failed to parse eszip header")?; - - let bufreader = loader.await.context("Failed to parse eszip archive")?; - - let mut metadata = String::new(); - - bufreader - .take(trailer.metadata_len()) - .read_to_string(&mut metadata) - .await - .context("Failed to read metadata from the current executable")?; - - let mut metadata: Metadata = serde_json::from_str(&metadata).unwrap(); - metadata.argv.reserve(cli_args.len() - 1); - for arg in cli_args.into_iter().skip(1) { - metadata.argv.push(arg.into_string().unwrap()); - } - - Ok((metadata, eszip)) + let fs_root = VfsRoot { + dir: vfs_dir, + root_path: root_path.clone(), + start_file_offset: 0, + }; + Arc::new(FileBackedVfs::new(Cow::Borrowed(vfs_files_data), fs_root)) + }; + let fs: Arc = + Arc::new(DenoCompileFileSystem::new(vfs.clone())); + Ok(Some(StandaloneData { + fs, + metadata, + modules: StandaloneModules { + remote_modules, + vfs: vfs.clone(), + }, + npm_snapshot, + root_path, + vfs, })) } -const TRAILER_SIZE: usize = std::mem::size_of::() + 8; // 8 bytes for the magic trailer string - -struct Trailer { - eszip_pos: u64, - metadata_pos: u64, - npm_vfs_pos: u64, - npm_files_pos: u64, -} - -impl Trailer { - pub fn parse(trailer: &[u8]) -> Result, AnyError> { - let (magic_trailer, rest) = trailer.split_at(8); - if magic_trailer != MAGIC_TRAILER { - return Ok(None); - } - - let (eszip_archive_pos, rest) = rest.split_at(8); - let (metadata_pos, rest) = rest.split_at(8); - let (npm_vfs_pos, npm_files_pos) = rest.split_at(8); - let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?; - let metadata_pos = u64_from_bytes(metadata_pos)?; - let npm_vfs_pos = u64_from_bytes(npm_vfs_pos)?; - let npm_files_pos = u64_from_bytes(npm_files_pos)?; - Ok(Some(Trailer { - eszip_pos: eszip_archive_pos, - metadata_pos, - npm_vfs_pos, - npm_files_pos, - })) - } - - pub fn metadata_len(&self) -> u64 { - self.npm_vfs_pos - self.metadata_pos - } - - pub fn npm_vfs_len(&self) -> u64 { - self.npm_files_pos - self.npm_vfs_pos - } - - pub fn as_bytes(&self) -> Vec { - let mut trailer = MAGIC_TRAILER.to_vec(); - trailer.write_all(&self.eszip_pos.to_be_bytes()).unwrap(); - trailer.write_all(&self.metadata_pos.to_be_bytes()).unwrap(); - trailer.write_all(&self.npm_vfs_pos.to_be_bytes()).unwrap(); - trailer - .write_all(&self.npm_files_pos.to_be_bytes()) - .unwrap(); - trailer - } -} - -fn u64_from_bytes(arr: &[u8]) -> Result { - let fixed_arr: &[u8; 8] = arr - .try_into() - .context("Failed to convert the buffer into a fixed-size array")?; - Ok(u64::from_be_bytes(*fixed_arr)) -} - pub struct DenoCompileBinaryWriter<'a> { deno_dir: &'a DenoDir, + emitter: &'a Emitter, file_fetcher: &'a FileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a dyn CliNpmResolver, @@ -343,6 +366,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( deno_dir: &'a DenoDir, + emitter: &'a Emitter, file_fetcher: &'a FileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a dyn CliNpmResolver, @@ -351,6 +375,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) -> Self { Self { deno_dir, + emitter, file_fetcher, http_client_provider, npm_resolver, @@ -362,8 +387,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { pub async fn write_bin( &self, writer: File, - eszip: eszip::EszipV2, - root_dir_url: EszipRelativeFileBaseUrl<'_>, + graph: &ModuleGraph, + root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, compile_flags: &CompileFlags, cli_options: &CliOptions, @@ -390,15 +415,17 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) } } - self.write_standalone_binary( - writer, - original_binary, - eszip, - root_dir_url, - entrypoint, - cli_options, - compile_flags, - ) + self + .write_standalone_binary( + writer, + original_binary, + graph, + root_dir_url, + entrypoint, + cli_options, + compile_flags, + ) + .await } async fn get_base_binary( @@ -493,12 +520,12 @@ impl<'a> DenoCompileBinaryWriter<'a> { /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. #[allow(clippy::too_many_arguments)] - fn write_standalone_binary( + async fn write_standalone_binary( &self, writer: File, original_bin: Vec, - mut eszip: eszip::EszipV2, - root_dir_url: EszipRelativeFileBaseUrl<'_>, + graph: &ModuleGraph, + root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, cli_options: &CliOptions, compile_flags: &CompileFlags, @@ -512,19 +539,17 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => None, }; let root_path = root_dir_url.inner().to_file_path().unwrap(); - let (npm_vfs, npm_files, node_modules) = match self.npm_resolver.as_inner() + let (maybe_npm_vfs, node_modules, npm_snapshot) = match self + .npm_resolver + .as_inner() { InnerCliNpmResolverRef::Managed(managed) => { let snapshot = managed.serialized_valid_snapshot_for_system(&self.npm_system_info); if !snapshot.as_serialized().packages.is_empty() { - let (root_dir, files) = self - .build_vfs(&root_path, cli_options)? - .into_dir_and_files(); - eszip.add_npm_snapshot(snapshot); + let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?; ( - Some(root_dir), - files, + Some(npm_vfs_builder), Some(NodeModules::Managed { node_modules_dir: self.npm_resolver.root_node_modules_path().map( |path| { @@ -536,18 +561,16 @@ impl<'a> DenoCompileBinaryWriter<'a> { }, ), }), + Some(snapshot), ) } else { - (None, Vec::new(), None) + (None, None, None) } } InnerCliNpmResolverRef::Byonm(resolver) => { - let (root_dir, files) = self - .build_vfs(&root_path, cli_options)? - .into_dir_and_files(); + let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?; ( - Some(root_dir), - files, + Some(npm_vfs_builder), Some(NodeModules::Byonm { root_node_modules_dir: resolver.root_node_modules_path().map( |node_modules_dir| { @@ -560,9 +583,67 @@ impl<'a> DenoCompileBinaryWriter<'a> { }, ), }), + None, ) } }; + let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs { + npm_vfs + } else { + VfsBuilder::new(root_path.clone())? + }; + let mut remote_modules_store = RemoteModulesStoreBuilder::default(); + for module in graph.modules() { + if module.specifier().scheme() == "data" { + continue; // don't store data urls as an entry as they're in the code + } + let (maybe_source, media_type) = match module { + deno_graph::Module::Js(m) => { + // todo(https://github.com/denoland/deno_media_type/pull/12): use is_emittable() + let is_emittable = matches!( + m.media_type, + MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Jsx + | MediaType::Tsx + ); + let source = if is_emittable { + let source = self + .emitter + .emit_parsed_source(&m.specifier, m.media_type, &m.source) + .await?; + source.to_vec() + } else { + m.source.as_bytes().to_vec() + }; + (Some(source), m.media_type) + } + deno_graph::Module::Json(m) => { + (Some(m.source.as_bytes().to_vec()), m.media_type) + } + deno_graph::Module::Npm(_) + | deno_graph::Module::Node(_) + | deno_graph::Module::External(_) => (None, MediaType::Unknown), + }; + if module.specifier().scheme() == "file" { + let file_path = deno_path_util::url_to_file_path(module.specifier())?; + vfs + .add_file_with_data( + &file_path, + match maybe_source { + Some(source) => source, + None => RealFs.read_file_sync(&file_path, None)?, + }, + ) + .with_context(|| { + format!("Failed adding '{}'", file_path.display()) + })?; + } else if let Some(source) = maybe_source { + remote_modules_store.add(module.specifier(), media_type, source); + } + } + remote_modules_store.add_redirects(&graph.redirects); let env_vars_from_env_file = match cli_options.env_file_name() { Some(env_filename) => { @@ -636,14 +717,14 @@ impl<'a> DenoCompileBinaryWriter<'a> { writer, original_bin, &metadata, - eszip, - npm_vfs.as_ref(), - &npm_files, + npm_snapshot.map(|s| s.into_serialized()), + &remote_modules_store, + vfs, compile_flags, ) } - fn build_vfs( + fn build_npm_vfs( &self, root_path: &Path, cli_options: &CliOptions, @@ -664,8 +745,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { } else { // DO NOT include the user's registry url as it may contain credentials, // but also don't make this dependent on the registry url - let root_path = npm_resolver.global_cache_root_folder(); - let mut builder = VfsBuilder::new(root_path)?; + let global_cache_root_path = npm_resolver.global_cache_root_folder(); + let mut builder = VfsBuilder::new(global_cache_root_path)?; let mut packages = npm_resolver.all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism @@ -675,12 +756,12 @@ impl<'a> DenoCompileBinaryWriter<'a> { builder.add_dir_recursive(&folder)?; } - // Flatten all the registries folders into a single "node_modules/localhost" folder + // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder // that will be used by denort when loading the npm cache. This avoids us exposing // the user's private registry information and means we don't have to bother // serializing all the different registry config into the binary. builder.with_root_dir(|root_dir| { - root_dir.name = "node_modules".to_string(); + root_dir.name = ".deno_compile_node_modules".to_string(); let mut new_entries = Vec::with_capacity(root_dir.entries.len()); let mut localhost_entries = IndexMap::new(); for entry in std::mem::take(&mut root_dir.entries) { @@ -715,6 +796,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { root_dir.entries = new_entries; }); + builder.set_new_root_path(root_path.to_path_buf())?; + Ok(builder) } } diff --git a/cli/standalone/file_system.rs b/cli/standalone/file_system.rs index 314444630b..712c6ee918 100644 --- a/cli/standalone/file_system.rs +++ b/cli/standalone/file_system.rs @@ -22,8 +22,8 @@ use super::virtual_fs::FileBackedVfs; pub struct DenoCompileFileSystem(Arc); impl DenoCompileFileSystem { - pub fn new(vfs: FileBackedVfs) -> Self { - Self(Arc::new(vfs)) + pub fn new(vfs: Arc) -> Self { + Self(vfs) } fn error_if_in_vfs(&self, path: &Path) -> FsResult<()> { diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 60018228b7..3a62b6ff96 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -5,6 +5,8 @@ #![allow(dead_code)] #![allow(unused_imports)] +use binary::StandaloneData; +use binary::StandaloneModules; use deno_ast::MediaType; use deno_cache_dir::npm::NpmCacheDir; use deno_config::workspace::MappedResolution; @@ -38,7 +40,6 @@ use deno_runtime::permissions::RuntimePermissionDescriptorParser; use deno_runtime::WorkerExecutionMode; use deno_runtime::WorkerLogLevel; use deno_semver::npm::NpmPackageReqReference; -use eszip::EszipRelativeFileBaseUrl; use import_map::parse_from_json; use node_resolver::analyze::NodeCodeTranslator; use node_resolver::NodeResolutionMode; @@ -54,6 +55,7 @@ use crate::args::CacheSetting; use crate::args::NpmInstallDepsProvider; use crate::args::StorageKeyResolver; use crate::cache::Caches; +use crate::cache::DenoCacheEnvFsAdapter; use crate::cache::DenoDirProvider; use crate::cache::NodeAnalysisCache; use crate::cache::RealDenoCacheEnv; @@ -78,52 +80,18 @@ use crate::worker::ModuleLoaderFactory; pub mod binary; mod file_system; +mod serialization; mod virtual_fs; pub use binary::extract_standalone; pub use binary::is_standalone_binary; pub use binary::DenoCompileBinaryWriter; -use self::binary::load_npm_vfs; use self::binary::Metadata; use self::file_system::DenoCompileFileSystem; -struct WorkspaceEszipModule { - specifier: ModuleSpecifier, - inner: eszip::Module, -} - -struct WorkspaceEszip { - eszip: eszip::EszipV2, - root_dir_url: Arc, -} - -impl WorkspaceEszip { - pub fn get_module( - &self, - specifier: &ModuleSpecifier, - ) -> Option { - if specifier.scheme() == "file" { - let specifier_key = EszipRelativeFileBaseUrl::new(&self.root_dir_url) - .specifier_key(specifier); - let module = self.eszip.get_module(&specifier_key)?; - let specifier = self.root_dir_url.join(&module.specifier).unwrap(); - Some(WorkspaceEszipModule { - specifier, - inner: module, - }) - } else { - let module = self.eszip.get_module(specifier.as_str())?; - Some(WorkspaceEszipModule { - specifier: ModuleSpecifier::parse(&module.specifier).unwrap(), - inner: module, - }) - } - } -} - struct SharedModuleLoaderState { - eszip: WorkspaceEszip, + modules: StandaloneModules, workspace_resolver: WorkspaceResolver, node_resolver: Arc, npm_module_loader: Arc, @@ -249,8 +217,10 @@ impl ModuleLoader for EmbeddedModuleLoader { } if specifier.scheme() == "jsr" { - if let Some(module) = self.shared.eszip.get_module(&specifier) { - return Ok(module.specifier); + if let Some(specifier) = + self.shared.modules.resolve_specifier(&specifier)? + { + return Ok(specifier.clone()); } } @@ -345,56 +315,30 @@ impl ModuleLoader for EmbeddedModuleLoader { ); } - let Some(module) = self.shared.eszip.get_module(original_specifier) else { - return deno_core::ModuleLoadResponse::Sync(Err(type_error(format!( - "{MODULE_NOT_FOUND}: {}", - original_specifier - )))); - }; - let original_specifier = original_specifier.clone(); - - deno_core::ModuleLoadResponse::Async( - async move { - let code = module.inner.source().await.ok_or_else(|| { - type_error(format!("Module not found: {}", original_specifier)) - })?; - let code = arc_u8_to_arc_str(code) - .map_err(|_| type_error("Module source is not utf-8"))?; - Ok(deno_core::ModuleSource::new_with_redirect( - match module.inner.kind { - eszip::ModuleKind::JavaScript => ModuleType::JavaScript, - eszip::ModuleKind::Json => ModuleType::Json, - eszip::ModuleKind::Jsonc => { - return Err(type_error("jsonc modules not supported")) - } - eszip::ModuleKind::OpaqueData => { - unreachable!(); - } - }, - ModuleSourceCode::String(code.into()), - &original_specifier, - &module.specifier, - None, + match self.shared.modules.read(original_specifier) { + Ok(Some(module)) => { + let (module_specifier, module_type, module_source) = + module.into_for_v8(); + deno_core::ModuleLoadResponse::Sync(Ok( + deno_core::ModuleSource::new_with_redirect( + module_type, + module_source, + original_specifier, + module_specifier, + None, + ), )) } - .boxed_local(), - ) + Ok(None) => deno_core::ModuleLoadResponse::Sync(Err(type_error( + format!("{MODULE_NOT_FOUND}: {}", original_specifier), + ))), + Err(err) => deno_core::ModuleLoadResponse::Sync(Err(type_error( + format!("{:?}", err), + ))), + } } } -fn arc_u8_to_arc_str( - arc_u8: Arc<[u8]>, -) -> Result, std::str::Utf8Error> { - // Check that the string is valid UTF-8. - std::str::from_utf8(&arc_u8)?; - // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as - // Arc. This is proven by the From> impl for Arc<[u8]> from the - // standard library. - Ok(unsafe { - std::mem::transmute::, std::sync::Arc>(arc_u8) - }) -} - struct StandaloneModuleLoaderFactory { shared: Arc, } @@ -439,13 +383,15 @@ impl RootCertStoreProvider for StandaloneRootCertStoreProvider { } } -pub async fn run( - mut eszip: eszip::EszipV2, - metadata: Metadata, -) -> Result { - let current_exe_path = std::env::current_exe().unwrap(); - let current_exe_name = - current_exe_path.file_name().unwrap().to_string_lossy(); +pub async fn run(data: StandaloneData) -> Result { + let StandaloneData { + fs, + metadata, + modules, + npm_snapshot, + root_path, + vfs, + } = data; let deno_dir_provider = Arc::new(DenoDirProvider::new(None)); let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider { ca_stores: metadata.ca_stores, @@ -459,112 +405,83 @@ pub async fn run( )); // use a dummy npm registry url let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap(); - let root_path = - std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name)); let root_dir_url = Arc::new(ModuleSpecifier::from_directory_path(&root_path).unwrap()); let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap(); - let root_node_modules_path = root_path.join("node_modules"); - let npm_cache_dir = NpmCacheDir::new( - &RealDenoCacheEnv, - root_node_modules_path.clone(), - vec![npm_registry_url.clone()], - ); - let npm_global_cache_dir = npm_cache_dir.get_cache_location(); + let npm_global_cache_dir = root_path.join(".deno_compile_node_modules"); let cache_setting = CacheSetting::Only; - let (fs, npm_resolver, maybe_vfs_root) = match metadata.node_modules { + let npm_resolver = match metadata.node_modules { Some(binary::NodeModules::Managed { node_modules_dir }) => { - // this will always have a snapshot - let snapshot = eszip.take_npm_snapshot().unwrap(); - let vfs_root_dir_path = if node_modules_dir.is_some() { - root_path.clone() - } else { - npm_cache_dir.root_dir().to_owned() - }; - let vfs = load_npm_vfs(vfs_root_dir_path.clone()) - .context("Failed to load npm vfs.")?; + let snapshot = npm_snapshot.unwrap(); let maybe_node_modules_path = node_modules_dir - .map(|node_modules_dir| vfs_root_dir_path.join(node_modules_dir)); - let fs = Arc::new(DenoCompileFileSystem::new(vfs)) - as Arc; - let npm_resolver = - create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( - CliNpmResolverManagedCreateOptions { - snapshot: CliNpmResolverManagedSnapshotOption::Specified(Some( - snapshot, - )), - maybe_lockfile: None, - fs: fs.clone(), - http_client_provider: http_client_provider.clone(), - npm_global_cache_dir, - cache_setting, - text_only_progress_bar: progress_bar, - maybe_node_modules_path, - npm_system_info: Default::default(), - npm_install_deps_provider: Arc::new( - // this is only used for installing packages, which isn't necessary with deno compile - NpmInstallDepsProvider::empty(), - ), - // create an npmrc that uses the fake npm_registry_url to resolve packages - npmrc: Arc::new(ResolvedNpmRc { - default_config: deno_npm::npm_rc::RegistryConfigWithUrl { - registry_url: npm_registry_url.clone(), - config: Default::default(), - }, - scopes: Default::default(), - registry_configs: Default::default(), - }), - lifecycle_scripts: Default::default(), - }, - )) - .await?; - (fs, npm_resolver, Some(vfs_root_dir_path)) + .map(|node_modules_dir| root_path.join(node_modules_dir)); + create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( + CliNpmResolverManagedCreateOptions { + snapshot: CliNpmResolverManagedSnapshotOption::Specified(Some( + snapshot, + )), + maybe_lockfile: None, + fs: fs.clone(), + http_client_provider: http_client_provider.clone(), + npm_global_cache_dir, + cache_setting, + text_only_progress_bar: progress_bar, + maybe_node_modules_path, + npm_system_info: Default::default(), + npm_install_deps_provider: Arc::new( + // this is only used for installing packages, which isn't necessary with deno compile + NpmInstallDepsProvider::empty(), + ), + // create an npmrc that uses the fake npm_registry_url to resolve packages + npmrc: Arc::new(ResolvedNpmRc { + default_config: deno_npm::npm_rc::RegistryConfigWithUrl { + registry_url: npm_registry_url.clone(), + config: Default::default(), + }, + scopes: Default::default(), + registry_configs: Default::default(), + }), + lifecycle_scripts: Default::default(), + }, + )) + .await? } Some(binary::NodeModules::Byonm { root_node_modules_dir, }) => { - let vfs_root_dir_path = root_path.clone(); - let vfs = load_npm_vfs(vfs_root_dir_path.clone()) - .context("Failed to load vfs.")?; let root_node_modules_dir = root_node_modules_dir.map(|p| vfs.root().join(p)); - let fs = Arc::new(DenoCompileFileSystem::new(vfs)) - as Arc; - let npm_resolver = create_cli_npm_resolver( - CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { + create_cli_npm_resolver(CliNpmResolverCreateOptions::Byonm( + CliByonmNpmResolverCreateOptions { fs: CliDenoResolverFs(fs.clone()), root_node_modules_dir, - }), - ) - .await?; - (fs, npm_resolver, Some(vfs_root_dir_path)) + }, + )) + .await? } None => { - let fs = Arc::new(deno_fs::RealFs) as Arc; - let npm_resolver = - create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( - CliNpmResolverManagedCreateOptions { - snapshot: CliNpmResolverManagedSnapshotOption::Specified(None), - maybe_lockfile: None, - fs: fs.clone(), - http_client_provider: http_client_provider.clone(), - npm_global_cache_dir, - cache_setting, - text_only_progress_bar: progress_bar, - maybe_node_modules_path: None, - npm_system_info: Default::default(), - npm_install_deps_provider: Arc::new( - // this is only used for installing packages, which isn't necessary with deno compile - NpmInstallDepsProvider::empty(), - ), - // Packages from different registries are already inlined in the ESZip, - // so no need to create actual `.npmrc` configuration. - npmrc: create_default_npmrc(), - lifecycle_scripts: Default::default(), - }, - )) - .await?; - (fs, npm_resolver, None) + create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( + CliNpmResolverManagedCreateOptions { + snapshot: CliNpmResolverManagedSnapshotOption::Specified(None), + maybe_lockfile: None, + fs: fs.clone(), + http_client_provider: http_client_provider.clone(), + npm_global_cache_dir, + cache_setting, + text_only_progress_bar: progress_bar, + maybe_node_modules_path: None, + npm_system_info: Default::default(), + npm_install_deps_provider: Arc::new( + // this is only used for installing packages, which isn't necessary with deno compile + NpmInstallDepsProvider::empty(), + ), + // Packages from different registries are already inlined in the binary, + // so no need to create actual `.npmrc` configuration. + npmrc: create_default_npmrc(), + lifecycle_scripts: Default::default(), + }, + )) + .await? } }; @@ -645,10 +562,7 @@ pub async fn run( }; let module_loader_factory = StandaloneModuleLoaderFactory { shared: Arc::new(SharedModuleLoaderState { - eszip: WorkspaceEszip { - eszip, - root_dir_url, - }, + modules, workspace_resolver, node_resolver: cli_node_resolver.clone(), npm_module_loader: Arc::new(NpmModuleLoader::new( @@ -663,19 +577,17 @@ pub async fn run( let permissions = { let mut permissions = metadata.permissions.to_options(/* cli_arg_urls */ &[]); - // if running with an npm vfs, grant read access to it - if let Some(vfs_root) = maybe_vfs_root { - match &mut permissions.allow_read { - Some(vec) if vec.is_empty() => { - // do nothing, already granted - } - Some(vec) => { - vec.push(vfs_root.to_string_lossy().to_string()); - } - None => { - permissions.allow_read = - Some(vec![vfs_root.to_string_lossy().to_string()]); - } + // grant read access to the vfs + match &mut permissions.allow_read { + Some(vec) if vec.is_empty() => { + // do nothing, already granted + } + Some(vec) => { + vec.push(root_path.to_string_lossy().to_string()); + } + None => { + permissions.allow_read = + Some(vec![root_path.to_string_lossy().to_string()]); } } diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs new file mode 100644 index 0000000000..12927845bc --- /dev/null +++ b/cli/standalone/serialization.rs @@ -0,0 +1,642 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::io::Write; + +use deno_ast::MediaType; +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::serde_json; +use deno_core::url::Url; +use deno_core::FastString; +use deno_core::ModuleSourceCode; +use deno_core::ModuleType; +use deno_npm::resolution::SerializedNpmResolutionSnapshot; +use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_npm::NpmPackageId; +use deno_semver::package::PackageReq; + +use crate::standalone::virtual_fs::VirtualDirectory; + +use super::binary::Metadata; +use super::virtual_fs::VfsBuilder; + +const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; + +/// Binary format: +/// * d3n0l4nd +/// * +/// * +/// * +/// * +/// * +/// * d3n0l4nd +pub fn serialize_binary_data_section( + metadata: &Metadata, + npm_snapshot: Option, + remote_modules: &RemoteModulesStoreBuilder, + vfs: VfsBuilder, +) -> Result, AnyError> { + fn write_bytes_with_len(bytes: &mut Vec, data: &[u8]) { + bytes.extend_from_slice(&(data.len() as u64).to_le_bytes()); + bytes.extend_from_slice(data); + } + + let mut bytes = Vec::new(); + bytes.extend_from_slice(MAGIC_BYTES); + + // 1. Metadata + { + let metadata = serde_json::to_string(metadata)?; + write_bytes_with_len(&mut bytes, metadata.as_bytes()); + } + // 2. Npm snapshot + { + let npm_snapshot = + npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default(); + write_bytes_with_len(&mut bytes, &npm_snapshot); + } + // 3. Remote modules + { + let update_index = bytes.len(); + bytes.extend_from_slice(&(0_u64).to_le_bytes()); + let start_index = bytes.len(); + remote_modules.write(&mut bytes)?; + let length = bytes.len() - start_index; + let length_bytes = (length as u64).to_le_bytes(); + bytes[update_index..update_index + length_bytes.len()] + .copy_from_slice(&length_bytes); + } + // 4. VFS + { + let (vfs, vfs_files) = vfs.into_dir_and_files(); + let vfs = serde_json::to_string(&vfs)?; + write_bytes_with_len(&mut bytes, vfs.as_bytes()); + let vfs_bytes_len = vfs_files.iter().map(|f| f.len() as u64).sum::(); + bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes()); + for file in &vfs_files { + bytes.extend_from_slice(file); + } + } + + // write the magic bytes at the end so we can use it + // to make sure we've deserialized correctly + bytes.extend_from_slice(MAGIC_BYTES); + + Ok(bytes) +} + +pub struct DeserializedDataSection { + pub metadata: Metadata, + pub npm_snapshot: Option, + pub remote_modules: RemoteModulesStore, + pub vfs_dir: VirtualDirectory, + pub vfs_files_data: &'static [u8], +} + +pub fn deserialize_binary_data_section( + data: &'static [u8], +) -> Result, AnyError> { + fn read_bytes_with_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> { + let (input, len) = read_u64(input)?; + let (input, data) = read_bytes(input, len as usize)?; + Ok((input, data)) + } + + fn read_magic_bytes(input: &[u8]) -> Result<(&[u8], bool), AnyError> { + if input.len() < MAGIC_BYTES.len() { + bail!("Unexpected end of data. Could not find magic bytes."); + } + let (magic_bytes, input) = input.split_at(MAGIC_BYTES.len()); + if magic_bytes != MAGIC_BYTES { + return Ok((input, false)); + } + Ok((input, true)) + } + + let (input, found) = read_magic_bytes(data)?; + if !found { + return Ok(None); + } + + // 1. Metadata + let (input, data) = read_bytes_with_len(input).context("reading metadata")?; + let metadata: Metadata = + serde_json::from_slice(data).context("deserializing metadata")?; + // 2. Npm snapshot + let (input, data) = + read_bytes_with_len(input).context("reading npm snapshot")?; + let npm_snapshot = if data.is_empty() { + None + } else { + Some(deserialize_npm_snapshot(data).context("deserializing npm snapshot")?) + }; + // 3. Remote modules + let (input, data) = + read_bytes_with_len(input).context("reading remote modules data")?; + let remote_modules = + RemoteModulesStore::build(data).context("deserializing remote modules")?; + // 4. VFS + let (input, data) = read_bytes_with_len(input).context("vfs")?; + let vfs_dir: VirtualDirectory = + serde_json::from_slice(data).context("deserializing vfs data")?; + let (input, vfs_files_data) = + read_bytes_with_len(input).context("reading vfs files data")?; + + // finally ensure we read the magic bytes at the end + let (_input, found) = read_magic_bytes(input)?; + if !found { + bail!("Could not find magic bytes at the end of the data."); + } + + Ok(Some(DeserializedDataSection { + metadata, + npm_snapshot, + remote_modules, + vfs_dir, + vfs_files_data, + })) +} + +#[derive(Default)] +pub struct RemoteModulesStoreBuilder { + specifiers: Vec<(String, u64)>, + data: Vec<(MediaType, Vec)>, + data_byte_len: u64, + redirects: Vec<(String, String)>, + redirects_len: u64, +} + +impl RemoteModulesStoreBuilder { + pub fn add(&mut self, specifier: &Url, media_type: MediaType, data: Vec) { + let specifier = specifier.to_string(); + self.specifiers.push((specifier, self.data_byte_len)); + self.data_byte_len += 1 + 8 + data.len() as u64; // media type (1 byte), data length (8 bytes), data + self.data.push((media_type, data)); + } + + pub fn add_redirects(&mut self, redirects: &BTreeMap) { + self.redirects.reserve(redirects.len()); + for (from, to) in redirects { + let from = from.to_string(); + let to = to.to_string(); + self.redirects_len += (4 + from.len() + 4 + to.len()) as u64; + self.redirects.push((from, to)); + } + } + + fn write(&self, writer: &mut dyn Write) -> Result<(), AnyError> { + writer.write_all(&(self.specifiers.len() as u32).to_le_bytes())?; + writer.write_all(&(self.redirects.len() as u32).to_le_bytes())?; + for (specifier, offset) in &self.specifiers { + writer.write_all(&(specifier.len() as u32).to_le_bytes())?; + writer.write_all(specifier.as_bytes())?; + writer.write_all(&offset.to_le_bytes())?; + } + for (from, to) in &self.redirects { + writer.write_all(&(from.len() as u32).to_le_bytes())?; + writer.write_all(from.as_bytes())?; + writer.write_all(&(to.len() as u32).to_le_bytes())?; + writer.write_all(to.as_bytes())?; + } + for (media_type, data) in &self.data { + writer.write_all(&[serialize_media_type(*media_type)])?; + writer.write_all(&(data.len() as u64).to_le_bytes())?; + writer.write_all(data)?; + } + Ok(()) + } +} + +pub struct DenoCompileModuleData<'a> { + pub specifier: &'a Url, + pub media_type: MediaType, + pub data: Cow<'static, [u8]>, +} + +impl<'a> DenoCompileModuleData<'a> { + pub fn into_for_v8(self) -> (&'a Url, ModuleType, ModuleSourceCode) { + fn into_bytes(data: Cow<'static, [u8]>) -> ModuleSourceCode { + ModuleSourceCode::Bytes(match data { + Cow::Borrowed(d) => d.into(), + Cow::Owned(d) => d.into_boxed_slice().into(), + }) + } + + fn into_string_unsafe(data: Cow<'static, [u8]>) -> ModuleSourceCode { + // todo(https://github.com/denoland/deno_core/pull/943): store whether + // the string is ascii or not ahead of time so we can avoid the is_ascii() + // check in FastString::from_static + match data { + Cow::Borrowed(d) => ModuleSourceCode::String( + // SAFETY: we know this is a valid utf8 string + unsafe { FastString::from_static(std::str::from_utf8_unchecked(d)) }, + ), + Cow::Owned(d) => ModuleSourceCode::Bytes(d.into_boxed_slice().into()), + } + } + + let (media_type, source) = match self.media_type { + MediaType::JavaScript + | MediaType::Jsx + | MediaType::Mjs + | MediaType::Cjs + | MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Dts + | MediaType::Dmts + | MediaType::Dcts + | MediaType::Tsx => { + (ModuleType::JavaScript, into_string_unsafe(self.data)) + } + MediaType::Json => (ModuleType::Json, into_string_unsafe(self.data)), + MediaType::Wasm => (ModuleType::Wasm, into_bytes(self.data)), + // just assume javascript if we made it here + MediaType::TsBuildInfo | MediaType::SourceMap | MediaType::Unknown => { + (ModuleType::JavaScript, into_bytes(self.data)) + } + }; + (self.specifier, media_type, source) + } +} + +enum RemoteModulesStoreSpecifierValue { + Data(usize), + Redirect(Url), +} + +pub struct RemoteModulesStore { + specifiers: HashMap, + files_data: &'static [u8], +} + +impl RemoteModulesStore { + fn build(data: &'static [u8]) -> Result { + fn read_specifier(input: &[u8]) -> Result<(&[u8], (Url, u64)), AnyError> { + let (input, specifier) = read_string_lossy(input)?; + let specifier = Url::parse(&specifier)?; + let (input, offset) = read_u64(input)?; + Ok((input, (specifier, offset))) + } + + fn read_redirect(input: &[u8]) -> Result<(&[u8], (Url, Url)), AnyError> { + let (input, from) = read_string_lossy(input)?; + let from = Url::parse(&from)?; + let (input, to) = read_string_lossy(input)?; + let to = Url::parse(&to)?; + Ok((input, (from, to))) + } + + fn read_headers( + input: &[u8], + ) -> Result<(&[u8], HashMap), AnyError> + { + let (input, specifiers_len) = read_u32_as_usize(input)?; + let (mut input, redirects_len) = read_u32_as_usize(input)?; + let mut specifiers = + HashMap::with_capacity(specifiers_len + redirects_len); + for _ in 0..specifiers_len { + let (current_input, (specifier, offset)) = + read_specifier(input).context("reading specifier")?; + input = current_input; + specifiers.insert( + specifier, + RemoteModulesStoreSpecifierValue::Data(offset as usize), + ); + } + + for _ in 0..redirects_len { + let (current_input, (from, to)) = read_redirect(input)?; + input = current_input; + specifiers.insert(from, RemoteModulesStoreSpecifierValue::Redirect(to)); + } + + Ok((input, specifiers)) + } + + let (files_data, specifiers) = read_headers(data)?; + + Ok(Self { + specifiers, + files_data, + }) + } + + pub fn resolve_specifier<'a>( + &'a self, + specifier: &'a Url, + ) -> Result, AnyError> { + let mut count = 0; + let mut current = specifier; + loop { + if count > 10 { + bail!("Too many redirects resolving '{}'", specifier); + } + match self.specifiers.get(current) { + Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => { + current = to; + count += 1; + } + Some(RemoteModulesStoreSpecifierValue::Data(_)) => { + return Ok(Some(current)); + } + None => { + return Ok(None); + } + } + } + } + + pub fn read<'a>( + &'a self, + specifier: &'a Url, + ) -> Result>, AnyError> { + let mut count = 0; + let mut current = specifier; + loop { + if count > 10 { + bail!("Too many redirects resolving '{}'", specifier); + } + match self.specifiers.get(current) { + Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => { + current = to; + count += 1; + } + Some(RemoteModulesStoreSpecifierValue::Data(offset)) => { + let input = &self.files_data[*offset..]; + let (input, media_type_byte) = read_bytes(input, 1)?; + let media_type = deserialize_media_type(media_type_byte[0])?; + let (input, len) = read_u64(input)?; + let (_input, data) = read_bytes(input, len as usize)?; + return Ok(Some(DenoCompileModuleData { + specifier, + media_type, + data: Cow::Borrowed(data), + })); + } + None => { + return Ok(None); + } + } + } + } +} + +fn serialize_npm_snapshot( + mut snapshot: SerializedNpmResolutionSnapshot, +) -> Vec { + fn append_string(bytes: &mut Vec, string: &str) { + let len = string.len() as u32; + bytes.extend_from_slice(&len.to_le_bytes()); + bytes.extend_from_slice(string.as_bytes()); + } + + snapshot.packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism + let ids_to_stored_ids = snapshot + .packages + .iter() + .enumerate() + .map(|(i, pkg)| (&pkg.id, i as u32)) + .collect::>(); + + let mut root_packages: Vec<_> = snapshot.root_packages.iter().collect(); + root_packages.sort(); + let mut bytes = Vec::new(); + + bytes.extend_from_slice(&(snapshot.packages.len() as u32).to_le_bytes()); + for pkg in &snapshot.packages { + append_string(&mut bytes, &pkg.id.as_serialized()); + } + + bytes.extend_from_slice(&(root_packages.len() as u32).to_le_bytes()); + for (req, id) in root_packages { + append_string(&mut bytes, &req.to_string()); + let id = ids_to_stored_ids.get(&id).unwrap(); + bytes.extend_from_slice(&id.to_le_bytes()); + } + + for pkg in &snapshot.packages { + let deps_len = pkg.dependencies.len() as u32; + bytes.extend_from_slice(&deps_len.to_le_bytes()); + let mut deps: Vec<_> = pkg.dependencies.iter().collect(); + deps.sort(); + for (req, id) in deps { + append_string(&mut bytes, req); + let id = ids_to_stored_ids.get(&id).unwrap(); + bytes.extend_from_slice(&id.to_le_bytes()); + } + } + + bytes +} + +fn deserialize_npm_snapshot( + input: &[u8], +) -> Result { + fn parse_id(input: &[u8]) -> Result<(&[u8], NpmPackageId), AnyError> { + let (input, id) = read_string_lossy(input)?; + let id = NpmPackageId::from_serialized(&id)?; + Ok((input, id)) + } + + #[allow(clippy::needless_lifetimes)] // clippy bug + fn parse_root_package<'a>( + id_to_npm_id: &'a impl Fn(usize) -> Result, + ) -> impl Fn(&[u8]) -> Result<(&[u8], (PackageReq, NpmPackageId)), AnyError> + 'a + { + |input| { + let (input, req) = read_string_lossy(input)?; + let req = PackageReq::from_str(&req)?; + let (input, id) = read_u32_as_usize(input)?; + Ok((input, (req, id_to_npm_id(id)?))) + } + } + + #[allow(clippy::needless_lifetimes)] // clippy bug + fn parse_package_dep<'a>( + id_to_npm_id: &'a impl Fn(usize) -> Result, + ) -> impl Fn(&[u8]) -> Result<(&[u8], (String, NpmPackageId)), AnyError> + 'a + { + |input| { + let (input, req) = read_string_lossy(input)?; + let (input, id) = read_u32_as_usize(input)?; + Ok((input, (req.into_owned(), id_to_npm_id(id)?))) + } + } + + fn parse_package<'a>( + input: &'a [u8], + id: NpmPackageId, + id_to_npm_id: &impl Fn(usize) -> Result, + ) -> Result<(&'a [u8], SerializedNpmResolutionSnapshotPackage), AnyError> { + let (input, deps_len) = read_u32_as_usize(input)?; + let (input, dependencies) = + parse_hashmap_n_times(input, deps_len, parse_package_dep(id_to_npm_id))?; + Ok(( + input, + SerializedNpmResolutionSnapshotPackage { + id, + system: Default::default(), + dist: Default::default(), + dependencies, + optional_dependencies: Default::default(), + bin: None, + scripts: Default::default(), + deprecated: Default::default(), + }, + )) + } + + let (input, packages_len) = read_u32_as_usize(input)?; + + // get a hashmap of all the npm package ids to their serialized ids + let (input, data_ids_to_npm_ids) = + parse_vec_n_times(input, packages_len, parse_id) + .context("deserializing id")?; + let data_id_to_npm_id = |id: usize| { + data_ids_to_npm_ids + .get(id) + .cloned() + .ok_or_else(|| deno_core::anyhow::anyhow!("Invalid npm package id")) + }; + + let (input, root_packages_len) = read_u32_as_usize(input)?; + let (input, root_packages) = parse_hashmap_n_times( + input, + root_packages_len, + parse_root_package(&data_id_to_npm_id), + ) + .context("deserializing root package")?; + let (input, packages) = + parse_vec_n_times_with_index(input, packages_len, |input, index| { + parse_package(input, data_id_to_npm_id(index)?, &data_id_to_npm_id) + }) + .context("deserializing package")?; + + if !input.is_empty() { + bail!("Unexpected data left over"); + } + + Ok( + SerializedNpmResolutionSnapshot { + packages, + root_packages, + } + // this is ok because we have already verified that all the + // identifiers found in the snapshot are valid via the + // npm package id -> npm package id mapping + .into_valid_unsafe(), + ) +} + +fn serialize_media_type(media_type: MediaType) -> u8 { + match media_type { + MediaType::JavaScript => 0, + MediaType::Jsx => 1, + MediaType::Mjs => 2, + MediaType::Cjs => 3, + MediaType::TypeScript => 4, + MediaType::Mts => 5, + MediaType::Cts => 6, + MediaType::Dts => 7, + MediaType::Dmts => 8, + MediaType::Dcts => 9, + MediaType::Tsx => 10, + MediaType::Json => 11, + MediaType::Wasm => 12, + MediaType::TsBuildInfo => 13, + MediaType::SourceMap => 14, + MediaType::Unknown => 15, + } +} + +fn deserialize_media_type(value: u8) -> Result { + match value { + 0 => Ok(MediaType::JavaScript), + 1 => Ok(MediaType::Jsx), + 2 => Ok(MediaType::Mjs), + 3 => Ok(MediaType::Cjs), + 4 => Ok(MediaType::TypeScript), + 5 => Ok(MediaType::Mts), + 6 => Ok(MediaType::Cts), + 7 => Ok(MediaType::Dts), + 8 => Ok(MediaType::Dmts), + 9 => Ok(MediaType::Dcts), + 10 => Ok(MediaType::Tsx), + 11 => Ok(MediaType::Json), + 12 => Ok(MediaType::Wasm), + 13 => Ok(MediaType::TsBuildInfo), + 14 => Ok(MediaType::SourceMap), + 15 => Ok(MediaType::Unknown), + _ => bail!("Unknown media type value: {}", value), + } +} + +fn parse_hashmap_n_times( + mut input: &[u8], + times: usize, + parse: impl Fn(&[u8]) -> Result<(&[u8], (TKey, TValue)), AnyError>, +) -> Result<(&[u8], HashMap), AnyError> { + let mut results = HashMap::with_capacity(times); + for _ in 0..times { + let result = parse(input); + let (new_input, (key, value)) = result?; + results.insert(key, value); + input = new_input; + } + Ok((input, results)) +} + +fn parse_vec_n_times( + input: &[u8], + times: usize, + parse: impl Fn(&[u8]) -> Result<(&[u8], TResult), AnyError>, +) -> Result<(&[u8], Vec), AnyError> { + parse_vec_n_times_with_index(input, times, |input, _index| parse(input)) +} + +fn parse_vec_n_times_with_index( + mut input: &[u8], + times: usize, + parse: impl Fn(&[u8], usize) -> Result<(&[u8], TResult), AnyError>, +) -> Result<(&[u8], Vec), AnyError> { + let mut results = Vec::with_capacity(times); + for i in 0..times { + let result = parse(input, i); + let (new_input, result) = result?; + results.push(result); + input = new_input; + } + Ok((input, results)) +} + +fn read_bytes(input: &[u8], len: usize) -> Result<(&[u8], &[u8]), AnyError> { + if input.len() < len { + bail!("Unexpected end of data.",); + } + let (len_bytes, input) = input.split_at(len); + Ok((input, len_bytes)) +} + +fn read_string_lossy(input: &[u8]) -> Result<(&[u8], Cow), AnyError> { + let (input, str_len) = read_u32_as_usize(input)?; + let (input, data_bytes) = read_bytes(input, str_len)?; + Ok((input, String::from_utf8_lossy(data_bytes))) +} + +fn read_u32_as_usize(input: &[u8]) -> Result<(&[u8], usize), AnyError> { + let (input, len_bytes) = read_bytes(input, 4)?; + let len = u32::from_le_bytes(len_bytes.try_into()?); + Ok((input, len as usize)) +} + +fn read_u64(input: &[u8]) -> Result<(&[u8], u64), AnyError> { + let (input, len_bytes) = read_bytes(input, 8)?; + let len = u64::from_le_bytes(len_bytes.try_into()?); + Ok((input, len)) +} diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 53d045b625..0ae00accbf 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -7,6 +7,7 @@ use std::fs::File; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; +use std::ops::Range; use std::path::Path; use std::path::PathBuf; use std::rc::Rc; @@ -67,6 +68,26 @@ impl VfsBuilder { }) } + pub fn set_new_root_path( + &mut self, + root_path: PathBuf, + ) -> Result<(), AnyError> { + let root_path = canonicalize_path(&root_path)?; + self.root_path = root_path; + self.root_dir = VirtualDirectory { + name: self + .root_path + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or("root".to_string()), + entries: vec![VfsEntry::Dir(VirtualDirectory { + name: std::mem::take(&mut self.root_dir.name), + entries: std::mem::take(&mut self.root_dir.entries), + })], + }; + Ok(()) + } + pub fn with_root_dir( &mut self, with_root: impl FnOnce(&mut VirtualDirectory) -> R, @@ -119,7 +140,7 @@ impl VfsBuilder { // inline the symlink and make the target file let file_bytes = std::fs::read(&target) .with_context(|| format!("Reading {}", path.display()))?; - self.add_file(&path, file_bytes)?; + self.add_file_with_data_inner(&path, file_bytes)?; } else { log::warn!( "{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.", @@ -191,16 +212,32 @@ impl VfsBuilder { self.add_file_at_path_not_symlink(&target_path) } - pub fn add_file_at_path_not_symlink( + fn add_file_at_path_not_symlink( &mut self, path: &Path, ) -> Result<(), AnyError> { let file_bytes = std::fs::read(path) .with_context(|| format!("Reading {}", path.display()))?; - self.add_file(path, file_bytes) + self.add_file_with_data_inner(path, file_bytes) } - fn add_file(&mut self, path: &Path, data: Vec) -> Result<(), AnyError> { + pub fn add_file_with_data( + &mut self, + path: &Path, + data: Vec, + ) -> Result<(), AnyError> { + let target_path = canonicalize_path(path)?; + if target_path != path { + self.add_symlink(path, &target_path)?; + } + self.add_file_with_data_inner(&target_path, data) + } + + fn add_file_with_data_inner( + &mut self, + path: &Path, + data: Vec, + ) -> Result<(), AnyError> { log::debug!("Adding file '{}'", path.display()); let checksum = util::checksum::gen(&[&data]); let offset = if let Some(offset) = self.file_offsets.get(&checksum) { @@ -249,8 +286,15 @@ impl VfsBuilder { path.display(), target.display() ); - let dest = self.path_relative_root(target)?; - if dest == self.path_relative_root(path)? { + let relative_target = self.path_relative_root(target)?; + let relative_path = match self.path_relative_root(path) { + Ok(path) => path, + Err(StripRootError { .. }) => { + // ignore if the original path is outside the root directory + return Ok(()); + } + }; + if relative_target == relative_path { // it's the same, ignore return Ok(()); } @@ -263,7 +307,7 @@ impl VfsBuilder { insert_index, VfsEntry::Symlink(VirtualSymlink { name: name.to_string(), - dest_parts: dest + dest_parts: relative_target .components() .map(|c| c.as_os_str().to_string_lossy().to_string()) .collect::>(), @@ -751,14 +795,14 @@ impl deno_io::fs::File for FileBackedVfsFile { #[derive(Debug)] pub struct FileBackedVfs { - file: Mutex>, + vfs_data: Cow<'static, [u8]>, fs_root: VfsRoot, } impl FileBackedVfs { - pub fn new(file: Vec, fs_root: VfsRoot) -> Self { + pub fn new(data: Cow<'static, [u8]>, fs_root: VfsRoot) -> Self { Self { - file: Mutex::new(file), + vfs_data: data, fs_root, } } @@ -827,10 +871,15 @@ impl FileBackedVfs { Ok(path) } - pub fn read_file_all(&self, file: &VirtualFile) -> std::io::Result> { - let mut buf = vec![0; file.len as usize]; - self.read_file(file, 0, &mut buf)?; - Ok(buf) + pub fn read_file_all( + &self, + file: &VirtualFile, + ) -> std::io::Result> { + let read_range = self.get_read_range(file, 0, file.len)?; + match &self.vfs_data { + Cow::Borrowed(data) => Ok(Cow::Borrowed(&data[read_range])), + Cow::Owned(data) => Ok(Cow::Owned(data[read_range].to_vec())), + } } pub fn read_file( @@ -839,18 +888,27 @@ impl FileBackedVfs { pos: u64, buf: &mut [u8], ) -> std::io::Result { - let data = self.file.lock(); + let read_range = self.get_read_range(file, pos, buf.len() as u64)?; + buf.copy_from_slice(&self.vfs_data[read_range]); + Ok(buf.len()) + } + + fn get_read_range( + &self, + file: &VirtualFile, + pos: u64, + len: u64, + ) -> std::io::Result> { + let data = &self.vfs_data; let start = self.fs_root.start_file_offset + file.offset + pos; - let end = start + buf.len() as u64; + let end = start + len; if end > data.len() as u64 { return Err(std::io::Error::new( std::io::ErrorKind::UnexpectedEof, "unexpected EOF", )); } - - buf.copy_from_slice(&data[start as usize..end as usize]); - Ok(buf.len()) + Ok(start as usize..end as usize) } pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> { @@ -888,7 +946,7 @@ mod test { #[track_caller] fn read_file(vfs: &FileBackedVfs, path: &Path) -> String { let file = vfs.file_entry(path).unwrap(); - String::from_utf8(vfs.read_file_all(file).unwrap()).unwrap() + String::from_utf8(vfs.read_file_all(file).unwrap().into_owned()).unwrap() } #[test] @@ -901,20 +959,23 @@ mod test { let src_path = src_path.to_path_buf(); let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); builder - .add_file(&src_path.join("a.txt"), "data".into()) + .add_file_with_data_inner(&src_path.join("a.txt"), "data".into()) .unwrap(); builder - .add_file(&src_path.join("b.txt"), "data".into()) + .add_file_with_data_inner(&src_path.join("b.txt"), "data".into()) .unwrap(); assert_eq!(builder.files.len(), 1); // because duplicate data builder - .add_file(&src_path.join("c.txt"), "c".into()) + .add_file_with_data_inner(&src_path.join("c.txt"), "c".into()) .unwrap(); builder - .add_file(&src_path.join("sub_dir").join("d.txt"), "d".into()) + .add_file_with_data_inner( + &src_path.join("sub_dir").join("d.txt"), + "d".into(), + ) .unwrap(); builder - .add_file(&src_path.join("e.txt"), "e".into()) + .add_file_with_data_inner(&src_path.join("e.txt"), "e".into()) .unwrap(); builder .add_symlink( @@ -1031,7 +1092,7 @@ mod test { ( dest_path.to_path_buf(), FileBackedVfs::new( - data, + Cow::Owned(data), VfsRoot { dir: root_dir, root_path: dest_path.to_path_buf(), @@ -1082,7 +1143,7 @@ mod test { let temp_path = temp_dir.path().canonicalize(); let mut builder = VfsBuilder::new(temp_path.to_path_buf()).unwrap(); builder - .add_file( + .add_file_with_data_inner( temp_path.join("a.txt").as_path(), "0123456789".to_string().into_bytes(), ) diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index 3cc4414fcb..5a4a938bb9 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -5,6 +5,7 @@ use crate::args::CompileFlags; use crate::args::Flags; use crate::factory::CliFactory; use crate::http_util::HttpClientProvider; +use crate::standalone::binary::StandaloneRelativeFileBaseUrl; use crate::standalone::is_standalone_binary; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; @@ -14,7 +15,6 @@ use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_graph::GraphKind; use deno_terminal::colors; -use eszip::EszipRelativeFileBaseUrl; use rand::Rng; use std::path::Path; use std::path::PathBuf; @@ -29,7 +29,6 @@ pub async fn compile( let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; let module_graph_creator = factory.module_graph_creator().await?; - let parsed_source_cache = factory.parsed_source_cache(); let binary_writer = factory.create_compile_binary_writer().await?; let http_client = factory.http_client_provider(); let module_specifier = cli_options.resolve_main_module()?; @@ -80,7 +79,7 @@ pub async fn compile( let graph = if cli_options.type_check_mode().is_true() { // In this case, the previous graph creation did type checking, which will // create a module graph with types information in it. We don't want to - // store that in the eszip so create a code only module graph from scratch. + // store that in the binary so create a code only module graph from scratch. module_graph_creator .create_graph(GraphKind::CodeOnly, module_roots) .await? @@ -91,11 +90,6 @@ pub async fn compile( let ts_config_for_emit = cli_options .resolve_ts_config_for_emit(deno_config::deno_json::TsConfigType::Emit)?; check_warn_tsconfig(&ts_config_for_emit); - let (transpile_options, emit_options) = - crate::args::ts_config_to_transpile_and_emit_options( - ts_config_for_emit.ts_config, - )?; - let parser = parsed_source_cache.as_capturing_parser(); let root_dir_url = resolve_root_dir_from_specifiers( cli_options.workspace().root_dir(), graph.specifiers().map(|(s, _)| s).chain( @@ -106,17 +100,6 @@ pub async fn compile( ), ); log::debug!("Binary root dir: {}", root_dir_url); - let root_dir_url = EszipRelativeFileBaseUrl::new(&root_dir_url); - let eszip = eszip::EszipV2::from_graph(eszip::FromGraphOptions { - graph, - parser, - transpile_options, - emit_options, - // make all the modules relative to the root folder - relative_file_base: Some(root_dir_url), - npm_packages: None, - })?; - log::info!( "{} {} to {}", colors::green("Compile"), @@ -143,15 +126,18 @@ pub async fn compile( let write_result = binary_writer .write_bin( file, - eszip, - root_dir_url, + &graph, + StandaloneRelativeFileBaseUrl::from(&root_dir_url), module_specifier, &compile_flags, cli_options, ) .await .with_context(|| { - format!("Writing temporary file '{}'", temp_path.display()) + format!( + "Writing deno compile executable to temporary file '{}'", + temp_path.display() + ) }); // set it as executable diff --git a/cli/util/text_encoding.rs b/cli/util/text_encoding.rs index 0b7601cb9c..df72cc2be6 100644 --- a/cli/util/text_encoding.rs +++ b/cli/util/text_encoding.rs @@ -103,6 +103,21 @@ pub fn arc_str_to_bytes(arc_str: Arc) -> Arc<[u8]> { unsafe { Arc::from_raw(raw as *const [u8]) } } +/// Converts an `Arc` to an `Arc` if able. +#[allow(dead_code)] +pub fn arc_u8_to_arc_str( + arc_u8: Arc<[u8]>, +) -> Result, std::str::Utf8Error> { + // Check that the string is valid UTF-8. + std::str::from_utf8(&arc_u8)?; + // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as + // Arc. This is proven by the From> impl for Arc<[u8]> from the + // standard library. + Ok(unsafe { + std::mem::transmute::, std::sync::Arc>(arc_u8) + }) +} + #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/tests/specs/compile/byonm_main_sub_dir/deno.json b/tests/specs/compile/byonm_main_sub_dir/deno.json index 6134d86d1c..fde86a1efb 100644 --- a/tests/specs/compile/byonm_main_sub_dir/deno.json +++ b/tests/specs/compile/byonm_main_sub_dir/deno.json @@ -1,3 +1,3 @@ { - "unstable": ["byonm"] + "nodeModulesDir": "manual" } diff --git a/tests/specs/compile/detect_cjs/output.out b/tests/specs/compile/detect_cjs/output.out index b53c443698..e1c27b8dcf 100644 --- a/tests/specs/compile/detect_cjs/output.out +++ b/tests/specs/compile/detect_cjs/output.out @@ -1 +1,2 @@ -error: Module not found: file:///[WILDLINE]/add.js +error: Uncaught SyntaxError: The requested module './add.js' does not provide an export named 'add' + at (file:///[WILDLINE]) From 09e36d41b007604378525828047085cbaa7b948c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 24 Oct 2024 20:55:38 +0100 Subject: [PATCH 15/63] fix: add 'fmt-component' to unstable features in schema file (#26526) Closes https://github.com/denoland/deno/issues/26510 --- cli/schemas/config-file.v1.json | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/schemas/config-file.v1.json b/cli/schemas/config-file.v1.json index db40e5062b..27c8499ea2 100644 --- a/cli/schemas/config-file.v1.json +++ b/cli/schemas/config-file.v1.json @@ -531,6 +531,7 @@ "detect-cjs", "ffi", "fs", + "fmt-component", "http", "kv", "net", From fd8bf0827104407dca12fa57259a0d32380adcd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 24 Oct 2024 22:19:58 +0100 Subject: [PATCH 16/63] bench: remove http benchmarks (#26484) These benchmarks run on GitHub Actions and are extremely noisy, thus not providing much value. --- cli/bench/http.rs | 167 --------------------- cli/bench/http/deno_flash_hono_router.js | 10 -- cli/bench/http/deno_flash_send_file.js | 14 -- cli/bench/http/deno_http_read_headers.lua | 5 - cli/bench/http/deno_http_serve.js | 11 -- cli/bench/http/deno_post_bin.lua | 5 - cli/bench/http/deno_post_json.lua | 3 - cli/bench/http/deno_reactdom_ssr_flash.jsx | 25 --- cli/bench/http/deno_tcp.ts | 34 ----- cli/bench/main.rs | 19 +-- 10 files changed, 2 insertions(+), 291 deletions(-) delete mode 100644 cli/bench/http.rs delete mode 100644 cli/bench/http/deno_flash_hono_router.js delete mode 100644 cli/bench/http/deno_flash_send_file.js delete mode 100644 cli/bench/http/deno_http_read_headers.lua delete mode 100644 cli/bench/http/deno_http_serve.js delete mode 100644 cli/bench/http/deno_post_bin.lua delete mode 100644 cli/bench/http/deno_post_json.lua delete mode 100644 cli/bench/http/deno_reactdom_ssr_flash.jsx delete mode 100644 cli/bench/http/deno_tcp.ts diff --git a/cli/bench/http.rs b/cli/bench/http.rs deleted file mode 100644 index f739b76ba8..0000000000 --- a/cli/bench/http.rs +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use std::collections::HashMap; -use std::net::TcpStream; -use std::path::Path; -use std::process::Command; -use std::sync::atomic::AtomicU16; -use std::sync::atomic::Ordering; -use std::time::Duration; -use std::time::Instant; - -use super::Result; - -pub use test_util::parse_wrk_output; -pub use test_util::WrkOutput as HttpBenchmarkResult; -// Some of the benchmarks in this file have been renamed. In case the history -// somehow gets messed up: -// "node_http" was once called "node" -// "deno_tcp" was once called "deno" -// "deno_http" was once called "deno_net_http" - -const DURATION: &str = "10s"; - -pub fn benchmark( - target_path: &Path, -) -> Result> { - let deno_exe = test_util::deno_exe_path(); - let deno_exe = deno_exe.to_string(); - - let hyper_hello_exe = target_path.join("test_server"); - let hyper_hello_exe = hyper_hello_exe.to_str().unwrap(); - - let mut res = HashMap::new(); - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); - let http_dir = manifest_dir.join("bench").join("http"); - for entry in std::fs::read_dir(&http_dir)? { - let entry = entry?; - let pathbuf = entry.path(); - let path = pathbuf.to_str().unwrap(); - if path.ends_with(".lua") { - continue; - } - let file_stem = pathbuf.file_stem().unwrap().to_str().unwrap(); - - let lua_script = http_dir.join(format!("{file_stem}.lua")); - let mut maybe_lua = None; - if lua_script.exists() { - maybe_lua = Some(lua_script.to_str().unwrap()); - } - - let port = get_port(); - // deno run -A --unstable-net - res.insert( - file_stem.to_string(), - run( - &[ - deno_exe.as_str(), - "run", - "--allow-all", - "--unstable-net", - "--enable-testing-features-do-not-use", - path, - &server_addr(port), - ], - port, - None, - None, - maybe_lua, - )?, - ); - } - - res.insert("hyper".to_string(), hyper_http(hyper_hello_exe)?); - - Ok(res) -} - -fn run( - server_cmd: &[&str], - port: u16, - env: Option>, - origin_cmd: Option<&[&str]>, - lua_script: Option<&str>, -) -> Result { - // Wait for port 4544 to become available. - // TODO Need to use SO_REUSEPORT with tokio::net::TcpListener. - std::thread::sleep(Duration::from_secs(5)); - - let mut origin = None; - if let Some(cmd) = origin_cmd { - let mut com = Command::new(cmd[0]); - com.args(&cmd[1..]); - if let Some(env) = env.clone() { - com.envs(env); - } - origin = Some(com.spawn()?); - }; - - println!("{}", server_cmd.join(" ")); - let mut server = { - let mut com = Command::new(server_cmd[0]); - com.args(&server_cmd[1..]); - if let Some(env) = env { - com.envs(env); - } - com.spawn()? - }; - - // Wait for server to wake up. - let now = Instant::now(); - let addr = format!("127.0.0.1:{port}"); - while now.elapsed().as_secs() < 30 { - if TcpStream::connect(&addr).is_ok() { - break; - } - std::thread::sleep(Duration::from_millis(10)); - } - TcpStream::connect(&addr).expect("Failed to connect to server in time"); - println!("Server took {} ms to start", now.elapsed().as_millis()); - - let wrk = test_util::prebuilt_tool_path("wrk"); - assert!(wrk.is_file()); - - let addr = format!("http://{addr}/"); - let wrk = wrk.to_string(); - let mut wrk_cmd = vec![wrk.as_str(), "-d", DURATION, "--latency", &addr]; - - if let Some(lua_script) = lua_script { - wrk_cmd.push("-s"); - wrk_cmd.push(lua_script); - } - - println!("{}", wrk_cmd.join(" ")); - let output = test_util::run_collect(&wrk_cmd, None, None, None, true).0; - - std::thread::sleep(Duration::from_secs(1)); // wait to capture failure. TODO racy. - - println!("{output}"); - assert!( - server.try_wait()?.map(|s| s.success()).unwrap_or(true), - "server ended with error" - ); - - server.kill()?; - if let Some(mut origin) = origin { - origin.kill()?; - } - - Ok(parse_wrk_output(&output)) -} - -static NEXT_PORT: AtomicU16 = AtomicU16::new(4544); -pub(crate) fn get_port() -> u16 { - let p = NEXT_PORT.load(Ordering::SeqCst); - NEXT_PORT.store(p.wrapping_add(1), Ordering::SeqCst); - p -} - -fn server_addr(port: u16) -> String { - format!("0.0.0.0:{port}") -} - -fn hyper_http(exe: &str) -> Result { - let port = get_port(); - println!("http_benchmark testing RUST hyper"); - run(&[exe, &port.to_string()], port, None, None, None) -} diff --git a/cli/bench/http/deno_flash_hono_router.js b/cli/bench/http/deno_flash_hono_router.js deleted file mode 100644 index baced0cece..0000000000 --- a/cli/bench/http/deno_flash_hono_router.js +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { Hono } from "https://deno.land/x/hono@v2.0.9/mod.ts"; - -const addr = Deno.args[0] || "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); - -const app = new Hono(); -app.get("/", (c) => c.text("Hello, World!")); - -Deno.serve({ port: Number(port), hostname }, app.fetch); diff --git a/cli/bench/http/deno_flash_send_file.js b/cli/bench/http/deno_flash_send_file.js deleted file mode 100644 index bf8541f8b0..0000000000 --- a/cli/bench/http/deno_flash_send_file.js +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -const addr = Deno.args[0] || "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); -const { serve } = Deno; - -const path = new URL("../testdata/128k.bin", import.meta.url).pathname; - -function handler() { - const file = Deno.openSync(path); - return new Response(file.readable); -} - -serve({ hostname, port: Number(port) }, handler); diff --git a/cli/bench/http/deno_http_read_headers.lua b/cli/bench/http/deno_http_read_headers.lua deleted file mode 100644 index 64f1923ff3..0000000000 --- a/cli/bench/http/deno_http_read_headers.lua +++ /dev/null @@ -1,5 +0,0 @@ -wrk.headers["foo"] = "bar" -wrk.headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36" -wrk.headers["Viewport-Width"] = "1920" -wrk.headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -wrk.headers["Accept-Language"] = "en,la;q=0.9" \ No newline at end of file diff --git a/cli/bench/http/deno_http_serve.js b/cli/bench/http/deno_http_serve.js deleted file mode 100644 index 639982ce60..0000000000 --- a/cli/bench/http/deno_http_serve.js +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -const addr = Deno.args[0] ?? "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); -const { serve } = Deno; - -function handler() { - return new Response("Hello World"); -} - -serve({ hostname, port: Number(port), reusePort: true }, handler); diff --git a/cli/bench/http/deno_post_bin.lua b/cli/bench/http/deno_post_bin.lua deleted file mode 100644 index c8f5d3e3f7..0000000000 --- a/cli/bench/http/deno_post_bin.lua +++ /dev/null @@ -1,5 +0,0 @@ -wrk.method = "POST" -wrk.headers["Content-Type"] = "application/octet-stream" - -file = io.open("./cli/bench/testdata/128k.bin", "rb") -wrk.body = file:read("*a") \ No newline at end of file diff --git a/cli/bench/http/deno_post_json.lua b/cli/bench/http/deno_post_json.lua deleted file mode 100644 index cc6c4e226d..0000000000 --- a/cli/bench/http/deno_post_json.lua +++ /dev/null @@ -1,3 +0,0 @@ -wrk.method = "POST" -wrk.headers["Content-Type"] = "application/json" -wrk.body = '{"hello":"deno"}' \ No newline at end of file diff --git a/cli/bench/http/deno_reactdom_ssr_flash.jsx b/cli/bench/http/deno_reactdom_ssr_flash.jsx deleted file mode 100644 index eaabf89121..0000000000 --- a/cli/bench/http/deno_reactdom_ssr_flash.jsx +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -import { renderToReadableStream } from "https://esm.run/react-dom/server"; -import * as React from "https://esm.run/react"; -const { serve } = Deno; -const addr = Deno.args[0] || "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); - -const App = () => ( - - -

Hello World

- - -); - -const headers = { - headers: { - "Content-Type": "text/html", - }, -}; - -serve({ hostname, port: Number(port) }, async () => { - return new Response(await renderToReadableStream(), headers); -}); diff --git a/cli/bench/http/deno_tcp.ts b/cli/bench/http/deno_tcp.ts deleted file mode 100644 index b795910737..0000000000 --- a/cli/bench/http/deno_tcp.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -// Used for benchmarking Deno's networking. -// TODO(bartlomieju): Replace this with a real HTTP server once -// https://github.com/denoland/deno/issues/726 is completed. -// Note: this is a keep-alive server. -// deno-lint-ignore-file no-console -const addr = Deno.args[0] || "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); -const listener = Deno.listen({ hostname, port: Number(port) }); -const response = new TextEncoder().encode( - "HTTP/1.1 200 OK\r\nContent-Length: 12\r\n\r\nHello World\n", -); -async function handle(conn: Deno.Conn): Promise { - const buffer = new Uint8Array(1024); - try { - while (true) { - await conn.read(buffer); - await conn.write(response); - } - } catch (e) { - if ( - !(e instanceof Deno.errors.BrokenPipe) && - !(e instanceof Deno.errors.ConnectionReset) - ) { - throw e; - } - } - conn.close(); -} - -console.log("Listening on", addr); -for await (const conn of listener) { - handle(conn); -} diff --git a/cli/bench/main.rs b/cli/bench/main.rs index 72fa7e9636..c3c42d2488 100644 --- a/cli/bench/main.rs +++ b/cli/bench/main.rs @@ -17,7 +17,6 @@ use std::process::Stdio; use std::time::SystemTime; use test_util::PathRef; -mod http; mod lsp; fn read_json(filename: &Path) -> Result { @@ -345,9 +344,11 @@ struct BenchResult { binary_size: HashMap, bundle_size: HashMap, cargo_deps: usize, + // TODO(bartlomieju): remove max_latency: HashMap, max_memory: HashMap, lsp_exec_time: HashMap, + // TODO(bartlomieju): remove req_per_sec: HashMap, syscall_count: HashMap, thread_count: HashMap, @@ -362,7 +363,6 @@ async fn main() -> Result<()> { "binary_size", "cargo_deps", "lsp", - "http", "strace", "mem_usage", ]; @@ -427,21 +427,6 @@ async fn main() -> Result<()> { new_data.lsp_exec_time = lsp_exec_times; } - if benchmarks.contains(&"http") && cfg!(not(target_os = "windows")) { - let stats = http::benchmark(target_dir.as_path())?; - let req_per_sec = stats - .iter() - .map(|(name, result)| (name.clone(), result.requests as i64)) - .collect(); - new_data.req_per_sec = req_per_sec; - let max_latency = stats - .iter() - .map(|(name, result)| (name.clone(), result.latency)) - .collect(); - - new_data.max_latency = max_latency; - } - if cfg!(target_os = "linux") && benchmarks.contains(&"strace") { use std::io::Read; From 8dd6177c624649d75ffcacca77e7c4f48cea07a2 Mon Sep 17 00:00:00 2001 From: Nicola Bovolato <61934734+nicolabovolato@users.noreply.github.com> Date: Fri, 25 Oct 2024 00:02:26 +0200 Subject: [PATCH 17/63] fix(ext/node): refactor http.ServerResponse into function class (#26210) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While testing, I found out that light-my-request relies on `ServerResponse.connection`, which is deprecated, so I added that and `socket`, the non deprecated property. It also relies on an undocumented `_header` property, apparently for [raw header processing](https://github.com/fastify/light-my-request/blob/v6.1.0/lib/response.js#L180-L186). I added it as an empty string, feel free to provide other approaches. Fixes #19901 Co-authored-by: Bartek Iwańczuk --- ext/node/polyfills/http.ts | 534 +++++++++++++++++++++-------------- tests/unit_node/http_test.ts | 80 +++++- 2 files changed, 407 insertions(+), 207 deletions(-) diff --git a/ext/node/polyfills/http.ts b/ext/node/polyfills/http.ts index 20bef30099..e7a860ff1c 100644 --- a/ext/node/polyfills/http.ts +++ b/ext/node/polyfills/http.ts @@ -34,6 +34,7 @@ import { finished, Readable as NodeReadable, Writable as NodeWritable, + WritableOptions as NodeWritableOptions, } from "node:stream"; import { kUniqueHeaders, @@ -70,6 +71,7 @@ import { resourceForReadableStream } from "ext:deno_web/06_streams.js"; import { UpgradedConn } from "ext:deno_net/01_net.js"; import { STATUS_CODES } from "node:_http_server"; import { methods as METHODS } from "node:_http_common"; +import { deprecate } from "node:util"; const { internalRidSymbol } = core; const { ArrayIsArray, StringPrototypeToLowerCase } = primordials; @@ -1184,49 +1186,95 @@ function onError(self, error, cb) { } } -export class ServerResponse extends NodeWritable { - statusCode = 200; - statusMessage?: string = undefined; - #headers: Record = { __proto__: null }; - #hasNonStringHeaders: boolean = false; - #readable: ReadableStream; - override writable = true; - // used by `npm:on-finished` - finished = false; - headersSent = false; - #resolve: (value: Response | PromiseLike) => void; +export type ServerResponse = { + statusCode: number; + statusMessage?: string; + + _headers: Record; + _hasNonStringHeaders: boolean; + + _readable: ReadableStream; + finished: boolean; + headersSent: boolean; + _resolve: (value: Response | PromiseLike) => void; // deno-lint-ignore no-explicit-any - #socketOverride: any | null = null; + _socketOverride: any | null; + // deno-lint-ignore no-explicit-any + socket: any | null; - static #enqueue(controller: ReadableStreamDefaultController, chunk: Chunk) { - try { - if (typeof chunk === "string") { - controller.enqueue(ENCODER.encode(chunk)); - } else { - controller.enqueue(chunk); - } - } catch (_) { - // The stream might have been closed. Ignore the error. - } - } + setHeader(name: string, value: string | string[]): void; + appendHeader(name: string, value: string | string[]): void; + getHeader(name: string): string | string[]; + removeHeader(name: string): void; + getHeaderNames(): string[]; + getHeaders(): Record; + hasHeader(name: string): boolean; - /** Returns true if the response body should be null with the given - * http status code */ - static #bodyShouldBeNull(status: number) { - return status === 101 || status === 204 || status === 205 || status === 304; - } + writeHead( + status: number, + statusMessage?: string, + headers?: + | Record + | Array<[string, string]>, + ): void; + writeHead( + status: number, + headers?: + | Record + | Array<[string, string]>, + ): void; - constructor( + _ensureHeaders(singleChunk?: Chunk): void; + + respond(final: boolean, singleChunk?: Chunk): void; + // deno-lint-ignore no-explicit-any + end(chunk?: any, encoding?: any, cb?: any): void; + + flushHeaders(): void; + _implicitHeader(): void; + + // Undocumented field used by `npm:light-my-request`. + _header: string; + + assignSocket(socket): void; + detachSocket(socket): void; +} & { -readonly [K in keyof NodeWritable]: NodeWritable[K] }; + +type ServerResponseStatic = { + new ( resolve: (value: Response | PromiseLike) => void, socket: FakeSocket, - ) { - let controller: ReadableByteStreamController; - const readable = new ReadableStream({ - start(c) { - controller = c as ReadableByteStreamController; - }, - }); - super({ + ): ServerResponse; + _enqueue(controller: ReadableStreamDefaultController, chunk: Chunk): void; + _bodyShouldBeNull(statusCode: number): boolean; +}; + +export const ServerResponse = function ( + this: ServerResponse, + resolve: (value: Response | PromiseLike) => void, + socket: FakeSocket, +) { + this.statusCode = 200; + this.statusMessage = undefined; + this._headers = { __proto__: null }; + this._hasNonStringHeaders = false; + this.writable = true; + + // used by `npm:on-finished` + this.finished = false; + this.headersSent = false; + this._socketOverride = null; + + let controller: ReadableByteStreamController; + const readable = new ReadableStream({ + start(c) { + controller = c as ReadableByteStreamController; + }, + }); + + NodeWritable.call( + this, + { autoDestroy: true, defaultEncoding: "utf-8", emitClose: true, @@ -1235,16 +1283,16 @@ export class ServerResponse extends NodeWritable { write: (chunk, encoding, cb) => { // Writes chunks are directly written to the socket if // one is assigned via assignSocket() - if (this.#socketOverride && this.#socketOverride.writable) { - this.#socketOverride.write(chunk, encoding); + if (this._socketOverride && this._socketOverride.writable) { + this._socketOverride.write(chunk, encoding); return cb(); } if (!this.headersSent) { - ServerResponse.#enqueue(controller, chunk); + ServerResponse._enqueue(controller, chunk); this.respond(false); return cb(); } - ServerResponse.#enqueue(controller, chunk); + ServerResponse._enqueue(controller, chunk); return cb(); }, final: (cb) => { @@ -1260,193 +1308,269 @@ export class ServerResponse extends NodeWritable { } return cb(null); }, - }); - this.#readable = readable; - this.#resolve = resolve; - this.socket = socket; - } + } satisfies NodeWritableOptions, + ); - setHeader(name: string, value: string | string[]) { - if (Array.isArray(value)) { - this.#hasNonStringHeaders = true; - } - this.#headers[StringPrototypeToLowerCase(name)] = value; - return this; - } + this._readable = readable; + this._resolve = resolve; + this.socket = socket; - appendHeader(name: string, value: string | string[]) { - const key = StringPrototypeToLowerCase(name); - if (this.#headers[key] === undefined) { - if (Array.isArray(value)) this.#hasNonStringHeaders = true; - this.#headers[key] = value; + this._header = ""; +} as unknown as ServerResponseStatic; + +Object.setPrototypeOf(ServerResponse.prototype, NodeWritable.prototype); +Object.setPrototypeOf(ServerResponse, NodeWritable); + +ServerResponse._enqueue = function ( + this: ServerResponse, + controller: ReadableStreamDefaultController, + chunk: Chunk, +) { + try { + if (typeof chunk === "string") { + controller.enqueue(ENCODER.encode(chunk)); } else { - this.#hasNonStringHeaders = true; - if (!Array.isArray(this.#headers[key])) { - this.#headers[key] = [this.#headers[key]]; + controller.enqueue(chunk); + } + } catch (_) { + // The stream might have been closed. Ignore the error. + } +}; + +/** Returns true if the response body should be null with the given + * http status code */ +ServerResponse._bodyShouldBeNull = function ( + this: ServerResponse, + status: number, +) { + return status === 101 || status === 204 || status === 205 || status === 304; +}; + +ServerResponse.prototype.setHeader = function ( + this: ServerResponse, + name: string, + value: string | string[], +) { + if (Array.isArray(value)) { + this._hasNonStringHeaders = true; + } + this._headers[StringPrototypeToLowerCase(name)] = value; + return this; +}; + +ServerResponse.prototype.appendHeader = function ( + this: ServerResponse, + name: string, + value: string | string[], +) { + const key = StringPrototypeToLowerCase(name); + if (this._headers[key] === undefined) { + if (Array.isArray(value)) this._hasNonStringHeaders = true; + this._headers[key] = value; + } else { + this._hasNonStringHeaders = true; + if (!Array.isArray(this._headers[key])) { + this._headers[key] = [this._headers[key]]; + } + const header = this._headers[key]; + if (Array.isArray(value)) { + header.push(...value); + } else { + header.push(value); + } + } + return this; +}; + +ServerResponse.prototype.getHeader = function ( + this: ServerResponse, + name: string, +) { + return this._headers[StringPrototypeToLowerCase(name)]; +}; + +ServerResponse.prototype.removeHeader = function ( + this: ServerResponse, + name: string, +) { + delete this._headers[StringPrototypeToLowerCase(name)]; +}; + +ServerResponse.prototype.getHeaderNames = function (this: ServerResponse) { + return Object.keys(this._headers); +}; + +ServerResponse.prototype.getHeaders = function ( + this: ServerResponse, +): Record { + return { __proto__: null, ...this._headers }; +}; + +ServerResponse.prototype.hasHeader = function ( + this: ServerResponse, + name: string, +) { + return Object.hasOwn(this._headers, name); +}; + +ServerResponse.prototype.writeHead = function ( + this: ServerResponse, + status: number, + statusMessageOrHeaders?: + | string + | Record + | Array<[string, string]>, + maybeHeaders?: + | Record + | Array<[string, string]>, +) { + this.statusCode = status; + + let headers = null; + if (typeof statusMessageOrHeaders === "string") { + this.statusMessage = statusMessageOrHeaders; + if (maybeHeaders !== undefined) { + headers = maybeHeaders; + } + } else if (statusMessageOrHeaders !== undefined) { + headers = statusMessageOrHeaders; + } + + if (headers !== null) { + if (ArrayIsArray(headers)) { + headers = headers as Array<[string, string]>; + for (let i = 0; i < headers.length; i++) { + this.appendHeader(headers[i][0], headers[i][1]); } - const header = this.#headers[key]; - if (Array.isArray(value)) { - header.push(...value); - } else { - header.push(value); + } else { + headers = headers as Record; + for (const k in headers) { + if (Object.hasOwn(headers, k)) { + this.setHeader(k, headers[k]); + } } } - return this; } - getHeader(name: string) { - return this.#headers[StringPrototypeToLowerCase(name)]; - } - removeHeader(name: string) { - delete this.#headers[StringPrototypeToLowerCase(name)]; - } - getHeaderNames() { - return Object.keys(this.#headers); - } - getHeaders(): Record { - // @ts-ignore Ignore null __proto__ - return { __proto__: null, ...this.#headers }; - } - hasHeader(name: string) { - return Object.hasOwn(this.#headers, name); - } + return this; +}; - writeHead( - status: number, - statusMessage?: string, - headers?: - | Record - | Array<[string, string]>, - ): this; - writeHead( - status: number, - headers?: - | Record - | Array<[string, string]>, - ): this; - writeHead( - status: number, - statusMessageOrHeaders?: - | string - | Record - | Array<[string, string]>, - maybeHeaders?: - | Record - | Array<[string, string]>, - ): this { - this.statusCode = status; +ServerResponse.prototype._ensureHeaders = function ( + this: ServerResponse, + singleChunk?: Chunk, +) { + if (this.statusCode === 200 && this.statusMessage === undefined) { + this.statusMessage = "OK"; + } + if (typeof singleChunk === "string" && !this.hasHeader("content-type")) { + this.setHeader("content-type", "text/plain;charset=UTF-8"); + } +}; - let headers = null; - if (typeof statusMessageOrHeaders === "string") { - this.statusMessage = statusMessageOrHeaders; - if (maybeHeaders !== undefined) { - headers = maybeHeaders; - } - } else if (statusMessageOrHeaders !== undefined) { - headers = statusMessageOrHeaders; - } - - if (headers !== null) { - if (ArrayIsArray(headers)) { - headers = headers as Array<[string, string]>; - for (let i = 0; i < headers.length; i++) { - this.appendHeader(headers[i][0], headers[i][1]); +ServerResponse.prototype.respond = function ( + this: ServerResponse, + final: boolean, + singleChunk?: Chunk, +) { + this.headersSent = true; + this._ensureHeaders(singleChunk); + let body = singleChunk ?? (final ? null : this._readable); + if (ServerResponse._bodyShouldBeNull(this.statusCode)) { + body = null; + } + let headers: Record | [string, string][] = this + ._headers as Record; + if (this._hasNonStringHeaders) { + headers = []; + // Guard is not needed as this is a null prototype object. + // deno-lint-ignore guard-for-in + for (const key in this._headers) { + const entry = this._headers[key]; + if (Array.isArray(entry)) { + for (const value of entry) { + headers.push([key, value]); } } else { - headers = headers as Record; - for (const k in headers) { - if (Object.hasOwn(headers, k)) { - this.setHeader(k, headers[k]); - } - } + headers.push([key, entry]); } } - - return this; - } - - #ensureHeaders(singleChunk?: Chunk) { - if (this.statusCode === 200 && this.statusMessage === undefined) { - this.statusMessage = "OK"; - } - if ( - typeof singleChunk === "string" && - !this.hasHeader("content-type") - ) { - this.setHeader("content-type", "text/plain;charset=UTF-8"); - } - } - - respond(final: boolean, singleChunk?: Chunk) { - this.headersSent = true; - this.#ensureHeaders(singleChunk); - let body = singleChunk ?? (final ? null : this.#readable); - if (ServerResponse.#bodyShouldBeNull(this.statusCode)) { - body = null; - } - let headers: Record | [string, string][] = this - .#headers as Record; - if (this.#hasNonStringHeaders) { - headers = []; - // Guard is not needed as this is a null prototype object. - // deno-lint-ignore guard-for-in - for (const key in this.#headers) { - const entry = this.#headers[key]; - if (Array.isArray(entry)) { - for (const value of entry) { - headers.push([key, value]); - } - } else { - headers.push([key, entry]); - } - } - } - this.#resolve( - new Response(body, { - headers, - status: this.statusCode, - statusText: this.statusMessage, - }), - ); } + this._resolve( + new Response(body, { + headers, + status: this.statusCode, + statusText: this.statusMessage, + }), + ); +}; +ServerResponse.prototype.end = function ( + this: ServerResponse, // deno-lint-ignore no-explicit-any - override end(chunk?: any, encoding?: any, cb?: any): this { - this.finished = true; - if (!chunk && "transfer-encoding" in this.#headers) { - // FIXME(bnoordhuis) Node sends a zero length chunked body instead, i.e., - // the trailing "0\r\n", but respondWith() just hangs when I try that. - this.#headers["content-length"] = "0"; - delete this.#headers["transfer-encoding"]; - } - - // @ts-expect-error The signature for cb is stricter than the one implemented here - return super.end(chunk, encoding, cb); + chunk?: any, + // deno-lint-ignore no-explicit-any + encoding?: any, + // deno-lint-ignore no-explicit-any + cb?: any, +) { + this.finished = true; + if (!chunk && "transfer-encoding" in this._headers) { + // FIXME(bnoordhuis) Node sends a zero length chunked body instead, i.e., + // the trailing "0\r\n", but respondWith() just hangs when I try that. + this._headers["content-length"] = "0"; + delete this._headers["transfer-encoding"]; } - flushHeaders() { - // no-op - } + // @ts-expect-error The signature for cb is stricter than the one implemented here + NodeWritable.prototype.end.call(this, chunk, encoding, cb); +}; - // Undocumented API used by `npm:compression`. - _implicitHeader() { - this.writeHead(this.statusCode); - } +ServerResponse.prototype.flushHeaders = function (this: ServerResponse) { + // no-op +}; - assignSocket(socket) { - if (socket._httpMessage) { - throw new ERR_HTTP_SOCKET_ASSIGNED(); - } - socket._httpMessage = this; - this.#socketOverride = socket; - } +// Undocumented API used by `npm:compression`. +ServerResponse.prototype._implicitHeader = function (this: ServerResponse) { + this.writeHead(this.statusCode); +}; - detachSocket(socket) { - assert(socket._httpMessage === this); - socket._httpMessage = null; - this.#socketOverride = null; +ServerResponse.prototype.assignSocket = function ( + this: ServerResponse, + socket, +) { + if (socket._httpMessage) { + throw new ERR_HTTP_SOCKET_ASSIGNED(); } -} + socket._httpMessage = this; + this._socketOverride = socket; +}; + +ServerResponse.prototype.detachSocket = function ( + this: ServerResponse, + socket, +) { + assert(socket._httpMessage === this); + socket._httpMessage = null; + this._socketOverride = null; +}; + +Object.defineProperty(ServerResponse.prototype, "connection", { + get: deprecate( + function (this: ServerResponse) { + return this._socketOverride; + }, + "ServerResponse.prototype.connection is deprecated", + "DEP0066", + ), + set: deprecate( + // deno-lint-ignore no-explicit-any + function (this: ServerResponse, socket: any) { + this._socketOverride = socket; + }, + "ServerResponse.prototype.connection is deprecated", + "DEP0066", + ), +}); // TODO(@AaronO): optimize export class IncomingMessageForServer extends NodeReadable { diff --git a/tests/unit_node/http_test.ts b/tests/unit_node/http_test.ts index 0faf7fb345..84d6f57279 100644 --- a/tests/unit_node/http_test.ts +++ b/tests/unit_node/http_test.ts @@ -3,10 +3,14 @@ // deno-lint-ignore-file no-console import EventEmitter from "node:events"; -import http, { type RequestOptions, type ServerResponse } from "node:http"; +import http, { + IncomingMessage, + type RequestOptions, + ServerResponse, +} from "node:http"; import url from "node:url"; import https from "node:https"; -import net from "node:net"; +import net, { Socket } from "node:net"; import fs from "node:fs"; import { text } from "node:stream/consumers"; @@ -1704,3 +1708,75 @@ Deno.test("[node/http] upgraded socket closes when the server closed without clo await clientSocketClosed.promise; await serverProcessClosed.promise; }); + +// deno-lint-ignore require-await +Deno.test("[node/http] ServerResponse.call()", async () => { + function Wrapper(this: unknown, req: IncomingMessage) { + ServerResponse.call(this, req); + } + Object.setPrototypeOf(Wrapper.prototype, ServerResponse.prototype); + + // deno-lint-ignore no-explicit-any + const wrapper = new (Wrapper as any)(new IncomingMessage(new Socket())); + + assert(wrapper instanceof ServerResponse); +}); + +Deno.test("[node/http] ServerResponse _header", async () => { + const { promise, resolve } = Promise.withResolvers(); + const server = http.createServer((_req, res) => { + assert(Object.hasOwn(res, "_header")); + res.end(); + }); + + server.listen(async () => { + const { port } = server.address() as { port: number }; + const res = await fetch(`http://localhost:${port}`); + await res.body?.cancel(); + server.close(() => { + resolve(); + }); + }); + + await promise; +}); + +Deno.test("[node/http] ServerResponse connection", async () => { + const { promise, resolve } = Promise.withResolvers(); + const server = http.createServer((_req, res) => { + assert(Object.hasOwn(res, "connection")); + assert(res.connection instanceof Socket); + res.end(); + }); + + server.listen(async () => { + const { port } = server.address() as { port: number }; + const res = await fetch(`http://localhost:${port}`); + await res.body?.cancel(); + server.close(() => { + resolve(); + }); + }); + + await promise; +}); + +Deno.test("[node/http] ServerResponse socket", async () => { + const { promise, resolve } = Promise.withResolvers(); + const server = http.createServer((_req, res) => { + assert(Object.hasOwn(res, "socket")); + assert(res.socket instanceof Socket); + res.end(); + }); + + server.listen(async () => { + const { port } = server.address() as { port: number }; + const res = await fetch(`http://localhost:${port}`); + await res.body?.cancel(); + server.close(() => { + resolve(); + }); + }); + + await promise; +}); From e16230624747eccea7636b20b952df5899719336 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 24 Oct 2024 18:06:17 -0400 Subject: [PATCH 18/63] chore: remove print debugging from test server (#26529) Accidentally added in https://github.com/denoland/deno/pull/26473/files --- tests/util/server/src/builders.rs | 1 + tests/util/server/src/fs.rs | 5 ++++- tests/util/server/src/lib.rs | 18 ++++++++++++------ tests/util/server/src/lsp.rs | 18 +++++++++++++----- tests/util/server/src/npm.rs | 8 -------- tests/util/server/src/pty.rs | 18 ++++++++++++------ tests/util/server/src/servers/hyper_utils.rs | 8 +++++++- tests/util/server/src/servers/mod.rs | 14 +++++++++----- tests/util/server/src/servers/ws.rs | 7 +++++++ 9 files changed, 65 insertions(+), 32 deletions(-) diff --git a/tests/util/server/src/builders.rs b/tests/util/server/src/builders.rs index 4a4b6a7613..3bf5c704a1 100644 --- a/tests/util/server/src/builders.rs +++ b/tests/util/server/src/builders.rs @@ -78,6 +78,7 @@ impl DiagnosticLogger { logger.write_all(text.as_ref().as_bytes()).unwrap(); logger.write_all(b"\n").unwrap(); } + #[allow(clippy::print_stderr)] None => eprintln!("{}", text.as_ref()), } } diff --git a/tests/util/server/src/fs.rs b/tests/util/server/src/fs.rs index 47d0d61fa1..7feb0799ae 100644 --- a/tests/util/server/src/fs.rs +++ b/tests/util/server/src/fs.rs @@ -285,7 +285,10 @@ impl PathRef { #[track_caller] pub fn assert_matches_file(&self, wildcard_file: impl AsRef) -> &Self { let wildcard_file = testdata_path().join(wildcard_file); - println!("output path {}", wildcard_file); + #[allow(clippy::print_stdout)] + { + println!("output path {}", wildcard_file); + } let expected_text = wildcard_file.read_to_string(); self.assert_matches_text(&expected_text) } diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index e4a2cc02e3..67dbcd820c 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -1,8 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -#![allow(clippy::print_stdout)] -#![allow(clippy::print_stderr)] - use std::collections::HashMap; use std::env; use std::io::Write; @@ -302,7 +299,10 @@ async fn get_tcp_listener_stream( .collect::>(); // Eye catcher for HttpServerCount - println!("ready: {name} on {:?}", addresses); + #[allow(clippy::print_stdout)] + { + println!("ready: {name} on {:?}", addresses); + } futures::stream::select_all(listeners) } @@ -345,7 +345,10 @@ struct HttpServerStarter { impl Default for HttpServerStarter { fn default() -> Self { - println!("test_server starting..."); + #[allow(clippy::print_stdout)] + { + println!("test_server starting..."); + } let mut test_server = Command::new(test_server_path()) .current_dir(testdata_path()) .stdout(Stdio::piped()) @@ -360,7 +363,6 @@ impl Default for HttpServerStarter { let mut ready_count = 0; for maybe_line in lines { if let Ok(line) = maybe_line { - eprintln!("LINE: {}", line); if line.starts_with("ready:") { ready_count += 1; } @@ -480,6 +482,7 @@ pub fn run_collect( } = prog.wait_with_output().expect("failed to wait on child"); let stdout = String::from_utf8(stdout).unwrap(); let stderr = String::from_utf8(stderr).unwrap(); + #[allow(clippy::print_stderr)] if expect_success != status.success() { eprintln!("stdout: <<<{stdout}>>>"); eprintln!("stderr: <<<{stderr}>>>"); @@ -540,6 +543,7 @@ pub fn run_and_collect_output_with_args( } = deno.wait_with_output().expect("failed to wait on child"); let stdout = String::from_utf8(stdout).unwrap(); let stderr = String::from_utf8(stderr).unwrap(); + #[allow(clippy::print_stderr)] if expect_success != status.success() { eprintln!("stdout: <<<{stdout}>>>"); eprintln!("stderr: <<<{stderr}>>>"); @@ -564,6 +568,7 @@ pub fn deno_cmd_with_deno_dir(deno_dir: &TempDir) -> TestCommandBuilder { .env("JSR_URL", jsr_registry_unset_url()) } +#[allow(clippy::print_stdout)] pub fn run_powershell_script_file( script_file_path: &str, args: Vec<&str>, @@ -655,6 +660,7 @@ impl<'a> CheckOutputIntegrationTest<'a> { } pub fn wildcard_match(pattern: &str, text: &str) -> bool { + #[allow(clippy::print_stderr)] match wildcard_match_detailed(pattern, text) { WildcardMatchResult::Success => true, WildcardMatchResult::Fail(debug_output) => { diff --git a/tests/util/server/src/lsp.rs b/tests/util/server/src/lsp.rs index 4e75cfadb9..d34deb2161 100644 --- a/tests/util/server/src/lsp.rs +++ b/tests/util/server/src/lsp.rs @@ -157,6 +157,7 @@ impl LspStdoutReader { self.pending_messages.0.lock().len() } + #[allow(clippy::print_stderr)] pub fn output_pending_messages(&self) { let messages = self.pending_messages.0.lock(); eprintln!("{:?}", messages); @@ -573,6 +574,7 @@ impl LspClientBuilder { for line in stderr.lines() { match line { Ok(line) => { + #[allow(clippy::print_stderr)] if print_stderr { eprintln!("{}", line); } @@ -587,7 +589,10 @@ impl LspClientBuilder { continue; } Err(err) => { - eprintln!("failed to parse perf record: {:#}", err); + #[allow(clippy::print_stderr)] + { + eprintln!("failed to parse perf record: {:#}", err); + } } } } @@ -782,11 +787,14 @@ impl LspClient { std::thread::sleep(Duration::from_millis(20)); } - eprintln!("==== STDERR OUTPUT ===="); - for line in found_lines { - eprintln!("{}", line) + #[allow(clippy::print_stderr)] + { + eprintln!("==== STDERR OUTPUT ===="); + for line in found_lines { + eprintln!("{}", line) + } + eprintln!("== END STDERR OUTPUT =="); } - eprintln!("== END STDERR OUTPUT =="); panic!("Timed out waiting on condition.") } diff --git a/tests/util/server/src/npm.rs b/tests/util/server/src/npm.rs index 4b17b95f72..31686fa854 100644 --- a/tests/util/server/src/npm.rs +++ b/tests/util/server/src/npm.rs @@ -103,7 +103,6 @@ impl TestNpmRegistry { } pub fn root_dir(&self) -> PathRef { - eprintln!("root {}", self.local_path); tests_path().join("registry").join(&self.local_path) } @@ -120,7 +119,6 @@ impl TestNpmRegistry { } pub fn registry_file(&self, name: &str) -> Result>> { - eprintln!("registry file {}", name); self.get_package_property(name, |p| p.registry_file.as_bytes().to_vec()) } @@ -138,7 +136,6 @@ impl TestNpmRegistry { package_name: &str, func: impl FnOnce(&CustomNpmPackage) -> TResult, ) -> Result> { - eprintln!("get package property {}", package_name); // it's ok if multiple threads race here as they will do the same work twice if !self.cache.lock().contains_key(package_name) { match get_npm_package(&self.hostname, &self.local_path, package_name)? { @@ -155,7 +152,6 @@ impl TestNpmRegistry { &self, uri_path: &'s str, ) -> Option<(&'s str, &'s str)> { - eprintln!("GEETT {}", uri_path); let prefix1 = format!("/{}/", DENOTEST_SCOPE_NAME); let prefix2 = format!("/{}%2f", DENOTEST_SCOPE_NAME); @@ -198,10 +194,6 @@ fn get_npm_package( local_path: &str, package_name: &str, ) -> Result> { - eprintln!( - "get npm package {} {} {}", - registry_hostname, local_path, package_name - ); let registry_hostname = if package_name == "@denotest/tarballs-privateserver2" { "http://localhost:4262" diff --git a/tests/util/server/src/pty.rs b/tests/util/server/src/pty.rs index 8d42fed78b..07659262cf 100644 --- a/tests/util/server/src/pty.rs +++ b/tests/util/server/src/pty.rs @@ -61,7 +61,10 @@ impl Pty { if is_windows && *IS_CI { // the pty tests don't really start up on the windows CI for some reason // so ignore them for now - eprintln!("Ignoring windows CI."); + #[allow(clippy::print_stderr)] + { + eprintln!("Ignoring windows CI."); + } false } else { true @@ -250,11 +253,14 @@ impl Pty { } let text = self.next_text(); - eprintln!( - "------ Start Full Text ------\n{:?}\n------- End Full Text -------", - String::from_utf8_lossy(&self.read_bytes) - ); - eprintln!("Next text: {:?}", text); + #[allow(clippy::print_stderr)] + { + eprintln!( + "------ Start Full Text ------\n{:?}\n------- End Full Text -------", + String::from_utf8_lossy(&self.read_bytes) + ); + eprintln!("Next text: {:?}", text); + } false } diff --git a/tests/util/server/src/servers/hyper_utils.rs b/tests/util/server/src/servers/hyper_utils.rs index c2db7ea668..8e01151ed4 100644 --- a/tests/util/server/src/servers/hyper_utils.rs +++ b/tests/util/server/src/servers/hyper_utils.rs @@ -42,7 +42,10 @@ where let fut: Pin>>> = async move { let listener = TcpListener::bind(options.addr).await?; - println!("ready: {}", options.addr); + #[allow(clippy::print_stdout)] + { + println!("ready: {}", options.addr); + } loop { let (stream, _) = listener.accept().await?; let io = TokioIo::new(stream); @@ -58,6 +61,7 @@ where if let Err(e) = fut.await { let err_str = e.to_string(); + #[allow(clippy::print_stderr)] if !err_str.contains("early eof") { eprintln!("{}: {:?}", options.error_msg, e); } @@ -89,6 +93,7 @@ pub async fn run_server_with_acceptor<'a, A, F, S>( if let Err(e) = fut.await { let err_str = e.to_string(); + #[allow(clippy::print_stderr)] if !err_str.contains("early eof") { eprintln!("{}: {:?}", error_msg, e); } @@ -135,6 +140,7 @@ async fn hyper_serve_connection( if let Err(e) = result { let err_str = e.to_string(); + #[allow(clippy::print_stderr)] if !err_str.contains("early eof") { eprintln!("{}: {:?}", error_msg, e); } diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs index 3a9c440104..6c960ce522 100644 --- a/tests/util/server/src/servers/mod.rs +++ b/tests/util/server/src/servers/mod.rs @@ -198,7 +198,6 @@ fn json_body(value: serde_json::Value) -> UnsyncBoxBody { /// Benchmark server that just serves "hello world" responses. async fn hyper_hello(port: u16) { - println!("hyper hello"); let addr = SocketAddr::from(([127, 0, 0, 1], port)); let handler = move |_: Request| async move { Ok::<_, anyhow::Error>(Response::new(UnsyncBoxBody::new( @@ -342,7 +341,10 @@ async fn get_tcp_listener_stream( .collect::>(); // Eye catcher for HttpServerCount - println!("ready: {name} on {:?}", addresses); + #[allow(clippy::print_stdout)] + { + println!("ready: {name} on {:?}", addresses); + } futures::stream::select_all(listeners) } @@ -358,7 +360,10 @@ async fn run_tls_client_auth_server(port: u16) { while let Some(Ok(mut tls_stream)) = tls.next().await { tokio::spawn(async move { let Ok(handshake) = tls_stream.handshake().await else { - eprintln!("Failed to handshake"); + #[allow(clippy::print_stderr)] + { + eprintln!("Failed to handshake"); + } return; }; // We only need to check for the presence of client certificates @@ -405,7 +410,6 @@ async fn absolute_redirect( .collect(); if let Some(url) = query_params.get("redirect_to") { - println!("URL: {url:?}"); let redirect = redirect_resp(url.to_owned()); return Ok(redirect); } @@ -413,7 +417,6 @@ async fn absolute_redirect( if path.starts_with("/REDIRECT") { let url = &req.uri().path()[9..]; - println!("URL: {url:?}"); let redirect = redirect_resp(url.to_string()); return Ok(redirect); } @@ -1357,6 +1360,7 @@ async fn wrap_client_auth_https_server(port: u16) { // here. Rusttls ensures that they are valid and signed by the CA. match handshake.has_peer_certificates { true => { yield Ok(tls); }, + #[allow(clippy::print_stderr)] false => { eprintln!("https_client_auth: no valid client certificate"); }, }; } diff --git a/tests/util/server/src/servers/ws.rs b/tests/util/server/src/servers/ws.rs index 815119b6a1..dd4efbf659 100644 --- a/tests/util/server/src/servers/ws.rs +++ b/tests/util/server/src/servers/ws.rs @@ -76,6 +76,7 @@ pub async fn run_wss2_server(port: u16) { let server: Handshake<_, Bytes> = h2.handshake(tls); let mut server = match server.await { Ok(server) => server, + #[allow(clippy::print_stdout)] Err(e) => { println!("Failed to handshake h2: {e:?}"); return; @@ -87,6 +88,7 @@ pub async fn run_wss2_server(port: u16) { }; let (recv, send) = match conn { Ok(conn) => conn, + #[allow(clippy::print_stdout)] Err(e) => { println!("Failed to accept a connection: {e:?}"); break; @@ -137,6 +139,7 @@ where .map_err(|e| anyhow!("Error upgrading websocket connection: {}", e)) .unwrap(); + #[allow(clippy::print_stderr)] if let Err(e) = handler(ws).await { eprintln!("Error in websocket connection: {}", e); } @@ -152,6 +155,7 @@ where .serve_connection(io, service) .with_upgrades(); + #[allow(clippy::print_stderr)] if let Err(e) = conn.await { eprintln!("websocket server error: {e:?}"); } @@ -162,16 +166,19 @@ async fn handle_wss_stream( recv: Request, mut send: SendResponse, ) -> Result<(), h2::Error> { + #[allow(clippy::print_stderr)] if recv.method() != Method::CONNECT { eprintln!("wss2: refusing non-CONNECT stream"); send.send_reset(Reason::REFUSED_STREAM); return Ok(()); } + #[allow(clippy::print_stderr)] let Some(protocol) = recv.extensions().get::() else { eprintln!("wss2: refusing no-:protocol stream"); send.send_reset(Reason::REFUSED_STREAM); return Ok(()); }; + #[allow(clippy::print_stderr)] if protocol.as_str() != "websocket" && protocol.as_str() != "WebSocket" { eprintln!("wss2: refusing non-websocket stream"); send.send_reset(Reason::REFUSED_STREAM); From 0060e74779e25eba4544ca540066f07a1c46d17b Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:46:48 -0700 Subject: [PATCH 19/63] fix(install): don't cache json exports of JSR packages (for now) (#26530) Temporary fix for #26509, so people don't get errors. --- cli/tools/registry/pm/cache_deps.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/tools/registry/pm/cache_deps.rs b/cli/tools/registry/pm/cache_deps.rs index 365622d11f..9883deb1d5 100644 --- a/cli/tools/registry/pm/cache_deps.rs +++ b/cli/tools/registry/pm/cache_deps.rs @@ -90,8 +90,13 @@ pub async fn cache_top_level_deps( while let Some(info_future) = info_futures.next().await { if let Some((specifier, info)) = info_future { let exports = info.exports(); - for (k, _) in exports { + for (k, v) in exports { if let Ok(spec) = specifier.join(k) { + if v.ends_with(".json") { + // TODO(nathanwhit): this should work, there's a bug with + // json roots in deno_graph. skip it for now + continue; + } roots.push(spec); } } From cead8af104af693558db07675a423761f174d260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 25 Oct 2024 11:53:34 +0100 Subject: [PATCH 20/63] build: use 'fs' feature of 'nix' crate in ext/fs (#26533) Hot-fix to unblock `v2.0.3` release --- ext/fs/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml index ab0bf22fd3..f5daa1bcdc 100644 --- a/ext/fs/Cargo.toml +++ b/ext/fs/Cargo.toml @@ -31,7 +31,7 @@ serde.workspace = true thiserror.workspace = true [target.'cfg(unix)'.dependencies] -nix = { workspace = true, features = ["user"] } +nix = { workspace = true, features = ["fs", "user"] } [target.'cfg(windows)'.dependencies] winapi = { workspace = true, features = ["winbase"] } From 730331622ee17cf603447f4eb53631b9cfd7bef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 25 Oct 2024 14:57:40 +0100 Subject: [PATCH 21/63] chore: forward v2.0.3 commit to main (#26535) Forwarding v2.0.3 commit to `main` Co-authored-by: denobot <33910674+denobot@users.noreply.github.com> Co-authored-by: bartlomieju --- .github/workflows/ci.generate.ts | 2 +- .github/workflows/ci.yml | 8 ++--- Cargo.lock | 58 ++++++++++++++++---------------- Cargo.toml | 56 +++++++++++++++--------------- Releases.md | 38 +++++++++++++++++++++ bench_util/Cargo.toml | 2 +- cli/Cargo.toml | 2 +- ext/broadcast_channel/Cargo.toml | 2 +- ext/cache/Cargo.toml | 2 +- ext/canvas/Cargo.toml | 2 +- ext/console/Cargo.toml | 2 +- ext/cron/Cargo.toml | 2 +- ext/crypto/Cargo.toml | 2 +- ext/fetch/Cargo.toml | 2 +- ext/ffi/Cargo.toml | 2 +- ext/fs/Cargo.toml | 2 +- ext/http/Cargo.toml | 2 +- ext/io/Cargo.toml | 2 +- ext/kv/Cargo.toml | 2 +- ext/napi/Cargo.toml | 2 +- ext/napi/sym/Cargo.toml | 2 +- ext/net/Cargo.toml | 2 +- ext/node/Cargo.toml | 2 +- ext/tls/Cargo.toml | 2 +- ext/url/Cargo.toml | 2 +- ext/web/Cargo.toml | 2 +- ext/webgpu/Cargo.toml | 2 +- ext/webidl/Cargo.toml | 2 +- ext/websocket/Cargo.toml | 2 +- ext/webstorage/Cargo.toml | 2 +- resolvers/deno/Cargo.toml | 2 +- resolvers/node/Cargo.toml | 2 +- runtime/Cargo.toml | 2 +- runtime/permissions/Cargo.toml | 2 +- 34 files changed, 129 insertions(+), 91 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index ceba91e355..3e2b079643 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify"; // Bump this number when you want to purge the cache. // Note: the tools/release/01_bump_crate_versions.ts script will update this version // automatically via regex, so ensure that this line maintains this format. -const cacheVersion = 21; +const cacheVersion = 22; const ubuntuX86Runner = "ubuntu-22.04"; const ubuntuX86XlRunner = "ubuntu-22.04-xl"; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 167fe5394d..5739537bb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -361,8 +361,8 @@ jobs: path: |- ~/.cargo/registry/index ~/.cargo/registry/cache - key: '21-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '21-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' + key: '22-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' + restore-keys: '22-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' if: '!(matrix.skip)' - name: Restore cache build output (PR) uses: actions/cache/restore@v4 @@ -375,7 +375,7 @@ jobs: !./target/*/*.zip !./target/*/*.tar.gz key: never_saved - restore-keys: '21-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' + restore-keys: '22-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' - name: Apply and update mtime cache if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))' uses: ./.github/mtime_cache @@ -685,7 +685,7 @@ jobs: !./target/*/*.zip !./target/*/*.sha256sum !./target/*/*.tar.gz - key: '21-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' + key: '22-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: name: publish canary runs-on: ubuntu-22.04 diff --git a/Cargo.lock b/Cargo.lock index f1209b32ac..6e76f27c2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1154,7 +1154,7 @@ dependencies = [ [[package]] name = "deno" -version = "2.0.2" +version = "2.0.3" dependencies = [ "anstream", "async-trait", @@ -1323,7 +1323,7 @@ dependencies = [ [[package]] name = "deno_bench_util" -version = "0.167.0" +version = "0.168.0" dependencies = [ "bencher", "deno_core", @@ -1332,7 +1332,7 @@ dependencies = [ [[package]] name = "deno_broadcast_channel" -version = "0.167.0" +version = "0.168.0" dependencies = [ "async-trait", "deno_core", @@ -1343,7 +1343,7 @@ dependencies = [ [[package]] name = "deno_cache" -version = "0.105.0" +version = "0.106.0" dependencies = [ "async-trait", "deno_core", @@ -1376,7 +1376,7 @@ dependencies = [ [[package]] name = "deno_canvas" -version = "0.42.0" +version = "0.43.0" dependencies = [ "deno_core", "deno_webgpu", @@ -1411,7 +1411,7 @@ dependencies = [ [[package]] name = "deno_console" -version = "0.173.0" +version = "0.174.0" dependencies = [ "deno_core", ] @@ -1456,7 +1456,7 @@ checksum = "a13951ea98c0a4c372f162d669193b4c9d991512de9f2381dd161027f34b26b1" [[package]] name = "deno_cron" -version = "0.53.0" +version = "0.54.0" dependencies = [ "anyhow", "async-trait", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "deno_crypto" -version = "0.187.0" +version = "0.188.0" dependencies = [ "aes", "aes-gcm", @@ -1531,7 +1531,7 @@ dependencies = [ [[package]] name = "deno_fetch" -version = "0.197.0" +version = "0.198.0" dependencies = [ "base64 0.21.7", "bytes", @@ -1564,7 +1564,7 @@ dependencies = [ [[package]] name = "deno_ffi" -version = "0.160.0" +version = "0.161.0" dependencies = [ "deno_core", "deno_permissions", @@ -1584,7 +1584,7 @@ dependencies = [ [[package]] name = "deno_fs" -version = "0.83.0" +version = "0.84.0" dependencies = [ "async-trait", "base32", @@ -1635,7 +1635,7 @@ dependencies = [ [[package]] name = "deno_http" -version = "0.171.0" +version = "0.172.0" dependencies = [ "async-compression", "async-trait", @@ -1674,7 +1674,7 @@ dependencies = [ [[package]] name = "deno_io" -version = "0.83.0" +version = "0.84.0" dependencies = [ "async-trait", "deno_core", @@ -1695,7 +1695,7 @@ dependencies = [ [[package]] name = "deno_kv" -version = "0.81.0" +version = "0.82.0" dependencies = [ "anyhow", "async-trait", @@ -1767,7 +1767,7 @@ dependencies = [ [[package]] name = "deno_napi" -version = "0.104.0" +version = "0.105.0" dependencies = [ "deno_core", "deno_permissions", @@ -1795,7 +1795,7 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.165.0" +version = "0.166.0" dependencies = [ "deno_core", "deno_permissions", @@ -1812,7 +1812,7 @@ dependencies = [ [[package]] name = "deno_node" -version = "0.110.0" +version = "0.111.0" dependencies = [ "aead-gcm-stream", "aes", @@ -1961,7 +1961,7 @@ dependencies = [ [[package]] name = "deno_permissions" -version = "0.33.0" +version = "0.34.0" dependencies = [ "deno_core", "deno_path_util", @@ -1978,7 +1978,7 @@ dependencies = [ [[package]] name = "deno_resolver" -version = "0.5.0" +version = "0.6.0" dependencies = [ "anyhow", "base32", @@ -1994,7 +1994,7 @@ dependencies = [ [[package]] name = "deno_runtime" -version = "0.182.0" +version = "0.183.0" dependencies = [ "color-print", "deno_ast", @@ -2112,7 +2112,7 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.160.0" +version = "0.161.0" dependencies = [ "deno_core", "deno_native_certs", @@ -2161,7 +2161,7 @@ dependencies = [ [[package]] name = "deno_url" -version = "0.173.0" +version = "0.174.0" dependencies = [ "deno_bench_util", "deno_console", @@ -2173,7 +2173,7 @@ dependencies = [ [[package]] name = "deno_web" -version = "0.204.0" +version = "0.205.0" dependencies = [ "async-trait", "base64-simd 0.8.0", @@ -2195,7 +2195,7 @@ dependencies = [ [[package]] name = "deno_webgpu" -version = "0.140.0" +version = "0.141.0" dependencies = [ "deno_core", "raw-window-handle", @@ -2208,7 +2208,7 @@ dependencies = [ [[package]] name = "deno_webidl" -version = "0.173.0" +version = "0.174.0" dependencies = [ "deno_bench_util", "deno_core", @@ -2216,7 +2216,7 @@ dependencies = [ [[package]] name = "deno_websocket" -version = "0.178.0" +version = "0.179.0" dependencies = [ "bytes", "deno_core", @@ -2238,7 +2238,7 @@ dependencies = [ [[package]] name = "deno_webstorage" -version = "0.168.0" +version = "0.169.0" dependencies = [ "deno_core", "deno_web", @@ -4483,7 +4483,7 @@ dependencies = [ [[package]] name = "napi_sym" -version = "0.103.0" +version = "0.104.0" dependencies = [ "quote", "serde", @@ -4538,7 +4538,7 @@ dependencies = [ [[package]] name = "node_resolver" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index a420a9968e..7ca53699ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,16 +48,16 @@ repository = "https://github.com/denoland/deno" deno_ast = { version = "=0.42.2", features = ["transpiling"] } deno_core = { version = "0.314.2" } -deno_bench_util = { version = "0.167.0", path = "./bench_util" } +deno_bench_util = { version = "0.168.0", path = "./bench_util" } deno_lockfile = "=0.23.1" deno_media_type = { version = "0.1.4", features = ["module_specifier"] } deno_npm = "=0.25.4" deno_path_util = "=0.2.1" -deno_permissions = { version = "0.33.0", path = "./runtime/permissions" } -deno_runtime = { version = "0.182.0", path = "./runtime" } +deno_permissions = { version = "0.34.0", path = "./runtime/permissions" } +deno_runtime = { version = "0.183.0", path = "./runtime" } deno_semver = "=0.5.16" deno_terminal = "0.2.0" -napi_sym = { version = "0.103.0", path = "./ext/napi/sym" } +napi_sym = { version = "0.104.0", path = "./ext/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } denokv_proto = "0.8.1" @@ -66,32 +66,32 @@ denokv_remote = "0.8.1" denokv_sqlite = { default-features = false, version = "0.8.2" } # exts -deno_broadcast_channel = { version = "0.167.0", path = "./ext/broadcast_channel" } -deno_cache = { version = "0.105.0", path = "./ext/cache" } -deno_canvas = { version = "0.42.0", path = "./ext/canvas" } -deno_console = { version = "0.173.0", path = "./ext/console" } -deno_cron = { version = "0.53.0", path = "./ext/cron" } -deno_crypto = { version = "0.187.0", path = "./ext/crypto" } -deno_fetch = { version = "0.197.0", path = "./ext/fetch" } -deno_ffi = { version = "0.160.0", path = "./ext/ffi" } -deno_fs = { version = "0.83.0", path = "./ext/fs" } -deno_http = { version = "0.171.0", path = "./ext/http" } -deno_io = { version = "0.83.0", path = "./ext/io" } -deno_kv = { version = "0.81.0", path = "./ext/kv" } -deno_napi = { version = "0.104.0", path = "./ext/napi" } -deno_net = { version = "0.165.0", path = "./ext/net" } -deno_node = { version = "0.110.0", path = "./ext/node" } -deno_tls = { version = "0.160.0", path = "./ext/tls" } -deno_url = { version = "0.173.0", path = "./ext/url" } -deno_web = { version = "0.204.0", path = "./ext/web" } -deno_webgpu = { version = "0.140.0", path = "./ext/webgpu" } -deno_webidl = { version = "0.173.0", path = "./ext/webidl" } -deno_websocket = { version = "0.178.0", path = "./ext/websocket" } -deno_webstorage = { version = "0.168.0", path = "./ext/webstorage" } +deno_broadcast_channel = { version = "0.168.0", path = "./ext/broadcast_channel" } +deno_cache = { version = "0.106.0", path = "./ext/cache" } +deno_canvas = { version = "0.43.0", path = "./ext/canvas" } +deno_console = { version = "0.174.0", path = "./ext/console" } +deno_cron = { version = "0.54.0", path = "./ext/cron" } +deno_crypto = { version = "0.188.0", path = "./ext/crypto" } +deno_fetch = { version = "0.198.0", path = "./ext/fetch" } +deno_ffi = { version = "0.161.0", path = "./ext/ffi" } +deno_fs = { version = "0.84.0", path = "./ext/fs" } +deno_http = { version = "0.172.0", path = "./ext/http" } +deno_io = { version = "0.84.0", path = "./ext/io" } +deno_kv = { version = "0.82.0", path = "./ext/kv" } +deno_napi = { version = "0.105.0", path = "./ext/napi" } +deno_net = { version = "0.166.0", path = "./ext/net" } +deno_node = { version = "0.111.0", path = "./ext/node" } +deno_tls = { version = "0.161.0", path = "./ext/tls" } +deno_url = { version = "0.174.0", path = "./ext/url" } +deno_web = { version = "0.205.0", path = "./ext/web" } +deno_webgpu = { version = "0.141.0", path = "./ext/webgpu" } +deno_webidl = { version = "0.174.0", path = "./ext/webidl" } +deno_websocket = { version = "0.179.0", path = "./ext/websocket" } +deno_webstorage = { version = "0.169.0", path = "./ext/webstorage" } # resolvers -deno_resolver = { version = "0.5.0", path = "./resolvers/deno" } -node_resolver = { version = "0.12.0", path = "./resolvers/node" } +deno_resolver = { version = "0.6.0", path = "./resolvers/deno" } +node_resolver = { version = "0.13.0", path = "./resolvers/node" } aes = "=0.8.3" anyhow = "1.0.57" diff --git a/Releases.md b/Releases.md index 08562134a1..5576d4bd8c 100644 --- a/Releases.md +++ b/Releases.md @@ -6,6 +6,44 @@ https://github.com/denoland/deno/releases We also have one-line install commands at: https://github.com/denoland/deno_install +### 2.0.3 / 2024.10.25 + +- feat(lsp): interactive inlay hints (#26382) +- fix: support node-api in denort (#26389) +- fix(check): support `--frozen` on deno check (#26479) +- fix(cli): increase size of blocking task threadpool on windows (#26465) +- fix(config): schemas for lint rule and tag autocompletion (#26515) +- fix(ext/console): ignore casing for named colors in css parsing (#26466) +- fix(ext/ffi): return u64/i64 as bigints from nonblocking ffi calls (#26486) +- fix(ext/node): cancel pending ipc writes on channel close (#26504) +- fix(ext/node): map `ERROR_INVALID_NAME` to `ENOENT` on windows (#26475) +- fix(ext/node): only set our end of child process pipe to nonblocking mode + (#26495) +- fix(ext/node): properly map reparse point error in readlink (#26375) +- fix(ext/node): refactor http.ServerResponse into function class (#26210) +- fix(ext/node): stub HTTPParser internal binding (#26401) +- fix(ext/node): use primordials in `ext/node/polyfills/https.ts` (#26323) +- fix(fmt): --ext flag requires to pass files (#26525) +- fix(fmt): upgrade formatters (#26469) +- fix(help): missing package specifier (#26380) +- fix(info): resolve workspace member mappings (#26350) +- fix(install): better json editing (#26450) +- fix(install): cache all exports of JSR packages listed in `deno.json` (#26501) +- fix(install): cache type only module deps in `deno install` (#26497) +- fix(install): don't cache json exports of JSR packages (for now) (#26530) +- fix(install): update lockfile when using package.json (#26458) +- fix(lsp): import-map-remap quickfix for type imports (#26454) +- fix(node/util): support array formats in `styleText` (#26507) +- fix(node:tls): set TLSSocket.alpnProtocol for client connections (#26476) +- fix(npm): ensure scoped package name is encoded in URLs (#26390) +- fix(npm): support version ranges with && or comma (#26453) +- fix: `.npmrc` settings not being passed to install/add command (#26473) +- fix: add 'fmt-component' to unstable features in schema file (#26526) +- fix: share inotify fd across watchers (#26200) +- fix: unpin tokio version (#26457) +- perf(compile): pass module source data from binary directly to v8 (#26494) +- perf: avoid multiple calls to runMicrotask (#26378) + ### 2.0.2 / 2024.10.17 - fix(cli): set napi object property properly (#26344) diff --git a/bench_util/Cargo.toml b/bench_util/Cargo.toml index fbf30b9fcf..eea9c69a3c 100644 --- a/bench_util/Cargo.toml +++ b/bench_util/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_bench_util" -version = "0.167.0" +version = "0.168.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 14eda5427e..db0248d846 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno" -version = "2.0.2" +version = "2.0.3" authors.workspace = true default-run = "deno" edition.workspace = true diff --git a/ext/broadcast_channel/Cargo.toml b/ext/broadcast_channel/Cargo.toml index 7ca058a44a..845544f0c8 100644 --- a/ext/broadcast_channel/Cargo.toml +++ b/ext/broadcast_channel/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_broadcast_channel" -version = "0.167.0" +version = "0.168.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cache/Cargo.toml b/ext/cache/Cargo.toml index dee4d7274d..a07a8f7207 100644 --- a/ext/cache/Cargo.toml +++ b/ext/cache/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cache" -version = "0.105.0" +version = "0.106.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 6ca3d76c05..4c125ea9b5 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_canvas" -version = "0.42.0" +version = "0.43.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/console/Cargo.toml b/ext/console/Cargo.toml index f83f7138d8..b89f67fece 100644 --- a/ext/console/Cargo.toml +++ b/ext/console/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_console" -version = "0.173.0" +version = "0.174.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cron/Cargo.toml b/ext/cron/Cargo.toml index ccd81de0cc..6c5547c35f 100644 --- a/ext/cron/Cargo.toml +++ b/ext/cron/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cron" -version = "0.53.0" +version = "0.54.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/crypto/Cargo.toml b/ext/crypto/Cargo.toml index 2f970ca535..e9761f48f1 100644 --- a/ext/crypto/Cargo.toml +++ b/ext/crypto/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_crypto" -version = "0.187.0" +version = "0.188.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml index afffd3ffb1..316a6eea81 100644 --- a/ext/fetch/Cargo.toml +++ b/ext/fetch/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fetch" -version = "0.197.0" +version = "0.198.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/ffi/Cargo.toml b/ext/ffi/Cargo.toml index 59dc13ed96..405030df29 100644 --- a/ext/ffi/Cargo.toml +++ b/ext/ffi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_ffi" -version = "0.160.0" +version = "0.161.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml index f5daa1bcdc..2a1c5bf2c6 100644 --- a/ext/fs/Cargo.toml +++ b/ext/fs/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fs" -version = "0.83.0" +version = "0.84.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/http/Cargo.toml b/ext/http/Cargo.toml index 6fa7598cb5..0bae99ef76 100644 --- a/ext/http/Cargo.toml +++ b/ext/http/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_http" -version = "0.171.0" +version = "0.172.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/io/Cargo.toml b/ext/io/Cargo.toml index 8f407c820c..f7ffe46bc3 100644 --- a/ext/io/Cargo.toml +++ b/ext/io/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_io" -version = "0.83.0" +version = "0.84.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/kv/Cargo.toml b/ext/kv/Cargo.toml index f7b28af675..a4e35193d5 100644 --- a/ext/kv/Cargo.toml +++ b/ext/kv/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_kv" -version = "0.81.0" +version = "0.82.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/Cargo.toml b/ext/napi/Cargo.toml index ed39d96df3..622429f7c8 100644 --- a/ext/napi/Cargo.toml +++ b/ext/napi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_napi" -version = "0.104.0" +version = "0.105.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/sym/Cargo.toml b/ext/napi/sym/Cargo.toml index 7c9bf208c7..9fe2d839ad 100644 --- a/ext/napi/sym/Cargo.toml +++ b/ext/napi/sym/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "napi_sym" -version = "0.103.0" +version = "0.104.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/net/Cargo.toml b/ext/net/Cargo.toml index 634dd7dda4..a458dbe9e3 100644 --- a/ext/net/Cargo.toml +++ b/ext/net/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_net" -version = "0.165.0" +version = "0.166.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index 55d53d0ce0..3743e6a85e 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_node" -version = "0.110.0" +version = "0.111.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/tls/Cargo.toml b/ext/tls/Cargo.toml index b060f28298..d163fd7eb4 100644 --- a/ext/tls/Cargo.toml +++ b/ext/tls/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_tls" -version = "0.160.0" +version = "0.161.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/url/Cargo.toml b/ext/url/Cargo.toml index ceef18f45a..ef7890e2ae 100644 --- a/ext/url/Cargo.toml +++ b/ext/url/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_url" -version = "0.173.0" +version = "0.174.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/web/Cargo.toml b/ext/web/Cargo.toml index 395ea182f9..0cb08e19c1 100644 --- a/ext/web/Cargo.toml +++ b/ext/web/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_web" -version = "0.204.0" +version = "0.205.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webgpu/Cargo.toml b/ext/webgpu/Cargo.toml index 7354919d4a..84ee4a012f 100644 --- a/ext/webgpu/Cargo.toml +++ b/ext/webgpu/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webgpu" -version = "0.140.0" +version = "0.141.0" authors = ["the Deno authors"] edition.workspace = true license = "MIT" diff --git a/ext/webidl/Cargo.toml b/ext/webidl/Cargo.toml index d069b32125..a403db2fea 100644 --- a/ext/webidl/Cargo.toml +++ b/ext/webidl/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webidl" -version = "0.173.0" +version = "0.174.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/websocket/Cargo.toml b/ext/websocket/Cargo.toml index ec359100d4..b48b39c99f 100644 --- a/ext/websocket/Cargo.toml +++ b/ext/websocket/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_websocket" -version = "0.178.0" +version = "0.179.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webstorage/Cargo.toml b/ext/webstorage/Cargo.toml index 0d6fcd6a0d..911e252414 100644 --- a/ext/webstorage/Cargo.toml +++ b/ext/webstorage/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webstorage" -version = "0.168.0" +version = "0.169.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/deno/Cargo.toml b/resolvers/deno/Cargo.toml index e75a9a1014..418e1cb522 100644 --- a/resolvers/deno/Cargo.toml +++ b/resolvers/deno/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_resolver" -version = "0.5.0" +version = "0.6.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/node/Cargo.toml b/resolvers/node/Cargo.toml index 44549bec19..9638899209 100644 --- a/resolvers/node/Cargo.toml +++ b/resolvers/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "node_resolver" -version = "0.12.0" +version = "0.13.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index fbf5d95811..703bd5a432 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_runtime" -version = "0.182.0" +version = "0.183.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/runtime/permissions/Cargo.toml b/runtime/permissions/Cargo.toml index d7f2364a56..45266791ae 100644 --- a/runtime/permissions/Cargo.toml +++ b/runtime/permissions/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_permissions" -version = "0.33.0" +version = "0.34.0" authors.workspace = true edition.workspace = true license.workspace = true From 38c7af456518ac4fdbd36e3bfa731c38195bb773 Mon Sep 17 00:00:00 2001 From: Nayeem Rahman Date: Fri, 25 Oct 2024 18:35:09 +0100 Subject: [PATCH 22/63] feat(lsp): "typescript.preferences.preferTypeOnlyAutoImports" setting (#26546) --- cli/lsp/config.rs | 5 +++++ cli/lsp/tsc.rs | 7 +++++- cli/tsc/99_main_compiler.js | 39 +++++----------------------------- tests/integration/lsp_tests.rs | 22 +++++++++++++------ 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index 74f3583d68..3ffc9e657a 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -439,6 +439,8 @@ pub struct LanguagePreferences { pub use_aliases_for_renames: bool, #[serde(default)] pub quote_style: QuoteStyle, + #[serde(default)] + pub prefer_type_only_auto_imports: bool, } impl Default for LanguagePreferences { @@ -449,6 +451,7 @@ impl Default for LanguagePreferences { auto_import_file_exclude_patterns: vec![], use_aliases_for_renames: true, quote_style: Default::default(), + prefer_type_only_auto_imports: false, } } } @@ -2251,6 +2254,7 @@ mod tests { auto_import_file_exclude_patterns: vec![], use_aliases_for_renames: true, quote_style: QuoteStyle::Auto, + prefer_type_only_auto_imports: false, }, suggest: CompletionSettings { complete_function_calls: false, @@ -2296,6 +2300,7 @@ mod tests { auto_import_file_exclude_patterns: vec![], use_aliases_for_renames: true, quote_style: QuoteStyle::Auto, + prefer_type_only_auto_imports: false, }, suggest: CompletionSettings { complete_function_calls: false, diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index 0bc7d1600d..0f31d7dd3b 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -4952,6 +4952,8 @@ pub struct UserPreferences { pub auto_import_file_exclude_patterns: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub interactive_inlay_hints: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prefer_type_only_auto_imports: Option, } impl UserPreferences { @@ -5074,6 +5076,9 @@ impl UserPreferences { } else { Some(language_settings.preferences.quote_style) }, + prefer_type_only_auto_imports: Some( + language_settings.preferences.prefer_type_only_auto_imports, + ), ..base_preferences } } @@ -6215,7 +6220,7 @@ mod tests { let change = changes.text_changes.first().unwrap(); assert_eq!( change.new_text, - "import type { someLongVariable } from './b.ts'\n" + "import { someLongVariable } from './b.ts'\n" ); } diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index 719f2b9824..e3a0bee597 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -516,7 +516,6 @@ delete Object.prototype.__proto__; /** @typedef {{ * ls: ts.LanguageService & { [k:string]: any }, * compilerOptions: ts.CompilerOptions, - * forceEnabledVerbatimModuleSyntax: boolean, * }} LanguageServiceEntry */ /** @type {{ unscoped: LanguageServiceEntry, byScope: Map }} */ const languageServiceEntries = { @@ -1026,7 +1025,7 @@ delete Object.prototype.__proto__; : ts.sortAndDeduplicateDiagnostics( checkFiles.map((s) => program.getSemanticDiagnostics(s)).flat(), )), - ].filter(filterMapDiagnostic.bind(null, false)); + ].filter(filterMapDiagnostic); // emit the tsbuildinfo file // @ts-ignore: emitBuildInfo is not exposed (https://github.com/microsoft/TypeScript/issues/49871) @@ -1041,28 +1040,11 @@ delete Object.prototype.__proto__; debug("<<< exec stop"); } - /** - * @param {boolean} isLsp - * @param {ts.Diagnostic} diagnostic - */ - function filterMapDiagnostic(isLsp, diagnostic) { + /** @param {ts.Diagnostic} diagnostic */ + function filterMapDiagnostic(diagnostic) { if (IGNORED_DIAGNOSTICS.includes(diagnostic.code)) { return false; } - if (isLsp) { - // TS1484: `...` is a type and must be imported using a type-only import when 'verbatimModuleSyntax' is enabled. - // We force-enable `verbatimModuleSyntax` in the LSP so the `type` - // modifier is used when auto-importing types. But we don't want this - // diagnostic unless it was explicitly enabled by the user. - if (diagnostic.code == 1484) { - const entry = (lastRequestScope - ? languageServiceEntries.byScope.get(lastRequestScope) - : null) ?? languageServiceEntries.unscoped; - if (entry.forceEnabledVerbatimModuleSyntax) { - return false; - } - } - } // make the diagnostic for using an `export =` in an es module a warning if (diagnostic.code === 1203) { diagnostic.category = ts.DiagnosticCategory.Warning; @@ -1159,12 +1141,10 @@ delete Object.prototype.__proto__; "strict": true, "target": "esnext", "useDefineForClassFields": true, - "verbatimModuleSyntax": true, "jsx": "react", "jsxFactory": "React.createElement", "jsxFragmentFactory": "React.Fragment", }), - forceEnabledVerbatimModuleSyntax: true, }; setLogDebug(enableDebugLogging, "TSLS"); debug("serverInit()"); @@ -1230,17 +1210,8 @@ delete Object.prototype.__proto__; const ls = oldEntry ? oldEntry.ls : ts.createLanguageService(host, documentRegistry); - let forceEnabledVerbatimModuleSyntax = false; - if (!config["verbatimModuleSyntax"]) { - config["verbatimModuleSyntax"] = true; - forceEnabledVerbatimModuleSyntax = true; - } const compilerOptions = lspTsConfigToCompilerOptions(config); - newByScope.set(scope, { - ls, - compilerOptions, - forceEnabledVerbatimModuleSyntax, - }); + newByScope.set(scope, { ls, compilerOptions }); languageServiceEntries.byScope.delete(scope); } for (const oldEntry of languageServiceEntries.byScope.values()) { @@ -1305,7 +1276,7 @@ delete Object.prototype.__proto__; ...ls.getSemanticDiagnostics(specifier), ...ls.getSuggestionDiagnostics(specifier), ...ls.getSyntacticDiagnostics(specifier), - ].filter(filterMapDiagnostic.bind(null, true))); + ].filter(filterMapDiagnostic)); } return respond(id, diagnosticMap); } catch (e) { diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index 35a412b288..79e6dc5c48 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -5955,7 +5955,7 @@ fn lsp_jsr_code_action_missing_declaration() { "character": 6, }, }, - "newText": "import type { ReturnType } from \"jsr:@denotest/types-file/types\";\n", + "newText": "import { ReturnType } from \"jsr:@denotest/types-file/types\";\n", }, { "range": { @@ -6469,6 +6469,16 @@ fn lsp_code_actions_imports() { let context = TestContextBuilder::new().use_temp_cwd().build(); let mut client = context.new_lsp_command().build(); client.initialize_default(); + client.change_configuration(json!({ + "deno": { + "enable": true, + }, + "typescript": { + "preferences": { + "preferTypeOnlyAutoImports": true, + }, + }, + })); client.did_open(json!({ "textDocument": { "uri": "file:///a/file00.ts", @@ -6768,7 +6778,7 @@ fn lsp_code_actions_imports_dts() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "import type { SomeType } from \"./decl.d.ts\";\n", + "newText": "import { SomeType } from \"./decl.d.ts\";\n", }], }], }, @@ -7199,7 +7209,7 @@ fn lsp_code_actions_imports_respects_fmt_config() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 } }, - "newText": "import type { DuckConfigOptions } from './file01.ts'\n" + "newText": "import { DuckConfigOptions } from './file01.ts'\n" }] }] } @@ -7252,7 +7262,7 @@ fn lsp_code_actions_imports_respects_fmt_config() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 } }, - "newText": "import type { DuckConfigOptions } from './file01.ts'\n" + "newText": "import { DuckConfigOptions } from './file01.ts'\n" }] }] }, @@ -7352,7 +7362,7 @@ fn lsp_quote_style_from_workspace_settings() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "import type { DuckConfigOptions } from './file01.ts';\n", + "newText": "import { DuckConfigOptions } from './file01.ts';\n", }], }], }, @@ -7396,7 +7406,7 @@ fn lsp_quote_style_from_workspace_settings() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "import type { DuckConfigOptions } from \"./file01.ts\";\n", + "newText": "import { DuckConfigOptions } from \"./file01.ts\";\n", }], }], }, From 4626b11d032e964c64887e884f24b4359d3642bf Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 25 Oct 2024 13:37:26 -0400 Subject: [PATCH 23/63] fix(install): regression - do not panic when config file contains \r\n newlines (#26547) This is specifically for `deno install`/`deno add` commands. * https://github.com/dprint/jsonc-parser/pull/49 Closes https://github.com/denoland/deno/issues/26543 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e76f27c2f..3612cc23b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4005,9 +4005,9 @@ dependencies = [ [[package]] name = "jsonc-parser" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c78ad024523b61a2f20b1cad47413dd24db744a15d3d1b7276e69d1bee106c" +checksum = "b558af6b49fd918e970471374e7a798b2c9bbcda624a210ffa3901ee5614bc8e" dependencies = [ "serde_json", ] diff --git a/Cargo.toml b/Cargo.toml index 7ca53699ef..37fed81681 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,7 +137,7 @@ hyper-util = { version = "=0.1.7", features = ["tokio", "client", "client-legacy hyper_v014 = { package = "hyper", version = "0.14.26", features = ["runtime", "http1"] } indexmap = { version = "2", features = ["serde"] } ipnet = "2.3" -jsonc-parser = { version = "=0.26.1", features = ["serde"] } +jsonc-parser = { version = "=0.26.2", features = ["serde"] } lazy-regex = "3" libc = "0.2.126" libz-sys = { version = "1.1.20", default-features = false } From e70341e65e9f4e44811210c9b24e67a29b2c497a Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 25 Oct 2024 13:56:40 -0400 Subject: [PATCH 24/63] fix(check): ignore resolving `jsxImportSource` when jsx is not used in graph (#26548) --- cli/tsc/mod.rs | 46 ++++++++++++++++--- .../__test__.jsonc | 4 ++ .../jsx_import_source_not_in_graph/deno.json | 6 +++ .../jsx_import_source_not_in_graph/main.out | 1 + .../jsx_import_source_not_in_graph/main.ts | 1 + 5 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 tests/specs/check/jsx_import_source_not_in_graph/__test__.jsonc create mode 100644 tests/specs/check/jsx_import_source_not_in_graph/deno.json create mode 100644 tests/specs/check/jsx_import_source_not_in_graph/main.out create mode 100644 tests/specs/check/jsx_import_source_not_in_graph/main.ts diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index 0e3387494a..de91889b62 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -5,6 +5,7 @@ use crate::args::TypeCheckMode; use crate::cache::FastInsecureHasher; use crate::node; use crate::npm::CliNpmResolver; +use crate::npm::ResolvePkgFolderFromDenoReqError; use crate::util::checksum; use crate::util::path::mapped_specifier_for_tsc; @@ -35,6 +36,7 @@ use deno_runtime::deno_node::NodeResolver; use deno_semver::npm::NpmPackageReqReference; use node_resolver::errors::NodeJsErrorCode; use node_resolver::errors::NodeJsErrorCoded; +use node_resolver::errors::ResolvePkgSubpathFromDenoModuleError; use node_resolver::NodeModuleKind; use node_resolver::NodeResolution; use node_resolver::NodeResolutionMode; @@ -45,6 +47,7 @@ use std::fmt; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use thiserror::Error; mod diagnostics; @@ -688,12 +691,30 @@ fn op_resolve_inner( Some(ResolutionResolved { specifier, .. }) => { resolve_graph_specifier_types(specifier, &referrer, state)? } - _ => resolve_non_graph_specifier_types( - &specifier, - &referrer, - referrer_kind, - state, - )?, + _ => { + match resolve_non_graph_specifier_types( + &specifier, + &referrer, + referrer_kind, + state, + ) { + Ok(maybe_result) => maybe_result, + Err( + err @ ResolveNonGraphSpecifierTypesError::ResolvePkgFolderFromDenoReq( + ResolvePkgFolderFromDenoReqError::Managed(_), + ), + ) => { + // it's most likely requesting the jsxImportSource, which isn't loaded + // into the graph when not using jsx, so just ignore this error + if specifier.ends_with("/jsx-runtime") { + None + } else { + return Err(err.into()); + } + } + Err(err) => return Err(err.into()), + } + } }; let result = match maybe_result { Some((specifier, media_type)) => { @@ -818,12 +839,23 @@ fn resolve_graph_specifier_types( } } +#[derive(Debug, Error)] +enum ResolveNonGraphSpecifierTypesError { + #[error(transparent)] + ResolvePkgFolderFromDenoReq(#[from] ResolvePkgFolderFromDenoReqError), + #[error(transparent)] + ResolvePkgSubpathFromDenoModule(#[from] ResolvePkgSubpathFromDenoModuleError), +} + fn resolve_non_graph_specifier_types( raw_specifier: &str, referrer: &ModuleSpecifier, referrer_kind: NodeModuleKind, state: &State, -) -> Result, AnyError> { +) -> Result< + Option<(ModuleSpecifier, MediaType)>, + ResolveNonGraphSpecifierTypesError, +> { let npm = match state.maybe_npm.as_ref() { Some(npm) => npm, None => return Ok(None), // we only support non-graph types for npm packages diff --git a/tests/specs/check/jsx_import_source_not_in_graph/__test__.jsonc b/tests/specs/check/jsx_import_source_not_in_graph/__test__.jsonc new file mode 100644 index 0000000000..de0339cfb4 --- /dev/null +++ b/tests/specs/check/jsx_import_source_not_in_graph/__test__.jsonc @@ -0,0 +1,4 @@ +{ + "args": "check main.ts", + "output": "main.out" +} diff --git a/tests/specs/check/jsx_import_source_not_in_graph/deno.json b/tests/specs/check/jsx_import_source_not_in_graph/deno.json new file mode 100644 index 0000000000..219ef515f6 --- /dev/null +++ b/tests/specs/check/jsx_import_source_not_in_graph/deno.json @@ -0,0 +1,6 @@ +{ + "compilerOptions": { + "jsx": "react-jsx", + "jsxImportSource": "npm:preact" + } +} diff --git a/tests/specs/check/jsx_import_source_not_in_graph/main.out b/tests/specs/check/jsx_import_source_not_in_graph/main.out new file mode 100644 index 0000000000..1830c3186d --- /dev/null +++ b/tests/specs/check/jsx_import_source_not_in_graph/main.out @@ -0,0 +1 @@ +Check file:///[WILDLINE]/main.ts diff --git a/tests/specs/check/jsx_import_source_not_in_graph/main.ts b/tests/specs/check/jsx_import_source_not_in_graph/main.ts new file mode 100644 index 0000000000..bf6b817963 --- /dev/null +++ b/tests/specs/check/jsx_import_source_not_in_graph/main.ts @@ -0,0 +1 @@ +console.log("Hello"); From a01edb394d2785b7d37da6435bb37381efc376ea Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 25 Oct 2024 15:58:28 -0400 Subject: [PATCH 25/63] fix(upgrade): stop running `deno lsp` processes on windows before attempting to replace executable (#26542) --- cli/tools/upgrade.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cli/tools/upgrade.rs b/cli/tools/upgrade.rs index b1b09d1a6f..77a9f72b80 100644 --- a/cli/tools/upgrade.rs +++ b/cli/tools/upgrade.rs @@ -579,6 +579,10 @@ pub async fn upgrade( let output_exe_path = full_path_output_flag.as_ref().unwrap_or(¤t_exe_path); + + #[cfg(windows)] + kill_running_deno_lsp_processes(); + let output_result = if *output_exe_path == current_exe_path { replace_exe(&new_exe_path, output_exe_path) } else { @@ -966,6 +970,34 @@ fn check_windows_access_denied_error( }) } +#[cfg(windows)] +fn kill_running_deno_lsp_processes() { + // limit this to `deno lsp` invocations to avoid killing important programs someone might be running + let is_debug = log::log_enabled!(log::Level::Debug); + let get_pipe = || { + if is_debug { + std::process::Stdio::inherit() + } else { + std::process::Stdio::null() + } + }; + let _ = Command::new("powershell.exe") + .args([ + "-Command", + r#"Get-WmiObject Win32_Process | Where-Object { + $_.Name -eq 'deno.exe' -and + $_.CommandLine -match '^(?:\"[^\"]+\"|\S+)\s+lsp\b' +} | ForEach-Object { + if ($_.Terminate()) { + Write-Host 'Terminated:' $_.ProcessId + } +}"#, + ]) + .stdout(get_pipe()) + .stderr(get_pipe()) + .output(); +} + fn set_exe_permissions( current_exe_path: &Path, output_exe_path: &Path, From ec968aa5aec068e92fb554fc7192d912bcddb82c Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:19:03 -0700 Subject: [PATCH 26/63] fix(install): cache json exports of JSR packages (#26552) Fixes https://github.com/denoland/deno/issues/26509. Ended up being a `deno_graph` bug causing the error to surface. This PR updates `deno_graph` to pick up the fix and reverts the temporary workaround that skipped JSON exports. --- Cargo.lock | 4 ++-- cli/Cargo.toml | 2 +- cli/tools/registry/pm/cache_deps.rs | 7 +------ .../registry/jsr/@denotest/multiple-exports/1.0.0/add.ts | 1 + .../jsr/@denotest/multiple-exports/1.0.0/data.json | 3 +++ .../jsr/@denotest/multiple-exports/1.0.0/subtract.ts | 1 + .../jsr/@denotest/multiple-exports/1.0.0_meta.json | 7 +++++++ tests/registry/jsr/@denotest/multiple-exports/meta.json | 5 +++++ tests/specs/install/jsr_exports/__test__.jsonc | 3 ++- tests/specs/install/jsr_exports/deno.json | 2 +- tests/specs/install/jsr_exports/install.out | 9 +++++---- tests/specs/install/jsr_exports/main.out | 3 +++ tests/specs/install/jsr_exports/main.ts | 7 +++++++ 13 files changed, 39 insertions(+), 15 deletions(-) create mode 100644 tests/registry/jsr/@denotest/multiple-exports/1.0.0/add.ts create mode 100644 tests/registry/jsr/@denotest/multiple-exports/1.0.0/data.json create mode 100644 tests/registry/jsr/@denotest/multiple-exports/1.0.0/subtract.ts create mode 100644 tests/registry/jsr/@denotest/multiple-exports/1.0.0_meta.json create mode 100644 tests/registry/jsr/@denotest/multiple-exports/meta.json create mode 100644 tests/specs/install/jsr_exports/main.out create mode 100644 tests/specs/install/jsr_exports/main.ts diff --git a/Cargo.lock b/Cargo.lock index 3612cc23b6..623d028ce6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1606,9 +1606,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.83.3" +version = "0.83.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77163c46755676d8f793fc19e365537ba660a8db173cd1e02d21eb010c0b3cef" +checksum = "5bd20bc0780071989c622cbfd5d4fb2e4fd05a247ccd7f791f13c8d2c3792228" dependencies = [ "anyhow", "async-trait", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index db0248d846..0065a2cbd0 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -73,7 +73,7 @@ deno_cache_dir = { workspace = true } deno_config = { version = "=0.37.2", features = ["workspace", "sync"] } deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_doc = { version = "0.154.0", default-features = false, features = ["rust", "html", "syntect"] } -deno_graph = { version = "=0.83.3" } +deno_graph = { version = "=0.83.4" } deno_lint = { version = "=0.67.0", features = ["docs"] } deno_lockfile.workspace = true deno_npm.workspace = true diff --git a/cli/tools/registry/pm/cache_deps.rs b/cli/tools/registry/pm/cache_deps.rs index 9883deb1d5..365622d11f 100644 --- a/cli/tools/registry/pm/cache_deps.rs +++ b/cli/tools/registry/pm/cache_deps.rs @@ -90,13 +90,8 @@ pub async fn cache_top_level_deps( while let Some(info_future) = info_futures.next().await { if let Some((specifier, info)) = info_future { let exports = info.exports(); - for (k, v) in exports { + for (k, _) in exports { if let Ok(spec) = specifier.join(k) { - if v.ends_with(".json") { - // TODO(nathanwhit): this should work, there's a bug with - // json roots in deno_graph. skip it for now - continue; - } roots.push(spec); } } diff --git a/tests/registry/jsr/@denotest/multiple-exports/1.0.0/add.ts b/tests/registry/jsr/@denotest/multiple-exports/1.0.0/add.ts new file mode 100644 index 0000000000..de02f69024 --- /dev/null +++ b/tests/registry/jsr/@denotest/multiple-exports/1.0.0/add.ts @@ -0,0 +1 @@ +export * from "jsr:@denotest/add@1"; diff --git a/tests/registry/jsr/@denotest/multiple-exports/1.0.0/data.json b/tests/registry/jsr/@denotest/multiple-exports/1.0.0/data.json new file mode 100644 index 0000000000..885e71c6cc --- /dev/null +++ b/tests/registry/jsr/@denotest/multiple-exports/1.0.0/data.json @@ -0,0 +1,3 @@ +{ + "a": 1 +} \ No newline at end of file diff --git a/tests/registry/jsr/@denotest/multiple-exports/1.0.0/subtract.ts b/tests/registry/jsr/@denotest/multiple-exports/1.0.0/subtract.ts new file mode 100644 index 0000000000..215c42310d --- /dev/null +++ b/tests/registry/jsr/@denotest/multiple-exports/1.0.0/subtract.ts @@ -0,0 +1 @@ +export * from "jsr:@denotest/subtract@1"; diff --git a/tests/registry/jsr/@denotest/multiple-exports/1.0.0_meta.json b/tests/registry/jsr/@denotest/multiple-exports/1.0.0_meta.json new file mode 100644 index 0000000000..d9f58b9a61 --- /dev/null +++ b/tests/registry/jsr/@denotest/multiple-exports/1.0.0_meta.json @@ -0,0 +1,7 @@ +{ + "exports": { + "./add": "./add.ts", + "./subtract": "./subtract.ts", + "./data-json": "./data.json" + } +} diff --git a/tests/registry/jsr/@denotest/multiple-exports/meta.json b/tests/registry/jsr/@denotest/multiple-exports/meta.json new file mode 100644 index 0000000000..02601e4d0d --- /dev/null +++ b/tests/registry/jsr/@denotest/multiple-exports/meta.json @@ -0,0 +1,5 @@ +{ + "versions": { + "1.0.0": {} + } +} diff --git a/tests/specs/install/jsr_exports/__test__.jsonc b/tests/specs/install/jsr_exports/__test__.jsonc index 934f3c5885..7ccc5da4d9 100644 --- a/tests/specs/install/jsr_exports/__test__.jsonc +++ b/tests/specs/install/jsr_exports/__test__.jsonc @@ -1,6 +1,7 @@ { "tempDir": true, "steps": [ - { "args": "install", "output": "install.out" } + { "args": "install", "output": "install.out" }, + { "args": "run --cached-only main.ts", "output": "main.out" } ] } diff --git a/tests/specs/install/jsr_exports/deno.json b/tests/specs/install/jsr_exports/deno.json index 21212f5794..4b281f80ae 100644 --- a/tests/specs/install/jsr_exports/deno.json +++ b/tests/specs/install/jsr_exports/deno.json @@ -1,5 +1,5 @@ { "imports": { - "@denotest/different-deps-per-export": "jsr:@denotest/different-deps-per-export@^1.0.0" + "@denotest/multiple-exports": "jsr:@denotest/multiple-exports@^1.0.0" } } diff --git a/tests/specs/install/jsr_exports/install.out b/tests/specs/install/jsr_exports/install.out index 8f64001636..bce0d79bb3 100644 --- a/tests/specs/install/jsr_exports/install.out +++ b/tests/specs/install/jsr_exports/install.out @@ -1,8 +1,9 @@ [UNORDERED_START] -Download http://127.0.0.1:4250/@denotest/different-deps-per-export/meta.json -Download http://127.0.0.1:4250/@denotest/different-deps-per-export/1.0.0_meta.json -Download http://127.0.0.1:4250/@denotest/different-deps-per-export/1.0.0/add.ts -Download http://127.0.0.1:4250/@denotest/different-deps-per-export/1.0.0/subtract.ts +Download http://127.0.0.1:4250/@denotest/multiple-exports/meta.json +Download http://127.0.0.1:4250/@denotest/multiple-exports/1.0.0_meta.json +Download http://127.0.0.1:4250/@denotest/multiple-exports/1.0.0/add.ts +Download http://127.0.0.1:4250/@denotest/multiple-exports/1.0.0/subtract.ts +Download http://127.0.0.1:4250/@denotest/multiple-exports/1.0.0/data.json Download http://127.0.0.1:4250/@denotest/add/meta.json Download http://127.0.0.1:4250/@denotest/subtract/meta.json Download http://127.0.0.1:4250/@denotest/add/1.0.0_meta.json diff --git a/tests/specs/install/jsr_exports/main.out b/tests/specs/install/jsr_exports/main.out new file mode 100644 index 0000000000..6ce184bada --- /dev/null +++ b/tests/specs/install/jsr_exports/main.out @@ -0,0 +1,3 @@ +3 +-1 +{ a: 1 } diff --git a/tests/specs/install/jsr_exports/main.ts b/tests/specs/install/jsr_exports/main.ts new file mode 100644 index 0000000000..f7efdb0e98 --- /dev/null +++ b/tests/specs/install/jsr_exports/main.ts @@ -0,0 +1,7 @@ +import { add } from "@denotest/multiple-exports/add"; +import { subtract } from "@denotest/multiple-exports/subtract"; +import data from "@denotest/multiple-exports/data-json" with { type: "json" }; + +console.log(add(1, 2)); +console.log(subtract(1, 2)); +console.log(data); From d92d2fe9b0bd5e6e29cb3b6f924472aec972b636 Mon Sep 17 00:00:00 2001 From: Nayeem Rahman Date: Fri, 25 Oct 2024 21:52:50 +0100 Subject: [PATCH 27/63] fix(lsp): make missing import action fix infallible (#26539) --- cli/lsp/analysis.rs | 103 +++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 55 deletions(-) diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index 8c2e8bb1dc..39f1ae6482 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -18,7 +18,6 @@ use deno_lint::diagnostic::LintDiagnosticRange; use deno_ast::SourceRange; use deno_ast::SourceRangedForSpanned; use deno_ast::SourceTextInfo; -use deno_core::anyhow::anyhow; use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::serde::Deserialize; @@ -40,6 +39,7 @@ use import_map::ImportMap; use node_resolver::NpmResolver; use once_cell::sync::Lazy; use regex::Regex; +use std::borrow::Cow; use std::cmp::Ordering; use std::collections::HashMap; use std::collections::HashSet; @@ -598,68 +598,62 @@ pub fn fix_ts_import_changes( /// Fix tsc import code actions so that the module specifier is correct for /// resolution by Deno (includes the extension). -fn fix_ts_import_action( +fn fix_ts_import_action<'a>( referrer: &ModuleSpecifier, - action: &tsc::CodeFixAction, + action: &'a tsc::CodeFixAction, import_mapper: &TsResponseImportMapper, -) -> Result, AnyError> { - if matches!( +) -> Option> { + if !matches!( action.fix_name.as_str(), "import" | "fixMissingFunctionDeclaration" ) { - let change = action + return Some(Cow::Borrowed(action)); + } + let specifier = (|| { + let text_change = action.changes.first()?.text_changes.first()?; + let captures = IMPORT_SPECIFIER_RE.captures(&text_change.new_text)?; + Some(captures.get(1)?.as_str()) + })(); + let Some(specifier) = specifier else { + return Some(Cow::Borrowed(action)); + }; + if let Some(new_specifier) = + import_mapper.check_unresolved_specifier(specifier, referrer) + { + let description = action.description.replace(specifier, &new_specifier); + let changes = action .changes - .first() - .ok_or_else(|| anyhow!("Unexpected action changes."))?; - let text_change = change - .text_changes - .first() - .ok_or_else(|| anyhow!("Missing text change."))?; - if let Some(captures) = IMPORT_SPECIFIER_RE.captures(&text_change.new_text) - { - let specifier = captures - .get(1) - .ok_or_else(|| anyhow!("Missing capture."))? - .as_str(); - if let Some(new_specifier) = - import_mapper.check_unresolved_specifier(specifier, referrer) - { - let description = action.description.replace(specifier, &new_specifier); - let changes = action - .changes + .iter() + .map(|c| { + let text_changes = c + .text_changes .iter() - .map(|c| { - let text_changes = c - .text_changes - .iter() - .map(|tc| tsc::TextChange { - span: tc.span.clone(), - new_text: tc.new_text.replace(specifier, &new_specifier), - }) - .collect(); - tsc::FileTextChanges { - file_name: c.file_name.clone(), - text_changes, - is_new_file: c.is_new_file, - } + .map(|tc| tsc::TextChange { + span: tc.span.clone(), + new_text: tc.new_text.replace(specifier, &new_specifier), }) .collect(); + tsc::FileTextChanges { + file_name: c.file_name.clone(), + text_changes, + is_new_file: c.is_new_file, + } + }) + .collect(); - return Ok(Some(tsc::CodeFixAction { - description, - changes, - commands: None, - fix_name: action.fix_name.clone(), - fix_id: None, - fix_all_description: None, - })); - } else if !import_mapper.is_valid_import(specifier, referrer) { - return Ok(None); - } - } + Some(Cow::Owned(tsc::CodeFixAction { + description, + changes, + commands: None, + fix_name: action.fix_name.clone(), + fix_id: None, + fix_all_description: None, + })) + } else if !import_mapper.is_valid_import(specifier, referrer) { + None + } else { + Some(Cow::Borrowed(action)) } - - Ok(Some(action.clone())) } /// Determines if two TypeScript diagnostic codes are effectively equivalent. @@ -1004,8 +998,7 @@ impl CodeActionCollection { specifier, action, &language_server.get_ts_response_import_mapper(specifier), - )? - else { + ) else { return Ok(()); }; let edit = ts_changes_to_edit(&action.changes, language_server)?; @@ -1027,7 +1020,7 @@ impl CodeActionCollection { }); self .actions - .push(CodeActionKind::Tsc(code_action, action.clone())); + .push(CodeActionKind::Tsc(code_action, action.as_ref().clone())); if let Some(fix_id) = &action.fix_id { if let Some(CodeActionKind::Tsc(existing_fix_all, existing_action)) = From f0f476e58453d502ab5c118fc91d1475e6829967 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 26 Oct 2024 13:41:09 -0400 Subject: [PATCH 28/63] perf: pass transpiled module to deno_core as known string (#26555) --- cli/cache/emit.rs | 25 +++++------------- cli/emit.rs | 55 +++++++++++++++++++++------------------ cli/module_loader.rs | 6 +++-- cli/standalone/binary.rs | 2 +- cli/tools/coverage/mod.rs | 5 ++-- 5 files changed, 43 insertions(+), 50 deletions(-) diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index 6807f06c10..74e1c1101b 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -39,7 +39,7 @@ impl EmitCache { &self, specifier: &ModuleSpecifier, expected_source_hash: u64, - ) -> Option> { + ) -> Option { let emit_filename = self.get_emit_filename(specifier)?; let bytes = self.disk_cache.get(&emit_filename).ok()?; self @@ -100,7 +100,7 @@ impl EmitFileSerializer { &self, mut bytes: Vec, expected_source_hash: u64, - ) -> Option> { + ) -> Option { let last_newline_index = bytes.iter().rposition(|&b| b == b'\n')?; let (content, last_line) = bytes.split_at(last_newline_index); let hashes = last_line.strip_prefix(LAST_LINE_PREFIX.as_bytes())?; @@ -120,7 +120,7 @@ impl EmitFileSerializer { // everything looks good, truncate and return it bytes.truncate(content.len()); - Some(bytes) + String::from_utf8(bytes).ok() } pub fn serialize(&self, code: &[u8], source_hash: u64) -> Vec { @@ -170,8 +170,6 @@ mod test { }, emit_failed_flag: Default::default(), }; - let to_string = - |bytes: Vec| -> String { String::from_utf8(bytes).unwrap() }; let specifier1 = ModuleSpecifier::from_file_path(temp_dir.path().join("file1.ts")) @@ -188,13 +186,10 @@ mod test { assert_eq!(cache.get_emit_code(&specifier1, 5), None); // providing the correct source hash assert_eq!( - cache.get_emit_code(&specifier1, 10).map(to_string), + cache.get_emit_code(&specifier1, 10), Some(emit_code1.clone()), ); - assert_eq!( - cache.get_emit_code(&specifier2, 2).map(to_string), - Some(emit_code2) - ); + assert_eq!(cache.get_emit_code(&specifier2, 2), Some(emit_code2)); // try changing the cli version (should not load previous ones) let cache = EmitCache { @@ -215,18 +210,12 @@ mod test { }, emit_failed_flag: Default::default(), }; - assert_eq!( - cache.get_emit_code(&specifier1, 5).map(to_string), - Some(emit_code1) - ); + assert_eq!(cache.get_emit_code(&specifier1, 5), Some(emit_code1)); // adding when already exists should not cause issue let emit_code3 = "asdf".to_string(); cache.set_emit_code(&specifier1, 20, emit_code3.as_bytes()); assert_eq!(cache.get_emit_code(&specifier1, 5), None); - assert_eq!( - cache.get_emit_code(&specifier1, 20).map(to_string), - Some(emit_code3) - ); + assert_eq!(cache.get_emit_code(&specifier1, 20), Some(emit_code3)); } } diff --git a/cli/emit.rs b/cli/emit.rs index ad200af050..8e93092e67 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -13,7 +13,6 @@ use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; use deno_core::futures::FutureExt; use deno_core::futures::StreamExt; -use deno_core::ModuleCodeBytes; use deno_core::ModuleSpecifier; use deno_graph::MediaType; use deno_graph::Module; @@ -94,7 +93,7 @@ impl Emitter { &self, specifier: &ModuleSpecifier, source: &str, - ) -> Option> { + ) -> Option { let source_hash = self.get_source_hash(source); self.emit_cache.get_emit_code(specifier, source_hash) } @@ -104,7 +103,7 @@ impl Emitter { specifier: &ModuleSpecifier, media_type: MediaType, source: &Arc, - ) -> Result { + ) -> Result { // Note: keep this in sync with the sync version below let helper = EmitParsedSourceHelper(self); match helper.pre_emit_parsed_source(specifier, source) { @@ -113,7 +112,7 @@ impl Emitter { let parsed_source_cache = self.parsed_source_cache.clone(); let transpile_and_emit_options = self.transpile_and_emit_options.clone(); - let transpile_result = deno_core::unsync::spawn_blocking({ + let transpiled_source = deno_core::unsync::spawn_blocking({ let specifier = specifier.clone(); let source = source.clone(); move || -> Result<_, AnyError> { @@ -129,11 +128,12 @@ impl Emitter { }) .await .unwrap()?; - Ok(helper.post_emit_parsed_source( + helper.post_emit_parsed_source( specifier, - transpile_result, + &transpiled_source, source_hash, - )) + ); + Ok(transpiled_source) } } } @@ -143,13 +143,13 @@ impl Emitter { specifier: &ModuleSpecifier, media_type: MediaType, source: &Arc, - ) -> Result { + ) -> Result { // Note: keep this in sync with the async version above let helper = EmitParsedSourceHelper(self); match helper.pre_emit_parsed_source(specifier, source) { PreEmitResult::Cached(emitted_text) => Ok(emitted_text), PreEmitResult::NotCached { source_hash } => { - let transpile_result = EmitParsedSourceHelper::transpile( + let transpiled_source = EmitParsedSourceHelper::transpile( &self.parsed_source_cache, specifier, source.clone(), @@ -157,11 +157,12 @@ impl Emitter { &self.transpile_and_emit_options.0, &self.transpile_and_emit_options.1, )?; - Ok(helper.post_emit_parsed_source( + helper.post_emit_parsed_source( specifier, - transpile_result, + &transpiled_source, source_hash, - )) + ); + Ok(transpiled_source) } } } @@ -227,7 +228,7 @@ impl Emitter { } enum PreEmitResult { - Cached(ModuleCodeBytes), + Cached(String), NotCached { source_hash: u64 }, } @@ -245,7 +246,7 @@ impl<'a> EmitParsedSourceHelper<'a> { if let Some(emit_code) = self.0.emit_cache.get_emit_code(specifier, source_hash) { - PreEmitResult::Cached(emit_code.into_boxed_slice().into()) + PreEmitResult::Cached(emit_code) } else { PreEmitResult::NotCached { source_hash } } @@ -258,21 +259,14 @@ impl<'a> EmitParsedSourceHelper<'a> { media_type: MediaType, transpile_options: &deno_ast::TranspileOptions, emit_options: &deno_ast::EmitOptions, - ) -> Result { + ) -> Result { // nothing else needs the parsed source at this point, so remove from // the cache in order to not transpile owned let parsed_source = parsed_source_cache .remove_or_parse_module(specifier, source, media_type)?; ensure_no_import_assertion(&parsed_source)?; - Ok(parsed_source.transpile(transpile_options, emit_options)?) - } - - pub fn post_emit_parsed_source( - &self, - specifier: &ModuleSpecifier, - transpile_result: TranspileResult, - source_hash: u64, - ) -> ModuleCodeBytes { + let transpile_result = + parsed_source.transpile(transpile_options, emit_options)?; let transpiled_source = match transpile_result { TranspileResult::Owned(source) => source, TranspileResult::Cloned(source) => { @@ -281,12 +275,21 @@ impl<'a> EmitParsedSourceHelper<'a> { } }; debug_assert!(transpiled_source.source_map.is_none()); + let text = String::from_utf8(transpiled_source.source)?; + Ok(text) + } + + pub fn post_emit_parsed_source( + &self, + specifier: &ModuleSpecifier, + transpiled_source: &str, + source_hash: u64, + ) { self.0.emit_cache.set_emit_code( specifier, source_hash, - &transpiled_source.source, + transpiled_source.as_bytes(), ); - transpiled_source.source.into_boxed_slice().into() } } diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 37d42f78e5..4a020516e7 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -541,7 +541,8 @@ impl self.parsed_source_cache.free(specifier); Ok(Some(ModuleCodeStringSource { - code: ModuleSourceCode::Bytes(transpile_result), + // note: it's faster to provide a string if we know it's a string + code: ModuleSourceCode::String(transpile_result.into()), found_url: specifier.clone(), media_type, })) @@ -571,7 +572,8 @@ impl self.parsed_source_cache.free(specifier); Ok(Some(ModuleCodeStringSource { - code: ModuleSourceCode::Bytes(transpile_result), + // note: it's faster to provide a string if we know it's a string + code: ModuleSourceCode::String(transpile_result.into()), found_url: specifier.clone(), media_type, })) diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 0f8b0b49d4..f41f3003ff 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -613,7 +613,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { .emitter .emit_parsed_source(&m.specifier, m.media_type, &m.source) .await?; - source.to_vec() + source.into_bytes() } else { m.source.as_bytes().to_vec() }; diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 260c0c8424..3b08f2c77a 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -571,7 +571,7 @@ pub async fn cover_files( | MediaType::Cjs | MediaType::Mjs | MediaType::Json => None, - MediaType::Dts | MediaType::Dmts | MediaType::Dcts => Some(Vec::new()), + MediaType::Dts | MediaType::Dmts | MediaType::Dcts => Some(String::new()), MediaType::TypeScript | MediaType::Jsx | MediaType::Mts @@ -593,8 +593,7 @@ pub async fn cover_files( } }; let runtime_code: String = match transpiled_code { - Some(code) => String::from_utf8(code) - .with_context(|| format!("Failed decoding {}", file.specifier))?, + Some(code) => code, None => original_source.to_string(), }; From 793b155cd31bfc79e4aac4447bd250c9a6d31f6b Mon Sep 17 00:00:00 2001 From: Mayank Kumar <103447831+mayank-Pareek@users.noreply.github.com> Date: Sat, 26 Oct 2024 13:42:14 -0400 Subject: [PATCH 29/63] fix(ext/node): use primordials in ext\node\polyfills\internal\crypto\_randomInt.ts (#26534) Towards #24236 --- .../polyfills/internal/crypto/_randomInt.ts | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/ext/node/polyfills/internal/crypto/_randomInt.ts b/ext/node/polyfills/internal/crypto/_randomInt.ts index 7f4d703ad4..e08b3e9639 100644 --- a/ext/node/polyfills/internal/crypto/_randomInt.ts +++ b/ext/node/polyfills/internal/crypto/_randomInt.ts @@ -1,9 +1,15 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -// TODO(petamoriken): enable prefer-primordials for node polyfills -// deno-lint-ignore-file prefer-primordials - import { op_node_random_int } from "ext:core/ops"; +import { primordials } from "ext:core/mod.js"; +const { + Error, + MathCeil, + MathFloor, + MathPow, + NumberIsSafeInteger, + RangeError, +} = primordials; export default function randomInt(max: number): number; export default function randomInt(min: number, max: number): number; @@ -23,7 +29,9 @@ export default function randomInt( cb?: (err: Error | null, n?: number) => void, ): number | void { if (typeof max === "number" && typeof min === "number") { - [max, min] = [min, max]; + const temp = max; + max = min; + min = temp; } if (min === undefined) min = 0; else if (typeof min === "function") { @@ -32,13 +40,13 @@ export default function randomInt( } if ( - !Number.isSafeInteger(min) || - typeof max === "number" && !Number.isSafeInteger(max) + !NumberIsSafeInteger(min) || + typeof max === "number" && !NumberIsSafeInteger(max) ) { throw new Error("max or min is not a Safe Number"); } - if (max - min > Math.pow(2, 48)) { + if (max - min > MathPow(2, 48)) { throw new RangeError("max - min should be less than 2^48!"); } @@ -46,8 +54,8 @@ export default function randomInt( throw new Error("Min is bigger than Max!"); } - min = Math.ceil(min); - max = Math.floor(max); + min = MathCeil(min); + max = MathFloor(max); const result = op_node_random_int(min, max); if (cb) { From 75f00eecb0cf5e0002fe52da94d764fc6cb0e817 Mon Sep 17 00:00:00 2001 From: Chris Veness Date: Sat, 26 Oct 2024 23:12:40 +0100 Subject: [PATCH 30/63] fix(cli): Make --watcher CLEAR_SCREEN clear scrollback buffer as well as visible screen (#25997) The --watch option should clear the screen scrollback buffer as well as the screen itself. On Ubuntu (22.04 Jammy) the 'clear' command generates "\x1B[H\x1B[2J\x1B[3J"; that is: - \E[H - cursor home - \E[2J - clear entire screen - \E[3J - clear entire screen & scrollback buffer. By contrast, Deno defined CLEAR_SCREEN as "\x1B[2J\x1B[1;1H", which fails to clear the scrollback buffer. The "\E[H\E[2J\E[3J" sequence works on MacOS (Sonoma) (using printf); I'm not able to test on Windows. Closes https://github.com/denoland/deno/issues/26514 --- cli/util/file_watcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/util/file_watcher.rs b/cli/util/file_watcher.rs index d92d880bc1..8d734af88e 100644 --- a/cli/util/file_watcher.rs +++ b/cli/util/file_watcher.rs @@ -30,7 +30,7 @@ use tokio::sync::mpsc; use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::sleep; -const CLEAR_SCREEN: &str = "\x1B[2J\x1B[1;1H"; +const CLEAR_SCREEN: &str = "\x1B[H\x1B[2J\x1B[3J"; const DEBOUNCE_INTERVAL: Duration = Duration::from_millis(200); struct DebouncedReceiver { From ab3d02a08174042c6337d130b02a6fa0f85673d2 Mon Sep 17 00:00:00 2001 From: Meir Blachman Date: Sun, 27 Oct 2024 01:31:44 +0200 Subject: [PATCH 31/63] docs(readme): Add winget instructions (#25136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Meir Blachman Co-authored-by: Bartek Iwańczuk --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 102319f4f2..19d4fa8a12 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ brew install deno choco install deno ``` +[WinGet](https://winstall.app/apps/DenoLand.Deno) (Windows): + +```powershell +winget install --id=DenoLand.Deno +``` + ### Build and install from source Complete instructions for building Deno from source can be found in the manual From 05868cc2367d6c827691d7624478157239048bea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sun, 27 Oct 2024 03:00:19 +0000 Subject: [PATCH 32/63] fix(watch): don't panic on invalid file specifiers (#26577) Removes an unwrap that falsely assumed the specifier is a valid file path. Fixes https://github.com/denoland/deno/issues/26209 --- cli/graph_util.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cli/graph_util.rs b/cli/graph_util.rs index e67ae7821b..2eaee228af 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -1009,7 +1009,11 @@ impl deno_graph::source::Reporter for FileWatcherReporter { ) { let mut file_paths = self.file_paths.lock(); if specifier.scheme() == "file" { - file_paths.push(specifier.to_file_path().unwrap()); + // Don't trust that the path is a valid path at this point: + // https://github.com/denoland/deno/issues/26209. + if let Ok(file_path) = specifier.to_file_path() { + file_paths.push(file_path); + } } if modules_done == modules_total { From c314b2d8577289078d6b00a0dd58f8f36ff6920a Mon Sep 17 00:00:00 2001 From: familyboat <84062528+familyboat@users.noreply.github.com> Date: Sun, 27 Oct 2024 11:04:35 +0800 Subject: [PATCH 33/63] fix(ext/node): add path to `fs.stat` and `fs.statSync` error (#26037) --- ext/node/polyfills/_fs/_fs_stat.ts | 14 ++++++++-- tests/unit_node/fs_test.ts | 43 ++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/ext/node/polyfills/_fs/_fs_stat.ts b/ext/node/polyfills/_fs/_fs_stat.ts index c4ed82d577..d00c81ffb6 100644 --- a/ext/node/polyfills/_fs/_fs_stat.ts +++ b/ext/node/polyfills/_fs/_fs_stat.ts @@ -383,7 +383,10 @@ export function stat( Deno.stat(path).then( (stat) => callback(null, CFISBIS(stat, options.bigint)), - (err) => callback(denoErrorToNodeError(err, { syscall: "stat" })), + (err) => + callback( + denoErrorToNodeError(err, { syscall: "stat", path: getPathname(path) }), + ), ); } @@ -417,9 +420,16 @@ export function statSync( return; } if (err instanceof Error) { - throw denoErrorToNodeError(err, { syscall: "stat" }); + throw denoErrorToNodeError(err, { + syscall: "stat", + path: getPathname(path), + }); } else { throw err; } } } + +function getPathname(path: string | URL) { + return typeof path === "string" ? path : path.pathname; +} diff --git a/tests/unit_node/fs_test.ts b/tests/unit_node/fs_test.ts index ef8f829cda..631608d7cc 100644 --- a/tests/unit_node/fs_test.ts +++ b/tests/unit_node/fs_test.ts @@ -26,6 +26,7 @@ import { cp, FileHandle, open, + stat, writeFile, } from "node:fs/promises"; import process from "node:process"; @@ -123,6 +124,48 @@ Deno.test( }, ); +Deno.test( + "[node/fs statSync] throw error with path information", + () => { + const file = "non-exist-file"; + const fileUrl = new URL(file, import.meta.url); + + assertThrows(() => { + statSync(file); + }, "Error: ENOENT: no such file or directory, stat 'non-exist-file'"); + + assertThrows(() => { + statSync(fileUrl); + }, `Error: ENOENT: no such file or directory, stat '${fileUrl.pathname}'`); + }, +); + +Deno.test( + "[node/fs/promises stat] throw error with path information", + async () => { + const file = "non-exist-file"; + const fileUrl = new URL(file, import.meta.url); + + try { + await stat(file); + } catch (error: unknown) { + assertEquals( + `${error}`, + "Error: ENOENT: no such file or directory, stat 'non-exist-file'", + ); + } + + try { + await stat(fileUrl); + } catch (error: unknown) { + assertEquals( + `${error}`, + `Error: ENOENT: no such file or directory, stat '${fileUrl.pathname}'`, + ); + } + }, +); + Deno.test( "[node/fs/promises cp] copy file", async () => { From 3a306c450ce99067c4322720792d909afaf78841 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 27 Oct 2024 22:03:30 -0400 Subject: [PATCH 34/63] fix: do not panic when failing to write to http cache (#26591) Closes https://github.com/denoland/deno/issues/26189 Closes https://github.com/denoland/deno/issues/26575 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 623d028ce6..2c8028a1e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1356,9 +1356,9 @@ dependencies = [ [[package]] name = "deno_cache_dir" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186a102b13b4512841f5f40784cd25822042d22954afe3b5b070d406d15eb4f2" +checksum = "693ca429aebf945de5fef30df232044f9f80be4cc5a5e7c8d767226c43880f5a" dependencies = [ "base32", "deno_media_type", diff --git a/Cargo.toml b/Cargo.toml index 37fed81681..b622cbe443 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,7 +111,7 @@ console_static_text = "=0.8.1" dashmap = "5.5.3" data-encoding = "2.3.3" data-url = "=0.3.0" -deno_cache_dir = "=0.13.0" +deno_cache_dir = "=0.13.1" deno_package_json = { version = "0.1.2", default-features = false } dlopen2 = "0.6.1" ecb = "=0.1.2" From 5389972ba5037b1ed48da11506f6798deec2b48e Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Mon, 28 Oct 2024 12:08:51 +0100 Subject: [PATCH 35/63] fix(npm): match npm bearer token generation (#26544) Spend some time stepping through the npm client code and noticed that the bearer token was different from ours. They do some double encoding and @dsherret helped me in matching the encoding behavior. Fixes https://github.com/denoland/deno/issues/26033 --- cli/npm/common.rs | 23 +++++++++++-------- .../specs/npm/npmrc_username_password/.npmrc | 6 +++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/cli/npm/common.rs b/cli/npm/common.rs index de282310a1..55f1bc086d 100644 --- a/cli/npm/common.rs +++ b/cli/npm/common.rs @@ -3,6 +3,7 @@ use base64::prelude::BASE64_STANDARD; use base64::Engine; use deno_core::anyhow::bail; +use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_npm::npm_rc::RegistryConfig; use http::header; @@ -36,17 +37,21 @@ pub fn maybe_auth_header_for_npm_registry( } if username.is_some() && password.is_some() { + // The npm client does some double encoding when generating the + // bearer token value, see + // https://github.com/npm/cli/blob/780afc50e3a345feb1871a28e33fa48235bc3bd5/workspaces/config/lib/index.js#L846-L851 + let pw_base64 = BASE64_STANDARD + .decode(password.unwrap()) + .with_context(|| "The password in npmrc is an invalid base64 string")?; + let bearer = BASE64_STANDARD.encode(format!( + "{}:{}", + username.unwrap(), + String::from_utf8_lossy(&pw_base64) + )); + return Ok(Some(( header::AUTHORIZATION, - header::HeaderValue::from_str(&format!( - "Basic {}", - BASE64_STANDARD.encode(format!( - "{}:{}", - username.unwrap(), - password.unwrap() - )) - )) - .unwrap(), + header::HeaderValue::from_str(&format!("Basic {}", bearer)).unwrap(), ))); } diff --git a/tests/specs/npm/npmrc_username_password/.npmrc b/tests/specs/npm/npmrc_username_password/.npmrc index c318678aea..9e1ded96a3 100644 --- a/tests/specs/npm/npmrc_username_password/.npmrc +++ b/tests/specs/npm/npmrc_username_password/.npmrc @@ -1,6 +1,8 @@ @denotest:registry=http://localhost:4261/ //localhost:4261/:username=deno -//localhost:4261/:_password=land +# base64 of land +//localhost:4261/:_password=bGFuZA== @denotest2:registry=http://localhost:4262/ //localhost:4262/:username=deno -//localhost:4262/:_password=land2 +# base64 of land2 +//localhost:4262/:_password=bGFuZDI= From f61af864df4d7a2513f738ddf2d5fddf79c878af Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 28 Oct 2024 09:31:58 -0400 Subject: [PATCH 36/63] fix(compile): regression handling redirects (#26586) Closes https://github.com/denoland/deno/issues/26583 --- cli/standalone/serialization.rs | 12 ++++++----- tests/specs/compile/redirects/__test__.jsonc | 22 ++++++++++++++++++++ tests/specs/compile/redirects/main.out | 1 + tests/specs/compile/redirects/main.ts | 1 + 4 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 tests/specs/compile/redirects/__test__.jsonc create mode 100644 tests/specs/compile/redirects/main.out create mode 100644 tests/specs/compile/redirects/main.ts diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index 12927845bc..7b63c584e7 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -173,6 +173,7 @@ pub struct RemoteModulesStoreBuilder { impl RemoteModulesStoreBuilder { pub fn add(&mut self, specifier: &Url, media_type: MediaType, data: Vec) { + log::debug!("Adding '{}' ({})", specifier, media_type); let specifier = specifier.to_string(); self.specifiers.push((specifier, self.data_byte_len)); self.data_byte_len += 1 + 8 + data.len() as u64; // media type (1 byte), data length (8 bytes), data @@ -182,6 +183,7 @@ impl RemoteModulesStoreBuilder { pub fn add_redirects(&mut self, redirects: &BTreeMap) { self.redirects.reserve(redirects.len()); for (from, to) in redirects { + log::debug!("Adding redirect '{}' -> '{}'", from, to); let from = from.to_string(); let to = to.to_string(); self.redirects_len += (4 + from.len() + 4 + to.len()) as u64; @@ -354,17 +356,17 @@ impl RemoteModulesStore { pub fn read<'a>( &'a self, - specifier: &'a Url, + original_specifier: &'a Url, ) -> Result>, AnyError> { let mut count = 0; - let mut current = specifier; + let mut specifier = original_specifier; loop { if count > 10 { - bail!("Too many redirects resolving '{}'", specifier); + bail!("Too many redirects resolving '{}'", original_specifier); } - match self.specifiers.get(current) { + match self.specifiers.get(specifier) { Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => { - current = to; + specifier = to; count += 1; } Some(RemoteModulesStoreSpecifierValue::Data(offset)) => { diff --git a/tests/specs/compile/redirects/__test__.jsonc b/tests/specs/compile/redirects/__test__.jsonc new file mode 100644 index 0000000000..07ce693999 --- /dev/null +++ b/tests/specs/compile/redirects/__test__.jsonc @@ -0,0 +1,22 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + "args": "compile -A --output main main.ts", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "main.out" + }, { + "if": "windows", + "args": "compile -A --output main.exe main.ts", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "main.out" + }] +} diff --git a/tests/specs/compile/redirects/main.out b/tests/specs/compile/redirects/main.out new file mode 100644 index 0000000000..e965047ad7 --- /dev/null +++ b/tests/specs/compile/redirects/main.out @@ -0,0 +1 @@ +Hello diff --git a/tests/specs/compile/redirects/main.ts b/tests/specs/compile/redirects/main.ts new file mode 100644 index 0000000000..4396319014 --- /dev/null +++ b/tests/specs/compile/redirects/main.ts @@ -0,0 +1 @@ +import "http://localhost:4546/run/003_relative_import.ts"; From 4e38fbd0a3e0ca139314e503494a8d4795007d8a Mon Sep 17 00:00:00 2001 From: snek Date: Mon, 28 Oct 2024 18:16:43 +0100 Subject: [PATCH 37/63] fix: report exceptions from nextTick (#26579) Fixes: https://github.com/denoland/deno/issues/24713 Fixes: https://github.com/denoland/deno/issues/25855 --- ext/node/polyfills/_next_tick.ts | 2 + tests/integration/node_unit_tests.rs | 4 ++ tests/node_compat/config.jsonc | 4 +- tests/node_compat/runner/TODO.md | 1 - .../test-child-process-ipc-next-tick.js | 46 +++++++++++++++++++ .../__test__.jsonc | 4 ++ .../next_tick_uncaught_exception/main.out | 2 + .../node/next_tick_uncaught_exception/main.ts | 13 ++++++ 8 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 tests/node_compat/test/parallel/test-child-process-ipc-next-tick.js create mode 100644 tests/specs/node/next_tick_uncaught_exception/__test__.jsonc create mode 100644 tests/specs/node/next_tick_uncaught_exception/main.out create mode 100644 tests/specs/node/next_tick_uncaught_exception/main.ts diff --git a/ext/node/polyfills/_next_tick.ts b/ext/node/polyfills/_next_tick.ts index 62470c564e..af306a29c8 100644 --- a/ext/node/polyfills/_next_tick.ts +++ b/ext/node/polyfills/_next_tick.ts @@ -62,6 +62,8 @@ export function processTicksAndRejections() { callback(...args); } } + } catch (e) { + reportError(e); } finally { // FIXME(bartlomieju): Deno currently doesn't support async hooks // if (destroyHooksExist()) diff --git a/tests/integration/node_unit_tests.rs b/tests/integration/node_unit_tests.rs index d66db5a40e..40bd7b2fbb 100644 --- a/tests/integration/node_unit_tests.rs +++ b/tests/integration/node_unit_tests.rs @@ -212,3 +212,7 @@ itest!(unhandled_rejection_web_process { envs: env_vars_for_npm_tests(), http_server: true, }); + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +// The itest macro is deprecated. Please move your new test to ~/tests/specs. +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/tests/node_compat/config.jsonc b/tests/node_compat/config.jsonc index 75f463342e..a99a427900 100644 --- a/tests/node_compat/config.jsonc +++ b/tests/node_compat/config.jsonc @@ -43,9 +43,6 @@ // TODO(littledivy): windows ipc streams not yet implemented "test-child-process-fork-ref.js", "test-child-process-fork-ref2.js", - // TODO(bartlomieju): this test is very flaky on CI - // https://github.com/denoland/deno/issues/25855 - // "test-child-process-ipc-next-tick.js", "test-child-process-ipc.js", "test-child-process-spawnsync-env.js", "test-child-process-stdio-inherit.js", @@ -240,6 +237,7 @@ "test-child-process-execfilesync-maxbuf.js", "test-child-process-execsync-maxbuf.js", "test-child-process-flush-stdio.js", + "test-child-process-ipc-next-tick.js", "test-child-process-kill.js", "test-child-process-set-blocking.js", "test-child-process-spawn-args.js", diff --git a/tests/node_compat/runner/TODO.md b/tests/node_compat/runner/TODO.md index 11b5d28053..35a67e72d2 100644 --- a/tests/node_compat/runner/TODO.md +++ b/tests/node_compat/runner/TODO.md @@ -280,7 +280,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-child-process-fork3.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-child-process-fork3.js) - [parallel/test-child-process-http-socket-leak.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-child-process-http-socket-leak.js) - [parallel/test-child-process-internal.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-child-process-internal.js) -- [parallel/test-child-process-ipc-next-tick.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-child-process-ipc-next-tick.js) - [parallel/test-child-process-ipc.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-child-process-ipc.js) - [parallel/test-child-process-no-deprecation.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-child-process-no-deprecation.js) - [parallel/test-child-process-pipe-dataflow.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-child-process-pipe-dataflow.js) diff --git a/tests/node_compat/test/parallel/test-child-process-ipc-next-tick.js b/tests/node_compat/test/parallel/test-child-process-ipc-next-tick.js new file mode 100644 index 0000000000..f511d25004 --- /dev/null +++ b/tests/node_compat/test/parallel/test-child-process-ipc-next-tick.js @@ -0,0 +1,46 @@ +// deno-fmt-ignore-file +// deno-lint-ignore-file + +// Copyright Joyent and Node contributors. All rights reserved. MIT license. +// Taken from Node 18.12.1 +// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. + +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const cp = require('child_process'); +const NUM_MESSAGES = 10; +const values = []; + +for (let i = 0; i < NUM_MESSAGES; ++i) { + values[i] = i; +} + +if (process.argv[2] === 'child') { + const received = values.map(() => { return false; }); + + process.on('uncaughtException', common.mustCall((err) => { + received[err] = true; + const done = received.every((element) => { return element === true; }); + + if (done) + process.disconnect(); + }, NUM_MESSAGES)); + + process.on('message', (msg) => { + // If messages are handled synchronously, throwing should break the IPC + // message processing. + throw msg; + }); + + process.send('ready'); +} else { + const child = cp.fork(__filename, ['child']); + + child.on('message', common.mustCall((msg) => { + assert.strictEqual(msg, 'ready'); + values.forEach((value) => { + child.send(value); + }); + })); +} diff --git a/tests/specs/node/next_tick_uncaught_exception/__test__.jsonc b/tests/specs/node/next_tick_uncaught_exception/__test__.jsonc new file mode 100644 index 0000000000..5517e693d6 --- /dev/null +++ b/tests/specs/node/next_tick_uncaught_exception/__test__.jsonc @@ -0,0 +1,4 @@ +{ + "args": "run main.ts", + "output": "main.out" +} diff --git a/tests/specs/node/next_tick_uncaught_exception/main.out b/tests/specs/node/next_tick_uncaught_exception/main.out new file mode 100644 index 0000000000..45b756515b --- /dev/null +++ b/tests/specs/node/next_tick_uncaught_exception/main.out @@ -0,0 +1,2 @@ +caught Error: thrown from next tick + at file:///[WILDCARD]/specs/node/next_tick_uncaught_exception/main.ts:4:15 diff --git a/tests/specs/node/next_tick_uncaught_exception/main.ts b/tests/specs/node/next_tick_uncaught_exception/main.ts new file mode 100644 index 0000000000..2679d3d54b --- /dev/null +++ b/tests/specs/node/next_tick_uncaught_exception/main.ts @@ -0,0 +1,13 @@ +import process from "node:process"; +import { strictEqual } from "node:assert"; + +const error = new Error("thrown from next tick"); + +process.on("uncaughtException", (caught) => { + strictEqual(caught, error); + console.log("caught", caught); +}); + +process.nextTick(() => { + throw error; +}); From 0e641632c38383a1d67aa610867496f64ae423ca Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 28 Oct 2024 17:43:41 -0400 Subject: [PATCH 38/63] fix(check): expose more globals from @types/node (#26603) Extracted out of https://github.com/denoland/deno/pull/26558 Closes https://github.com/denoland/deno/issues/26578 --- cli/tsc/99_main_compiler.js | 6 ------ tests/specs/npm/compare_globals/__test__.jsonc | 4 ++-- .../compare_globals/{compare_globals => }/main.out | 3 ++- .../npm/compare_globals/{compare_globals => }/main.ts | 11 +++++++++-- 4 files changed, 13 insertions(+), 11 deletions(-) rename tests/specs/npm/compare_globals/{compare_globals => }/main.out (90%) rename tests/specs/npm/compare_globals/{compare_globals => }/main.ts (84%) diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index e3a0bee597..6011dece76 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -1337,18 +1337,12 @@ delete Object.prototype.__proto__; "console", "Console", "ErrorConstructor", - "exports", "gc", "Global", "ImportMeta", "localStorage", - "module", - "NodeModule", - "NodeRequire", - "process", "queueMicrotask", "RequestInit", - "require", "ResponseInit", "sessionStorage", "setImmediate", diff --git a/tests/specs/npm/compare_globals/__test__.jsonc b/tests/specs/npm/compare_globals/__test__.jsonc index e096ddeb22..96458306a7 100644 --- a/tests/specs/npm/compare_globals/__test__.jsonc +++ b/tests/specs/npm/compare_globals/__test__.jsonc @@ -1,4 +1,4 @@ { - "args": "run --allow-read --check=all compare_globals/main.ts", - "output": "compare_globals/main.out" + "args": "run --allow-read --check=all main.ts", + "output": "main.out" } diff --git a/tests/specs/npm/compare_globals/compare_globals/main.out b/tests/specs/npm/compare_globals/main.out similarity index 90% rename from tests/specs/npm/compare_globals/compare_globals/main.out rename to tests/specs/npm/compare_globals/main.out index 5be6125b08..290e9c3b23 100644 --- a/tests/specs/npm/compare_globals/compare_globals/main.out +++ b/tests/specs/npm/compare_globals/main.out @@ -8,10 +8,11 @@ Download http://localhost:4260/@denotest/globals/1.0.0.tgz Download http://localhost:4260/@types/node/node-22.5.4.tgz Download http://localhost:4260/undici-types/undici-types-6.19.8.tgz [UNORDERED_END] -Check file:///[WILDCARD]/compare_globals/main.ts +Check file:///[WILDCARD]/main.ts true true [] +process equals process true setTimeout 1 false setTimeout 2 function setTimeout 3 function diff --git a/tests/specs/npm/compare_globals/compare_globals/main.ts b/tests/specs/npm/compare_globals/main.ts similarity index 84% rename from tests/specs/npm/compare_globals/compare_globals/main.ts rename to tests/specs/npm/compare_globals/main.ts index 9482798d8c..308ce9b23f 100644 --- a/tests/specs/npm/compare_globals/compare_globals/main.ts +++ b/tests/specs/npm/compare_globals/main.ts @@ -5,10 +5,17 @@ console.log(globals.global === globals.globalThis); // @ts-expect-error even though these are the same object, they have different types console.log(globals.globalThis === globalThis); console.log(globals.process.execArgv); +console.log("process equals process", process === globals.process); type AssertTrue = never; -type _TestNoProcessGlobal = AssertTrue< - typeof globalThis extends { process: any } ? false : true +type _TestHasProcessGlobal = AssertTrue< + typeof globalThis extends { process: any } ? true : false +>; +type _TestProcessGlobalVersion = AssertTrue< + typeof process.versions.node extends string ? true : false +>; +type _TestNoBufferGlogal = AssertTrue< + typeof globalThis extends { Buffer: any } ? false : true >; type _TestHasNodeJsGlobal = NodeJS.Architecture; From 484f8ca9c30909914eb3348b2df10d44e2753248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 28 Oct 2024 23:55:51 +0000 Subject: [PATCH 39/63] fix: provide hints in terminal errors for Node.js globals (#26610) Add info/hint for terminal errors related to Node.js globals: - __filename - __dirname - Buffer - global - setImmediate - clearImmediate Closes https://github.com/denoland/deno/issues/17494 --- runtime/fmt_errors.rs | 42 +++++++++++++++++++ .../main.out | 3 ++ .../run/node_globals_hints/__test__.jsonc | 34 +++++++++++++++ tests/specs/run/node_globals_hints/buffer.js | 1 + tests/specs/run/node_globals_hints/buffer.out | 7 ++++ .../run/node_globals_hints/clear_immediate.js | 1 + .../node_globals_hints/clear_immediate.out | 7 ++++ tests/specs/run/node_globals_hints/dirname.js | 1 + .../specs/run/node_globals_hints/dirname.out | 7 ++++ .../specs/run/node_globals_hints/filename.js | 1 + .../specs/run/node_globals_hints/filename.out | 7 ++++ tests/specs/run/node_globals_hints/global.js | 1 + tests/specs/run/node_globals_hints/global.out | 7 ++++ .../run/node_globals_hints/set_immediate.js | 1 + .../run/node_globals_hints/set_immediate.out | 7 ++++ 15 files changed, 127 insertions(+) create mode 100644 tests/specs/run/node_globals_hints/__test__.jsonc create mode 100644 tests/specs/run/node_globals_hints/buffer.js create mode 100644 tests/specs/run/node_globals_hints/buffer.out create mode 100644 tests/specs/run/node_globals_hints/clear_immediate.js create mode 100644 tests/specs/run/node_globals_hints/clear_immediate.out create mode 100644 tests/specs/run/node_globals_hints/dirname.js create mode 100644 tests/specs/run/node_globals_hints/dirname.out create mode 100644 tests/specs/run/node_globals_hints/filename.js create mode 100644 tests/specs/run/node_globals_hints/filename.out create mode 100644 tests/specs/run/node_globals_hints/global.js create mode 100644 tests/specs/run/node_globals_hints/global.out create mode 100644 tests/specs/run/node_globals_hints/set_immediate.js create mode 100644 tests/specs/run/node_globals_hints/set_immediate.out diff --git a/runtime/fmt_errors.rs b/runtime/fmt_errors.rs index 7c6cf6d397..4cd8a06345 100644 --- a/runtime/fmt_errors.rs +++ b/runtime/fmt_errors.rs @@ -321,6 +321,48 @@ fn get_suggestions_for_terminal_errors(e: &JsError) -> Vec { ]), FixSuggestion::docs("https://docs.deno.com/go/commonjs"), ]; + } else if msg.contains("__filename is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "__filename global is not available in ES modules." + )), + FixSuggestion::hint(cstr!("Use import.meta.filename instead.")), + ]; + } else if msg.contains("__dirname is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "__dirname global is not available in ES modules." + )), + FixSuggestion::hint(cstr!("Use import.meta.dirname instead.")), + ]; + } else if msg.contains("Buffer is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "Buffer is not available in the global scope in Deno." + )), + FixSuggestion::hint(cstr!("Import it explicitly with import { Buffer } from \"node:buffer\";.")), + ]; + } else if msg.contains("clearImmediate is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "clearImmediate is not available in the global scope in Deno." + )), + FixSuggestion::hint(cstr!("Import it explicitly with import { clearImmediate } from \"node:timers\";.")), + ]; + } else if msg.contains("setImmediate is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "setImmediate is not available in the global scope in Deno." + )), + FixSuggestion::hint(cstr!("Import it explicitly with import { setImmediate } from \"node:timers\";.")), + ]; + } else if msg.contains("global is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "global is not available in the global scope in Deno." + )), + FixSuggestion::hint(cstr!("Use globalThis instead, or assign globalThis.global = globalThis.")), + ]; } else if msg.contains("openKv is not a function") { return vec![ FixSuggestion::info("Deno.openKv() is an unstable API."), diff --git a/tests/specs/lint/node_globals_no_duplicate_imports/main.out b/tests/specs/lint/node_globals_no_duplicate_imports/main.out index 56df10eba4..058b80795d 100644 --- a/tests/specs/lint/node_globals_no_duplicate_imports/main.out +++ b/tests/specs/lint/node_globals_no_duplicate_imports/main.out @@ -2,3 +2,6 @@ error: Uncaught (in promise) ReferenceError: setImmediate is not defined const _foo = setImmediate; ^ at [WILDCARD]main.ts:3:14 + + info: setImmediate is not available in the global scope in Deno. + hint: Import it explicitly with import { setImmediate } from "node:timers";. diff --git a/tests/specs/run/node_globals_hints/__test__.jsonc b/tests/specs/run/node_globals_hints/__test__.jsonc new file mode 100644 index 0000000000..c5c5d6e42e --- /dev/null +++ b/tests/specs/run/node_globals_hints/__test__.jsonc @@ -0,0 +1,34 @@ +{ + "tests": { + "__dirname": { + "args": "run dirname.js", + "output": "dirname.out", + "exitCode": 1 + }, + "__filename": { + "args": "run filename.js", + "output": "filename.out", + "exitCode": 1 + }, + "clearImmediate": { + "args": "run clear_immediate.js", + "output": "clear_immediate.out", + "exitCode": 1 + }, + "buffer": { + "args": "run buffer.js", + "output": "buffer.out", + "exitCode": 1 + }, + "global": { + "args": "run global.js", + "output": "global.out", + "exitCode": 1 + }, + "setImmediate": { + "args": "run set_immediate.js", + "output": "set_immediate.out", + "exitCode": 1 + } + } +} diff --git a/tests/specs/run/node_globals_hints/buffer.js b/tests/specs/run/node_globals_hints/buffer.js new file mode 100644 index 0000000000..9809e1656e --- /dev/null +++ b/tests/specs/run/node_globals_hints/buffer.js @@ -0,0 +1 @@ +Buffer; diff --git a/tests/specs/run/node_globals_hints/buffer.out b/tests/specs/run/node_globals_hints/buffer.out new file mode 100644 index 0000000000..4980e6d129 --- /dev/null +++ b/tests/specs/run/node_globals_hints/buffer.out @@ -0,0 +1,7 @@ +error: Uncaught (in promise) ReferenceError: Buffer is not defined +Buffer; +^ + at [WILDCARD]buffer.js:1:1 + + info: Buffer is not available in the global scope in Deno. + hint: Import it explicitly with import { Buffer } from "node:buffer";. diff --git a/tests/specs/run/node_globals_hints/clear_immediate.js b/tests/specs/run/node_globals_hints/clear_immediate.js new file mode 100644 index 0000000000..94b26ae682 --- /dev/null +++ b/tests/specs/run/node_globals_hints/clear_immediate.js @@ -0,0 +1 @@ +clearImmediate; diff --git a/tests/specs/run/node_globals_hints/clear_immediate.out b/tests/specs/run/node_globals_hints/clear_immediate.out new file mode 100644 index 0000000000..ecd34babef --- /dev/null +++ b/tests/specs/run/node_globals_hints/clear_immediate.out @@ -0,0 +1,7 @@ +error: Uncaught (in promise) ReferenceError: clearImmediate is not defined +clearImmediate; +^ + at [WILDCARD]clear_immediate.js:1:1 + + info: clearImmediate is not available in the global scope in Deno. + hint: Import it explicitly with import { clearImmediate } from "node:timers";. diff --git a/tests/specs/run/node_globals_hints/dirname.js b/tests/specs/run/node_globals_hints/dirname.js new file mode 100644 index 0000000000..58d9e93794 --- /dev/null +++ b/tests/specs/run/node_globals_hints/dirname.js @@ -0,0 +1 @@ +__dirname; diff --git a/tests/specs/run/node_globals_hints/dirname.out b/tests/specs/run/node_globals_hints/dirname.out new file mode 100644 index 0000000000..0aa3e4f51b --- /dev/null +++ b/tests/specs/run/node_globals_hints/dirname.out @@ -0,0 +1,7 @@ +error: Uncaught (in promise) ReferenceError: __dirname is not defined +__dirname; +^ + at [WILDCARD]dirname.js:1:1 + + info: __dirname global is not available in ES modules. + hint: Use import.meta.dirname instead. diff --git a/tests/specs/run/node_globals_hints/filename.js b/tests/specs/run/node_globals_hints/filename.js new file mode 100644 index 0000000000..31770bb141 --- /dev/null +++ b/tests/specs/run/node_globals_hints/filename.js @@ -0,0 +1 @@ +__filename; diff --git a/tests/specs/run/node_globals_hints/filename.out b/tests/specs/run/node_globals_hints/filename.out new file mode 100644 index 0000000000..47b4264fbd --- /dev/null +++ b/tests/specs/run/node_globals_hints/filename.out @@ -0,0 +1,7 @@ +error: Uncaught (in promise) ReferenceError: __filename is not defined +__filename; +^ + at [WILDCARD]filename.js:1:1 + + info: __filename global is not available in ES modules. + hint: Use import.meta.filename instead. diff --git a/tests/specs/run/node_globals_hints/global.js b/tests/specs/run/node_globals_hints/global.js new file mode 100644 index 0000000000..56676d726a --- /dev/null +++ b/tests/specs/run/node_globals_hints/global.js @@ -0,0 +1 @@ +global; diff --git a/tests/specs/run/node_globals_hints/global.out b/tests/specs/run/node_globals_hints/global.out new file mode 100644 index 0000000000..e090d32e29 --- /dev/null +++ b/tests/specs/run/node_globals_hints/global.out @@ -0,0 +1,7 @@ +error: Uncaught (in promise) ReferenceError: global is not defined +global; +^ + at [WILDCARD]global.js:1:1 + + info: global is not available in the global scope in Deno. + hint: Use globalThis instead, or assign globalThis.global = globalThis. diff --git a/tests/specs/run/node_globals_hints/set_immediate.js b/tests/specs/run/node_globals_hints/set_immediate.js new file mode 100644 index 0000000000..fa29d1e714 --- /dev/null +++ b/tests/specs/run/node_globals_hints/set_immediate.js @@ -0,0 +1 @@ +setImmediate; diff --git a/tests/specs/run/node_globals_hints/set_immediate.out b/tests/specs/run/node_globals_hints/set_immediate.out new file mode 100644 index 0000000000..38859ff5d7 --- /dev/null +++ b/tests/specs/run/node_globals_hints/set_immediate.out @@ -0,0 +1,7 @@ +error: Uncaught (in promise) ReferenceError: setImmediate is not defined +setImmediate; +^ + at [WILDCARD]set_immediate.js:1:1 + + info: setImmediate is not available in the global scope in Deno. + hint: Import it explicitly with import { setImmediate } from "node:timers";. From 46e5ed1a642edcd998d1e22c9b37e0c4aa298e1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 29 Oct 2024 00:41:02 +0000 Subject: [PATCH 40/63] Revert "fix(ext/node): use primordials in `ext/node/polyfills/https.ts` (#26323)" (#26613) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …s` (#26323)" This reverts commit afb33b3c2597c9ec943f71218b236486fbc86e23. Reverting because it caused a regression - https://github.com/denoland/deno/issues/26612. Closes https://github.com/denoland/deno/issues/26612. --- ext/node/polyfills/https.ts | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/ext/node/polyfills/https.ts b/ext/node/polyfills/https.ts index dd24cf048c..f60c5e471a 100644 --- a/ext/node/polyfills/https.ts +++ b/ext/node/polyfills/https.ts @@ -1,6 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. // Copyright Joyent and Node contributors. All rights reserved. MIT license. +// TODO(petamoriken): enable prefer-primordials for node polyfills +// deno-lint-ignore-file prefer-primordials + import { notImplemented } from "ext:deno_node/_utils.ts"; import { urlToHttpOptions } from "ext:deno_node/internal/url.ts"; import { @@ -14,14 +17,6 @@ import { type ServerHandler, ServerImpl as HttpServer } from "node:http"; import { validateObject } from "ext:deno_node/internal/validators.mjs"; import { kEmptyObject } from "ext:deno_node/internal/util.mjs"; import { Buffer } from "node:buffer"; -import { primordials } from "ext:core/mod.js"; -const { - ArrayPrototypeShift, - ArrayPrototypeUnshift, - ArrayIsArray, - ObjectPrototypeIsPrototypeOf, - ObjectAssign, -} = primordials; export class Server extends HttpServer { constructor(opts, requestListener?: ServerHandler) { @@ -34,11 +29,11 @@ export class Server extends HttpServer { validateObject(opts, "options"); } - if (opts.cert && ArrayIsArray(opts.cert)) { + if (opts.cert && Array.isArray(opts.cert)) { notImplemented("https.Server.opts.cert array type"); } - if (opts.key && ArrayIsArray(opts.key)) { + if (opts.key && Array.isArray(opts.key)) { notImplemented("https.Server.opts.key array type"); } @@ -47,12 +42,10 @@ export class Server extends HttpServer { _additionalServeOptions() { return { - cert: ObjectPrototypeIsPrototypeOf(Buffer, this._opts.cert) - // deno-lint-ignore prefer-primordials + cert: this._opts.cert instanceof Buffer ? this._opts.cert.toString() : this._opts.cert, - key: ObjectPrototypeIsPrototypeOf(Buffer, this._opts.key) - // deno-lint-ignore prefer-primordials + key: this._opts.key instanceof Buffer ? this._opts.key.toString() : this._opts.key, }; @@ -166,18 +159,18 @@ export function request(...args: any[]) { let options = {}; if (typeof args[0] === "string") { - const urlStr = ArrayPrototypeShift(args); + const urlStr = args.shift(); options = urlToHttpOptions(new URL(urlStr)); - } else if (ObjectPrototypeIsPrototypeOf(URL, args[0])) { - options = urlToHttpOptions(ArrayPrototypeShift(args)); + } else if (args[0] instanceof URL) { + options = urlToHttpOptions(args.shift()); } if (args[0] && typeof args[0] !== "function") { - ObjectAssign(options, ArrayPrototypeShift(args)); + Object.assign(options, args.shift()); } options._defaultAgent = globalAgent; - ArrayPrototypeUnshift(args, options); + args.unshift(options); return new HttpsClientRequest(args[0], args[1]); } From aa2a354190f15645a7db0563eebe49255a083ff9 Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Tue, 29 Oct 2024 14:37:21 +0900 Subject: [PATCH 41/63] refactor(init): inline routing in deno init --serve template (#26595) --- cli/tools/init/mod.rs | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/cli/tools/init/mod.rs b/cli/tools/init/mod.rs index 2d6a894e13..4e4a686c5f 100644 --- a/cli/tools/init/mod.rs +++ b/cli/tools/init/mod.rs @@ -24,32 +24,29 @@ pub fn init_project(init_flags: InitFlags) -> Result<(), AnyError> { create_file( &dir, "main.ts", - r#"import { type Route, route, serveDir } from "@std/http"; + r#"import { serveDir } from "@std/http"; -const routes: Route[] = [ - { - pattern: new URLPattern({ pathname: "/" }), - handler: () => new Response("Home page"), - }, - { - pattern: new URLPattern({ pathname: "/users/:id" }), - handler: (_req, _info, params) => new Response(params?.pathname.groups.id), - }, - { - pattern: new URLPattern({ pathname: "/static/*" }), - handler: (req) => serveDir(req), - }, -]; - -function defaultHandler(_req: Request) { - return new Response("Not found", { status: 404 }); -} - -const handler = route(routes, defaultHandler); +const userPagePattern = new URLPattern({ pathname: "/users/:id" }); +const staticPathPattern = new URLPattern({ pathname: "/static/*" }); export default { fetch(req) { - return handler(req); + const url = new URL(req.url); + + if (url.pathname === "/") { + return new Response("Home page"); + } + + const userPageMatch = userPagePattern.exec(url); + if (userPageMatch) { + return new Response(userPageMatch.pathname.groups.id); + } + + if (staticPathPattern.test(url)) { + return serveDir(req); + } + + return new Response("Not found", { status: 404 }); }, } satisfies Deno.ServeDefaultExport; "#, From efb5e912e5ef6745fc7fdfb1a84b5bd362548d61 Mon Sep 17 00:00:00 2001 From: Volker Schlecht <47375452+VlkrS@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:10:32 +0100 Subject: [PATCH 42/63] fix(ext/node): compatibility with {Free,Open}BSD (#26604) Ports for both BSDs contain patches to the same effect. See https://github.com/freebsd/freebsd-ports/blob/main/www/deno/files/patch-ext_node_ops_fs.rs and https://github.com/openbsd/ports/blob/8644910cae24458306b6a7c4ef4ef7811bc3d1f5/lang/deno/patches/patch-ext_node_ops_fs_rs --- ext/node/ops/fs.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/ext/node/ops/fs.rs b/ext/node/ops/fs.rs index 6645792e74..98b3c46a14 100644 --- a/ext/node/ops/fs.rs +++ b/ext/node/ops/fs.rs @@ -152,13 +152,21 @@ where let mut cpath = path.as_bytes().to_vec(); cpath.push(0); if bigint { - #[cfg(not(target_os = "macos"))] + #[cfg(not(any( + target_os = "macos", + target_os = "freebsd", + target_os = "openbsd" + )))] // SAFETY: `cpath` is NUL-terminated and result is pointer to valid statfs memory. let (code, result) = unsafe { let mut result: libc::statfs64 = std::mem::zeroed(); (libc::statfs64(cpath.as_ptr() as _, &mut result), result) }; - #[cfg(target_os = "macos")] + #[cfg(any( + target_os = "macos", + target_os = "freebsd", + target_os = "openbsd" + ))] // SAFETY: `cpath` is NUL-terminated and result is pointer to valid statfs memory. let (code, result) = unsafe { let mut result: libc::statfs = std::mem::zeroed(); @@ -168,7 +176,10 @@ where return Err(std::io::Error::last_os_error().into()); } Ok(StatFs { + #[cfg(not(target_os = "openbsd"))] typ: result.f_type as _, + #[cfg(target_os = "openbsd")] + typ: 0 as _, bsize: result.f_bsize as _, blocks: result.f_blocks as _, bfree: result.f_bfree as _, @@ -186,7 +197,10 @@ where return Err(std::io::Error::last_os_error().into()); } Ok(StatFs { + #[cfg(not(target_os = "openbsd"))] typ: result.f_type as _, + #[cfg(target_os = "openbsd")] + typ: 0 as _, bsize: result.f_bsize as _, blocks: result.f_blocks as _, bfree: result.f_bfree as _, From 51978a76540cac72dd35528dd023c3fc6d5eeca6 Mon Sep 17 00:00:00 2001 From: Volker Schlecht <47375452+VlkrS@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:11:49 +0100 Subject: [PATCH 43/63] fix(ext/napi): export dynamic symbols list for {Free,Open}BSD (#26605) The two BSD ports are reusing the Linux code here. --- ext/napi/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ext/napi/lib.rs b/ext/napi/lib.rs index 0d41bb40fe..20f924bdbf 100644 --- a/ext/napi/lib.rs +++ b/ext/napi/lib.rs @@ -662,7 +662,11 @@ pub fn print_linker_flags(name: &str) { symbols_path, ); - #[cfg(target_os = "linux")] + #[cfg(any( + target_os = "linux", + target_os = "freebsd", + target_os = "openbsd" + ))] println!( "cargo:rustc-link-arg-bin={name}=-Wl,--export-dynamic-symbol-list={}", symbols_path, From a69224ea5bd02f08108aac867c492754095f2d34 Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Wed, 30 Oct 2024 02:41:16 +0900 Subject: [PATCH 44/63] Revert "fix(ext/node): fix dns.lookup result ordering (#26264)" (#26621) This reverts commit d59599fc187c559ee231882773e1c5a2b932fc3d. Closes #26588 --- ext/node/polyfills/http.ts | 6 ++--- ext/node/polyfills/internal/dns/utils.ts | 14 +++++++++-- .../polyfills/internal_binding/cares_wrap.ts | 15 ++++-------- ext/node/polyfills/net.ts | 24 +++++++++++++------ tests/unit_node/http_test.ts | 8 ++----- tests/unit_node/tls_test.ts | 12 +++------- 6 files changed, 41 insertions(+), 38 deletions(-) diff --git a/ext/node/polyfills/http.ts b/ext/node/polyfills/http.ts index e7a860ff1c..e8a189f70f 100644 --- a/ext/node/polyfills/http.ts +++ b/ext/node/polyfills/http.ts @@ -51,7 +51,6 @@ import { urlToHttpOptions } from "ext:deno_node/internal/url.ts"; import { kEmptyObject } from "ext:deno_node/internal/util.mjs"; import { constants, TCP } from "ext:deno_node/internal_binding/tcp_wrap.ts"; import { notImplemented, warnNotImplemented } from "ext:deno_node/_utils.ts"; -import { isWindows } from "ext:deno_node/_util/os.ts"; import { connResetException, ERR_HTTP_HEADERS_SENT, @@ -1712,8 +1711,9 @@ export class ServerImpl extends EventEmitter { port = options.port | 0; } - // Use 0.0.0.0 for Windows, and [::] for other platforms. - let hostname = options.host ?? (isWindows ? "0.0.0.0" : "[::]"); + // TODO(bnoordhuis) Node prefers [::] when host is omitted, + // we on the other hand default to 0.0.0.0. + let hostname = options.host ?? "0.0.0.0"; if (hostname == "localhost") { hostname = "127.0.0.1"; } diff --git a/ext/node/polyfills/internal/dns/utils.ts b/ext/node/polyfills/internal/dns/utils.ts index 1e0c3d9ed6..226fce93dd 100644 --- a/ext/node/polyfills/internal/dns/utils.ts +++ b/ext/node/polyfills/internal/dns/utils.ts @@ -416,10 +416,20 @@ export function emitInvalidHostnameWarning(hostname: string) { ); } -let dnsOrder = getOptionValue("--dns-result-order") || "verbatim"; +let dnsOrder = getOptionValue("--dns-result-order") || "ipv4first"; export function getDefaultVerbatim() { - return dnsOrder !== "ipv4first"; + switch (dnsOrder) { + case "verbatim": { + return true; + } + case "ipv4first": { + return false; + } + default: { + return false; + } + } } /** diff --git a/ext/node/polyfills/internal_binding/cares_wrap.ts b/ext/node/polyfills/internal_binding/cares_wrap.ts index cbfea40b22..6feb7faf0d 100644 --- a/ext/node/polyfills/internal_binding/cares_wrap.ts +++ b/ext/node/polyfills/internal_binding/cares_wrap.ts @@ -75,18 +75,11 @@ export function getaddrinfo( const recordTypes: ("A" | "AAAA")[] = []; - if (family === 6) { + if (family === 0 || family === 4) { + recordTypes.push("A"); + } + if (family === 0 || family === 6) { recordTypes.push("AAAA"); - } else if (family === 4) { - recordTypes.push("A"); - } else if (family === 0 && hostname === "localhost") { - // Ipv6 is preferred over Ipv4 for localhost - recordTypes.push("AAAA"); - recordTypes.push("A"); - } else if (family === 0) { - // Only get Ipv4 addresses for the other hostnames - // This simulates what `getaddrinfo` does when the family is not specified - recordTypes.push("A"); } (async () => { diff --git a/ext/node/polyfills/net.ts b/ext/node/polyfills/net.ts index 35d273be93..48e1d0de87 100644 --- a/ext/node/polyfills/net.ts +++ b/ext/node/polyfills/net.ts @@ -1871,13 +1871,23 @@ function _setupListenHandle( // Try to bind to the unspecified IPv6 address, see if IPv6 is available if (!address && typeof fd !== "number") { - if (isWindows) { - address = DEFAULT_IPV4_ADDR; - addressType = 4; - } else { - address = DEFAULT_IPV6_ADDR; - addressType = 6; - } + // TODO(@bartlomieju): differs from Node which tries to bind to IPv6 first + // when no address is provided. + // + // Forcing IPv4 as a workaround for Deno not aligning with Node on + // implicit binding on Windows. + // + // REF: https://github.com/denoland/deno/issues/10762 + // rval = _createServerHandle(DEFAULT_IPV6_ADDR, port, 6, fd, flags); + + // if (typeof rval === "number") { + // rval = null; + address = DEFAULT_IPV4_ADDR; + addressType = 4; + // } else { + // address = DEFAULT_IPV6_ADDR; + // addressType = 6; + // } } if (rval === null) { diff --git a/tests/unit_node/http_test.ts b/tests/unit_node/http_test.ts index 84d6f57279..2b3b8f509f 100644 --- a/tests/unit_node/http_test.ts +++ b/tests/unit_node/http_test.ts @@ -322,14 +322,10 @@ Deno.test("[node/http] IncomingRequest socket has remoteAddress + remotePort", a // deno-lint-ignore no-explicit-any const port = (server.address() as any).port; const res = await fetch( - `http://localhost:${port}/`, + `http://127.0.0.1:${port}/`, ); await res.arrayBuffer(); - if (Deno.build.os === "windows") { - assertEquals(remoteAddress, "127.0.0.1"); - } else { - assertEquals(remoteAddress, "::1"); - } + assertEquals(remoteAddress, "127.0.0.1"); assertEquals(typeof remotePort, "number"); server.close(() => resolve()); }); diff --git a/tests/unit_node/tls_test.ts b/tests/unit_node/tls_test.ts index 236dab2086..43d6205b0b 100644 --- a/tests/unit_node/tls_test.ts +++ b/tests/unit_node/tls_test.ts @@ -32,15 +32,13 @@ for ( ) { Deno.test(`tls.connect sends correct ALPN: '${alpnServer}' + '${alpnClient}' = '${expected}'`, async () => { const listener = Deno.listenTls({ - hostname: "localhost", port: 0, key, cert, alpnProtocols: alpnServer, }); const outgoing = tls.connect({ - host: "::1", - servername: "localhost", + host: "localhost", port: listener.addr.port, ALPNProtocols: alpnClient, secureContext: { @@ -63,7 +61,6 @@ Deno.test("tls.connect makes tls connection", async () => { const ctl = new AbortController(); let port; const serve = Deno.serve({ - hostname: "localhost", port: 0, key, cert, @@ -74,8 +71,7 @@ Deno.test("tls.connect makes tls connection", async () => { await delay(200); const conn = tls.connect({ - host: "::1", - servername: "localhost", + host: "localhost", port, secureContext: { ca: rootCaCert, @@ -106,7 +102,6 @@ Deno.test("tls.connect mid-read tcp->tls upgrade", async () => { const { promise, resolve } = Promise.withResolvers(); const ctl = new AbortController(); const serve = Deno.serve({ - hostname: "localhost", port: 8443, key, cert, @@ -116,8 +111,7 @@ Deno.test("tls.connect mid-read tcp->tls upgrade", async () => { await delay(200); const conn = tls.connect({ - host: "::1", - servername: "localhost", + host: "localhost", port: 8443, secureContext: { ca: rootCaCert, From 3b28446000d8d063c07ef320314ef493a9e0bffa Mon Sep 17 00:00:00 2001 From: HasanAlrimawi <141642411+HasanAlrimawi@users.noreply.github.com> Date: Tue, 29 Oct 2024 23:55:41 +0200 Subject: [PATCH 45/63] fix: support watch flag to enable watching other files than the main module on serve subcommand (#26622) Closes #26618 --- cli/args/mod.rs | 4 ++ tests/integration/watcher_tests.rs | 70 ++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 927f43e85a..f5ecc04496 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -1672,6 +1672,10 @@ impl CliOptions { if let DenoSubcommand::Run(RunFlags { watch: Some(WatchFlagsWithPaths { paths, .. }), .. + }) + | DenoSubcommand::Serve(ServeFlags { + watch: Some(WatchFlagsWithPaths { paths, .. }), + .. }) = &self.flags.subcommand { full_paths.extend(paths.iter().map(|path| self.initial_cwd.join(path))); diff --git a/tests/integration/watcher_tests.rs b/tests/integration/watcher_tests.rs index 122353bba9..d4705b8d53 100644 --- a/tests/integration/watcher_tests.rs +++ b/tests/integration/watcher_tests.rs @@ -566,6 +566,76 @@ async fn run_watch_no_dynamic() { check_alive_then_kill(child); } +#[flaky_test(tokio)] +async fn serve_watch_all() { + let t = TempDir::new(); + let main_file_to_watch = t.path().join("main_file_to_watch.js"); + main_file_to_watch.write( + "export default { + fetch(_request: Request) { + return new Response(\"aaaaaaqqq!\"); + }, + };", + ); + + let mut child = util::deno_cmd() + .current_dir(t.path()) + .arg("serve") + .arg("--watch=another_file.js") + .arg("-L") + .arg("debug") + .arg(&main_file_to_watch) + .env("NO_COLOR", "1") + .piped_output() + .spawn() + .unwrap(); + let (mut stdout_lines, mut stderr_lines) = child_lines(&mut child); + + wait_for_watcher("main_file_to_watch.js", &mut stderr_lines).await; + + // Change content of the file + main_file_to_watch.write( + "export default { + fetch(_request: Request) { + return new Response(\"aaaaaaqqq123!\"); + }, + };", + ); + wait_contains("Restarting", &mut stderr_lines).await; + wait_for_watcher("main_file_to_watch.js", &mut stderr_lines).await; + + let another_file = t.path().join("another_file.js"); + another_file.write("export const foo = 0;"); + // Confirm that the added file is watched as well + wait_contains("Restarting", &mut stderr_lines).await; + + main_file_to_watch + .write("import { foo } from './another_file.js'; console.log(foo);"); + wait_contains("Restarting", &mut stderr_lines).await; + wait_contains("0", &mut stdout_lines).await; + + another_file.write("export const foo = 42;"); + wait_contains("Restarting", &mut stderr_lines).await; + wait_contains("42", &mut stdout_lines).await; + + // Confirm that watch continues even with wrong syntax error + another_file.write("syntax error ^^"); + + wait_contains("Restarting", &mut stderr_lines).await; + wait_contains("error:", &mut stderr_lines).await; + + main_file_to_watch.write( + "export default { + fetch(_request: Request) { + return new Response(\"aaaaaaqqq!\"); + }, + };", + ); + wait_contains("Restarting", &mut stderr_lines).await; + wait_for_watcher("main_file_to_watch.js", &mut stderr_lines).await; + check_alive_then_kill(child); +} + #[flaky_test(tokio)] async fn run_watch_npm_specifier() { let _g = util::http_server(); From c8d229dbf03177e81f4fd2f43c7daf74a7f2e283 Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Wed, 30 Oct 2024 00:35:42 +0100 Subject: [PATCH 46/63] fix(install): percent encodings in interactive progress bar (#26600) Fixes percent encodings showing up when installing scoped packages via `deno add` or `deno install`. The issue is caused by us trying to map back the package name from the resolved http url. This doesn't and has never worked with private registries. The proper solution would be to pass the original specifier into here, but that's a bit of a bigger refactor. So for now the quickest workaround is to replace `%2f` back to `/`. Fixes https://github.com/denoland/deno/issues/26576 --- cli/util/progress_bar/renderer.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/util/progress_bar/renderer.rs b/cli/util/progress_bar/renderer.rs index a83ceb3334..6b08dada12 100644 --- a/cli/util/progress_bar/renderer.rs +++ b/cli/util/progress_bar/renderer.rs @@ -193,10 +193,16 @@ impl ProgressBarRenderer for TextOnlyProgressBarRenderer { } }; + // TODO(@marvinhagemeister): We're trying to reconstruct the original + // specifier from the resolved one, but we lack the information about + // private registries URLs and other things here. let message = display_entry .message .replace("https://registry.npmjs.org/", "npm:") - .replace("https://jsr.io/", "jsr:"); + .replace("https://jsr.io/", "jsr:") + .replace("%2f", "/") + .replace("%2F", "/"); + display_str.push_str( &colors::gray(format!(" - {}{}\n", message, bytes_text)).to_string(), ); From a1473d82c5612676c50af00ded0467dbb29bc0a8 Mon Sep 17 00:00:00 2001 From: denobot <33910674+denobot@users.noreply.github.com> Date: Wed, 30 Oct 2024 08:46:31 -0400 Subject: [PATCH 47/63] chore: forward v2.0.4 release commit to main (#26636) This is the release commit being forwarded back to main for 2.0.4 Co-authored-by: bartlomieju --- .github/workflows/ci.generate.ts | 2 +- .github/workflows/ci.yml | 8 ++--- Cargo.lock | 58 ++++++++++++++++---------------- Cargo.toml | 56 +++++++++++++++--------------- Releases.md | 32 ++++++++++++++++++ bench_util/Cargo.toml | 2 +- cli/Cargo.toml | 2 +- ext/broadcast_channel/Cargo.toml | 2 +- ext/cache/Cargo.toml | 2 +- ext/canvas/Cargo.toml | 2 +- ext/console/Cargo.toml | 2 +- ext/cron/Cargo.toml | 2 +- ext/crypto/Cargo.toml | 2 +- ext/fetch/Cargo.toml | 2 +- ext/ffi/Cargo.toml | 2 +- ext/fs/Cargo.toml | 2 +- ext/http/Cargo.toml | 2 +- ext/io/Cargo.toml | 2 +- ext/kv/Cargo.toml | 2 +- ext/napi/Cargo.toml | 2 +- ext/napi/sym/Cargo.toml | 2 +- ext/net/Cargo.toml | 2 +- ext/node/Cargo.toml | 2 +- ext/tls/Cargo.toml | 2 +- ext/url/Cargo.toml | 2 +- ext/web/Cargo.toml | 2 +- ext/webgpu/Cargo.toml | 2 +- ext/webidl/Cargo.toml | 2 +- ext/websocket/Cargo.toml | 2 +- ext/webstorage/Cargo.toml | 2 +- resolvers/deno/Cargo.toml | 2 +- resolvers/node/Cargo.toml | 2 +- runtime/Cargo.toml | 2 +- runtime/permissions/Cargo.toml | 2 +- 34 files changed, 123 insertions(+), 91 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 3e2b079643..d5ee67b654 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify"; // Bump this number when you want to purge the cache. // Note: the tools/release/01_bump_crate_versions.ts script will update this version // automatically via regex, so ensure that this line maintains this format. -const cacheVersion = 22; +const cacheVersion = 23; const ubuntuX86Runner = "ubuntu-22.04"; const ubuntuX86XlRunner = "ubuntu-22.04-xl"; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5739537bb0..a6421d9244 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -361,8 +361,8 @@ jobs: path: |- ~/.cargo/registry/index ~/.cargo/registry/cache - key: '22-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '22-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' + key: '23-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' + restore-keys: '23-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' if: '!(matrix.skip)' - name: Restore cache build output (PR) uses: actions/cache/restore@v4 @@ -375,7 +375,7 @@ jobs: !./target/*/*.zip !./target/*/*.tar.gz key: never_saved - restore-keys: '22-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' + restore-keys: '23-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' - name: Apply and update mtime cache if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))' uses: ./.github/mtime_cache @@ -685,7 +685,7 @@ jobs: !./target/*/*.zip !./target/*/*.sha256sum !./target/*/*.tar.gz - key: '22-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' + key: '23-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: name: publish canary runs-on: ubuntu-22.04 diff --git a/Cargo.lock b/Cargo.lock index 2c8028a1e2..36ba70d9d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1154,7 +1154,7 @@ dependencies = [ [[package]] name = "deno" -version = "2.0.3" +version = "2.0.4" dependencies = [ "anstream", "async-trait", @@ -1323,7 +1323,7 @@ dependencies = [ [[package]] name = "deno_bench_util" -version = "0.168.0" +version = "0.169.0" dependencies = [ "bencher", "deno_core", @@ -1332,7 +1332,7 @@ dependencies = [ [[package]] name = "deno_broadcast_channel" -version = "0.168.0" +version = "0.169.0" dependencies = [ "async-trait", "deno_core", @@ -1343,7 +1343,7 @@ dependencies = [ [[package]] name = "deno_cache" -version = "0.106.0" +version = "0.107.0" dependencies = [ "async-trait", "deno_core", @@ -1376,7 +1376,7 @@ dependencies = [ [[package]] name = "deno_canvas" -version = "0.43.0" +version = "0.44.0" dependencies = [ "deno_core", "deno_webgpu", @@ -1411,7 +1411,7 @@ dependencies = [ [[package]] name = "deno_console" -version = "0.174.0" +version = "0.175.0" dependencies = [ "deno_core", ] @@ -1456,7 +1456,7 @@ checksum = "a13951ea98c0a4c372f162d669193b4c9d991512de9f2381dd161027f34b26b1" [[package]] name = "deno_cron" -version = "0.54.0" +version = "0.55.0" dependencies = [ "anyhow", "async-trait", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "deno_crypto" -version = "0.188.0" +version = "0.189.0" dependencies = [ "aes", "aes-gcm", @@ -1531,7 +1531,7 @@ dependencies = [ [[package]] name = "deno_fetch" -version = "0.198.0" +version = "0.199.0" dependencies = [ "base64 0.21.7", "bytes", @@ -1564,7 +1564,7 @@ dependencies = [ [[package]] name = "deno_ffi" -version = "0.161.0" +version = "0.162.0" dependencies = [ "deno_core", "deno_permissions", @@ -1584,7 +1584,7 @@ dependencies = [ [[package]] name = "deno_fs" -version = "0.84.0" +version = "0.85.0" dependencies = [ "async-trait", "base32", @@ -1635,7 +1635,7 @@ dependencies = [ [[package]] name = "deno_http" -version = "0.172.0" +version = "0.173.0" dependencies = [ "async-compression", "async-trait", @@ -1674,7 +1674,7 @@ dependencies = [ [[package]] name = "deno_io" -version = "0.84.0" +version = "0.85.0" dependencies = [ "async-trait", "deno_core", @@ -1695,7 +1695,7 @@ dependencies = [ [[package]] name = "deno_kv" -version = "0.82.0" +version = "0.83.0" dependencies = [ "anyhow", "async-trait", @@ -1767,7 +1767,7 @@ dependencies = [ [[package]] name = "deno_napi" -version = "0.105.0" +version = "0.106.0" dependencies = [ "deno_core", "deno_permissions", @@ -1795,7 +1795,7 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.166.0" +version = "0.167.0" dependencies = [ "deno_core", "deno_permissions", @@ -1812,7 +1812,7 @@ dependencies = [ [[package]] name = "deno_node" -version = "0.111.0" +version = "0.112.0" dependencies = [ "aead-gcm-stream", "aes", @@ -1961,7 +1961,7 @@ dependencies = [ [[package]] name = "deno_permissions" -version = "0.34.0" +version = "0.35.0" dependencies = [ "deno_core", "deno_path_util", @@ -1978,7 +1978,7 @@ dependencies = [ [[package]] name = "deno_resolver" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "base32", @@ -1994,7 +1994,7 @@ dependencies = [ [[package]] name = "deno_runtime" -version = "0.183.0" +version = "0.184.0" dependencies = [ "color-print", "deno_ast", @@ -2112,7 +2112,7 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.161.0" +version = "0.162.0" dependencies = [ "deno_core", "deno_native_certs", @@ -2161,7 +2161,7 @@ dependencies = [ [[package]] name = "deno_url" -version = "0.174.0" +version = "0.175.0" dependencies = [ "deno_bench_util", "deno_console", @@ -2173,7 +2173,7 @@ dependencies = [ [[package]] name = "deno_web" -version = "0.205.0" +version = "0.206.0" dependencies = [ "async-trait", "base64-simd 0.8.0", @@ -2195,7 +2195,7 @@ dependencies = [ [[package]] name = "deno_webgpu" -version = "0.141.0" +version = "0.142.0" dependencies = [ "deno_core", "raw-window-handle", @@ -2208,7 +2208,7 @@ dependencies = [ [[package]] name = "deno_webidl" -version = "0.174.0" +version = "0.175.0" dependencies = [ "deno_bench_util", "deno_core", @@ -2216,7 +2216,7 @@ dependencies = [ [[package]] name = "deno_websocket" -version = "0.179.0" +version = "0.180.0" dependencies = [ "bytes", "deno_core", @@ -2238,7 +2238,7 @@ dependencies = [ [[package]] name = "deno_webstorage" -version = "0.169.0" +version = "0.170.0" dependencies = [ "deno_core", "deno_web", @@ -4483,7 +4483,7 @@ dependencies = [ [[package]] name = "napi_sym" -version = "0.104.0" +version = "0.105.0" dependencies = [ "quote", "serde", @@ -4538,7 +4538,7 @@ dependencies = [ [[package]] name = "node_resolver" -version = "0.13.0" +version = "0.14.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index b622cbe443..aab9e149b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,16 +48,16 @@ repository = "https://github.com/denoland/deno" deno_ast = { version = "=0.42.2", features = ["transpiling"] } deno_core = { version = "0.314.2" } -deno_bench_util = { version = "0.168.0", path = "./bench_util" } +deno_bench_util = { version = "0.169.0", path = "./bench_util" } deno_lockfile = "=0.23.1" deno_media_type = { version = "0.1.4", features = ["module_specifier"] } deno_npm = "=0.25.4" deno_path_util = "=0.2.1" -deno_permissions = { version = "0.34.0", path = "./runtime/permissions" } -deno_runtime = { version = "0.183.0", path = "./runtime" } +deno_permissions = { version = "0.35.0", path = "./runtime/permissions" } +deno_runtime = { version = "0.184.0", path = "./runtime" } deno_semver = "=0.5.16" deno_terminal = "0.2.0" -napi_sym = { version = "0.104.0", path = "./ext/napi/sym" } +napi_sym = { version = "0.105.0", path = "./ext/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } denokv_proto = "0.8.1" @@ -66,32 +66,32 @@ denokv_remote = "0.8.1" denokv_sqlite = { default-features = false, version = "0.8.2" } # exts -deno_broadcast_channel = { version = "0.168.0", path = "./ext/broadcast_channel" } -deno_cache = { version = "0.106.0", path = "./ext/cache" } -deno_canvas = { version = "0.43.0", path = "./ext/canvas" } -deno_console = { version = "0.174.0", path = "./ext/console" } -deno_cron = { version = "0.54.0", path = "./ext/cron" } -deno_crypto = { version = "0.188.0", path = "./ext/crypto" } -deno_fetch = { version = "0.198.0", path = "./ext/fetch" } -deno_ffi = { version = "0.161.0", path = "./ext/ffi" } -deno_fs = { version = "0.84.0", path = "./ext/fs" } -deno_http = { version = "0.172.0", path = "./ext/http" } -deno_io = { version = "0.84.0", path = "./ext/io" } -deno_kv = { version = "0.82.0", path = "./ext/kv" } -deno_napi = { version = "0.105.0", path = "./ext/napi" } -deno_net = { version = "0.166.0", path = "./ext/net" } -deno_node = { version = "0.111.0", path = "./ext/node" } -deno_tls = { version = "0.161.0", path = "./ext/tls" } -deno_url = { version = "0.174.0", path = "./ext/url" } -deno_web = { version = "0.205.0", path = "./ext/web" } -deno_webgpu = { version = "0.141.0", path = "./ext/webgpu" } -deno_webidl = { version = "0.174.0", path = "./ext/webidl" } -deno_websocket = { version = "0.179.0", path = "./ext/websocket" } -deno_webstorage = { version = "0.169.0", path = "./ext/webstorage" } +deno_broadcast_channel = { version = "0.169.0", path = "./ext/broadcast_channel" } +deno_cache = { version = "0.107.0", path = "./ext/cache" } +deno_canvas = { version = "0.44.0", path = "./ext/canvas" } +deno_console = { version = "0.175.0", path = "./ext/console" } +deno_cron = { version = "0.55.0", path = "./ext/cron" } +deno_crypto = { version = "0.189.0", path = "./ext/crypto" } +deno_fetch = { version = "0.199.0", path = "./ext/fetch" } +deno_ffi = { version = "0.162.0", path = "./ext/ffi" } +deno_fs = { version = "0.85.0", path = "./ext/fs" } +deno_http = { version = "0.173.0", path = "./ext/http" } +deno_io = { version = "0.85.0", path = "./ext/io" } +deno_kv = { version = "0.83.0", path = "./ext/kv" } +deno_napi = { version = "0.106.0", path = "./ext/napi" } +deno_net = { version = "0.167.0", path = "./ext/net" } +deno_node = { version = "0.112.0", path = "./ext/node" } +deno_tls = { version = "0.162.0", path = "./ext/tls" } +deno_url = { version = "0.175.0", path = "./ext/url" } +deno_web = { version = "0.206.0", path = "./ext/web" } +deno_webgpu = { version = "0.142.0", path = "./ext/webgpu" } +deno_webidl = { version = "0.175.0", path = "./ext/webidl" } +deno_websocket = { version = "0.180.0", path = "./ext/websocket" } +deno_webstorage = { version = "0.170.0", path = "./ext/webstorage" } # resolvers -deno_resolver = { version = "0.6.0", path = "./resolvers/deno" } -node_resolver = { version = "0.13.0", path = "./resolvers/node" } +deno_resolver = { version = "0.7.0", path = "./resolvers/deno" } +node_resolver = { version = "0.14.0", path = "./resolvers/node" } aes = "=0.8.3" anyhow = "1.0.57" diff --git a/Releases.md b/Releases.md index 5576d4bd8c..a183e01ecd 100644 --- a/Releases.md +++ b/Releases.md @@ -6,6 +6,38 @@ https://github.com/denoland/deno/releases We also have one-line install commands at: https://github.com/denoland/deno_install +### 2.0.4 / 2024.10.29 + +- Revert "fix(ext/node): fix dns.lookup result ordering (#26264)" (#26621) +- Revert "fix(ext/node): use primordials in `ext/node/polyfills/https.ts` + (#26323)" (#26613) +- feat(lsp): "typescript.preferences.preferTypeOnlyAutoImports" setting (#26546) +- fix(check): expose more globals from @types/node (#26603) +- fix(check): ignore resolving `jsxImportSource` when jsx is not used in graph + (#26548) +- fix(cli): Make --watcher CLEAR_SCREEN clear scrollback buffer as well as + visible screen (#25997) +- fix(compile): regression handling redirects (#26586) +- fix(ext/napi): export dynamic symbols list for {Free,Open}BSD (#26605) +- fix(ext/node): add path to `fs.stat` and `fs.statSync` error (#26037) +- fix(ext/node): compatibility with {Free,Open}BSD (#26604) +- fix(ext/node): use primordials in + ext\node\polyfills\internal\crypto\_randomInt.ts (#26534) +- fix(install): cache json exports of JSR packages (#26552) +- fix(install): regression - do not panic when config file contains \r\n + newlines (#26547) +- fix(lsp): make missing import action fix infallible (#26539) +- fix(npm): match npm bearer token generation (#26544) +- fix(upgrade): stop running `deno lsp` processes on windows before attempting + to replace executable (#26542) +- fix(watch): don't panic on invalid file specifiers (#26577) +- fix: do not panic when failing to write to http cache (#26591) +- fix: provide hints in terminal errors for Node.js globals (#26610) +- fix: report exceptions from nextTick (#26579) +- fix: support watch flag to enable watching other files than the main module on + serve subcommand (#26622) +- perf: pass transpiled module to deno_core as known string (#26555) + ### 2.0.3 / 2024.10.25 - feat(lsp): interactive inlay hints (#26382) diff --git a/bench_util/Cargo.toml b/bench_util/Cargo.toml index eea9c69a3c..31c27b9a9d 100644 --- a/bench_util/Cargo.toml +++ b/bench_util/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_bench_util" -version = "0.168.0" +version = "0.169.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0065a2cbd0..ef92947d21 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno" -version = "2.0.3" +version = "2.0.4" authors.workspace = true default-run = "deno" edition.workspace = true diff --git a/ext/broadcast_channel/Cargo.toml b/ext/broadcast_channel/Cargo.toml index 845544f0c8..e6eeeed5ec 100644 --- a/ext/broadcast_channel/Cargo.toml +++ b/ext/broadcast_channel/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_broadcast_channel" -version = "0.168.0" +version = "0.169.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cache/Cargo.toml b/ext/cache/Cargo.toml index a07a8f7207..7cae8d88a2 100644 --- a/ext/cache/Cargo.toml +++ b/ext/cache/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cache" -version = "0.106.0" +version = "0.107.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 4c125ea9b5..5d2c371999 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_canvas" -version = "0.43.0" +version = "0.44.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/console/Cargo.toml b/ext/console/Cargo.toml index b89f67fece..b3543ed85f 100644 --- a/ext/console/Cargo.toml +++ b/ext/console/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_console" -version = "0.174.0" +version = "0.175.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cron/Cargo.toml b/ext/cron/Cargo.toml index 6c5547c35f..0d52cb6a97 100644 --- a/ext/cron/Cargo.toml +++ b/ext/cron/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cron" -version = "0.54.0" +version = "0.55.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/crypto/Cargo.toml b/ext/crypto/Cargo.toml index e9761f48f1..95981bba3b 100644 --- a/ext/crypto/Cargo.toml +++ b/ext/crypto/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_crypto" -version = "0.188.0" +version = "0.189.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml index 316a6eea81..93fc88ae64 100644 --- a/ext/fetch/Cargo.toml +++ b/ext/fetch/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fetch" -version = "0.198.0" +version = "0.199.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/ffi/Cargo.toml b/ext/ffi/Cargo.toml index 405030df29..58144790bf 100644 --- a/ext/ffi/Cargo.toml +++ b/ext/ffi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_ffi" -version = "0.161.0" +version = "0.162.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml index 2a1c5bf2c6..0ccde7f7e2 100644 --- a/ext/fs/Cargo.toml +++ b/ext/fs/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fs" -version = "0.84.0" +version = "0.85.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/http/Cargo.toml b/ext/http/Cargo.toml index 0bae99ef76..a1a4cb20be 100644 --- a/ext/http/Cargo.toml +++ b/ext/http/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_http" -version = "0.172.0" +version = "0.173.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/io/Cargo.toml b/ext/io/Cargo.toml index f7ffe46bc3..7e858d298c 100644 --- a/ext/io/Cargo.toml +++ b/ext/io/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_io" -version = "0.84.0" +version = "0.85.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/kv/Cargo.toml b/ext/kv/Cargo.toml index a4e35193d5..bbd2349011 100644 --- a/ext/kv/Cargo.toml +++ b/ext/kv/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_kv" -version = "0.82.0" +version = "0.83.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/Cargo.toml b/ext/napi/Cargo.toml index 622429f7c8..c31367118e 100644 --- a/ext/napi/Cargo.toml +++ b/ext/napi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_napi" -version = "0.105.0" +version = "0.106.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/sym/Cargo.toml b/ext/napi/sym/Cargo.toml index 9fe2d839ad..c23054eb40 100644 --- a/ext/napi/sym/Cargo.toml +++ b/ext/napi/sym/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "napi_sym" -version = "0.104.0" +version = "0.105.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/net/Cargo.toml b/ext/net/Cargo.toml index a458dbe9e3..5ffd1b4504 100644 --- a/ext/net/Cargo.toml +++ b/ext/net/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_net" -version = "0.166.0" +version = "0.167.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index 3743e6a85e..ed31dbf17e 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_node" -version = "0.111.0" +version = "0.112.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/tls/Cargo.toml b/ext/tls/Cargo.toml index d163fd7eb4..6871faa5b8 100644 --- a/ext/tls/Cargo.toml +++ b/ext/tls/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_tls" -version = "0.161.0" +version = "0.162.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/url/Cargo.toml b/ext/url/Cargo.toml index ef7890e2ae..24d81abf83 100644 --- a/ext/url/Cargo.toml +++ b/ext/url/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_url" -version = "0.174.0" +version = "0.175.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/web/Cargo.toml b/ext/web/Cargo.toml index 0cb08e19c1..a02cff98fc 100644 --- a/ext/web/Cargo.toml +++ b/ext/web/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_web" -version = "0.205.0" +version = "0.206.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webgpu/Cargo.toml b/ext/webgpu/Cargo.toml index 84ee4a012f..3bb814c25c 100644 --- a/ext/webgpu/Cargo.toml +++ b/ext/webgpu/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webgpu" -version = "0.141.0" +version = "0.142.0" authors = ["the Deno authors"] edition.workspace = true license = "MIT" diff --git a/ext/webidl/Cargo.toml b/ext/webidl/Cargo.toml index a403db2fea..1685bb1fd0 100644 --- a/ext/webidl/Cargo.toml +++ b/ext/webidl/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webidl" -version = "0.174.0" +version = "0.175.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/websocket/Cargo.toml b/ext/websocket/Cargo.toml index b48b39c99f..6a575cb416 100644 --- a/ext/websocket/Cargo.toml +++ b/ext/websocket/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_websocket" -version = "0.179.0" +version = "0.180.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webstorage/Cargo.toml b/ext/webstorage/Cargo.toml index 911e252414..079ab2e4ee 100644 --- a/ext/webstorage/Cargo.toml +++ b/ext/webstorage/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webstorage" -version = "0.169.0" +version = "0.170.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/deno/Cargo.toml b/resolvers/deno/Cargo.toml index 418e1cb522..d8569d7219 100644 --- a/resolvers/deno/Cargo.toml +++ b/resolvers/deno/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_resolver" -version = "0.6.0" +version = "0.7.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/node/Cargo.toml b/resolvers/node/Cargo.toml index 9638899209..95f7eabad8 100644 --- a/resolvers/node/Cargo.toml +++ b/resolvers/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "node_resolver" -version = "0.13.0" +version = "0.14.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 703bd5a432..c4c0c8ff9b 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_runtime" -version = "0.183.0" +version = "0.184.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/runtime/permissions/Cargo.toml b/runtime/permissions/Cargo.toml index 45266791ae..bfa37b4898 100644 --- a/runtime/permissions/Cargo.toml +++ b/runtime/permissions/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_permissions" -version = "0.34.0" +version = "0.35.0" authors.workspace = true edition.workspace = true license.workspace = true From 1431ffa9f8142e4f76f565c51301f4fb174e449a Mon Sep 17 00:00:00 2001 From: McSneaky Date: Wed, 30 Oct 2024 17:32:04 +0200 Subject: [PATCH 48/63] docs(console): Update docstrings for install and uninstall (#26623) When running `deno -h` then `install` and `uninstall` scripts had description since deno 1 times :) --- cli/args/flags.rs | 4 ++-- tests/integration/flags_tests.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 235dd53a42..1a1213aac2 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -1179,8 +1179,8 @@ static DENO_HELP: &str = cstr!( Dependency management: add Add dependencies deno add jsr:@std/assert | deno add npm:express - install Install script as an executable - uninstall Uninstall a script previously installed with deno install + install Installs dependencies either in the local project or globally to a bin directory + uninstall Uninstalls a dependency or an executable script in the installation root's bin directory remove Remove dependencies from the configuration file Tooling: diff --git a/tests/integration/flags_tests.rs b/tests/integration/flags_tests.rs index 455507b9fa..663da363da 100644 --- a/tests/integration/flags_tests.rs +++ b/tests/integration/flags_tests.rs @@ -18,8 +18,8 @@ fn help_output() { "Start an interactive Read-Eval-Print Loop (REPL) for Deno", "Evaluate a script from the command line", "Add dependencies", - "Install script as an executable", - "Uninstall a script previously installed with deno install", + "Installs dependencies either in the local project or globally to a bin directory", + "Uninstalls a dependency or an executable script in the installation root's bin directory", "Run benchmarks", "Type-check the dependencies", "Compile the script into a self contained executable", From 5f5ac4063142c6c411486581f2f26cf51a32d7e3 Mon Sep 17 00:00:00 2001 From: HasanAlrimawi <141642411+HasanAlrimawi@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:32:18 +0200 Subject: [PATCH 49/63] fix(serve): support serve hmr (#26078) This PR addresses issue #25600 Changes: Updated `fn has_hmr` to check `serve` subcommand and return its hmr value if found, in order to run the worker in serve mode with hmr_runner. Thus the hmr event can be dispatched upon changes on the file served. --- cli/args/mod.rs | 6 ++++++ cli/tools/serve.rs | 11 ++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cli/args/mod.rs b/cli/args/mod.rs index f5ecc04496..754cd38fa9 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -1452,6 +1452,12 @@ impl CliOptions { watch: Some(WatchFlagsWithPaths { hmr, .. }), .. }) = &self.flags.subcommand + { + *hmr + } else if let DenoSubcommand::Serve(ServeFlags { + watch: Some(WatchFlagsWithPaths { hmr, .. }), + .. + }) = &self.flags.subcommand { *hmr } else { diff --git a/cli/tools/serve.rs b/cli/tools/serve.rs index 4ce1cad6f2..e3f9e94f8e 100644 --- a/cli/tools/serve.rs +++ b/cli/tools/serve.rs @@ -44,12 +44,15 @@ pub async fn serve( maybe_npm_install(&factory).await?; let worker_factory = factory.create_cli_main_worker_factory().await?; - + let hmr = serve_flags + .watch + .map(|watch_flags| watch_flags.hmr) + .unwrap_or(false); do_serve( worker_factory, main_module.clone(), serve_flags.worker_count, - false, + hmr, ) .await } @@ -109,8 +112,6 @@ async fn do_serve( } } Ok(exit_code) - - // main.await? } async fn run_worker( @@ -119,7 +120,7 @@ async fn run_worker( main_module: ModuleSpecifier, hmr: bool, ) -> Result { - let mut worker = worker_factory + let mut worker: crate::worker::CliMainWorker = worker_factory .create_main_worker( deno_runtime::WorkerExecutionMode::Serve { is_main: false, From a346071dd2bd822a7c2abcb187406efe7d1d1471 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:13:32 -0700 Subject: [PATCH 50/63] fix(ext/node): return `this` from `http.Server.ref/unref()` (#26647) Fixes https://github.com/denoland/deno/issues/26642 --- ext/node/polyfills/http.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/node/polyfills/http.ts b/ext/node/polyfills/http.ts index e8a189f70f..9a920adeee 100644 --- a/ext/node/polyfills/http.ts +++ b/ext/node/polyfills/http.ts @@ -1802,6 +1802,8 @@ export class ServerImpl extends EventEmitter { this.#server.ref(); } this.#unref = false; + + return this; } unref() { @@ -1809,6 +1811,8 @@ export class ServerImpl extends EventEmitter { this.#server.unref(); } this.#unref = true; + + return this; } close(cb?: (err?: Error) => void): this { From a8846eb70f96445753fe5f8f56e6155cb2d0fac6 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 30 Oct 2024 16:43:22 -0400 Subject: [PATCH 51/63] fix: remove permission check in op_require_node_module_paths (#26645) --- ext/node/ops/require.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ext/node/ops/require.rs b/ext/node/ops/require.rs index f4607f4e8a..43867053b0 100644 --- a/ext/node/ops/require.rs +++ b/ext/node/ops/require.rs @@ -131,9 +131,6 @@ where normalize_path(current_dir.join(from)) }; - let from = ensure_read_permission::

(state, &from) - .map_err(RequireError::Permission)?; - if cfg!(windows) { // return root node_modules when path is 'D:\\'. let from_str = from.to_str().unwrap(); @@ -154,7 +151,7 @@ where } let mut paths = Vec::with_capacity(from.components().count()); - let mut current_path = from.as_ref(); + let mut current_path = from.as_path(); let mut maybe_parent = Some(current_path); while let Some(parent) = maybe_parent { if !parent.ends_with("node_modules") { From 8bfd134da6d730cc1d182ab430b1713901d7a5b5 Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Thu, 31 Oct 2024 10:10:07 +0530 Subject: [PATCH 52/63] fix: clamp smi in fast calls by default (#26506) Fixes https://github.com/denoland/deno/issues/26480 Ref https://github.com/denoland/deno_core/commit/d2945fb65bca56ebfa7bb80556a4c8f4330d2315 --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- ext/node/ops/crypto/mod.rs | 8 ++++---- runtime/inspector_server.rs | 7 +++++++ runtime/worker.rs | 8 +++++++- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 36ba70d9d2..3fa6203842 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1418,9 +1418,9 @@ dependencies = [ [[package]] name = "deno_core" -version = "0.314.2" +version = "0.316.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83138917579676069b423c3eb9be3c1e579f60dc022d85f6ded4c792456255ff" +checksum = "94f68061c88ced959c6b0417f0f0d0b3dbeaeb18013b55f86c505e9fba705cf8" dependencies = [ "anyhow", "bincode", @@ -1921,9 +1921,9 @@ dependencies = [ [[package]] name = "deno_ops" -version = "0.190.1" +version = "0.192.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f46d4e4f52f26c882b74a9b58810ea75252b807cf0966166ec333077cdfd85" +checksum = "bdb7096887508456349d7e7e09e326d157d4dba46ef1f5849bc544592ea3042a" dependencies = [ "proc-macro-rules", "proc-macro2", @@ -6169,9 +6169,9 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.223.1" +version = "0.225.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf3d859dda87ee96423c01244f10af864fa6d6a9fcdc2b77e0595078ea0ea11" +checksum = "ce4b71200ef49a9e629edaea3d13fc98c25ede07e1496558df7f09354e37976f" dependencies = [ "num-bigint", "serde", diff --git a/Cargo.toml b/Cargo.toml index aab9e149b0..285c077809 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ repository = "https://github.com/denoland/deno" [workspace.dependencies] deno_ast = { version = "=0.42.2", features = ["transpiling"] } -deno_core = { version = "0.314.2" } +deno_core = { version = "0.316.0" } deno_bench_util = { version = "0.169.0", path = "./bench_util" } deno_lockfile = "=0.23.1" diff --git a/ext/node/ops/crypto/mod.rs b/ext/node/ops/crypto/mod.rs index 600d315587..0cf34511b4 100644 --- a/ext/node/ops/crypto/mod.rs +++ b/ext/node/ops/crypto/mod.rs @@ -519,11 +519,11 @@ pub fn op_node_dh_compute_secret( } #[op2(fast)] -#[smi] +#[number] pub fn op_node_random_int( - #[smi] min: i32, - #[smi] max: i32, -) -> Result { + #[number] min: i64, + #[number] max: i64, +) -> Result { let mut rng = rand::thread_rng(); // Uniform distribution is required to avoid Modulo Bias // https://en.wikipedia.org/wiki/Fisher–Yates_shuffle#Modulo_bias diff --git a/runtime/inspector_server.rs b/runtime/inspector_server.rs index 1f8cd5e714..a789dd3dca 100644 --- a/runtime/inspector_server.rs +++ b/runtime/inspector_server.rs @@ -19,6 +19,8 @@ use deno_core::serde_json::Value; use deno_core::unsync::spawn; use deno_core::url::Url; use deno_core::InspectorMsg; +use deno_core::InspectorSessionKind; +use deno_core::InspectorSessionOptions; use deno_core::InspectorSessionProxy; use deno_core::JsRuntime; use fastwebsockets::Frame; @@ -192,6 +194,11 @@ fn handle_ws_request( let inspector_session_proxy = InspectorSessionProxy { tx: outbound_tx, rx: inbound_rx, + options: InspectorSessionOptions { + kind: InspectorSessionKind::NonBlocking { + wait_for_disconnect: true, + }, + }, }; log::info!("Debugger session started."); diff --git a/runtime/worker.rs b/runtime/worker.rs index 477d3b880c..b780aefc1b 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -20,6 +20,8 @@ use deno_core::CompiledWasmModuleStore; use deno_core::Extension; use deno_core::FeatureChecker; use deno_core::GetErrorClassFn; +use deno_core::InspectorSessionKind; +use deno_core::InspectorSessionOptions; use deno_core::JsRuntime; use deno_core::LocalInspectorSession; use deno_core::ModuleCodeString; @@ -792,7 +794,11 @@ impl MainWorker { /// was not configured to create inspector. pub fn create_inspector_session(&mut self) -> LocalInspectorSession { self.js_runtime.maybe_init_inspector(); - self.js_runtime.inspector().borrow().create_local_session() + self.js_runtime.inspector().borrow().create_local_session( + InspectorSessionOptions { + kind: InspectorSessionKind::Blocking, + }, + ) } pub async fn run_event_loop( From 2f0c25d33fc0c8bb8e6e92826b2549f429d68d42 Mon Sep 17 00:00:00 2001 From: Nayeem Rahman Date: Thu, 31 Oct 2024 10:52:43 +0000 Subject: [PATCH 53/63] fix(lsp): include unstable features from editor settings (#26655) --- cli/lsp/config.rs | 12 +++++++++++- cli/lsp/language_server.rs | 10 +++------- cli/lsp/resolver.rs | 4 ++-- tests/integration/lsp_tests.rs | 30 ++++++++++++++---------------- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index 3ffc9e657a..cac0637a05 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -41,6 +41,7 @@ use deno_runtime::deno_node::PackageJson; use indexmap::IndexSet; use lsp_types::ClientCapabilities; use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::collections::HashMap; use std::ops::Deref; use std::ops::DerefMut; @@ -1190,6 +1191,7 @@ pub struct ConfigData { pub resolver: Arc, pub sloppy_imports_resolver: Option>, pub import_map_from_settings: Option, + pub unstable: BTreeSet, watched_files: HashMap, } @@ -1587,9 +1589,16 @@ impl ConfigData { .join("\n") ); } + let unstable = member_dir + .workspace + .unstable_features() + .iter() + .chain(settings.unstable.as_deref()) + .cloned() + .collect::>(); let unstable_sloppy_imports = std::env::var("DENO_UNSTABLE_SLOPPY_IMPORTS") .is_ok() - || member_dir.workspace.has_unstable("sloppy-imports"); + || unstable.contains("sloppy-imports"); let sloppy_imports_resolver = unstable_sloppy_imports.then(|| { Arc::new(CliSloppyImportsResolver::new( SloppyImportsCachedFs::new_without_stat_cache(Arc::new( @@ -1630,6 +1639,7 @@ impl ConfigData { lockfile, npmrc, import_map_from_settings, + unstable, watched_files, } } diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 33ae539f85..a592245ce7 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -1384,14 +1384,10 @@ impl Inner { .clone(); fmt_options.use_tabs = Some(!params.options.insert_spaces); fmt_options.indent_width = Some(params.options.tab_size as u8); - let maybe_workspace = self - .config - .tree - .data_for_specifier(&specifier) - .map(|d| &d.member_dir.workspace); + let config_data = self.config.tree.data_for_specifier(&specifier); let unstable_options = UnstableFmtOptions { - component: maybe_workspace - .map(|w| w.has_unstable("fmt-component")) + component: config_data + .map(|d| d.unstable.contains("fmt-component")) .unwrap_or(false), }; let document = document.clone(); diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index c89273147a..00587f8f50 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -555,8 +555,8 @@ fn create_graph_resolver( workspace.to_maybe_jsx_import_source_config().ok().flatten() }), maybe_vendor_dir: config_data.and_then(|d| d.vendor_dir.as_ref()), - bare_node_builtins_enabled: workspace - .is_some_and(|workspace| workspace.has_unstable("bare-node-builtins")), + bare_node_builtins_enabled: config_data + .is_some_and(|d| d.unstable.contains("bare-node-builtins")), sloppy_imports_resolver: config_data .and_then(|d| d.sloppy_imports_resolver.clone()), })) diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index 79e6dc5c48..2376aebd90 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -15484,25 +15484,23 @@ fn lsp_sloppy_imports() { fn lsp_sloppy_imports_prefers_dts() { let context = TestContextBuilder::new().use_temp_cwd().build(); let temp_dir = context.temp_dir(); - let temp_dir = temp_dir.path(); - - temp_dir - .join("deno.json") - .write(r#"{ "unstable": ["sloppy-imports"] }"#); - - let mut client: LspClient = context - .new_lsp_command() - .set_root_dir(temp_dir.clone()) - .build(); - client.initialize_default(); - - temp_dir.join("a.js").write("export const foo: number;"); - - let a_dts = source_file(temp_dir.join("a.d.ts"), "export const foo = 3;"); + temp_dir.write("deno.json", json!({}).to_string()); + temp_dir.write("a.js", "export const foo: number;"); + let a_dts = + source_file(temp_dir.path().join("a.d.ts"), "export const foo = 3;"); let file = source_file( - temp_dir.join("file.ts"), + temp_dir.path().join("file.ts"), "import { foo } from './a.js';\nconsole.log(foo);", ); + let mut client: LspClient = context.new_lsp_command().build(); + client.initialize_default(); + client.change_configuration(json!({ + "deno": { + "enable": true, + "unstable": ["sloppy-imports"], + }, + })); + let diagnostics = client.did_open_file(&file); // no other warnings because "a.js" exists assert_eq!( From 56f25af2c7b8b158ae21271c129ea0f0bf3c258f Mon Sep 17 00:00:00 2001 From: Pig Fang Date: Thu, 31 Oct 2024 21:50:58 +0800 Subject: [PATCH 54/63] fix(fmt): fix several HTML and components issues (#26654) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #26245 Close #26324 Fix #26508 Fix #26540 Fix #26562 --------- Co-authored-by: Bartek Iwańczuk --- .dprint.json | 2 +- Cargo.lock | 4 ++-- cli/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.dprint.json b/.dprint.json index 7877f17647..ea99f2aeda 100644 --- a/.dprint.json +++ b/.dprint.json @@ -68,7 +68,7 @@ "third_party" ], "plugins": [ - "https://plugins.dprint.dev/typescript-0.93.0.wasm", + "https://plugins.dprint.dev/typescript-0.93.1.wasm", "https://plugins.dprint.dev/json-0.19.4.wasm", "https://plugins.dprint.dev/markdown-0.17.8.wasm", "https://plugins.dprint.dev/toml-0.6.3.wasm", diff --git a/Cargo.lock b/Cargo.lock index 3fa6203842..e0780b2268 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4311,9 +4311,9 @@ dependencies = [ [[package]] name = "markup_fmt" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f15d7b24ae4ea9b87279bc0696462a4fb6c2168847f2cc162a2da05fe1a0f61" +checksum = "ebae65c91eab3d42231232bf48107f351e5a8d511454927218c53aeb68bbdb6f" dependencies = [ "aho-corasick", "css_dataset", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ef92947d21..782adfd903 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -129,7 +129,7 @@ libz-sys.workspace = true log = { workspace = true, features = ["serde"] } lsp-types.workspace = true malva = "=0.11.0" -markup_fmt = "=0.14.0" +markup_fmt = "=0.15.0" memmem.workspace = true monch.workspace = true notify.workspace = true From 50ea707b58dff85b479905ebbeef866c95bc3cad Mon Sep 17 00:00:00 2001 From: Taku Amano Date: Thu, 31 Oct 2024 23:20:26 +0900 Subject: [PATCH 55/63] fix(coverage): exclude comment lines from coverage reports (#25939) --- cli/tools/coverage/mod.rs | 19 ++++- tests/testdata/coverage/complex.ts | 13 +++ tests/testdata/coverage/complex_expected.lcov | 82 ++++++++----------- tests/testdata/coverage/complex_expected.out | 31 +++---- 4 files changed, 80 insertions(+), 65 deletions(-) diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 3b08f2c77a..48922f1449 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -327,6 +327,7 @@ fn generate_coverage_report( coverage_report.found_lines = if let Some(source_map) = maybe_source_map.as_ref() { + let script_source_lines = script_source.lines().collect::>(); let mut found_lines = line_counts .iter() .enumerate() @@ -334,7 +335,23 @@ fn generate_coverage_report( // get all the mappings from this destination line to a different src line let mut results = source_map .tokens() - .filter(move |token| token.get_dst_line() as usize == index) + .filter(|token| { + let dst_line = token.get_dst_line() as usize; + dst_line == index && { + let dst_col = token.get_dst_col() as usize; + let content = script_source_lines + .get(dst_line) + .and_then(|line| { + line.get(dst_col..std::cmp::min(dst_col + 2, line.len())) + }) + .unwrap_or(""); + + !content.is_empty() + && content != "/*" + && content != "*/" + && content != "//" + } + }) .map(move |token| (token.get_src_line() as usize, *count)) .collect::>(); // only keep the results that point at different src lines diff --git a/tests/testdata/coverage/complex.ts b/tests/testdata/coverage/complex.ts index d128b54373..316986cd22 100644 --- a/tests/testdata/coverage/complex.ts +++ b/tests/testdata/coverage/complex.ts @@ -1,3 +1,8 @@ +/** + * @module + * Complex module + */ + // This entire interface should be completely ignored by the coverage tool. export interface Complex { // These comments should be ignored. @@ -19,6 +24,7 @@ function dependency( bar: string, baz: string, ): Complex { + // inline comment in tested function return { foo, bar, @@ -34,6 +40,9 @@ export function complex( bar: string, baz: string, ): Complex { + /* + * block comment in tested function + */ return dependency( foo, bar, @@ -48,6 +57,7 @@ export function unused( bar: string, baz: string, ): Complex { + // inline comment in untested function return complex( foo, bar, @@ -62,6 +72,9 @@ export const π = Math.PI; // And same applies for this one, this one is unused and will show up in // lacking coverage. export function ƒ(): number { + /* + * block comment in untested function + */ return ( 0 ); diff --git a/tests/testdata/coverage/complex_expected.lcov b/tests/testdata/coverage/complex_expected.lcov index 94b86465ae..7e5c0f40f6 100644 --- a/tests/testdata/coverage/complex_expected.lcov +++ b/tests/testdata/coverage/complex_expected.lcov @@ -1,8 +1,8 @@ SF:[WILDCARD]complex.ts -FN:17,dependency -FN:32,complex -FN:46,unused -FN:64,ƒ +FN:22,dependency +FN:38,complex +FN:55,unused +FN:74,ƒ FNDA:1,dependency FNDA:1,complex FNDA:0,unused @@ -11,57 +11,41 @@ FNF:4 FNH:2 BRF:0 BRH:0 -DA:1,1 -DA:13,1 -DA:14,1 -DA:15,1 -DA:16,1 -DA:17,2 -DA:18,2 -DA:19,2 -DA:20,2 DA:22,2 DA:23,2 DA:24,2 DA:25,2 -DA:26,2 -DA:27,2 -DA:29,1 -DA:30,1 -DA:31,1 -DA:32,1 -DA:33,1 -DA:34,1 -DA:35,1 -DA:37,2 -DA:38,2 -DA:39,2 -DA:40,2 -DA:42,2 -DA:44,1 -DA:45,1 -DA:46,0 -DA:47,0 -DA:48,0 -DA:49,0 -DA:51,0 -DA:52,0 -DA:53,0 -DA:54,0 +DA:28,2 +DA:29,2 +DA:30,2 +DA:31,2 +DA:32,2 +DA:33,2 +DA:38,1 +DA:39,1 +DA:40,1 +DA:41,1 +DA:46,2 +DA:47,2 +DA:48,2 +DA:49,2 +DA:51,2 +DA:55,0 DA:56,0 -DA:58,1 -DA:59,1 -DA:60,1 -DA:62,1 -DA:63,1 +DA:57,0 +DA:58,0 +DA:61,0 +DA:62,0 +DA:63,0 DA:64,0 -DA:65,0 DA:66,0 -DA:68,0 DA:70,1 -DA:71,0 -DA:73,1 -DA:74,1 -LH:37 -LF:51 +DA:74,0 +DA:78,0 +DA:79,0 +DA:81,0 +DA:84,0 +DA:87,1 +LH:21 +LF:35 end_of_record diff --git a/tests/testdata/coverage/complex_expected.out b/tests/testdata/coverage/complex_expected.out index 3d5f6a0ab6..f626024b96 100644 --- a/tests/testdata/coverage/complex_expected.out +++ b/tests/testdata/coverage/complex_expected.out @@ -1,20 +1,21 @@ -cover [WILDCARD]/coverage/complex.ts ... 72.549% (37/51) - 46 | export function unused( - 47 | foo: string, - 48 | bar: string, - 49 | baz: string, +cover [WILDCARD]/coverage/complex.ts ... 60.000% (21/35) + 55 | export function unused( + 56 | foo: string, + 57 | bar: string, + 58 | baz: string, -----|----- - 51 | return complex( - 52 | foo, - 53 | bar, - 54 | baz, + 61 | return complex( + 62 | foo, + 63 | bar, + 64 | baz, -----|----- - 56 | } + 66 | } -----|----- - 64 | export function ƒ(): number { - 65 | return ( - 66 | 0 + 74 | export function ƒ(): number { -----|----- - 68 | } + 78 | return ( + 79 | 0 -----|----- - 71 | console.log("%s", () => 1); + 81 | } +-----|----- + 84 | console.log("%s", () => 1); From 90edca21a26fd2decd0603fea37af10d1e11e454 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 31 Oct 2024 11:35:17 -0400 Subject: [PATCH 56/63] fix: surface package.json location on dep parse failure (#26665) Related: https://github.com/denoland/deno/issues/26653 --- cli/args/mod.rs | 1 + cli/args/package_json.rs | 25 ++++++++++++++++--- cli/npm/managed/mod.rs | 20 +++++++++------ .../install/invalid_scheme/__test__.jsonc | 5 ++++ .../specs/install/invalid_scheme/install.out | 5 ++++ .../specs/install/invalid_scheme/package.json | 6 +++++ .../run/package_json/invalid_value/add.out | 3 ++- .../package_json/invalid_value/install.out | 3 ++- 8 files changed, 56 insertions(+), 12 deletions(-) create mode 100644 tests/specs/install/invalid_scheme/__test__.jsonc create mode 100644 tests/specs/install/invalid_scheme/install.out create mode 100644 tests/specs/install/invalid_scheme/package.json diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 754cd38fa9..d725264623 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -46,6 +46,7 @@ pub use flags::*; pub use lockfile::CliLockfile; pub use lockfile::CliLockfileReadFromPathOptions; pub use package_json::NpmInstallDepsProvider; +pub use package_json::PackageJsonDepValueParseWithLocationError; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; diff --git a/cli/args/package_json.rs b/cli/args/package_json.rs index 2ef39a30d2..7dc75550c3 100644 --- a/cli/args/package_json.rs +++ b/cli/args/package_json.rs @@ -5,10 +5,12 @@ use std::sync::Arc; use deno_config::workspace::Workspace; use deno_core::serde_json; +use deno_core::url::Url; use deno_package_json::PackageJsonDepValue; use deno_package_json::PackageJsonDepValueParseError; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageReq; +use thiserror::Error; #[derive(Debug)] pub struct InstallNpmRemotePkg { @@ -23,11 +25,20 @@ pub struct InstallNpmWorkspacePkg { pub target_dir: PathBuf, } +#[derive(Debug, Error, Clone)] +#[error("Failed to install '{}'\n at {}", alias, location)] +pub struct PackageJsonDepValueParseWithLocationError { + pub location: Url, + pub alias: String, + #[source] + pub source: PackageJsonDepValueParseError, +} + #[derive(Debug, Default)] pub struct NpmInstallDepsProvider { remote_pkgs: Vec, workspace_pkgs: Vec, - pkg_json_dep_errors: Vec, + pkg_json_dep_errors: Vec, } impl NpmInstallDepsProvider { @@ -89,7 +100,13 @@ impl NpmInstallDepsProvider { let dep = match dep { Ok(dep) => dep, Err(err) => { - pkg_json_dep_errors.push(err); + pkg_json_dep_errors.push( + PackageJsonDepValueParseWithLocationError { + location: pkg_json.specifier(), + alias, + source: err, + }, + ); continue; } }; @@ -150,7 +167,9 @@ impl NpmInstallDepsProvider { &self.workspace_pkgs } - pub fn pkg_json_dep_errors(&self) -> &[PackageJsonDepValueParseError] { + pub fn pkg_json_dep_errors( + &self, + ) -> &[PackageJsonDepValueParseWithLocationError] { &self.pkg_json_dep_errors } } diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index d0880557fe..a0754812bb 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -38,6 +38,7 @@ use crate::args::LifecycleScriptsConfig; use crate::args::NpmInstallDepsProvider; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; +use crate::args::PackageJsonDepValueParseWithLocationError; use crate::cache::DenoCacheEnvFsAdapter; use crate::cache::FastInsecureHasher; use crate::http_util::HttpClientProvider; @@ -480,19 +481,24 @@ impl ManagedCliNpmResolver { self.resolution.resolve_pkg_id_from_pkg_req(req) } - pub fn ensure_no_pkg_json_dep_errors(&self) -> Result<(), AnyError> { + pub fn ensure_no_pkg_json_dep_errors( + &self, + ) -> Result<(), Box> { for err in self.npm_install_deps_provider.pkg_json_dep_errors() { - match err { + match &err.source { deno_package_json::PackageJsonDepValueParseError::VersionReq(_) => { - return Err( - AnyError::from(err.clone()) - .context("Failed to install from package.json"), - ); + return Err(Box::new(err.clone())); } deno_package_json::PackageJsonDepValueParseError::Unsupported { .. } => { - log::warn!("{} {} in package.json", colors::yellow("Warning"), err) + // only warn for this one + log::warn!( + "{} {}\n at {}", + colors::yellow("Warning"), + err.source, + err.location, + ) } } } diff --git a/tests/specs/install/invalid_scheme/__test__.jsonc b/tests/specs/install/invalid_scheme/__test__.jsonc new file mode 100644 index 0000000000..c4d3ebbf06 --- /dev/null +++ b/tests/specs/install/invalid_scheme/__test__.jsonc @@ -0,0 +1,5 @@ +{ + "tempDir": true, + "args": "install", + "output": "install.out" +} diff --git a/tests/specs/install/invalid_scheme/install.out b/tests/specs/install/invalid_scheme/install.out new file mode 100644 index 0000000000..cf100216bd --- /dev/null +++ b/tests/specs/install/invalid_scheme/install.out @@ -0,0 +1,5 @@ +Warning Not implemented scheme 'git' + at file:///[WILDLINE]/package.json +Download http://localhost:4260/@denotest%2fadd +Download http://localhost:4260/@denotest/add/1.0.0.tgz +Initialize @denotest/add@1.0.0 diff --git a/tests/specs/install/invalid_scheme/package.json b/tests/specs/install/invalid_scheme/package.json new file mode 100644 index 0000000000..5e50ae591d --- /dev/null +++ b/tests/specs/install/invalid_scheme/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "@denotest/add": "*", + "deno": "git:denoland/deno" + } +} diff --git a/tests/specs/run/package_json/invalid_value/add.out b/tests/specs/run/package_json/invalid_value/add.out index 9b7493c1a2..32d9acaed6 100644 --- a/tests/specs/run/package_json/invalid_value/add.out +++ b/tests/specs/run/package_json/invalid_value/add.out @@ -1,5 +1,6 @@ Add npm:cowsay@1.5.0 -error: Failed to install from package.json +error: Failed to install '@denotest/cjs-default-export' + at file:///[WILDLINE]/package.json Caused by: 0: Invalid version requirement diff --git a/tests/specs/run/package_json/invalid_value/install.out b/tests/specs/run/package_json/invalid_value/install.out index cc82b345b0..ca3492ab22 100644 --- a/tests/specs/run/package_json/invalid_value/install.out +++ b/tests/specs/run/package_json/invalid_value/install.out @@ -1,4 +1,5 @@ -error: Failed to install from package.json +error: Failed to install '@denotest/cjs-default-export' + at file:///[WILDLINE]/package.json Caused by: 0: Invalid version requirement From 951103fc8de952f32386121d3f0e4803fe267ea4 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Thu, 31 Oct 2024 10:02:17 -0700 Subject: [PATCH 57/63] fix(ext/node): convert errors from `fs.readFile/fs.readFileSync` to node format (#26632) Fixes the original issue reported in #26404. storybook runs into other errors after this PR (the new errors will be fixed in other PRs). Some code used by a dependency of storybook does a [string comparison on the error message](https://github.com/chromaui/chromatic-cli/blob/ce30b2be343cb96a0826390b95ea42befb2be547/node-src/lib/getConfiguration.ts#L88-L92) thrown here to check for a file not found error. --- ext/node/polyfills/_fs/_fs_readFile.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ext/node/polyfills/_fs/_fs_readFile.ts b/ext/node/polyfills/_fs/_fs_readFile.ts index 0f05ee1678..cf7e0305d8 100644 --- a/ext/node/polyfills/_fs/_fs_readFile.ts +++ b/ext/node/polyfills/_fs/_fs_readFile.ts @@ -19,6 +19,7 @@ import { TextEncodings, } from "ext:deno_node/_utils.ts"; import { FsFile } from "ext:deno_fs/30_fs.js"; +import { denoErrorToNodeError } from "ext:deno_node/internal/errors.ts"; function maybeDecode(data: Uint8Array, encoding: TextEncodings): string; function maybeDecode( @@ -87,7 +88,7 @@ export function readFile( } const buffer = maybeDecode(data, encoding); (cb as BinaryCallback)(null, buffer); - }, (err) => cb && cb(err)); + }, (err) => cb && cb(denoErrorToNodeError(err))); } } @@ -117,7 +118,12 @@ export function readFileSync( opt?: FileOptionsArgument, ): string | Buffer { path = path instanceof URL ? pathFromURL(path) : path; - const data = Deno.readFileSync(path); + let data; + try { + data = Deno.readFileSync(path); + } catch (err) { + throw denoErrorToNodeError(err); + } const encoding = getEncoding(opt); if (encoding && encoding !== "binary") { const text = maybeDecode(data, encoding); From 6d44952d4daaaab8ed9ec82212d2256c058eb05d Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Thu, 31 Oct 2024 10:02:31 -0700 Subject: [PATCH 58/63] fix(ext/node): resolve exports even if parent module filename isn't present (#26553) Fixes https://github.com/denoland/deno/issues/26505 I'm not exactly sure how this case comes about (I tried to write tests for it but couldn't manage to reproduce it), but what happens is the parent filename ends up null, and we bail out of resolving the specifier in package exports. I've checked, and in node the parent filename is also null (so that's not a bug on our part), but node continues to resolve even in that case. So this PR should match node's behavior more closely than we currently do. --- ext/node/ops/require.rs | 8 ++++++-- ext/node/polyfills/01_require.js | 6 +----- .../cjs-multiple-exports/1.0.0/package.json | 8 ++++++++ .../cjs-multiple-exports/1.0.0/src/add.js | 3 +++ .../cjs-multiple-exports/1.0.0/src/index.js | 3 +++ .../__test__.jsonc | 13 +++++++++++++ .../main.cjs | 16 ++++++++++++++++ .../package.json | 5 +++++ 8 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/package.json create mode 100644 tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/src/add.js create mode 100644 tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/src/index.js create mode 100644 tests/specs/node/require_export_from_parent_with_no_filename/__test__.jsonc create mode 100644 tests/specs/node/require_export_from_parent_with_no_filename/main.cjs create mode 100644 tests/specs/node/require_export_from_parent_with_no_filename/package.json diff --git a/ext/node/ops/require.rs b/ext/node/ops/require.rs index 43867053b0..7524fb43c7 100644 --- a/ext/node/ops/require.rs +++ b/ext/node/ops/require.rs @@ -529,12 +529,16 @@ where return Ok(None); }; - let referrer = Url::from_file_path(parent_path).unwrap(); + let referrer = if parent_path.is_empty() { + None + } else { + Some(Url::from_file_path(parent_path).unwrap()) + }; let r = node_resolver.package_exports_resolve( &pkg.path, &format!(".{expansion}"), exports, - Some(&referrer), + referrer.as_ref(), NodeModuleKind::Cjs, REQUIRE_CONDITIONS, NodeResolutionMode::Execution, diff --git a/ext/node/polyfills/01_require.js b/ext/node/polyfills/01_require.js index 5b0980c310..7935903a8d 100644 --- a/ext/node/polyfills/01_require.js +++ b/ext/node/polyfills/01_require.js @@ -523,17 +523,13 @@ function resolveExports( return; } - if (!parentPath) { - return false; - } - return op_require_resolve_exports( usesLocalNodeModulesDir, modulesPath, request, name, expansion, - parentPath, + parentPath ?? "", ) ?? false; } diff --git a/tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/package.json b/tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/package.json new file mode 100644 index 0000000000..43f07a2351 --- /dev/null +++ b/tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/package.json @@ -0,0 +1,8 @@ +{ + "name": "@denotest/cjs-multiple-exports", + "version": "1.0.0", + "exports": { + ".": "./src/index.js", + "./add": "./src/add.js" + } +} \ No newline at end of file diff --git a/tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/src/add.js b/tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/src/add.js new file mode 100644 index 0000000000..42c8a7c604 --- /dev/null +++ b/tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/src/add.js @@ -0,0 +1,3 @@ +module.exports = function add(a, b) { + return a + b; +}; \ No newline at end of file diff --git a/tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/src/index.js b/tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/src/index.js new file mode 100644 index 0000000000..432ed652ed --- /dev/null +++ b/tests/registry/npm/@denotest/cjs-multiple-exports/1.0.0/src/index.js @@ -0,0 +1,3 @@ +module.exports = { + hello: "world" +}; \ No newline at end of file diff --git a/tests/specs/node/require_export_from_parent_with_no_filename/__test__.jsonc b/tests/specs/node/require_export_from_parent_with_no_filename/__test__.jsonc new file mode 100644 index 0000000000..a3de08e460 --- /dev/null +++ b/tests/specs/node/require_export_from_parent_with_no_filename/__test__.jsonc @@ -0,0 +1,13 @@ +{ + "tempDir": true, + "steps": [ + { + "args": "install", + "output": "[WILDCARD]" + }, + { + "args": "run -A main.cjs", + "output": "3\n" + } + ] +} diff --git a/tests/specs/node/require_export_from_parent_with_no_filename/main.cjs b/tests/specs/node/require_export_from_parent_with_no_filename/main.cjs new file mode 100644 index 0000000000..3335ae1bf9 --- /dev/null +++ b/tests/specs/node/require_export_from_parent_with_no_filename/main.cjs @@ -0,0 +1,16 @@ +const path = require("node:path"); +const Module = require("node:module"); +function requireFromString(code, filename) { + const paths = Module._nodeModulePaths((0, path.dirname)(filename)); + const m = new Module(filename, module.parent); + m.paths = paths; + m._compile(code, filename); + return m.exports; +} + +const code = ` +const add = require("@denotest/cjs-multiple-exports/add"); + +console.log(add(1, 2)); +`; +requireFromString(code, "fake.js"); diff --git a/tests/specs/node/require_export_from_parent_with_no_filename/package.json b/tests/specs/node/require_export_from_parent_with_no_filename/package.json new file mode 100644 index 0000000000..9cd6438953 --- /dev/null +++ b/tests/specs/node/require_export_from_parent_with_no_filename/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "@denotest/cjs-multiple-exports": "1.0.0" + } +} From 6c6bbeb97495e8c3e8eac7bea27bf836f02b575f Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Thu, 31 Oct 2024 22:18:33 -0700 Subject: [PATCH 59/63] fix(node): Implement `os.userInfo` properly, add missing `toPrimitive` (#24702) Fixes the implementation of `os.userInfo`, and adds a missing `toPrimitive` for `tmpdir`. This allows us to enable the corresponding node_compat test. --- ext/node/lib.rs | 2 +- ext/node/ops/os/mod.rs | 156 ++++++++++- ext/node/polyfills/internal/errors.ts | 13 - ext/node/polyfills/os.ts | 53 ++-- runtime/errors.rs | 1 + runtime/permissions/lib.rs | 8 +- tests/node_compat/config.jsonc | 3 +- tests/node_compat/runner/TODO.md | 1 - tests/node_compat/test/parallel/test-os.js | 286 +++++++++++++++++++++ 9 files changed, 469 insertions(+), 54 deletions(-) create mode 100644 tests/node_compat/test/parallel/test-os.js diff --git a/ext/node/lib.rs b/ext/node/lib.rs index 32624f38b7..b34bea815b 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -348,7 +348,7 @@ deno_core::extension!(deno_node, ops::http2::op_http2_send_response, ops::os::op_node_os_get_priority

, ops::os::op_node_os_set_priority

, - ops::os::op_node_os_username

, + ops::os::op_node_os_user_info

, ops::os::op_geteuid

, ops::os::op_getegid

, ops::os::op_cpus

, diff --git a/ext/node/ops/os/mod.rs b/ext/node/ops/os/mod.rs index b4c9eaa8ca..ea7e6b99fd 100644 --- a/ext/node/ops/os/mod.rs +++ b/ext/node/ops/os/mod.rs @@ -1,5 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::mem::MaybeUninit; + use crate::NodePermissions; use deno_core::op2; use deno_core::OpState; @@ -15,6 +17,8 @@ pub enum OsError { Permission(deno_core::error::AnyError), #[error("Failed to get cpu info")] FailedToGetCpuInfo, + #[error("Failed to get user info")] + FailedToGetUserInfo(#[source] std::io::Error), } #[op2(fast)] @@ -54,20 +58,162 @@ where priority::set_priority(pid, priority).map_err(OsError::Priority) } +#[derive(serde::Serialize)] +pub struct UserInfo { + username: String, + homedir: String, + shell: Option, +} + +#[cfg(unix)] +fn get_user_info(uid: u32) -> Result { + use std::ffi::CStr; + let mut pw: MaybeUninit = MaybeUninit::uninit(); + let mut result: *mut libc::passwd = std::ptr::null_mut(); + // SAFETY: libc call, no invariants + let max_buf_size = unsafe { libc::sysconf(libc::_SC_GETPW_R_SIZE_MAX) }; + let buf_size = if max_buf_size < 0 { + // from the man page + 16_384 + } else { + max_buf_size as usize + }; + let mut buf = { + let mut b = Vec::>::with_capacity(buf_size); + // SAFETY: MaybeUninit has no initialization invariants, and len == cap + unsafe { + b.set_len(buf_size); + } + b + }; + // SAFETY: libc call, args are correct + let s = unsafe { + libc::getpwuid_r( + uid, + pw.as_mut_ptr(), + buf.as_mut_ptr().cast(), + buf_size, + std::ptr::addr_of_mut!(result), + ) + }; + if result.is_null() { + if s != 0 { + return Err( + OsError::FailedToGetUserInfo(std::io::Error::last_os_error()), + ); + } else { + return Err(OsError::FailedToGetUserInfo(std::io::Error::from( + std::io::ErrorKind::NotFound, + ))); + } + } + // SAFETY: pw was initialized by the call to `getpwuid_r` above + let pw = unsafe { pw.assume_init() }; + // SAFETY: initialized above, pw alive until end of function, nul terminated + let username = unsafe { CStr::from_ptr(pw.pw_name) }; + // SAFETY: initialized above, pw alive until end of function, nul terminated + let homedir = unsafe { CStr::from_ptr(pw.pw_dir) }; + // SAFETY: initialized above, pw alive until end of function, nul terminated + let shell = unsafe { CStr::from_ptr(pw.pw_shell) }; + Ok(UserInfo { + username: username.to_string_lossy().into_owned(), + homedir: homedir.to_string_lossy().into_owned(), + shell: Some(shell.to_string_lossy().into_owned()), + }) +} + +#[cfg(windows)] +fn get_user_info(_uid: u32) -> Result { + use std::ffi::OsString; + use std::os::windows::ffi::OsStringExt; + + use windows_sys::Win32::Foundation::CloseHandle; + use windows_sys::Win32::Foundation::GetLastError; + use windows_sys::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER; + use windows_sys::Win32::Foundation::HANDLE; + use windows_sys::Win32::System::Threading::GetCurrentProcess; + use windows_sys::Win32::System::Threading::OpenProcessToken; + use windows_sys::Win32::UI::Shell::GetUserProfileDirectoryW; + struct Handle(HANDLE); + impl Drop for Handle { + fn drop(&mut self) { + // SAFETY: win32 call + unsafe { + CloseHandle(self.0); + } + } + } + let mut token: MaybeUninit = MaybeUninit::uninit(); + + // Get a handle to the current process + // SAFETY: win32 call + unsafe { + if OpenProcessToken( + GetCurrentProcess(), + windows_sys::Win32::Security::TOKEN_READ, + token.as_mut_ptr(), + ) == 0 + { + return Err( + OsError::FailedToGetUserInfo(std::io::Error::last_os_error()), + ); + } + } + + // SAFETY: initialized by call above + let token = Handle(unsafe { token.assume_init() }); + + let mut bufsize = 0; + // get the size for the homedir buf (it'll end up in `bufsize`) + // SAFETY: win32 call + unsafe { + GetUserProfileDirectoryW(token.0, std::ptr::null_mut(), &mut bufsize); + let err = GetLastError(); + if err != ERROR_INSUFFICIENT_BUFFER { + return Err(OsError::FailedToGetUserInfo( + std::io::Error::from_raw_os_error(err as i32), + )); + } + } + let mut path = vec![0; bufsize as usize]; + // Actually get the homedir + // SAFETY: path is `bufsize` elements + unsafe { + if GetUserProfileDirectoryW(token.0, path.as_mut_ptr(), &mut bufsize) == 0 { + return Err( + OsError::FailedToGetUserInfo(std::io::Error::last_os_error()), + ); + } + } + // remove trailing nul + path.pop(); + let homedir_wide = OsString::from_wide(&path); + let homedir = homedir_wide.to_string_lossy().into_owned(); + + Ok(UserInfo { + username: deno_whoami::username(), + homedir, + shell: None, + }) +} + #[op2] -#[string] -pub fn op_node_os_username

( +#[serde] +pub fn op_node_os_user_info

( state: &mut OpState, -) -> Result + #[smi] uid: u32, +) -> Result where P: NodePermissions + 'static, { { let permissions = state.borrow_mut::

(); - permissions.check_sys("username", "node:os.userInfo()")?; + permissions + .check_sys("userInfo", "node:os.userInfo()") + .map_err(OsError::Permission)?; } - Ok(deno_whoami::username()) + get_user_info(uid) } #[op2(fast)] diff --git a/ext/node/polyfills/internal/errors.ts b/ext/node/polyfills/internal/errors.ts index 51bd7a0250..5a3d4437a1 100644 --- a/ext/node/polyfills/internal/errors.ts +++ b/ext/node/polyfills/internal/errors.ts @@ -2558,19 +2558,6 @@ export class ERR_FS_RMDIR_ENOTDIR extends NodeSystemError { } } -export class ERR_OS_NO_HOMEDIR extends NodeSystemError { - constructor() { - const code = isWindows ? "ENOENT" : "ENOTDIR"; - const ctx: NodeSystemErrorCtx = { - message: "not a directory", - syscall: "home", - code, - errno: isWindows ? osConstants.errno.ENOENT : osConstants.errno.ENOTDIR, - }; - super(code, ctx, "Path is not a directory"); - } -} - export class ERR_HTTP_SOCKET_ASSIGNED extends NodeError { constructor() { super( diff --git a/ext/node/polyfills/os.ts b/ext/node/polyfills/os.ts index e47e8679ec..edc89ed2c3 100644 --- a/ext/node/polyfills/os.ts +++ b/ext/node/polyfills/os.ts @@ -28,16 +28,17 @@ import { op_homedir, op_node_os_get_priority, op_node_os_set_priority, - op_node_os_username, + op_node_os_user_info, } from "ext:core/ops"; import { validateIntegerRange } from "ext:deno_node/_utils.ts"; import process from "node:process"; import { isWindows } from "ext:deno_node/_util/os.ts"; -import { ERR_OS_NO_HOMEDIR } from "ext:deno_node/internal/errors.ts"; import { os } from "ext:deno_node/internal_binding/constants.ts"; import { osUptime } from "ext:runtime/30_os.js"; import { Buffer } from "ext:deno_node/internal/buffer.mjs"; +import { primordials } from "ext:core/mod.js"; +const { StringPrototypeEndsWith, StringPrototypeSlice } = primordials; export const constants = os; @@ -136,6 +137,8 @@ export function arch(): string { (uptime as any)[Symbol.toPrimitive] = (): number => uptime(); // deno-lint-ignore no-explicit-any (machine as any)[Symbol.toPrimitive] = (): string => machine(); +// deno-lint-ignore no-explicit-any +(tmpdir as any)[Symbol.toPrimitive] = (): string | null => tmpdir(); export function cpus(): CPUCoreInfo[] { return op_cpus(); @@ -268,26 +271,27 @@ export function setPriority(pid: number, priority?: number) { export function tmpdir(): string | null { /* This follows the node js implementation, but has a few differences: - * On windows, if none of the environment variables are defined, - we return null. - * On unix we use a plain Deno.env.get, instead of safeGetenv, + * We use a plain Deno.env.get, instead of safeGetenv, which special cases setuid binaries. - * Node removes a single trailing / or \, we remove all. */ if (isWindows) { - const temp = Deno.env.get("TEMP") || Deno.env.get("TMP"); - if (temp) { - return temp.replace(/(? 1 && StringPrototypeEndsWith(temp, "\\") && + !StringPrototypeEndsWith(temp, ":\\") + ) { + temp = StringPrototypeSlice(temp, 0, -1); } - const base = Deno.env.get("SYSTEMROOT") || Deno.env.get("WINDIR"); - if (base) { - return base + "\\temp"; - } - return null; + + return temp; } else { // !isWindows - const temp = Deno.env.get("TMPDIR") || Deno.env.get("TMP") || + let temp = Deno.env.get("TMPDIR") || Deno.env.get("TMP") || Deno.env.get("TEMP") || "/tmp"; - return temp.replace(/(? 1 && StringPrototypeEndsWith(temp, "/")) { + temp = StringPrototypeSlice(temp, 0, -1); + } + return temp; } } @@ -320,7 +324,6 @@ export function uptime(): number { return osUptime(); } -/** Not yet implemented */ export function userInfo( options: UserInfoOptions = { encoding: "utf-8" }, ): UserInfo { @@ -331,20 +334,10 @@ export function userInfo( uid = -1; gid = -1; } - - // TODO(@crowlKats): figure out how to do this correctly: - // The value of homedir returned by os.userInfo() is provided by the operating system. - // This differs from the result of os.homedir(), which queries environment - // variables for the home directory before falling back to the operating system response. - let _homedir = homedir(); - if (!_homedir) { - throw new ERR_OS_NO_HOMEDIR(); - } - let shell = isWindows ? null : (Deno.env.get("SHELL") || null); - let username = op_node_os_username(); + let { username, homedir, shell } = op_node_os_user_info(uid); if (options?.encoding === "buffer") { - _homedir = _homedir ? Buffer.from(_homedir) : _homedir; + homedir = homedir ? Buffer.from(homedir) : homedir; shell = shell ? Buffer.from(shell) : shell; username = Buffer.from(username); } @@ -352,7 +345,7 @@ export function userInfo( return { uid, gid, - homedir: _homedir, + homedir, shell, username, }; diff --git a/runtime/errors.rs b/runtime/errors.rs index 07bf694dc1..a5c436e751 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -1086,6 +1086,7 @@ mod node { }, OsError::Permission(e) => get_error_class_name(e).unwrap_or("Error"), OsError::FailedToGetCpuInfo => "TypeError", + OsError::FailedToGetUserInfo(e) => get_io_error_class(e), } } diff --git a/runtime/permissions/lib.rs b/runtime/permissions/lib.rs index 1e1321bb2f..84503f025b 100644 --- a/runtime/permissions/lib.rs +++ b/runtime/permissions/lib.rs @@ -1368,8 +1368,12 @@ impl SysDescriptor { match kind.as_str() { "hostname" | "osRelease" | "osUptime" | "loadavg" | "networkInterfaces" | "systemMemoryInfo" | "uid" | "gid" | "cpus" - | "homedir" | "getegid" | "username" | "statfs" | "getPriority" - | "setPriority" => Ok(Self(kind)), + | "homedir" | "getegid" | "statfs" | "getPriority" | "setPriority" + | "userInfo" => Ok(Self(kind)), + + // the underlying permission check changed to `userInfo` to better match the API, + // alias this to avoid breaking existing projects with `--allow-sys=username` + "username" => Ok(Self("userInfo".into())), _ => Err(type_error(format!("unknown system info kind \"{kind}\""))), } } diff --git a/tests/node_compat/config.jsonc b/tests/node_compat/config.jsonc index a99a427900..16951d9ede 100644 --- a/tests/node_compat/config.jsonc +++ b/tests/node_compat/config.jsonc @@ -87,8 +87,6 @@ "test-net-server-try-ports.js", "test-net-socket-timeout.js", "test-net-write-arguments.js", - // TODO(nathanwhit): Disable os.userInfo is slightly incorrect - // "test-os.js", "test-path-resolve.js", "test-querystring.js", "test-readline-interface.js", @@ -448,6 +446,7 @@ "test-next-tick-when-exiting.js", "test-next-tick.js", "test-nodeeventtarget.js", + "test-os.js", "test-outgoing-message-destroy.js", "test-outgoing-message-pipe.js", "test-parse-args.mjs", diff --git a/tests/node_compat/runner/TODO.md b/tests/node_compat/runner/TODO.md index 35a67e72d2..231a4f62c9 100644 --- a/tests/node_compat/runner/TODO.md +++ b/tests/node_compat/runner/TODO.md @@ -1878,7 +1878,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-os-homedir-no-envvar.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-os-homedir-no-envvar.js) - [parallel/test-os-process-priority.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-os-process-priority.js) - [parallel/test-os-userinfo-handles-getter-errors.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-os-userinfo-handles-getter-errors.js) -- [parallel/test-os.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-os.js) - [parallel/test-path-posix-relative-on-windows.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-path-posix-relative-on-windows.js) - [parallel/test-pending-deprecation.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-pending-deprecation.js) - [parallel/test-perf-gc-crash.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-perf-gc-crash.js) diff --git a/tests/node_compat/test/parallel/test-os.js b/tests/node_compat/test/parallel/test-os.js new file mode 100644 index 0000000000..f7c24342ad --- /dev/null +++ b/tests/node_compat/test/parallel/test-os.js @@ -0,0 +1,286 @@ +// deno-fmt-ignore-file +// deno-lint-ignore-file + +// Copyright Joyent and Node contributors. All rights reserved. MIT license. +// Taken from Node 18.12.1 +// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. + +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const os = require('os'); +const path = require('path'); +const { inspect } = require('util'); + +const is = { + number: (value, key) => { + assert(!Number.isNaN(value), `${key} should not be NaN`); + assert.strictEqual(typeof value, 'number'); + }, + string: (value) => { assert.strictEqual(typeof value, 'string'); }, + array: (value) => { assert.ok(Array.isArray(value)); }, + object: (value) => { + assert.strictEqual(typeof value, 'object'); + assert.notStrictEqual(value, null); + } +}; + +process.env.TMPDIR = '/tmpdir'; +process.env.TMP = '/tmp'; +process.env.TEMP = '/temp'; +if (common.isWindows) { + assert.strictEqual(os.tmpdir(), '/temp'); + process.env.TEMP = ''; + assert.strictEqual(os.tmpdir(), '/tmp'); + process.env.TMP = ''; + const expected = `${process.env.SystemRoot || process.env.windir}\\temp`; + assert.strictEqual(os.tmpdir(), expected); + process.env.TEMP = '\\temp\\'; + assert.strictEqual(os.tmpdir(), '\\temp'); + process.env.TEMP = '\\tmpdir/'; + assert.strictEqual(os.tmpdir(), '\\tmpdir/'); + process.env.TEMP = '\\'; + assert.strictEqual(os.tmpdir(), '\\'); + process.env.TEMP = 'C:\\'; + assert.strictEqual(os.tmpdir(), 'C:\\'); +} else { + assert.strictEqual(os.tmpdir(), '/tmpdir'); + process.env.TMPDIR = ''; + assert.strictEqual(os.tmpdir(), '/tmp'); + process.env.TMP = ''; + assert.strictEqual(os.tmpdir(), '/temp'); + process.env.TEMP = ''; + assert.strictEqual(os.tmpdir(), '/tmp'); + process.env.TMPDIR = '/tmpdir/'; + assert.strictEqual(os.tmpdir(), '/tmpdir'); + process.env.TMPDIR = '/tmpdir\\'; + assert.strictEqual(os.tmpdir(), '/tmpdir\\'); + process.env.TMPDIR = '/'; + assert.strictEqual(os.tmpdir(), '/'); +} + +const endianness = os.endianness(); +is.string(endianness); +assert.match(endianness, /[BL]E/); + +const hostname = os.hostname(); +is.string(hostname); +assert.ok(hostname.length > 0); + +// IBMi process priority is different. +if (!common.isIBMi) { + const DUMMY_PRIORITY = 10; + os.setPriority(DUMMY_PRIORITY); + const priority = os.getPriority(); + is.number(priority); + assert.strictEqual(priority, DUMMY_PRIORITY); +} + +// On IBMi, os.uptime() returns 'undefined' +if (!common.isIBMi) { + const uptime = os.uptime(); + is.number(uptime); + assert.ok(uptime > 0); +} + +const cpus = os.cpus(); +is.array(cpus); +assert.ok(cpus.length > 0); +for (const cpu of cpus) { + assert.strictEqual(typeof cpu.model, 'string'); + assert.strictEqual(typeof cpu.speed, 'number'); + assert.strictEqual(typeof cpu.times.user, 'number'); + assert.strictEqual(typeof cpu.times.nice, 'number'); + assert.strictEqual(typeof cpu.times.sys, 'number'); + assert.strictEqual(typeof cpu.times.idle, 'number'); + assert.strictEqual(typeof cpu.times.irq, 'number'); +} + +const type = os.type(); +is.string(type); +assert.ok(type.length > 0); + +const release = os.release(); +is.string(release); +assert.ok(release.length > 0); +// TODO: Check format on more than just AIX +if (common.isAIX) + assert.match(release, /^\d+\.\d+$/); + +const platform = os.platform(); +is.string(platform); +assert.ok(platform.length > 0); + +const arch = os.arch(); +is.string(arch); +assert.ok(arch.length > 0); + +if (!common.isSunOS) { + // not implemented yet + assert.ok(os.loadavg().length > 0); + assert.ok(os.freemem() > 0); + assert.ok(os.totalmem() > 0); +} + +const interfaces = os.networkInterfaces(); +switch (platform) { + case 'linux': { + const filter = (e) => + e.address === '127.0.0.1' && + e.netmask === '255.0.0.0'; + + const actual = interfaces.lo.filter(filter); + const expected = [{ + address: '127.0.0.1', + netmask: '255.0.0.0', + family: 'IPv4', + mac: '00:00:00:00:00:00', + internal: true, + cidr: '127.0.0.1/8' + }]; + assert.deepStrictEqual(actual, expected); + break; + } + case 'win32': { + const filter = (e) => + e.address === '127.0.0.1'; + + const actual = interfaces['Loopback Pseudo-Interface 1'].filter(filter); + const expected = [{ + address: '127.0.0.1', + netmask: '255.0.0.0', + family: 'IPv4', + mac: '00:00:00:00:00:00', + internal: true, + cidr: '127.0.0.1/8' + }]; + assert.deepStrictEqual(actual, expected); + break; + } +} +const netmaskToCIDRSuffixMap = new Map(Object.entries({ + '255.0.0.0': 8, + '255.255.255.0': 24, + 'ffff:ffff:ffff:ffff::': 64, + 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff': 128 +})); + +Object.values(interfaces) + .flat(Infinity) + .map((v) => ({ v, mask: netmaskToCIDRSuffixMap.get(v.netmask) })) + .forEach(({ v, mask }) => { + assert.ok('cidr' in v, `"cidr" prop not found in ${inspect(v)}`); + if (mask) { + assert.strictEqual(v.cidr, `${v.address}/${mask}`); + } + }); + +const EOL = os.EOL; +if (common.isWindows) { + assert.strictEqual(EOL, '\r\n'); +} else { + assert.strictEqual(EOL, '\n'); +} + +const home = os.homedir(); +is.string(home); +assert.ok(home.includes(path.sep)); + +const version = os.version(); +assert.strictEqual(typeof version, 'string'); +assert(version); + +if (common.isWindows && process.env.USERPROFILE) { + assert.strictEqual(home, process.env.USERPROFILE); + delete process.env.USERPROFILE; + assert.ok(os.homedir().includes(path.sep)); + process.env.USERPROFILE = home; +} else if (!common.isWindows && process.env.HOME) { + assert.strictEqual(home, process.env.HOME); + delete process.env.HOME; + assert.ok(os.homedir().includes(path.sep)); + process.env.HOME = home; +} + +const pwd = os.userInfo(); +is.object(pwd); +const pwdBuf = os.userInfo({ encoding: 'buffer' }); + +if (common.isWindows) { + assert.strictEqual(pwd.uid, -1); + assert.strictEqual(pwd.gid, -1); + assert.strictEqual(pwd.shell, null); + assert.strictEqual(pwdBuf.uid, -1); + assert.strictEqual(pwdBuf.gid, -1); + assert.strictEqual(pwdBuf.shell, null); +} else { + is.number(pwd.uid); + is.number(pwd.gid); + assert.strictEqual(typeof pwd.shell, 'string'); + // It's possible for /etc/passwd to leave the user's shell blank. + if (pwd.shell.length > 0) { + assert(pwd.shell.includes(path.sep)); + } + assert.strictEqual(pwd.uid, pwdBuf.uid); + assert.strictEqual(pwd.gid, pwdBuf.gid); + assert.strictEqual(pwd.shell, pwdBuf.shell.toString('utf8')); +} + +is.string(pwd.username); +assert.ok(pwd.homedir.includes(path.sep)); +assert.strictEqual(pwd.username, pwdBuf.username.toString('utf8')); +assert.strictEqual(pwd.homedir, pwdBuf.homedir.toString('utf8')); + +assert.strictEqual(`${os.hostname}`, os.hostname()); +assert.strictEqual(`${os.homedir}`, os.homedir()); +assert.strictEqual(`${os.release}`, os.release()); +assert.strictEqual(`${os.type}`, os.type()); +assert.strictEqual(`${os.endianness}`, os.endianness()); +assert.strictEqual(`${os.tmpdir}`, os.tmpdir()); +assert.strictEqual(`${os.arch}`, os.arch()); +assert.strictEqual(`${os.platform}`, os.platform()); +assert.strictEqual(`${os.version}`, os.version()); +assert.strictEqual(`${os.machine}`, os.machine()); +assert.strictEqual(+os.totalmem, os.totalmem()); + +// Assert that the following values are coercible to numbers. +// On IBMi, os.uptime() returns 'undefined' +if (!common.isIBMi) { + is.number(+os.uptime, 'uptime'); + is.number(os.uptime(), 'uptime'); +} + +is.number(+os.availableParallelism, 'availableParallelism'); +is.number(os.availableParallelism(), 'availableParallelism'); +is.number(+os.freemem, 'freemem'); +is.number(os.freemem(), 'freemem'); + +const devNull = os.devNull; +if (common.isWindows) { + assert.strictEqual(devNull, '\\\\.\\nul'); +} else { + assert.strictEqual(devNull, '/dev/null'); +} + +assert.ok(os.availableParallelism() > 0); From 04ae1a551726dd6e0047a942b459d18e1dcb1935 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Thu, 31 Oct 2024 22:19:19 -0700 Subject: [PATCH 60/63] fix(cli): set `npm_config_user_agent` when running npm packages or tasks (#26639) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #25342. Still not sure on the exact user agent to set (should it include `node`?). After this PR, here's the state of running some `create-*` packages (just ones I could think of off the top of my head): | package | prints/runs/suggests deno install | notes | | ---------------- | ------------- | ------ | | `create-next-app` | ❌ | falls back to npm, needs a PR ([code](https://github.com/vercel/next.js/blob/c32e2802097c03fd9f95b1dae228d6e0257569c0/packages/create-next-app/helpers/get-pkg-manager.ts#L3)) | `sv create` | ❌ | uses `package-manager-detector`, needs a PR ([code](https://github.com/antfu-collective/package-manager-detector/tree/main)) | `create-qwik` | ✅ | runs `deno install` but suggests `deno start` which doesn't work (should be `deno task start` or `deno run start`) | `create-astro` | ✅ | runs `deno install` but suggests `npm run dev` later in output, probably needs a PR | `nuxi init` | ❌ | deno not an option in dialog, needs a PR ([code](https://github.com/nuxt/cli/blob/f04e2e894446f597da9d971b7eb03191d5a0da7e/src/commands/init.ts#L96-L102)) | `create-react-app` | ❌ | uses npm | `ng new` (`@angular/cli`) | ❌ | uses npm | `create-vite` | ✅ | suggests working deno commands 🎉 | `create-solid` | ❌ | suggests npm commands, needs PR It's possible that fixing `package-manager-detector` or other packages might make some of these just work, but haven't looked too carefully at each --- cli/npm/mod.rs | 12 +++++ cli/task_runner.rs | 10 +++- cli/tools/run/mod.rs | 18 ++++++++ .../print-npm-user-agent/1.0.0/index.js | 2 + .../print-npm-user-agent/1.0.0/package.json | 10 ++++ .../npm/user_agent_env_var/__test__.jsonc | 46 +++++++++++++++++++ .../npm/user_agent_env_var/bin_command.out | 2 + tests/specs/npm/user_agent_env_var/deno.jsonc | 3 ++ tests/specs/npm/user_agent_env_var/main.ts | 1 + .../specs/npm/user_agent_env_var/package.json | 8 ++++ .../npm/user_agent_env_var/postinstall.out | 10 ++++ tests/specs/npm/user_agent_env_var/run.out | 1 + tests/specs/npm/user_agent_env_var/test.mjs | 1 + 13 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 tests/registry/npm/@denotest/print-npm-user-agent/1.0.0/index.js create mode 100644 tests/registry/npm/@denotest/print-npm-user-agent/1.0.0/package.json create mode 100644 tests/specs/npm/user_agent_env_var/__test__.jsonc create mode 100644 tests/specs/npm/user_agent_env_var/bin_command.out create mode 100644 tests/specs/npm/user_agent_env_var/deno.jsonc create mode 100644 tests/specs/npm/user_agent_env_var/main.ts create mode 100644 tests/specs/npm/user_agent_env_var/package.json create mode 100644 tests/specs/npm/user_agent_env_var/postinstall.out create mode 100644 tests/specs/npm/user_agent_env_var/run.out create mode 100644 tests/specs/npm/user_agent_env_var/test.mjs diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index f48f7a7405..3971b1a0b3 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -189,3 +189,15 @@ impl NpmFetchResolver { info } } + +pub const NPM_CONFIG_USER_AGENT_ENV_VAR: &str = "npm_config_user_agent"; + +pub fn get_npm_config_user_agent() -> String { + format!( + "deno/{} npm/? deno/{} {} {}", + env!("CARGO_PKG_VERSION"), + env!("CARGO_PKG_VERSION"), + std::env::consts::OS, + std::env::consts::ARCH + ) +} diff --git a/cli/task_runner.rs b/cli/task_runner.rs index 418043b23f..43840e868d 100644 --- a/cli/task_runner.rs +++ b/cli/task_runner.rs @@ -155,6 +155,12 @@ fn prepare_env_vars( initial_cwd.to_string_lossy().to_string(), ); } + if !env_vars.contains_key(crate::npm::NPM_CONFIG_USER_AGENT_ENV_VAR) { + env_vars.insert( + crate::npm::NPM_CONFIG_USER_AGENT_ENV_VAR.into(), + crate::npm::get_npm_config_user_agent(), + ); + } if let Some(node_modules_dir) = node_modules_dir { prepend_to_path( &mut env_vars, @@ -204,7 +210,7 @@ impl ShellCommand for NpmCommand { mut context: ShellCommandContext, ) -> LocalBoxFuture<'static, ExecuteResult> { if context.args.first().map(|s| s.as_str()) == Some("run") - && context.args.len() > 2 + && context.args.len() >= 2 // for now, don't run any npm scripts that have a flag because // we don't handle stuff like `--workspaces` properly && !context.args.iter().any(|s| s.starts_with('-')) @@ -267,10 +273,12 @@ impl ShellCommand for NodeCommand { ) .execute(context); } + args.extend(["run", "-A"].into_iter().map(|s| s.to_string())); args.extend(context.args.iter().cloned()); let mut state = context.state; + state.apply_env_var(USE_PKG_JSON_HIDDEN_ENV_VAR_NAME, "1"); ExecutableCommand::new("deno".to_string(), std::env::current_exe().unwrap()) .execute(ShellCommandContext { diff --git a/cli/tools/run/mod.rs b/cli/tools/run/mod.rs index 152e2650bd..bebb3f5881 100644 --- a/cli/tools/run/mod.rs +++ b/cli/tools/run/mod.rs @@ -30,6 +30,16 @@ To grant permissions, set them before the script argument. For example: } } +fn set_npm_user_agent() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + std::env::set_var( + crate::npm::NPM_CONFIG_USER_AGENT_ENV_VAR, + crate::npm::get_npm_config_user_agent(), + ); + }); +} + pub async fn run_script( mode: WorkerExecutionMode, flags: Arc, @@ -58,6 +68,10 @@ pub async fn run_script( let main_module = cli_options.resolve_main_module()?; + if main_module.scheme() == "npm" { + set_npm_user_agent(); + } + maybe_npm_install(&factory).await?; let worker_factory = factory.create_cli_main_worker_factory().await?; @@ -119,6 +133,10 @@ async fn run_with_watch( let cli_options = factory.cli_options()?; let main_module = cli_options.resolve_main_module()?; + if main_module.scheme() == "npm" { + set_npm_user_agent(); + } + maybe_npm_install(&factory).await?; let _ = watcher_communicator.watch_paths(cli_options.watch_paths()); diff --git a/tests/registry/npm/@denotest/print-npm-user-agent/1.0.0/index.js b/tests/registry/npm/@denotest/print-npm-user-agent/1.0.0/index.js new file mode 100644 index 0000000000..b835b8e25e --- /dev/null +++ b/tests/registry/npm/@denotest/print-npm-user-agent/1.0.0/index.js @@ -0,0 +1,2 @@ +#!/usr/bin/env node +console.log(`npm_config_user_agent: ${process.env["npm_config_user_agent"]}`); \ No newline at end of file diff --git a/tests/registry/npm/@denotest/print-npm-user-agent/1.0.0/package.json b/tests/registry/npm/@denotest/print-npm-user-agent/1.0.0/package.json new file mode 100644 index 0000000000..9071173467 --- /dev/null +++ b/tests/registry/npm/@denotest/print-npm-user-agent/1.0.0/package.json @@ -0,0 +1,10 @@ +{ + "name": "@denotest/print-npm-user-agent", + "version": "1.0.0", + "bin": { + "print-npm-user-agent": "index.js" + }, + "scripts": { + "postinstall": "echo postinstall && node index.js && exit 1" + } +} \ No newline at end of file diff --git a/tests/specs/npm/user_agent_env_var/__test__.jsonc b/tests/specs/npm/user_agent_env_var/__test__.jsonc new file mode 100644 index 0000000000..a2af970a0e --- /dev/null +++ b/tests/specs/npm/user_agent_env_var/__test__.jsonc @@ -0,0 +1,46 @@ +{ + "tempDir": true, + "tests": { + "set_for_npm_package": { + "steps": [ + { + "args": "install", + "output": "[WILDCARD]" + }, + { + "args": "run -A npm:@denotest/print-npm-user-agent", + "output": "run.out" + } + ] + }, + "unset_for_local_file": { + "steps": [ + { + "args": "run -A main.ts", + "output": "Download [WILDCARD]\nnpm_config_user_agent: undefined\n" + } + ] + }, + "set_for_tasks": { + "steps": [ + { + "args": "install", + "output": "[WILDCARD]" + }, + { + "args": "task run-via-bin", + "output": "bin_command.out" + } + ] + }, + "set_for_lifecycle_scripts": { + "steps": [ + { + "args": "install --allow-scripts", + "output": "postinstall.out", + "exitCode": 1 + } + ] + } + } +} diff --git a/tests/specs/npm/user_agent_env_var/bin_command.out b/tests/specs/npm/user_agent_env_var/bin_command.out new file mode 100644 index 0000000000..c8e9413038 --- /dev/null +++ b/tests/specs/npm/user_agent_env_var/bin_command.out @@ -0,0 +1,2 @@ +Task run-via-bin print-npm-user-agent +npm_config_user_agent: deno/[WILDCARD] npm/? deno/[WILDCARD] [WILDCARD] [WILDCARD] diff --git a/tests/specs/npm/user_agent_env_var/deno.jsonc b/tests/specs/npm/user_agent_env_var/deno.jsonc new file mode 100644 index 0000000000..fbd70ec480 --- /dev/null +++ b/tests/specs/npm/user_agent_env_var/deno.jsonc @@ -0,0 +1,3 @@ +{ + "nodeModulesDir": "auto" +} diff --git a/tests/specs/npm/user_agent_env_var/main.ts b/tests/specs/npm/user_agent_env_var/main.ts new file mode 100644 index 0000000000..2c51107e3c --- /dev/null +++ b/tests/specs/npm/user_agent_env_var/main.ts @@ -0,0 +1 @@ +console.log(`npm_config_user_agent: ${Deno.env.get("npm_config_user_agent")}`); diff --git a/tests/specs/npm/user_agent_env_var/package.json b/tests/specs/npm/user_agent_env_var/package.json new file mode 100644 index 0000000000..12cc0cc6fb --- /dev/null +++ b/tests/specs/npm/user_agent_env_var/package.json @@ -0,0 +1,8 @@ +{ + "scripts": { + "run-via-bin": "print-npm-user-agent" + }, + "dependencies": { + "@denotest/print-npm-user-agent": "1.0.0" + } +} diff --git a/tests/specs/npm/user_agent_env_var/postinstall.out b/tests/specs/npm/user_agent_env_var/postinstall.out new file mode 100644 index 0000000000..19a08598fb --- /dev/null +++ b/tests/specs/npm/user_agent_env_var/postinstall.out @@ -0,0 +1,10 @@ +Download http://localhost:4260/@denotest%2fprint-npm-user-agent +Download http://localhost:4260/@denotest/print-npm-user-agent/1.0.0.tgz +Initialize @denotest/print-npm-user-agent@1.0.0 +Initialize @denotest/print-npm-user-agent@1.0.0: running 'postinstall' script +error: script 'postinstall' in '@denotest/print-npm-user-agent@1.0.0' failed with exit code 1 +stdout: +postinstall +npm_config_user_agent: deno/[WILDCARD] npm/? deno/[WILDCARD] [WILDCARD] [WILDCARD] + +error: failed to run scripts for packages: @denotest/print-npm-user-agent@1.0.0 diff --git a/tests/specs/npm/user_agent_env_var/run.out b/tests/specs/npm/user_agent_env_var/run.out new file mode 100644 index 0000000000..a630ac412f --- /dev/null +++ b/tests/specs/npm/user_agent_env_var/run.out @@ -0,0 +1 @@ +npm_config_user_agent: deno/[WILDCARD] npm/? deno/[WILDCARD] [WILDCARD] [WILDCARD] diff --git a/tests/specs/npm/user_agent_env_var/test.mjs b/tests/specs/npm/user_agent_env_var/test.mjs new file mode 100644 index 0000000000..ae035d40ef --- /dev/null +++ b/tests/specs/npm/user_agent_env_var/test.mjs @@ -0,0 +1 @@ +console.log(process.env.npm_config_user_agent); From 4774eab64d5176e997b6431f03f075782321b3d9 Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Fri, 1 Nov 2024 16:13:02 +0530 Subject: [PATCH 61/63] chore: upgrade to rust 1.82 and LLVM 19 (#26615) Upgrade to rust 1.82 and LLVM 19 . Removes one webusb test because `requestAdapter` not working on new ubuntu 24 runners --- .github/workflows/ci.generate.ts | 10 +++--- .github/workflows/ci.yml | 36 +++++++++++----------- cli/tools/info.rs | 4 +-- ext/node/ops/crypto/sign.rs | 30 ++++++++---------- runtime/permissions/prompter.rs | 2 +- rust-toolchain.toml | 2 +- tests/napi/src/async.rs | 2 +- tests/specs/future/unstable_flags/main.js | 1 - tests/specs/future/unstable_flags/main.out | 1 - 9 files changed, 40 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index d5ee67b654..49d11107d5 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -7,8 +7,8 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify"; // automatically via regex, so ensure that this line maintains this format. const cacheVersion = 23; -const ubuntuX86Runner = "ubuntu-22.04"; -const ubuntuX86XlRunner = "ubuntu-22.04-xl"; +const ubuntuX86Runner = "ubuntu-24.04"; +const ubuntuX86XlRunner = "ubuntu-24.04-xl"; const ubuntuARMRunner = "ubicloud-standard-16-arm"; const windowsX86Runner = "windows-2022"; const windowsX86XlRunner = "windows-2022-xl"; @@ -59,7 +59,7 @@ const prCacheKeyPrefix = `${cacheVersion}-cargo-target-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ matrix.profile }}-\${{ matrix.job }}-`; // Note that you may need to add more version to the `apt-get remove` line below if you change this -const llvmVersion = 18; +const llvmVersion = 19; const installPkgsCommand = `sudo apt-get install --no-install-recommends clang-${llvmVersion} lld-${llvmVersion} clang-tools-${llvmVersion} clang-format-${llvmVersion} clang-tidy-${llvmVersion}`; const sysRootStep = { @@ -71,7 +71,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get -qq remove --purge -y man-db > /dev/null 2> /dev/null # Remove older clang before we install sudo apt-get -qq remove \ - 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' > /dev/null 2> /dev/null + 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'clang-17*' 'clang-18*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' 'lld-17*' 'lld-18*' > /dev/null 2> /dev/null # Install clang-XXX, lld-XXX, and debootstrap. echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${llvmVersion} main" | @@ -86,7 +86,7 @@ ${installPkgsCommand} || echo 'Failed. Trying again.' && sudo apt-get clean && s (yes '' | sudo update-alternatives --force --all) > /dev/null 2> /dev/null || true echo "Decompressing sysroot..." -wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20240528/sysroot-\`uname -m\`.tar.xz -O /tmp/sysroot.tar.xz +wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20241030/sysroot-\`uname -m\`.tar.xz -O /tmp/sysroot.tar.xz cd / xzcat /tmp/sysroot.tar.xz | sudo tar -x sudo mount --rbind /dev /sysroot/dev diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6421d9244..a9c06de17a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: profile: debug - os: macos arch: x86_64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || ''macos-13'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || ''macos-13'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' @@ -73,7 +73,7 @@ jobs: profile: debug - os: macos arch: aarch64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || ''macos-14'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || ''macos-14'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' @@ -84,33 +84,33 @@ jobs: profile: debug - os: windows arch: x86_64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || github.repository == ''denoland/deno'' && ''windows-2022-xl'' || ''windows-2022'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && ''windows-2022-xl'' || ''windows-2022'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' - os: linux arch: x86_64 - runner: '${{ github.repository == ''denoland/deno'' && ''ubuntu-22.04-xl'' || ''ubuntu-22.04'' }}' + runner: '${{ github.repository == ''denoland/deno'' && ''ubuntu-24.04-xl'' || ''ubuntu-24.04'' }}' job: test profile: release use_sysroot: true wpt: '${{ !startsWith(github.ref, ''refs/tags/'') }}' - os: linux arch: x86_64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'' && !contains(github.event.pull_request.labels.*.name, ''ci-bench''))) && ''ubuntu-22.04'' || github.repository == ''denoland/deno'' && ''ubuntu-22.04-xl'' || ''ubuntu-22.04'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'' && !contains(github.event.pull_request.labels.*.name, ''ci-bench''))) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && ''ubuntu-24.04-xl'' || ''ubuntu-24.04'' }}' job: bench profile: release use_sysroot: true skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'' && !contains(github.event.pull_request.labels.*.name, ''ci-bench'')) }}' - os: linux arch: x86_64 - runner: ubuntu-22.04 + runner: ubuntu-24.04 job: test profile: debug use_sysroot: true - os: linux arch: x86_64 - runner: ubuntu-22.04 + runner: ubuntu-24.04 job: lint profile: debug - os: linux @@ -252,22 +252,22 @@ jobs: # to complete. sudo apt-get -qq remove --purge -y man-db > /dev/null 2> /dev/null # Remove older clang before we install - sudo apt-get -qq remove 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' > /dev/null 2> /dev/null + sudo apt-get -qq remove 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'clang-17*' 'clang-18*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' 'lld-17*' 'lld-18*' > /dev/null 2> /dev/null # Install clang-XXX, lld-XXX, and debootstrap. - echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" | - sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-jammy-18.list + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main" | + sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-jammy-19.list curl https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor | sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg sudo apt-get update # this was unreliable sometimes, so try again if it fails - sudo apt-get install --no-install-recommends clang-18 lld-18 clang-tools-18 clang-format-18 clang-tidy-18 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends clang-18 lld-18 clang-tools-18 clang-format-18 clang-tidy-18 + sudo apt-get install --no-install-recommends clang-19 lld-19 clang-tools-19 clang-format-19 clang-tidy-19 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends clang-19 lld-19 clang-tools-19 clang-format-19 clang-tidy-19 # Fix alternatives (yes '' | sudo update-alternatives --force --all) > /dev/null 2> /dev/null || true echo "Decompressing sysroot..." - wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20240528/sysroot-`uname -m`.tar.xz -O /tmp/sysroot.tar.xz + wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20241030/sysroot-`uname -m`.tar.xz -O /tmp/sysroot.tar.xz cd / xzcat /tmp/sysroot.tar.xz | sudo tar -x sudo mount --rbind /dev /sysroot/dev @@ -299,8 +299,8 @@ jobs: CARGO_PROFILE_RELEASE_LTO=false RUSTFLAGS<<__1 -C linker-plugin-lto=true - -C linker=clang-18 - -C link-arg=-fuse-ld=lld-18 + -C linker=clang-19 + -C link-arg=-fuse-ld=lld-19 -C link-arg=-ldl -C link-arg=-Wl,--allow-shlib-undefined -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache @@ -310,8 +310,8 @@ jobs: __1 RUSTDOCFLAGS<<__1 -C linker-plugin-lto=true - -C linker=clang-18 - -C link-arg=-fuse-ld=lld-18 + -C linker=clang-19 + -C link-arg=-fuse-ld=lld-19 -C link-arg=-ldl -C link-arg=-Wl,--allow-shlib-undefined -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache @@ -319,7 +319,7 @@ jobs: --cfg tokio_unstable $RUSTFLAGS __1 - CC=/usr/bin/clang-18 + CC=/usr/bin/clang-19 CFLAGS=-flto=thin $CFLAGS " > $GITHUB_ENV - name: Remove macOS cURL --ipv4 flag @@ -688,7 +688,7 @@ jobs: key: '23-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: name: publish canary - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - build if: github.repository == 'denoland/deno' && github.ref == 'refs/heads/main' diff --git a/cli/tools/info.rs b/cli/tools/info.rs index 3febaff579..b53485bd6b 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -530,7 +530,7 @@ impl<'a> GraphDisplayContext<'a> { fn build_module_info(&mut self, module: &Module, type_dep: bool) -> TreeNode { enum PackageOrSpecifier { - Package(NpmResolutionPackage), + Package(Box), Specifier(ModuleSpecifier), } @@ -538,7 +538,7 @@ impl<'a> GraphDisplayContext<'a> { let package_or_specifier = match module.npm() { Some(npm) => match self.npm_info.resolve_package(npm.nv_reference.nv()) { - Some(package) => Package(package.clone()), + Some(package) => Package(Box::new(package.clone())), None => Specifier(module.specifier().clone()), // should never happen }, None => Specifier(module.specifier().clone()), diff --git a/ext/node/ops/crypto/sign.rs b/ext/node/ops/crypto/sign.rs index b7779a5d80..31744f86ba 100644 --- a/ext/node/ops/crypto/sign.rs +++ b/ext/node/ops/crypto/sign.rs @@ -79,18 +79,15 @@ impl KeyObjectHandle { AsymmetricPrivateKey::RsaPss(key) => { let mut hash_algorithm = None; let mut salt_length = None; - match &key.details { - Some(details) => { - if details.hash_algorithm != details.mf1_hash_algorithm { - return Err(type_error( + if let Some(details) = &key.details { + if details.hash_algorithm != details.mf1_hash_algorithm { + return Err(type_error( "rsa-pss with different mf1 hash algorithm and hash algorithm is not supported", )); - } - hash_algorithm = Some(details.hash_algorithm); - salt_length = Some(details.salt_length as usize); } - None => {} - }; + hash_algorithm = Some(details.hash_algorithm); + salt_length = Some(details.salt_length as usize); + } if let Some(s) = pss_salt_length { salt_length = Some(s as usize); } @@ -215,18 +212,15 @@ impl KeyObjectHandle { AsymmetricPublicKey::RsaPss(key) => { let mut hash_algorithm = None; let mut salt_length = None; - match &key.details { - Some(details) => { - if details.hash_algorithm != details.mf1_hash_algorithm { - return Err(type_error( + if let Some(details) = &key.details { + if details.hash_algorithm != details.mf1_hash_algorithm { + return Err(type_error( "rsa-pss with different mf1 hash algorithm and hash algorithm is not supported", )); - } - hash_algorithm = Some(details.hash_algorithm); - salt_length = Some(details.salt_length as usize); } - None => {} - }; + hash_algorithm = Some(details.hash_algorithm); + salt_length = Some(details.salt_length as usize); + } if let Some(s) = pss_salt_length { salt_length = Some(s as usize); } diff --git a/runtime/permissions/prompter.rs b/runtime/permissions/prompter.rs index 316911edc1..b582b4f53e 100644 --- a/runtime/permissions/prompter.rs +++ b/runtime/permissions/prompter.rs @@ -269,7 +269,7 @@ fn get_stdin_metadata() -> std::io::Result { unsafe { let stdin = std::fs::File::from_raw_fd(0); let metadata = stdin.metadata().unwrap(); - stdin.into_raw_fd(); + let _ = stdin.into_raw_fd(); Ok(metadata) } } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f19c7df470..3d572e0d69 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.81.0" +channel = "1.82.0" components = ["rustfmt", "clippy"] diff --git a/tests/napi/src/async.rs b/tests/napi/src/async.rs index 3d3827b51c..367d2e9ef0 100644 --- a/tests/napi/src/async.rs +++ b/tests/napi/src/async.rs @@ -95,7 +95,7 @@ extern "C" fn test_async_work( )); let mut baton = unsafe { Box::from_raw(baton_ptr as *mut Baton) }; baton.task = async_work; - Box::into_raw(baton); + let _ = Box::into_raw(baton); assert_napi_ok!(napi_queue_async_work(env, async_work)); ptr::null_mut() diff --git a/tests/specs/future/unstable_flags/main.js b/tests/specs/future/unstable_flags/main.js index 51af892f6b..8cc8960d97 100644 --- a/tests/specs/future/unstable_flags/main.js +++ b/tests/specs/future/unstable_flags/main.js @@ -1,4 +1,3 @@ -console.log(typeof await navigator.gpu.requestAdapter() === "object"); // Throws without `--unstable-gpu` console.log(typeof Deno.dlopen === "function"); // Undefined without `--unstable-ffi` console.log( // Undefined without `--unstable-fs` diff --git a/tests/specs/future/unstable_flags/main.out b/tests/specs/future/unstable_flags/main.out index b979d62f4f..bb101b641b 100644 --- a/tests/specs/future/unstable_flags/main.out +++ b/tests/specs/future/unstable_flags/main.out @@ -1,3 +1,2 @@ true true -true From 826e42a5b5880c974ae019a7a21aade6a718062c Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 1 Nov 2024 12:27:00 -0400 Subject: [PATCH 62/63] fix: improved support for cjs and cts modules (#26558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cts support * better cjs/cts type checking * deno compile cjs/cts support * More efficient detect cjs (going towards stabilization) * Determination of whether .js, .ts, .jsx, or .tsx is cjs or esm is only done after loading * Support `import x = require(...);` Co-authored-by: Bartek Iwańczuk --- Cargo.lock | 28 +- Cargo.toml | 6 +- cli/Cargo.toml | 8 +- cli/args/mod.rs | 11 + cli/cache/cache_db.rs | 6 +- cli/cache/emit.rs | 2 + cli/cache/mod.rs | 101 ++--- cli/cache/module_info.rs | 140 +++++-- cli/cache/parsed_source.rs | 60 +-- cli/emit.rs | 77 ++-- cli/factory.rs | 274 ++++++++----- cli/graph_util.rs | 80 ++-- cli/lsp/analysis.rs | 21 +- cli/lsp/code_lens.rs | 4 +- cli/lsp/config.rs | 2 +- cli/lsp/documents.rs | 20 +- cli/lsp/language_server.rs | 2 +- cli/lsp/resolver.rs | 230 ++++++++--- cli/lsp/testing/collectors.rs | 2 +- cli/lsp/tsc.rs | 27 +- cli/main.rs | 2 +- cli/module_loader.rs | 310 ++++++++++++--- cli/node.rs | 134 +++---- cli/npm/byonm.rs | 43 +- cli/npm/managed/cache/mod.rs | 12 +- cli/npm/managed/mod.rs | 91 +++-- cli/npm/managed/resolvers/common.rs | 4 - cli/npm/managed/resolvers/global.rs | 7 +- cli/npm/managed/resolvers/local.rs | 4 - cli/npm/mod.rs | 34 +- cli/resolver.rs | 354 +++++++++++------ cli/standalone/binary.rs | 38 +- cli/standalone/mod.rs | 370 ++++++++++++------ cli/standalone/serialization.rs | 49 ++- cli/tools/bench/mod.rs | 2 +- cli/tools/check.rs | 11 +- cli/tools/compile.rs | 10 - cli/tools/coverage/mod.rs | 23 +- cli/tools/doc.rs | 6 +- cli/tools/registry/tar.rs | 2 +- cli/tools/repl/session.rs | 6 +- cli/tools/run/hmr.rs | 25 +- cli/tools/test/mod.rs | 2 +- cli/tsc/99_main_compiler.js | 2 + cli/tsc/diagnostics.rs | 2 +- cli/tsc/mod.rs | 153 ++++++-- cli/util/extract.rs | 8 +- cli/util/logger.rs | 2 + cli/util/path.rs | 27 -- cli/util/text_encoding.rs | 1 + cli/worker.rs | 171 +++----- ext/node/lib.rs | 47 +-- ext/node/ops/require.rs | 62 +-- ext/node/ops/worker_threads.rs | 40 +- ext/node/polyfills/01_require.js | 31 +- resolvers/deno/fs.rs | 10 - resolvers/deno/npm/byonm.rs | 41 +- resolvers/deno/npm/mod.rs | 1 + resolvers/deno/sloppy_imports.rs | 2 +- resolvers/node/analyze.rs | 50 ++- resolvers/node/errors.rs | 58 --- resolvers/node/lib.rs | 5 +- resolvers/node/npm.rs | 6 + resolvers/node/package_json.rs | 106 ++++- resolvers/node/resolution.rs | 196 ++-------- runtime/errors.rs | 14 +- runtime/shared.rs | 9 +- tests/integration/run_tests.rs | 3 +- .../@denotest/type-commonjs/1.0.0/index.js | 6 +- .../css_import/exists_run_with_check.out | 2 +- tests/specs/compile/cjs/__test__.jsonc | 24 ++ tests/specs/compile/cjs/add.cjs | 1 + tests/specs/compile/cjs/divide.cts | 1 + tests/specs/compile/cjs/main.js | 5 + tests/specs/compile/cjs/multiply.cts | 4 + tests/specs/compile/cjs/output.out | 2 + tests/specs/compile/cjs/reexport.cjs | 1 + tests/specs/compile/detect_cjs/__test__.jsonc | 11 +- tests/specs/compile/detect_cjs/add.js | 2 +- tests/specs/compile/detect_cjs/compile.out | 1 - tests/specs/compile/detect_cjs/output.out | 3 +- tests/specs/compile/detect_cjs/package.json | 5 +- tests/specs/compile/detect_cjs/subtract.ts | 2 + .../npm/require_type_commonjs/__test__.jsonc | 3 +- .../specs/npm/require_type_commonjs/main.out | 5 +- tests/specs/npm/require_type_commonjs/main.ts | 3 +- .../typescript_file_in_package/__test__.jsonc | 4 +- .../npm/typescript_file_in_package/main.out | 3 + .../{typescript_file_in_package => }/main.ts | 0 .../typescript_file_in_package/main.out | 6 - .../specs/run/cjs/main_module/__test__.jsonc | 4 + tests/specs/run/cjs/main_module/main.cjs | 1 + tests/specs/run/cjs/main_module/main.out | 5 + .../reexport_non_analyzable}/__test__.jsonc | 0 .../reexport_non_analyzable}/deno.json | 0 .../reexport_non_analyzable}/main.ts | 0 .../node_modules/foo.cjs | 0 tests/specs/run/cjs/unprepared/__test__.jsonc | 4 + tests/specs/run/cjs/unprepared/file.cjs | 7 + tests/specs/run/cjs/unprepared/main.out | 1 + tests/specs/run/cjs/unprepared/main.ts | 7 + tests/specs/run/cjs/unprepared/output.cjs | 3 + .../run/cts/cjs_import_cts/__test__.jsonc | 13 + .../add.js => cts/cjs_import_cts/add.cts} | 0 tests/specs/run/cts/cjs_import_cts/check.out | 17 + tests/specs/run/cts/cjs_import_cts/main.js | 3 + tests/specs/run/cts/cjs_import_cts/main.out | 1 + .../specs/run/cts/cjs_import_cts/subtract.cjs | 3 + .../cts/import_export_equals/__test__.jsonc | 17 + .../run/cts/import_export_equals/add.cts | 3 + .../run/cts/import_export_equals/main.cts | 3 + .../run/cts/import_export_equals/main.out | 2 + .../run/cts/import_export_equals/mod.mts | 3 + .../import_export_equals/mod.mts.check.out | 5 + .../run/cts/import_export_equals/mod.mts.out | 1 + tests/specs/run/cts/main/__test__.jsonc | 4 + tests/specs/run/cts/main/import_main.cjs | 1 + tests/specs/run/cts/main/main.cts | 5 + tests/specs/run/cts/main/main.out | 1 + .../commonjs/{ => basic}/__test__.jsonc | 0 .../package_json_type/commonjs/basic/add.js | 3 + .../commonjs/{ => basic}/deno.jsonc | 0 .../{ => basic}/import_import_meta.js | 0 .../commonjs/{ => basic}/import_meta.js | 0 .../commonjs/{ => basic}/main_cjs.js | 0 .../commonjs/{ => basic}/main_esm.js | 0 .../{ => basic}/main_esm_import_meta.js | 0 .../{ => basic}/main_esm_import_meta.out | 0 .../commonjs/{ => basic}/main_mix.js | 0 .../commonjs/{ => basic}/main_mix.out | 0 .../commonjs/{ => basic}/not_import_meta.js | 0 .../commonjs/{ => basic}/package.json | 0 .../commonjs/{ => basic}/tla.js | 0 .../commonjs/jsx/__test__.jsonc | 5 + .../run/package_json_type/commonjs/jsx/add.js | 3 + .../package_json_type/commonjs/jsx/deno.jsonc | 10 + .../package_json_type/commonjs/jsx/main.jsx | 7 + .../package_json_type/commonjs/jsx/main.out | 4 + .../commonjs/jsx/package.json | 7 + .../package_json_type/commonjs/jsx/tsx.tsx | 5 + tests/specs/run/remote_cjs_main/output.out | 2 +- tests/specs/run/require_esm/main.out | 2 +- .../npm/deno_run_cowsay_no_permissions.out | 2 +- 143 files changed, 2418 insertions(+), 1527 deletions(-) create mode 100644 tests/specs/compile/cjs/__test__.jsonc create mode 100644 tests/specs/compile/cjs/add.cjs create mode 100644 tests/specs/compile/cjs/divide.cts create mode 100644 tests/specs/compile/cjs/main.js create mode 100644 tests/specs/compile/cjs/multiply.cts create mode 100644 tests/specs/compile/cjs/output.out create mode 100644 tests/specs/compile/cjs/reexport.cjs create mode 100644 tests/specs/compile/detect_cjs/subtract.ts create mode 100644 tests/specs/npm/typescript_file_in_package/main.out rename tests/specs/npm/typescript_file_in_package/{typescript_file_in_package => }/main.ts (100%) delete mode 100644 tests/specs/npm/typescript_file_in_package/typescript_file_in_package/main.out create mode 100644 tests/specs/run/cjs/main_module/__test__.jsonc create mode 100644 tests/specs/run/cjs/main_module/main.cjs create mode 100644 tests/specs/run/cjs/main_module/main.out rename tests/specs/run/{cjs_reexport_non_analyzable => cjs/reexport_non_analyzable}/__test__.jsonc (100%) rename tests/specs/run/{cjs_reexport_non_analyzable => cjs/reexport_non_analyzable}/deno.json (100%) rename tests/specs/run/{cjs_reexport_non_analyzable => cjs/reexport_non_analyzable}/main.ts (100%) rename tests/specs/run/{cjs_reexport_non_analyzable => cjs/reexport_non_analyzable}/node_modules/foo.cjs (100%) create mode 100644 tests/specs/run/cjs/unprepared/__test__.jsonc create mode 100644 tests/specs/run/cjs/unprepared/file.cjs create mode 100644 tests/specs/run/cjs/unprepared/main.out create mode 100644 tests/specs/run/cjs/unprepared/main.ts create mode 100644 tests/specs/run/cjs/unprepared/output.cjs create mode 100644 tests/specs/run/cts/cjs_import_cts/__test__.jsonc rename tests/specs/run/{package_json_type/commonjs/add.js => cts/cjs_import_cts/add.cts} (100%) create mode 100644 tests/specs/run/cts/cjs_import_cts/check.out create mode 100644 tests/specs/run/cts/cjs_import_cts/main.js create mode 100644 tests/specs/run/cts/cjs_import_cts/main.out create mode 100644 tests/specs/run/cts/cjs_import_cts/subtract.cjs create mode 100644 tests/specs/run/cts/import_export_equals/__test__.jsonc create mode 100644 tests/specs/run/cts/import_export_equals/add.cts create mode 100644 tests/specs/run/cts/import_export_equals/main.cts create mode 100644 tests/specs/run/cts/import_export_equals/main.out create mode 100644 tests/specs/run/cts/import_export_equals/mod.mts create mode 100644 tests/specs/run/cts/import_export_equals/mod.mts.check.out create mode 100644 tests/specs/run/cts/import_export_equals/mod.mts.out create mode 100644 tests/specs/run/cts/main/__test__.jsonc create mode 100644 tests/specs/run/cts/main/import_main.cjs create mode 100644 tests/specs/run/cts/main/main.cts create mode 100644 tests/specs/run/cts/main/main.out rename tests/specs/run/package_json_type/commonjs/{ => basic}/__test__.jsonc (100%) create mode 100644 tests/specs/run/package_json_type/commonjs/basic/add.js rename tests/specs/run/package_json_type/commonjs/{ => basic}/deno.jsonc (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/import_import_meta.js (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/import_meta.js (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/main_cjs.js (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/main_esm.js (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/main_esm_import_meta.js (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/main_esm_import_meta.out (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/main_mix.js (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/main_mix.out (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/not_import_meta.js (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/package.json (100%) rename tests/specs/run/package_json_type/commonjs/{ => basic}/tla.js (100%) create mode 100644 tests/specs/run/package_json_type/commonjs/jsx/__test__.jsonc create mode 100644 tests/specs/run/package_json_type/commonjs/jsx/add.js create mode 100644 tests/specs/run/package_json_type/commonjs/jsx/deno.jsonc create mode 100644 tests/specs/run/package_json_type/commonjs/jsx/main.jsx create mode 100644 tests/specs/run/package_json_type/commonjs/jsx/main.out create mode 100644 tests/specs/run/package_json_type/commonjs/jsx/package.json create mode 100644 tests/specs/run/package_json_type/commonjs/jsx/tsx.tsx diff --git a/Cargo.lock b/Cargo.lock index e0780b2268..8c7b35ca9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1279,9 +1279,9 @@ dependencies = [ [[package]] name = "deno_ast" -version = "0.42.2" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b9d03b1bbeeecdac54367f075d572131736d06c5be3bc49037855bc5ab1bbb" +checksum = "48d00b724e06d2081a141ec1155756a0b465d413d8e2a7515221f61d482eb2ee" dependencies = [ "base64 0.21.7", "deno_media_type", @@ -1356,9 +1356,9 @@ dependencies = [ [[package]] name = "deno_cache_dir" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693ca429aebf945de5fef30df232044f9f80be4cc5a5e7c8d767226c43880f5a" +checksum = "08c1f52170cd7715f8006da54cde1444863a0d6fbd9c11d037a737db2dec8e22" dependencies = [ "base32", "deno_media_type", @@ -1506,9 +1506,9 @@ dependencies = [ [[package]] name = "deno_doc" -version = "0.154.0" +version = "0.156.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e204e45b0d79750880114e37b34abe19ad0710d8435a8da8f23a528fe98de4" +checksum = "2585b98d6ad76dae30bf2d7b6d71b8363cae041158b8780d14a2f4fe17590a61" dependencies = [ "anyhow", "cfg-if", @@ -1606,9 +1606,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.83.4" +version = "0.84.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bd20bc0780071989c622cbfd5d4fb2e4fd05a247ccd7f791f13c8d2c3792228" +checksum = "cd4f4a14aa069087be41c2998077b0453f0191747898f96e6343f700abfc2c18" dependencies = [ "anyhow", "async-trait", @@ -1726,9 +1726,9 @@ dependencies = [ [[package]] name = "deno_lint" -version = "0.67.0" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871b60e32bfb6c110cbb9b0688dbf048f81e5d347fe4ce5a42239263de9dd938" +checksum = "bb994e6d1b18223df0a756c7948143b35682941d615edffef60d5b38822f38ac" dependencies = [ "anyhow", "deno_ast", @@ -1756,9 +1756,9 @@ dependencies = [ [[package]] name = "deno_media_type" -version = "0.1.4" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8978229b82552bf8457a0125aa20863f023619cfc21ebb007b1e571d68fd85b" +checksum = "7fcf552fbdedbe81c89705349d7d2485c7051382b000dfddbdbf7fc25931cf83" dependencies = [ "data-url", "serde", @@ -2608,9 +2608,9 @@ dependencies = [ [[package]] name = "dprint-plugin-typescript" -version = "0.93.0" +version = "0.93.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9308d98b923b7c0335c2ee1560199e3f2321b1be82803107b4ba4ed5dac46cc" +checksum = "5abfd78fe3cde4f5a6699d65f760c8d44da130cf446b6f80a7a9bc6580e156ab" dependencies = [ "anyhow", "deno_ast", diff --git a/Cargo.toml b/Cargo.toml index 285c077809..f57563e0b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,12 +45,12 @@ license = "MIT" repository = "https://github.com/denoland/deno" [workspace.dependencies] -deno_ast = { version = "=0.42.2", features = ["transpiling"] } +deno_ast = { version = "=0.43.3", features = ["transpiling"] } deno_core = { version = "0.316.0" } deno_bench_util = { version = "0.169.0", path = "./bench_util" } deno_lockfile = "=0.23.1" -deno_media_type = { version = "0.1.4", features = ["module_specifier"] } +deno_media_type = { version = "0.2.0", features = ["module_specifier"] } deno_npm = "=0.25.4" deno_path_util = "=0.2.1" deno_permissions = { version = "0.35.0", path = "./runtime/permissions" } @@ -111,7 +111,7 @@ console_static_text = "=0.8.1" dashmap = "5.5.3" data-encoding = "2.3.3" data-url = "=0.3.0" -deno_cache_dir = "=0.13.1" +deno_cache_dir = "=0.13.2" deno_package_json = { version = "0.1.2", default-features = false } dlopen2 = "0.6.1" ecb = "=0.1.2" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 782adfd903..a3e2b71f00 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -72,9 +72,9 @@ deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "proposa deno_cache_dir = { workspace = true } deno_config = { version = "=0.37.2", features = ["workspace", "sync"] } deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } -deno_doc = { version = "0.154.0", default-features = false, features = ["rust", "html", "syntect"] } -deno_graph = { version = "=0.83.4" } -deno_lint = { version = "=0.67.0", features = ["docs"] } +deno_doc = { version = "0.156.0", default-features = false, features = ["rust", "html", "syntect"] } +deno_graph = { version = "=0.84.1" } +deno_lint = { version = "=0.68.0", features = ["docs"] } deno_lockfile.workspace = true deno_npm.workspace = true deno_package_json.workspace = true @@ -107,7 +107,7 @@ dotenvy = "0.15.7" dprint-plugin-json = "=0.19.4" dprint-plugin-jupyter = "=0.1.5" dprint-plugin-markdown = "=0.17.8" -dprint-plugin-typescript = "=0.93.0" +dprint-plugin-typescript = "=0.93.1" env_logger = "=0.10.0" fancy-regex = "=0.10.0" faster-hex.workspace = true diff --git a/cli/args/mod.rs b/cli/args/mod.rs index d725264623..e19025f8b1 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -201,6 +201,8 @@ pub fn ts_config_to_transpile_and_emit_options( precompile_jsx_dynamic_props: None, transform_jsx, var_decl_imports: false, + // todo(dsherret): support verbatim_module_syntax here properly + verbatim_module_syntax: false, }, deno_ast::EmitOptions { inline_sources: options.inline_sources, @@ -1602,6 +1604,15 @@ impl CliOptions { } pub fn use_byonm(&self) -> bool { + if matches!( + self.sub_command(), + DenoSubcommand::Install(_) + | DenoSubcommand::Add(_) + | DenoSubcommand::Remove(_) + ) { + // For `deno install/add/remove` we want to force the managed resolver so it can set up `node_modules/` directory. + return false; + } if self.node_modules_dir().ok().flatten().is_none() && self.maybe_node_modules_folder.is_some() && self diff --git a/cli/cache/cache_db.rs b/cli/cache/cache_db.rs index b24078f29b..329ed2d970 100644 --- a/cli/cache/cache_db.rs +++ b/cli/cache/cache_db.rs @@ -57,7 +57,7 @@ impl rusqlite::types::FromSql for CacheDBHash { } /// What should the cache should do on failure? -#[derive(Default)] +#[derive(Debug, Default)] pub enum CacheFailure { /// Return errors if failure mode otherwise unspecified. #[default] @@ -69,6 +69,7 @@ pub enum CacheFailure { } /// Configuration SQL and other parameters for a [`CacheDB`]. +#[derive(Debug)] pub struct CacheDBConfiguration { /// SQL to run for a new database. pub table_initializer: &'static str, @@ -98,6 +99,7 @@ impl CacheDBConfiguration { } } +#[derive(Debug)] enum ConnectionState { Connected(Connection), Blackhole, @@ -106,7 +108,7 @@ enum ConnectionState { /// A cache database that eagerly initializes itself off-thread, preventing initialization operations /// from blocking the main thread. -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct CacheDB { // TODO(mmastrac): We can probably simplify our thread-safe implementation here conn: Arc>>, diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index 74e1c1101b..3c9eecfcbd 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -10,6 +10,7 @@ use deno_core::unsync::sync::AtomicFlag; use super::DiskCache; /// The cache that stores previously emitted files. +#[derive(Debug)] pub struct EmitCache { disk_cache: DiskCache, emit_failed_flag: AtomicFlag, @@ -91,6 +92,7 @@ impl EmitCache { const LAST_LINE_PREFIX: &str = "\n// denoCacheMetadata="; +#[derive(Debug)] struct EmitFileSerializer { cli_version: &'static str, } diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index bf8f1b1f0b..50fc135ddf 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -8,14 +8,9 @@ use crate::file_fetcher::FetchOptions; use crate::file_fetcher::FetchPermissionsOptionRef; use crate::file_fetcher::FileFetcher; use crate::file_fetcher::FileOrRedirect; -use crate::npm::CliNpmResolver; -use crate::resolver::CliNodeResolver; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::atomic_write_file_with_retries_and_fs; use crate::util::fs::AtomicWriteFileFsAdapter; -use crate::util::path::specifier_has_extension; -use crate::util::text_encoding::arc_str_to_bytes; -use crate::util::text_encoding::from_utf8_lossy_owned; use deno_ast::MediaType; use deno_core::futures; @@ -25,7 +20,9 @@ use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; use deno_graph::source::LoadResponse; use deno_graph::source::Loader; +use deno_runtime::deno_fs; use deno_runtime::deno_permissions::PermissionsContainer; +use node_resolver::InNpmPackageChecker; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; @@ -60,7 +57,6 @@ pub use fast_check::FastCheckCache; pub use incremental::IncrementalCache; pub use module_info::ModuleInfoCache; pub use node::NodeAnalysisCache; -pub use parsed_source::EsmOrCjsChecker; pub use parsed_source::LazyGraphSourceParser; pub use parsed_source::ParsedSourceCache; @@ -181,46 +177,40 @@ pub struct FetchCacherOptions { pub permissions: PermissionsContainer, /// If we're publishing for `deno publish`. pub is_deno_publish: bool, - pub unstable_detect_cjs: bool, } /// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides /// a concise interface to the DENO_DIR when building module graphs. pub struct FetchCacher { pub file_header_overrides: HashMap>, - esm_or_cjs_checker: Arc, file_fetcher: Arc, + fs: Arc, global_http_cache: Arc, - node_resolver: Arc, - npm_resolver: Arc, + in_npm_pkg_checker: Arc, module_info_cache: Arc, permissions: PermissionsContainer, is_deno_publish: bool, - unstable_detect_cjs: bool, cache_info_enabled: bool, } impl FetchCacher { pub fn new( - esm_or_cjs_checker: Arc, file_fetcher: Arc, + fs: Arc, global_http_cache: Arc, - node_resolver: Arc, - npm_resolver: Arc, + in_npm_pkg_checker: Arc, module_info_cache: Arc, options: FetchCacherOptions, ) -> Self { Self { file_fetcher, - esm_or_cjs_checker, + fs, global_http_cache, - node_resolver, - npm_resolver, + in_npm_pkg_checker, module_info_cache, file_header_overrides: options.file_header_overrides, permissions: options.permissions, is_deno_publish: options.is_deno_publish, - unstable_detect_cjs: options.unstable_detect_cjs, cache_info_enabled: false, } } @@ -271,70 +261,23 @@ impl Loader for FetchCacher { ) -> LoadFuture { use deno_graph::source::CacheSetting as LoaderCacheSetting; - if specifier.scheme() == "file" { - if specifier.path().contains("/node_modules/") { - // The specifier might be in a completely different symlinked tree than - // what the node_modules url is in (ex. `/my-project-1/node_modules` - // symlinked to `/my-project-2/node_modules`), so first we checked if the path - // is in a node_modules dir to avoid needlessly canonicalizing, then now compare - // against the canonicalized specifier. - let specifier = - crate::node::resolve_specifier_into_node_modules(specifier); - if self.npm_resolver.in_npm_package(&specifier) { - return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::External { specifier }, - )))); - } - } - - // make local CJS modules external to the graph - if specifier_has_extension(specifier, "cjs") { + if specifier.scheme() == "file" + && specifier.path().contains("/node_modules/") + { + // The specifier might be in a completely different symlinked tree than + // what the node_modules url is in (ex. `/my-project-1/node_modules` + // symlinked to `/my-project-2/node_modules`), so first we checked if the path + // is in a node_modules dir to avoid needlessly canonicalizing, then now compare + // against the canonicalized specifier. + let specifier = crate::node::resolve_specifier_into_node_modules( + specifier, + self.fs.as_ref(), + ); + if self.in_npm_pkg_checker.in_npm_package(&specifier) { return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::External { - specifier: specifier.clone(), - }, + LoadResponse::External { specifier }, )))); } - - if self.unstable_detect_cjs && specifier_has_extension(specifier, "js") { - if let Ok(Some(pkg_json)) = - self.node_resolver.get_closest_package_json(specifier) - { - if pkg_json.typ == "commonjs" { - if let Ok(path) = specifier.to_file_path() { - if let Ok(bytes) = std::fs::read(&path) { - let text: Arc = from_utf8_lossy_owned(bytes).into(); - let is_es_module = match self.esm_or_cjs_checker.is_esm( - specifier, - text.clone(), - MediaType::JavaScript, - ) { - Ok(value) => value, - Err(err) => { - return Box::pin(futures::future::ready(Err(err.into()))); - } - }; - if !is_es_module { - self.node_resolver.mark_cjs_resolution(specifier.clone()); - return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::External { - specifier: specifier.clone(), - }, - )))); - } else { - return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::Module { - specifier: specifier.clone(), - content: arc_str_to_bytes(text), - maybe_headers: None, - }, - )))); - } - } - } - } - } - } } if self.is_deno_publish diff --git a/cli/cache/module_info.rs b/cli/cache/module_info.rs index 4dbb01c37b..060a6f4f0c 100644 --- a/cli/cache/module_info.rs +++ b/cli/cache/module_info.rs @@ -44,18 +44,32 @@ pub static MODULE_INFO_CACHE_DB: CacheDBConfiguration = CacheDBConfiguration { /// A cache of `deno_graph::ModuleInfo` objects. Using this leads to a considerable /// performance improvement because when it exists we can skip parsing a module for /// deno_graph. +#[derive(Debug)] pub struct ModuleInfoCache { conn: CacheDB, + parsed_source_cache: Arc, } impl ModuleInfoCache { #[cfg(test)] - pub fn new_in_memory(version: &'static str) -> Self { - Self::new(CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version)) + pub fn new_in_memory( + version: &'static str, + parsed_source_cache: Arc, + ) -> Self { + Self::new( + CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version), + parsed_source_cache, + ) } - pub fn new(conn: CacheDB) -> Self { - Self { conn } + pub fn new( + conn: CacheDB, + parsed_source_cache: Arc, + ) -> Self { + Self { + conn, + parsed_source_cache, + } } /// Useful for testing: re-create this cache DB with a different current version. @@ -63,6 +77,7 @@ impl ModuleInfoCache { pub(crate) fn recreate_with_version(self, version: &'static str) -> Self { Self { conn: self.conn.recreate_with_version(version), + parsed_source_cache: self.parsed_source_cache, } } @@ -113,13 +128,10 @@ impl ModuleInfoCache { Ok(()) } - pub fn as_module_analyzer<'a>( - &'a self, - parsed_source_cache: &'a Arc, - ) -> ModuleInfoCacheModuleAnalyzer<'a> { + pub fn as_module_analyzer(&self) -> ModuleInfoCacheModuleAnalyzer { ModuleInfoCacheModuleAnalyzer { module_info_cache: self, - parsed_source_cache, + parsed_source_cache: &self.parsed_source_cache, } } } @@ -129,6 +141,84 @@ pub struct ModuleInfoCacheModuleAnalyzer<'a> { parsed_source_cache: &'a Arc, } +impl<'a> ModuleInfoCacheModuleAnalyzer<'a> { + fn load_cached_module_info( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + source_hash: CacheDBHash, + ) -> Option { + match self.module_info_cache.get_module_info( + specifier, + media_type, + source_hash, + ) { + Ok(Some(info)) => Some(info), + Ok(None) => None, + Err(err) => { + log::debug!( + "Error loading module cache info for {}. {:#}", + specifier, + err + ); + None + } + } + } + + fn save_module_info_to_cache( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + source_hash: CacheDBHash, + module_info: &ModuleInfo, + ) { + if let Err(err) = self.module_info_cache.set_module_info( + specifier, + media_type, + source_hash, + module_info, + ) { + log::debug!( + "Error saving module cache info for {}. {:#}", + specifier, + err + ); + } + } + + pub fn analyze_sync( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + source: &Arc, + ) -> Result { + // attempt to load from the cache + let source_hash = CacheDBHash::from_source(source); + if let Some(info) = + self.load_cached_module_info(specifier, media_type, source_hash) + { + return Ok(info); + } + + // otherwise, get the module info from the parsed source cache + let parser = self.parsed_source_cache.as_capturing_parser(); + let analyzer = ParserModuleAnalyzer::new(&parser); + let module_info = + analyzer.analyze_sync(specifier, source.clone(), media_type)?; + + // then attempt to cache it + self.save_module_info_to_cache( + specifier, + media_type, + source_hash, + &module_info, + ); + + Ok(module_info) + } +} + #[async_trait::async_trait(?Send)] impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { async fn analyze( @@ -139,20 +229,10 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { ) -> Result { // attempt to load from the cache let source_hash = CacheDBHash::from_source(&source); - match self.module_info_cache.get_module_info( - specifier, - media_type, - source_hash, - ) { - Ok(Some(info)) => return Ok(info), - Ok(None) => {} - Err(err) => { - log::debug!( - "Error loading module cache info for {}. {:#}", - specifier, - err - ); - } + if let Some(info) = + self.load_cached_module_info(specifier, media_type, source_hash) + { + return Ok(info); } // otherwise, get the module info from the parsed source cache @@ -169,18 +249,12 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { .unwrap()?; // then attempt to cache it - if let Err(err) = self.module_info_cache.set_module_info( + self.save_module_info_to_cache( specifier, media_type, source_hash, &module_info, - ) { - log::debug!( - "Error saving module cache info for {}. {:#}", - specifier, - err - ); - } + ); Ok(module_info) } @@ -202,7 +276,7 @@ fn serialize_media_type(media_type: MediaType) -> i64 { Tsx => 11, Json => 12, Wasm => 13, - TsBuildInfo => 14, + Css => 14, SourceMap => 15, Unknown => 16, } @@ -217,7 +291,7 @@ mod test { #[test] pub fn module_info_cache_general_use() { - let cache = ModuleInfoCache::new_in_memory("1.0.0"); + let cache = ModuleInfoCache::new_in_memory("1.0.0", Default::default()); let specifier1 = ModuleSpecifier::parse("https://localhost/mod.ts").unwrap(); let specifier2 = diff --git a/cli/cache/parsed_source.rs b/cli/cache/parsed_source.rs index df6e45c35e..7e819ae998 100644 --- a/cli/cache/parsed_source.rs +++ b/cli/cache/parsed_source.rs @@ -5,12 +5,11 @@ use std::sync::Arc; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; -use deno_ast::ParseDiagnostic; use deno_ast::ParsedSource; use deno_core::parking_lot::Mutex; -use deno_graph::CapturingModuleParser; -use deno_graph::DefaultModuleParser; -use deno_graph::ModuleParser; +use deno_graph::CapturingEsParser; +use deno_graph::DefaultEsParser; +use deno_graph::EsParser; use deno_graph::ParseOptions; use deno_graph::ParsedSourceStore; @@ -47,7 +46,7 @@ impl<'a> LazyGraphSourceParser<'a> { } } -#[derive(Default)] +#[derive(Debug, Default)] pub struct ParsedSourceCache { sources: Mutex>, } @@ -58,12 +57,11 @@ impl ParsedSourceCache { module: &deno_graph::JsModule, ) -> Result { let parser = self.as_capturing_parser(); - // this will conditionally parse because it's using a CapturingModuleParser - parser.parse_module(ParseOptions { + // this will conditionally parse because it's using a CapturingEsParser + parser.parse_program(ParseOptions { specifier: &module.specifier, source: module.source.clone(), media_type: module.media_type, - // don't bother enabling because this method is currently only used for vendoring scope_analysis: false, }) } @@ -87,10 +85,9 @@ impl ParsedSourceCache { specifier, source, media_type, - // don't bother enabling because this method is currently only used for emitting scope_analysis: false, }; - DefaultModuleParser.parse_module(options) + DefaultEsParser.parse_program(options) } /// Frees the parsed source from memory. @@ -100,8 +97,8 @@ impl ParsedSourceCache { /// Creates a parser that will reuse a ParsedSource from the store /// if it exists, or else parse. - pub fn as_capturing_parser(&self) -> CapturingModuleParser { - CapturingModuleParser::new(None, self) + pub fn as_capturing_parser(&self) -> CapturingEsParser { + CapturingEsParser::new(None, self) } } @@ -150,42 +147,3 @@ impl deno_graph::ParsedSourceStore for ParsedSourceCache { } } } - -pub struct EsmOrCjsChecker { - parsed_source_cache: Arc, -} - -impl EsmOrCjsChecker { - pub fn new(parsed_source_cache: Arc) -> Self { - Self { - parsed_source_cache, - } - } - - pub fn is_esm( - &self, - specifier: &ModuleSpecifier, - source: Arc, - media_type: MediaType, - ) -> Result { - // todo(dsherret): add a file cache here to avoid parsing with swc on each run - let source = match self.parsed_source_cache.get_parsed_source(specifier) { - Some(source) => source.clone(), - None => { - let source = deno_ast::parse_program(deno_ast::ParseParams { - specifier: specifier.clone(), - text: source, - media_type, - capture_tokens: true, // capture because it's used for cjs export analysis - scope_analysis: false, - maybe_syntax: None, - })?; - self - .parsed_source_cache - .set_parsed_source(specifier.clone(), source.clone()); - source - } - }; - Ok(source.is_module()) - } -} diff --git a/cli/emit.rs b/cli/emit.rs index 8e93092e67..8c4f2091cf 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -3,11 +3,14 @@ use crate::cache::EmitCache; use crate::cache::FastInsecureHasher; use crate::cache::ParsedSourceCache; +use crate::resolver::CjsTracker; +use deno_ast::ModuleKind; use deno_ast::SourceMapOption; use deno_ast::SourceRange; use deno_ast::SourceRanged; use deno_ast::SourceRangedForSpanned; +use deno_ast::TranspileModuleOptions; use deno_ast::TranspileResult; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; @@ -19,7 +22,9 @@ use deno_graph::Module; use deno_graph::ModuleGraph; use std::sync::Arc; +#[derive(Debug)] pub struct Emitter { + cjs_tracker: Arc, emit_cache: Arc, parsed_source_cache: Arc, transpile_and_emit_options: @@ -30,6 +35,7 @@ pub struct Emitter { impl Emitter { pub fn new( + cjs_tracker: Arc, emit_cache: Arc, parsed_source_cache: Arc, transpile_options: deno_ast::TranspileOptions, @@ -42,6 +48,7 @@ impl Emitter { hasher.finish() }; Self { + cjs_tracker, emit_cache, parsed_source_cache, transpile_and_emit_options: Arc::new((transpile_options, emit_options)), @@ -59,21 +66,19 @@ impl Emitter { continue; }; - // todo(https://github.com/denoland/deno_media_type/pull/12): use is_emittable() - let is_emittable = matches!( - module.media_type, - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx - ); - if is_emittable { + if module.media_type.is_emittable() { futures.push( self .emit_parsed_source( &module.specifier, module.media_type, + ModuleKind::from_is_cjs( + self.cjs_tracker.is_cjs_with_known_is_script( + &module.specifier, + module.media_type, + module.is_script, + )?, + ), &module.source, ) .boxed_local(), @@ -92,9 +97,10 @@ impl Emitter { pub fn maybe_cached_emit( &self, specifier: &ModuleSpecifier, + module_kind: deno_ast::ModuleKind, source: &str, ) -> Option { - let source_hash = self.get_source_hash(source); + let source_hash = self.get_source_hash(module_kind, source); self.emit_cache.get_emit_code(specifier, source_hash) } @@ -102,11 +108,12 @@ impl Emitter { &self, specifier: &ModuleSpecifier, media_type: MediaType, + module_kind: deno_ast::ModuleKind, source: &Arc, ) -> Result { // Note: keep this in sync with the sync version below let helper = EmitParsedSourceHelper(self); - match helper.pre_emit_parsed_source(specifier, source) { + match helper.pre_emit_parsed_source(specifier, module_kind, source) { PreEmitResult::Cached(emitted_text) => Ok(emitted_text), PreEmitResult::NotCached { source_hash } => { let parsed_source_cache = self.parsed_source_cache.clone(); @@ -119,8 +126,9 @@ impl Emitter { EmitParsedSourceHelper::transpile( &parsed_source_cache, &specifier, - source.clone(), media_type, + module_kind, + source.clone(), &transpile_and_emit_options.0, &transpile_and_emit_options.1, ) @@ -142,18 +150,20 @@ impl Emitter { &self, specifier: &ModuleSpecifier, media_type: MediaType, + module_kind: deno_ast::ModuleKind, source: &Arc, ) -> Result { // Note: keep this in sync with the async version above let helper = EmitParsedSourceHelper(self); - match helper.pre_emit_parsed_source(specifier, source) { + match helper.pre_emit_parsed_source(specifier, module_kind, source) { PreEmitResult::Cached(emitted_text) => Ok(emitted_text), PreEmitResult::NotCached { source_hash } => { let transpiled_source = EmitParsedSourceHelper::transpile( &self.parsed_source_cache, specifier, - source.clone(), media_type, + module_kind, + source.clone(), &self.transpile_and_emit_options.0, &self.transpile_and_emit_options.1, )?; @@ -171,6 +181,7 @@ impl Emitter { pub async fn load_and_emit_for_hmr( &self, specifier: &ModuleSpecifier, + module_kind: deno_ast::ModuleKind, ) -> Result { let media_type = MediaType::from_specifier(specifier); let source_code = tokio::fs::read_to_string( @@ -193,9 +204,14 @@ impl Emitter { let mut options = self.transpile_and_emit_options.1.clone(); options.source_map = SourceMapOption::None; let transpiled_source = parsed_source - .transpile(&self.transpile_and_emit_options.0, &options)? - .into_source() - .into_string()?; + .transpile( + &self.transpile_and_emit_options.0, + &deno_ast::TranspileModuleOptions { + module_kind: Some(module_kind), + }, + &options, + )? + .into_source(); Ok(transpiled_source.text) } MediaType::JavaScript @@ -206,7 +222,7 @@ impl Emitter { | MediaType::Dcts | MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Unknown => { // clear this specifier from the parsed source cache as it's now out of date @@ -219,10 +235,11 @@ impl Emitter { /// A hashing function that takes the source code and uses the global emit /// options then generates a string hash which can be stored to /// determine if the cached emit is valid or not. - fn get_source_hash(&self, source_text: &str) -> u64 { + fn get_source_hash(&self, module_kind: ModuleKind, source_text: &str) -> u64 { FastInsecureHasher::new_without_deno_version() // stored in the transpile_and_emit_options_hash .write_str(source_text) .write_u64(self.transpile_and_emit_options_hash) + .write_hashable(module_kind) .finish() } } @@ -239,9 +256,10 @@ impl<'a> EmitParsedSourceHelper<'a> { pub fn pre_emit_parsed_source( &self, specifier: &ModuleSpecifier, + module_kind: deno_ast::ModuleKind, source: &Arc, ) -> PreEmitResult { - let source_hash = self.0.get_source_hash(source); + let source_hash = self.0.get_source_hash(module_kind, source); if let Some(emit_code) = self.0.emit_cache.get_emit_code(specifier, source_hash) @@ -255,8 +273,9 @@ impl<'a> EmitParsedSourceHelper<'a> { pub fn transpile( parsed_source_cache: &ParsedSourceCache, specifier: &ModuleSpecifier, - source: Arc, media_type: MediaType, + module_kind: deno_ast::ModuleKind, + source: Arc, transpile_options: &deno_ast::TranspileOptions, emit_options: &deno_ast::EmitOptions, ) -> Result { @@ -265,8 +284,13 @@ impl<'a> EmitParsedSourceHelper<'a> { let parsed_source = parsed_source_cache .remove_or_parse_module(specifier, source, media_type)?; ensure_no_import_assertion(&parsed_source)?; - let transpile_result = - parsed_source.transpile(transpile_options, emit_options)?; + let transpile_result = parsed_source.transpile( + transpile_options, + &TranspileModuleOptions { + module_kind: Some(module_kind), + }, + emit_options, + )?; let transpiled_source = match transpile_result { TranspileResult::Owned(source) => source, TranspileResult::Cloned(source) => { @@ -275,8 +299,7 @@ impl<'a> EmitParsedSourceHelper<'a> { } }; debug_assert!(transpiled_source.source_map.is_none()); - let text = String::from_utf8(transpiled_source.source)?; - Ok(text) + Ok(transpiled_source.text) } pub fn post_emit_parsed_source( @@ -321,7 +344,7 @@ fn ensure_no_import_assertion( deno_core::anyhow::anyhow!("{}", msg) } - let Some(module) = parsed_source.program_ref().as_module() else { + let deno_ast::ProgramRef::Module(module) = parsed_source.program_ref() else { return Ok(()); }; diff --git a/cli/factory.rs b/cli/factory.rs index d5ef4fd8b3..4a36c75ba2 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -11,10 +11,10 @@ use crate::args::StorageKeyResolver; use crate::args::TsConfigType; use crate::cache::Caches; use crate::cache::CodeCache; +use crate::cache::DenoCacheEnvFsAdapter; use crate::cache::DenoDir; use crate::cache::DenoDirProvider; use crate::cache::EmitCache; -use crate::cache::EsmOrCjsChecker; use crate::cache::GlobalHttpCache; use crate::cache::HttpCache; use crate::cache::LocalHttpCache; @@ -33,12 +33,16 @@ use crate::module_loader::ModuleLoadPreparer; use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::npm::create_cli_npm_resolver; +use crate::npm::create_in_npm_pkg_checker; use crate::npm::CliByonmNpmResolverCreateOptions; +use crate::npm::CliManagedInNpmPkgCheckerCreateOptions; +use crate::npm::CliManagedNpmResolverCreateOptions; use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolverCreateOptions; -use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; -use crate::resolver::CjsResolutionStore; +use crate::npm::CreateInNpmPkgCheckerOptions; +use crate::resolver::CjsTracker; +use crate::resolver::CjsTrackerOptions; use crate::resolver::CliDenoResolverFs; use crate::resolver::CliGraphResolver; use crate::resolver::CliGraphResolverOptions; @@ -51,6 +55,7 @@ use crate::tools::check::TypeChecker; use crate::tools::coverage::CoverageCollector; use crate::tools::lint::LintRuleProvider; use crate::tools::run::hmr::HmrRunner; +use crate::tsc::TypeCheckingCjsTracker; use crate::util::file_watcher::WatcherCommunicator; use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; @@ -59,6 +64,7 @@ use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerOptions; use std::path::PathBuf; +use deno_cache_dir::npm::NpmCacheDir; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::WorkspaceResolver; use deno_core::error::AnyError; @@ -68,6 +74,7 @@ use deno_core::FeatureChecker; use deno_runtime::deno_fs; use deno_runtime::deno_node::DenoFsNodeResolverEnv; use deno_runtime::deno_node::NodeResolver; +use deno_runtime::deno_node::PackageJsonResolver; use deno_runtime::deno_permissions::Permissions; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_tls::rustls::RootCertStore; @@ -77,6 +84,7 @@ use deno_runtime::inspector_server::InspectorServer; use deno_runtime::permissions::RuntimePermissionDescriptorParser; use log::warn; use node_resolver::analyze::NodeCodeTranslator; +use node_resolver::InNpmPackageChecker; use once_cell::sync::OnceCell; use std::future::Future; use std::sync::Arc; @@ -164,39 +172,41 @@ impl Deferred { #[derive(Default)] struct CliFactoryServices { - cli_options: Deferred>, + blob_store: Deferred>, caches: Deferred>, + cjs_tracker: Deferred>, + cli_node_resolver: Deferred>, + cli_options: Deferred>, + code_cache: Deferred>, + emit_cache: Deferred>, + emitter: Deferred>, + feature_checker: Deferred>, file_fetcher: Deferred>, + fs: Deferred>, global_http_cache: Deferred>, http_cache: Deferred>, http_client_provider: Deferred>, - emit_cache: Deferred>, - emitter: Deferred>, - esm_or_cjs_checker: Deferred>, - fs: Deferred>, + in_npm_pkg_checker: Deferred>, main_graph_container: Deferred>, - maybe_inspector_server: Deferred>>, - root_cert_store_provider: Deferred>, - blob_store: Deferred>, - module_info_cache: Deferred>, - parsed_source_cache: Deferred>, - resolver: Deferred>, maybe_file_watcher_reporter: Deferred>, + maybe_inspector_server: Deferred>>, module_graph_builder: Deferred>, module_graph_creator: Deferred>, + module_info_cache: Deferred>, module_load_preparer: Deferred>, node_code_translator: Deferred>, node_resolver: Deferred>, + npm_cache_dir: Deferred>, npm_resolver: Deferred>, + parsed_source_cache: Deferred>, permission_desc_parser: Deferred>, + pkg_json_resolver: Deferred>, + resolver: Deferred>, + root_cert_store_provider: Deferred>, root_permissions_container: Deferred, sloppy_imports_resolver: Deferred>>, text_only_progress_bar: Deferred, type_checker: Deferred>, - cjs_resolutions: Deferred>, - cli_node_resolver: Deferred>, - feature_checker: Deferred>, - code_cache: Deferred>, workspace_resolver: Deferred>, } @@ -300,12 +310,6 @@ impl CliFactory { .get_or_init(|| ProgressBar::new(ProgressBarStyle::TextOnly)) } - pub fn esm_or_cjs_checker(&self) -> &Arc { - self.services.esm_or_cjs_checker.get_or_init(|| { - Arc::new(EsmOrCjsChecker::new(self.parsed_source_cache().clone())) - }) - } - pub fn global_http_cache(&self) -> Result<&Arc, AnyError> { self.services.global_http_cache.get_or_try_init(|| { Ok(Arc::new(GlobalHttpCache::new( @@ -359,56 +363,112 @@ impl CliFactory { self.services.fs.get_or_init(|| Arc::new(deno_fs::RealFs)) } + pub fn in_npm_pkg_checker( + &self, + ) -> Result<&Arc, AnyError> { + self.services.in_npm_pkg_checker.get_or_try_init(|| { + let cli_options = self.cli_options()?; + let options = if cli_options.use_byonm() { + CreateInNpmPkgCheckerOptions::Byonm + } else { + CreateInNpmPkgCheckerOptions::Managed( + CliManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: self.npm_cache_dir()?.root_dir_url(), + maybe_node_modules_path: cli_options + .node_modules_dir_path() + .map(|p| p.as_path()), + }, + ) + }; + Ok(create_in_npm_pkg_checker(options)) + }) + } + + pub fn npm_cache_dir(&self) -> Result<&Arc, AnyError> { + self.services.npm_cache_dir.get_or_try_init(|| { + let fs = self.fs(); + let global_path = self.deno_dir()?.npm_folder_path(); + let cli_options = self.cli_options()?; + Ok(Arc::new(NpmCacheDir::new( + &DenoCacheEnvFsAdapter(fs.as_ref()), + global_path, + cli_options.npmrc().get_all_known_registries_urls(), + ))) + }) + } + pub async fn npm_resolver( &self, ) -> Result<&Arc, AnyError> { self .services .npm_resolver - .get_or_try_init_async(async { - let fs = self.fs(); - let cli_options = self.cli_options()?; - // For `deno install` we want to force the managed resolver so it can set up `node_modules/` directory. - create_cli_npm_resolver(if cli_options.use_byonm() && !matches!(cli_options.sub_command(), DenoSubcommand::Install(_) | DenoSubcommand::Add(_) | DenoSubcommand::Remove(_)) { - CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { - fs: CliDenoResolverFs(fs.clone()), - root_node_modules_dir: Some(match cli_options.node_modules_dir_path() { - Some(node_modules_path) => node_modules_path.to_path_buf(), - // path needs to be canonicalized for node resolution - // (node_modules_dir_path above is already canonicalized) - None => canonicalize_path_maybe_not_exists(cli_options.initial_cwd())? - .join("node_modules"), - }), - }) - } else { - CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions { - snapshot: match cli_options.resolve_npm_resolution_snapshot()? { - Some(snapshot) => { - CliNpmResolverManagedSnapshotOption::Specified(Some(snapshot)) - } - None => match cli_options.maybe_lockfile() { - Some(lockfile) => { - CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( - lockfile.clone(), - ) - } - None => CliNpmResolverManagedSnapshotOption::Specified(None), + .get_or_try_init_async( + async { + let fs = self.fs(); + let cli_options = self.cli_options()?; + create_cli_npm_resolver(if cli_options.use_byonm() { + CliNpmResolverCreateOptions::Byonm( + CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(fs.clone()), + pkg_json_resolver: self.pkg_json_resolver().clone(), + root_node_modules_dir: Some( + match cli_options.node_modules_dir_path() { + Some(node_modules_path) => node_modules_path.to_path_buf(), + // path needs to be canonicalized for node resolution + // (node_modules_dir_path above is already canonicalized) + None => canonicalize_path_maybe_not_exists( + cli_options.initial_cwd(), + )? + .join("node_modules"), + }, + ), }, - }, - maybe_lockfile: cli_options.maybe_lockfile().cloned(), - fs: fs.clone(), - http_client_provider: self.http_client_provider().clone(), - npm_global_cache_dir: self.deno_dir()?.npm_folder_path(), - cache_setting: cli_options.cache_setting(), - text_only_progress_bar: self.text_only_progress_bar().clone(), - maybe_node_modules_path: cli_options.node_modules_dir_path().cloned(), - npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::from_workspace(cli_options.workspace())), - npm_system_info: cli_options.npm_system_info(), - npmrc: cli_options.npmrc().clone(), - lifecycle_scripts: cli_options.lifecycle_scripts_config(), + ) + } else { + CliNpmResolverCreateOptions::Managed( + CliManagedNpmResolverCreateOptions { + snapshot: match cli_options.resolve_npm_resolution_snapshot()? { + Some(snapshot) => { + CliNpmResolverManagedSnapshotOption::Specified(Some( + snapshot, + )) + } + None => match cli_options.maybe_lockfile() { + Some(lockfile) => { + CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( + lockfile.clone(), + ) + } + None => { + CliNpmResolverManagedSnapshotOption::Specified(None) + } + }, + }, + maybe_lockfile: cli_options.maybe_lockfile().cloned(), + fs: fs.clone(), + http_client_provider: self.http_client_provider().clone(), + npm_cache_dir: self.npm_cache_dir()?.clone(), + cache_setting: cli_options.cache_setting(), + text_only_progress_bar: self.text_only_progress_bar().clone(), + maybe_node_modules_path: cli_options + .node_modules_dir_path() + .cloned(), + npm_install_deps_provider: Arc::new( + NpmInstallDepsProvider::from_workspace( + cli_options.workspace(), + ), + ), + npm_system_info: cli_options.npm_system_info(), + npmrc: cli_options.npmrc().clone(), + lifecycle_scripts: cli_options.lifecycle_scripts_config(), + }, + ) }) - }).await - }.boxed_local()) + .await + } + .boxed_local(), + ) .await } @@ -513,6 +573,7 @@ impl CliFactory { self.services.module_info_cache.get_or_try_init(|| { Ok(Arc::new(ModuleInfoCache::new( self.caches()?.dep_analysis_db(), + self.parsed_source_cache().clone(), ))) }) } @@ -541,6 +602,7 @@ impl CliFactory { ts_config_result.ts_config, )?; Ok(Arc::new(Emitter::new( + self.cjs_tracker()?.clone(), self.emit_cache()?.clone(), self.parsed_source_cache().clone(), transpile_options, @@ -564,7 +626,9 @@ impl CliFactory { async { Ok(Arc::new(NodeResolver::new( DenoFsNodeResolverEnv::new(self.fs().clone()), + self.in_npm_pkg_checker()?.clone(), self.npm_resolver().await?.clone().into_npm_resolver(), + self.pkg_json_resolver().clone(), ))) } .boxed_local(), @@ -582,24 +646,35 @@ impl CliFactory { let caches = self.caches()?; let node_analysis_cache = NodeAnalysisCache::new(caches.node_analysis_db()); - let node_resolver = self.cli_node_resolver().await?.clone(); + let node_resolver = self.node_resolver().await?.clone(); let cjs_esm_analyzer = CliCjsCodeAnalyzer::new( node_analysis_cache, + self.cjs_tracker()?.clone(), self.fs().clone(), - node_resolver, Some(self.parsed_source_cache().clone()), + self.cli_options()?.is_npm_main(), ); Ok(Arc::new(NodeCodeTranslator::new( cjs_esm_analyzer, DenoFsNodeResolverEnv::new(self.fs().clone()), - self.node_resolver().await?.clone(), + self.in_npm_pkg_checker()?.clone(), + node_resolver, self.npm_resolver().await?.clone().into_npm_resolver(), + self.pkg_json_resolver().clone(), ))) }) .await } + pub fn pkg_json_resolver(&self) -> &Arc { + self.services.pkg_json_resolver.get_or_init(|| { + Arc::new(PackageJsonResolver::new(DenoFsNodeResolverEnv::new( + self.fs().clone(), + ))) + }) + } + pub async fn type_checker(&self) -> Result<&Arc, AnyError> { self .services @@ -608,6 +683,10 @@ impl CliFactory { let cli_options = self.cli_options()?; Ok(Arc::new(TypeChecker::new( self.caches()?.clone(), + Arc::new(TypeCheckingCjsTracker::new( + self.cjs_tracker()?.clone(), + self.module_info_cache()?.clone(), + )), cli_options.clone(), self.module_graph_builder().await?.clone(), self.node_resolver().await?.clone(), @@ -626,19 +705,18 @@ impl CliFactory { .get_or_try_init_async(async { let cli_options = self.cli_options()?; Ok(Arc::new(ModuleGraphBuilder::new( - cli_options.clone(), self.caches()?.clone(), - self.esm_or_cjs_checker().clone(), + cli_options.clone(), + self.file_fetcher()?.clone(), self.fs().clone(), - self.resolver().await?.clone(), - self.cli_node_resolver().await?.clone(), - self.npm_resolver().await?.clone(), - self.module_info_cache()?.clone(), - self.parsed_source_cache().clone(), + self.global_http_cache()?.clone(), + self.in_npm_pkg_checker()?.clone(), cli_options.maybe_lockfile().cloned(), self.maybe_file_watcher_reporter().clone(), - self.file_fetcher()?.clone(), - self.global_http_cache()?.clone(), + self.module_info_cache()?.clone(), + self.npm_resolver().await?.clone(), + self.parsed_source_cache().clone(), + self.resolver().await?.clone(), self.root_permissions_container()?.clone(), ))) }) @@ -710,8 +788,17 @@ impl CliFactory { .await } - pub fn cjs_resolutions(&self) -> &Arc { - self.services.cjs_resolutions.get_or_init(Default::default) + pub fn cjs_tracker(&self) -> Result<&Arc, AnyError> { + self.services.cjs_tracker.get_or_try_init(|| { + let options = self.cli_options()?; + Ok(Arc::new(CjsTracker::new( + self.in_npm_pkg_checker()?.clone(), + self.pkg_json_resolver().clone(), + CjsTrackerOptions { + unstable_detect_cjs: options.unstable_detect_cjs(), + }, + ))) + }) } pub async fn cli_node_resolver( @@ -722,8 +809,9 @@ impl CliFactory { .cli_node_resolver .get_or_try_init_async(async { Ok(Arc::new(CliNodeResolver::new( - self.cjs_resolutions().clone(), + self.cjs_tracker()?.clone(), self.fs().clone(), + self.in_npm_pkg_checker()?.clone(), self.node_resolver().await?.clone(), self.npm_resolver().await?.clone(), ))) @@ -761,6 +849,7 @@ impl CliFactory { ) -> Result { let cli_options = self.cli_options()?; Ok(DenoCompileBinaryWriter::new( + self.cjs_tracker()?, self.deno_dir()?, self.emitter()?, self.file_fetcher()?, @@ -791,53 +880,60 @@ impl CliFactory { &self, ) -> Result { let cli_options = self.cli_options()?; + let fs = self.fs(); let node_resolver = self.node_resolver().await?; let npm_resolver = self.npm_resolver().await?; - let fs = self.fs(); let cli_node_resolver = self.cli_node_resolver().await?; let cli_npm_resolver = self.npm_resolver().await?.clone(); + let in_npm_pkg_checker = self.in_npm_pkg_checker()?; let maybe_file_watcher_communicator = if cli_options.has_hmr() { Some(self.watcher_communicator.clone().unwrap()) } else { None }; + let node_code_translator = self.node_code_translator().await?; + let cjs_tracker = self.cjs_tracker()?.clone(); + let pkg_json_resolver = self.pkg_json_resolver().clone(); Ok(CliMainWorkerFactory::new( self.blob_store().clone(), - self.cjs_resolutions().clone(), if cli_options.code_cache_enabled() { Some(self.code_cache()?.clone()) } else { None }, self.feature_checker()?.clone(), - self.fs().clone(), + fs.clone(), maybe_file_watcher_communicator, self.maybe_inspector_server()?.clone(), cli_options.maybe_lockfile().cloned(), Box::new(CliModuleLoaderFactory::new( cli_options, + cjs_tracker, if cli_options.code_cache_enabled() { Some(self.code_cache()?.clone()) } else { None }, self.emitter()?.clone(), + fs.clone(), + in_npm_pkg_checker.clone(), self.main_module_graph_container().await?.clone(), self.module_load_preparer().await?.clone(), + node_code_translator.clone(), cli_node_resolver.clone(), cli_npm_resolver.clone(), NpmModuleLoader::new( - self.cjs_resolutions().clone(), - self.node_code_translator().await?.clone(), + self.cjs_tracker()?.clone(), fs.clone(), - cli_node_resolver.clone(), + node_code_translator.clone(), ), self.parsed_source_cache().clone(), self.resolver().await?.clone(), )), node_resolver.clone(), npm_resolver.clone(), + pkg_json_resolver, self.root_cert_store_provider().clone(), self.root_permissions_container()?.clone(), StorageKeyResolver::from_options(cli_options), @@ -853,8 +949,10 @@ impl CliFactory { let create_hmr_runner = if cli_options.has_hmr() { let watcher_communicator = self.watcher_communicator.clone().unwrap(); let emitter = self.emitter()?.clone(); + let cjs_tracker = self.cjs_tracker()?.clone(); let fn_: crate::worker::CreateHmrRunnerCb = Box::new(move |session| { Box::new(HmrRunner::new( + cjs_tracker.clone(), emitter.clone(), session, watcher_communicator.clone(), @@ -891,7 +989,6 @@ impl CliFactory { inspect_wait: cli_options.inspect_wait().is_some(), strace_ops: cli_options.strace_ops().clone(), is_inspecting: cli_options.is_inspecting(), - is_npm_main: cli_options.is_npm_main(), location: cli_options.location_flag().clone(), // if the user ran a binary command, we'll need to set process.argv[0] // to be the name of the binary command instead of deno @@ -909,7 +1006,6 @@ impl CliFactory { node_ipc: cli_options.node_ipc_fd(), serve_port: cli_options.serve_port(), serve_host: cli_options.serve_host(), - unstable_detect_cjs: cli_options.unstable_detect_cjs(), }) } } diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 2eaee228af..46257cf785 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -6,7 +6,6 @@ use crate::args::CliLockfile; use crate::args::CliOptions; use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; use crate::cache; -use crate::cache::EsmOrCjsChecker; use crate::cache::GlobalHttpCache; use crate::cache::ModuleInfoCache; use crate::cache::ParsedSourceCache; @@ -15,7 +14,6 @@ use crate::errors::get_error_class_name; use crate::file_fetcher::FileFetcher; use crate::npm::CliNpmResolver; use crate::resolver::CliGraphResolver; -use crate::resolver::CliNodeResolver; use crate::resolver::CliSloppyImportsResolver; use crate::resolver::SloppyImportsCachedFs; use crate::tools::check; @@ -50,6 +48,7 @@ use deno_runtime::deno_permissions::PermissionsContainer; use deno_semver::jsr::JsrDepPackageReq; use deno_semver::package::PackageNv; use import_map::ImportMapError; +use node_resolver::InNpmPackageChecker; use std::collections::HashSet; use std::error::Error; use std::ops::Deref; @@ -379,54 +378,51 @@ pub struct BuildFastCheckGraphOptions<'a> { } pub struct ModuleGraphBuilder { - options: Arc, caches: Arc, - esm_or_cjs_checker: Arc, + cli_options: Arc, + file_fetcher: Arc, fs: Arc, - resolver: Arc, - node_resolver: Arc, - npm_resolver: Arc, - module_info_cache: Arc, - parsed_source_cache: Arc, + global_http_cache: Arc, + in_npm_pkg_checker: Arc, lockfile: Option>, maybe_file_watcher_reporter: Option, - file_fetcher: Arc, - global_http_cache: Arc, + module_info_cache: Arc, + npm_resolver: Arc, + parsed_source_cache: Arc, + resolver: Arc, root_permissions_container: PermissionsContainer, } impl ModuleGraphBuilder { #[allow(clippy::too_many_arguments)] pub fn new( - options: Arc, caches: Arc, - esm_or_cjs_checker: Arc, + cli_options: Arc, + file_fetcher: Arc, fs: Arc, - resolver: Arc, - node_resolver: Arc, - npm_resolver: Arc, - module_info_cache: Arc, - parsed_source_cache: Arc, + global_http_cache: Arc, + in_npm_pkg_checker: Arc, lockfile: Option>, maybe_file_watcher_reporter: Option, - file_fetcher: Arc, - global_http_cache: Arc, + module_info_cache: Arc, + npm_resolver: Arc, + parsed_source_cache: Arc, + resolver: Arc, root_permissions_container: PermissionsContainer, ) -> Self { Self { - options, caches, - esm_or_cjs_checker, + cli_options, + file_fetcher, fs, - resolver, - node_resolver, - npm_resolver, - module_info_cache, - parsed_source_cache, + global_http_cache, + in_npm_pkg_checker, lockfile, maybe_file_watcher_reporter, - file_fetcher, - global_http_cache, + module_info_cache, + npm_resolver, + parsed_source_cache, + resolver, root_permissions_container, } } @@ -512,13 +508,11 @@ impl ModuleGraphBuilder { } let maybe_imports = if options.graph_kind.include_types() { - self.options.to_compiler_option_types()? + self.cli_options.to_compiler_option_types()? } else { Vec::new() }; - let analyzer = self - .module_info_cache - .as_module_analyzer(&self.parsed_source_cache); + let analyzer = self.module_info_cache.as_module_analyzer(); let mut loader = match options.loader { Some(loader) => MutLoaderRef::Borrowed(loader), None => MutLoaderRef::Owned(self.create_graph_loader()), @@ -566,7 +560,7 @@ impl ModuleGraphBuilder { // ensure an "npm install" is done if the user has explicitly // opted into using a node_modules directory if self - .options + .cli_options .node_modules_dir()? .map(|m| m.uses_node_modules_dir()) .unwrap_or(false) @@ -677,10 +671,10 @@ impl ModuleGraphBuilder { graph.build_fast_check_type_graph( deno_graph::BuildFastCheckTypeGraphOptions { - jsr_url_provider: &CliJsrUrlProvider, + es_parser: Some(&parser), fast_check_cache: fast_check_cache.as_ref().map(|c| c as _), fast_check_dts: false, - module_parser: Some(&parser), + jsr_url_provider: &CliJsrUrlProvider, resolver: Some(graph_resolver), npm_resolver: Some(&graph_npm_resolver), workspace_fast_check: options.workspace_fast_check, @@ -699,20 +693,18 @@ impl ModuleGraphBuilder { permissions: PermissionsContainer, ) -> cache::FetchCacher { cache::FetchCacher::new( - self.esm_or_cjs_checker.clone(), self.file_fetcher.clone(), + self.fs.clone(), self.global_http_cache.clone(), - self.node_resolver.clone(), - self.npm_resolver.clone(), + self.in_npm_pkg_checker.clone(), self.module_info_cache.clone(), cache::FetchCacherOptions { - file_header_overrides: self.options.resolve_file_header_overrides(), + file_header_overrides: self.cli_options.resolve_file_header_overrides(), permissions, is_deno_publish: matches!( - self.options.sub_command(), + self.cli_options.sub_command(), crate::args::DenoSubcommand::Publish { .. } ), - unstable_detect_cjs: self.options.unstable_detect_cjs(), }, ) } @@ -737,12 +729,12 @@ impl ModuleGraphBuilder { &self.fs, roots, GraphValidOptions { - kind: if self.options.type_check_mode().is_true() { + kind: if self.cli_options.type_check_mode().is_true() { GraphKind::All } else { GraphKind::CodeOnly }, - check_js: self.options.check_js(), + check_js: self.cli_options.check_js(), exit_integrity_errors: true, }, ) diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index 39f1ae6482..0769c90477 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -36,7 +36,6 @@ use deno_semver::package::PackageReq; use deno_semver::package::PackageReqReference; use deno_semver::Version; use import_map::ImportMap; -use node_resolver::NpmResolver; use once_cell::sync::Lazy; use regex::Regex; use std::borrow::Cow; @@ -336,7 +335,12 @@ impl<'a> TsResponseImportMapper<'a> { .resolver .maybe_managed_npm_resolver(Some(&self.file_referrer)) { - if npm_resolver.in_npm_package(specifier) { + let in_npm_pkg = self + .resolver + .maybe_node_resolver(Some(&self.file_referrer)) + .map(|n| n.in_npm_package(specifier)) + .unwrap_or(false); + if in_npm_pkg { if let Ok(Some(pkg_id)) = npm_resolver.resolve_pkg_id_from_specifier(specifier) { @@ -1199,14 +1203,11 @@ impl CodeActionCollection { }), ); - match parsed_source.program_ref() { - deno_ast::swc::ast::Program::Module(module) => module - .body - .iter() - .find(|i| i.range().contains(&specifier_range)) - .map(|i| text_info.line_and_column_index(i.range().start)), - deno_ast::swc::ast::Program::Script(_) => None, - } + parsed_source + .program_ref() + .body() + .find(|i| i.range().contains(&specifier_range)) + .map(|i| text_info.line_and_column_index(i.range().start)) } async fn deno_types_for_npm_action( diff --git a/cli/lsp/code_lens.rs b/cli/lsp/code_lens.rs index e117888fba..a57ca3ac9f 100644 --- a/cli/lsp/code_lens.rs +++ b/cli/lsp/code_lens.rs @@ -421,7 +421,7 @@ pub fn collect_test( ) -> Result, AnyError> { let mut collector = DenoTestCollector::new(specifier.clone(), parsed_source.clone()); - parsed_source.module().visit_with(&mut collector); + parsed_source.program().visit_with(&mut collector); Ok(collector.take()) } @@ -581,7 +581,7 @@ mod tests { .unwrap(); let mut collector = DenoTestCollector::new(specifier, parsed_module.clone()); - parsed_module.module().visit_with(&mut collector); + parsed_module.program().visit_with(&mut collector); assert_eq!( collector.take(), vec![ diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index cac0637a05..34bf64446d 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -985,7 +985,7 @@ impl Config { | MediaType::Tsx => Some(&workspace_settings.typescript), MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Unknown => None, } diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index 7d1ca6810d..8609aed05e 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -272,7 +272,7 @@ fn get_maybe_test_module_fut( parsed_source.specifier().clone(), parsed_source.text_info_lazy().clone(), ); - parsed_source.module().visit_with(&mut collector); + parsed_source.program().visit_with(&mut collector); Arc::new(collector.take()) }) .map(Result::ok) @@ -332,12 +332,8 @@ impl Document { .filter(|s| cache.is_valid_file_referrer(s)) .cloned() .or(file_referrer); - let media_type = resolve_media_type( - &specifier, - maybe_headers.as_ref(), - maybe_language_id, - &resolver, - ); + let media_type = + resolve_media_type(&specifier, maybe_headers.as_ref(), maybe_language_id); let (maybe_parsed_source, maybe_module) = if media_type_is_diagnosable(media_type) { parse_and_analyze_module( @@ -399,7 +395,6 @@ impl Document { &self.specifier, self.maybe_headers.as_ref(), self.maybe_language_id, - &resolver, ); let dependencies; let maybe_types_dependency; @@ -764,14 +759,7 @@ fn resolve_media_type( specifier: &ModuleSpecifier, maybe_headers: Option<&HashMap>, maybe_language_id: Option, - resolver: &LspResolver, ) -> MediaType { - if resolver.in_node_modules(specifier) { - if let Some(media_type) = resolver.node_media_type(specifier) { - return media_type; - } - } - if let Some(language_id) = maybe_language_id { return MediaType::from_specifier_and_content_type( specifier, @@ -1561,7 +1549,7 @@ fn parse_source( text: Arc, media_type: MediaType, ) -> ParsedSourceResult { - deno_ast::parse_module(deno_ast::ParseParams { + deno_ast::parse_program(deno_ast::ParseParams { specifier, text, media_type, diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index a592245ce7..4fa0e3afbc 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -904,7 +904,7 @@ impl Inner { | MediaType::Tsx => {} MediaType::Wasm | MediaType::SourceMap - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::Unknown => { if path.extension().and_then(|s| s.to_str()) != Some("jsonc") { continue; diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index 00587f8f50..9ce76005e5 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -2,6 +2,8 @@ use dashmap::DashMap; use deno_ast::MediaType; +use deno_ast::ParsedSource; +use deno_cache_dir::npm::NpmCacheDir; use deno_cache_dir::HttpCache; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::WorkspaceResolver; @@ -14,15 +16,15 @@ use deno_path_util::url_to_file_path; use deno_runtime::deno_fs; use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_node::PackageJson; +use deno_runtime::deno_node::PackageJsonResolver; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use indexmap::IndexMap; use node_resolver::errors::ClosestPkgJsonError; -use node_resolver::NodeResolution; +use node_resolver::InNpmPackageChecker; use node_resolver::NodeResolutionMode; -use node_resolver::NpmResolver; use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -36,6 +38,7 @@ use crate::args::create_default_npmrc; use crate::args::CacheSetting; use crate::args::CliLockfile; use crate::args::NpmInstallDepsProvider; +use crate::cache::DenoCacheEnvFsAdapter; use crate::graph_util::CliJsrUrlProvider; use crate::http_util::HttpClientProvider; use crate::lsp::config::Config; @@ -43,26 +46,32 @@ use crate::lsp::config::ConfigData; use crate::lsp::logging::lsp_warn; use crate::npm::create_cli_npm_resolver_for_lsp; use crate::npm::CliByonmNpmResolverCreateOptions; +use crate::npm::CliManagedInNpmPkgCheckerCreateOptions; +use crate::npm::CliManagedNpmResolverCreateOptions; use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolverCreateOptions; -use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; +use crate::npm::CreateInNpmPkgCheckerOptions; use crate::npm::ManagedCliNpmResolver; -use crate::resolver::CjsResolutionStore; +use crate::resolver::CjsTracker; +use crate::resolver::CjsTrackerOptions; use crate::resolver::CliDenoResolverFs; use crate::resolver::CliGraphResolver; use crate::resolver::CliGraphResolverOptions; use crate::resolver::CliNodeResolver; use crate::resolver::WorkerCliNpmGraphResolver; +use crate::tsc::into_specifier_and_media_type; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; #[derive(Debug, Clone)] struct LspScopeResolver { + cjs_tracker: Option>, graph_resolver: Arc, jsr_resolver: Option>, npm_resolver: Option>, node_resolver: Option>, + pkg_json_resolver: Option>, redirect_resolver: Option>, graph_imports: Arc>, config_data: Option>, @@ -71,10 +80,12 @@ struct LspScopeResolver { impl Default for LspScopeResolver { fn default() -> Self { Self { + cjs_tracker: None, graph_resolver: create_graph_resolver(None, None, None), jsr_resolver: None, npm_resolver: None, node_resolver: None, + pkg_json_resolver: None, redirect_resolver: None, graph_imports: Default::default(), config_data: None, @@ -90,14 +101,35 @@ impl LspScopeResolver { ) -> Self { let mut npm_resolver = None; let mut node_resolver = None; + let mut lsp_cjs_tracker = None; + let fs = Arc::new(deno_fs::RealFs); + let pkg_json_resolver = Arc::new(PackageJsonResolver::new( + deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), + )); if let Some(http_client) = http_client_provider { npm_resolver = create_npm_resolver( config_data.map(|d| d.as_ref()), cache, http_client, + &pkg_json_resolver, ) .await; - node_resolver = create_node_resolver(npm_resolver.as_ref()); + if let Some(npm_resolver) = &npm_resolver { + let in_npm_pkg_checker = create_in_npm_pkg_checker(npm_resolver); + let cjs_tracker = create_cjs_tracker( + in_npm_pkg_checker.clone(), + pkg_json_resolver.clone(), + ); + lsp_cjs_tracker = + Some(Arc::new(LspCjsTracker::new(cjs_tracker.clone()))); + node_resolver = Some(create_node_resolver( + cjs_tracker, + fs.clone(), + in_npm_pkg_checker, + npm_resolver, + pkg_json_resolver.clone(), + )); + } } let graph_resolver = create_graph_resolver( config_data.map(|d| d.as_ref()), @@ -134,10 +166,12 @@ impl LspScopeResolver { }) .unwrap_or_default(); Self { + cjs_tracker: lsp_cjs_tracker, graph_resolver, jsr_resolver, npm_resolver, node_resolver, + pkg_json_resolver: Some(pkg_json_resolver), redirect_resolver, graph_imports, config_data: config_data.cloned(), @@ -147,18 +181,40 @@ impl LspScopeResolver { fn snapshot(&self) -> Arc { let npm_resolver = self.npm_resolver.as_ref().map(|r| r.clone_snapshotted()); - let node_resolver = create_node_resolver(npm_resolver.as_ref()); + let fs = Arc::new(deno_fs::RealFs); + let pkg_json_resolver = Arc::new(PackageJsonResolver::new( + deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), + )); + let mut node_resolver = None; + let mut lsp_cjs_tracker = None; + if let Some(npm_resolver) = &npm_resolver { + let in_npm_pkg_checker = create_in_npm_pkg_checker(npm_resolver); + let cjs_tracker = create_cjs_tracker( + in_npm_pkg_checker.clone(), + pkg_json_resolver.clone(), + ); + lsp_cjs_tracker = Some(Arc::new(LspCjsTracker::new(cjs_tracker.clone()))); + node_resolver = Some(create_node_resolver( + cjs_tracker, + fs, + in_npm_pkg_checker, + npm_resolver, + pkg_json_resolver.clone(), + )); + } let graph_resolver = create_graph_resolver( self.config_data.as_deref(), npm_resolver.as_ref(), node_resolver.as_ref(), ); Arc::new(Self { + cjs_tracker: lsp_cjs_tracker, graph_resolver, jsr_resolver: self.jsr_resolver.clone(), npm_resolver, node_resolver, redirect_resolver: self.redirect_resolver.clone(), + pkg_json_resolver: Some(pkg_json_resolver), graph_imports: self.graph_imports.clone(), config_data: self.config_data.clone(), }) @@ -261,6 +317,22 @@ impl LspResolver { resolver.graph_resolver.create_graph_npm_resolver() } + pub fn maybe_cjs_tracker( + &self, + file_referrer: Option<&ModuleSpecifier>, + ) -> Option<&Arc> { + let resolver = self.get_scope_resolver(file_referrer); + resolver.cjs_tracker.as_ref() + } + + pub fn maybe_node_resolver( + &self, + file_referrer: Option<&ModuleSpecifier>, + ) -> Option<&Arc> { + let resolver = self.get_scope_resolver(file_referrer); + resolver.node_resolver.as_ref() + } + pub fn maybe_managed_npm_resolver( &self, file_referrer: Option<&ModuleSpecifier>, @@ -328,7 +400,7 @@ impl LspResolver { ) -> Option<(ModuleSpecifier, MediaType)> { let resolver = self.get_scope_resolver(file_referrer); let node_resolver = resolver.node_resolver.as_ref()?; - Some(NodeResolution::into_specifier_and_media_type(Some( + Some(into_specifier_and_media_type(Some( node_resolver .resolve_req_reference(req_ref, referrer, NodeResolutionMode::Types) .ok()?, @@ -346,14 +418,10 @@ impl LspResolver { .contains("/node_modules/") } - let global_npm_resolver = self - .get_scope_resolver(Some(specifier)) - .npm_resolver - .as_ref() - .and_then(|npm_resolver| npm_resolver.as_managed()) - .filter(|r| r.root_node_modules_path().is_none()); - if let Some(npm_resolver) = &global_npm_resolver { - if npm_resolver.in_npm_package(specifier) { + if let Some(node_resolver) = + &self.get_scope_resolver(Some(specifier)).node_resolver + { + if node_resolver.in_npm_package(specifier) { return true; } } @@ -361,18 +429,6 @@ impl LspResolver { has_node_modules_dir(specifier) } - pub fn node_media_type( - &self, - specifier: &ModuleSpecifier, - ) -> Option { - let resolver = self.get_scope_resolver(Some(specifier)); - let node_resolver = resolver.node_resolver.as_ref()?; - let resolution = node_resolver - .url_to_node_resolution(specifier.clone()) - .ok()?; - Some(NodeResolution::into_specifier_and_media_type(Some(resolution)).1) - } - pub fn is_bare_package_json_dep( &self, specifier_text: &str, @@ -398,10 +454,10 @@ impl LspResolver { referrer: &ModuleSpecifier, ) -> Result>, ClosestPkgJsonError> { let resolver = self.get_scope_resolver(Some(referrer)); - let Some(node_resolver) = resolver.node_resolver.as_ref() else { + let Some(pkg_json_resolver) = resolver.pkg_json_resolver.as_ref() else { return Ok(None); }; - node_resolver.get_closest_package_json(referrer) + pkg_json_resolver.get_closest_package_json(referrer) } pub fn resolve_redirects( @@ -457,11 +513,13 @@ async fn create_npm_resolver( config_data: Option<&ConfigData>, cache: &LspCache, http_client_provider: &Arc, + pkg_json_resolver: &Arc, ) -> Option> { let enable_byonm = config_data.map(|d| d.byonm).unwrap_or(false); let options = if enable_byonm { CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { fs: CliDenoResolverFs(Arc::new(deno_fs::RealFs)), + pkg_json_resolver: pkg_json_resolver.clone(), root_node_modules_dir: config_data.and_then(|config_data| { config_data.node_modules_dir.clone().or_else(|| { url_to_file_path(&config_data.scope) @@ -471,7 +529,15 @@ async fn create_npm_resolver( }), }) } else { - CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions { + let npmrc = config_data + .and_then(|d| d.npmrc.clone()) + .unwrap_or_else(create_default_npmrc); + let npm_cache_dir = Arc::new(NpmCacheDir::new( + &DenoCacheEnvFsAdapter(&deno_fs::RealFs), + cache.deno_dir().npm_folder_path(), + npmrc.get_all_known_registries_urls(), + )); + CliNpmResolverCreateOptions::Managed(CliManagedNpmResolverCreateOptions { http_client_provider: http_client_provider.clone(), snapshot: match config_data.and_then(|d| d.lockfile.as_ref()) { Some(lockfile) => { @@ -485,7 +551,7 @@ async fn create_npm_resolver( // updating it. Only the cache request should update the lockfile. maybe_lockfile: None, fs: Arc::new(deno_fs::RealFs), - npm_global_cache_dir: cache.deno_dir().npm_folder_path(), + npm_cache_dir, // Use an "only" cache setting in order to make the // user do an explicit "cache" command and prevent // the cache from being filled with lots of packages while @@ -496,9 +562,7 @@ async fn create_npm_resolver( .and_then(|d| d.node_modules_dir.clone()), // only used for top level install, so we can ignore this npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()), - npmrc: config_data - .and_then(|d| d.npmrc.clone()) - .unwrap_or_else(create_default_npmrc), + npmrc, npm_system_info: NpmSystemInfo::default(), lifecycle_scripts: Default::default(), }) @@ -506,28 +570,59 @@ async fn create_npm_resolver( Some(create_cli_npm_resolver_for_lsp(options).await) } +fn create_cjs_tracker( + in_npm_pkg_checker: Arc, + pkg_json_resolver: Arc, +) -> Arc { + Arc::new(CjsTracker::new( + in_npm_pkg_checker, + pkg_json_resolver, + CjsTrackerOptions { + // todo(dsherret): support in the lsp by stabilizing the feature + // so that we don't have to pipe the config in here + unstable_detect_cjs: false, + }, + )) +} + +fn create_in_npm_pkg_checker( + npm_resolver: &Arc, +) -> Arc { + crate::npm::create_in_npm_pkg_checker(match npm_resolver.as_inner() { + crate::npm::InnerCliNpmResolverRef::Byonm(_) => { + CreateInNpmPkgCheckerOptions::Byonm + } + crate::npm::InnerCliNpmResolverRef::Managed(m) => { + CreateInNpmPkgCheckerOptions::Managed( + CliManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: m.global_cache_root_url(), + maybe_node_modules_path: m.maybe_node_modules_path(), + }, + ) + } + }) +} + fn create_node_resolver( - npm_resolver: Option<&Arc>, -) -> Option> { - use once_cell::sync::Lazy; - - // it's not ideal to share this across all scopes and to - // never clear it, but it's fine for the time being - static CJS_RESOLUTIONS: Lazy> = - Lazy::new(Default::default); - - let npm_resolver = npm_resolver?; - let fs = Arc::new(deno_fs::RealFs); + cjs_tracker: Arc, + fs: Arc, + in_npm_pkg_checker: Arc, + npm_resolver: &Arc, + pkg_json_resolver: Arc, +) -> Arc { let node_resolver_inner = Arc::new(NodeResolver::new( deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), + in_npm_pkg_checker.clone(), npm_resolver.clone().into_npm_resolver(), + pkg_json_resolver.clone(), )); - Some(Arc::new(CliNodeResolver::new( - CJS_RESOLUTIONS.clone(), + Arc::new(CliNodeResolver::new( + cjs_tracker.clone(), fs, + in_npm_pkg_checker, node_resolver_inner, npm_resolver.clone(), - ))) + )) } fn create_graph_resolver( @@ -702,6 +797,45 @@ impl RedirectResolver { } } +#[derive(Debug)] +pub struct LspCjsTracker { + cjs_tracker: Arc, +} + +impl LspCjsTracker { + pub fn new(cjs_tracker: Arc) -> Self { + Self { cjs_tracker } + } + + pub fn is_cjs( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + maybe_parsed_source: Option<&ParsedSource>, + ) -> bool { + if let Some(module_kind) = + self.cjs_tracker.get_known_kind(specifier, media_type) + { + module_kind.is_cjs() + } else { + let maybe_is_script = maybe_parsed_source.map(|p| p.compute_is_script()); + maybe_is_script + .and_then(|is_script| { + self + .cjs_tracker + .is_cjs_with_known_is_script(specifier, media_type, is_script) + .ok() + }) + .unwrap_or_else(|| { + self + .cjs_tracker + .is_maybe_cjs(specifier, media_type) + .unwrap_or(false) + }) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/cli/lsp/testing/collectors.rs b/cli/lsp/testing/collectors.rs index 2f2ddb8773..2dd7ec0d96 100644 --- a/cli/lsp/testing/collectors.rs +++ b/cli/lsp/testing/collectors.rs @@ -650,7 +650,7 @@ pub mod tests { .unwrap(); let text_info = parsed_module.text_info_lazy().clone(); let mut collector = TestCollector::new(specifier, text_info); - parsed_module.module().visit_with(&mut collector); + parsed_module.program().visit_with(&mut collector); collector.take() } diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index 0f31d7dd3b..5fcdb3575a 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -4362,14 +4362,25 @@ fn op_load<'s>( None } else { let asset_or_document = state.get_asset_or_document(&specifier); - asset_or_document.map(|doc| LoadResponse { - data: doc.text(), - script_kind: crate::tsc::as_ts_script_kind(doc.media_type()), - version: state.script_version(&specifier), - is_cjs: matches!( - doc.media_type(), - MediaType::Cjs | MediaType::Cts | MediaType::Dcts - ), + asset_or_document.map(|doc| { + let maybe_cjs_tracker = state + .state_snapshot + .resolver + .maybe_cjs_tracker(Some(&specifier)); + LoadResponse { + data: doc.text(), + script_kind: crate::tsc::as_ts_script_kind(doc.media_type()), + version: state.script_version(&specifier), + is_cjs: maybe_cjs_tracker + .map(|t| { + t.is_cjs( + &specifier, + doc.media_type(), + doc.maybe_parsed_source().and_then(|p| p.as_ref().ok()), + ) + }) + .unwrap_or(false), + } }) }; diff --git a/cli/main.rs b/cli/main.rs index c2639c9088..04daff6700 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -135,7 +135,7 @@ async fn run_subcommand(flags: Arc) -> Result { tools::compile::compile(flags, compile_flags).await }), DenoSubcommand::Coverage(coverage_flags) => spawn_subcommand(async { - tools::coverage::cover_files(flags, coverage_flags).await + tools::coverage::cover_files(flags, coverage_flags) }), DenoSubcommand::Fmt(fmt_flags) => { spawn_subcommand( diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 4a020516e7..43c9e1aa07 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use std::cell::RefCell; +use std::path::Path; use std::path::PathBuf; use std::pin::Pin; use std::rc::Rc; @@ -23,19 +24,23 @@ use crate::graph_container::ModuleGraphUpdatePermit; use crate::graph_util::CreateGraphOptions; use crate::graph_util::ModuleGraphBuilder; use crate::node; +use crate::node::CliNodeCodeTranslator; use crate::npm::CliNpmResolver; +use crate::resolver::CjsTracker; use crate::resolver::CliGraphResolver; use crate::resolver::CliNodeResolver; use crate::resolver::ModuleCodeStringSource; +use crate::resolver::NotSupportedKindInNpmError; use crate::resolver::NpmModuleLoader; use crate::tools::check; use crate::tools::check::TypeChecker; use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; -use crate::worker::ModuleLoaderAndSourceMapGetter; +use crate::worker::CreateModuleLoaderResult; use crate::worker::ModuleLoaderFactory; use deno_ast::MediaType; +use deno_ast::ModuleKind; use deno_core::anyhow::anyhow; use deno_core::anyhow::bail; use deno_core::anyhow::Context; @@ -63,9 +68,12 @@ use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::Resolution; use deno_runtime::code_cache; +use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::create_host_defined_options; +use deno_runtime::deno_node::NodeRequireLoader; use deno_runtime::deno_permissions::PermissionsContainer; use deno_semver::npm::NpmPackageReqReference; +use node_resolver::InNpmPackageChecker; use node_resolver::NodeResolutionMode; pub struct ModuleLoadPreparer { @@ -198,11 +206,16 @@ struct SharedCliModuleLoaderState { lib_worker: TsTypeLib, initial_cwd: PathBuf, is_inspecting: bool, + is_npm_main: bool, is_repl: bool, + cjs_tracker: Arc, code_cache: Option>, emitter: Arc, + fs: Arc, + in_npm_pkg_checker: Arc, main_module_graph_container: Arc, module_load_preparer: Arc, + node_code_translator: Arc, node_resolver: Arc, npm_resolver: Arc, npm_module_loader: NpmModuleLoader, @@ -218,10 +231,14 @@ impl CliModuleLoaderFactory { #[allow(clippy::too_many_arguments)] pub fn new( options: &CliOptions, + cjs_tracker: Arc, code_cache: Option>, emitter: Arc, + fs: Arc, + in_npm_pkg_checker: Arc, main_module_graph_container: Arc, module_load_preparer: Arc, + node_code_translator: Arc, node_resolver: Arc, npm_resolver: Arc, npm_module_loader: NpmModuleLoader, @@ -235,14 +252,19 @@ impl CliModuleLoaderFactory { lib_worker: options.ts_type_lib_worker(), initial_cwd: options.initial_cwd().to_path_buf(), is_inspecting: options.is_inspecting(), + is_npm_main: options.is_npm_main(), is_repl: matches!( options.sub_command(), DenoSubcommand::Repl(_) | DenoSubcommand::Jupyter(_) ), + cjs_tracker, code_cache, emitter, + fs, + in_npm_pkg_checker, main_module_graph_container, module_load_preparer, + node_code_translator, node_resolver, npm_resolver, npm_module_loader, @@ -259,19 +281,30 @@ impl CliModuleLoaderFactory { is_worker: bool, parent_permissions: PermissionsContainer, permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { - let loader = Rc::new(CliModuleLoader(Rc::new(CliModuleLoaderInner { - lib, - is_worker, - parent_permissions, - permissions, + ) -> CreateModuleLoaderResult { + let module_loader = + Rc::new(CliModuleLoader(Rc::new(CliModuleLoaderInner { + lib, + is_worker, + is_npm_main: self.shared.is_npm_main, + parent_permissions, + permissions, + graph_container: graph_container.clone(), + node_code_translator: self.shared.node_code_translator.clone(), + emitter: self.shared.emitter.clone(), + parsed_source_cache: self.shared.parsed_source_cache.clone(), + shared: self.shared.clone(), + }))); + let node_require_loader = Rc::new(CliNodeRequireLoader::new( + self.shared.emitter.clone(), + self.shared.fs.clone(), graph_container, - emitter: self.shared.emitter.clone(), - parsed_source_cache: self.shared.parsed_source_cache.clone(), - shared: self.shared.clone(), - }))); - ModuleLoaderAndSourceMapGetter { - module_loader: loader, + self.shared.in_npm_pkg_checker.clone(), + self.shared.npm_resolver.clone(), + )); + CreateModuleLoaderResult { + module_loader, + node_require_loader, } } } @@ -280,7 +313,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { fn create_for_main( &self, root_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { + ) -> CreateModuleLoaderResult { self.create_with_lib( (*self.shared.main_module_graph_container).clone(), self.shared.lib_window, @@ -294,7 +327,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { &self, parent_permissions: PermissionsContainer, permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { + ) -> CreateModuleLoaderResult { self.create_with_lib( // create a fresh module graph for the worker WorkerModuleGraphContainer::new(Arc::new(ModuleGraph::new( @@ -310,6 +343,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { struct CliModuleLoaderInner { lib: TsTypeLib, + is_npm_main: bool, is_worker: bool, /// The initial set of permissions used to resolve the static imports in the /// worker. These are "allow all" for main worker, and parent thread @@ -318,6 +352,7 @@ struct CliModuleLoaderInner { permissions: PermissionsContainer, shared: Arc, emitter: Arc, + node_code_translator: Arc, parsed_source_cache: Arc, graph_container: TGraphContainer, } @@ -331,24 +366,7 @@ impl maybe_referrer: Option<&ModuleSpecifier>, requested_module_type: RequestedModuleType, ) -> Result { - let code_source = match self.load_prepared_module(specifier).await? { - Some(code_source) => code_source, - None => { - if self.shared.npm_module_loader.if_in_npm_package(specifier) { - self - .shared - .npm_module_loader - .load(specifier, maybe_referrer) - .await? - } else { - let mut msg = format!("Loading unprepared module: {specifier}"); - if let Some(referrer) = maybe_referrer { - msg = format!("{}, imported from: {}", msg, referrer.as_str()); - } - return Err(anyhow!(msg)); - } - } - }; + let code_source = self.load_code_source(specifier, maybe_referrer).await?; let code = if self.shared.is_inspecting { // we need the code with the source map in order for // it to work with --inspect or --inspect-brk @@ -402,6 +420,29 @@ impl )) } + async fn load_code_source( + &self, + specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + ) -> Result { + if let Some(code_source) = self.load_prepared_module(specifier).await? { + return Ok(code_source); + } + if self.shared.node_resolver.in_npm_package(specifier) { + return self + .shared + .npm_module_loader + .load(specifier, maybe_referrer) + .await; + } + + let mut msg = format!("Loading unprepared module: {specifier}"); + if let Some(referrer) = maybe_referrer { + msg = format!("{}, imported from: {}", msg, referrer.as_str()); + } + Err(anyhow!(msg)) + } + fn resolve_referrer( &self, referrer: &str, @@ -474,15 +515,11 @@ impl if self.shared.is_repl { if let Ok(reference) = NpmPackageReqReference::from_specifier(&specifier) { - return self - .shared - .node_resolver - .resolve_req_reference( - &reference, - referrer, - NodeResolutionMode::Execution, - ) - .map(|res| res.into_url()); + return self.shared.node_resolver.resolve_req_reference( + &reference, + referrer, + NodeResolutionMode::Execution, + ); } } @@ -506,13 +543,15 @@ impl .with_context(|| { format!("Could not resolve '{}'.", module.nv_reference) })? - .into_url() } Some(Module::Node(module)) => module.specifier.clone(), Some(Module::Js(module)) => module.specifier.clone(), Some(Module::Json(module)) => module.specifier.clone(), Some(Module::External(module)) => { - node::resolve_specifier_into_node_modules(&module.specifier) + node::resolve_specifier_into_node_modules( + &module.specifier, + self.shared.fs.as_ref(), + ) } None => specifier.into_owned(), }; @@ -534,7 +573,7 @@ impl }) => { let transpile_result = self .emitter - .emit_parsed_source(specifier, media_type, source) + .emit_parsed_source(specifier, media_type, ModuleKind::Esm, source) .await?; // at this point, we no longer need the parsed source in memory, so free it @@ -547,11 +586,19 @@ impl media_type, })) } + Some(CodeOrDeferredEmit::Cjs { + specifier, + media_type, + source, + }) => self + .load_maybe_cjs(specifier, media_type, source) + .await + .map(Some), None => Ok(None), } } - fn load_prepared_module_sync( + fn load_prepared_module_for_source_map_sync( &self, specifier: &ModuleSpecifier, ) -> Result, AnyError> { @@ -564,9 +611,12 @@ impl media_type, source, }) => { - let transpile_result = self - .emitter - .emit_parsed_source_sync(specifier, media_type, source)?; + let transpile_result = self.emitter.emit_parsed_source_sync( + specifier, + media_type, + ModuleKind::Esm, + source, + )?; // at this point, we no longer need the parsed source in memory, so free it self.parsed_source_cache.free(specifier); @@ -578,6 +628,14 @@ impl media_type, })) } + Some(CodeOrDeferredEmit::Cjs { .. }) => { + self.parsed_source_cache.free(specifier); + + // todo(dsherret): to make this work, we should probably just + // rely on the CJS export cache. At the moment this is hard because + // cjs export analysis is only async + Ok(None) + } None => Ok(None), } } @@ -607,20 +665,40 @@ impl source, media_type, specifier, + is_script, .. })) => { + // todo(dsherret): revert in https://github.com/denoland/deno/pull/26439 + if self.is_npm_main && *is_script + || self.shared.cjs_tracker.is_cjs_with_known_is_script( + specifier, + *media_type, + *is_script, + )? + { + return Ok(Some(CodeOrDeferredEmit::Cjs { + specifier, + media_type: *media_type, + source, + })); + } let code: ModuleCodeString = match media_type { MediaType::JavaScript | MediaType::Unknown - | MediaType::Cjs | MediaType::Mjs | MediaType::Json => source.clone().into(), MediaType::Dts | MediaType::Dcts | MediaType::Dmts => { Default::default() } + MediaType::Cjs | MediaType::Cts => { + return Ok(Some(CodeOrDeferredEmit::Cjs { + specifier, + media_type: *media_type, + source, + })); + } MediaType::TypeScript | MediaType::Mts - | MediaType::Cts | MediaType::Jsx | MediaType::Tsx => { return Ok(Some(CodeOrDeferredEmit::DeferredEmit { @@ -629,7 +707,7 @@ impl source, })); } - MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { + MediaType::Css | MediaType::Wasm | MediaType::SourceMap => { panic!("Unexpected media type {media_type} for {specifier}") } }; @@ -651,6 +729,48 @@ impl | None => Ok(None), } } + + async fn load_maybe_cjs( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + original_source: &Arc, + ) -> Result { + let js_source = if media_type.is_emittable() { + Cow::Owned( + self + .emitter + .emit_parsed_source( + specifier, + media_type, + ModuleKind::Cjs, + original_source, + ) + .await?, + ) + } else { + Cow::Borrowed(original_source.as_ref()) + }; + let text = self + .node_code_translator + .translate_cjs_to_esm(specifier, Some(js_source)) + .await?; + // at this point, we no longer need the parsed source in memory, so free it + self.parsed_source_cache.free(specifier); + Ok(ModuleCodeStringSource { + code: match text { + // perf: if the text is borrowed, that means it didn't make any changes + // to the original source, so we can just provide that instead of cloning + // the borrowed text + Cow::Borrowed(_) => { + ModuleSourceCode::String(original_source.clone().into()) + } + Cow::Owned(text) => ModuleSourceCode::String(text.into()), + }, + found_url: specifier.clone(), + media_type, + }) + } } enum CodeOrDeferredEmit<'a> { @@ -660,6 +780,11 @@ enum CodeOrDeferredEmit<'a> { media_type: MediaType, source: &'a Arc, }, + Cjs { + specifier: &'a ModuleSpecifier, + media_type: MediaType, + source: &'a Arc, + }, } // todo(dsherret): this double Rc boxing is not ideal @@ -821,7 +946,10 @@ impl ModuleLoader "wasm" | "file" | "http" | "https" | "data" | "blob" => (), _ => return None, } - let source = self.0.load_prepared_module_sync(&specifier).ok()??; + let source = self + .0 + .load_prepared_module_for_source_map_sync(&specifier) + .ok()??; source_map_from_code(source.code.as_bytes()) } @@ -900,3 +1028,79 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit { drop(self.permit); // explicit drop for clarity } } + +#[derive(Debug)] +struct CliNodeRequireLoader { + emitter: Arc, + fs: Arc, + graph_container: TGraphContainer, + in_npm_pkg_checker: Arc, + npm_resolver: Arc, +} + +impl + CliNodeRequireLoader +{ + pub fn new( + emitter: Arc, + fs: Arc, + graph_container: TGraphContainer, + in_npm_pkg_checker: Arc, + npm_resolver: Arc, + ) -> Self { + Self { + emitter, + fs, + graph_container, + in_npm_pkg_checker, + npm_resolver, + } + } +} + +impl NodeRequireLoader + for CliNodeRequireLoader +{ + fn ensure_read_permission<'a>( + &self, + permissions: &mut dyn deno_runtime::deno_node::NodePermissions, + path: &'a Path, + ) -> Result, AnyError> { + if let Ok(url) = deno_path_util::url_from_file_path(path) { + // allow reading if it's in the module graph + if self.graph_container.graph().get(&url).is_some() { + return Ok(std::borrow::Cow::Borrowed(path)); + } + } + self.npm_resolver.ensure_read_permission(permissions, path) + } + + fn load_text_file_lossy(&self, path: &Path) -> Result { + // todo(dsherret): use the preloaded module from the graph if available? + let media_type = MediaType::from_path(path); + let text = self.fs.read_text_file_lossy_sync(path, None)?; + if media_type.is_emittable() { + let specifier = deno_path_util::url_from_file_path(path)?; + if self.in_npm_pkg_checker.in_npm_package(&specifier) { + return Err( + NotSupportedKindInNpmError { + media_type, + specifier, + } + .into(), + ); + } + self.emitter.emit_parsed_source_sync( + &specifier, + media_type, + // this is probably not super accurate due to require esm, but probably ok. + // If we find this causes a lot of churn in the emit cache then we should + // investigate how we can make this better + ModuleKind::Cjs, + &text.into(), + ) + } else { + Ok(text) + } + } +} diff --git a/cli/node.rs b/cli/node.rs index 733d5f8717..1d410a726a 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -1,11 +1,14 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; use std::sync::Arc; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_graph::ParsedSourceStore; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_runtime::deno_fs; use deno_runtime::deno_node::DenoFsNodeResolverEnv; use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis; @@ -18,8 +21,8 @@ use serde::Serialize; use crate::cache::CacheDBHash; use crate::cache::NodeAnalysisCache; use crate::cache::ParsedSourceCache; -use crate::resolver::CliNodeResolver; -use crate::util::fs::canonicalize_path_maybe_not_exists; +use crate::resolver::CjsTracker; +use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; pub type CliNodeCodeTranslator = NodeCodeTranslator; @@ -32,14 +35,14 @@ pub type CliNodeCodeTranslator = /// because the node_modules folder might not exist at that time. pub fn resolve_specifier_into_node_modules( specifier: &ModuleSpecifier, + fs: &dyn deno_fs::FileSystem, ) -> ModuleSpecifier { - specifier - .to_file_path() + url_to_file_path(specifier) .ok() // this path might not exist at the time the graph is being created // because the node_modules folder might not yet exist - .and_then(|path| canonicalize_path_maybe_not_exists(&path).ok()) - .and_then(|path| ModuleSpecifier::from_file_path(path).ok()) + .and_then(|path| canonicalize_path_maybe_not_exists_with_fs(&path, fs).ok()) + .and_then(|path| url_from_file_path(&path).ok()) .unwrap_or_else(|| specifier.clone()) } @@ -56,23 +59,29 @@ pub enum CliCjsAnalysis { pub struct CliCjsCodeAnalyzer { cache: NodeAnalysisCache, + cjs_tracker: Arc, fs: deno_fs::FileSystemRc, - node_resolver: Arc, parsed_source_cache: Option>, + // todo(dsherret): hack, remove in https://github.com/denoland/deno/pull/26439 + // For example, this does not properly handle if cjs analysis was already done + // and has been cached. + is_npm_main: bool, } impl CliCjsCodeAnalyzer { pub fn new( cache: NodeAnalysisCache, + cjs_tracker: Arc, fs: deno_fs::FileSystemRc, - node_resolver: Arc, parsed_source_cache: Option>, + is_npm_main: bool, ) -> Self { Self { cache, + cjs_tracker, fs, - node_resolver, parsed_source_cache, + is_npm_main, } } @@ -88,7 +97,7 @@ impl CliCjsCodeAnalyzer { return Ok(analysis); } - let mut media_type = MediaType::from_specifier(specifier); + let media_type = MediaType::from_specifier(specifier); if media_type == MediaType::Json { return Ok(CliCjsAnalysis::Cjs { exports: vec![], @@ -96,62 +105,53 @@ impl CliCjsCodeAnalyzer { }); } - if media_type == MediaType::JavaScript { - if let Some(package_json) = - self.node_resolver.get_closest_package_json(specifier)? - { - match package_json.typ.as_str() { - "commonjs" => { - media_type = MediaType::Cjs; - } - "module" => { - media_type = MediaType::Mjs; - } - _ => {} - } - } - } + let cjs_tracker = self.cjs_tracker.clone(); + let is_npm_main = self.is_npm_main; + let is_maybe_cjs = + cjs_tracker.is_maybe_cjs(specifier, media_type)? || is_npm_main; + let analysis = if is_maybe_cjs { + let maybe_parsed_source = self + .parsed_source_cache + .as_ref() + .and_then(|c| c.remove_parsed_source(specifier)); - let maybe_parsed_source = self - .parsed_source_cache - .as_ref() - .and_then(|c| c.remove_parsed_source(specifier)); - - let analysis = deno_core::unsync::spawn_blocking({ - let specifier = specifier.clone(); - let source: Arc = source.into(); - move || -> Result<_, deno_ast::ParseDiagnostic> { - let parsed_source = - maybe_parsed_source.map(Ok).unwrap_or_else(|| { - deno_ast::parse_program(deno_ast::ParseParams { - specifier, - text: source, - media_type, - capture_tokens: true, - scope_analysis: false, - maybe_syntax: None, + deno_core::unsync::spawn_blocking({ + let specifier = specifier.clone(); + let source: Arc = source.into(); + move || -> Result<_, AnyError> { + let parsed_source = + maybe_parsed_source.map(Ok).unwrap_or_else(|| { + deno_ast::parse_program(deno_ast::ParseParams { + specifier, + text: source, + media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + }) + })?; + let is_script = parsed_source.compute_is_script(); + let is_cjs = cjs_tracker.is_cjs_with_known_is_script( + parsed_source.specifier(), + media_type, + is_script, + )? || is_script && is_npm_main; + if is_cjs { + let analysis = parsed_source.analyze_cjs(); + Ok(CliCjsAnalysis::Cjs { + exports: analysis.exports, + reexports: analysis.reexports, }) - })?; - if parsed_source.is_script() { - let analysis = parsed_source.analyze_cjs(); - Ok(CliCjsAnalysis::Cjs { - exports: analysis.exports, - reexports: analysis.reexports, - }) - } else if media_type == MediaType::Cjs { - // FIXME: `deno_ast` should internally handle MediaType::Cjs implying that - // the result must never be Esm - Ok(CliCjsAnalysis::Cjs { - exports: vec![], - reexports: vec![], - }) - } else { - Ok(CliCjsAnalysis::Esm) + } else { + Ok(CliCjsAnalysis::Esm) + } } - } - }) - .await - .unwrap()?; + }) + .await + .unwrap()? + } else { + CliCjsAnalysis::Esm + }; self .cache @@ -163,11 +163,11 @@ impl CliCjsCodeAnalyzer { #[async_trait::async_trait(?Send)] impl CjsCodeAnalyzer for CliCjsCodeAnalyzer { - async fn analyze_cjs( + async fn analyze_cjs<'a>( &self, specifier: &ModuleSpecifier, - source: Option, - ) -> Result { + source: Option>, + ) -> Result, AnyError> { let source = match source { Some(source) => source, None => { @@ -175,7 +175,7 @@ impl CjsCodeAnalyzer for CliCjsCodeAnalyzer { if let Ok(source_from_file) = self.fs.read_text_file_lossy_async(path, None).await { - source_from_file + Cow::Owned(source_from_file) } else { return Ok(ExtNodeCjsAnalysis::Cjs(CjsAnalysisExports { exports: vec![], diff --git a/cli/npm/byonm.rs b/cli/npm/byonm.rs index fc095ab16f..4535b07fc5 100644 --- a/cli/npm/byonm.rs +++ b/cli/npm/byonm.rs @@ -10,8 +10,8 @@ use deno_core::serde_json; use deno_core::url::Url; use deno_resolver::npm::ByonmNpmResolver; use deno_resolver::npm::ByonmNpmResolverCreateOptions; +use deno_runtime::deno_node::DenoFsNodeResolverEnv; use deno_runtime::deno_node::NodePermissions; -use deno_runtime::deno_node::NodeRequireResolver; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageReq; use node_resolver::NpmResolver; @@ -25,30 +25,14 @@ use super::InnerCliNpmResolverRef; use super::ResolvePkgFolderFromDenoReqError; pub type CliByonmNpmResolverCreateOptions = - ByonmNpmResolverCreateOptions; -pub type CliByonmNpmResolver = ByonmNpmResolver; + ByonmNpmResolverCreateOptions; +pub type CliByonmNpmResolver = + ByonmNpmResolver; // todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple. #[derive(Debug)] struct CliByonmWrapper(Arc); -impl NodeRequireResolver for CliByonmWrapper { - fn ensure_read_permission<'a>( - &self, - permissions: &mut dyn NodePermissions, - path: &'a Path, - ) -> Result, AnyError> { - if !path - .components() - .any(|c| c.as_os_str().to_ascii_lowercase() == "node_modules") - { - permissions.check_read_path(path) - } else { - Ok(Cow::Borrowed(path)) - } - } -} - impl NpmProcessStateProvider for CliByonmWrapper { fn get_npm_process_state(&self) -> String { serde_json::to_string(&NpmProcessState { @@ -67,10 +51,6 @@ impl CliNpmResolver for CliByonmNpmResolver { self } - fn into_require_resolver(self: Arc) -> Arc { - Arc::new(CliByonmWrapper(self)) - } - fn into_process_state_provider( self: Arc, ) -> Arc { @@ -100,6 +80,21 @@ impl CliNpmResolver for CliByonmNpmResolver { .map_err(ResolvePkgFolderFromDenoReqError::Byonm) } + fn ensure_read_permission<'a>( + &self, + permissions: &mut dyn NodePermissions, + path: &'a Path, + ) -> Result, AnyError> { + if !path + .components() + .any(|c| c.as_os_str().to_ascii_lowercase() == "node_modules") + { + permissions.check_read_path(path) + } else { + Ok(Cow::Borrowed(path)) + } + } + fn check_state_hash(&self) -> Option { // it is very difficult to determine the check state hash for byonm // so we just return None to signify check caching is not supported diff --git a/cli/npm/managed/cache/mod.rs b/cli/npm/managed/cache/mod.rs index aaec486681..8ae99f41e0 100644 --- a/cli/npm/managed/cache/mod.rs +++ b/cli/npm/managed/cache/mod.rs @@ -36,7 +36,7 @@ pub use tarball::TarballCache; /// Stores a single copy of npm packages in a cache. #[derive(Debug)] pub struct NpmCache { - cache_dir: NpmCacheDir, + cache_dir: Arc, cache_setting: CacheSetting, npmrc: Arc, /// ensures a package is only downloaded once per run @@ -45,7 +45,7 @@ pub struct NpmCache { impl NpmCache { pub fn new( - cache_dir: NpmCacheDir, + cache_dir: Arc, cache_setting: CacheSetting, npmrc: Arc, ) -> Self { @@ -61,6 +61,10 @@ impl NpmCache { &self.cache_setting } + pub fn root_dir_path(&self) -> &Path { + self.cache_dir.root_dir() + } + pub fn root_dir_url(&self) -> &Url { self.cache_dir.root_dir_url() } @@ -152,10 +156,6 @@ impl NpmCache { self.cache_dir.package_name_folder(name, registry_url) } - pub fn root_folder(&self) -> PathBuf { - self.cache_dir.root_dir().to_owned() - } - pub fn resolve_package_folder_id_from_specifier( &self, specifier: &ModuleSpecifier, diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index a0754812bb..4a91bc3474 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -12,6 +12,7 @@ use deno_cache_dir::npm::NpmCacheDir; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::serde_json; +use deno_core::url::Url; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; use deno_npm::registry::NpmRegistryApi; @@ -24,12 +25,12 @@ use deno_npm::NpmSystemInfo; use deno_runtime::colors; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::NodePermissions; -use deno_runtime::deno_node::NodeRequireResolver; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use node_resolver::errors::PackageFolderResolveError; use node_resolver::errors::PackageFolderResolveIoError; +use node_resolver::InNpmPackageChecker; use node_resolver::NpmResolver; use resolution::AddPkgReqsResult; @@ -39,7 +40,6 @@ use crate::args::NpmInstallDepsProvider; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; use crate::args::PackageJsonDepValueParseWithLocationError; -use crate::cache::DenoCacheEnvFsAdapter; use crate::cache::FastInsecureHasher; use crate::http_util::HttpClientProvider; use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; @@ -66,12 +66,12 @@ pub enum CliNpmResolverManagedSnapshotOption { Specified(Option), } -pub struct CliNpmResolverManagedCreateOptions { +pub struct CliManagedNpmResolverCreateOptions { pub snapshot: CliNpmResolverManagedSnapshotOption, pub maybe_lockfile: Option>, pub fs: Arc, pub http_client_provider: Arc, - pub npm_global_cache_dir: PathBuf, + pub npm_cache_dir: Arc, pub cache_setting: crate::args::CacheSetting, pub text_only_progress_bar: crate::util::progress_bar::ProgressBar, pub maybe_node_modules_path: Option, @@ -82,7 +82,7 @@ pub struct CliNpmResolverManagedCreateOptions { } pub async fn create_managed_npm_resolver_for_lsp( - options: CliNpmResolverManagedCreateOptions, + options: CliManagedNpmResolverCreateOptions, ) -> Arc { let npm_cache = create_cache(&options); let npm_api = create_api(&options, npm_cache.clone()); @@ -115,7 +115,7 @@ pub async fn create_managed_npm_resolver_for_lsp( } pub async fn create_managed_npm_resolver( - options: CliNpmResolverManagedCreateOptions, + options: CliManagedNpmResolverCreateOptions, ) -> Result, AnyError> { let npm_cache = create_cache(&options); let npm_api = create_api(&options, npm_cache.clone()); @@ -189,20 +189,16 @@ fn create_inner( )) } -fn create_cache(options: &CliNpmResolverManagedCreateOptions) -> Arc { +fn create_cache(options: &CliManagedNpmResolverCreateOptions) -> Arc { Arc::new(NpmCache::new( - NpmCacheDir::new( - &DenoCacheEnvFsAdapter(options.fs.as_ref()), - options.npm_global_cache_dir.clone(), - options.npmrc.get_all_known_registries_urls(), - ), + options.npm_cache_dir.clone(), options.cache_setting.clone(), options.npmrc.clone(), )) } fn create_api( - options: &CliNpmResolverManagedCreateOptions, + options: &CliManagedNpmResolverCreateOptions, npm_cache: Arc, ) -> Arc { Arc::new(CliNpmRegistryApi::new( @@ -259,6 +255,35 @@ async fn snapshot_from_lockfile( Ok(snapshot) } +#[derive(Debug)] +struct ManagedInNpmPackageChecker { + root_dir: Url, +} + +impl InNpmPackageChecker for ManagedInNpmPackageChecker { + fn in_npm_package(&self, specifier: &Url) -> bool { + specifier.as_ref().starts_with(self.root_dir.as_str()) + } +} + +pub struct CliManagedInNpmPkgCheckerCreateOptions<'a> { + pub root_cache_dir_url: &'a Url, + pub maybe_node_modules_path: Option<&'a Path>, +} + +pub fn create_managed_in_npm_pkg_checker( + options: CliManagedInNpmPkgCheckerCreateOptions, +) -> Arc { + let root_dir = match options.maybe_node_modules_path { + Some(node_modules_folder) => { + deno_path_util::url_from_directory_path(node_modules_folder).unwrap() + } + None => options.root_cache_dir_url.clone(), + }; + debug_assert!(root_dir.as_str().ends_with('/')); + Arc::new(ManagedInNpmPackageChecker { root_dir }) +} + /// An npm resolver where the resolution is managed by Deno rather than /// the user bringing their own node_modules (BYONM) on the file system. pub struct ManagedCliNpmResolver { @@ -555,8 +580,16 @@ impl ManagedCliNpmResolver { .map_err(|err| err.into()) } - pub fn global_cache_root_folder(&self) -> PathBuf { - self.npm_cache.root_folder() + pub fn maybe_node_modules_path(&self) -> Option<&Path> { + self.fs_resolver.node_modules_path() + } + + pub fn global_cache_root_path(&self) -> &Path { + self.npm_cache.root_dir_path() + } + + pub fn global_cache_root_url(&self) -> &Url { + self.npm_cache.root_dir_url() } } @@ -591,22 +624,6 @@ impl NpmResolver for ManagedCliNpmResolver { log::debug!("Resolved {} from {} to {}", name, referrer, path.display()); Ok(path) } - - fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - let root_dir_url = self.fs_resolver.root_dir_url(); - debug_assert!(root_dir_url.as_str().ends_with('/')); - specifier.as_ref().starts_with(root_dir_url.as_str()) - } -} - -impl NodeRequireResolver for ManagedCliNpmResolver { - fn ensure_read_permission<'a>( - &self, - permissions: &mut dyn NodePermissions, - path: &'a Path, - ) -> Result, AnyError> { - self.fs_resolver.ensure_read_permission(permissions, path) - } } impl NpmProcessStateProvider for ManagedCliNpmResolver { @@ -623,10 +640,6 @@ impl CliNpmResolver for ManagedCliNpmResolver { self } - fn into_require_resolver(self: Arc) -> Arc { - self - } - fn into_process_state_provider( self: Arc, ) -> Arc { @@ -687,6 +700,14 @@ impl CliNpmResolver for ManagedCliNpmResolver { .map_err(ResolvePkgFolderFromDenoReqError::Managed) } + fn ensure_read_permission<'a>( + &self, + permissions: &mut dyn NodePermissions, + path: &'a Path, + ) -> Result, AnyError> { + self.fs_resolver.ensure_read_permission(permissions, path) + } + fn check_state_hash(&self) -> Option { // We could go further and check all the individual // npm packages, but that's probably overkill. diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs index 867bb4168a..590f8fb25c 100644 --- a/cli/npm/managed/resolvers/common.rs +++ b/cli/npm/managed/resolvers/common.rs @@ -17,7 +17,6 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures; use deno_core::futures::StreamExt; -use deno_core::url::Url; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; @@ -30,9 +29,6 @@ use crate::npm::managed::cache::TarballCache; /// Part of the resolution that interacts with the file system. #[async_trait(?Send)] pub trait NpmPackageFsResolver: Send + Sync { - /// Specifier for the root directory. - fn root_dir_url(&self) -> &Url; - /// The local node_modules folder if it is applicable to the implementation. fn node_modules_path(&self) -> Option<&Path>; diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs index 5be315e992..f0193e78e9 100644 --- a/cli/npm/managed/resolvers/global.rs +++ b/cli/npm/managed/resolvers/global.rs @@ -11,7 +11,6 @@ use crate::colors; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; -use deno_core::url::Url; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; @@ -56,7 +55,7 @@ impl GlobalNpmPackageResolver { Self { registry_read_permission_checker: RegistryReadPermissionChecker::new( fs, - cache.root_folder(), + cache.root_dir_path().to_path_buf(), ), cache, tarball_cache, @@ -69,10 +68,6 @@ impl GlobalNpmPackageResolver { #[async_trait(?Send)] impl NpmPackageFsResolver for GlobalNpmPackageResolver { - fn root_dir_url(&self) -> &Url { - self.cache.root_dir_url() - } - fn node_modules_path(&self) -> Option<&Path> { None } diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index 54f7576ade..0968be8a7e 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -155,10 +155,6 @@ impl LocalNpmPackageResolver { #[async_trait(?Send)] impl NpmPackageFsResolver for LocalNpmPackageResolver { - fn root_dir_url(&self) -> &Url { - &self.root_node_modules_url - } - fn node_modules_path(&self) -> Option<&Path> { Some(self.root_node_modules_path.as_ref()) } diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 3971b1a0b3..0d434ca27f 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -4,6 +4,7 @@ mod byonm; mod common; mod managed; +use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -15,13 +16,16 @@ use deno_core::error::AnyError; use deno_core::serde_json; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; +use deno_resolver::npm::ByonmInNpmPackageChecker; use deno_resolver::npm::ByonmNpmResolver; use deno_resolver::npm::ByonmResolvePkgFolderFromDenoReqError; -use deno_runtime::deno_node::NodeRequireResolver; +use deno_runtime::deno_node::NodePermissions; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use managed::cache::registry_info::get_package_url; +use managed::create_managed_in_npm_pkg_checker; +use node_resolver::InNpmPackageChecker; use node_resolver::NpmResolver; use thiserror::Error; @@ -29,7 +33,8 @@ use crate::file_fetcher::FileFetcher; pub use self::byonm::CliByonmNpmResolver; pub use self::byonm::CliByonmNpmResolverCreateOptions; -pub use self::managed::CliNpmResolverManagedCreateOptions; +pub use self::managed::CliManagedInNpmPkgCheckerCreateOptions; +pub use self::managed::CliManagedNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::ManagedCliNpmResolver; @@ -42,7 +47,7 @@ pub enum ResolvePkgFolderFromDenoReqError { } pub enum CliNpmResolverCreateOptions { - Managed(CliNpmResolverManagedCreateOptions), + Managed(CliManagedNpmResolverCreateOptions), Byonm(CliByonmNpmResolverCreateOptions), } @@ -68,6 +73,22 @@ pub async fn create_cli_npm_resolver( } } +pub enum CreateInNpmPkgCheckerOptions<'a> { + Managed(CliManagedInNpmPkgCheckerCreateOptions<'a>), + Byonm, +} + +pub fn create_in_npm_pkg_checker( + options: CreateInNpmPkgCheckerOptions, +) -> Arc { + match options { + CreateInNpmPkgCheckerOptions::Managed(options) => { + create_managed_in_npm_pkg_checker(options) + } + CreateInNpmPkgCheckerOptions::Byonm => Arc::new(ByonmInNpmPackageChecker), + } +} + pub enum InnerCliNpmResolverRef<'a> { Managed(&'a ManagedCliNpmResolver), #[allow(dead_code)] @@ -76,7 +97,6 @@ pub enum InnerCliNpmResolverRef<'a> { pub trait CliNpmResolver: NpmResolver { fn into_npm_resolver(self: Arc) -> Arc; - fn into_require_resolver(self: Arc) -> Arc; fn into_process_state_provider( self: Arc, ) -> Arc; @@ -107,6 +127,12 @@ pub trait CliNpmResolver: NpmResolver { referrer: &ModuleSpecifier, ) -> Result; + fn ensure_read_permission<'a>( + &self, + permissions: &mut dyn NodePermissions, + path: &'a Path, + ) -> Result, AnyError>; + /// Returns a hash returning the state of the npm resolver /// or `None` if the state currently can't be determined. fn check_state_hash(&self) -> Option; diff --git a/cli/resolver.rs b/cli/resolver.rs index 84c671268a..710b975093 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use dashmap::DashMap; use dashmap::DashSet; use deno_ast::MediaType; +use deno_ast::ModuleKind; use deno_config::workspace::MappedResolution; use deno_config::workspace::MappedResolutionDiagnostic; use deno_config::workspace::MappedResolutionError; @@ -11,6 +12,7 @@ use deno_config::workspace::WorkspaceResolver; use deno_core::anyhow::anyhow; use deno_core::anyhow::Context; use deno_core::error::AnyError; +use deno_core::url::Url; use deno_core::ModuleSourceCode; use deno_core::ModuleSpecifier; use deno_graph::source::ResolutionMode; @@ -29,6 +31,7 @@ use deno_runtime::deno_fs; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::is_builtin_node_module; use deno_runtime::deno_node::NodeResolver; +use deno_runtime::deno_node::PackageJsonResolver; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageReq; use node_resolver::errors::ClosestPkgJsonError; @@ -38,21 +41,22 @@ use node_resolver::errors::PackageFolderResolveErrorKind; use node_resolver::errors::PackageFolderResolveIoError; use node_resolver::errors::PackageNotFoundError; use node_resolver::errors::PackageResolveErrorKind; -use node_resolver::errors::UrlToNodeResolutionError; +use node_resolver::errors::PackageSubpathResolveError; +use node_resolver::InNpmPackageChecker; use node_resolver::NodeModuleKind; use node_resolver::NodeResolution; use node_resolver::NodeResolutionMode; -use node_resolver::PackageJson; +use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use thiserror::Error; use crate::args::JsxImportSourceConfig; use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; use crate::node::CliNodeCodeTranslator; use crate::npm::CliNpmResolver; use crate::npm::InnerCliNpmResolverRef; -use crate::util::path::specifier_has_extension; use crate::util::sync::AtomicFlag; use crate::util::text_encoding::from_utf8_lossy_owned; @@ -104,36 +108,32 @@ impl deno_resolver::fs::DenoResolverFs for CliDenoResolverFs { #[derive(Debug)] pub struct CliNodeResolver { - cjs_resolutions: Arc, + cjs_tracker: Arc, fs: Arc, + in_npm_pkg_checker: Arc, node_resolver: Arc, npm_resolver: Arc, } impl CliNodeResolver { pub fn new( - cjs_resolutions: Arc, + cjs_tracker: Arc, fs: Arc, + in_npm_pkg_checker: Arc, node_resolver: Arc, npm_resolver: Arc, ) -> Self { Self { - cjs_resolutions, + cjs_tracker, fs, + in_npm_pkg_checker, node_resolver, npm_resolver, } } pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - self.npm_resolver.in_npm_package(specifier) - } - - pub fn get_closest_package_json( - &self, - referrer: &ModuleSpecifier, - ) -> Result>, ClosestPkgJsonError> { - self.node_resolver.get_closest_package_json(referrer) + self.in_npm_pkg_checker.in_npm_package(specifier) } pub fn resolve_if_for_npm_pkg( @@ -153,8 +153,7 @@ impl CliNodeResolver { | NodeResolveErrorKind::UnsupportedEsmUrlScheme(_) | NodeResolveErrorKind::DataUrlReferrer(_) | NodeResolveErrorKind::TypesNotFound(_) - | NodeResolveErrorKind::FinalizeResolution(_) - | NodeResolveErrorKind::UrlToNodeResolution(_) => Err(err.into()), + | NodeResolveErrorKind::FinalizeResolution(_) => Err(err.into()), NodeResolveErrorKind::PackageResolve(err) => { let err = err.into_kind(); match err { @@ -216,7 +215,11 @@ impl CliNodeResolver { referrer: &ModuleSpecifier, mode: NodeResolutionMode, ) -> Result { - let referrer_kind = if self.cjs_resolutions.is_known_cjs(referrer) { + let referrer_kind = if self + .cjs_tracker + .is_maybe_cjs(referrer, MediaType::from_specifier(referrer)) + .map_err(|err| NodeResolveErrorKind::PackageResolve(err.into()))? + { NodeModuleKind::Cjs } else { NodeModuleKind::Esm @@ -226,7 +229,7 @@ impl CliNodeResolver { self .node_resolver .resolve(specifier, referrer, referrer_kind, mode)?; - Ok(self.handle_node_resolution(res)) + Ok(res) } pub fn resolve_req_reference( @@ -234,7 +237,7 @@ impl CliNodeResolver { req_ref: &NpmPackageReqReference, referrer: &ModuleSpecifier, mode: NodeResolutionMode, - ) -> Result { + ) -> Result { self.resolve_req_with_sub_path( req_ref.req(), req_ref.sub_path(), @@ -249,7 +252,7 @@ impl CliNodeResolver { sub_path: Option<&str>, referrer: &ModuleSpecifier, mode: NodeResolutionMode, - ) -> Result { + ) -> Result { let package_folder = self .npm_resolver .resolve_pkg_folder_from_deno_module_req(req, referrer)?; @@ -260,7 +263,7 @@ impl CliNodeResolver { mode, ); match resolution_result { - Ok(resolution) => Ok(resolution), + Ok(url) => Ok(url), Err(err) => { if self.npm_resolver.as_byonm().is_some() { let package_json_path = package_folder.join("package.json"); @@ -271,7 +274,7 @@ impl CliNodeResolver { )); } } - Err(err) + Err(err.into()) } } } @@ -282,16 +285,13 @@ impl CliNodeResolver { sub_path: Option<&str>, maybe_referrer: Option<&ModuleSpecifier>, mode: NodeResolutionMode, - ) -> Result { - let res = self - .node_resolver - .resolve_package_subpath_from_deno_module( - package_folder, - sub_path, - maybe_referrer, - mode, - )?; - Ok(self.handle_node_resolution(res)) + ) -> Result { + self.node_resolver.resolve_package_subpath_from_deno_module( + package_folder, + sub_path, + maybe_referrer, + mode, + ) } pub fn handle_if_in_node_modules( @@ -306,71 +306,45 @@ impl CliNodeResolver { // so canoncalize then check if it's in the node_modules directory. // If so, check if we need to store this specifier as being a CJS // resolution. - let specifier = - crate::node::resolve_specifier_into_node_modules(specifier); - if self.in_npm_package(&specifier) { - let resolution = - self.node_resolver.url_to_node_resolution(specifier)?; - let resolution = self.handle_node_resolution(resolution); - return Ok(Some(resolution.into_url())); - } + let specifier = crate::node::resolve_specifier_into_node_modules( + specifier, + self.fs.as_ref(), + ); + return Ok(Some(specifier)); } Ok(None) } - - pub fn url_to_node_resolution( - &self, - specifier: ModuleSpecifier, - ) -> Result { - self.node_resolver.url_to_node_resolution(specifier) - } - - fn handle_node_resolution( - &self, - resolution: NodeResolution, - ) -> NodeResolution { - if let NodeResolution::CommonJs(specifier) = &resolution { - // remember that this was a common js resolution - self.mark_cjs_resolution(specifier.clone()); - } - resolution - } - - pub fn mark_cjs_resolution(&self, specifier: ModuleSpecifier) { - self.cjs_resolutions.insert(specifier); - } } -// todo(dsherret): move to module_loader.rs +#[derive(Debug, Error)] +#[error("{media_type} files are not supported in npm packages: {specifier}")] +pub struct NotSupportedKindInNpmError { + pub media_type: MediaType, + pub specifier: Url, +} + +// todo(dsherret): move to module_loader.rs (it seems to be here due to use in standalone) #[derive(Clone)] pub struct NpmModuleLoader { - cjs_resolutions: Arc, - node_code_translator: Arc, + cjs_tracker: Arc, fs: Arc, - node_resolver: Arc, + node_code_translator: Arc, } impl NpmModuleLoader { pub fn new( - cjs_resolutions: Arc, - node_code_translator: Arc, + cjs_tracker: Arc, fs: Arc, - node_resolver: Arc, + node_code_translator: Arc, ) -> Self { Self { - cjs_resolutions, + cjs_tracker, node_code_translator, fs, - node_resolver, } } - pub fn if_in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - self.node_resolver.in_npm_package(specifier) - || self.cjs_resolutions.is_known_cjs(specifier) - } - pub async fn load( &self, specifier: &ModuleSpecifier, @@ -413,20 +387,30 @@ impl NpmModuleLoader { } })?; - let code = if self.cjs_resolutions.is_known_cjs(specifier) { + let media_type = MediaType::from_specifier(specifier); + if media_type.is_emittable() { + return Err(AnyError::from(NotSupportedKindInNpmError { + media_type, + specifier: specifier.clone(), + })); + } + + let code = if self.cjs_tracker.is_maybe_cjs(specifier, media_type)? { // translate cjs to esm if it's cjs and inject node globals let code = from_utf8_lossy_owned(code); ModuleSourceCode::String( self .node_code_translator - .translate_cjs_to_esm(specifier, Some(code)) + .translate_cjs_to_esm(specifier, Some(Cow::Owned(code))) .await? + .into_owned() .into(), ) } else { // esm and json code is untouched ModuleSourceCode::Bytes(code.into_boxed_slice().into()) }; + Ok(ModuleCodeStringSource { code, found_url: specifier.clone(), @@ -435,21 +419,165 @@ impl NpmModuleLoader { } } +pub struct CjsTrackerOptions { + pub unstable_detect_cjs: bool, +} + /// Keeps track of what module specifiers were resolved as CJS. -#[derive(Debug, Default)] -pub struct CjsResolutionStore(DashSet); +/// +/// Modules that are `.js` or `.ts` are only known to be CJS or +/// ESM after they're loaded based on their contents. So these files +/// will be "maybe CJS" until they're loaded. +#[derive(Debug)] +pub struct CjsTracker { + in_npm_pkg_checker: Arc, + pkg_json_resolver: Arc, + unstable_detect_cjs: bool, + known: DashMap, +} -impl CjsResolutionStore { - pub fn is_known_cjs(&self, specifier: &ModuleSpecifier) -> bool { - if specifier.scheme() != "file" { - return false; +impl CjsTracker { + pub fn new( + in_npm_pkg_checker: Arc, + pkg_json_resolver: Arc, + options: CjsTrackerOptions, + ) -> Self { + Self { + in_npm_pkg_checker, + pkg_json_resolver, + unstable_detect_cjs: options.unstable_detect_cjs, + known: Default::default(), } - - specifier_has_extension(specifier, "cjs") || self.0.contains(specifier) } - pub fn insert(&self, specifier: ModuleSpecifier) { - self.0.insert(specifier); + /// Checks whether the file might be treated as CJS, but it's not for sure + /// yet because the source hasn't been loaded to see whether it contains + /// imports or exports. + pub fn is_maybe_cjs( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + ) -> Result { + self.treat_as_cjs_with_is_script(specifier, media_type, None) + } + + /// Gets whether the file is CJS. If true, this is for sure + /// cjs because `is_script` is provided. + /// + /// `is_script` should be `true` when the contents of the file at the + /// provided specifier are known to be a script and not an ES module. + pub fn is_cjs_with_known_is_script( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + is_script: bool, + ) -> Result { + self.treat_as_cjs_with_is_script(specifier, media_type, Some(is_script)) + } + + fn treat_as_cjs_with_is_script( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + is_script: Option, + ) -> Result { + let kind = match self + .get_known_kind_with_is_script(specifier, media_type, is_script) + { + Some(kind) => kind, + None => self.check_based_on_pkg_json(specifier)?, + }; + Ok(kind.is_cjs()) + } + + pub fn get_known_kind( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + ) -> Option { + self.get_known_kind_with_is_script(specifier, media_type, None) + } + + fn get_known_kind_with_is_script( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + is_script: Option, + ) -> Option { + if specifier.scheme() != "file" { + return Some(ModuleKind::Esm); + } + + match media_type { + MediaType::Mts | MediaType::Mjs | MediaType::Dmts => Some(ModuleKind::Esm), + MediaType::Cjs | MediaType::Cts | MediaType::Dcts => Some(ModuleKind::Cjs), + MediaType::Dts => { + // dts files are always determined based on the package.json because + // they contain imports/exports even when considered CJS + if let Some(value) = self.known.get(specifier).map(|v| *v) { + Some(value) + } else { + let value = self.check_based_on_pkg_json(specifier).ok(); + if let Some(value) = value { + self.known.insert(specifier.clone(), value); + } + Some(value.unwrap_or(ModuleKind::Esm)) + } + } + MediaType::Wasm | + MediaType::Json => Some(ModuleKind::Esm), + MediaType::JavaScript + | MediaType::Jsx + | MediaType::TypeScript + | MediaType::Tsx + // treat these as unknown + | MediaType::Css + | MediaType::SourceMap + | MediaType::Unknown => { + if let Some(value) = self.known.get(specifier).map(|v| *v) { + if value.is_cjs() && is_script == Some(false) { + // we now know this is actually esm + self.known.insert(specifier.clone(), ModuleKind::Esm); + Some(ModuleKind::Esm) + } else { + Some(value) + } + } else if is_script == Some(false) { + // we know this is esm + self.known.insert(specifier.clone(), ModuleKind::Esm); + Some(ModuleKind::Esm) + } else { + None + } + } + } + } + + fn check_based_on_pkg_json( + &self, + specifier: &ModuleSpecifier, + ) -> Result { + if self.in_npm_pkg_checker.in_npm_package(specifier) { + if let Some(pkg_json) = + self.pkg_json_resolver.get_closest_package_json(specifier)? + { + let is_file_location_cjs = pkg_json.typ != "module"; + Ok(ModuleKind::from_is_cjs(is_file_location_cjs)) + } else { + Ok(ModuleKind::Cjs) + } + } else if self.unstable_detect_cjs { + if let Some(pkg_json) = + self.pkg_json_resolver.get_closest_package_json(specifier)? + { + let is_cjs_type = pkg_json.typ == "commonjs"; + Ok(ModuleKind::from_is_cjs(is_cjs_type)) + } else { + Ok(ModuleKind::Esm) + } + } else { + Ok(ModuleKind::Esm) + } } } @@ -633,8 +761,7 @@ impl Resolver for CliGraphResolver { Some(referrer), to_node_mode(mode), ) - .map_err(ResolveError::Other) - .map(|res| res.into_url()), + .map_err(|e| ResolveError::Other(e.into())), MappedResolution::PackageJson { dep_result, alias, @@ -665,19 +792,17 @@ impl Resolver for CliGraphResolver { ) .map_err(|e| ResolveError::Other(e.into())) .and_then(|pkg_folder| { - Ok( - self - .node_resolver - .as_ref() - .unwrap() - .resolve_package_sub_path_from_deno_module( - pkg_folder, - sub_path.as_deref(), - Some(referrer), - to_node_mode(mode), - )? - .into_url(), - ) + self + .node_resolver + .as_ref() + .unwrap() + .resolve_package_sub_path_from_deno_module( + pkg_folder, + sub_path.as_deref(), + Some(referrer), + to_node_mode(mode), + ) + .map_err(|e| ResolveError::Other(e.into())) }), }) } @@ -717,23 +842,20 @@ impl Resolver for CliGraphResolver { npm_req_ref.req(), ) { - return Ok( - node_resolver - .resolve_package_sub_path_from_deno_module( - pkg_folder, - npm_req_ref.sub_path(), - Some(referrer), - to_node_mode(mode), - )? - .into_url(), - ); + return node_resolver + .resolve_package_sub_path_from_deno_module( + pkg_folder, + npm_req_ref.sub_path(), + Some(referrer), + to_node_mode(mode), + ) + .map_err(|e| ResolveError::Other(e.into())); } // do npm resolution for byonm if is_byonm { return node_resolver .resolve_req_reference(&npm_req_ref, referrer, to_node_mode(mode)) - .map(|res| res.into_url()) .map_err(|err| err.into()); } } @@ -751,9 +873,7 @@ impl Resolver for CliGraphResolver { .map_err(ResolveError::Other)?; if let Some(res) = maybe_resolution { match res { - NodeResolution::Esm(url) | NodeResolution::CommonJs(url) => { - return Ok(url) - } + NodeResolution::Module(url) => return Ok(url), NodeResolution::BuiltIn(_) => { // don't resolve bare specifiers for built-in modules via node resolution } diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index f41f3003ff..9e26512268 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -21,6 +21,7 @@ use std::process::Command; use std::sync::Arc; use deno_ast::MediaType; +use deno_ast::ModuleKind; use deno_ast::ModuleSpecifier; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::ResolverWorkspaceJsrPackage; @@ -67,6 +68,7 @@ use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClientProvider; use crate::npm::CliNpmResolver; use crate::npm::InnerCliNpmResolverRef; +use crate::resolver::CjsTracker; use crate::shared::ReleaseChannel; use crate::standalone::virtual_fs::VfsEntry; use crate::util::archive; @@ -257,6 +259,10 @@ impl StandaloneModules { } } + pub fn has_file(&self, path: &Path) -> bool { + self.vfs.file_entry(path).is_ok() + } + pub fn read<'a>( &'a self, specifier: &'a ModuleSpecifier, @@ -353,6 +359,7 @@ pub fn extract_standalone( } pub struct DenoCompileBinaryWriter<'a> { + cjs_tracker: &'a CjsTracker, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a FileFetcher, @@ -365,6 +372,7 @@ pub struct DenoCompileBinaryWriter<'a> { impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( + cjs_tracker: &'a CjsTracker, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a FileFetcher, @@ -374,6 +382,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { npm_system_info: NpmSystemInfo, ) -> Self { Self { + cjs_tracker, deno_dir, emitter, file_fetcher, @@ -599,19 +608,21 @@ impl<'a> DenoCompileBinaryWriter<'a> { } let (maybe_source, media_type) = match module { deno_graph::Module::Js(m) => { - // todo(https://github.com/denoland/deno_media_type/pull/12): use is_emittable() - let is_emittable = matches!( - m.media_type, - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx - ); - let source = if is_emittable { + let source = if m.media_type.is_emittable() { + let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script( + &m.specifier, + m.media_type, + m.is_script, + )?; + let module_kind = ModuleKind::from_is_cjs(is_cjs); let source = self .emitter - .emit_parsed_source(&m.specifier, m.media_type, &m.source) + .emit_parsed_source( + &m.specifier, + m.media_type, + module_kind, + &m.source, + ) .await?; source.into_bytes() } else { @@ -745,8 +756,9 @@ impl<'a> DenoCompileBinaryWriter<'a> { } else { // DO NOT include the user's registry url as it may contain credentials, // but also don't make this dependent on the registry url - let global_cache_root_path = npm_resolver.global_cache_root_folder(); - let mut builder = VfsBuilder::new(global_cache_root_path)?; + let global_cache_root_path = npm_resolver.global_cache_root_path(); + let mut builder = + VfsBuilder::new(global_cache_root_path.to_path_buf())?; let mut packages = npm_resolver.all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 3a62b6ff96..85610f4c20 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -19,6 +19,7 @@ use deno_core::error::type_error; use deno_core::error::AnyError; use deno_core::futures::FutureExt; use deno_core::v8_set_flags; +use deno_core::FastString; use deno_core::FeatureChecker; use deno_core::ModuleLoader; use deno_core::ModuleSourceCode; @@ -30,7 +31,9 @@ use deno_npm::npm_rc::ResolvedNpmRc; use deno_package_json::PackageJsonDepValue; use deno_runtime::deno_fs; use deno_runtime::deno_node::create_host_defined_options; +use deno_runtime::deno_node::NodeRequireLoader; use deno_runtime::deno_node::NodeResolver; +use deno_runtime::deno_node::PackageJsonResolver; use deno_runtime::deno_permissions::Permissions; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_tls::rustls::RootCertStore; @@ -43,6 +46,7 @@ use deno_semver::npm::NpmPackageReqReference; use import_map::parse_from_json; use node_resolver::analyze::NodeCodeTranslator; use node_resolver::NodeResolutionMode; +use serialization::DenoCompileModuleSource; use std::borrow::Cow; use std::rc::Rc; use std::sync::Arc; @@ -61,12 +65,18 @@ use crate::cache::NodeAnalysisCache; use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClientProvider; use crate::node::CliCjsCodeAnalyzer; +use crate::node::CliNodeCodeTranslator; use crate::npm::create_cli_npm_resolver; +use crate::npm::create_in_npm_pkg_checker; use crate::npm::CliByonmNpmResolverCreateOptions; +use crate::npm::CliManagedInNpmPkgCheckerCreateOptions; +use crate::npm::CliManagedNpmResolverCreateOptions; +use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolverCreateOptions; -use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; -use crate::resolver::CjsResolutionStore; +use crate::npm::CreateInNpmPkgCheckerOptions; +use crate::resolver::CjsTracker; +use crate::resolver::CjsTrackerOptions; use crate::resolver::CliDenoResolverFs; use crate::resolver::CliNodeResolver; use crate::resolver::NpmModuleLoader; @@ -75,7 +85,7 @@ use crate::util::progress_bar::ProgressBarStyle; use crate::util::v8::construct_v8_flags; use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerOptions; -use crate::worker::ModuleLoaderAndSourceMapGetter; +use crate::worker::CreateModuleLoaderResult; use crate::worker::ModuleLoaderFactory; pub mod binary; @@ -91,10 +101,14 @@ use self::binary::Metadata; use self::file_system::DenoCompileFileSystem; struct SharedModuleLoaderState { + cjs_tracker: Arc, + fs: Arc, modules: StandaloneModules, - workspace_resolver: WorkspaceResolver, + node_code_translator: Arc, node_resolver: Arc, npm_module_loader: Arc, + npm_resolver: Arc, + workspace_resolver: WorkspaceResolver, } #[derive(Clone)] @@ -102,6 +116,12 @@ struct EmbeddedModuleLoader { shared: Arc, } +impl std::fmt::Debug for EmbeddedModuleLoader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EmbeddedModuleLoader").finish() + } +} + pub const MODULE_NOT_FOUND: &str = "Module not found"; pub const UNSUPPORTED_SCHEME: &str = "Unsupported scheme"; @@ -159,8 +179,7 @@ impl ModuleLoader for EmbeddedModuleLoader { sub_path.as_deref(), Some(&referrer), NodeResolutionMode::Execution, - )? - .into_url(), + )?, ), Ok(MappedResolution::PackageJson { dep_result, @@ -168,16 +187,14 @@ impl ModuleLoader for EmbeddedModuleLoader { alias, .. }) => match dep_result.as_ref().map_err(|e| AnyError::from(e.clone()))? { - PackageJsonDepValue::Req(req) => self - .shared - .node_resolver - .resolve_req_with_sub_path( + PackageJsonDepValue::Req(req) => { + self.shared.node_resolver.resolve_req_with_sub_path( req, sub_path.as_deref(), &referrer, NodeResolutionMode::Execution, ) - .map(|res| res.into_url()), + } PackageJsonDepValue::Workspace(version_req) => { let pkg_folder = self .shared @@ -195,8 +212,7 @@ impl ModuleLoader for EmbeddedModuleLoader { sub_path.as_deref(), Some(&referrer), NodeResolutionMode::Execution, - )? - .into_url(), + )?, ) } }, @@ -205,15 +221,11 @@ impl ModuleLoader for EmbeddedModuleLoader { if let Ok(reference) = NpmPackageReqReference::from_specifier(&specifier) { - return self - .shared - .node_resolver - .resolve_req_reference( - &reference, - &referrer, - NodeResolutionMode::Execution, - ) - .map(|res| res.into_url()); + return self.shared.node_resolver.resolve_req_reference( + &reference, + &referrer, + NodeResolutionMode::Execution, + ); } if specifier.scheme() == "jsr" { @@ -317,17 +329,72 @@ impl ModuleLoader for EmbeddedModuleLoader { match self.shared.modules.read(original_specifier) { Ok(Some(module)) => { + let media_type = module.media_type; let (module_specifier, module_type, module_source) = - module.into_for_v8(); - deno_core::ModuleLoadResponse::Sync(Ok( - deno_core::ModuleSource::new_with_redirect( - module_type, - module_source, - original_specifier, - module_specifier, - None, - ), - )) + module.into_parts(); + let is_maybe_cjs = match self + .shared + .cjs_tracker + .is_maybe_cjs(original_specifier, media_type) + { + Ok(is_maybe_cjs) => is_maybe_cjs, + Err(err) => { + return deno_core::ModuleLoadResponse::Sync(Err(type_error( + format!("{:?}", err), + ))); + } + }; + if is_maybe_cjs { + let original_specifier = original_specifier.clone(); + let module_specifier = module_specifier.clone(); + let shared = self.shared.clone(); + deno_core::ModuleLoadResponse::Async( + async move { + let source = match module_source { + DenoCompileModuleSource::String(string) => { + Cow::Borrowed(string) + } + DenoCompileModuleSource::Bytes(module_code_bytes) => { + match module_code_bytes { + Cow::Owned(bytes) => Cow::Owned( + crate::util::text_encoding::from_utf8_lossy_owned(bytes), + ), + Cow::Borrowed(bytes) => String::from_utf8_lossy(bytes), + } + } + }; + let source = shared + .node_code_translator + .translate_cjs_to_esm(&module_specifier, Some(source)) + .await?; + let module_source = match source { + Cow::Owned(source) => ModuleSourceCode::String(source.into()), + Cow::Borrowed(source) => { + ModuleSourceCode::String(FastString::from_static(source)) + } + }; + Ok(deno_core::ModuleSource::new_with_redirect( + module_type, + module_source, + &original_specifier, + &module_specifier, + None, + )) + } + .boxed_local(), + ) + } else { + let module_source = module_source.into_for_v8(); + deno_core::ModuleLoadResponse::Sync(Ok( + deno_core::ModuleSource::new_with_redirect( + module_type, + module_source, + original_specifier, + module_specifier, + None, + ), + )) + } } Ok(None) => deno_core::ModuleLoadResponse::Sync(Err(type_error( format!("{MODULE_NOT_FOUND}: {}", original_specifier), @@ -339,32 +406,61 @@ impl ModuleLoader for EmbeddedModuleLoader { } } +impl NodeRequireLoader for EmbeddedModuleLoader { + fn ensure_read_permission<'a>( + &self, + permissions: &mut dyn deno_runtime::deno_node::NodePermissions, + path: &'a std::path::Path, + ) -> Result, AnyError> { + if self.shared.modules.has_file(path) { + // allow reading if the file is in the snapshot + return Ok(Cow::Borrowed(path)); + } + + self + .shared + .npm_resolver + .ensure_read_permission(permissions, path) + } + + fn load_text_file_lossy( + &self, + path: &std::path::Path, + ) -> Result { + Ok(self.shared.fs.read_text_file_lossy_sync(path, None)?) + } +} + struct StandaloneModuleLoaderFactory { shared: Arc, } +impl StandaloneModuleLoaderFactory { + pub fn create_result(&self) -> CreateModuleLoaderResult { + let loader = Rc::new(EmbeddedModuleLoader { + shared: self.shared.clone(), + }); + CreateModuleLoaderResult { + module_loader: loader.clone(), + node_require_loader: loader, + } + } +} + impl ModuleLoaderFactory for StandaloneModuleLoaderFactory { fn create_for_main( &self, _root_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { - ModuleLoaderAndSourceMapGetter { - module_loader: Rc::new(EmbeddedModuleLoader { - shared: self.shared.clone(), - }), - } + ) -> CreateModuleLoaderResult { + self.create_result() } fn create_for_worker( &self, _parent_permissions: PermissionsContainer, _permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { - ModuleLoaderAndSourceMapGetter { - module_loader: Rc::new(EmbeddedModuleLoader { - shared: self.shared.clone(), - }), - } + ) -> CreateModuleLoaderResult { + self.create_result() } } @@ -410,106 +506,155 @@ pub async fn run(data: StandaloneData) -> Result { let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap(); let npm_global_cache_dir = root_path.join(".deno_compile_node_modules"); let cache_setting = CacheSetting::Only; - let npm_resolver = match metadata.node_modules { + let pkg_json_resolver = Arc::new(PackageJsonResolver::new( + deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), + )); + let (in_npm_pkg_checker, npm_resolver) = match metadata.node_modules { Some(binary::NodeModules::Managed { node_modules_dir }) => { + // create an npmrc that uses the fake npm_registry_url to resolve packages + let npmrc = Arc::new(ResolvedNpmRc { + default_config: deno_npm::npm_rc::RegistryConfigWithUrl { + registry_url: npm_registry_url.clone(), + config: Default::default(), + }, + scopes: Default::default(), + registry_configs: Default::default(), + }); + let npm_cache_dir = Arc::new(NpmCacheDir::new( + &DenoCacheEnvFsAdapter(fs.as_ref()), + npm_global_cache_dir, + npmrc.get_all_known_registries_urls(), + )); let snapshot = npm_snapshot.unwrap(); let maybe_node_modules_path = node_modules_dir .map(|node_modules_dir| root_path.join(node_modules_dir)); - create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( - CliNpmResolverManagedCreateOptions { - snapshot: CliNpmResolverManagedSnapshotOption::Specified(Some( - snapshot, - )), - maybe_lockfile: None, - fs: fs.clone(), - http_client_provider: http_client_provider.clone(), - npm_global_cache_dir, - cache_setting, - text_only_progress_bar: progress_bar, - maybe_node_modules_path, - npm_system_info: Default::default(), - npm_install_deps_provider: Arc::new( - // this is only used for installing packages, which isn't necessary with deno compile - NpmInstallDepsProvider::empty(), - ), - // create an npmrc that uses the fake npm_registry_url to resolve packages - npmrc: Arc::new(ResolvedNpmRc { - default_config: deno_npm::npm_rc::RegistryConfigWithUrl { - registry_url: npm_registry_url.clone(), - config: Default::default(), - }, - scopes: Default::default(), - registry_configs: Default::default(), - }), - lifecycle_scripts: Default::default(), - }, - )) - .await? + let in_npm_pkg_checker = + create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed( + CliManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: npm_cache_dir.root_dir_url(), + maybe_node_modules_path: maybe_node_modules_path.as_deref(), + }, + )); + let npm_resolver = + create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( + CliManagedNpmResolverCreateOptions { + snapshot: CliNpmResolverManagedSnapshotOption::Specified(Some( + snapshot, + )), + maybe_lockfile: None, + fs: fs.clone(), + http_client_provider: http_client_provider.clone(), + npm_cache_dir, + cache_setting, + text_only_progress_bar: progress_bar, + maybe_node_modules_path, + npm_system_info: Default::default(), + npm_install_deps_provider: Arc::new( + // this is only used for installing packages, which isn't necessary with deno compile + NpmInstallDepsProvider::empty(), + ), + npmrc, + lifecycle_scripts: Default::default(), + }, + )) + .await?; + (in_npm_pkg_checker, npm_resolver) } Some(binary::NodeModules::Byonm { root_node_modules_dir, }) => { let root_node_modules_dir = root_node_modules_dir.map(|p| vfs.root().join(p)); - create_cli_npm_resolver(CliNpmResolverCreateOptions::Byonm( - CliByonmNpmResolverCreateOptions { + let in_npm_pkg_checker = + create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Byonm); + let npm_resolver = create_cli_npm_resolver( + CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { fs: CliDenoResolverFs(fs.clone()), + pkg_json_resolver: pkg_json_resolver.clone(), root_node_modules_dir, - }, - )) - .await? + }), + ) + .await?; + (in_npm_pkg_checker, npm_resolver) } None => { - create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( - CliNpmResolverManagedCreateOptions { - snapshot: CliNpmResolverManagedSnapshotOption::Specified(None), - maybe_lockfile: None, - fs: fs.clone(), - http_client_provider: http_client_provider.clone(), - npm_global_cache_dir, - cache_setting, - text_only_progress_bar: progress_bar, - maybe_node_modules_path: None, - npm_system_info: Default::default(), - npm_install_deps_provider: Arc::new( - // this is only used for installing packages, which isn't necessary with deno compile - NpmInstallDepsProvider::empty(), - ), - // Packages from different registries are already inlined in the binary, - // so no need to create actual `.npmrc` configuration. - npmrc: create_default_npmrc(), - lifecycle_scripts: Default::default(), - }, - )) - .await? + // Packages from different registries are already inlined in the binary, + // so no need to create actual `.npmrc` configuration. + let npmrc = create_default_npmrc(); + let npm_cache_dir = Arc::new(NpmCacheDir::new( + &DenoCacheEnvFsAdapter(fs.as_ref()), + npm_global_cache_dir, + npmrc.get_all_known_registries_urls(), + )); + let in_npm_pkg_checker = + create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed( + CliManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: npm_cache_dir.root_dir_url(), + maybe_node_modules_path: None, + }, + )); + let npm_resolver = + create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( + CliManagedNpmResolverCreateOptions { + snapshot: CliNpmResolverManagedSnapshotOption::Specified(None), + maybe_lockfile: None, + fs: fs.clone(), + http_client_provider: http_client_provider.clone(), + npm_cache_dir, + cache_setting, + text_only_progress_bar: progress_bar, + maybe_node_modules_path: None, + npm_system_info: Default::default(), + npm_install_deps_provider: Arc::new( + // this is only used for installing packages, which isn't necessary with deno compile + NpmInstallDepsProvider::empty(), + ), + npmrc: create_default_npmrc(), + lifecycle_scripts: Default::default(), + }, + )) + .await?; + (in_npm_pkg_checker, npm_resolver) } }; let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some(); let node_resolver = Arc::new(NodeResolver::new( deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), + in_npm_pkg_checker.clone(), npm_resolver.clone().into_npm_resolver(), + pkg_json_resolver.clone(), + )); + let cjs_tracker = Arc::new(CjsTracker::new( + in_npm_pkg_checker.clone(), + pkg_json_resolver.clone(), + CjsTrackerOptions { + unstable_detect_cjs: metadata.unstable_config.detect_cjs, + }, )); - let cjs_resolutions = Arc::new(CjsResolutionStore::default()); let cache_db = Caches::new(deno_dir_provider.clone()); let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db()); let cli_node_resolver = Arc::new(CliNodeResolver::new( - cjs_resolutions.clone(), + cjs_tracker.clone(), fs.clone(), + in_npm_pkg_checker.clone(), node_resolver.clone(), npm_resolver.clone(), )); let cjs_esm_code_analyzer = CliCjsCodeAnalyzer::new( node_analysis_cache, + cjs_tracker.clone(), fs.clone(), - cli_node_resolver.clone(), None, + false, ); let node_code_translator = Arc::new(NodeCodeTranslator::new( cjs_esm_code_analyzer, deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), + in_npm_pkg_checker, node_resolver.clone(), npm_resolver.clone().into_npm_resolver(), + pkg_json_resolver.clone(), )); let workspace_resolver = { let import_map = match metadata.workspace_resolver.import_map { @@ -562,15 +707,18 @@ pub async fn run(data: StandaloneData) -> Result { }; let module_loader_factory = StandaloneModuleLoaderFactory { shared: Arc::new(SharedModuleLoaderState { + cjs_tracker: cjs_tracker.clone(), + fs: fs.clone(), modules, - workspace_resolver, + node_code_translator: node_code_translator.clone(), node_resolver: cli_node_resolver.clone(), npm_module_loader: Arc::new(NpmModuleLoader::new( - cjs_resolutions.clone(), - node_code_translator, + cjs_tracker.clone(), fs.clone(), - cli_node_resolver, + node_code_translator, )), + npm_resolver: npm_resolver.clone(), + workspace_resolver, }), }; @@ -609,7 +757,6 @@ pub async fn run(data: StandaloneData) -> Result { }); let worker_factory = CliMainWorkerFactory::new( Arc::new(BlobStore::default()), - cjs_resolutions, // Code cache is not supported for standalone binary yet. None, feature_checker, @@ -620,6 +767,7 @@ pub async fn run(data: StandaloneData) -> Result { Box::new(module_loader_factory), node_resolver, npm_resolver, + pkg_json_resolver, root_cert_store_provider, permissions, StorageKeyResolver::empty(), @@ -635,7 +783,6 @@ pub async fn run(data: StandaloneData) -> Result { inspect_wait: false, strace_ops: None, is_inspecting: false, - is_npm_main: main_module.scheme() == "npm", skip_op_registration: true, location: metadata.location, argv0: NpmPackageReqReference::from_specifier(&main_module) @@ -652,7 +799,6 @@ pub async fn run(data: StandaloneData) -> Result { node_ipc: None, serve_port: None, serve_host: None, - unstable_detect_cjs: metadata.unstable_config.detect_cjs, }, ); diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index 7b63c584e7..a5eb649bfd 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -214,14 +214,13 @@ impl RemoteModulesStoreBuilder { } } -pub struct DenoCompileModuleData<'a> { - pub specifier: &'a Url, - pub media_type: MediaType, - pub data: Cow<'static, [u8]>, +pub enum DenoCompileModuleSource { + String(&'static str), + Bytes(Cow<'static, [u8]>), } -impl<'a> DenoCompileModuleData<'a> { - pub fn into_for_v8(self) -> (&'a Url, ModuleType, ModuleSourceCode) { +impl DenoCompileModuleSource { + pub fn into_for_v8(self) -> ModuleSourceCode { fn into_bytes(data: Cow<'static, [u8]>) -> ModuleSourceCode { ModuleSourceCode::Bytes(match data { Cow::Borrowed(d) => d.into(), @@ -229,16 +228,31 @@ impl<'a> DenoCompileModuleData<'a> { }) } - fn into_string_unsafe(data: Cow<'static, [u8]>) -> ModuleSourceCode { + match self { // todo(https://github.com/denoland/deno_core/pull/943): store whether // the string is ascii or not ahead of time so we can avoid the is_ascii() // check in FastString::from_static + Self::String(s) => ModuleSourceCode::String(FastString::from_static(s)), + Self::Bytes(b) => into_bytes(b), + } + } +} + +pub struct DenoCompileModuleData<'a> { + pub specifier: &'a Url, + pub media_type: MediaType, + pub data: Cow<'static, [u8]>, +} + +impl<'a> DenoCompileModuleData<'a> { + pub fn into_parts(self) -> (&'a Url, ModuleType, DenoCompileModuleSource) { + fn into_string_unsafe(data: Cow<'static, [u8]>) -> DenoCompileModuleSource { match data { - Cow::Borrowed(d) => ModuleSourceCode::String( + Cow::Borrowed(d) => DenoCompileModuleSource::String( // SAFETY: we know this is a valid utf8 string - unsafe { FastString::from_static(std::str::from_utf8_unchecked(d)) }, + unsafe { std::str::from_utf8_unchecked(d) }, ), - Cow::Owned(d) => ModuleSourceCode::Bytes(d.into_boxed_slice().into()), + Cow::Owned(d) => DenoCompileModuleSource::Bytes(Cow::Owned(d)), } } @@ -257,11 +271,14 @@ impl<'a> DenoCompileModuleData<'a> { (ModuleType::JavaScript, into_string_unsafe(self.data)) } MediaType::Json => (ModuleType::Json, into_string_unsafe(self.data)), - MediaType::Wasm => (ModuleType::Wasm, into_bytes(self.data)), - // just assume javascript if we made it here - MediaType::TsBuildInfo | MediaType::SourceMap | MediaType::Unknown => { - (ModuleType::JavaScript, into_bytes(self.data)) + MediaType::Wasm => { + (ModuleType::Wasm, DenoCompileModuleSource::Bytes(self.data)) } + // just assume javascript if we made it here + MediaType::Css | MediaType::SourceMap | MediaType::Unknown => ( + ModuleType::JavaScript, + DenoCompileModuleSource::Bytes(self.data), + ), }; (self.specifier, media_type, source) } @@ -551,7 +568,7 @@ fn serialize_media_type(media_type: MediaType) -> u8 { MediaType::Tsx => 10, MediaType::Json => 11, MediaType::Wasm => 12, - MediaType::TsBuildInfo => 13, + MediaType::Css => 13, MediaType::SourceMap => 14, MediaType::Unknown => 15, } @@ -572,7 +589,7 @@ fn deserialize_media_type(value: u8) -> Result { 10 => Ok(MediaType::Tsx), 11 => Ok(MediaType::Json), 12 => Ok(MediaType::Wasm), - 13 => Ok(MediaType::TsBuildInfo), + 13 => Ok(MediaType::Css), 14 => Ok(MediaType::SourceMap), 15 => Ok(MediaType::Unknown), _ => bail!("Unknown media type value: {}", value), diff --git a/cli/tools/bench/mod.rs b/cli/tools/bench/mod.rs index be5d0ad0e1..272d063355 100644 --- a/cli/tools/bench/mod.rs +++ b/cli/tools/bench/mod.rs @@ -193,7 +193,7 @@ async fn bench_specifier_inner( .await?; // We execute the main module as a side module so that import.meta.main is not set. - worker.execute_side_module_possibly_with_npm().await?; + worker.execute_side_module().await?; let mut worker = worker.into_main_worker(); diff --git a/cli/tools/check.rs b/cli/tools/check.rs index 7edb392d48..d880278884 100644 --- a/cli/tools/check.rs +++ b/cli/tools/check.rs @@ -32,6 +32,7 @@ use crate::graph_util::ModuleGraphBuilder; use crate::npm::CliNpmResolver; use crate::tsc; use crate::tsc::Diagnostics; +use crate::tsc::TypeCheckingCjsTracker; use crate::util::extract; use crate::util::path::to_percent_decoded_str; @@ -99,6 +100,7 @@ pub struct CheckOptions { pub struct TypeChecker { caches: Arc, + cjs_tracker: Arc, cli_options: Arc, module_graph_builder: Arc, node_resolver: Arc, @@ -108,6 +110,7 @@ pub struct TypeChecker { impl TypeChecker { pub fn new( caches: Arc, + cjs_tracker: Arc, cli_options: Arc, module_graph_builder: Arc, node_resolver: Arc, @@ -115,6 +118,7 @@ impl TypeChecker { ) -> Self { Self { caches, + cjs_tracker, cli_options, module_graph_builder, node_resolver, @@ -244,6 +248,7 @@ impl TypeChecker { graph: graph.clone(), hash_data, maybe_npm: Some(tsc::RequestNpmState { + cjs_tracker: self.cjs_tracker.clone(), node_resolver: self.node_resolver.clone(), npm_resolver: self.npm_resolver.clone(), }), @@ -346,7 +351,7 @@ fn get_check_hash( } } MediaType::Json - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Wasm | MediaType::Unknown => continue, @@ -428,7 +433,7 @@ fn get_tsc_roots( } MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Unknown => None, }, @@ -536,7 +541,7 @@ fn has_ts_check(media_type: MediaType, file_text: &str) -> bool { | MediaType::Tsx | MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Unknown => false, } diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index 5a4a938bb9..b3e9993379 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -53,16 +53,6 @@ pub async fn compile( ); } - if cli_options.unstable_detect_cjs() { - log::warn!( - concat!( - "{} --unstable-detect-cjs is not properly supported in deno compile. ", - "The compiled executable may encounter runtime errors.", - ), - crate::colors::yellow("Warning"), - ); - } - let output_path = resolve_compile_executable_output_path( http_client, &compile_flags, diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 48922f1449..f593332475 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -6,12 +6,12 @@ use crate::args::FileFlags; use crate::args::Flags; use crate::cdp; use crate::factory::CliFactory; -use crate::npm::CliNpmResolver; use crate::tools::fmt::format_json; use crate::tools::test::is_supported_test_path; use crate::util::text_encoding::source_map_from_code; use deno_ast::MediaType; +use deno_ast::ModuleKind; use deno_ast::ModuleSpecifier; use deno_config::glob::FileCollector; use deno_config::glob::FilePatterns; @@ -25,6 +25,7 @@ use deno_core::serde_json; use deno_core::sourcemap::SourceMap; use deno_core::url::Url; use deno_core::LocalInspectorSession; +use node_resolver::InNpmPackageChecker; use regex::Regex; use std::fs; use std::fs::File; @@ -461,7 +462,7 @@ fn filter_coverages( coverages: Vec, include: Vec, exclude: Vec, - npm_resolver: &dyn CliNpmResolver, + in_npm_pkg_checker: &dyn InNpmPackageChecker, ) -> Vec { let include: Vec = include.iter().map(|e| Regex::new(e).unwrap()).collect(); @@ -485,7 +486,7 @@ fn filter_coverages( || doc_test_re.is_match(e.url.as_str()) || Url::parse(&e.url) .ok() - .map(|url| npm_resolver.in_npm_package(&url)) + .map(|url| in_npm_pkg_checker.in_npm_package(&url)) .unwrap_or(false); let is_included = include.iter().any(|p| p.is_match(&e.url)); @@ -496,7 +497,7 @@ fn filter_coverages( .collect::>() } -pub async fn cover_files( +pub fn cover_files( flags: Arc, coverage_flags: CoverageFlags, ) -> Result<(), AnyError> { @@ -506,9 +507,10 @@ pub async fn cover_files( let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; - let npm_resolver = factory.npm_resolver().await?; + let in_npm_pkg_checker = factory.in_npm_pkg_checker()?; let file_fetcher = factory.file_fetcher()?; let emitter = factory.emitter()?; + let cjs_tracker = factory.cjs_tracker()?; assert!(!coverage_flags.files.include.is_empty()); @@ -528,7 +530,7 @@ pub async fn cover_files( script_coverages, coverage_flags.include, coverage_flags.exclude, - npm_resolver.as_ref(), + in_npm_pkg_checker.as_ref(), ); if script_coverages.is_empty() { return Err(generic_error("No covered files included in the report")); @@ -585,6 +587,8 @@ pub async fn cover_files( let transpiled_code = match file.media_type { MediaType::JavaScript | MediaType::Unknown + | MediaType::Css + | MediaType::Wasm | MediaType::Cjs | MediaType::Mjs | MediaType::Json => None, @@ -594,7 +598,10 @@ pub async fn cover_files( | MediaType::Mts | MediaType::Cts | MediaType::Tsx => { - Some(match emitter.maybe_cached_emit(&file.specifier, &file.source) { + let module_kind = ModuleKind::from_is_cjs( + cjs_tracker.is_maybe_cjs(&file.specifier, file.media_type)?, + ); + Some(match emitter.maybe_cached_emit(&file.specifier, module_kind, &file.source) { Some(code) => code, None => { return Err(anyhow!( @@ -605,7 +612,7 @@ pub async fn cover_files( } }) } - MediaType::Wasm | MediaType::TsBuildInfo | MediaType::SourceMap => { + MediaType::SourceMap => { unreachable!() } }; diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index 5e18546a28..e33da4efb2 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -22,9 +22,9 @@ use deno_core::serde_json; use deno_doc as doc; use deno_doc::html::UrlResolveKind; use deno_graph::source::NullFileSystem; +use deno_graph::EsParser; use deno_graph::GraphKind; use deno_graph::ModuleAnalyzer; -use deno_graph::ModuleParser; use deno_graph::ModuleSpecifier; use doc::html::ShortPath; use doc::DocDiagnostic; @@ -37,7 +37,7 @@ const JSON_SCHEMA_VERSION: u8 = 1; async fn generate_doc_nodes_for_builtin_types( doc_flags: DocFlags, - parser: &dyn ModuleParser, + parser: &dyn EsParser, analyzer: &dyn ModuleAnalyzer, ) -> Result>, AnyError> { let source_file_specifier = @@ -96,7 +96,7 @@ pub async fn doc( let module_info_cache = factory.module_info_cache()?; let parsed_source_cache = factory.parsed_source_cache(); let capturing_parser = parsed_source_cache.as_capturing_parser(); - let analyzer = module_info_cache.as_module_analyzer(parsed_source_cache); + let analyzer = module_info_cache.as_module_analyzer(); let doc_nodes_by_url = match doc_flags.source_files { DocSourceFileFlag::Builtin => { diff --git a/cli/tools/registry/tar.rs b/cli/tools/registry/tar.rs index aca125e00b..6d1801ce69 100644 --- a/cli/tools/registry/tar.rs +++ b/cli/tools/registry/tar.rs @@ -120,7 +120,7 @@ fn resolve_content_maybe_unfurling( | MediaType::Unknown | MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo => { + | MediaType::Css => { // not unfurlable data return Ok(data); } diff --git a/cli/tools/repl/session.rs b/cli/tools/repl/session.rs index 484664dae4..23b0f11ac5 100644 --- a/cli/tools/repl/session.rs +++ b/cli/tools/repl/session.rs @@ -25,6 +25,7 @@ use deno_ast::swc::visit::noop_visit_type; use deno_ast::swc::visit::Visit; use deno_ast::swc::visit::VisitWith; use deno_ast::ImportsNotUsedAsValues; +use deno_ast::ModuleKind; use deno_ast::ModuleSpecifier; use deno_ast::ParseDiagnosticsError; use deno_ast::ParsedSource; @@ -641,6 +642,10 @@ impl ReplSession { jsx_fragment_factory: self.jsx.frag_factory.clone(), jsx_import_source: self.jsx.import_source.clone(), var_decl_imports: true, + verbatim_module_syntax: false, + }, + &deno_ast::TranspileModuleOptions { + module_kind: Some(ModuleKind::Esm), }, &deno_ast::EmitOptions { source_map: deno_ast::SourceMapOption::None, @@ -651,7 +656,6 @@ impl ReplSession { }, )? .into_source() - .into_string()? .text; let value = self diff --git a/cli/tools/run/hmr.rs b/cli/tools/run/hmr.rs index 6ccf8e344b..6cebedd012 100644 --- a/cli/tools/run/hmr.rs +++ b/cli/tools/run/hmr.rs @@ -1,9 +1,11 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use crate::cdp; -use crate::emit::Emitter; -use crate::util::file_watcher::WatcherCommunicator; -use crate::util::file_watcher::WatcherRestartMode; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_ast::MediaType; +use deno_ast::ModuleKind; use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures::StreamExt; @@ -12,11 +14,14 @@ use deno_core::serde_json::{self}; use deno_core::url::Url; use deno_core::LocalInspectorSession; use deno_terminal::colors; -use std::collections::HashMap; -use std::path::PathBuf; -use std::sync::Arc; use tokio::select; +use crate::cdp; +use crate::emit::Emitter; +use crate::resolver::CjsTracker; +use crate::util::file_watcher::WatcherCommunicator; +use crate::util::file_watcher::WatcherRestartMode; + fn explain(status: &cdp::Status) -> &'static str { match status { cdp::Status::Ok => "OK", @@ -58,6 +63,7 @@ pub struct HmrRunner { session: LocalInspectorSession, watcher_communicator: Arc, script_ids: HashMap, + cjs_tracker: Arc, emitter: Arc, } @@ -139,7 +145,8 @@ impl crate::worker::HmrRunner for HmrRunner { }; let source_code = self.emitter.load_and_emit_for_hmr( - &module_url + &module_url, + ModuleKind::from_is_cjs(self.cjs_tracker.is_maybe_cjs(&module_url, MediaType::from_specifier(&module_url))?), ).await?; let mut tries = 1; @@ -172,12 +179,14 @@ impl crate::worker::HmrRunner for HmrRunner { impl HmrRunner { pub fn new( + cjs_tracker: Arc, emitter: Arc, session: LocalInspectorSession, watcher_communicator: Arc, ) -> Self { Self { session, + cjs_tracker, emitter, watcher_communicator, script_ids: HashMap::new(), diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs index e81abad0b2..fa849614fa 100644 --- a/cli/tools/test/mod.rs +++ b/cli/tools/test/mod.rs @@ -631,7 +631,7 @@ async fn configure_main_worker( "Deno[Deno.internal].core.setLeakTracingEnabled(true);", )?; } - let res = worker.execute_side_module_possibly_with_npm().await; + let res = worker.execute_side_module().await; let mut worker = worker.into_main_worker(); match res { Ok(()) => Ok(()), diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index 6011dece76..bdc4340e33 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -845,6 +845,8 @@ delete Object.prototype.__proto__; jqueryMessage, "Cannot_find_name_0_Do_you_need_to_install_type_definitions_for_jQuery_Try_npm_i_save_dev_types_Slash_2592": jqueryMessage, + "Module_0_was_resolved_to_1_but_allowArbitraryExtensions_is_not_set_6263": + "Module '{0}' was resolved to '{1}', but importing these modules is not supported.", }; })()); diff --git a/cli/tsc/diagnostics.rs b/cli/tsc/diagnostics.rs index b0394ec177..d3795706eb 100644 --- a/cli/tsc/diagnostics.rs +++ b/cli/tsc/diagnostics.rs @@ -323,7 +323,7 @@ impl Diagnostics { // todo(dsherret): use a short lived cache to prevent parsing // source maps so often if let Ok(source_map) = - SourceMap::from_slice(&fast_check_module.source_map) + SourceMap::from_slice(fast_check_module.source_map.as_bytes()) { if let Some(start) = d.start.as_mut() { let maybe_token = source_map diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index de91889b62..dc7fc38f7a 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -3,9 +3,11 @@ use crate::args::TsConfig; use crate::args::TypeCheckMode; use crate::cache::FastInsecureHasher; +use crate::cache::ModuleInfoCache; use crate::node; use crate::npm::CliNpmResolver; use crate::npm::ResolvePkgFolderFromDenoReqError; +use crate::resolver::CjsTracker; use crate::util::checksum; use crate::util::path::mapped_specifier_for_tsc; @@ -32,13 +34,13 @@ use deno_graph::GraphKind; use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::ResolutionResolved; +use deno_runtime::deno_fs; use deno_runtime::deno_node::NodeResolver; use deno_semver::npm::NpmPackageReqReference; use node_resolver::errors::NodeJsErrorCode; use node_resolver::errors::NodeJsErrorCoded; -use node_resolver::errors::ResolvePkgSubpathFromDenoModuleError; +use node_resolver::errors::PackageSubpathResolveError; use node_resolver::NodeModuleKind; -use node_resolver::NodeResolution; use node_resolver::NodeResolutionMode; use once_cell::sync::Lazy; use std::borrow::Cow; @@ -302,8 +304,76 @@ pub struct EmittedFile { pub media_type: MediaType, } +pub fn into_specifier_and_media_type( + specifier: Option, +) -> (ModuleSpecifier, MediaType) { + match specifier { + Some(specifier) => { + let media_type = MediaType::from_specifier(&specifier); + + (specifier, media_type) + } + None => ( + Url::parse("internal:///missing_dependency.d.ts").unwrap(), + MediaType::Dts, + ), + } +} + +#[derive(Debug)] +pub struct TypeCheckingCjsTracker { + cjs_tracker: Arc, + module_info_cache: Arc, +} + +impl TypeCheckingCjsTracker { + pub fn new( + cjs_tracker: Arc, + module_info_cache: Arc, + ) -> Self { + Self { + cjs_tracker, + module_info_cache, + } + } + + pub fn is_cjs( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + code: &Arc, + ) -> bool { + if let Some(module_kind) = + self.cjs_tracker.get_known_kind(specifier, media_type) + { + module_kind.is_cjs() + } else { + let maybe_is_script = self + .module_info_cache + .as_module_analyzer() + .analyze_sync(specifier, media_type, code) + .ok() + .map(|info| info.is_script); + maybe_is_script + .and_then(|is_script| { + self + .cjs_tracker + .is_cjs_with_known_is_script(specifier, media_type, is_script) + .ok() + }) + .unwrap_or_else(|| { + self + .cjs_tracker + .is_maybe_cjs(specifier, media_type) + .unwrap_or(false) + }) + } + } +} + #[derive(Debug)] pub struct RequestNpmState { + pub cjs_tracker: Arc, pub node_resolver: Arc, pub npm_resolver: Arc, } @@ -456,7 +526,7 @@ pub fn as_ts_script_kind(media_type: MediaType) -> i32 { MediaType::Tsx => 4, MediaType::Json => 6, MediaType::SourceMap - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::Wasm | MediaType::Unknown => 0, } @@ -489,25 +559,22 @@ fn op_load_inner( ) -> Result, AnyError> { fn load_from_node_modules( specifier: &ModuleSpecifier, - node_resolver: Option<&NodeResolver>, + npm_state: Option<&RequestNpmState>, media_type: &mut MediaType, is_cjs: &mut bool, ) -> Result { *media_type = MediaType::from_specifier(specifier); - *is_cjs = node_resolver - .map(|node_resolver| { - match node_resolver.url_to_node_resolution(specifier.clone()) { - Ok(NodeResolution::CommonJs(_)) => true, - Ok(NodeResolution::Esm(_)) - | Ok(NodeResolution::BuiltIn(_)) - | Err(_) => false, - } - }) - .unwrap_or(false); let file_path = specifier.to_file_path().unwrap(); let code = std::fs::read_to_string(&file_path) .with_context(|| format!("Unable to load {}", file_path.display()))?; - Ok(code) + let code: Arc = code.into(); + *is_cjs = npm_state + .map(|npm_state| { + npm_state.cjs_tracker.is_cjs(specifier, *media_type, &code) + }) + .unwrap_or(false); + // todo(dsherret): how to avoid cloning here? + Ok(code.to_string()) } let state = state.borrow_mut::(); @@ -560,6 +627,9 @@ fn op_load_inner( match module { Module::Js(module) => { media_type = module.media_type; + if matches!(media_type, MediaType::Cjs | MediaType::Cts) { + is_cjs = true; + } let source = module .fast_check_module() .map(|m| &*m.source) @@ -573,11 +643,13 @@ fn op_load_inner( Module::Npm(_) | Module::Node(_) => None, Module::External(module) => { // means it's Deno code importing an npm module - let specifier = - node::resolve_specifier_into_node_modules(&module.specifier); + let specifier = node::resolve_specifier_into_node_modules( + &module.specifier, + &deno_fs::RealFs, + ); Some(Cow::Owned(load_from_node_modules( &specifier, - state.maybe_npm.as_ref().map(|n| n.node_resolver.as_ref()), + state.maybe_npm.as_ref(), &mut media_type, &mut is_cjs, )?)) @@ -590,7 +662,7 @@ fn op_load_inner( { Some(Cow::Owned(load_from_node_modules( specifier, - Some(npm.node_resolver.as_ref()), + Some(npm), &mut media_type, &mut is_cjs, )?)) @@ -739,7 +811,13 @@ fn op_resolve_inner( } } }; - (specifier_str, media_type.as_ts_extension()) + ( + specifier_str, + match media_type { + MediaType::Css => ".js", // surface these as .js for typescript + media_type => media_type.as_ts_extension(), + }, + ) } None => ( MISSING_DEPENDENCY_SPECIFIER.to_string(), @@ -810,29 +888,27 @@ fn resolve_graph_specifier_types( Some(referrer), NodeResolutionMode::Types, ); - let maybe_resolution = match res_result { - Ok(res) => Some(res), + let maybe_url = match res_result { + Ok(url) => Some(url), Err(err) => match err.code() { NodeJsErrorCode::ERR_TYPES_NOT_FOUND | NodeJsErrorCode::ERR_MODULE_NOT_FOUND => None, _ => return Err(err.into()), }, }; - Ok(Some(NodeResolution::into_specifier_and_media_type( - maybe_resolution, - ))) + Ok(Some(into_specifier_and_media_type(maybe_url))) } else { Ok(None) } } Some(Module::External(module)) => { // we currently only use "External" for when the module is in an npm package - Ok(state.maybe_npm.as_ref().map(|npm| { - let specifier = - node::resolve_specifier_into_node_modules(&module.specifier); - NodeResolution::into_specifier_and_media_type( - npm.node_resolver.url_to_node_resolution(specifier).ok(), - ) + Ok(state.maybe_npm.as_ref().map(|_| { + let specifier = node::resolve_specifier_into_node_modules( + &module.specifier, + &deno_fs::RealFs, + ); + into_specifier_and_media_type(Some(specifier)) })) } Some(Module::Node(_)) | None => Ok(None), @@ -844,7 +920,7 @@ enum ResolveNonGraphSpecifierTypesError { #[error(transparent)] ResolvePkgFolderFromDenoReq(#[from] ResolvePkgFolderFromDenoReqError), #[error(transparent)] - ResolvePkgSubpathFromDenoModule(#[from] ResolvePkgSubpathFromDenoModuleError), + PackageSubpathResolve(#[from] PackageSubpathResolveError), } fn resolve_non_graph_specifier_types( @@ -863,7 +939,7 @@ fn resolve_non_graph_specifier_types( let node_resolver = &npm.node_resolver; if node_resolver.in_npm_package(referrer) { // we're in an npm package, so use node resolution - Ok(Some(NodeResolution::into_specifier_and_media_type( + Ok(Some(into_specifier_and_media_type( node_resolver .resolve( raw_specifier, @@ -871,7 +947,8 @@ fn resolve_non_graph_specifier_types( referrer_kind, NodeResolutionMode::Types, ) - .ok(), + .ok() + .map(|res| res.into_url()), ))) } else if let Ok(npm_req_ref) = NpmPackageReqReference::from_str(raw_specifier) @@ -890,17 +967,15 @@ fn resolve_non_graph_specifier_types( Some(referrer), NodeResolutionMode::Types, ); - let maybe_resolution = match res_result { - Ok(res) => Some(res), + let maybe_url = match res_result { + Ok(url) => Some(url), Err(err) => match err.code() { NodeJsErrorCode::ERR_TYPES_NOT_FOUND | NodeJsErrorCode::ERR_MODULE_NOT_FOUND => None, _ => return Err(err.into()), }, }; - Ok(Some(NodeResolution::into_specifier_and_media_type( - maybe_resolution, - ))) + Ok(Some(into_specifier_and_media_type(maybe_url))) } else { Ok(None) } diff --git a/cli/util/extract.rs b/cli/util/extract.rs index 873b7e7f2d..f577cbefec 100644 --- a/cli/util/extract.rs +++ b/cli/util/extract.rs @@ -64,7 +64,7 @@ fn extract_inner( }) { Ok(parsed) => { let mut c = ExportCollector::default(); - c.visit_program(parsed.program_ref()); + c.visit_program(parsed.program().as_ref()); c } Err(_) => ExportCollector::default(), @@ -570,14 +570,14 @@ fn generate_pseudo_file( })?; let top_level_atoms = swc_utils::collect_decls_with_ctxt::( - parsed.program_ref(), + &parsed.program_ref(), parsed.top_level_context(), ); let transformed = parsed .program_ref() - .clone() + .to_owned() .fold_with(&mut as_folder(Transform { specifier: &file.specifier, base_file_specifier, @@ -1416,7 +1416,7 @@ console.log(Foo); }) .unwrap(); - collector.visit_program(parsed.program_ref()); + parsed.program_ref().visit_with(&mut collector); collector } diff --git a/cli/util/logger.rs b/cli/util/logger.rs index cdc89411fe..d93753dfd3 100644 --- a/cli/util/logger.rs +++ b/cli/util/logger.rs @@ -65,6 +65,8 @@ pub fn init(maybe_level: Option) { .filter_module("swc_ecma_parser", log::LevelFilter::Error) // Suppress span lifecycle logs since they are too verbose .filter_module("tracing::span", log::LevelFilter::Off) + // for deno_compile, this is too verbose + .filter_module("editpe", log::LevelFilter::Error) .format(|buf, record| { let mut target = record.target().to_string(); if let Some(line_no) = record.line() { diff --git a/cli/util/path.rs b/cli/util/path.rs index e4ae6e7cb1..58bed664f9 100644 --- a/cli/util/path.rs +++ b/cli/util/path.rs @@ -42,21 +42,6 @@ pub fn get_extension(file_path: &Path) -> Option { .map(|e| e.to_lowercase()); } -pub fn specifier_has_extension( - specifier: &ModuleSpecifier, - searching_ext: &str, -) -> bool { - let Some((_, ext)) = specifier.path().rsplit_once('.') else { - return false; - }; - let searching_ext = searching_ext.strip_prefix('.').unwrap_or(searching_ext); - debug_assert!(!searching_ext.contains('.')); // exts like .d.ts are not implemented here - if ext.len() != searching_ext.len() { - return false; - } - ext.eq_ignore_ascii_case(searching_ext) -} - pub fn get_atomic_dir_path(file_path: &Path) -> PathBuf { let rand = gen_rand_path_component(); let new_file_name = format!( @@ -350,18 +335,6 @@ mod test { } } - #[test] - fn test_specifier_has_extension() { - fn get(specifier: &str, ext: &str) -> bool { - specifier_has_extension(&ModuleSpecifier::parse(specifier).unwrap(), ext) - } - - assert!(get("file:///a/b/c.ts", "ts")); - assert!(get("file:///a/b/c.ts", ".ts")); - assert!(!get("file:///a/b/c.ts", ".cts")); - assert!(get("file:///a/b/c.CtS", ".cts")); - } - #[test] fn test_to_percent_decoded_str() { let str = to_percent_decoded_str("%F0%9F%A6%95"); diff --git a/cli/util/text_encoding.rs b/cli/util/text_encoding.rs index df72cc2be6..8524e63ebb 100644 --- a/cli/util/text_encoding.rs +++ b/cli/util/text_encoding.rs @@ -97,6 +97,7 @@ fn find_source_map_range(code: &[u8]) -> Option> { } /// Converts an `Arc` to an `Arc<[u8]>`. +#[allow(dead_code)] pub fn arc_str_to_bytes(arc_str: Arc) -> Arc<[u8]> { let raw = Arc::into_raw(arc_str); // SAFETY: This is safe because they have the same memory layout. diff --git a/cli/worker.rs b/cli/worker.rs index e230197d2b..baacd681a1 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -14,16 +14,17 @@ use deno_core::v8; use deno_core::CompiledWasmModuleStore; use deno_core::Extension; use deno_core::FeatureChecker; -use deno_core::ModuleId; use deno_core::ModuleLoader; use deno_core::PollEventLoopOptions; use deno_core::SharedArrayBufferStore; use deno_runtime::code_cache; use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; use deno_runtime::deno_fs; -use deno_runtime::deno_node; use deno_runtime::deno_node::NodeExtInitServices; +use deno_runtime::deno_node::NodeRequireLoader; +use deno_runtime::deno_node::NodeRequireLoaderRc; use deno_runtime::deno_node::NodeResolver; +use deno_runtime::deno_node::PackageJsonResolver; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_tls::RootCertStoreProvider; use deno_runtime::deno_web::BlobStore; @@ -42,7 +43,6 @@ use deno_runtime::WorkerExecutionMode; use deno_runtime::WorkerLogLevel; use deno_semver::npm::NpmPackageReqReference; use deno_terminal::colors; -use node_resolver::NodeResolution; use node_resolver::NodeResolutionMode; use tokio::select; @@ -51,28 +51,27 @@ use crate::args::DenoSubcommand; use crate::args::StorageKeyResolver; use crate::errors; use crate::npm::CliNpmResolver; -use crate::resolver::CjsResolutionStore; use crate::util::checksum; use crate::util::file_watcher::WatcherCommunicator; use crate::util::file_watcher::WatcherRestartMode; -use crate::util::path::specifier_has_extension; use crate::version; -pub struct ModuleLoaderAndSourceMapGetter { +pub struct CreateModuleLoaderResult { pub module_loader: Rc, + pub node_require_loader: Rc, } pub trait ModuleLoaderFactory: Send + Sync { fn create_for_main( &self, root_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter; + ) -> CreateModuleLoaderResult; fn create_for_worker( &self, parent_permissions: PermissionsContainer, permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter; + ) -> CreateModuleLoaderResult; } #[async_trait::async_trait(?Send)] @@ -109,7 +108,6 @@ pub struct CliMainWorkerOptions { pub inspect_wait: bool, pub strace_ops: Option>, pub is_inspecting: bool, - pub is_npm_main: bool, pub location: Option, pub argv0: Option, pub node_debug: Option, @@ -122,13 +120,11 @@ pub struct CliMainWorkerOptions { pub node_ipc: Option, pub serve_port: Option, pub serve_host: Option, - pub unstable_detect_cjs: bool, } struct SharedWorkerState { blob_store: Arc, broadcast_channel: InMemoryBroadcastChannel, - cjs_resolution_store: Arc, code_cache: Option>, compiled_wasm_module_store: CompiledWasmModuleStore, feature_checker: Arc, @@ -139,6 +135,7 @@ struct SharedWorkerState { module_loader_factory: Box, node_resolver: Arc, npm_resolver: Arc, + pkg_json_resolver: Arc, root_cert_store_provider: Arc, root_permissions: PermissionsContainer, shared_array_buffer_store: SharedArrayBufferStore, @@ -148,11 +145,15 @@ struct SharedWorkerState { } impl SharedWorkerState { - pub fn create_node_init_services(&self) -> NodeExtInitServices { + pub fn create_node_init_services( + &self, + node_require_loader: NodeRequireLoaderRc, + ) -> NodeExtInitServices { NodeExtInitServices { - node_require_resolver: self.npm_resolver.clone().into_require_resolver(), + node_require_loader, node_resolver: self.node_resolver.clone(), npm_resolver: self.npm_resolver.clone().into_npm_resolver(), + pkg_json_resolver: self.pkg_json_resolver.clone(), } } @@ -163,7 +164,6 @@ impl SharedWorkerState { pub struct CliMainWorker { main_module: ModuleSpecifier, - is_main_cjs: bool, worker: MainWorker, shared: Arc, } @@ -185,17 +185,7 @@ impl CliMainWorker { log::debug!("main_module {}", self.main_module); - if self.is_main_cjs { - deno_node::load_cjs_module( - &mut self.worker.js_runtime, - &self.main_module.to_file_path().unwrap().to_string_lossy(), - true, - self.shared.options.inspect_brk, - )?; - } else { - self.execute_main_module_possibly_with_npm().await?; - } - + self.execute_main_module().await?; self.worker.dispatch_load_event()?; loop { @@ -283,22 +273,7 @@ impl CliMainWorker { /// Execute the given main module emitting load and unload events before and after execution /// respectively. pub async fn execute(&mut self) -> Result<(), AnyError> { - if self.inner.is_main_cjs { - deno_node::load_cjs_module( - &mut self.inner.worker.js_runtime, - &self - .inner - .main_module - .to_file_path() - .unwrap() - .to_string_lossy(), - true, - self.inner.shared.options.inspect_brk, - )?; - } else { - self.inner.execute_main_module_possibly_with_npm().await?; - } - + self.inner.execute_main_module().await?; self.inner.worker.dispatch_load_event()?; self.pending_unload = true; @@ -339,24 +314,13 @@ impl CliMainWorker { executor.execute().await } - pub async fn execute_main_module_possibly_with_npm( - &mut self, - ) -> Result<(), AnyError> { + pub async fn execute_main_module(&mut self) -> Result<(), AnyError> { let id = self.worker.preload_main_module(&self.main_module).await?; - self.evaluate_module_possibly_with_npm(id).await + self.worker.evaluate_module(id).await } - pub async fn execute_side_module_possibly_with_npm( - &mut self, - ) -> Result<(), AnyError> { + pub async fn execute_side_module(&mut self) -> Result<(), AnyError> { let id = self.worker.preload_side_module(&self.main_module).await?; - self.evaluate_module_possibly_with_npm(id).await - } - - async fn evaluate_module_possibly_with_npm( - &mut self, - id: ModuleId, - ) -> Result<(), AnyError> { self.worker.evaluate_module(id).await } @@ -426,7 +390,6 @@ impl CliMainWorkerFactory { #[allow(clippy::too_many_arguments)] pub fn new( blob_store: Arc, - cjs_resolution_store: Arc, code_cache: Option>, feature_checker: Arc, fs: Arc, @@ -436,6 +399,7 @@ impl CliMainWorkerFactory { module_loader_factory: Box, node_resolver: Arc, npm_resolver: Arc, + pkg_json_resolver: Arc, root_cert_store_provider: Arc, root_permissions: PermissionsContainer, storage_key_resolver: StorageKeyResolver, @@ -446,7 +410,6 @@ impl CliMainWorkerFactory { shared: Arc::new(SharedWorkerState { blob_store, broadcast_channel: Default::default(), - cjs_resolution_store, code_cache, compiled_wasm_module_store: Default::default(), feature_checker, @@ -457,6 +420,7 @@ impl CliMainWorkerFactory { module_loader_factory, node_resolver, npm_resolver, + pkg_json_resolver, root_cert_store_provider, root_permissions, shared_array_buffer_store: Default::default(), @@ -492,10 +456,13 @@ impl CliMainWorkerFactory { stdio: deno_runtime::deno_io::Stdio, ) -> Result { let shared = &self.shared; - let ModuleLoaderAndSourceMapGetter { module_loader } = shared + let CreateModuleLoaderResult { + module_loader, + node_require_loader, + } = shared .module_loader_factory .create_for_main(permissions.clone()); - let (main_module, is_main_cjs) = if let Ok(package_ref) = + let main_module = if let Ok(package_ref) = NpmPackageReqReference::from_specifier(&main_module) { if let Some(npm_resolver) = shared.npm_resolver.as_managed() { @@ -515,9 +482,8 @@ impl CliMainWorkerFactory { package_ref.req(), &referrer, )?; - let node_resolution = self + let main_module = self .resolve_binary_entrypoint(&package_folder, package_ref.sub_path())?; - let is_main_cjs = matches!(node_resolution, NodeResolution::CommonJs(_)); if let Some(lockfile) = &shared.maybe_lockfile { // For npm binary commands, ensure that the lockfile gets updated @@ -526,36 +492,9 @@ impl CliMainWorkerFactory { lockfile.write_if_changed()?; } - (node_resolution.into_url(), is_main_cjs) - } else if shared.options.is_npm_main - || shared.node_resolver.in_npm_package(&main_module) - { - let node_resolution = - shared.node_resolver.url_to_node_resolution(main_module)?; - let is_main_cjs = matches!(node_resolution, NodeResolution::CommonJs(_)); - (node_resolution.into_url(), is_main_cjs) + main_module } else { - let is_maybe_cjs_js_ext = self.shared.options.unstable_detect_cjs - && specifier_has_extension(&main_module, "js") - && self - .shared - .node_resolver - .get_closest_package_json(&main_module) - .ok() - .flatten() - .map(|pkg_json| pkg_json.typ == "commonjs") - .unwrap_or(false); - let is_cjs = if is_maybe_cjs_js_ext { - // fill the cjs resolution store by preparing the module load - module_loader - .prepare_load(&main_module, None, false) - .await?; - self.shared.cjs_resolution_store.is_known_cjs(&main_module) - } else { - main_module.scheme() == "file" - && specifier_has_extension(&main_module, "cjs") - }; - (main_module, is_cjs) + main_module }; let maybe_inspector_server = shared.maybe_inspector_server.clone(); @@ -597,7 +536,9 @@ impl CliMainWorkerFactory { root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), module_loader, fs: shared.fs.clone(), - node_services: Some(shared.create_node_init_services()), + node_services: Some( + shared.create_node_init_services(node_require_loader), + ), npm_process_state_provider: Some(shared.npm_process_state_provider()), blob_store: shared.blob_store.clone(), broadcast_channel: shared.broadcast_channel.clone(), @@ -682,7 +623,6 @@ impl CliMainWorkerFactory { Ok(CliMainWorker { main_module, - is_main_cjs, worker, shared: shared.clone(), }) @@ -692,19 +632,19 @@ impl CliMainWorkerFactory { &self, package_folder: &Path, sub_path: Option<&str>, - ) -> Result { + ) -> Result { match self .shared .node_resolver .resolve_binary_export(package_folder, sub_path) { - Ok(node_resolution) => Ok(node_resolution), + Ok(specifier) => Ok(specifier), Err(original_err) => { // if the binary entrypoint was not found, fallback to regular node resolution let result = self.resolve_binary_entrypoint_fallback(package_folder, sub_path); match result { - Ok(Some(resolution)) => Ok(resolution), + Ok(Some(specifier)) => Ok(specifier), Ok(None) => Err(original_err.into()), Err(fallback_err) => { bail!("{:#}\n\nFallback failed: {:#}", original_err, fallback_err) @@ -719,7 +659,7 @@ impl CliMainWorkerFactory { &self, package_folder: &Path, sub_path: Option<&str>, - ) -> Result, AnyError> { + ) -> Result, AnyError> { // only fallback if the user specified a sub path if sub_path.is_none() { // it's confusing to users if the package doesn't have any binary @@ -728,7 +668,7 @@ impl CliMainWorkerFactory { return Ok(None); } - let resolution = self + let specifier = self .shared .node_resolver .resolve_package_subpath_from_deno_module( @@ -737,19 +677,14 @@ impl CliMainWorkerFactory { /* referrer */ None, NodeResolutionMode::Execution, )?; - match &resolution { - NodeResolution::BuiltIn(_) => Ok(None), - NodeResolution::CommonJs(specifier) | NodeResolution::Esm(specifier) => { - if specifier - .to_file_path() - .map(|p| p.exists()) - .unwrap_or(false) - { - Ok(Some(resolution)) - } else { - bail!("Cannot find module '{}'", specifier) - } - } + if specifier + .to_file_path() + .map(|p| p.exists()) + .unwrap_or(false) + { + Ok(Some(specifier)) + } else { + bail!("Cannot find module '{}'", specifier) } } } @@ -761,11 +696,13 @@ fn create_web_worker_callback( Arc::new(move |args| { let maybe_inspector_server = shared.maybe_inspector_server.clone(); - let ModuleLoaderAndSourceMapGetter { module_loader } = - shared.module_loader_factory.create_for_worker( - args.parent_permissions.clone(), - args.permissions.clone(), - ); + let CreateModuleLoaderResult { + module_loader, + node_require_loader, + } = shared.module_loader_factory.create_for_worker( + args.parent_permissions.clone(), + args.permissions.clone(), + ); let create_web_worker_cb = create_web_worker_callback(shared.clone(), stdio.clone()); @@ -795,7 +732,9 @@ fn create_web_worker_callback( root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), module_loader, fs: shared.fs.clone(), - node_services: Some(shared.create_node_init_services()), + node_services: Some( + shared.create_node_init_services(node_require_loader), + ), blob_store: shared.blob_store.clone(), broadcast_channel: shared.broadcast_channel.clone(), shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), diff --git a/ext/node/lib.rs b/ext/node/lib.rs index b34bea815b..db6d08e11e 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -9,15 +9,11 @@ use std::path::Path; use std::path::PathBuf; use deno_core::error::AnyError; -use deno_core::located_script_name; use deno_core::op2; use deno_core::url::Url; #[allow(unused_imports)] use deno_core::v8; use deno_core::v8::ExternalReference; -use deno_core::JsRuntime; -use deno_fs::sync::MaybeSend; -use deno_fs::sync::MaybeSync; use node_resolver::NpmResolverRc; use once_cell::sync::Lazy; @@ -125,16 +121,17 @@ impl NodePermissions for deno_permissions::PermissionsContainer { } #[allow(clippy::disallowed_types)] -pub type NodeRequireResolverRc = - deno_fs::sync::MaybeArc; +pub type NodeRequireLoaderRc = std::rc::Rc; -pub trait NodeRequireResolver: std::fmt::Debug + MaybeSend + MaybeSync { +pub trait NodeRequireLoader { #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn ensure_read_permission<'a>( &self, permissions: &mut dyn NodePermissions, path: &'a Path, ) -> Result, AnyError>; + + fn load_text_file_lossy(&self, path: &Path) -> Result; } pub static NODE_ENV_VAR_ALLOWLIST: Lazy> = Lazy::new(|| { @@ -152,10 +149,12 @@ fn op_node_build_os() -> String { env!("TARGET").split('-').nth(2).unwrap().to_string() } +#[derive(Clone)] pub struct NodeExtInitServices { - pub node_require_resolver: NodeRequireResolverRc, + pub node_require_loader: NodeRequireLoaderRc, pub node_resolver: NodeResolverRc, pub npm_resolver: NpmResolverRc, + pub pkg_json_resolver: PackageJsonResolverRc, } deno_core::extension!(deno_node, @@ -639,9 +638,10 @@ deno_core::extension!(deno_node, state.put(options.fs.clone()); if let Some(init) = &options.maybe_init { - state.put(init.node_require_resolver.clone()); + state.put(init.node_require_loader.clone()); state.put(init.node_resolver.clone()); state.put(init.npm_resolver.clone()); + state.put(init.pkg_json_resolver.clone()); } }, global_template_middleware = global_template_middleware, @@ -761,33 +761,16 @@ deno_core::extension!(deno_node, }, ); -pub fn load_cjs_module( - js_runtime: &mut JsRuntime, - module: &str, - main: bool, - inspect_brk: bool, -) -> Result<(), AnyError> { - fn escape_for_single_quote_string(text: &str) -> String { - text.replace('\\', r"\\").replace('\'', r"\'") - } - - let source_code = format!( - r#"(function loadCjsModule(moduleName, isMain, inspectBrk) {{ - Deno[Deno.internal].node.loadCjsModule(moduleName, isMain, inspectBrk); - }})('{module}', {main}, {inspect_brk});"#, - main = main, - module = escape_for_single_quote_string(module), - inspect_brk = inspect_brk, - ); - - js_runtime.execute_script(located_script_name!(), source_code)?; - Ok(()) -} - pub type NodeResolver = node_resolver::NodeResolver; #[allow(clippy::disallowed_types)] pub type NodeResolverRc = deno_fs::sync::MaybeArc>; +pub type PackageJsonResolver = + node_resolver::PackageJsonResolver; +#[allow(clippy::disallowed_types)] +pub type PackageJsonResolverRc = deno_fs::sync::MaybeArc< + node_resolver::PackageJsonResolver, +>; #[derive(Debug)] pub struct DenoFsNodeResolverEnv { diff --git a/ext/node/ops/require.rs b/ext/node/ops/require.rs index 7524fb43c7..30db8b6293 100644 --- a/ext/node/ops/require.rs +++ b/ext/node/ops/require.rs @@ -9,6 +9,7 @@ use deno_core::OpState; use deno_fs::FileSystemRc; use deno_package_json::PackageJsonRc; use deno_path_util::normalize_path; +use deno_path_util::url_to_file_path; use node_resolver::NodeModuleKind; use node_resolver::NodeResolutionMode; use node_resolver::REQUIRE_CONDITIONS; @@ -19,9 +20,10 @@ use std::path::PathBuf; use std::rc::Rc; use crate::NodePermissions; -use crate::NodeRequireResolverRc; +use crate::NodeRequireLoaderRc; use crate::NodeResolverRc; use crate::NpmResolverRc; +use crate::PackageJsonResolverRc; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn ensure_read_permission<'a, P>( @@ -31,9 +33,9 @@ fn ensure_read_permission<'a, P>( where P: NodePermissions + 'static, { - let resolver = state.borrow::().clone(); + let loader = state.borrow::().clone(); let permissions = state.borrow_mut::

(); - resolver.ensure_read_permission(permissions, file_path) + loader.ensure_read_permission(permissions, file_path) } #[derive(Debug, thiserror::Error)] @@ -54,10 +56,14 @@ pub enum RequireError { PackageImportsResolve( #[from] node_resolver::errors::PackageImportsResolveError, ), - #[error("failed to convert '{0}' to file path")] - FilePathConversion(Url), + #[error(transparent)] + FilePathConversion(#[from] deno_path_util::UrlToFilePathError), + #[error(transparent)] + UrlConversion(#[from] deno_path_util::PathToUrlError), #[error(transparent)] Fs(#[from] deno_io::fs::FsError), + #[error(transparent)] + ReadModule(deno_core::error::AnyError), #[error("Unable to get CWD: {0}")] UnableToGetCwd(deno_io::fs::FsError), } @@ -229,8 +235,11 @@ pub fn op_require_is_deno_dir_package( state: &mut OpState, #[string] path: String, ) -> bool { - let resolver = state.borrow::(); - resolver.in_npm_package_at_file_path(&PathBuf::from(path)) + let resolver = state.borrow::(); + match deno_path_util::url_from_file_path(&PathBuf::from(path)) { + Ok(specifier) => resolver.in_npm_package(&specifier), + Err(_) => false, + } } #[op2] @@ -411,8 +420,8 @@ where return Ok(None); } - let node_resolver = state.borrow::(); - let pkg = node_resolver + let pkg_json_resolver = state.borrow::(); + let pkg = pkg_json_resolver .get_closest_package_json_from_path(&PathBuf::from(parent_path.unwrap())) .ok() .flatten(); @@ -441,6 +450,7 @@ where let referrer = deno_core::url::Url::from_file_path(&pkg.path).unwrap(); if let Some(exports) = &pkg.exports { + let node_resolver = state.borrow::(); let r = node_resolver.package_exports_resolve( &pkg.path, &expansion, @@ -470,10 +480,13 @@ where P: NodePermissions + 'static, { let file_path = PathBuf::from(file_path); + // todo(dsherret): there's multiple borrows to NodeRequireLoaderRc here let file_path = ensure_read_permission::

(state, &file_path) .map_err(RequireError::Permission)?; - let fs = state.borrow::(); - Ok(fs.read_text_file_lossy_sync(&file_path, None)?) + let loader = state.borrow::(); + loader + .load_text_file_lossy(&file_path) + .map_err(RequireError::ReadModule) } #[op2] @@ -503,11 +516,12 @@ where P: NodePermissions + 'static, { let fs = state.borrow::(); - let npm_resolver = state.borrow::(); let node_resolver = state.borrow::(); + let pkg_json_resolver = state.borrow::(); let modules_path = PathBuf::from(&modules_path_str); - let pkg_path = if npm_resolver.in_npm_package_at_file_path(&modules_path) + let modules_specifier = deno_path_util::url_from_file_path(&modules_path)?; + let pkg_path = if node_resolver.in_npm_package(&modules_specifier) && !uses_local_node_modules_dir { modules_path @@ -521,7 +535,7 @@ where } }; let Some(pkg) = - node_resolver.load_package_json(&pkg_path.join("package.json"))? + pkg_json_resolver.load_package_json(&pkg_path.join("package.json"))? else { return Ok(None); }; @@ -561,8 +575,8 @@ where { let filename = PathBuf::from(filename); // permissions: allow reading the closest package.json files - let node_resolver = state.borrow::().clone(); - node_resolver.get_closest_package_json_from_path(&filename) + let pkg_json_resolver = state.borrow::(); + pkg_json_resolver.get_closest_package_json_from_path(&filename) } #[op2] @@ -574,13 +588,13 @@ pub fn op_require_read_package_scope

( where P: NodePermissions + 'static, { - let node_resolver = state.borrow::().clone(); + let pkg_json_resolver = state.borrow::(); let package_json_path = PathBuf::from(package_json_path); if package_json_path.file_name() != Some("package.json".as_ref()) { // permissions: do not allow reading a non-package.json file return None; } - node_resolver + pkg_json_resolver .load_package_json(&package_json_path) .ok() .flatten() @@ -599,14 +613,15 @@ where let referrer_path = PathBuf::from(&referrer_filename); let referrer_path = ensure_read_permission::

(state, &referrer_path) .map_err(RequireError::Permission)?; - let node_resolver = state.borrow::(); + let pkg_json_resolver = state.borrow::(); let Some(pkg) = - node_resolver.get_closest_package_json_from_path(&referrer_path)? + pkg_json_resolver.get_closest_package_json_from_path(&referrer_path)? else { return Ok(None); }; if pkg.imports.is_some() { + let node_resolver = state.borrow::(); let referrer_url = Url::from_file_path(&referrer_filename).unwrap(); let url = node_resolver.package_imports_resolve( &request, @@ -637,13 +652,6 @@ fn url_to_file_path_string(url: &Url) -> Result { Ok(file_path.to_string_lossy().into_owned()) } -fn url_to_file_path(url: &Url) -> Result { - match url.to_file_path() { - Ok(file_path) => Ok(file_path), - Err(()) => Err(RequireError::FilePathConversion(url.clone())), - } -} - #[op2(fast)] pub fn op_require_can_parse_as_esm( scope: &mut v8::HandleScope, diff --git a/ext/node/ops/worker_threads.rs b/ext/node/ops/worker_threads.rs index e33cf91574..d2e5758826 100644 --- a/ext/node/ops/worker_threads.rs +++ b/ext/node/ops/worker_threads.rs @@ -4,14 +4,12 @@ use deno_core::op2; use deno_core::url::Url; use deno_core::OpState; use deno_fs::FileSystemRc; -use node_resolver::NodeResolution; use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use crate::NodePermissions; -use crate::NodeRequireResolverRc; -use crate::NodeResolverRc; +use crate::NodeRequireLoaderRc; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn ensure_read_permission<'a, P>( @@ -21,9 +19,9 @@ fn ensure_read_permission<'a, P>( where P: NodePermissions + 'static, { - let resolver = state.borrow::().clone(); + let loader = state.borrow::().clone(); let permissions = state.borrow_mut::

(); - resolver.ensure_read_permission(permissions, file_path) + loader.ensure_read_permission(permissions, file_path) } #[derive(Debug, thiserror::Error)] @@ -42,14 +40,11 @@ pub enum WorkerThreadsFilenameError { UrlToPath, #[error("File not found [{0:?}]")] FileNotFound(PathBuf), - #[error("Neither ESM nor CJS")] - NeitherEsmNorCjs, - #[error("{0}")] - UrlToNodeResolution(node_resolver::errors::UrlToNodeResolutionError), #[error(transparent)] Fs(#[from] deno_io::fs::FsError), } +// todo(dsherret): we should remove this and do all this work inside op_create_worker #[op2] #[string] pub fn op_worker_threads_filename

( @@ -88,30 +83,5 @@ where url_path.to_path_buf(), )); } - let node_resolver = state.borrow::(); - match node_resolver - .url_to_node_resolution(url) - .map_err(WorkerThreadsFilenameError::UrlToNodeResolution)? - { - NodeResolution::Esm(u) => Ok(u.to_string()), - NodeResolution::CommonJs(u) => wrap_cjs(u), - NodeResolution::BuiltIn(_) => { - Err(WorkerThreadsFilenameError::NeitherEsmNorCjs) - } - } -} - -/// -/// Wrap a CJS file-URL and the required setup in a stringified `data:`-URL -/// -fn wrap_cjs(url: Url) -> Result { - let path = url - .to_file_path() - .map_err(|_| WorkerThreadsFilenameError::UrlToPath)?; - let filename = path.file_name().unwrap().to_string_lossy(); - Ok(format!( - "data:text/javascript,import {{ createRequire }} from \"node:module\";\ - const require = createRequire(\"{}\"); require(\"./{}\");", - url, filename, - )) + Ok(url.to_string()) } diff --git a/ext/node/polyfills/01_require.js b/ext/node/polyfills/01_require.js index 7935903a8d..296d819aa5 100644 --- a/ext/node/polyfills/01_require.js +++ b/ext/node/polyfills/01_require.js @@ -1071,13 +1071,35 @@ Module._extensions[".js"] = function (module, filename) { } else if (pkg?.type === "commonjs") { format = "commonjs"; } - } else if (StringPrototypeEndsWith(filename, ".cjs")) { - format = "commonjs"; } module._compile(content, filename, format); }; +Module._extensions[".ts"] = + Module._extensions[".jsx"] = + Module._extensions[".tsx"] = + function (module, filename) { + const content = op_require_read_file(filename); + + let format; + const pkg = op_require_read_closest_package_json(filename); + if (pkg?.type === "module") { + format = "module"; + } else if (pkg?.type === "commonjs") { + format = "commonjs"; + } + + module._compile(content, filename, format); + }; + +Module._extensions[".cjs"] = + Module._extensions[".cts"] = + function (module, filename) { + const content = op_require_read_file(filename); + module._compile(content, filename, "commonjs"); + }; + function loadESMFromCJS(module, filename, code) { const namespace = op_import_sync( url.pathToFileURL(filename).toString(), @@ -1087,7 +1109,10 @@ function loadESMFromCJS(module, filename, code) { module.exports = namespace; } -Module._extensions[".mjs"] = function (module, filename) { +Module._extensions[".mjs"] = Module._extensions[".mts"] = function ( + module, + filename, +) { loadESMFromCJS(module, filename); }; diff --git a/resolvers/deno/fs.rs b/resolvers/deno/fs.rs index b08be37982..44495fa7c2 100644 --- a/resolvers/deno/fs.rs +++ b/resolvers/deno/fs.rs @@ -15,13 +15,3 @@ pub trait DenoResolverFs { fn is_dir_sync(&self, path: &Path) -> bool; fn read_dir_sync(&self, dir_path: &Path) -> std::io::Result>; } - -pub(crate) struct DenoPkgJsonFsAdapter<'a, Fs: DenoResolverFs>(pub &'a Fs); - -impl<'a, Fs: DenoResolverFs> deno_package_json::fs::DenoPkgJsonFs - for DenoPkgJsonFsAdapter<'a, Fs> -{ - fn read_to_string_lossy(&self, path: &Path) -> std::io::Result { - self.0.read_to_string_lossy(path) - } -} diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs index 3394b3e501..b85117052c 100644 --- a/resolvers/deno/npm/byonm.rs +++ b/resolvers/deno/npm/byonm.rs @@ -10,16 +10,17 @@ use deno_package_json::PackageJsonDepValue; use deno_path_util::url_to_file_path; use deno_semver::package::PackageReq; use deno_semver::Version; +use node_resolver::env::NodeResolverEnv; use node_resolver::errors::PackageFolderResolveError; use node_resolver::errors::PackageFolderResolveIoError; use node_resolver::errors::PackageJsonLoadError; use node_resolver::errors::PackageNotFoundError; -use node_resolver::load_pkg_json; +use node_resolver::InNpmPackageChecker; use node_resolver::NpmResolver; +use node_resolver::PackageJsonResolverRc; use thiserror::Error; use url::Url; -use crate::fs::DenoPkgJsonFsAdapter; use crate::fs::DenoResolverFs; use super::local::normalize_pkg_name_for_node_modules_deno_folder; @@ -36,32 +37,41 @@ pub enum ByonmResolvePkgFolderFromDenoReqError { Io(#[from] std::io::Error), } -pub struct ByonmNpmResolverCreateOptions { - pub fs: Fs, +pub struct ByonmNpmResolverCreateOptions< + Fs: DenoResolverFs, + TEnv: NodeResolverEnv, +> { // todo(dsherret): investigate removing this pub root_node_modules_dir: Option, + pub fs: Fs, + pub pkg_json_resolver: PackageJsonResolverRc, } #[derive(Debug)] -pub struct ByonmNpmResolver { +pub struct ByonmNpmResolver { fs: Fs, + pkg_json_resolver: PackageJsonResolverRc, root_node_modules_dir: Option, } -impl Clone for ByonmNpmResolver { +impl Clone + for ByonmNpmResolver +{ fn clone(&self) -> Self { Self { fs: self.fs.clone(), + pkg_json_resolver: self.pkg_json_resolver.clone(), root_node_modules_dir: self.root_node_modules_dir.clone(), } } } -impl ByonmNpmResolver { - pub fn new(options: ByonmNpmResolverCreateOptions) -> Self { +impl ByonmNpmResolver { + pub fn new(options: ByonmNpmResolverCreateOptions) -> Self { Self { - fs: options.fs, root_node_modules_dir: options.root_node_modules_dir, + fs: options.fs, + pkg_json_resolver: options.pkg_json_resolver, } } @@ -73,7 +83,7 @@ impl ByonmNpmResolver { &self, path: &Path, ) -> Result>, PackageJsonLoadError> { - load_pkg_json(&DenoPkgJsonFsAdapter(&self.fs), path) + self.pkg_json_resolver.load_package_json(path) } /// Finds the ancestor package.json that contains the specified dependency. @@ -290,8 +300,10 @@ impl ByonmNpmResolver { } } -impl NpmResolver - for ByonmNpmResolver +impl< + Fs: DenoResolverFs + Send + Sync + std::fmt::Debug, + TEnv: NodeResolverEnv, + > NpmResolver for ByonmNpmResolver { fn resolve_package_folder_from_package( &self, @@ -342,7 +354,12 @@ impl NpmResolver .into() }) } +} +#[derive(Debug)] +pub struct ByonmInNpmPackageChecker; + +impl InNpmPackageChecker for ByonmInNpmPackageChecker { fn in_npm_package(&self, specifier: &Url) -> bool { specifier.scheme() == "file" && specifier diff --git a/resolvers/deno/npm/mod.rs b/resolvers/deno/npm/mod.rs index 9d885cad31..45e2341c78 100644 --- a/resolvers/deno/npm/mod.rs +++ b/resolvers/deno/npm/mod.rs @@ -3,6 +3,7 @@ mod byonm; mod local; +pub use byonm::ByonmInNpmPackageChecker; pub use byonm::ByonmNpmResolver; pub use byonm::ByonmNpmResolverCreateOptions; pub use byonm::ByonmResolvePkgFolderFromDenoReqError; diff --git a/resolvers/deno/sloppy_imports.rs b/resolvers/deno/sloppy_imports.rs index e215e87686..7aba5b771a 100644 --- a/resolvers/deno/sloppy_imports.rs +++ b/resolvers/deno/sloppy_imports.rs @@ -232,7 +232,7 @@ impl SloppyImportsResolver { | MediaType::Tsx | MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap => { return None; } diff --git a/resolvers/node/analyze.rs b/resolvers/node/analyze.rs index 009296006a..c7415933d7 100644 --- a/resolvers/node/analyze.rs +++ b/resolvers/node/analyze.rs @@ -19,18 +19,19 @@ use anyhow::Error as AnyError; use url::Url; use crate::env::NodeResolverEnv; -use crate::package_json::load_pkg_json; +use crate::npm::InNpmPackageCheckerRc; use crate::resolution::NodeResolverRc; use crate::NodeModuleKind; use crate::NodeResolutionMode; use crate::NpmResolverRc; +use crate::PackageJsonResolverRc; use crate::PathClean; #[derive(Debug, Clone)] -pub enum CjsAnalysis { +pub enum CjsAnalysis<'a> { /// File was found to be an ES module and the translator should /// load the code as ESM. - Esm(String), + Esm(Cow<'a, str>), Cjs(CjsAnalysisExports), } @@ -50,11 +51,11 @@ pub trait CjsCodeAnalyzer { /// already has it. If the source is needed by the implementation, /// then it can use the provided source, or otherwise load it if /// necessary. - async fn analyze_cjs( + async fn analyze_cjs<'a>( &self, specifier: &Url, - maybe_source: Option, - ) -> Result; + maybe_source: Option>, + ) -> Result, AnyError>; } pub struct NodeCodeTranslator< @@ -63,8 +64,10 @@ pub struct NodeCodeTranslator< > { cjs_code_analyzer: TCjsCodeAnalyzer, env: TNodeResolverEnv, + in_npm_pkg_checker: InNpmPackageCheckerRc, node_resolver: NodeResolverRc, npm_resolver: NpmResolverRc, + pkg_json_resolver: PackageJsonResolverRc, } impl @@ -73,14 +76,18 @@ impl pub fn new( cjs_code_analyzer: TCjsCodeAnalyzer, env: TNodeResolverEnv, + in_npm_pkg_checker: InNpmPackageCheckerRc, node_resolver: NodeResolverRc, npm_resolver: NpmResolverRc, + pkg_json_resolver: PackageJsonResolverRc, ) -> Self { Self { cjs_code_analyzer, env, + in_npm_pkg_checker, node_resolver, npm_resolver, + pkg_json_resolver, } } @@ -90,11 +97,11 @@ impl /// For all discovered reexports the analysis will be performed recursively. /// /// If successful a source code for equivalent ES module is returned. - pub async fn translate_cjs_to_esm( + pub async fn translate_cjs_to_esm<'a>( &self, entry_specifier: &Url, - source: Option, - ) -> Result { + source: Option>, + ) -> Result, AnyError> { let mut temp_var_count = 0; let analysis = self @@ -108,7 +115,7 @@ impl }; let mut source = vec![ - r#"import {createRequire as __internalCreateRequire} from "node:module"; + r#"import {createRequire as __internalCreateRequire, Module as __internalModule } from "node:module"; const require = __internalCreateRequire(import.meta.url);"# .to_string(), ]; @@ -135,7 +142,12 @@ impl } source.push(format!( - "const mod = require(\"{}\");", + r#"let mod; + if (import.meta.main) {{ + mod = __internalModule._load("{0}", null, true) + }} else {{ + mod = require("{0}"); + }}"#, url_to_file_path(entry_specifier) .unwrap() .to_str() @@ -159,7 +171,7 @@ impl source.push("export default mod;".to_string()); let translated_source = source.join("\n"); - Ok(translated_source) + Ok(Cow::Owned(translated_source)) } async fn analyze_reexports<'a>( @@ -174,7 +186,7 @@ impl struct Analysis { reexport_specifier: url::Url, referrer: url::Url, - analysis: CjsAnalysis, + analysis: CjsAnalysis<'static>, } type AnalysisFuture<'a> = LocalBoxFuture<'a, Result>; @@ -329,8 +341,9 @@ impl }?; let package_json_path = module_dir.join("package.json"); - let maybe_package_json = - load_pkg_json(self.env.pkg_json_fs(), &package_json_path)?; + let maybe_package_json = self + .pkg_json_resolver + .load_package_json(&package_json_path)?; if let Some(package_json) = maybe_package_json { if let Some(exports) = &package_json.exports { return Some( @@ -356,8 +369,9 @@ impl if self.env.is_dir_sync(&d) { // subdir might have a package.json that specifies the entrypoint let package_json_path = d.join("package.json"); - let maybe_package_json = - load_pkg_json(self.env.pkg_json_fs(), &package_json_path)?; + let maybe_package_json = self + .pkg_json_resolver + .load_package_json(&package_json_path)?; if let Some(package_json) = maybe_package_json { if let Some(main) = package_json.main(NodeModuleKind::Cjs) { return Ok(Some(url_from_file_path(&d.join(main).clean())?)); @@ -382,7 +396,7 @@ impl // as a fallback, attempt to resolve it via the ancestor directories let mut last = referrer_path.as_path(); while let Some(parent) = last.parent() { - if !self.npm_resolver.in_npm_package_at_dir_path(parent) { + if !self.in_npm_pkg_checker.in_npm_package_at_dir_path(parent) { break; } let path = if parent.ends_with("node_modules") { diff --git a/resolvers/node/errors.rs b/resolvers/node/errors.rs index 4ba829eda5..aacbecefb4 100644 --- a/resolvers/node/errors.rs +++ b/resolvers/node/errors.rs @@ -81,29 +81,6 @@ pub trait NodeJsErrorCoded { fn code(&self) -> NodeJsErrorCode; } -kinded_err!( - ResolvePkgSubpathFromDenoModuleError, - ResolvePkgSubpathFromDenoModuleErrorKind -); - -impl NodeJsErrorCoded for ResolvePkgSubpathFromDenoModuleError { - fn code(&self) -> NodeJsErrorCode { - use ResolvePkgSubpathFromDenoModuleErrorKind::*; - match self.as_kind() { - PackageSubpathResolve(e) => e.code(), - UrlToNodeResolution(e) => e.code(), - } - } -} - -#[derive(Debug, Error)] -pub enum ResolvePkgSubpathFromDenoModuleErrorKind { - #[error(transparent)] - PackageSubpathResolve(#[from] PackageSubpathResolveError), - #[error(transparent)] - UrlToNodeResolution(#[from] UrlToNodeResolutionError), -} - // todo(https://github.com/denoland/deno_core/issues/810): make this a TypeError #[derive(Debug, Clone, Error)] #[error( @@ -394,37 +371,6 @@ impl NodeJsErrorCoded for CanonicalizingPkgJsonDirError { } } -#[derive(Debug, Error)] -#[error("TypeScript files are not supported in npm packages: {specifier}")] -pub struct TypeScriptNotSupportedInNpmError { - pub specifier: Url, -} - -impl NodeJsErrorCoded for TypeScriptNotSupportedInNpmError { - fn code(&self) -> NodeJsErrorCode { - NodeJsErrorCode::ERR_UNKNOWN_FILE_EXTENSION - } -} - -kinded_err!(UrlToNodeResolutionError, UrlToNodeResolutionErrorKind); - -impl NodeJsErrorCoded for UrlToNodeResolutionError { - fn code(&self) -> NodeJsErrorCode { - match self.as_kind() { - UrlToNodeResolutionErrorKind::TypeScriptNotSupported(e) => e.code(), - UrlToNodeResolutionErrorKind::ClosestPkgJson(e) => e.code(), - } - } -} - -#[derive(Debug, Error)] -pub enum UrlToNodeResolutionErrorKind { - #[error(transparent)] - TypeScriptNotSupported(#[from] TypeScriptNotSupportedInNpmError), - #[error(transparent)] - ClosestPkgJson(#[from] ClosestPkgJsonError), -} - // todo(https://github.com/denoland/deno_core/issues/810): make this a TypeError #[derive(Debug, Error)] #[error( @@ -533,8 +479,6 @@ pub enum NodeResolveErrorKind { TypesNotFound(#[from] TypesNotFoundError), #[error(transparent)] FinalizeResolution(#[from] FinalizeResolutionError), - #[error(transparent)] - UrlToNodeResolution(#[from] UrlToNodeResolutionError), } kinded_err!(FinalizeResolutionError, FinalizeResolutionErrorKind); @@ -728,8 +672,6 @@ pub enum ResolvePkgJsonBinExportError { MissingPkgJson { pkg_json_path: PathBuf }, #[error("Failed resolving binary export. {message}")] InvalidBinProperty { message: String }, - #[error(transparent)] - UrlToNodeResolution(#[from] UrlToNodeResolutionError), } #[derive(Debug, Error)] diff --git a/resolvers/node/lib.rs b/resolvers/node/lib.rs index f03f770486..18b0a85363 100644 --- a/resolvers/node/lib.rs +++ b/resolvers/node/lib.rs @@ -13,9 +13,12 @@ mod resolution; mod sync; pub use deno_package_json::PackageJson; +pub use npm::InNpmPackageChecker; +pub use npm::InNpmPackageCheckerRc; pub use npm::NpmResolver; pub use npm::NpmResolverRc; -pub use package_json::load_pkg_json; +pub use package_json::PackageJsonResolver; +pub use package_json::PackageJsonResolverRc; pub use package_json::PackageJsonThreadLocalCache; pub use path::PathClean; pub use resolution::parse_npm_pkg_name; diff --git a/resolvers/node/npm.rs b/resolvers/node/npm.rs index 6b5f21db62..2132f0b545 100644 --- a/resolvers/node/npm.rs +++ b/resolvers/node/npm.rs @@ -22,7 +22,13 @@ pub trait NpmResolver: std::fmt::Debug + MaybeSend + MaybeSync { specifier: &str, referrer: &Url, ) -> Result; +} +#[allow(clippy::disallowed_types)] +pub type InNpmPackageCheckerRc = crate::sync::MaybeArc; + +/// Checks if a provided specifier is in an npm package. +pub trait InNpmPackageChecker: std::fmt::Debug + MaybeSend + MaybeSync { fn in_npm_package(&self, specifier: &Url) -> bool; fn in_npm_package_at_dir_path(&self, path: &Path) -> bool { diff --git a/resolvers/node/package_json.rs b/resolvers/node/package_json.rs index de750f1d7e..6967779e5d 100644 --- a/resolvers/node/package_json.rs +++ b/resolvers/node/package_json.rs @@ -2,15 +2,21 @@ use deno_package_json::PackageJson; use deno_package_json::PackageJsonRc; +use deno_path_util::strip_unc_prefix; use std::cell::RefCell; use std::collections::HashMap; use std::io::ErrorKind; use std::path::Path; use std::path::PathBuf; +use url::Url; +use crate::env::NodeResolverEnv; +use crate::errors::CanonicalizingPkgJsonDirError; +use crate::errors::ClosestPkgJsonError; use crate::errors::PackageJsonLoadError; -// use a thread local cache so that workers have their own distinct cache +// todo(dsherret): this isn't exactly correct and we should change it to instead +// be created per worker and passed down as a ctor arg to the pkg json resolver thread_local! { static CACHE: RefCell> = RefCell::new(HashMap::new()); } @@ -33,21 +39,91 @@ impl deno_package_json::PackageJsonCache for PackageJsonThreadLocalCache { } } -/// Helper to load a package.json file using the thread local cache -/// in node_resolver. -pub fn load_pkg_json( - fs: &dyn deno_package_json::fs::DenoPkgJsonFs, - path: &Path, -) -> Result, PackageJsonLoadError> { - let result = - PackageJson::load_from_path(path, fs, Some(&PackageJsonThreadLocalCache)); - match result { - Ok(pkg_json) => Ok(Some(pkg_json)), - Err(deno_package_json::PackageJsonLoadError::Io { source, .. }) - if source.kind() == ErrorKind::NotFound => - { +#[allow(clippy::disallowed_types)] +pub type PackageJsonResolverRc = + crate::sync::MaybeArc>; + +#[derive(Debug)] +pub struct PackageJsonResolver { + env: TEnv, +} + +impl PackageJsonResolver { + pub fn new(env: TEnv) -> Self { + Self { env } + } + + pub fn get_closest_package_json( + &self, + url: &Url, + ) -> Result, ClosestPkgJsonError> { + let Ok(file_path) = deno_path_util::url_to_file_path(url) else { + return Ok(None); + }; + self.get_closest_package_json_from_path(&file_path) + } + + pub fn get_closest_package_json_from_path( + &self, + file_path: &Path, + ) -> Result, ClosestPkgJsonError> { + // we use this for deno compile using byonm because the script paths + // won't be in virtual file system, but the package.json paths will be + fn canonicalize_first_ancestor_exists( + dir_path: &Path, + env: &TEnv, + ) -> Result, std::io::Error> { + for ancestor in dir_path.ancestors() { + match env.realpath_sync(ancestor) { + Ok(dir_path) => return Ok(Some(dir_path)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + // keep searching + } + Err(err) => return Err(err), + } + } Ok(None) } - Err(err) => Err(PackageJsonLoadError(err)), + + let parent_dir = file_path.parent().unwrap(); + let Some(start_dir) = canonicalize_first_ancestor_exists( + parent_dir, &self.env, + ) + .map_err(|source| CanonicalizingPkgJsonDirError { + dir_path: parent_dir.to_path_buf(), + source, + })? + else { + return Ok(None); + }; + let start_dir = strip_unc_prefix(start_dir); + for current_dir in start_dir.ancestors() { + let package_json_path = current_dir.join("package.json"); + if let Some(pkg_json) = self.load_package_json(&package_json_path)? { + return Ok(Some(pkg_json)); + } + } + + Ok(None) + } + + pub fn load_package_json( + &self, + path: &Path, + ) -> Result, PackageJsonLoadError> { + let result = PackageJson::load_from_path( + path, + self.env.pkg_json_fs(), + Some(&PackageJsonThreadLocalCache), + ); + match result { + Ok(pkg_json) => Ok(Some(pkg_json)), + Err(deno_package_json::PackageJsonLoadError::Io { source, .. }) + if source.kind() == ErrorKind::NotFound => + { + Ok(None) + } + Err(err) => Err(PackageJsonLoadError(err)), + } } } diff --git a/resolvers/node/resolution.rs b/resolvers/node/resolution.rs index 811583a5ee..d44539e978 100644 --- a/resolvers/node/resolution.rs +++ b/resolvers/node/resolution.rs @@ -6,9 +6,6 @@ use std::path::PathBuf; use anyhow::bail; use anyhow::Error as AnyError; -use deno_media_type::MediaType; -use deno_package_json::PackageJsonRc; -use deno_path_util::strip_unc_prefix; use deno_path_util::url_from_file_path; use serde_json::Map; use serde_json::Value; @@ -16,8 +13,6 @@ use url::Url; use crate::env::NodeResolverEnv; use crate::errors; -use crate::errors::CanonicalizingPkgJsonDirError; -use crate::errors::ClosestPkgJsonError; use crate::errors::DataUrlReferrerError; use crate::errors::FinalizeResolutionError; use crate::errors::InvalidModuleSpecifierError; @@ -32,7 +27,6 @@ use crate::errors::PackageExportsResolveError; use crate::errors::PackageImportNotDefinedError; use crate::errors::PackageImportsResolveError; use crate::errors::PackageImportsResolveErrorKind; -use crate::errors::PackageJsonLoadError; use crate::errors::PackagePathNotExportedError; use crate::errors::PackageResolveError; use crate::errors::PackageSubpathResolveError; @@ -42,14 +36,13 @@ use crate::errors::PackageTargetResolveError; use crate::errors::PackageTargetResolveErrorKind; use crate::errors::ResolveBinaryCommandsError; use crate::errors::ResolvePkgJsonBinExportError; -use crate::errors::ResolvePkgSubpathFromDenoModuleError; -use crate::errors::TypeScriptNotSupportedInNpmError; use crate::errors::TypesNotFoundError; use crate::errors::TypesNotFoundErrorData; use crate::errors::UnsupportedDirImportError; use crate::errors::UnsupportedEsmUrlSchemeError; -use crate::errors::UrlToNodeResolutionError; +use crate::npm::InNpmPackageCheckerRc; use crate::NpmResolverRc; +use crate::PackageJsonResolverRc; use crate::PathClean; use deno_package_json::PackageJson; @@ -73,16 +66,14 @@ impl NodeResolutionMode { #[derive(Debug)] pub enum NodeResolution { - Esm(Url), - CommonJs(Url), + Module(Url), BuiltIn(String), } impl NodeResolution { pub fn into_url(self) -> Url { match self { - Self::Esm(u) => u, - Self::CommonJs(u) => u, + Self::Module(u) => u, Self::BuiltIn(specifier) => { if specifier.starts_with("node:") { Url::parse(&specifier).unwrap() @@ -92,42 +83,6 @@ impl NodeResolution { } } } - - pub fn into_specifier_and_media_type( - resolution: Option, - ) -> (Url, MediaType) { - match resolution { - Some(NodeResolution::CommonJs(specifier)) => { - let media_type = MediaType::from_specifier(&specifier); - ( - specifier, - match media_type { - MediaType::JavaScript | MediaType::Jsx => MediaType::Cjs, - MediaType::TypeScript | MediaType::Tsx => MediaType::Cts, - MediaType::Dts => MediaType::Dcts, - _ => media_type, - }, - ) - } - Some(NodeResolution::Esm(specifier)) => { - let media_type = MediaType::from_specifier(&specifier); - ( - specifier, - match media_type { - MediaType::JavaScript | MediaType::Jsx => MediaType::Mjs, - MediaType::TypeScript | MediaType::Tsx => MediaType::Mts, - MediaType::Dts => MediaType::Dmts, - _ => media_type, - }, - ) - } - Some(resolution) => (resolution.into_url(), MediaType::Dts), - None => ( - Url::parse("internal:///missing_dependency.d.ts").unwrap(), - MediaType::Dts, - ), - } - } } #[allow(clippy::disallowed_types)] @@ -136,16 +91,28 @@ pub type NodeResolverRc = crate::sync::MaybeArc>; #[derive(Debug)] pub struct NodeResolver { env: TEnv, + in_npm_pkg_checker: InNpmPackageCheckerRc, npm_resolver: NpmResolverRc, + pkg_json_resolver: PackageJsonResolverRc, } impl NodeResolver { - pub fn new(env: TEnv, npm_resolver: NpmResolverRc) -> Self { - Self { env, npm_resolver } + pub fn new( + env: TEnv, + in_npm_pkg_checker: InNpmPackageCheckerRc, + npm_resolver: NpmResolverRc, + pkg_json_resolver: PackageJsonResolverRc, + ) -> Self { + Self { + env, + in_npm_pkg_checker, + npm_resolver, + pkg_json_resolver, + } } pub fn in_npm_package(&self, specifier: &Url) -> bool { - self.npm_resolver.in_npm_package(specifier) + self.in_npm_pkg_checker.in_npm_package(specifier) } /// This function is an implementation of `defaultResolve` in @@ -166,7 +133,7 @@ impl NodeResolver { if let Ok(url) = Url::parse(specifier) { if url.scheme() == "data" { - return Ok(NodeResolution::Esm(url)); + return Ok(NodeResolution::Module(url)); } if let Some(module_name) = @@ -191,7 +158,7 @@ impl NodeResolver { let url = referrer .join(specifier) .map_err(|source| DataUrlReferrerError { source })?; - return Ok(NodeResolution::Esm(url)); + return Ok(NodeResolution::Module(url)); } } @@ -212,7 +179,7 @@ impl NodeResolver { }; let url = self.finalize_resolution(url, Some(referrer))?; - let resolve_response = self.url_to_node_resolution(url)?; + let resolve_response = NodeResolution::Module(url); // TODO(bartlomieju): skipped checking errors for commonJS resolution and // "preserveSymlinksMain"/"preserveSymlinks" options. Ok(resolve_response) @@ -236,6 +203,7 @@ impl NodeResolver { })?) } else if specifier.starts_with('#') { let pkg_config = self + .pkg_json_resolver .get_closest_package_json(referrer) .map_err(PackageImportsResolveErrorKind::ClosestPkgJson) .map_err(|err| PackageImportsResolveError(Box::new(err)))?; @@ -332,7 +300,7 @@ impl NodeResolver { package_subpath: Option<&str>, maybe_referrer: Option<&Url>, mode: NodeResolutionMode, - ) -> Result { + ) -> Result { let node_module_kind = NodeModuleKind::Esm; let package_subpath = package_subpath .map(|s| format!("./{s}")) @@ -345,10 +313,9 @@ impl NodeResolver { DEFAULT_CONDITIONS, mode, )?; - let resolve_response = self.url_to_node_resolution(resolved_url)?; // TODO(bartlomieju): skipped checking errors for commonJS resolution and // "preserveSymlinksMain"/"preserveSymlinks" options. - Ok(resolve_response) + Ok(resolved_url) } pub fn resolve_binary_commands( @@ -356,7 +323,9 @@ impl NodeResolver { package_folder: &Path, ) -> Result, ResolveBinaryCommandsError> { let pkg_json_path = package_folder.join("package.json"); - let Some(package_json) = self.load_package_json(&pkg_json_path)? else { + let Some(package_json) = + self.pkg_json_resolver.load_package_json(&pkg_json_path)? + else { return Ok(Vec::new()); }; @@ -381,9 +350,11 @@ impl NodeResolver { &self, package_folder: &Path, sub_path: Option<&str>, - ) -> Result { + ) -> Result { let pkg_json_path = package_folder.join("package.json"); - let Some(package_json) = self.load_package_json(&pkg_json_path)? else { + let Some(package_json) = + self.pkg_json_resolver.load_package_json(&pkg_json_path)? + else { return Err(ResolvePkgJsonBinExportError::MissingPkgJson { pkg_json_path, }); @@ -396,37 +367,9 @@ impl NodeResolver { })?; let url = url_from_file_path(&package_folder.join(bin_entry)).unwrap(); - let resolve_response = self.url_to_node_resolution(url)?; // TODO(bartlomieju): skipped checking errors for commonJS resolution and // "preserveSymlinksMain"/"preserveSymlinks" options. - Ok(resolve_response) - } - - pub fn url_to_node_resolution( - &self, - url: Url, - ) -> Result { - let url_str = url.as_str().to_lowercase(); - if url_str.starts_with("http") || url_str.ends_with(".json") { - Ok(NodeResolution::Esm(url)) - } else if url_str.ends_with(".js") || url_str.ends_with(".d.ts") { - let maybe_package_config = self.get_closest_package_json(&url)?; - match maybe_package_config { - Some(c) if c.typ == "module" => Ok(NodeResolution::Esm(url)), - Some(_) => Ok(NodeResolution::CommonJs(url)), - None => Ok(NodeResolution::Esm(url)), - } - } else if url_str.ends_with(".mjs") || url_str.ends_with(".d.mts") { - Ok(NodeResolution::Esm(url)) - } else if url_str.ends_with(".ts") || url_str.ends_with(".mts") { - if self.in_npm_package(&url) { - Err(TypeScriptNotSupportedInNpmError { specifier: url }.into()) - } else { - Ok(NodeResolution::Esm(url)) - } - } else { - Ok(NodeResolution::CommonJs(url)) - } + Ok(url) } /// Checks if the resolved file has a corresponding declaration file. @@ -1101,7 +1044,9 @@ impl NodeResolver { let (package_name, package_subpath, _is_scoped) = parse_npm_pkg_name(specifier, referrer)?; - if let Some(package_config) = self.get_closest_package_json(referrer)? { + if let Some(package_config) = + self.pkg_json_resolver.get_closest_package_json(referrer)? + { // ResolveSelf if package_config.name.as_ref() == Some(&package_name) { if let Some(exports) = &package_config.exports { @@ -1216,7 +1161,10 @@ impl NodeResolver { mode: NodeResolutionMode, ) -> Result { let package_json_path = package_dir_path.join("package.json"); - match self.load_package_json(&package_json_path)? { + match self + .pkg_json_resolver + .load_package_json(&package_json_path)? + { Some(pkg_json) => self.resolve_package_subpath( &pkg_json, package_subpath, @@ -1337,70 +1285,6 @@ impl NodeResolver { } } - pub fn get_closest_package_json( - &self, - url: &Url, - ) -> Result, ClosestPkgJsonError> { - let Ok(file_path) = deno_path_util::url_to_file_path(url) else { - return Ok(None); - }; - self.get_closest_package_json_from_path(&file_path) - } - - pub fn get_closest_package_json_from_path( - &self, - file_path: &Path, - ) -> Result, ClosestPkgJsonError> { - // we use this for deno compile using byonm because the script paths - // won't be in virtual file system, but the package.json paths will be - fn canonicalize_first_ancestor_exists( - dir_path: &Path, - env: &dyn NodeResolverEnv, - ) -> Result, std::io::Error> { - for ancestor in dir_path.ancestors() { - match env.realpath_sync(ancestor) { - Ok(dir_path) => return Ok(Some(dir_path)), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - // keep searching - } - Err(err) => return Err(err), - } - } - Ok(None) - } - - let parent_dir = file_path.parent().unwrap(); - let Some(start_dir) = canonicalize_first_ancestor_exists( - parent_dir, &self.env, - ) - .map_err(|source| CanonicalizingPkgJsonDirError { - dir_path: parent_dir.to_path_buf(), - source, - })? - else { - return Ok(None); - }; - let start_dir = strip_unc_prefix(start_dir); - for current_dir in start_dir.ancestors() { - let package_json_path = current_dir.join("package.json"); - if let Some(pkg_json) = self.load_package_json(&package_json_path)? { - return Ok(Some(pkg_json)); - } - } - - Ok(None) - } - - pub fn load_package_json( - &self, - package_json_path: &Path, - ) -> Result, PackageJsonLoadError> { - crate::package_json::load_pkg_json( - self.env.pkg_json_fs(), - package_json_path, - ) - } - pub(super) fn legacy_main_resolve( &self, package_json: &PackageJson, diff --git a/runtime/errors.rs b/runtime/errors.rs index a5c436e751..ada26ec35f 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -1048,8 +1048,6 @@ mod node { WorkerThreadsFilenameError::UrlToPathString => "Error", WorkerThreadsFilenameError::UrlToPath => "Error", WorkerThreadsFilenameError::FileNotFound(_) => "Error", - WorkerThreadsFilenameError::NeitherEsmNorCjs => "Error", - WorkerThreadsFilenameError::UrlToNodeResolution(_) => "Error", WorkerThreadsFilenameError::Fs(e) => super::get_fs_error(e), } } @@ -1058,11 +1056,13 @@ mod node { match error { RequireError::UrlParse(e) => get_url_parse_error_class(e), RequireError::Permission(e) => get_error_class_name(e).unwrap_or("Error"), - RequireError::PackageExportsResolve(_) => "Error", - RequireError::PackageJsonLoad(_) => "Error", - RequireError::ClosestPkgJson(_) => "Error", - RequireError::FilePathConversion(_) => "Error", - RequireError::PackageImportsResolve(_) => "Error", + RequireError::PackageExportsResolve(_) + | RequireError::PackageJsonLoad(_) + | RequireError::ClosestPkgJson(_) + | RequireError::FilePathConversion(_) + | RequireError::UrlConversion(_) + | RequireError::ReadModule(_) + | RequireError::PackageImportsResolve(_) => "Error", RequireError::Fs(e) | RequireError::UnableToGetCwd(e) => { super::get_fs_error(e) } diff --git a/runtime/shared.rs b/runtime/shared.rs index 02dfd18719..f7d76f67a7 100644 --- a/runtime/shared.rs +++ b/runtime/shared.rs @@ -98,6 +98,7 @@ pub fn maybe_transpile_source( imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, ..Default::default() }, + &deno_ast::TranspileModuleOptions::default(), &deno_ast::EmitOptions { source_map: if cfg!(debug_assertions) { SourceMapOption::Separate @@ -109,9 +110,9 @@ pub fn maybe_transpile_source( )? .into_source(); - let maybe_source_map: Option = - transpiled_source.source_map.map(|sm| sm.into()); - let source_text = String::from_utf8(transpiled_source.source)?; - + let maybe_source_map: Option = transpiled_source + .source_map + .map(|sm| sm.into_bytes().into()); + let source_text = transpiled_source.text; Ok((source_text.into(), maybe_source_map)) } diff --git a/tests/integration/run_tests.rs b/tests/integration/run_tests.rs index db9f79556e..a07dd56c83 100644 --- a/tests/integration/run_tests.rs +++ b/tests/integration/run_tests.rs @@ -3605,7 +3605,8 @@ fn running_declaration_files() { temp_dir.write(file, ""); context .new_command() - .args_vec(["run", file]) + // todo(dsherret): investigate why --allow-read is required here + .args_vec(["run", "--allow-read", file]) .run() .skip_output_check() .assert_exit_code(0); diff --git a/tests/registry/npm/@denotest/type-commonjs/1.0.0/index.js b/tests/registry/npm/@denotest/type-commonjs/1.0.0/index.js index cb0ff5c3b5..d3f80a0495 100644 --- a/tests/registry/npm/@denotest/type-commonjs/1.0.0/index.js +++ b/tests/registry/npm/@denotest/type-commonjs/1.0.0/index.js @@ -1 +1,5 @@ -export {}; +// this module is declared as CommonJS, but during loading we'll +// discover it's ESM and load it fine +export function add(a, b) { + return a + b; +} diff --git a/tests/specs/check/css_import/exists_run_with_check.out b/tests/specs/check/css_import/exists_run_with_check.out index 1a1dafeb74..315769e40c 100644 --- a/tests/specs/check/css_import/exists_run_with_check.out +++ b/tests/specs/check/css_import/exists_run_with_check.out @@ -1,3 +1,3 @@ -error: Expected a JavaScript or TypeScript module, but identified a Unknown module. Importing these types of modules is currently not supported. +error: Expected a JavaScript or TypeScript module, but identified a Css module. Importing these types of modules is currently not supported. Specifier: file:///[WILDLINE]/app.css at file:///[WILDLINE]/exists.ts:2:8 diff --git a/tests/specs/compile/cjs/__test__.jsonc b/tests/specs/compile/cjs/__test__.jsonc new file mode 100644 index 0000000000..9bdcf4724f --- /dev/null +++ b/tests/specs/compile/cjs/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + "args": "compile --output main main.js", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "output.out", + "exitCode": 0 + }, { + "if": "windows", + "args": "compile --output main.exe main.js", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "output.out", + "exitCode": 0 + }] +} diff --git a/tests/specs/compile/cjs/add.cjs b/tests/specs/compile/cjs/add.cjs new file mode 100644 index 0000000000..bf90601267 --- /dev/null +++ b/tests/specs/compile/cjs/add.cjs @@ -0,0 +1 @@ +module.exports = (a, b) => a + b; diff --git a/tests/specs/compile/cjs/divide.cts b/tests/specs/compile/cjs/divide.cts new file mode 100644 index 0000000000..d89a600a4e --- /dev/null +++ b/tests/specs/compile/cjs/divide.cts @@ -0,0 +1 @@ +module.exports.divide = (a: number, b: number) => a / b; diff --git a/tests/specs/compile/cjs/main.js b/tests/specs/compile/cjs/main.js new file mode 100644 index 0000000000..c2a8c7be02 --- /dev/null +++ b/tests/specs/compile/cjs/main.js @@ -0,0 +1,5 @@ +import { add } from "./reexport.cjs"; +import { multiply } from "./multiply.cts"; + +console.log(add(1, 2)); +console.log(multiply(2, 3)); diff --git a/tests/specs/compile/cjs/multiply.cts b/tests/specs/compile/cjs/multiply.cts new file mode 100644 index 0000000000..3c0618cfc5 --- /dev/null +++ b/tests/specs/compile/cjs/multiply.cts @@ -0,0 +1,4 @@ +/// +exports.multiply = function (a: number, b: number): number { + return require("./divide.cts").divide(a, 1 / b); +}; diff --git a/tests/specs/compile/cjs/output.out b/tests/specs/compile/cjs/output.out new file mode 100644 index 0000000000..2559e5c49e --- /dev/null +++ b/tests/specs/compile/cjs/output.out @@ -0,0 +1,2 @@ +3 +6 diff --git a/tests/specs/compile/cjs/reexport.cjs b/tests/specs/compile/cjs/reexport.cjs new file mode 100644 index 0000000000..af7cecfdf4 --- /dev/null +++ b/tests/specs/compile/cjs/reexport.cjs @@ -0,0 +1 @@ +module.exports.add = require("./add.cjs"); diff --git a/tests/specs/compile/detect_cjs/__test__.jsonc b/tests/specs/compile/detect_cjs/__test__.jsonc index 32bebb7a57..0abf121f05 100644 --- a/tests/specs/compile/detect_cjs/__test__.jsonc +++ b/tests/specs/compile/detect_cjs/__test__.jsonc @@ -1,24 +1,27 @@ { "tempDir": true, "steps": [{ + "args": "install", + "output": "[WILDCARD]" + }, { "if": "unix", - "args": "compile --allow-read --output main main.js", + "args": "compile --output main main.js", "output": "compile.out" }, { "if": "unix", "commandName": "./main", "args": [], "output": "output.out", - "exitCode": 1 + "exitCode": 0 }, { "if": "windows", - "args": "compile --allow-read --output main.exe main.js", + "args": "compile --output main.exe main.js", "output": "compile.out" }, { "if": "windows", "commandName": "./main.exe", "args": [], "output": "output.out", - "exitCode": 1 + "exitCode": 0 }] } diff --git a/tests/specs/compile/detect_cjs/add.js b/tests/specs/compile/detect_cjs/add.js index 2a886fbc18..94b0263f0e 100644 --- a/tests/specs/compile/detect_cjs/add.js +++ b/tests/specs/compile/detect_cjs/add.js @@ -1,3 +1,3 @@ module.exports.add = function (a, b) { - return a + b; + return require("./subtract.ts").subtract(a, -b); }; diff --git a/tests/specs/compile/detect_cjs/compile.out b/tests/specs/compile/detect_cjs/compile.out index 6509b7f29c..913e363c3e 100644 --- a/tests/specs/compile/detect_cjs/compile.out +++ b/tests/specs/compile/detect_cjs/compile.out @@ -1,3 +1,2 @@ -Warning --unstable-detect-cjs is not properly supported in deno compile. The compiled executable may encounter runtime errors. Check file:///[WILDLINE]/main.js Compile file:///[WILDLINE] diff --git a/tests/specs/compile/detect_cjs/output.out b/tests/specs/compile/detect_cjs/output.out index e1c27b8dcf..00750edc07 100644 --- a/tests/specs/compile/detect_cjs/output.out +++ b/tests/specs/compile/detect_cjs/output.out @@ -1,2 +1 @@ -error: Uncaught SyntaxError: The requested module './add.js' does not provide an export named 'add' - at (file:///[WILDLINE]) +3 diff --git a/tests/specs/compile/detect_cjs/package.json b/tests/specs/compile/detect_cjs/package.json index 5bbefffbab..6e65b32ed5 100644 --- a/tests/specs/compile/detect_cjs/package.json +++ b/tests/specs/compile/detect_cjs/package.json @@ -1,3 +1,6 @@ { - "type": "commonjs" + "type": "commonjs", + "dependencies": { + "@types/node": "*" + } } diff --git a/tests/specs/compile/detect_cjs/subtract.ts b/tests/specs/compile/detect_cjs/subtract.ts new file mode 100644 index 0000000000..e4f6760b77 --- /dev/null +++ b/tests/specs/compile/detect_cjs/subtract.ts @@ -0,0 +1,2 @@ +/// +module.exports.subtract = (a: number, b: number) => a - b; diff --git a/tests/specs/npm/require_type_commonjs/__test__.jsonc b/tests/specs/npm/require_type_commonjs/__test__.jsonc index c9ba97ff52..a71173ce37 100644 --- a/tests/specs/npm/require_type_commonjs/__test__.jsonc +++ b/tests/specs/npm/require_type_commonjs/__test__.jsonc @@ -1,5 +1,4 @@ { "args": "run --allow-read --quiet main.ts", - "output": "main.out", - "exitCode": 1 + "output": "main.out" } diff --git a/tests/specs/npm/require_type_commonjs/main.out b/tests/specs/npm/require_type_commonjs/main.out index d715db8a94..00750edc07 100644 --- a/tests/specs/npm/require_type_commonjs/main.out +++ b/tests/specs/npm/require_type_commonjs/main.out @@ -1,4 +1 @@ -error: 'import', and 'export' cannot be used outside of module code at file://[WILDCARD]/@denotest/type-commonjs/1.0.0/index.js:1:1 - - export {}; - ~~~~~~ +3 diff --git a/tests/specs/npm/require_type_commonjs/main.ts b/tests/specs/npm/require_type_commonjs/main.ts index 243eb216e4..95ec6099ec 100644 --- a/tests/specs/npm/require_type_commonjs/main.ts +++ b/tests/specs/npm/require_type_commonjs/main.ts @@ -1 +1,2 @@ -import "npm:@denotest/type-commonjs"; +import { add } from "npm:@denotest/type-commonjs"; +console.log(add(1, 2)); diff --git a/tests/specs/npm/typescript_file_in_package/__test__.jsonc b/tests/specs/npm/typescript_file_in_package/__test__.jsonc index 08979ed257..7b5c5e1b67 100644 --- a/tests/specs/npm/typescript_file_in_package/__test__.jsonc +++ b/tests/specs/npm/typescript_file_in_package/__test__.jsonc @@ -1,5 +1,5 @@ { - "args": "run typescript_file_in_package/main.ts", - "output": "typescript_file_in_package/main.out", + "args": "run main.ts", + "output": "main.out", "exitCode": 1 } diff --git a/tests/specs/npm/typescript_file_in_package/main.out b/tests/specs/npm/typescript_file_in_package/main.out new file mode 100644 index 0000000000..58290a8737 --- /dev/null +++ b/tests/specs/npm/typescript_file_in_package/main.out @@ -0,0 +1,3 @@ +Download http://localhost:4260/@denotest%2ftypescript-file +Download http://localhost:4260/@denotest/typescript-file/1.0.0.tgz +error: TypeScript files are not supported in npm packages: file:///[WILDCARD]/@denotest/typescript-file/1.0.0/index.ts diff --git a/tests/specs/npm/typescript_file_in_package/typescript_file_in_package/main.ts b/tests/specs/npm/typescript_file_in_package/main.ts similarity index 100% rename from tests/specs/npm/typescript_file_in_package/typescript_file_in_package/main.ts rename to tests/specs/npm/typescript_file_in_package/main.ts diff --git a/tests/specs/npm/typescript_file_in_package/typescript_file_in_package/main.out b/tests/specs/npm/typescript_file_in_package/typescript_file_in_package/main.out deleted file mode 100644 index b3faa87900..0000000000 --- a/tests/specs/npm/typescript_file_in_package/typescript_file_in_package/main.out +++ /dev/null @@ -1,6 +0,0 @@ -Download http://localhost:4260/@denotest%2ftypescript-file -Download http://localhost:4260/@denotest/typescript-file/1.0.0.tgz -error: Could not resolve 'npm:@denotest/typescript-file@1.0.0'. - -Caused by: - TypeScript files are not supported in npm packages: file:///[WILDCARD]/@denotest/typescript-file/1.0.0/index.ts diff --git a/tests/specs/run/cjs/main_module/__test__.jsonc b/tests/specs/run/cjs/main_module/__test__.jsonc new file mode 100644 index 0000000000..e756a63627 --- /dev/null +++ b/tests/specs/run/cjs/main_module/__test__.jsonc @@ -0,0 +1,4 @@ +{ + "args": "run --allow-read main.cjs", + "output": "main.out" +} diff --git a/tests/specs/run/cjs/main_module/main.cjs b/tests/specs/run/cjs/main_module/main.cjs new file mode 100644 index 0000000000..2a4c57ab1c --- /dev/null +++ b/tests/specs/run/cjs/main_module/main.cjs @@ -0,0 +1 @@ +console.log(require.main); diff --git a/tests/specs/run/cjs/main_module/main.out b/tests/specs/run/cjs/main_module/main.out new file mode 100644 index 0000000000..93b86d27e3 --- /dev/null +++ b/tests/specs/run/cjs/main_module/main.out @@ -0,0 +1,5 @@ +Module { + id: ".", + [WILDCARD] + filename: "[WILDCARD]main.cjs", +[WILDCARD] \ No newline at end of file diff --git a/tests/specs/run/cjs_reexport_non_analyzable/__test__.jsonc b/tests/specs/run/cjs/reexport_non_analyzable/__test__.jsonc similarity index 100% rename from tests/specs/run/cjs_reexport_non_analyzable/__test__.jsonc rename to tests/specs/run/cjs/reexport_non_analyzable/__test__.jsonc diff --git a/tests/specs/run/cjs_reexport_non_analyzable/deno.json b/tests/specs/run/cjs/reexport_non_analyzable/deno.json similarity index 100% rename from tests/specs/run/cjs_reexport_non_analyzable/deno.json rename to tests/specs/run/cjs/reexport_non_analyzable/deno.json diff --git a/tests/specs/run/cjs_reexport_non_analyzable/main.ts b/tests/specs/run/cjs/reexport_non_analyzable/main.ts similarity index 100% rename from tests/specs/run/cjs_reexport_non_analyzable/main.ts rename to tests/specs/run/cjs/reexport_non_analyzable/main.ts diff --git a/tests/specs/run/cjs_reexport_non_analyzable/node_modules/foo.cjs b/tests/specs/run/cjs/reexport_non_analyzable/node_modules/foo.cjs similarity index 100% rename from tests/specs/run/cjs_reexport_non_analyzable/node_modules/foo.cjs rename to tests/specs/run/cjs/reexport_non_analyzable/node_modules/foo.cjs diff --git a/tests/specs/run/cjs/unprepared/__test__.jsonc b/tests/specs/run/cjs/unprepared/__test__.jsonc new file mode 100644 index 0000000000..f816bad869 --- /dev/null +++ b/tests/specs/run/cjs/unprepared/__test__.jsonc @@ -0,0 +1,4 @@ +{ + "args": "run -A main.ts", + "output": "main.out" +} diff --git a/tests/specs/run/cjs/unprepared/file.cjs b/tests/specs/run/cjs/unprepared/file.cjs new file mode 100644 index 0000000000..ba0c6fa6ff --- /dev/null +++ b/tests/specs/run/cjs/unprepared/file.cjs @@ -0,0 +1,7 @@ +// non-analyzable +const moduleName = "./output.cjs"; +function getModuleName() { + return moduleName; +} + +require(getModuleName()); diff --git a/tests/specs/run/cjs/unprepared/main.out b/tests/specs/run/cjs/unprepared/main.out new file mode 100644 index 0000000000..e965047ad7 --- /dev/null +++ b/tests/specs/run/cjs/unprepared/main.out @@ -0,0 +1 @@ +Hello diff --git a/tests/specs/run/cjs/unprepared/main.ts b/tests/specs/run/cjs/unprepared/main.ts new file mode 100644 index 0000000000..5630b4fbc3 --- /dev/null +++ b/tests/specs/run/cjs/unprepared/main.ts @@ -0,0 +1,7 @@ +// non-analyzable +const moduleName = "./output.cjs"; +function getModuleName() { + return moduleName; +} + +await import(getModuleName()); diff --git a/tests/specs/run/cjs/unprepared/output.cjs b/tests/specs/run/cjs/unprepared/output.cjs new file mode 100644 index 0000000000..54ed3702f5 --- /dev/null +++ b/tests/specs/run/cjs/unprepared/output.cjs @@ -0,0 +1,3 @@ +console.log("Hello"); + +module.exports = 1; diff --git a/tests/specs/run/cts/cjs_import_cts/__test__.jsonc b/tests/specs/run/cts/cjs_import_cts/__test__.jsonc new file mode 100644 index 0000000000..2205183d07 --- /dev/null +++ b/tests/specs/run/cts/cjs_import_cts/__test__.jsonc @@ -0,0 +1,13 @@ +{ + "tests": { + "no_check": { + "args": "run --allow-read main.js", + "output": "main.out" + }, + "check": { + "args": "check main.js", + "output": "check.out", + "exitCode": 1 + } + } +} diff --git a/tests/specs/run/package_json_type/commonjs/add.js b/tests/specs/run/cts/cjs_import_cts/add.cts similarity index 100% rename from tests/specs/run/package_json_type/commonjs/add.js rename to tests/specs/run/cts/cjs_import_cts/add.cts diff --git a/tests/specs/run/cts/cjs_import_cts/check.out b/tests/specs/run/cts/cjs_import_cts/check.out new file mode 100644 index 0000000000..a27e8d7af7 --- /dev/null +++ b/tests/specs/run/cts/cjs_import_cts/check.out @@ -0,0 +1,17 @@ +Check file:///[WILDLINE]main.js +error: TS2580 [ERROR]: Cannot find name 'module'. +module.exports.add = function (a, b) { +~~~~~~ + at file:///[WILDLINE] + +TS7006 [ERROR]: Parameter 'a' implicitly has an 'any' type. +module.exports.add = function (a, b) { + ^ + at file:///[WILDLINE] + +TS7006 [ERROR]: Parameter 'b' implicitly has an 'any' type. +module.exports.add = function (a, b) { + ^ + at file:///[WILDLINE] + +Found 3 errors. diff --git a/tests/specs/run/cts/cjs_import_cts/main.js b/tests/specs/run/cts/cjs_import_cts/main.js new file mode 100644 index 0000000000..9546a0fea5 --- /dev/null +++ b/tests/specs/run/cts/cjs_import_cts/main.js @@ -0,0 +1,3 @@ +import { subtract } from "./subtract.cjs"; + +console.log(subtract(1, 2)); diff --git a/tests/specs/run/cts/cjs_import_cts/main.out b/tests/specs/run/cts/cjs_import_cts/main.out new file mode 100644 index 0000000000..3a2e3f4984 --- /dev/null +++ b/tests/specs/run/cts/cjs_import_cts/main.out @@ -0,0 +1 @@ +-1 diff --git a/tests/specs/run/cts/cjs_import_cts/subtract.cjs b/tests/specs/run/cts/cjs_import_cts/subtract.cjs new file mode 100644 index 0000000000..7dee54346f --- /dev/null +++ b/tests/specs/run/cts/cjs_import_cts/subtract.cjs @@ -0,0 +1,3 @@ +module.exports.subtract = function (a, b) { + return require("./add.cts").add(a, -b); +}; diff --git a/tests/specs/run/cts/import_export_equals/__test__.jsonc b/tests/specs/run/cts/import_export_equals/__test__.jsonc new file mode 100644 index 0000000000..6a5c6db420 --- /dev/null +++ b/tests/specs/run/cts/import_export_equals/__test__.jsonc @@ -0,0 +1,17 @@ +{ + "tests": { + "main": { + "args": "run --check --allow-read=. main.cts", + "output": "main.out" + }, + "mts": { + "args": "run --allow-read=. mod.mts", + "output": "mod.mts.out" + }, + "mts_check": { + "args": "check mod.mts", + "output": "mod.mts.check.out", + "exitCode": 1 + } + } +} diff --git a/tests/specs/run/cts/import_export_equals/add.cts b/tests/specs/run/cts/import_export_equals/add.cts new file mode 100644 index 0000000000..adf3503ac5 --- /dev/null +++ b/tests/specs/run/cts/import_export_equals/add.cts @@ -0,0 +1,3 @@ +export = function (a: number, b: number) { + return a + b; +}; diff --git a/tests/specs/run/cts/import_export_equals/main.cts b/tests/specs/run/cts/import_export_equals/main.cts new file mode 100644 index 0000000000..e5c45f92e0 --- /dev/null +++ b/tests/specs/run/cts/import_export_equals/main.cts @@ -0,0 +1,3 @@ +import add = require("./add.cts"); + +console.log(add(1, 2)); diff --git a/tests/specs/run/cts/import_export_equals/main.out b/tests/specs/run/cts/import_export_equals/main.out new file mode 100644 index 0000000000..e7a973a7d0 --- /dev/null +++ b/tests/specs/run/cts/import_export_equals/main.out @@ -0,0 +1,2 @@ +Check file:///[WILDLINE]/main.cts +3 diff --git a/tests/specs/run/cts/import_export_equals/mod.mts b/tests/specs/run/cts/import_export_equals/mod.mts new file mode 100644 index 0000000000..5fbbd6c6a0 --- /dev/null +++ b/tests/specs/run/cts/import_export_equals/mod.mts @@ -0,0 +1,3 @@ +import add from "./add.cts"; + +console.log(add(1, "test")); diff --git a/tests/specs/run/cts/import_export_equals/mod.mts.check.out b/tests/specs/run/cts/import_export_equals/mod.mts.check.out new file mode 100644 index 0000000000..8703539019 --- /dev/null +++ b/tests/specs/run/cts/import_export_equals/mod.mts.check.out @@ -0,0 +1,5 @@ +Check file:///[WILDLINE]/mod.mts +error: TS2345 [ERROR]: Argument of type 'string' is not assignable to parameter of type 'number'. +console.log(add(1, "test")); + ~~~~~~ + at file:///[WILDLINE]/mod.mts:3:20 diff --git a/tests/specs/run/cts/import_export_equals/mod.mts.out b/tests/specs/run/cts/import_export_equals/mod.mts.out new file mode 100644 index 0000000000..208465a080 --- /dev/null +++ b/tests/specs/run/cts/import_export_equals/mod.mts.out @@ -0,0 +1 @@ +1test diff --git a/tests/specs/run/cts/main/__test__.jsonc b/tests/specs/run/cts/main/__test__.jsonc new file mode 100644 index 0000000000..0157b44e96 --- /dev/null +++ b/tests/specs/run/cts/main/__test__.jsonc @@ -0,0 +1,4 @@ +{ + "args": "run --allow-read main.cts", + "output": "main.out" +} diff --git a/tests/specs/run/cts/main/import_main.cjs b/tests/specs/run/cts/main/import_main.cjs new file mode 100644 index 0000000000..f61e0da2a6 --- /dev/null +++ b/tests/specs/run/cts/main/import_main.cjs @@ -0,0 +1 @@ +require("./main.cts").sayHello(); diff --git a/tests/specs/run/cts/main/main.cts b/tests/specs/run/cts/main/main.cts new file mode 100644 index 0000000000..0b0330686b --- /dev/null +++ b/tests/specs/run/cts/main/main.cts @@ -0,0 +1,5 @@ +module.exports.sayHello = function () { + console.log("Hello"); +}; + +require("./import_main.cjs"); diff --git a/tests/specs/run/cts/main/main.out b/tests/specs/run/cts/main/main.out new file mode 100644 index 0000000000..e965047ad7 --- /dev/null +++ b/tests/specs/run/cts/main/main.out @@ -0,0 +1 @@ +Hello diff --git a/tests/specs/run/package_json_type/commonjs/__test__.jsonc b/tests/specs/run/package_json_type/commonjs/basic/__test__.jsonc similarity index 100% rename from tests/specs/run/package_json_type/commonjs/__test__.jsonc rename to tests/specs/run/package_json_type/commonjs/basic/__test__.jsonc diff --git a/tests/specs/run/package_json_type/commonjs/basic/add.js b/tests/specs/run/package_json_type/commonjs/basic/add.js new file mode 100644 index 0000000000..2a886fbc18 --- /dev/null +++ b/tests/specs/run/package_json_type/commonjs/basic/add.js @@ -0,0 +1,3 @@ +module.exports.add = function (a, b) { + return a + b; +}; diff --git a/tests/specs/run/package_json_type/commonjs/deno.jsonc b/tests/specs/run/package_json_type/commonjs/basic/deno.jsonc similarity index 100% rename from tests/specs/run/package_json_type/commonjs/deno.jsonc rename to tests/specs/run/package_json_type/commonjs/basic/deno.jsonc diff --git a/tests/specs/run/package_json_type/commonjs/import_import_meta.js b/tests/specs/run/package_json_type/commonjs/basic/import_import_meta.js similarity index 100% rename from tests/specs/run/package_json_type/commonjs/import_import_meta.js rename to tests/specs/run/package_json_type/commonjs/basic/import_import_meta.js diff --git a/tests/specs/run/package_json_type/commonjs/import_meta.js b/tests/specs/run/package_json_type/commonjs/basic/import_meta.js similarity index 100% rename from tests/specs/run/package_json_type/commonjs/import_meta.js rename to tests/specs/run/package_json_type/commonjs/basic/import_meta.js diff --git a/tests/specs/run/package_json_type/commonjs/main_cjs.js b/tests/specs/run/package_json_type/commonjs/basic/main_cjs.js similarity index 100% rename from tests/specs/run/package_json_type/commonjs/main_cjs.js rename to tests/specs/run/package_json_type/commonjs/basic/main_cjs.js diff --git a/tests/specs/run/package_json_type/commonjs/main_esm.js b/tests/specs/run/package_json_type/commonjs/basic/main_esm.js similarity index 100% rename from tests/specs/run/package_json_type/commonjs/main_esm.js rename to tests/specs/run/package_json_type/commonjs/basic/main_esm.js diff --git a/tests/specs/run/package_json_type/commonjs/main_esm_import_meta.js b/tests/specs/run/package_json_type/commonjs/basic/main_esm_import_meta.js similarity index 100% rename from tests/specs/run/package_json_type/commonjs/main_esm_import_meta.js rename to tests/specs/run/package_json_type/commonjs/basic/main_esm_import_meta.js diff --git a/tests/specs/run/package_json_type/commonjs/main_esm_import_meta.out b/tests/specs/run/package_json_type/commonjs/basic/main_esm_import_meta.out similarity index 100% rename from tests/specs/run/package_json_type/commonjs/main_esm_import_meta.out rename to tests/specs/run/package_json_type/commonjs/basic/main_esm_import_meta.out diff --git a/tests/specs/run/package_json_type/commonjs/main_mix.js b/tests/specs/run/package_json_type/commonjs/basic/main_mix.js similarity index 100% rename from tests/specs/run/package_json_type/commonjs/main_mix.js rename to tests/specs/run/package_json_type/commonjs/basic/main_mix.js diff --git a/tests/specs/run/package_json_type/commonjs/main_mix.out b/tests/specs/run/package_json_type/commonjs/basic/main_mix.out similarity index 100% rename from tests/specs/run/package_json_type/commonjs/main_mix.out rename to tests/specs/run/package_json_type/commonjs/basic/main_mix.out diff --git a/tests/specs/run/package_json_type/commonjs/not_import_meta.js b/tests/specs/run/package_json_type/commonjs/basic/not_import_meta.js similarity index 100% rename from tests/specs/run/package_json_type/commonjs/not_import_meta.js rename to tests/specs/run/package_json_type/commonjs/basic/not_import_meta.js diff --git a/tests/specs/run/package_json_type/commonjs/package.json b/tests/specs/run/package_json_type/commonjs/basic/package.json similarity index 100% rename from tests/specs/run/package_json_type/commonjs/package.json rename to tests/specs/run/package_json_type/commonjs/basic/package.json diff --git a/tests/specs/run/package_json_type/commonjs/tla.js b/tests/specs/run/package_json_type/commonjs/basic/tla.js similarity index 100% rename from tests/specs/run/package_json_type/commonjs/tla.js rename to tests/specs/run/package_json_type/commonjs/basic/tla.js diff --git a/tests/specs/run/package_json_type/commonjs/jsx/__test__.jsonc b/tests/specs/run/package_json_type/commonjs/jsx/__test__.jsonc new file mode 100644 index 0000000000..f815fd72cb --- /dev/null +++ b/tests/specs/run/package_json_type/commonjs/jsx/__test__.jsonc @@ -0,0 +1,5 @@ +{ + "tempDir": true, + "args": "run -A --quiet main.jsx", + "output": "main.out" +} diff --git a/tests/specs/run/package_json_type/commonjs/jsx/add.js b/tests/specs/run/package_json_type/commonjs/jsx/add.js new file mode 100644 index 0000000000..2a886fbc18 --- /dev/null +++ b/tests/specs/run/package_json_type/commonjs/jsx/add.js @@ -0,0 +1,3 @@ +module.exports.add = function (a, b) { + return a + b; +}; diff --git a/tests/specs/run/package_json_type/commonjs/jsx/deno.jsonc b/tests/specs/run/package_json_type/commonjs/jsx/deno.jsonc new file mode 100644 index 0000000000..192ddb98c4 --- /dev/null +++ b/tests/specs/run/package_json_type/commonjs/jsx/deno.jsonc @@ -0,0 +1,10 @@ +{ + "nodeModulesDir": "auto", + "compilerOptions": { + "jsx": "react-jsx", + "jsxImportSource": "react" + }, + "unstable": [ + "detect-cjs" + ] +} diff --git a/tests/specs/run/package_json_type/commonjs/jsx/main.jsx b/tests/specs/run/package_json_type/commonjs/jsx/main.jsx new file mode 100644 index 0000000000..1922fce1bb --- /dev/null +++ b/tests/specs/run/package_json_type/commonjs/jsx/main.jsx @@ -0,0 +1,7 @@ +const { add } = require("./add.js"); + +console.log(add(1, 2)); + +console.log(

!= null); + +require("./tsx.tsx"); diff --git a/tests/specs/run/package_json_type/commonjs/jsx/main.out b/tests/specs/run/package_json_type/commonjs/jsx/main.out new file mode 100644 index 0000000000..cdcf045418 --- /dev/null +++ b/tests/specs/run/package_json_type/commonjs/jsx/main.out @@ -0,0 +1,4 @@ +3 +true +4 +true diff --git a/tests/specs/run/package_json_type/commonjs/jsx/package.json b/tests/specs/run/package_json_type/commonjs/jsx/package.json new file mode 100644 index 0000000000..88afcdcd95 --- /dev/null +++ b/tests/specs/run/package_json_type/commonjs/jsx/package.json @@ -0,0 +1,7 @@ +{ + "type": "commonjs", + "dependencies": { + "@types/react": "*", + "react": "*" + } +} diff --git a/tests/specs/run/package_json_type/commonjs/jsx/tsx.tsx b/tests/specs/run/package_json_type/commonjs/jsx/tsx.tsx new file mode 100644 index 0000000000..ad8f0c0a9c --- /dev/null +++ b/tests/specs/run/package_json_type/commonjs/jsx/tsx.tsx @@ -0,0 +1,5 @@ +import mod = require("./add.js"); + +console.log(mod.add(2, 2)); + +console.log(
!= null); diff --git a/tests/specs/run/remote_cjs_main/output.out b/tests/specs/run/remote_cjs_main/output.out index f75c33907a..360934acb7 100644 --- a/tests/specs/run/remote_cjs_main/output.out +++ b/tests/specs/run/remote_cjs_main/output.out @@ -1,3 +1,3 @@ Download http://localhost:4545/run/add.cjs -error: Expected a JavaScript or TypeScript module, but identified a Cjs module. Importing these types of modules is currently not supported. +error: Remote CJS modules are not supported. Specifier: http://localhost:4545/run/add.cjs diff --git a/tests/specs/run/require_esm/main.out b/tests/specs/run/require_esm/main.out index d17b1ead55..57b842b345 100644 --- a/tests/specs/run/require_esm/main.out +++ b/tests/specs/run/require_esm/main.out @@ -1,6 +1,6 @@ [Module: null prototype] { sync_js: 1 } [Module: null prototype] { sync_mjs: 1 } -error: Uncaught Error: Top-level await is not allowed in synchronous evaluation +error: Uncaught (in promise) Error: Top-level await is not allowed in synchronous evaluation at loadESMFromCJS (node:module:[WILDCARD]) at Module._compile (node:module:[WILDCARD]) at Object.Module._extensions..js (node:module:[WILDCARD]) diff --git a/tests/testdata/npm/deno_run_cowsay_no_permissions.out b/tests/testdata/npm/deno_run_cowsay_no_permissions.out index 6434620e2e..25b79d9a7d 100644 --- a/tests/testdata/npm/deno_run_cowsay_no_permissions.out +++ b/tests/testdata/npm/deno_run_cowsay_no_permissions.out @@ -1,2 +1,2 @@ -error: Uncaught NotCapable: Requires read access to , specify the required permissions during compilation using `deno compile --allow-read` +error: Uncaught (in promise) NotCapable: Requires read access to , specify the required permissions during compilation using `deno compile --allow-read` [WILDCARD] From 2c8a0e791732ab00907ca11c3a4918ad26ead03f Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Fri, 1 Nov 2024 19:10:35 -0700 Subject: [PATCH 63/63] fix(add): only add npm deps to package.json if it's at least as close as deno.json (#26683) Fixes https://github.com/denoland/deno/issues/26653 --- cli/tools/registry/pm.rs | 26 ++++++++++++++++++- .../package_json_and_deno_json/__test__.jsonc | 23 ++++++++++++++++ .../subdir/deno.json | 4 +++ .../package_json_and_deno_json/subdir/mod.ts | 0 .../subdir/prefer_if_closer_deno.json.out | 9 +++++++ 5 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/specs/add/package_json_and_deno_json/subdir/deno.json create mode 100644 tests/specs/add/package_json_and_deno_json/subdir/mod.ts create mode 100644 tests/specs/add/package_json_and_deno_json/subdir/prefer_if_closer_deno.json.out diff --git a/cli/tools/registry/pm.rs b/cli/tools/registry/pm.rs index d1be901d67..4e03879983 100644 --- a/cli/tools/registry/pm.rs +++ b/cli/tools/registry/pm.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -333,6 +334,14 @@ fn load_configs( Ok((cli_factory, npm_config, deno_config)) } +fn path_distance(a: &Path, b: &Path) -> usize { + let diff = pathdiff::diff_paths(a, b); + let Some(diff) = diff else { + return usize::MAX; + }; + diff.components().count() +} + pub async fn add( flags: Arc, add_flags: AddFlags, @@ -357,6 +366,21 @@ pub async fn add( } } + let start_dir = cli_factory.cli_options()?.start_dir.dir_path(); + + // only prefer to add npm deps to `package.json` if there isn't a closer deno.json. + // example: if deno.json is in the CWD and package.json is in the parent, we should add + // npm deps to deno.json, since it's closer + let prefer_npm_config = match (npm_config.as_ref(), deno_config.as_ref()) { + (Some(npm), Some(deno)) => { + let npm_distance = path_distance(&npm.path, &start_dir); + let deno_distance = path_distance(&deno.path, &start_dir); + npm_distance <= deno_distance + } + (Some(_), None) => true, + (None, _) => false, + }; + let http_client = cli_factory.http_client_provider(); let deps_http_cache = cli_factory.global_http_cache()?; let mut deps_file_fetcher = FileFetcher::new( @@ -455,7 +479,7 @@ pub async fn add( selected_package.selected_version ); - if selected_package.package_name.starts_with("npm:") { + if selected_package.package_name.starts_with("npm:") && prefer_npm_config { if let Some(npm) = &mut npm_config { npm.add(selected_package, dev); } else { diff --git a/tests/specs/add/package_json_and_deno_json/__test__.jsonc b/tests/specs/add/package_json_and_deno_json/__test__.jsonc index 0beee02d15..8d67a07c82 100644 --- a/tests/specs/add/package_json_and_deno_json/__test__.jsonc +++ b/tests/specs/add/package_json_and_deno_json/__test__.jsonc @@ -41,6 +41,29 @@ "output": "good\n" } ] + }, + "only_prefers_package_json_if_closer": { + "steps": [ + { + "cwd": "./subdir", + "args": "add npm:@denotest/esm-basic jsr:@denotest/add npm:@denotest/say-hello", + "output": "[WILDCARD]" + }, + { + "args": [ + "eval", + "console.log(Deno.readTextFileSync('package.json').trim())" + ], + "output": "{}\n" + }, + { + "args": [ + "eval", + "console.log(Deno.readTextFileSync('./subdir/deno.json').trim())" + ], + "output": "subdir/prefer_if_closer_deno.json.out" + } + ] } } } diff --git a/tests/specs/add/package_json_and_deno_json/subdir/deno.json b/tests/specs/add/package_json_and_deno_json/subdir/deno.json new file mode 100644 index 0000000000..cc54bbd8a2 --- /dev/null +++ b/tests/specs/add/package_json_and_deno_json/subdir/deno.json @@ -0,0 +1,4 @@ +{ + "name": "@test/subdir", + "exports": "./mod.ts" +} diff --git a/tests/specs/add/package_json_and_deno_json/subdir/mod.ts b/tests/specs/add/package_json_and_deno_json/subdir/mod.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/specs/add/package_json_and_deno_json/subdir/prefer_if_closer_deno.json.out b/tests/specs/add/package_json_and_deno_json/subdir/prefer_if_closer_deno.json.out new file mode 100644 index 0000000000..8a5819d503 --- /dev/null +++ b/tests/specs/add/package_json_and_deno_json/subdir/prefer_if_closer_deno.json.out @@ -0,0 +1,9 @@ +{ + "name": "@test/subdir", + "exports": "./mod.ts", + "imports": { + "@denotest/add": "jsr:@denotest/add@^1.0.0", + "@denotest/esm-basic": "npm:@denotest/esm-basic@^1.0.0", + "@denotest/say-hello": "npm:@denotest/say-hello@^1.0.0" + } +}