foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-27 16:10:57 -05:00
Code Issues Packages 1 Wiki Activity

Merge branch 'main' into fix-internal-test-types

Browse source
This commit is contained in:
Satya Rohith 2024-11-02 15:09:34 +05:30 committed by GitHub
commit 83272f7e8b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
761 changed files with 16267 additions and 8143 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.dprint.json
View file

@ -68,8 +68,8 @@
"third_party"
],
"plugins": [
"https://plugins.dprint.dev/typescript-0.93.0.wasm",
"https://plugins.dprint.dev/json-0.19.3.wasm",
"https://plugins.dprint.dev/typescript-0.93.1.wasm",
"https://plugins.dprint.dev/json-0.19.4.wasm",
"https://plugins.dprint.dev/markdown-0.17.8.wasm",
"https://plugins.dprint.dev/toml-0.6.3.wasm",
"https://plugins.dprint.dev/exec-0.5.0.json@8d9972eee71fa1590e04873540421f3eda7674d0f1aae3d7c788615e7b7413d0",

7
.github/workflows/cargo_publish.yml vendored
View file

@ -2,6 +2,11 @@ name: cargo_publish
on: workflow_dispatch
# Ensures only one publish is running at a time
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: true
jobs:
build:
name: cargo publish
@ -28,7 +33,7 @@ jobs:
- uses: dsherret/rust-toolchain-file@v1
- name: Install deno
uses: denoland/setup-deno@v1
uses: denoland/setup-deno@v2
with:
deno-version: v1.x

46
.github/workflows/ci.generate.ts vendored
View file

@ -5,10 +5,10 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify";
// Bump this number when you want to purge the cache.
// Note: the tools/release/01_bump_crate_versions.ts script will update this version
// automatically via regex, so ensure that this line maintains this format.
const cacheVersion = 18;
const cacheVersion = 23;
const ubuntuX86Runner = "ubuntu-22.04";
const ubuntuX86XlRunner = "ubuntu-22.04-xl";
const ubuntuX86Runner = "ubuntu-24.04";
const ubuntuX86XlRunner = "ubuntu-24.04-xl";
const ubuntuARMRunner = "ubicloud-standard-16-arm";
const windowsX86Runner = "windows-2022";
const windowsX86XlRunner = "windows-2022-xl";
@ -59,7 +59,7 @@ const prCacheKeyPrefix =
`${cacheVersion}-cargo-target-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ matrix.profile }}-\${{ matrix.job }}-`;
// Note that you may need to add more version to the `apt-get remove` line below if you change this
const llvmVersion = 18;
const llvmVersion = 19;
const installPkgsCommand =
`sudo apt-get install --no-install-recommends clang-${llvmVersion} lld-${llvmVersion} clang-tools-${llvmVersion} clang-format-${llvmVersion} clang-tidy-${llvmVersion}`;
const sysRootStep = {
@ -71,7 +71,7 @@ export DEBIAN_FRONTEND=noninteractive
sudo apt-get -qq remove --purge -y man-db > /dev/null 2> /dev/null
# Remove older clang before we install
sudo apt-get -qq remove \
'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' > /dev/null 2> /dev/null
'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'clang-17*' 'clang-18*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' 'lld-17*' 'lld-18*' > /dev/null 2> /dev/null
# Install clang-XXX, lld-XXX, and debootstrap.
echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${llvmVersion} main" |
@ -86,7 +86,7 @@ ${installPkgsCommand} || echo 'Failed. Trying again.' && sudo apt-get clean && s
(yes '' | sudo update-alternatives --force --all) > /dev/null 2> /dev/null || true
echo "Decompressing sysroot..."
wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20240528/sysroot-\`uname -m\`.tar.xz -O /tmp/sysroot.tar.xz
wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20241030/sysroot-\`uname -m\`.tar.xz -O /tmp/sysroot.tar.xz
cd /
xzcat /tmp/sysroot.tar.xz | sudo tar -x
sudo mount --rbind /dev /sysroot/dev
@ -193,7 +193,7 @@ const installNodeStep = {
};
const installDenoStep = {
name: "Install Deno",
uses: "denoland/setup-deno@v1",
uses: "denoland/setup-deno@v2",
with: { "deno-version": "v1.x" },
};
@ -751,11 +751,11 @@ const ci = {
].join("\n"),
run: [
"cd target/release",
"shasum -a 256 deno > deno-${{ matrix.arch }}-unknown-linux-gnu.sha256sum",
"zip -r deno-${{ matrix.arch }}-unknown-linux-gnu.zip deno",
"shasum -a 256 deno-${{ matrix.arch }}-unknown-linux-gnu.zip > deno-${{ matrix.arch }}-unknown-linux-gnu.zip.sha256sum",
"strip denort",
"shasum -a 256 denort > denort-${{ matrix.arch }}-unknown-linux-gnu.sha256sum",
"zip -r denort-${{ matrix.arch }}-unknown-linux-gnu.zip denort",
"shasum -a 256 denort-${{ matrix.arch }}-unknown-linux-gnu.zip > denort-${{ matrix.arch }}-unknown-linux-gnu.zip.sha256sum",
"./deno types > lib.deno.d.ts",
].join("\n"),
},
@ -779,11 +779,11 @@ const ci = {
"--p12-file=<(echo $APPLE_CODESIGN_KEY | base64 -d) " +
"--entitlements-xml-file=cli/entitlements.plist",
"cd target/release",
"shasum -a 256 deno > deno-${{ matrix.arch }}-apple-darwin.sha256sum",
"zip -r deno-${{ matrix.arch }}-apple-darwin.zip deno",
"shasum -a 256 deno-${{ matrix.arch }}-apple-darwin.zip > deno-${{ matrix.arch }}-apple-darwin.zip.sha256sum",
"strip denort",
"shasum -a 256 denort > denort-${{ matrix.arch }}-apple-darwin.sha256sum",
"zip -r denort-${{ matrix.arch }}-apple-darwin.zip denort",
"shasum -a 256 denort-${{ matrix.arch }}-apple-darwin.zip > denort-${{ matrix.arch }}-apple-darwin.zip.sha256sum",
]
.join("\n"),
},
@ -797,10 +797,10 @@ const ci = {
].join("\n"),
shell: "pwsh",
run: [
"Get-FileHash target/release/deno.exe -Algorithm SHA256 | Format-List > target/release/deno-${{ matrix.arch }}-pc-windows-msvc.sha256sum",
"Compress-Archive -CompressionLevel Optimal -Force -Path target/release/deno.exe -DestinationPath target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip",
"Get-FileHash target/release/denort.exe -Algorithm SHA256 | Format-List > target/release/denort-${{ matrix.arch }}-pc-windows-msvc.sha256sum",
"Get-FileHash target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip -Algorithm SHA256 | Format-List > target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip.sha256sum",
"Compress-Archive -CompressionLevel Optimal -Force -Path target/release/denort.exe -DestinationPath target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip",
"Get-FileHash target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip -Algorithm SHA256 | Format-List > target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip.sha256sum",
].join("\n"),
},
{
@ -1045,25 +1045,25 @@ const ci = {
with: {
files: [
"target/release/deno-x86_64-pc-windows-msvc.zip",
"target/release/deno-x86_64-pc-windows-msvc.sha256sum",
"target/release/deno-x86_64-pc-windows-msvc.zip.sha256sum",
"target/release/denort-x86_64-pc-windows-msvc.zip",
"target/release/denort-x86_64-pc-windows-msvc.sha256sum",
"target/release/denort-x86_64-pc-windows-msvc.zip.sha256sum",
"target/release/deno-x86_64-unknown-linux-gnu.zip",
"target/release/deno-x86_64-unknown-linux-gnu.sha256sum",
"target/release/deno-x86_64-unknown-linux-gnu.zip.sha256sum",
"target/release/denort-x86_64-unknown-linux-gnu.zip",
"target/release/denort-x86_64-unknown-linux-gnu.sha256sum",
"target/release/denort-x86_64-unknown-linux-gnu.zip.sha256sum",
"target/release/deno-x86_64-apple-darwin.zip",
"target/release/deno-x86_64-apple-darwin.sha256sum",
"target/release/deno-x86_64-apple-darwin.zip.sha256sum",
"target/release/denort-x86_64-apple-darwin.zip",
"target/release/denort-x86_64-apple-darwin.sha256sum",
"target/release/denort-x86_64-apple-darwin.zip.sha256sum",
"target/release/deno-aarch64-unknown-linux-gnu.zip",
"target/release/deno-aarch64-unknown-linux-gnu.sha256sum",
"target/release/deno-aarch64-unknown-linux-gnu.zip.sha256sum",
"target/release/denort-aarch64-unknown-linux-gnu.zip",
"target/release/denort-aarch64-unknown-linux-gnu.sha256sum",
"target/release/denort-aarch64-unknown-linux-gnu.zip.sha256sum",
"target/release/deno-aarch64-apple-darwin.zip",
"target/release/deno-aarch64-apple-darwin.sha256sum",
"target/release/deno-aarch64-apple-darwin.zip.sha256sum",
"target/release/denort-aarch64-apple-darwin.zip",
"target/release/denort-aarch64-apple-darwin.sha256sum",
"target/release/denort-aarch64-apple-darwin.zip.sha256sum",
"target/release/deno_src.tar.gz",
"target/release/lib.deno.d.ts",
].join("\n"),

78
.github/workflows/ci.yml vendored
View file

@ -62,7 +62,7 @@ jobs:
profile: debug
- os: macos
arch: x86_64
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || ''macos-13'' }}'
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || ''macos-13'' }}'
job: test
profile: release
skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}'
@ -73,7 +73,7 @@ jobs:
profile: debug
- os: macos
arch: aarch64
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || ''macos-14'' }}'
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || ''macos-14'' }}'
job: test
profile: release
skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}'
@ -84,33 +84,33 @@ jobs:
profile: debug
- os: windows
arch: x86_64
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || github.repository == ''denoland/deno'' && ''windows-2022-xl'' || ''windows-2022'' }}'
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && ''windows-2022-xl'' || ''windows-2022'' }}'
job: test
profile: release
skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}'
- os: linux
arch: x86_64
runner: '${{ github.repository == ''denoland/deno'' && ''ubuntu-22.04-xl'' || ''ubuntu-22.04'' }}'
runner: '${{ github.repository == ''denoland/deno'' && ''ubuntu-24.04-xl'' || ''ubuntu-24.04'' }}'
job: test
profile: release
use_sysroot: true
wpt: '${{ !startsWith(github.ref, ''refs/tags/'') }}'
- os: linux
arch: x86_64
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'' && !contains(github.event.pull_request.labels.*.name, ''ci-bench''))) && ''ubuntu-22.04'' || github.repository == ''denoland/deno'' && ''ubuntu-22.04-xl'' || ''ubuntu-22.04'' }}'
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'' && !contains(github.event.pull_request.labels.*.name, ''ci-bench''))) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && ''ubuntu-24.04-xl'' || ''ubuntu-24.04'' }}'
job: bench
profile: release
use_sysroot: true
skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'' && !contains(github.event.pull_request.labels.*.name, ''ci-bench'')) }}'
- os: linux
arch: x86_64
runner: ubuntu-22.04
runner: ubuntu-24.04
job: test
profile: debug
use_sysroot: true
- os: linux
arch: x86_64
runner: ubuntu-22.04
runner: ubuntu-24.04
job: lint
profile: debug
- os: linux
@ -178,7 +178,7 @@ jobs:
if: '!(matrix.skip)'
- if: '!(matrix.skip) && (matrix.job == ''lint'' || matrix.job == ''test'' || matrix.job == ''bench'')'
name: Install Deno
uses: denoland/setup-deno@v1
uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- name: Install Python
@ -252,22 +252,22 @@ jobs:
# to complete.
sudo apt-get -qq remove --purge -y man-db > /dev/null 2> /dev/null
# Remove older clang before we install
sudo apt-get -qq remove 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' > /dev/null 2> /dev/null
sudo apt-get -qq remove 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'clang-17*' 'clang-18*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' 'lld-17*' 'lld-18*' > /dev/null 2> /dev/null
# Install clang-XXX, lld-XXX, and debootstrap.
echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" |
sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-jammy-18.list
echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main" |
sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-jammy-19.list
curl https://apt.llvm.org/llvm-snapshot.gpg.key |
gpg --dearmor |
sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg
sudo apt-get update
# this was unreliable sometimes, so try again if it fails
sudo apt-get install --no-install-recommends clang-18 lld-18 clang-tools-18 clang-format-18 clang-tidy-18 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends clang-18 lld-18 clang-tools-18 clang-format-18 clang-tidy-18
sudo apt-get install --no-install-recommends clang-19 lld-19 clang-tools-19 clang-format-19 clang-tidy-19 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends clang-19 lld-19 clang-tools-19 clang-format-19 clang-tidy-19
# Fix alternatives
(yes '' | sudo update-alternatives --force --all) > /dev/null 2> /dev/null || true
echo "Decompressing sysroot..."
wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20240528/sysroot-`uname -m`.tar.xz -O /tmp/sysroot.tar.xz
wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20241030/sysroot-`uname -m`.tar.xz -O /tmp/sysroot.tar.xz
cd /
xzcat /tmp/sysroot.tar.xz | sudo tar -x
sudo mount --rbind /dev /sysroot/dev
@ -299,8 +299,8 @@ jobs:
CARGO_PROFILE_RELEASE_LTO=false
RUSTFLAGS<<__1
-C linker-plugin-lto=true
-C linker=clang-18
-C link-arg=-fuse-ld=lld-18
-C linker=clang-19
-C link-arg=-fuse-ld=lld-19
-C link-arg=-ldl
-C link-arg=-Wl,--allow-shlib-undefined
-C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache
@ -310,8 +310,8 @@ jobs:
__1
RUSTDOCFLAGS<<__1
-C linker-plugin-lto=true
-C linker=clang-18
-C link-arg=-fuse-ld=lld-18
-C linker=clang-19
-C link-arg=-fuse-ld=lld-19
-C link-arg=-ldl
-C link-arg=-Wl,--allow-shlib-undefined
-C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache
@ -319,7 +319,7 @@ jobs:
--cfg tokio_unstable
$RUSTFLAGS
__1
CC=/usr/bin/clang-18
CC=/usr/bin/clang-19
CFLAGS=-flto=thin $CFLAGS
" > $GITHUB_ENV
- name: Remove macOS cURL --ipv4 flag
@ -361,8 +361,8 @@ jobs:
path: |-
~/.cargo/registry/index
~/.cargo/registry/cache
key: '18-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '18-cargo-home-${{ matrix.os }}-${{ matrix.arch }}'
key: '23-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '23-cargo-home-${{ matrix.os }}-${{ matrix.arch }}'
if: '!(matrix.skip)'
- name: Restore cache build output (PR)
uses: actions/cache/restore@v4
@ -375,7 +375,7 @@ jobs:
!./target/*/*.zip
!./target/*/*.tar.gz
key: never_saved
restore-keys: '18-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-'
restore-keys: '23-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-'
- name: Apply and update mtime cache
if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))'
uses: ./.github/mtime_cache
@ -442,11 +442,11 @@ jobs:
github.repository == 'denoland/deno')
run: |-
cd target/release
shasum -a 256 deno > deno-${{ matrix.arch }}-unknown-linux-gnu.sha256sum
zip -r deno-${{ matrix.arch }}-unknown-linux-gnu.zip deno
shasum -a 256 deno-${{ matrix.arch }}-unknown-linux-gnu.zip > deno-${{ matrix.arch }}-unknown-linux-gnu.zip.sha256sum
strip denort
shasum -a 256 denort > denort-${{ matrix.arch }}-unknown-linux-gnu.sha256sum
zip -r denort-${{ matrix.arch }}-unknown-linux-gnu.zip denort
shasum -a 256 denort-${{ matrix.arch }}-unknown-linux-gnu.zip > denort-${{ matrix.arch }}-unknown-linux-gnu.zip.sha256sum
./deno types > lib.deno.d.ts
- name: Pre-release (mac)
if: |-
@ -461,11 +461,11 @@ jobs:
echo "Key is $(echo $APPLE_CODESIGN_KEY | base64 -d | wc -c) bytes"
rcodesign sign target/release/deno --code-signature-flags=runtime --p12-password="$APPLE_CODESIGN_PASSWORD" --p12-file=<(echo $APPLE_CODESIGN_KEY | base64 -d) --entitlements-xml-file=cli/entitlements.plist
cd target/release
shasum -a 256 deno > deno-${{ matrix.arch }}-apple-darwin.sha256sum
zip -r deno-${{ matrix.arch }}-apple-darwin.zip deno
shasum -a 256 deno-${{ matrix.arch }}-apple-darwin.zip > deno-${{ matrix.arch }}-apple-darwin.zip.sha256sum
strip denort
shasum -a 256 denort > denort-${{ matrix.arch }}-apple-darwin.sha256sum
zip -r denort-${{ matrix.arch }}-apple-darwin.zip denort
shasum -a 256 denort-${{ matrix.arch }}-apple-darwin.zip > denort-${{ matrix.arch }}-apple-darwin.zip.sha256sum
- name: Pre-release (windows)
if: |-
!(matrix.skip) && (matrix.os == 'windows' &&
@ -474,10 +474,10 @@ jobs:
github.repository == 'denoland/deno')
shell: pwsh
run: |-
Get-FileHash target/release/deno.exe -Algorithm SHA256 | Format-List > target/release/deno-${{ matrix.arch }}-pc-windows-msvc.sha256sum
Compress-Archive -CompressionLevel Optimal -Force -Path target/release/deno.exe -DestinationPath target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip
Get-FileHash target/release/denort.exe -Algorithm SHA256 | Format-List > target/release/denort-${{ matrix.arch }}-pc-windows-msvc.sha256sum
Get-FileHash target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip -Algorithm SHA256 | Format-List > target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip.sha256sum
Compress-Archive -CompressionLevel Optimal -Force -Path target/release/denort.exe -DestinationPath target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip
Get-FileHash target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip -Algorithm SHA256 | Format-List > target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip.sha256sum
- name: Upload canary to dl.deno.land
if: |-
!(matrix.skip) && (matrix.job == 'test' &&
@ -652,25 +652,25 @@ jobs:
with:
files: |-
target/release/deno-x86_64-pc-windows-msvc.zip
target/release/deno-x86_64-pc-windows-msvc.sha256sum
target/release/deno-x86_64-pc-windows-msvc.zip.sha256sum
target/release/denort-x86_64-pc-windows-msvc.zip
target/release/denort-x86_64-pc-windows-msvc.sha256sum
target/release/denort-x86_64-pc-windows-msvc.zip.sha256sum
target/release/deno-x86_64-unknown-linux-gnu.zip
target/release/deno-x86_64-unknown-linux-gnu.sha256sum
target/release/deno-x86_64-unknown-linux-gnu.zip.sha256sum
target/release/denort-x86_64-unknown-linux-gnu.zip
target/release/denort-x86_64-unknown-linux-gnu.sha256sum
target/release/denort-x86_64-unknown-linux-gnu.zip.sha256sum
target/release/deno-x86_64-apple-darwin.zip
target/release/deno-x86_64-apple-darwin.sha256sum
target/release/deno-x86_64-apple-darwin.zip.sha256sum
target/release/denort-x86_64-apple-darwin.zip
target/release/denort-x86_64-apple-darwin.sha256sum
target/release/denort-x86_64-apple-darwin.zip.sha256sum
target/release/deno-aarch64-unknown-linux-gnu.zip
target/release/deno-aarch64-unknown-linux-gnu.sha256sum
target/release/deno-aarch64-unknown-linux-gnu.zip.sha256sum
target/release/denort-aarch64-unknown-linux-gnu.zip
target/release/denort-aarch64-unknown-linux-gnu.sha256sum
target/release/denort-aarch64-unknown-linux-gnu.zip.sha256sum
target/release/deno-aarch64-apple-darwin.zip
target/release/deno-aarch64-apple-darwin.sha256sum
target/release/deno-aarch64-apple-darwin.zip.sha256sum
target/release/denort-aarch64-apple-darwin.zip
target/release/denort-aarch64-apple-darwin.sha256sum
target/release/denort-aarch64-apple-darwin.zip.sha256sum
target/release/deno_src.tar.gz
target/release/lib.deno.d.ts
body_path: target/release/release-notes.md
@ -685,10 +685,10 @@ jobs:
!./target/*/*.zip
!./target/*/*.sha256sum
!./target/*/*.tar.gz
key: '18-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
key: '23-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
publish-canary:
name: publish canary
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
needs:
- build
if: github.repository == 'denoland/deno' && github.ref == 'refs/heads/main'

2
.github/workflows/promote_to_release.yml vendored
View file

@ -40,7 +40,7 @@ jobs:
project_id: denoland
- name: Install deno
uses: denoland/setup-deno@v1
uses: denoland/setup-deno@v2
with:
deno-version: v1.x

2
.github/workflows/start_release.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
uses: actions/checkout@v4
- name: Install deno
uses: denoland/setup-deno@v1
uses: denoland/setup-deno@v2
with:
deno-version: v1.x

2
.github/workflows/version_bump.yml vendored
View file

@ -39,7 +39,7 @@ jobs:
- uses: dsherret/rust-toolchain-file@v1
- name: Install deno
uses: denoland/setup-deno@v1
uses: denoland/setup-deno@v2
with:
deno-version: v1.x

2
.github/workflows/wpt_epoch.yml vendored
View file

@ -30,7 +30,7 @@ jobs:
persist-credentials: false
- name: Setup Deno
uses: denoland/setup-deno@v1
uses: denoland/setup-deno@v2
with:
deno-version: ${{ matrix.deno-version }}

449
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

77
Cargo.toml
View file

@ -5,7 +5,6 @@ resolver = "2"
members = [
"bench_util",
"cli",
"cli/napi/sym",
"ext/broadcast_channel",
"ext/cache",
"ext/canvas",
@ -19,6 +18,7 @@ members = [
"ext/io",
"ext/kv",
"ext/napi",
"ext/napi/sym",
"ext/net",
"ext/node",
"ext/url",
@ -45,19 +45,19 @@ license = "MIT"
repository = "https://github.com/denoland/deno"
[workspace.dependencies]
deno_ast = { version = "=0.42.2", features = ["transpiling"] }
deno_core = { version = "0.311.0" }
deno_ast = { version = "=0.43.3", features = ["transpiling"] }
deno_core = { version = "0.316.0" }
deno_bench_util = { version = "0.165.0", path = "./bench_util" }
deno_bench_util = { version = "0.169.0", path = "./bench_util" }
deno_lockfile = "=0.23.1"
deno_media_type = { version = "0.1.4", features = ["module_specifier"] }
deno_npm = "=0.25.3"
deno_media_type = { version = "0.2.0", features = ["module_specifier"] }
deno_npm = "=0.25.4"
deno_path_util = "=0.2.1"
deno_permissions = { version = "0.31.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.180.0", path = "./runtime" }
deno_semver = "=0.5.14"
deno_permissions = { version = "0.35.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.184.0", path = "./runtime" }
deno_semver = "=0.5.16"
deno_terminal = "0.2.0"
napi_sym = { version = "0.101.0", path = "./cli/napi/sym" }
napi_sym = { version = "0.105.0", path = "./ext/napi/sym" }
test_util = { package = "test_server", path = "./tests/util/server" }
denokv_proto = "0.8.1"
@ -66,32 +66,32 @@ denokv_remote = "0.8.1"
denokv_sqlite = { default-features = false, version = "0.8.2" }
# exts
deno_broadcast_channel = { version = "0.165.0", path = "./ext/broadcast_channel" }
deno_cache = { version = "0.103.0", path = "./ext/cache" }
deno_canvas = { version = "0.40.0", path = "./ext/canvas" }
deno_console = { version = "0.171.0", path = "./ext/console" }
deno_cron = { version = "0.51.0", path = "./ext/cron" }
deno_crypto = { version = "0.185.0", path = "./ext/crypto" }
deno_fetch = { version = "0.195.0", path = "./ext/fetch" }
deno_ffi = { version = "0.158.0", path = "./ext/ffi" }
deno_fs = { version = "0.81.0", path = "./ext/fs" }
deno_http = { version = "0.169.0", path = "./ext/http" }
deno_io = { version = "0.81.0", path = "./ext/io" }
deno_kv = { version = "0.79.0", path = "./ext/kv" }
deno_napi = { version = "0.102.0", path = "./ext/napi" }
deno_net = { version = "0.163.0", path = "./ext/net" }
deno_node = { version = "0.108.0", path = "./ext/node" }
deno_tls = { version = "0.158.0", path = "./ext/tls" }
deno_url = { version = "0.171.0", path = "./ext/url" }
deno_web = { version = "0.202.0", path = "./ext/web" }
deno_webgpu = { version = "0.138.0", path = "./ext/webgpu" }
deno_webidl = { version = "0.171.0", path = "./ext/webidl" }
deno_websocket = { version = "0.176.0", path = "./ext/websocket" }
deno_webstorage = { version = "0.166.0", path = "./ext/webstorage" }
deno_broadcast_channel = { version = "0.169.0", path = "./ext/broadcast_channel" }
deno_cache = { version = "0.107.0", path = "./ext/cache" }
deno_canvas = { version = "0.44.0", path = "./ext/canvas" }
deno_console = { version = "0.175.0", path = "./ext/console" }
deno_cron = { version = "0.55.0", path = "./ext/cron" }
deno_crypto = { version = "0.189.0", path = "./ext/crypto" }
deno_fetch = { version = "0.199.0", path = "./ext/fetch" }
deno_ffi = { version = "0.162.0", path = "./ext/ffi" }
deno_fs = { version = "0.85.0", path = "./ext/fs" }
deno_http = { version = "0.173.0", path = "./ext/http" }
deno_io = { version = "0.85.0", path = "./ext/io" }
deno_kv = { version = "0.83.0", path = "./ext/kv" }
deno_napi = { version = "0.106.0", path = "./ext/napi" }
deno_net = { version = "0.167.0", path = "./ext/net" }
deno_node = { version = "0.112.0", path = "./ext/node" }
deno_tls = { version = "0.162.0", path = "./ext/tls" }
deno_url = { version = "0.175.0", path = "./ext/url" }
deno_web = { version = "0.206.0", path = "./ext/web" }
deno_webgpu = { version = "0.142.0", path = "./ext/webgpu" }
deno_webidl = { version = "0.175.0", path = "./ext/webidl" }
deno_websocket = { version = "0.180.0", path = "./ext/websocket" }
deno_webstorage = { version = "0.170.0", path = "./ext/webstorage" }
# resolvers
deno_resolver = { version = "0.3.0", path = "./resolvers/deno" }
node_resolver = { version = "0.10.0", path = "./resolvers/node" }
deno_resolver = { version = "0.7.0", path = "./resolvers/deno" }
node_resolver = { version = "0.14.0", path = "./resolvers/node" }
aes = "=0.8.3"
anyhow = "1.0.57"
@ -106,11 +106,12 @@ cbc = { version = "=0.1.2", features = ["alloc"] }
# Note: Do not use the "clock" feature of chrono, as it links us to CoreFoundation on macOS.
# Instead use util::time::utc_now()
chrono = { version = "0.4", default-features = false, features = ["std", "serde"] }
color-print = "0.3.5"
console_static_text = "=0.8.1"
dashmap = "5.5.3"
data-encoding = "2.3.3"
data-url = "=0.3.0"
deno_cache_dir = "=0.13.0"
deno_cache_dir = "=0.13.2"
deno_package_json = { version = "0.1.2", default-features = false }
dlopen2 = "0.6.1"
ecb = "=0.1.2"
@ -136,7 +137,7 @@ hyper-util = { version = "=0.1.7", features = ["tokio", "client", "client-legacy
hyper_v014 = { package = "hyper", version = "0.14.26", features = ["runtime", "http1"] }
indexmap = { version = "2", features = ["serde"] }
ipnet = "2.3"
jsonc-parser = { version = "=0.23.0", features = ["serde"] }
jsonc-parser = { version = "=0.26.2", features = ["serde"] }
lazy-regex = "3"
libc = "0.2.126"
libz-sys = { version = "1.1.20", default-features = false }
@ -186,7 +187,7 @@ tar = "=0.4.40"
tempfile = "3.4.0"
termcolor = "1.1.3"
thiserror = "1.0.61"
tokio = { version = "=1.36.0", features = ["full"] }
tokio = { version = "1.36.0", features = ["full"] }
tokio-metrics = { version = "0.3.0", features = ["rt"] }
tokio-rustls = { version = "0.26.0", default-features = false, features = ["ring", "tls12"] }
tokio-socks = "0.5.1"
@ -220,7 +221,7 @@ quote = "1"
syn = { version = "2", features = ["full", "extra-traits"] }
# unix
nix = "=0.26.2"
nix = "=0.27.1"
# windows deps
junction = "=0.2.0"

6
README.md
View file

@ -46,6 +46,12 @@ brew install deno
choco install deno
```
[WinGet](https://winstall.app/apps/DenoLand.Deno) (Windows):
```powershell
winget install --id=DenoLand.Deno
```
### Build and install from source
Complete instructions for building Deno from source can be found in the manual

141
Releases.md
View file

@ -6,6 +6,147 @@ https://github.com/denoland/deno/releases
We also have one-line install commands at:
https://github.com/denoland/deno_install
### 2.0.4 / 2024.10.29
- Revert "fix(ext/node): fix dns.lookup result ordering (#26264)" (#26621)
- Revert "fix(ext/node): use primordials in `ext/node/polyfills/https.ts`
(#26323)" (#26613)
- feat(lsp): "typescript.preferences.preferTypeOnlyAutoImports" setting (#26546)
- fix(check): expose more globals from @types/node (#26603)
- fix(check): ignore resolving `jsxImportSource` when jsx is not used in graph
(#26548)
- fix(cli): Make --watcher CLEAR_SCREEN clear scrollback buffer as well as
visible screen (#25997)
- fix(compile): regression handling redirects (#26586)
- fix(ext/napi): export dynamic symbols list for {Free,Open}BSD (#26605)
- fix(ext/node): add path to `fs.stat` and `fs.statSync` error (#26037)
- fix(ext/node): compatibility with {Free,Open}BSD (#26604)
- fix(ext/node): use primordials in
ext\node\polyfills\internal\crypto\_randomInt.ts (#26534)
- fix(install): cache json exports of JSR packages (#26552)
- fix(install): regression - do not panic when config file contains \r\n
newlines (#26547)
- fix(lsp): make missing import action fix infallible (#26539)
- fix(npm): match npm bearer token generation (#26544)
- fix(upgrade): stop running `deno lsp` processes on windows before attempting
to replace executable (#26542)
- fix(watch): don't panic on invalid file specifiers (#26577)
- fix: do not panic when failing to write to http cache (#26591)
- fix: provide hints in terminal errors for Node.js globals (#26610)
- fix: report exceptions from nextTick (#26579)
- fix: support watch flag to enable watching other files than the main module on
serve subcommand (#26622)
- perf: pass transpiled module to deno_core as known string (#26555)
### 2.0.3 / 2024.10.25
- feat(lsp): interactive inlay hints (#26382)
- fix: support node-api in denort (#26389)
- fix(check): support `--frozen` on deno check (#26479)
- fix(cli): increase size of blocking task threadpool on windows (#26465)
- fix(config): schemas for lint rule and tag autocompletion (#26515)
- fix(ext/console): ignore casing for named colors in css parsing (#26466)
- fix(ext/ffi): return u64/i64 as bigints from nonblocking ffi calls (#26486)
- fix(ext/node): cancel pending ipc writes on channel close (#26504)
- fix(ext/node): map `ERROR_INVALID_NAME` to `ENOENT` on windows (#26475)
- fix(ext/node): only set our end of child process pipe to nonblocking mode
(#26495)
- fix(ext/node): properly map reparse point error in readlink (#26375)
- fix(ext/node): refactor http.ServerResponse into function class (#26210)
- fix(ext/node): stub HTTPParser internal binding (#26401)
- fix(ext/node): use primordials in `ext/node/polyfills/https.ts` (#26323)
- fix(fmt): --ext flag requires to pass files (#26525)
- fix(fmt): upgrade formatters (#26469)
- fix(help): missing package specifier (#26380)
- fix(info): resolve workspace member mappings (#26350)
- fix(install): better json editing (#26450)
- fix(install): cache all exports of JSR packages listed in `deno.json` (#26501)
- fix(install): cache type only module deps in `deno install` (#26497)
- fix(install): don't cache json exports of JSR packages (for now) (#26530)
- fix(install): update lockfile when using package.json (#26458)
- fix(lsp): import-map-remap quickfix for type imports (#26454)
- fix(node/util): support array formats in `styleText` (#26507)
- fix(node:tls): set TLSSocket.alpnProtocol for client connections (#26476)
- fix(npm): ensure scoped package name is encoded in URLs (#26390)
- fix(npm): support version ranges with && or comma (#26453)
- fix: `.npmrc` settings not being passed to install/add command (#26473)
- fix: add 'fmt-component' to unstable features in schema file (#26526)
- fix: share inotify fd across watchers (#26200)
- fix: unpin tokio version (#26457)
- perf(compile): pass module source data from binary directly to v8 (#26494)
- perf: avoid multiple calls to runMicrotask (#26378)
### 2.0.2 / 2024.10.17
- fix(cli): set napi object property properly (#26344)
- fix(ext/node): add null check for kStreamBaseField (#26368)
- fix(install): don't attempt to cache specifiers that point to directories
(#26369)
- fix(jupyter): fix panics for overslow subtraction (#26371)
- fix(jupyter): update to the new logo (#26353)
- fix(net): don't try to set nodelay on upgrade streams (#26342)
- fix(node/fs): copyFile with `COPYFILE_EXCL` should not throw if the
destination doesn't exist (#26360)
- fix(node/http): normalize header names in `ServerResponse` (#26339)
- fix(runtime): send ws ping frames from inspector server (#26352)
- fix: don't warn on ignored signals on windows (#26332)
### 2.0.1 / 2024.10.16
- feat(lsp): "deno/didRefreshDenoConfigurationTree" notifications (#26215)
- feat(unstable): `--unstable-detect-cjs` for respecting explicit
`"type": "commonjs"` (#26149)
- fix(add): create deno.json when running `deno add jsr:<pkg>` (#26275)
- fix(add): exact version should not have range `^` specifier (#26302)
- fix(child_process): map node `--no-warnings` flag to `--quiet` (#26288)
- fix(cli): add prefix to install commands in help (#26318)
- fix(cli): consolidate pkg parser for install & remove (#26298)
- fix(cli): named export takes precedence over default export in doc testing
(#26112)
- fix(cli): improve deno info output for npm packages (#25906)
- fix(console/ext/repl): support using parseFloat() (#25900)
- fix(ext/console): apply coloring for console.table (#26280)
- fix(ext/napi): pass user context to napi_threadsafe_fn finalizers (#26229)
- fix(ext/node): allow writing to tty columns (#26201)
- fix(ext/node): compute pem length (upper bound) for key exports (#26231)
- fix(ext/node): fix dns.lookup result ordering (#26264)
- fix(ext/node): handle http2 server ending stream (#26235)
- fix(ext/node): implement TCP.setNoDelay (#26263)
- fix(ext/node): timingSafeEqual account for AB byteOffset (#26292)
- fix(ext/node): use primordials in `ext/node/polyfills/internal/buffer.mjs`
(#24993)
- fix(ext/webgpu): allow GL backend on Windows (#26206)
- fix(install): duplicate dependencies in `package.json` (#26128)
- fix(install): handle pkg with dep on self when pkg part of peer dep resolution
(#26277)
- fix(install): retry downloads of registry info / tarballs (#26278)
- fix(install): support installing npm package with alias (#26246)
- fix(jupyter): copy kernels icons to the kernel directory (#26084)
- fix(jupyter): keep running event loop when waiting for messages (#26049)
- fix(lsp): relative completions for bare import-mapped specifiers (#26137)
- fix(node): make `process.stdout.isTTY` writable (#26130)
- fix(node/util): export `styleText` from `node:util` (#26194)
- fix(npm): support `--allow-scripts` on `deno run` (and `deno add`,
`deno test`, etc) (#26075)
- fix(repl): importing json files (#26053)
- fix(repl): remove check flags (#26140)
- fix(unstable/worker): ensure import permissions are passed (#26101)
- fix: add hint for missing `document` global in terminal error (#26218)
- fix: do not panic on wsl share file paths on windows (#26081)
- fix: do not panic running remote cjs module (#26259)
- fix: do not panic when using methods on classes and interfaces in deno doc
html output (#26100)
- fix: improve suggestions and hints when using CommonJS modules (#26287)
- fix: node-api function call should use preamble (#26297)
- fix: panic in `prepare_stack_trace_callback` when global interceptor throws
(#26241)
- fix: use syntect for deno doc html generation (#26322)
- perf(http): avoid clone getting request method and url (#26250)
- perf(http): cache webidl.converters lookups in ext/fetch/23_response.js
(#26256)
- perf(http): make heap allocation for path conditional (#26289)
- perf: use fast calls for microtask ops (#26236)
### 2.0.0 / 2024.10.09
Read announcement blog post at: https://deno.com/blog/v2

2
bench_util/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_bench_util"
version = "0.165.0"
version = "0.169.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

35
cli/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno"
version = "2.0.0"
version = "2.0.4"
authors.workspace = true
default-run = "deno"
edition.workspace = true
@ -38,6 +38,11 @@ path = "./bench/lsp_bench_standalone.rs"
[features]
default = ["upgrade", "__vendored_zlib_ng"]
# A feature that enables heap profiling with dhat on Linux.
# 1. Compile with `cargo build --profile=release-with-debug --features=dhat-heap`
# 2. Run the executable. It will output a dhat-heap.json file.
# 3. Open the json file in https://nnethercote.github.io/dh_view/dh_view.html
dhat-heap = ["dhat"]
# A feature that enables the upgrade subcommand and the background check for
# available updates (of deno binary). This is typically disabled for (Linux)
# distribution packages.
@ -65,11 +70,11 @@ winres.workspace = true
[dependencies]
deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "proposal", "react", "sourcemap", "transforms", "typescript", "view", "visit"] }
deno_cache_dir = { workspace = true }
deno_config = { version = "=0.37.1", features = ["workspace", "sync"] }
deno_config = { version = "=0.37.2", features = ["workspace", "sync"] }
deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
deno_doc = { version = "0.153.0", features = ["html"] }
deno_graph = { version = "=0.83.3" }
deno_lint = { version = "=0.67.0", features = ["docs"] }
deno_doc = { version = "0.156.0", default-features = false, features = ["rust", "html", "syntect"] }
deno_graph = { version = "=0.84.1" }
deno_lint = { version = "=0.68.0", features = ["docs"] }
deno_lockfile.workspace = true
deno_npm.workspace = true
deno_package_json.workspace = true
@ -79,9 +84,7 @@ deno_runtime = { workspace = true, features = ["include_js_files_for_snapshottin
deno_semver.workspace = true
deno_task_shell = "=0.18.1"
deno_terminal.workspace = true
eszip = "=0.79.1"
libsui = "0.4.0"
napi_sym.workspace = true
node_resolver.workspace = true
anstream = "0.6.14"
@ -94,16 +97,17 @@ chrono = { workspace = true, features = ["now"] }
clap = { version = "=4.5.16", features = ["env", "string", "wrap_help", "error-context"] }
clap_complete = "=4.5.24"
clap_complete_fig = "=4.5.2"
color-print = "0.3.5"
color-print.workspace = true
console_static_text.workspace = true
dashmap.workspace = true
data-encoding.workspace = true
dhat = { version = "0.3.3", optional = true }
dissimilar = "=1.0.4"
dotenvy = "0.15.7"
dprint-plugin-json = "=0.19.3"
dprint-plugin-jupyter = "=0.1.3"
dprint-plugin-json = "=0.19.4"
dprint-plugin-jupyter = "=0.1.5"
dprint-plugin-markdown = "=0.17.8"
dprint-plugin-typescript = "=0.93.0"
dprint-plugin-typescript = "=0.93.1"
env_logger = "=0.10.0"
fancy-regex = "=0.10.0"
faster-hex.workspace = true
@ -117,15 +121,15 @@ http-body-util.workspace = true
hyper-util.workspace = true
import_map = { version = "=0.20.1", features = ["ext"] }
indexmap.workspace = true
jsonc-parser.workspace = true
jsonc-parser = { workspace = true, features = ["cst", "serde"] }
jupyter_runtime = { package = "runtimelib", version = "=0.14.0" }
lazy-regex.workspace = true
libc.workspace = true
libz-sys.workspace = true
log = { workspace = true, features = ["serde"] }
lsp-types.workspace = true
malva = "=0.10.1"
markup_fmt = "=0.13.1"
malva = "=0.11.0"
markup_fmt = "=0.15.0"
memmem.workspace = true
monch.workspace = true
notify.workspace = true
@ -162,7 +166,6 @@ typed-arena = "=2.0.2"
uuid = { workspace = true, features = ["serde"] }
walkdir = "=2.3.2"
which.workspace = true
yoke.workspace = true
zeromq.workspace = true
zip = { version = "2.1.6", default-features = false, features = ["deflate-flate2"] }
zstd.workspace = true
@ -170,14 +173,12 @@ zstd.workspace = true
[target.'cfg(windows)'.dependencies]
junction.workspace = true
winapi = { workspace = true, features = ["knownfolders", "mswsock", "objbase", "shlobj", "tlhelp32", "winbase", "winerror", "winsock2"] }
windows-sys.workspace = true
[target.'cfg(unix)'.dependencies]
nix.workspace = true
[dev-dependencies]
deno_bench_util.workspace = true
libuv-sys-lite = "=1.48.2"
pretty_assertions.workspace = true
test_util.workspace = true

248
cli/args/flags.rs
View file

@ -575,7 +575,8 @@ fn parse_packages_allowed_scripts(s: &str) -> Result<String, AnyError> {
pub struct UnstableConfig {
// TODO(bartlomieju): remove in Deno 2.5
pub legacy_flag_enabled: bool, // --unstable
pub bare_node_builtins: bool, // --unstable-bare-node-builts
pub bare_node_builtins: bool,
pub detect_cjs: bool,
pub sloppy_imports: bool,
pub features: Vec<String>, // --unstabe-kv --unstable-cron
}
@ -1177,9 +1178,9 @@ static DENO_HELP: &str = cstr!(
<y>Dependency management:</>
<g>add</> Add dependencies
<p(245)>deno add @std/assert | deno add npm:express</>
<g>install</> Install script as an executable
<g>uninstall</> Uninstall a script previously installed with deno install
<p(245)>deno add jsr:@std/assert | deno add npm:express</>
<g>install</> Installs dependencies either in the local project or globally to a bin directory
<g>uninstall</> Uninstalls a dependency or an executable script in the installation root's bin directory
<g>remove</> Remove dependencies from the configuration file
<y>Tooling:</>
@ -1342,7 +1343,7 @@ pub fn flags_from_vec(args: Vec<OsString>) -> clap::error::Result<Flags> {
}
match subcommand.as_str() {
"add" => add_parse(&mut flags, &mut m),
"add" => add_parse(&mut flags, &mut m)?,
"remove" => remove_parse(&mut flags, &mut m),
"bench" => bench_parse(&mut flags, &mut m)?,
"bundle" => bundle_parse(&mut flags, &mut m),
@ -1528,7 +1529,7 @@ pub fn clap_root() -> Command {
);
run_args(Command::new("deno"), true)
.args(unstable_args(UnstableArgsConfig::ResolutionAndRuntime))
.with_unstable_args(UnstableArgsConfig::ResolutionAndRuntime)
.next_line_help(false)
.bin_name("deno")
.styles(
@ -1630,7 +1631,7 @@ fn command(
) -> Command {
Command::new(name)
.about(about)
.args(unstable_args(unstable_args_config))
.with_unstable_args(unstable_args_config)
}
fn help_subcommand(app: &Command) -> Command {
@ -1658,10 +1659,10 @@ fn add_subcommand() -> Command {
"add",
cstr!(
"Add dependencies to your configuration file.
<p(245)>deno add @std/path</>
<p(245)>deno add jsr:@std/path</>
You can add multiple dependencies at once:
<p(245)>deno add @std/path @std/assert</>"
<p(245)>deno add jsr:@std/path jsr:@std/assert</>"
),
UnstableArgsConfig::None,
)
@ -1675,6 +1676,7 @@ You can add multiple dependencies at once:
.action(ArgAction::Append),
)
.arg(add_dev_arg())
.arg(allow_scripts_arg())
})
}
@ -1717,7 +1719,7 @@ If you specify a directory instead of a file, the path is expanded to all contai
UnstableArgsConfig::ResolutionAndRuntime,
)
.defer(|cmd| {
runtime_args(cmd, true, false)
runtime_args(cmd, true, false, true)
.arg(check_arg(true))
.arg(
Arg::new("json")
@ -1854,6 +1856,7 @@ Unless --reload is specified, this command will not re-download already cached d
.required_unless_present("help")
.value_hint(ValueHint::FilePath),
)
.arg(frozen_lockfile_arg())
.arg(allow_import_arg())
}
)
@ -1881,7 +1884,7 @@ On the first invocation with deno will download the proper binary and cache it i
UnstableArgsConfig::ResolutionAndRuntime,
)
.defer(|cmd| {
runtime_args(cmd, true, false)
runtime_args(cmd, true, false, true)
.arg(check_arg(true))
.arg(
Arg::new("include")
@ -2202,7 +2205,7 @@ This command has implicit access to all permissions.
UnstableArgsConfig::ResolutionAndRuntime,
)
.defer(|cmd| {
runtime_args(cmd, false, true)
runtime_args(cmd, false, true, true)
.arg(check_arg(false))
.arg(executable_ext_arg())
.arg(
@ -2271,7 +2274,7 @@ Ignore formatting a file by adding an ignore comment at the top of the file:
"sass", "less", "html", "svelte", "vue", "astro", "yml", "yaml",
"ipynb",
])
.help_heading(FMT_HEADING),
.help_heading(FMT_HEADING).requires("files"),
)
.arg(
Arg::new("ignore")
@ -2468,7 +2471,7 @@ in the package cache. If no dependency is specified, installs all dependencies l
If the <p(245)>--entrypoint</> flag is passed, installs the dependencies of the specified entrypoint(s).
<p(245)>deno install</>
<p(245)>deno install @std/bytes</>
<p(245)>deno install jsr:@std/bytes</>
<p(245)>deno install npm:chalk</>
<p(245)>deno install --entrypoint entry1.ts entry2.ts</>
@ -2501,7 +2504,7 @@ The installation root is determined, in order of precedence:
These must be added to the path manually if required."), UnstableArgsConfig::ResolutionAndRuntime)
.visible_alias("i")
.defer(|cmd| {
permission_args(runtime_args(cmd, false, true), Some("global"))
permission_args(runtime_args(cmd, false, true, false), Some("global"))
.arg(check_arg(true))
.arg(allow_scripts_arg())
.arg(
@ -2767,8 +2770,13 @@ It is especially useful for quick prototyping and checking snippets of code.
TypeScript is supported, however it is not type-checked, only transpiled."
), UnstableArgsConfig::ResolutionAndRuntime)
.defer(|cmd| runtime_args(cmd, true, true)
.arg(check_arg(false))
.defer(|cmd| {
let cmd = compile_args_without_check_args(cmd);
let cmd = inspect_args(cmd);
let cmd = permission_args(cmd, None);
let cmd = runtime_misc_args(cmd);
cmd
.arg(
Arg::new("eval-file")
.long("eval-file")
@ -2787,7 +2795,7 @@ TypeScript is supported, however it is not type-checked, only transpiled."
.after_help(cstr!("<y>Environment variables:</>
<g>DENO_REPL_HISTORY</> Set REPL history file path. History file is disabled when the value is empty.
<p(245)>[default: $DENO_DIR/deno_history.txt]</>"))
)
})
.arg(env_file_arg())
.arg(
Arg::new("args")
@ -2799,7 +2807,7 @@ TypeScript is supported, however it is not type-checked, only transpiled."
}
fn run_args(command: Command, top_level: bool) -> Command {
runtime_args(command, true, true)
runtime_args(command, true, true, true)
.arg(check_arg(false))
.arg(watch_arg(true))
.arg(hmr_arg(true))
@ -2855,7 +2863,7 @@ Start a server defined in server.ts:
Start a server defined in server.ts, watching for changes and running on port 5050:
<p(245)>deno serve --watch --port 5050 server.ts</>
<y>Read more:</> <c>https://docs.deno.com/go/serve</>"), UnstableArgsConfig::ResolutionAndRuntime), true, true)
<y>Read more:</> <c>https://docs.deno.com/go/serve</>"), UnstableArgsConfig::ResolutionAndRuntime), true, true, true)
.arg(
Arg::new("port")
.long("port")
@ -2929,7 +2937,7 @@ or <c>**/__tests__/**</>:
UnstableArgsConfig::ResolutionAndRuntime
)
.defer(|cmd|
runtime_args(cmd, true, true)
runtime_args(cmd, true, true, true)
.arg(check_arg(true))
.arg(
Arg::new("ignore")
@ -3642,6 +3650,7 @@ fn runtime_args(
app: Command,
include_perms: bool,
include_inspector: bool,
include_allow_scripts: bool,
) -> Command {
let app = compile_args(app);
let app = if include_perms {
@ -3654,6 +3663,15 @@ fn runtime_args(
} else {
app
};
let app = if include_allow_scripts {
app.arg(allow_scripts_arg())
} else {
app
};
runtime_misc_args(app)
}
fn runtime_misc_args(app: Command) -> Command {
app
.arg(frozen_lockfile_arg())
.arg(cached_only_arg())
@ -4135,23 +4153,29 @@ enum UnstableArgsConfig {
ResolutionAndRuntime,
}
struct UnstableArgsIter {
idx: usize,
cfg: UnstableArgsConfig,
trait CommandExt {
fn with_unstable_args(self, cfg: UnstableArgsConfig) -> Self;
}
impl Iterator for UnstableArgsIter {
type Item = Arg;
impl CommandExt for Command {
fn with_unstable_args(self, cfg: UnstableArgsConfig) -> Self {
let mut next_display_order = {
let mut value = 1000;
move || {
value += 1;
value
}
};
fn next(&mut self) -> Option<Self::Item> {
let arg = if self.idx == 0 {
let mut cmd = self.arg(
Arg::new("unstable")
.long("unstable")
.help(cstr!("Enable all unstable features and APIs. Instead of using this flag, consider enabling individual unstable features
.long("unstable")
.help(cstr!("Enable all unstable features and APIs. Instead of using this flag, consider enabling individual unstable features
<p(245)>To view the list of individual unstable feature flags, run this command again with --help=unstable</>"))
.action(ArgAction::SetTrue)
.hide(matches!(self.cfg, UnstableArgsConfig::None))
} else if self.idx == 1 {
.action(ArgAction::SetTrue)
.hide(matches!(cfg, UnstableArgsConfig::None))
.display_order(next_display_order())
).arg(
Arg::new("unstable-bare-node-builtins")
.long("unstable-bare-node-builtins")
.help("Enable unstable bare node builtins feature")
@ -4159,20 +4183,36 @@ impl Iterator for UnstableArgsIter {
.value_parser(FalseyValueParser::new())
.action(ArgAction::SetTrue)
.hide(true)
.long_help(match self.cfg {
.long_help(match cfg {
UnstableArgsConfig::None => None,
UnstableArgsConfig::ResolutionOnly
| UnstableArgsConfig::ResolutionAndRuntime => Some("true"),
})
.help_heading(UNSTABLE_HEADING)
} else if self.idx == 2 {
.display_order(next_display_order()),
).arg(
Arg::new("unstable-detect-cjs")
.long("unstable-detect-cjs")
.help("Reads the package.json type field in a project to treat .js files as .cjs")
.value_parser(FalseyValueParser::new())
.action(ArgAction::SetTrue)
.hide(true)
.long_help(match cfg {
UnstableArgsConfig::None => None,
UnstableArgsConfig::ResolutionOnly
| UnstableArgsConfig::ResolutionAndRuntime => Some("true"),
})
.help_heading(UNSTABLE_HEADING)
.display_order(next_display_order())
).arg(
Arg::new("unstable-byonm")
.long("unstable-byonm")
.value_parser(FalseyValueParser::new())
.action(ArgAction::SetTrue)
.hide(true)
.help_heading(UNSTABLE_HEADING)
} else if self.idx == 3 {
.display_order(next_display_order()),
).arg(
Arg::new("unstable-sloppy-imports")
.long("unstable-sloppy-imports")
.help("Enable unstable resolving of specifiers by extension probing, .js to .ts, and directory probing")
@ -4180,40 +4220,39 @@ impl Iterator for UnstableArgsIter {
.value_parser(FalseyValueParser::new())
.action(ArgAction::SetTrue)
.hide(true)
.long_help(match self.cfg {
.long_help(match cfg {
UnstableArgsConfig::None => None,
UnstableArgsConfig::ResolutionOnly | UnstableArgsConfig::ResolutionAndRuntime => Some("true")
})
.help_heading(UNSTABLE_HEADING)
} else if self.idx > 3 {
let granular_flag = crate::UNSTABLE_GRANULAR_FLAGS.get(self.idx - 4)?;
Arg::new(format!("unstable-{}", granular_flag.name))
.long(format!("unstable-{}", granular_flag.name))
.help(granular_flag.help_text)
.action(ArgAction::SetTrue)
.hide(true)
.help_heading(UNSTABLE_HEADING)
// we don't render long help, so using it here as a sort of metadata
.long_help(if granular_flag.show_in_help {
match self.cfg {
UnstableArgsConfig::None | UnstableArgsConfig::ResolutionOnly => {
None
}
UnstableArgsConfig::ResolutionAndRuntime => Some("true"),
}
} else {
None
})
} else {
return None;
};
self.idx += 1;
Some(arg.display_order(self.idx + 1000))
}
}
.display_order(next_display_order())
);
fn unstable_args(cfg: UnstableArgsConfig) -> impl IntoIterator<Item = Arg> {
UnstableArgsIter { idx: 0, cfg }
for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS.iter() {
cmd = cmd.arg(
Arg::new(format!("unstable-{}", granular_flag.name))
.long(format!("unstable-{}", granular_flag.name))
.help(granular_flag.help_text)
.action(ArgAction::SetTrue)
.hide(true)
.help_heading(UNSTABLE_HEADING)
// we don't render long help, so using it here as a sort of metadata
.long_help(if granular_flag.show_in_help {
match cfg {
UnstableArgsConfig::None | UnstableArgsConfig::ResolutionOnly => {
None
}
UnstableArgsConfig::ResolutionAndRuntime => Some("true"),
}
} else {
None
})
.display_order(next_display_order()),
);
}
cmd
}
}
fn allow_scripts_arg_parse(
@ -4235,8 +4274,13 @@ fn allow_scripts_arg_parse(
Ok(())
}
fn add_parse(flags: &mut Flags, matches: &mut ArgMatches) {
fn add_parse(
flags: &mut Flags,
matches: &mut ArgMatches,
) -> clap::error::Result<()> {
allow_scripts_arg_parse(flags, matches)?;
flags.subcommand = DenoSubcommand::Add(add_parse_inner(matches, None));
Ok(())
}
fn add_parse_inner(
@ -4262,7 +4306,7 @@ fn bench_parse(
) -> clap::error::Result<()> {
flags.type_check_mode = TypeCheckMode::Local;
runtime_args_parse(flags, matches, true, false)?;
runtime_args_parse(flags, matches, true, false, true)?;
ext_arg_parse(flags, matches);
// NOTE: `deno bench` always uses `--no-prompt`, tests shouldn't ever do
@ -4330,6 +4374,7 @@ fn check_parse(
flags.type_check_mode = TypeCheckMode::Local;
compile_args_without_check_parse(flags, matches)?;
unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionAndRuntime);
frozen_lockfile_arg_parse(flags, matches);
let files = matches.remove_many::<String>("file").unwrap().collect();
if matches.get_flag("all") || matches.get_flag("remote") {
flags.type_check_mode = TypeCheckMode::All;
@ -4352,7 +4397,7 @@ fn compile_parse(
matches: &mut ArgMatches,
) -> clap::error::Result<()> {
flags.type_check_mode = TypeCheckMode::Local;
runtime_args_parse(flags, matches, true, false)?;
runtime_args_parse(flags, matches, true, false, true)?;
let mut script = matches.remove_many::<String>("script_arg").unwrap();
let source_file = script.next().unwrap();
@ -4527,7 +4572,7 @@ fn eval_parse(
flags: &mut Flags,
matches: &mut ArgMatches,
) -> clap::error::Result<()> {
runtime_args_parse(flags, matches, false, true)?;
runtime_args_parse(flags, matches, false, true, false)?;
unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionAndRuntime);
flags.allow_all();
@ -4620,7 +4665,7 @@ fn install_parse(
flags: &mut Flags,
matches: &mut ArgMatches,
) -> clap::error::Result<()> {
runtime_args_parse(flags, matches, true, true)?;
runtime_args_parse(flags, matches, true, true, false)?;
let global = matches.get_flag("global");
if global {
@ -4846,8 +4891,18 @@ fn repl_parse(
flags: &mut Flags,
matches: &mut ArgMatches,
) -> clap::error::Result<()> {
runtime_args_parse(flags, matches, true, true)?;
unsafely_ignore_certificate_errors_parse(flags, matches);
unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionAndRuntime);
compile_args_without_check_parse(flags, matches)?;
cached_only_arg_parse(flags, matches);
frozen_lockfile_arg_parse(flags, matches);
permission_args_parse(flags, matches)?;
inspect_arg_parse(flags, matches);
location_arg_parse(flags, matches);
v8_flags_arg_parse(flags, matches);
seed_arg_parse(flags, matches);
enable_testing_features_arg_parse(flags, matches);
env_file_arg_parse(flags, matches);
strace_ops_parse(flags, matches);
let eval_files = matches
.remove_many::<String>("eval-file")
@ -4879,7 +4934,7 @@ fn run_parse(
mut app: Command,
bare: bool,
) -> clap::error::Result<()> {
runtime_args_parse(flags, matches, true, true)?;
runtime_args_parse(flags, matches, true, true, true)?;
ext_arg_parse(flags, matches);
flags.code_cache_enabled = !matches.get_flag("no-code-cache");
@ -4920,7 +4975,7 @@ fn serve_parse(
let worker_count = parallel_arg_parse(matches).map(|v| v.get());
runtime_args_parse(flags, matches, true, true)?;
runtime_args_parse(flags, matches, true, true, true)?;
// If the user didn't pass --allow-net, add this port to the network
// allowlist. If the host is 0.0.0.0, we add :{port} and allow the same network perms
// as if it was passed to --allow-net directly.
@ -5015,7 +5070,7 @@ fn test_parse(
matches: &mut ArgMatches,
) -> clap::error::Result<()> {
flags.type_check_mode = TypeCheckMode::Local;
runtime_args_parse(flags, matches, true, true)?;
runtime_args_parse(flags, matches, true, true, true)?;
ext_arg_parse(flags, matches);
// NOTE: `deno test` always uses `--no-prompt`, tests shouldn't ever do
@ -5380,6 +5435,7 @@ fn runtime_args_parse(
matches: &mut ArgMatches,
include_perms: bool,
include_inspector: bool,
include_allow_scripts: bool,
) -> clap::error::Result<()> {
unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionAndRuntime);
compile_args_parse(flags, matches)?;
@ -5391,6 +5447,9 @@ fn runtime_args_parse(
if include_inspector {
inspect_arg_parse(flags, matches);
}
if include_allow_scripts {
allow_scripts_arg_parse(flags, matches)?;
}
location_arg_parse(flags, matches);
v8_flags_arg_parse(flags, matches);
seed_arg_parse(flags, matches);
@ -5662,6 +5721,7 @@ fn unstable_args_parse(
flags.unstable_config.bare_node_builtins =
matches.get_flag("unstable-bare-node-builtins");
flags.unstable_config.detect_cjs = matches.get_flag("unstable-detect-cjs");
flags.unstable_config.sloppy_imports =
matches.get_flag("unstable-sloppy-imports");
@ -6742,6 +6802,32 @@ mod tests {
..Flags::default()
}
);
let r = flags_from_vec(svec!["deno", "fmt", "--ext", "html"]);
assert!(r.is_err());
let r = flags_from_vec(svec!["deno", "fmt", "--ext", "html", "./**"]);
assert_eq!(
r.unwrap(),
Flags {
subcommand: DenoSubcommand::Fmt(FmtFlags {
check: false,
files: FileFlags {
include: vec!["./**".to_string()],
ignore: vec![],
},
use_tabs: None,
line_width: None,
indent_width: None,
single_quote: None,
prose_wrap: None,
no_semicolons: None,
unstable_component: false,
watch: Default::default(),
}),
ext: Some("html".to_string()),
..Flags::default()
}
);
}
#[test]
@ -7390,7 +7476,7 @@ mod tests {
#[test]
fn repl_with_flags() {
#[rustfmt::skip]
let r = flags_from_vec(svec!["deno", "repl", "-A", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--no-check", "--reload", "--lock", "lock.json", "--cert", "example.crt", "--cached-only", "--location", "https:foo", "--v8-flags=--help", "--seed", "1", "--inspect=127.0.0.1:9229", "--unsafely-ignore-certificate-errors", "--env=.example.env"]);
let r = flags_from_vec(svec!["deno", "repl", "-A", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--reload", "--lock", "lock.json", "--cert", "example.crt", "--cached-only", "--location", "https:foo", "--v8-flags=--help", "--seed", "1", "--inspect=127.0.0.1:9229", "--unsafely-ignore-certificate-errors", "--env=.example.env"]);
assert_eq!(
r.unwrap(),
Flags {
@ -7438,7 +7524,6 @@ mod tests {
allow_write: Some(vec![]),
..Default::default()
},
type_check_mode: TypeCheckMode::None,
..Flags::default()
}
);
@ -7460,7 +7545,6 @@ mod tests {
eval: None,
is_default_command: false,
}),
type_check_mode: TypeCheckMode::None,
..Flags::default()
}
);
@ -8862,8 +8946,12 @@ mod tests {
#[test]
fn test_no_colon_in_value_name() {
let app =
runtime_args(Command::new("test_inspect_completion_value"), true, true);
let app = runtime_args(
Command::new("test_inspect_completion_value"),
true,
true,
false,
);
let inspect_args = app
.get_arguments()
.filter(|arg| arg.get_id() == "inspect")

57
cli/args/mod.rs
View file

@ -46,6 +46,7 @@ pub use flags::*;
pub use lockfile::CliLockfile;
pub use lockfile::CliLockfileReadFromPathOptions;
pub use package_json::NpmInstallDepsProvider;
pub use package_json::PackageJsonDepValueParseWithLocationError;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
@ -200,6 +201,8 @@ pub fn ts_config_to_transpile_and_emit_options(
precompile_jsx_dynamic_props: None,
transform_jsx,
var_decl_imports: false,
// todo(dsherret): support verbatim_module_syntax here properly
verbatim_module_syntax: false,
},
deno_ast::EmitOptions {
inline_sources: options.inline_sources,
@ -578,6 +581,7 @@ fn discover_npmrc(
let resolved = npmrc
.as_resolved(npm_registry_url())
.context("Failed to resolve .npmrc options")?;
log::debug!(".npmrc found at: '{}'", path.display());
Ok(Arc::new(resolved))
}
@ -963,6 +967,9 @@ impl CliOptions {
match self.sub_command() {
DenoSubcommand::Cache(_) => GraphKind::All,
DenoSubcommand::Check(_) => GraphKind::TypesOnly,
DenoSubcommand::Install(InstallFlags {
kind: InstallKind::Local(_),
}) => GraphKind::All,
_ => self.type_check_mode().as_graph_kind(),
}
}
@ -1448,6 +1455,12 @@ impl CliOptions {
watch: Some(WatchFlagsWithPaths { hmr, .. }),
..
}) = &self.flags.subcommand
{
*hmr
} else if let DenoSubcommand::Serve(ServeFlags {
watch: Some(WatchFlagsWithPaths { hmr, .. }),
..
}) = &self.flags.subcommand
{
*hmr
} else {
@ -1576,6 +1589,11 @@ impl CliOptions {
|| self.workspace().has_unstable("bare-node-builtins")
}
pub fn unstable_detect_cjs(&self) -> bool {
self.flags.unstable_config.detect_cjs
|| self.workspace().has_unstable("detect-cjs")
}
fn byonm_enabled(&self) -> bool {
// check if enabled via unstable
self.node_modules_dir().ok().flatten() == Some(NodeModulesDirMode::Manual)
@ -1586,6 +1604,15 @@ impl CliOptions {
}
pub fn use_byonm(&self) -> bool {
if matches!(
self.sub_command(),
DenoSubcommand::Install(_)
| DenoSubcommand::Add(_)
| DenoSubcommand::Remove(_)
) {
// For `deno install/add/remove` we want to force the managed resolver so it can set up `node_modules/` directory.
return false;
}
if self.node_modules_dir().ok().flatten().is_none()
&& self.maybe_node_modules_folder.is_some()
&& self
@ -1620,21 +1647,17 @@ impl CliOptions {
});
if !from_config_file.is_empty() {
// collect unstable granular flags
let mut all_valid_unstable_flags: Vec<&str> =
crate::UNSTABLE_GRANULAR_FLAGS
.iter()
.map(|granular_flag| granular_flag.name)
.collect();
let mut another_unstable_flags = Vec::from([
"sloppy-imports",
"byonm",
"bare-node-builtins",
"fmt-component",
]);
// add more unstable flags to the same vector holding granular flags
all_valid_unstable_flags.append(&mut another_unstable_flags);
let all_valid_unstable_flags: Vec<&str> = crate::UNSTABLE_GRANULAR_FLAGS
.iter()
.map(|granular_flag| granular_flag.name)
.chain([
"sloppy-imports",
"byonm",
"bare-node-builtins",
"fmt-component",
"detect-cjs",
])
.collect();
// check and warn if the unstable flag of config file isn't supported, by
// iterating through the vector holding the unstable flags
@ -1667,6 +1690,10 @@ impl CliOptions {
if let DenoSubcommand::Run(RunFlags {
watch: Some(WatchFlagsWithPaths { paths, .. }),
..
})
| DenoSubcommand::Serve(ServeFlags {
watch: Some(WatchFlagsWithPaths { paths, .. }),
..
}) = &self.flags.subcommand
{
full_paths.extend(paths.iter().map(|path| self.initial_cwd.join(path)));

25
cli/args/package_json.rs
View file

@ -5,10 +5,12 @@ use std::sync::Arc;
use deno_config::workspace::Workspace;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_package_json::PackageJsonDepValue;
use deno_package_json::PackageJsonDepValueParseError;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageReq;
use thiserror::Error;
#[derive(Debug)]
pub struct InstallNpmRemotePkg {
@ -23,11 +25,20 @@ pub struct InstallNpmWorkspacePkg {
pub target_dir: PathBuf,
}
#[derive(Debug, Error, Clone)]
#[error("Failed to install '{}'\n at {}", alias, location)]
pub struct PackageJsonDepValueParseWithLocationError {
pub location: Url,
pub alias: String,
#[source]
pub source: PackageJsonDepValueParseError,
}
#[derive(Debug, Default)]
pub struct NpmInstallDepsProvider {
remote_pkgs: Vec<InstallNpmRemotePkg>,
workspace_pkgs: Vec<InstallNpmWorkspacePkg>,
pkg_json_dep_errors: Vec<PackageJsonDepValueParseError>,
pkg_json_dep_errors: Vec<PackageJsonDepValueParseWithLocationError>,
}
impl NpmInstallDepsProvider {
@ -89,7 +100,13 @@ impl NpmInstallDepsProvider {
let dep = match dep {
Ok(dep) => dep,
Err(err) => {
pkg_json_dep_errors.push(err);
pkg_json_dep_errors.push(
PackageJsonDepValueParseWithLocationError {
location: pkg_json.specifier(),
alias,
source: err,
},
);
continue;
}
};
@ -150,7 +167,9 @@ impl NpmInstallDepsProvider {
&self.workspace_pkgs
}
pub fn pkg_json_dep_errors(&self) -> &[PackageJsonDepValueParseError] {
pub fn pkg_json_dep_errors(
&self,
) -> &[PackageJsonDepValueParseWithLocationError] {
&self.pkg_json_dep_errors
}
}

167
cli/bench/http.rs
View file

@ -1,167 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashMap;
use std::net::TcpStream;
use std::path::Path;
use std::process::Command;
use std::sync::atomic::AtomicU16;
use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
use super::Result;
pub use test_util::parse_wrk_output;
pub use test_util::WrkOutput as HttpBenchmarkResult;
// Some of the benchmarks in this file have been renamed. In case the history
// somehow gets messed up:
// "node_http" was once called "node"
// "deno_tcp" was once called "deno"
// "deno_http" was once called "deno_net_http"
const DURATION: &str = "10s";
pub fn benchmark(
target_path: &Path,
) -> Result<HashMap<String, HttpBenchmarkResult>> {
let deno_exe = test_util::deno_exe_path();
let deno_exe = deno_exe.to_string();
let hyper_hello_exe = target_path.join("test_server");
let hyper_hello_exe = hyper_hello_exe.to_str().unwrap();
let mut res = HashMap::new();
let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let http_dir = manifest_dir.join("bench").join("http");
for entry in std::fs::read_dir(&http_dir)? {
let entry = entry?;
let pathbuf = entry.path();
let path = pathbuf.to_str().unwrap();
if path.ends_with(".lua") {
continue;
}
let file_stem = pathbuf.file_stem().unwrap().to_str().unwrap();
let lua_script = http_dir.join(format!("{file_stem}.lua"));
let mut maybe_lua = None;
if lua_script.exists() {
maybe_lua = Some(lua_script.to_str().unwrap());
}
let port = get_port();
// deno run -A --unstable-net <path> <addr>
res.insert(
file_stem.to_string(),
run(
&[
deno_exe.as_str(),
"run",
"--allow-all",
"--unstable-net",
"--enable-testing-features-do-not-use",
path,
&server_addr(port),
],
port,
None,
None,
maybe_lua,
)?,
);
}
res.insert("hyper".to_string(), hyper_http(hyper_hello_exe)?);
Ok(res)
}
fn run(
server_cmd: &[&str],
port: u16,
env: Option<Vec<(String, String)>>,
origin_cmd: Option<&[&str]>,
lua_script: Option<&str>,
) -> Result<HttpBenchmarkResult> {
// Wait for port 4544 to become available.
// TODO Need to use SO_REUSEPORT with tokio::net::TcpListener.
std::thread::sleep(Duration::from_secs(5));
let mut origin = None;
if let Some(cmd) = origin_cmd {
let mut com = Command::new(cmd[0]);
com.args(&cmd[1..]);
if let Some(env) = env.clone() {
com.envs(env);
}
origin = Some(com.spawn()?);
};
println!("{}", server_cmd.join(" "));
let mut server = {
let mut com = Command::new(server_cmd[0]);
com.args(&server_cmd[1..]);
if let Some(env) = env {
com.envs(env);
}
com.spawn()?
};
// Wait for server to wake up.
let now = Instant::now();
let addr = format!("127.0.0.1:{port}");
while now.elapsed().as_secs() < 30 {
if TcpStream::connect(&addr).is_ok() {
break;
}
std::thread::sleep(Duration::from_millis(10));
}
TcpStream::connect(&addr).expect("Failed to connect to server in time");
println!("Server took {} ms to start", now.elapsed().as_millis());
let wrk = test_util::prebuilt_tool_path("wrk");
assert!(wrk.is_file());
let addr = format!("http://{addr}/");
let wrk = wrk.to_string();
let mut wrk_cmd = vec![wrk.as_str(), "-d", DURATION, "--latency", &addr];
if let Some(lua_script) = lua_script {
wrk_cmd.push("-s");
wrk_cmd.push(lua_script);
}
println!("{}", wrk_cmd.join(" "));
let output = test_util::run_collect(&wrk_cmd, None, None, None, true).0;
std::thread::sleep(Duration::from_secs(1)); // wait to capture failure. TODO racy.
println!("{output}");
assert!(
server.try_wait()?.map(|s| s.success()).unwrap_or(true),
"server ended with error"
);
server.kill()?;
if let Some(mut origin) = origin {
origin.kill()?;
}
Ok(parse_wrk_output(&output))
}
static NEXT_PORT: AtomicU16 = AtomicU16::new(4544);
pub(crate) fn get_port() -> u16 {
let p = NEXT_PORT.load(Ordering::SeqCst);
NEXT_PORT.store(p.wrapping_add(1), Ordering::SeqCst);
p
}
fn server_addr(port: u16) -> String {
format!("0.0.0.0:{port}")
}
fn hyper_http(exe: &str) -> Result<HttpBenchmarkResult> {
let port = get_port();
println!("http_benchmark testing RUST hyper");
run(&[exe, &port.to_string()], port, None, None, None)
}

10
cli/bench/http/deno_flash_hono_router.js
View file

@ -1,10 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
import { Hono } from "https://deno.land/x/hono@v2.0.9/mod.ts";
const addr = Deno.args[0] || "127.0.0.1:4500";
const [hostname, port] = addr.split(":");
const app = new Hono();
app.get("/", (c) => c.text("Hello, World!"));
Deno.serve({ port: Number(port), hostname }, app.fetch);

14
cli/bench/http/deno_flash_send_file.js
View file

@ -1,14 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
const addr = Deno.args[0] || "127.0.0.1:4500";
const [hostname, port] = addr.split(":");
const { serve } = Deno;
const path = new URL("../testdata/128k.bin", import.meta.url).pathname;
function handler() {
const file = Deno.openSync(path);
return new Response(file.readable);
}
serve({ hostname, port: Number(port) }, handler);

5
cli/bench/http/deno_http_read_headers.lua
View file

@ -1,5 +0,0 @@
wrk.headers["foo"] = "bar"
wrk.headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"
wrk.headers["Viewport-Width"] = "1920"
wrk.headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
wrk.headers["Accept-Language"] = "en,la;q=0.9"

11
cli/bench/http/deno_http_serve.js
View file

@ -1,11 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
const addr = Deno.args[0] ?? "127.0.0.1:4500";
const [hostname, port] = addr.split(":");
const { serve } = Deno;
function handler() {
return new Response("Hello World");
}
serve({ hostname, port: Number(port), reusePort: true }, handler);

5
cli/bench/http/deno_post_bin.lua
View file

@ -1,5 +0,0 @@
wrk.method = "POST"
wrk.headers["Content-Type"] = "application/octet-stream"
file = io.open("./cli/bench/testdata/128k.bin", "rb")
wrk.body = file:read("*a")

3
cli/bench/http/deno_post_json.lua
View file

@ -1,3 +0,0 @@
wrk.method = "POST"
wrk.headers["Content-Type"] = "application/json"
wrk.body = '{"hello":"deno"}'

25
cli/bench/http/deno_reactdom_ssr_flash.jsx
View file

@ -1,25 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
import { renderToReadableStream } from "https://esm.run/react-dom/server";
import * as React from "https://esm.run/react";
const { serve } = Deno;
const addr = Deno.args[0] || "127.0.0.1:4500";
const [hostname, port] = addr.split(":");
const App = () => (
<html>
<body>
<h1>Hello World</h1>
</body>
</html>
);
const headers = {
headers: {
"Content-Type": "text/html",
},
};
serve({ hostname, port: Number(port) }, async () => {
return new Response(await renderToReadableStream(<App />), headers);
});

34
cli/bench/http/deno_tcp.ts
View file

@ -1,34 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
// Used for benchmarking Deno's networking.
// TODO(bartlomieju): Replace this with a real HTTP server once
// https://github.com/denoland/deno/issues/726 is completed.
// Note: this is a keep-alive server.
// deno-lint-ignore-file no-console
const addr = Deno.args[0] || "127.0.0.1:4500";
const [hostname, port] = addr.split(":");
const listener = Deno.listen({ hostname, port: Number(port) });
const response = new TextEncoder().encode(
"HTTP/1.1 200 OK\r\nContent-Length: 12\r\n\r\nHello World\n",
);
async function handle(conn: Deno.Conn): Promise<void> {
const buffer = new Uint8Array(1024);
try {
while (true) {
await conn.read(buffer);
await conn.write(response);
}
} catch (e) {
if (
!(e instanceof Deno.errors.BrokenPipe) &&
!(e instanceof Deno.errors.ConnectionReset)
) {
throw e;
}
}
conn.close();
}
console.log("Listening on", addr);
for await (const conn of listener) {
handle(conn);
}

16
cli/bench/lsp.rs
View file

@ -150,7 +150,11 @@ fn bench_big_file_edits(deno_exe: &Path) -> Duration {
.deno_exe(deno_exe)
.build();
client.initialize_default();
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
client.change_configuration(json!({ "deno": { "enable": true } }));
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
client.write_notification(
"textDocument/didOpen",
@ -206,6 +210,8 @@ fn bench_code_lens(deno_exe: &Path) -> Duration {
.deno_exe(deno_exe)
.build();
client.initialize_default();
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
client.change_configuration(json!({ "deno": {
"enable": true,
"codeLens": {
@ -214,6 +220,8 @@ fn bench_code_lens(deno_exe: &Path) -> Duration {
"test": true,
},
} }));
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
client.write_notification(
"textDocument/didOpen",
@ -257,7 +265,11 @@ fn bench_find_replace(deno_exe: &Path) -> Duration {
.deno_exe(deno_exe)
.build();
client.initialize_default();
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
client.change_configuration(json!({ "deno": { "enable": true } }));
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
for i in 0..10 {
client.write_notification(
@ -341,7 +353,11 @@ fn bench_startup_shutdown(deno_exe: &Path) -> Duration {
.deno_exe(deno_exe)
.build();
client.initialize_default();
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
client.change_configuration(json!({ "deno": { "enable": true } }));
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
client.write_notification(
"textDocument/didOpen",

4
cli/bench/lsp_bench_standalone.rs
View file

@ -13,7 +13,11 @@ use test_util::lsp::LspClientBuilder;
fn incremental_change_wait(bench: &mut Bencher) {
let mut client = LspClientBuilder::new().use_diagnostic_sync(false).build();
client.initialize_default();
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
client.change_configuration(json!({ "deno": { "enable": true } }));
let (method, _): (String, Option<Value>) = client.read_notification();
assert_eq!(method, "deno/didRefreshDenoConfigurationTree");
client.write_notification(
"textDocument/didOpen",

19
cli/bench/main.rs
View file

@ -17,7 +17,6 @@ use std::process::Stdio;
use std::time::SystemTime;
use test_util::PathRef;
mod http;
mod lsp;
fn read_json(filename: &Path) -> Result<Value> {
@ -345,9 +344,11 @@ struct BenchResult {
binary_size: HashMap<String, i64>,
bundle_size: HashMap<String, i64>,
cargo_deps: usize,
// TODO(bartlomieju): remove
max_latency: HashMap<String, f64>,
max_memory: HashMap<String, i64>,
lsp_exec_time: HashMap<String, i64>,
// TODO(bartlomieju): remove
req_per_sec: HashMap<String, i64>,
syscall_count: HashMap<String, i64>,
thread_count: HashMap<String, i64>,
@ -362,7 +363,6 @@ async fn main() -> Result<()> {
"binary_size",
"cargo_deps",
"lsp",
"http",
"strace",
"mem_usage",
];
@ -427,21 +427,6 @@ async fn main() -> Result<()> {
new_data.lsp_exec_time = lsp_exec_times;
}
if benchmarks.contains(&"http") && cfg!(not(target_os = "windows")) {
let stats = http::benchmark(target_dir.as_path())?;
let req_per_sec = stats
.iter()
.map(|(name, result)| (name.clone(), result.requests as i64))
.collect();
new_data.req_per_sec = req_per_sec;
let max_latency = stats
.iter()
.map(|(name, result)| (name.clone(), result.latency))
.collect();
new_data.max_latency = max_latency;
}
if cfg!(target_os = "linux") && benchmarks.contains(&"strace") {
use std::io::Read;

55
cli/build.rs
View file

@ -365,6 +365,9 @@ fn main() {
return;
}
deno_napi::print_linker_flags("deno");
deno_napi::print_linker_flags("denort");
// Host snapshots won't work when cross compiling.
let target = env::var("TARGET").unwrap();
let host = env::var("HOST").unwrap();
@ -374,58 +377,6 @@ fn main() {
panic!("Cross compiling with snapshot is not supported.");
}
let symbols_file_name = match env::consts::OS {
"android" | "freebsd" | "openbsd" => {
"generated_symbol_exports_list_linux.def".to_string()
}
os => format!("generated_symbol_exports_list_{}.def", os),
};
let symbols_path = std::path::Path::new("napi")
.join(symbols_file_name)
.canonicalize()
.expect(
"Missing symbols list! Generate using tools/napi/generate_symbols_lists.js",
);
println!("cargo:rustc-rerun-if-changed={}", symbols_path.display());
#[cfg(target_os = "windows")]
println!(
"cargo:rustc-link-arg-bin=deno=/DEF:{}",
symbols_path.display()
);
#[cfg(target_os = "macos")]
println!(
"cargo:rustc-link-arg-bin=deno=-Wl,-exported_symbols_list,{}",
symbols_path.display()
);
#[cfg(target_os = "linux")]
{
// If a custom compiler is set, the glibc version is not reliable.
// Here, we assume that if a custom compiler is used, that it will be modern enough to support a dynamic symbol list.
if env::var("CC").is_err()
&& glibc_version::get_version()
.map(|ver| ver.major <= 2 && ver.minor < 35)
.unwrap_or(false)
{
println!("cargo:warning=Compiling with all symbols exported, this will result in a larger binary. Please use glibc 2.35 or later for an optimised build.");
println!("cargo:rustc-link-arg-bin=deno=-rdynamic");
} else {
println!(
"cargo:rustc-link-arg-bin=deno=-Wl,--export-dynamic-symbol-list={}",
symbols_path.display()
);
}
}
#[cfg(target_os = "android")]
println!(
"cargo:rustc-link-arg-bin=deno=-Wl,--export-dynamic-symbol-list={}",
symbols_path.display()
);
// To debug snapshot issues uncomment:
// op_fetch_asset::trace_serializer();

6
cli/cache/cache_db.rs vendored
View file

@ -57,7 +57,7 @@ impl rusqlite::types::FromSql for CacheDBHash {
}
/// What should the cache should do on failure?
#[derive(Default)]
#[derive(Debug, Default)]
pub enum CacheFailure {
/// Return errors if failure mode otherwise unspecified.
#[default]
@ -69,6 +69,7 @@ pub enum CacheFailure {
}
/// Configuration SQL and other parameters for a [`CacheDB`].
#[derive(Debug)]
pub struct CacheDBConfiguration {
/// SQL to run for a new database.
pub table_initializer: &'static str,
@ -98,6 +99,7 @@ impl CacheDBConfiguration {
}
}
#[derive(Debug)]
enum ConnectionState {
Connected(Connection),
Blackhole,
@ -106,7 +108,7 @@ enum ConnectionState {
/// A cache database that eagerly initializes itself off-thread, preventing initialization operations
/// from blocking the main thread.
#[derive(Clone)]
#[derive(Debug, Clone)]
pub struct CacheDB {
// TODO(mmastrac): We can probably simplify our thread-safe implementation here
conn: Arc<Mutex<OnceCell<ConnectionState>>>,

27
cli/cache/emit.rs vendored
View file

@ -10,6 +10,7 @@ use deno_core::unsync::sync::AtomicFlag;
use super::DiskCache;
/// The cache that stores previously emitted files.
#[derive(Debug)]
pub struct EmitCache {
disk_cache: DiskCache,
emit_failed_flag: AtomicFlag,
@ -39,7 +40,7 @@ impl EmitCache {
&self,
specifier: &ModuleSpecifier,
expected_source_hash: u64,
) -> Option<Vec<u8>> {
) -> Option<String> {
let emit_filename = self.get_emit_filename(specifier)?;
let bytes = self.disk_cache.get(&emit_filename).ok()?;
self
@ -91,6 +92,7 @@ impl EmitCache {
const LAST_LINE_PREFIX: &str = "\n// denoCacheMetadata=";
#[derive(Debug)]
struct EmitFileSerializer {
cli_version: &'static str,
}
@ -100,7 +102,7 @@ impl EmitFileSerializer {
&self,
mut bytes: Vec<u8>,
expected_source_hash: u64,
) -> Option<Vec<u8>> {
) -> Option<String> {
let last_newline_index = bytes.iter().rposition(|&b| b == b'\n')?;
let (content, last_line) = bytes.split_at(last_newline_index);
let hashes = last_line.strip_prefix(LAST_LINE_PREFIX.as_bytes())?;
@ -120,7 +122,7 @@ impl EmitFileSerializer {
// everything looks good, truncate and return it
bytes.truncate(content.len());
Some(bytes)
String::from_utf8(bytes).ok()
}
pub fn serialize(&self, code: &[u8], source_hash: u64) -> Vec<u8> {
@ -170,8 +172,6 @@ mod test {
},
emit_failed_flag: Default::default(),
};
let to_string =
|bytes: Vec<u8>| -> String { String::from_utf8(bytes).unwrap() };
let specifier1 =
ModuleSpecifier::from_file_path(temp_dir.path().join("file1.ts"))
@ -188,13 +188,10 @@ mod test {
assert_eq!(cache.get_emit_code(&specifier1, 5), None);
// providing the correct source hash
assert_eq!(
cache.get_emit_code(&specifier1, 10).map(to_string),
cache.get_emit_code(&specifier1, 10),
Some(emit_code1.clone()),
);
assert_eq!(
cache.get_emit_code(&specifier2, 2).map(to_string),
Some(emit_code2)
);
assert_eq!(cache.get_emit_code(&specifier2, 2), Some(emit_code2));
// try changing the cli version (should not load previous ones)
let cache = EmitCache {
@ -215,18 +212,12 @@ mod test {
},
emit_failed_flag: Default::default(),
};
assert_eq!(
cache.get_emit_code(&specifier1, 5).map(to_string),
Some(emit_code1)
);
assert_eq!(cache.get_emit_code(&specifier1, 5), Some(emit_code1));
// adding when already exists should not cause issue
let emit_code3 = "asdf".to_string();
cache.set_emit_code(&specifier1, 20, emit_code3.as_bytes());
assert_eq!(cache.get_emit_code(&specifier1, 5), None);
assert_eq!(
cache.get_emit_code(&specifier1, 20).map(to_string),
Some(emit_code3)
);
assert_eq!(cache.get_emit_code(&specifier1, 20), Some(emit_code3));
}
}

53
cli/cache/mod.rs vendored
View file

@ -8,11 +8,9 @@ use crate::file_fetcher::FetchOptions;
use crate::file_fetcher::FetchPermissionsOptionRef;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::FileOrRedirect;
use crate::npm::CliNpmResolver;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::AtomicWriteFileFsAdapter;
use crate::util::path::specifier_has_extension;
use deno_ast::MediaType;
use deno_core::futures;
@ -22,7 +20,9 @@ use deno_graph::source::CacheInfo;
use deno_graph::source::LoadFuture;
use deno_graph::source::LoadResponse;
use deno_graph::source::Loader;
use deno_runtime::deno_fs;
use deno_runtime::deno_permissions::PermissionsContainer;
use node_resolver::InNpmPackageChecker;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
@ -182,28 +182,31 @@ pub struct FetchCacherOptions {
/// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides
/// a concise interface to the DENO_DIR when building module graphs.
pub struct FetchCacher {
file_fetcher: Arc<FileFetcher>,
pub file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
file_fetcher: Arc<FileFetcher>,
fs: Arc<dyn deno_fs::FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
npm_resolver: Arc<dyn CliNpmResolver>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
permissions: PermissionsContainer,
cache_info_enabled: bool,
is_deno_publish: bool,
cache_info_enabled: bool,
}
impl FetchCacher {
pub fn new(
file_fetcher: Arc<FileFetcher>,
fs: Arc<dyn deno_fs::FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
npm_resolver: Arc<dyn CliNpmResolver>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
options: FetchCacherOptions,
) -> Self {
Self {
file_fetcher,
fs,
global_http_cache,
npm_resolver,
in_npm_pkg_checker,
module_info_cache,
file_header_overrides: options.file_header_overrides,
permissions: options.permissions,
@ -258,28 +261,21 @@ impl Loader for FetchCacher {
) -> LoadFuture {
use deno_graph::source::CacheSetting as LoaderCacheSetting;
if specifier.scheme() == "file" {
if specifier.path().contains("/node_modules/") {
// The specifier might be in a completely different symlinked tree than
// what the node_modules url is in (ex. `/my-project-1/node_modules`
// symlinked to `/my-project-2/node_modules`), so first we checked if the path
// is in a node_modules dir to avoid needlessly canonicalizing, then now compare
// against the canonicalized specifier.
let specifier =
crate::node::resolve_specifier_into_node_modules(specifier);
if self.npm_resolver.in_npm_package(&specifier) {
return Box::pin(futures::future::ready(Ok(Some(
LoadResponse::External { specifier },
))));
}
}
// make local CJS modules external to the graph
if specifier_has_extension(specifier, "cjs") {
if specifier.scheme() == "file"
&& specifier.path().contains("/node_modules/")
{
// The specifier might be in a completely different symlinked tree than
// what the node_modules url is in (ex. `/my-project-1/node_modules`
// symlinked to `/my-project-2/node_modules`), so first we checked if the path
// is in a node_modules dir to avoid needlessly canonicalizing, then now compare
// against the canonicalized specifier.
let specifier = crate::node::resolve_specifier_into_node_modules(
specifier,
self.fs.as_ref(),
);
if self.in_npm_pkg_checker.in_npm_package(&specifier) {
return Box::pin(futures::future::ready(Ok(Some(
LoadResponse::External {
specifier: specifier.clone(),
},
LoadResponse::External { specifier },
))));
}
}
@ -325,6 +321,7 @@ impl Loader for FetchCacher {
} else {
FetchPermissionsOptionRef::DynamicContainer(&permissions)
},
maybe_auth: None,
maybe_accept: None,
maybe_cache_setting: maybe_cache_setting.as_ref(),
},

140
cli/cache/module_info.rs vendored
View file

@ -44,18 +44,32 @@ pub static MODULE_INFO_CACHE_DB: CacheDBConfiguration = CacheDBConfiguration {
/// A cache of `deno_graph::ModuleInfo` objects. Using this leads to a considerable
/// performance improvement because when it exists we can skip parsing a module for
/// deno_graph.
#[derive(Debug)]
pub struct ModuleInfoCache {
conn: CacheDB,
parsed_source_cache: Arc<ParsedSourceCache>,
}
impl ModuleInfoCache {
#[cfg(test)]
pub fn new_in_memory(version: &'static str) -> Self {
Self::new(CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version))
pub fn new_in_memory(
version: &'static str,
parsed_source_cache: Arc<ParsedSourceCache>,
) -> Self {
Self::new(
CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version),
parsed_source_cache,
)
}
pub fn new(conn: CacheDB) -> Self {
Self { conn }
pub fn new(
conn: CacheDB,
parsed_source_cache: Arc<ParsedSourceCache>,
) -> Self {
Self {
conn,
parsed_source_cache,
}
}
/// Useful for testing: re-create this cache DB with a different current version.
@ -63,6 +77,7 @@ impl ModuleInfoCache {
pub(crate) fn recreate_with_version(self, version: &'static str) -> Self {
Self {
conn: self.conn.recreate_with_version(version),
parsed_source_cache: self.parsed_source_cache,
}
}
@ -113,13 +128,10 @@ impl ModuleInfoCache {
Ok(())
}
pub fn as_module_analyzer<'a>(
&'a self,
parsed_source_cache: &'a Arc<ParsedSourceCache>,
) -> ModuleInfoCacheModuleAnalyzer<'a> {
pub fn as_module_analyzer(&self) -> ModuleInfoCacheModuleAnalyzer {
ModuleInfoCacheModuleAnalyzer {
module_info_cache: self,
parsed_source_cache,
parsed_source_cache: &self.parsed_source_cache,
}
}
}
@ -129,6 +141,84 @@ pub struct ModuleInfoCacheModuleAnalyzer<'a> {
parsed_source_cache: &'a Arc<ParsedSourceCache>,
}
impl<'a> ModuleInfoCacheModuleAnalyzer<'a> {
fn load_cached_module_info(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
source_hash: CacheDBHash,
) -> Option<ModuleInfo> {
match self.module_info_cache.get_module_info(
specifier,
media_type,
source_hash,
) {
Ok(Some(info)) => Some(info),
Ok(None) => None,
Err(err) => {
log::debug!(
"Error loading module cache info for {}. {:#}",
specifier,
err
);
None
}
}
}
fn save_module_info_to_cache(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
source_hash: CacheDBHash,
module_info: &ModuleInfo,
) {
if let Err(err) = self.module_info_cache.set_module_info(
specifier,
media_type,
source_hash,
module_info,
) {
log::debug!(
"Error saving module cache info for {}. {:#}",
specifier,
err
);
}
}
pub fn analyze_sync(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
source: &Arc<str>,
) -> Result<ModuleInfo, deno_ast::ParseDiagnostic> {
// attempt to load from the cache
let source_hash = CacheDBHash::from_source(source);
if let Some(info) =
self.load_cached_module_info(specifier, media_type, source_hash)
{
return Ok(info);
}
// otherwise, get the module info from the parsed source cache
let parser = self.parsed_source_cache.as_capturing_parser();
let analyzer = ParserModuleAnalyzer::new(&parser);
let module_info =
analyzer.analyze_sync(specifier, source.clone(), media_type)?;
// then attempt to cache it
self.save_module_info_to_cache(
specifier,
media_type,
source_hash,
&module_info,
);
Ok(module_info)
}
}
#[async_trait::async_trait(?Send)]
impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
async fn analyze(
@ -139,20 +229,10 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
) -> Result<ModuleInfo, deno_ast::ParseDiagnostic> {
// attempt to load from the cache
let source_hash = CacheDBHash::from_source(&source);
match self.module_info_cache.get_module_info(
specifier,
media_type,
source_hash,
) {
Ok(Some(info)) => return Ok(info),
Ok(None) => {}
Err(err) => {
log::debug!(
"Error loading module cache info for {}. {:#}",
specifier,
err
);
}
if let Some(info) =
self.load_cached_module_info(specifier, media_type, source_hash)
{
return Ok(info);
}
// otherwise, get the module info from the parsed source cache
@ -169,18 +249,12 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
.unwrap()?;
// then attempt to cache it
if let Err(err) = self.module_info_cache.set_module_info(
self.save_module_info_to_cache(
specifier,
media_type,
source_hash,
&module_info,
) {
log::debug!(
"Error saving module cache info for {}. {:#}",
specifier,
err
);
}
);
Ok(module_info)
}
@ -202,7 +276,7 @@ fn serialize_media_type(media_type: MediaType) -> i64 {
Tsx => 11,
Json => 12,
Wasm => 13,
TsBuildInfo => 14,
Css => 14,
SourceMap => 15,
Unknown => 16,
}
@ -217,7 +291,7 @@ mod test {
#[test]
pub fn module_info_cache_general_use() {
let cache = ModuleInfoCache::new_in_memory("1.0.0");
let cache = ModuleInfoCache::new_in_memory("1.0.0", Default::default());
let specifier1 =
ModuleSpecifier::parse("https://localhost/mod.ts").unwrap();
let specifier2 =

20
cli/cache/parsed_source.rs vendored
View file

@ -7,9 +7,9 @@ use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_ast::ParsedSource;
use deno_core::parking_lot::Mutex;
use deno_graph::CapturingModuleParser;
use deno_graph::DefaultModuleParser;
use deno_graph::ModuleParser;
use deno_graph::CapturingEsParser;
use deno_graph::DefaultEsParser;
use deno_graph::EsParser;
use deno_graph::ParseOptions;
use deno_graph::ParsedSourceStore;
@ -46,7 +46,7 @@ impl<'a> LazyGraphSourceParser<'a> {
}
}
#[derive(Default)]
#[derive(Debug, Default)]
pub struct ParsedSourceCache {
sources: Mutex<HashMap<ModuleSpecifier, ParsedSource>>,
}
@ -57,12 +57,11 @@ impl ParsedSourceCache {
module: &deno_graph::JsModule,
) -> Result<ParsedSource, deno_ast::ParseDiagnostic> {
let parser = self.as_capturing_parser();
// this will conditionally parse because it's using a CapturingModuleParser
parser.parse_module(ParseOptions {
// this will conditionally parse because it's using a CapturingEsParser
parser.parse_program(ParseOptions {
specifier: &module.specifier,
source: module.source.clone(),
media_type: module.media_type,
// don't bother enabling because this method is currently only used for vendoring
scope_analysis: false,
})
}
@ -86,10 +85,9 @@ impl ParsedSourceCache {
specifier,
source,
media_type,
// don't bother enabling because this method is currently only used for emitting
scope_analysis: false,
};
DefaultModuleParser.parse_module(options)
DefaultEsParser.parse_program(options)
}
/// Frees the parsed source from memory.
@ -99,8 +97,8 @@ impl ParsedSourceCache {
/// Creates a parser that will reuse a ParsedSource from the store
/// if it exists, or else parse.
pub fn as_capturing_parser(&self) -> CapturingModuleParser {
CapturingModuleParser::new(None, self)
pub fn as_capturing_parser(&self) -> CapturingEsParser {
CapturingEsParser::new(None, self)
}
}

123
cli/emit.rs
View file

@ -3,24 +3,28 @@
use crate::cache::EmitCache;
use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::resolver::CjsTracker;
use deno_ast::ModuleKind;
use deno_ast::SourceMapOption;
use deno_ast::SourceRange;
use deno_ast::SourceRanged;
use deno_ast::SourceRangedForSpanned;
use deno_ast::TranspileModuleOptions;
use deno_ast::TranspileResult;
use deno_core::error::AnyError;
use deno_core::futures::stream::FuturesUnordered;
use deno_core::futures::FutureExt;
use deno_core::futures::StreamExt;
use deno_core::ModuleCodeBytes;
use deno_core::ModuleSpecifier;
use deno_graph::MediaType;
use deno_graph::Module;
use deno_graph::ModuleGraph;
use std::sync::Arc;
#[derive(Debug)]
pub struct Emitter {
cjs_tracker: Arc<CjsTracker>,
emit_cache: Arc<EmitCache>,
parsed_source_cache: Arc<ParsedSourceCache>,
transpile_and_emit_options:
@ -31,6 +35,7 @@ pub struct Emitter {
impl Emitter {
pub fn new(
cjs_tracker: Arc<CjsTracker>,
emit_cache: Arc<EmitCache>,
parsed_source_cache: Arc<ParsedSourceCache>,
transpile_options: deno_ast::TranspileOptions,
@ -43,6 +48,7 @@ impl Emitter {
hasher.finish()
};
Self {
cjs_tracker,
emit_cache,
parsed_source_cache,
transpile_and_emit_options: Arc::new((transpile_options, emit_options)),
@ -60,20 +66,19 @@ impl Emitter {
continue;
};
let is_emittable = matches!(
module.media_type,
MediaType::TypeScript
| MediaType::Mts
| MediaType::Cts
| MediaType::Jsx
| MediaType::Tsx
);
if is_emittable {
if module.media_type.is_emittable() {
futures.push(
self
.emit_parsed_source(
&module.specifier,
module.media_type,
ModuleKind::from_is_cjs(
self.cjs_tracker.is_cjs_with_known_is_script(
&module.specifier,
module.media_type,
module.is_script,
)?,
),
&module.source,
)
.boxed_local(),
@ -92,9 +97,10 @@ impl Emitter {
pub fn maybe_cached_emit(
&self,
specifier: &ModuleSpecifier,
module_kind: deno_ast::ModuleKind,
source: &str,
) -> Option<Vec<u8>> {
let source_hash = self.get_source_hash(source);
) -> Option<String> {
let source_hash = self.get_source_hash(module_kind, source);
self.emit_cache.get_emit_code(specifier, source_hash)
}
@ -102,25 +108,27 @@ impl Emitter {
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
module_kind: deno_ast::ModuleKind,
source: &Arc<str>,
) -> Result<ModuleCodeBytes, AnyError> {
) -> Result<String, AnyError> {
// Note: keep this in sync with the sync version below
let helper = EmitParsedSourceHelper(self);
match helper.pre_emit_parsed_source(specifier, source) {
match helper.pre_emit_parsed_source(specifier, module_kind, source) {
PreEmitResult::Cached(emitted_text) => Ok(emitted_text),
PreEmitResult::NotCached { source_hash } => {
let parsed_source_cache = self.parsed_source_cache.clone();
let transpile_and_emit_options =
self.transpile_and_emit_options.clone();
let transpile_result = deno_core::unsync::spawn_blocking({
let transpiled_source = deno_core::unsync::spawn_blocking({
let specifier = specifier.clone();
let source = source.clone();
move || -> Result<_, AnyError> {
EmitParsedSourceHelper::transpile(
&parsed_source_cache,
&specifier,
source.clone(),
media_type,
module_kind,
source.clone(),
&transpile_and_emit_options.0,
&transpile_and_emit_options.1,
)
@ -128,11 +136,12 @@ impl Emitter {
})
.await
.unwrap()?;
Ok(helper.post_emit_parsed_source(
helper.post_emit_parsed_source(
specifier,
transpile_result,
&transpiled_source,
source_hash,
))
);
Ok(transpiled_source)
}
}
}
@ -141,26 +150,29 @@ impl Emitter {
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
module_kind: deno_ast::ModuleKind,
source: &Arc<str>,
) -> Result<ModuleCodeBytes, AnyError> {
) -> Result<String, AnyError> {
// Note: keep this in sync with the async version above
let helper = EmitParsedSourceHelper(self);
match helper.pre_emit_parsed_source(specifier, source) {
match helper.pre_emit_parsed_source(specifier, module_kind, source) {
PreEmitResult::Cached(emitted_text) => Ok(emitted_text),
PreEmitResult::NotCached { source_hash } => {
let transpile_result = EmitParsedSourceHelper::transpile(
let transpiled_source = EmitParsedSourceHelper::transpile(
&self.parsed_source_cache,
specifier,
source.clone(),
media_type,
module_kind,
source.clone(),
&self.transpile_and_emit_options.0,
&self.transpile_and_emit_options.1,
)?;
Ok(helper.post_emit_parsed_source(
helper.post_emit_parsed_source(
specifier,
transpile_result,
&transpiled_source,
source_hash,
))
);
Ok(transpiled_source)
}
}
}
@ -169,6 +181,7 @@ impl Emitter {
pub async fn load_and_emit_for_hmr(
&self,
specifier: &ModuleSpecifier,
module_kind: deno_ast::ModuleKind,
) -> Result<String, AnyError> {
let media_type = MediaType::from_specifier(specifier);
let source_code = tokio::fs::read_to_string(
@ -191,9 +204,14 @@ impl Emitter {
let mut options = self.transpile_and_emit_options.1.clone();
options.source_map = SourceMapOption::None;
let transpiled_source = parsed_source
.transpile(&self.transpile_and_emit_options.0, &options)?
.into_source()
.into_string()?;
.transpile(
&self.transpile_and_emit_options.0,
&deno_ast::TranspileModuleOptions {
module_kind: Some(module_kind),
},
&options,
)?
.into_source();
Ok(transpiled_source.text)
}
MediaType::JavaScript
@ -204,7 +222,7 @@ impl Emitter {
| MediaType::Dcts
| MediaType::Json
| MediaType::Wasm
| MediaType::TsBuildInfo
| MediaType::Css
| MediaType::SourceMap
| MediaType::Unknown => {
// clear this specifier from the parsed source cache as it's now out of date
@ -217,16 +235,17 @@ impl Emitter {
/// A hashing function that takes the source code and uses the global emit
/// options then generates a string hash which can be stored to
/// determine if the cached emit is valid or not.
fn get_source_hash(&self, source_text: &str) -> u64 {
fn get_source_hash(&self, module_kind: ModuleKind, source_text: &str) -> u64 {
FastInsecureHasher::new_without_deno_version() // stored in the transpile_and_emit_options_hash
.write_str(source_text)
.write_u64(self.transpile_and_emit_options_hash)
.write_hashable(module_kind)
.finish()
}
}
enum PreEmitResult {
Cached(ModuleCodeBytes),
Cached(String),
NotCached { source_hash: u64 },
}
@ -237,14 +256,15 @@ impl<'a> EmitParsedSourceHelper<'a> {
pub fn pre_emit_parsed_source(
&self,
specifier: &ModuleSpecifier,
module_kind: deno_ast::ModuleKind,
source: &Arc<str>,
) -> PreEmitResult {
let source_hash = self.0.get_source_hash(source);
let source_hash = self.0.get_source_hash(module_kind, source);
if let Some(emit_code) =
self.0.emit_cache.get_emit_code(specifier, source_hash)
{
PreEmitResult::Cached(emit_code.into_boxed_slice().into())
PreEmitResult::Cached(emit_code)
} else {
PreEmitResult::NotCached { source_hash }
}
@ -253,25 +273,24 @@ impl<'a> EmitParsedSourceHelper<'a> {
pub fn transpile(
parsed_source_cache: &ParsedSourceCache,
specifier: &ModuleSpecifier,
source: Arc<str>,
media_type: MediaType,
module_kind: deno_ast::ModuleKind,
source: Arc<str>,
transpile_options: &deno_ast::TranspileOptions,
emit_options: &deno_ast::EmitOptions,
) -> Result<TranspileResult, AnyError> {
) -> Result<String, AnyError> {
// nothing else needs the parsed source at this point, so remove from
// the cache in order to not transpile owned
let parsed_source = parsed_source_cache
.remove_or_parse_module(specifier, source, media_type)?;
ensure_no_import_assertion(&parsed_source)?;
Ok(parsed_source.transpile(transpile_options, emit_options)?)
}
pub fn post_emit_parsed_source(
&self,
specifier: &ModuleSpecifier,
transpile_result: TranspileResult,
source_hash: u64,
) -> ModuleCodeBytes {
let transpile_result = parsed_source.transpile(
transpile_options,
&TranspileModuleOptions {
module_kind: Some(module_kind),
},
emit_options,
)?;
let transpiled_source = match transpile_result {
TranspileResult::Owned(source) => source,
TranspileResult::Cloned(source) => {
@ -280,12 +299,20 @@ impl<'a> EmitParsedSourceHelper<'a> {
}
};
debug_assert!(transpiled_source.source_map.is_none());
Ok(transpiled_source.text)
}
pub fn post_emit_parsed_source(
&self,
specifier: &ModuleSpecifier,
transpiled_source: &str,
source_hash: u64,
) {
self.0.emit_cache.set_emit_code(
specifier,
source_hash,
&transpiled_source.source,
transpiled_source.as_bytes(),
);
transpiled_source.source.into_boxed_slice().into()
}
}
@ -317,7 +344,7 @@ fn ensure_no_import_assertion(
deno_core::anyhow::anyhow!("{}", msg)
}
let Some(module) = parsed_source.program_ref().as_module() else {
let deno_ast::ProgramRef::Module(module) = parsed_source.program_ref() else {
return Ok(());
};

264
cli/factory.rs
View file

@ -11,6 +11,7 @@ use crate::args::StorageKeyResolver;
use crate::args::TsConfigType;
use crate::cache::Caches;
use crate::cache::CodeCache;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::DenoDir;
use crate::cache::DenoDirProvider;
use crate::cache::EmitCache;
@ -32,12 +33,16 @@ use crate::module_loader::ModuleLoadPreparer;
use crate::node::CliCjsCodeAnalyzer;
use crate::node::CliNodeCodeTranslator;
use crate::npm::create_cli_npm_resolver;
use crate::npm::create_in_npm_pkg_checker;
use crate::npm::CliByonmNpmResolverCreateOptions;
use crate::npm::CliManagedInNpmPkgCheckerCreateOptions;
use crate::npm::CliManagedNpmResolverCreateOptions;
use crate::npm::CliNpmResolver;
use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::resolver::CjsResolutionStore;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::resolver::CjsTracker;
use crate::resolver::CjsTrackerOptions;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliGraphResolver;
use crate::resolver::CliGraphResolverOptions;
@ -50,6 +55,7 @@ use crate::tools::check::TypeChecker;
use crate::tools::coverage::CoverageCollector;
use crate::tools::lint::LintRuleProvider;
use crate::tools::run::hmr::HmrRunner;
use crate::tsc::TypeCheckingCjsTracker;
use crate::util::file_watcher::WatcherCommunicator;
use crate::util::fs::canonicalize_path_maybe_not_exists;
use crate::util::progress_bar::ProgressBar;
@ -58,6 +64,7 @@ use crate::worker::CliMainWorkerFactory;
use crate::worker::CliMainWorkerOptions;
use std::path::PathBuf;
use deno_cache_dir::npm::NpmCacheDir;
use deno_config::workspace::PackageJsonDepResolution;
use deno_config::workspace::WorkspaceResolver;
use deno_core::error::AnyError;
@ -67,6 +74,7 @@ use deno_core::FeatureChecker;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_permissions::Permissions;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::rustls::RootCertStore;
@ -76,6 +84,7 @@ use deno_runtime::inspector_server::InspectorServer;
use deno_runtime::permissions::RuntimePermissionDescriptorParser;
use log::warn;
use node_resolver::analyze::NodeCodeTranslator;
use node_resolver::InNpmPackageChecker;
use once_cell::sync::OnceCell;
use std::future::Future;
use std::sync::Arc;
@ -163,38 +172,41 @@ impl<T> Deferred<T> {
#[derive(Default)]
struct CliFactoryServices {
cli_options: Deferred<Arc<CliOptions>>,
blob_store: Deferred<Arc<BlobStore>>,
caches: Deferred<Arc<Caches>>,
cjs_tracker: Deferred<Arc<CjsTracker>>,
cli_node_resolver: Deferred<Arc<CliNodeResolver>>,
cli_options: Deferred<Arc<CliOptions>>,
code_cache: Deferred<Arc<CodeCache>>,
emit_cache: Deferred<Arc<EmitCache>>,
emitter: Deferred<Arc<Emitter>>,
feature_checker: Deferred<Arc<FeatureChecker>>,
file_fetcher: Deferred<Arc<FileFetcher>>,
fs: Deferred<Arc<dyn deno_fs::FileSystem>>,
global_http_cache: Deferred<Arc<GlobalHttpCache>>,
http_cache: Deferred<Arc<dyn HttpCache>>,
http_client_provider: Deferred<Arc<HttpClientProvider>>,
emit_cache: Deferred<Arc<EmitCache>>,
emitter: Deferred<Arc<Emitter>>,
fs: Deferred<Arc<dyn deno_fs::FileSystem>>,
in_npm_pkg_checker: Deferred<Arc<dyn InNpmPackageChecker>>,
main_graph_container: Deferred<Arc<MainModuleGraphContainer>>,
maybe_inspector_server: Deferred<Option<Arc<InspectorServer>>>,
root_cert_store_provider: Deferred<Arc<dyn RootCertStoreProvider>>,
blob_store: Deferred<Arc<BlobStore>>,
module_info_cache: Deferred<Arc<ModuleInfoCache>>,
parsed_source_cache: Deferred<Arc<ParsedSourceCache>>,
resolver: Deferred<Arc<CliGraphResolver>>,
maybe_file_watcher_reporter: Deferred<Option<FileWatcherReporter>>,
maybe_inspector_server: Deferred<Option<Arc<InspectorServer>>>,
module_graph_builder: Deferred<Arc<ModuleGraphBuilder>>,
module_graph_creator: Deferred<Arc<ModuleGraphCreator>>,
module_info_cache: Deferred<Arc<ModuleInfoCache>>,
module_load_preparer: Deferred<Arc<ModuleLoadPreparer>>,
node_code_translator: Deferred<Arc<CliNodeCodeTranslator>>,
node_resolver: Deferred<Arc<NodeResolver>>,
npm_cache_dir: Deferred<Arc<NpmCacheDir>>,
npm_resolver: Deferred<Arc<dyn CliNpmResolver>>,
parsed_source_cache: Deferred<Arc<ParsedSourceCache>>,
permission_desc_parser: Deferred<Arc<RuntimePermissionDescriptorParser>>,
pkg_json_resolver: Deferred<Arc<PackageJsonResolver>>,
resolver: Deferred<Arc<CliGraphResolver>>,
root_cert_store_provider: Deferred<Arc<dyn RootCertStoreProvider>>,
root_permissions_container: Deferred<PermissionsContainer>,
sloppy_imports_resolver: Deferred<Option<Arc<CliSloppyImportsResolver>>>,
text_only_progress_bar: Deferred<ProgressBar>,
type_checker: Deferred<Arc<TypeChecker>>,
cjs_resolutions: Deferred<Arc<CjsResolutionStore>>,
cli_node_resolver: Deferred<Arc<CliNodeResolver>>,
feature_checker: Deferred<Arc<FeatureChecker>>,
code_cache: Deferred<Arc<CodeCache>>,
workspace_resolver: Deferred<Arc<WorkspaceResolver>>,
}
@ -351,56 +363,112 @@ impl CliFactory {
self.services.fs.get_or_init(|| Arc::new(deno_fs::RealFs))
}
pub fn in_npm_pkg_checker(
&self,
) -> Result<&Arc<dyn InNpmPackageChecker>, AnyError> {
self.services.in_npm_pkg_checker.get_or_try_init(|| {
let cli_options = self.cli_options()?;
let options = if cli_options.use_byonm() {
CreateInNpmPkgCheckerOptions::Byonm
} else {
CreateInNpmPkgCheckerOptions::Managed(
CliManagedInNpmPkgCheckerCreateOptions {
root_cache_dir_url: self.npm_cache_dir()?.root_dir_url(),
maybe_node_modules_path: cli_options
.node_modules_dir_path()
.map(|p| p.as_path()),
},
)
};
Ok(create_in_npm_pkg_checker(options))
})
}
pub fn npm_cache_dir(&self) -> Result<&Arc<NpmCacheDir>, AnyError> {
self.services.npm_cache_dir.get_or_try_init(|| {
let fs = self.fs();
let global_path = self.deno_dir()?.npm_folder_path();
let cli_options = self.cli_options()?;
Ok(Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
global_path,
cli_options.npmrc().get_all_known_registries_urls(),
)))
})
}
pub async fn npm_resolver(
&self,
) -> Result<&Arc<dyn CliNpmResolver>, AnyError> {
self
.services
.npm_resolver
.get_or_try_init_async(async {
let fs = self.fs();
let cli_options = self.cli_options()?;
// For `deno install` we want to force the managed resolver so it can set up `node_modules/` directory.
create_cli_npm_resolver(if cli_options.use_byonm() && !matches!(cli_options.sub_command(), DenoSubcommand::Install(_) | DenoSubcommand::Add(_) | DenoSubcommand::Remove(_)) {
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(fs.clone()),
root_node_modules_dir: Some(match cli_options.node_modules_dir_path() {
Some(node_modules_path) => node_modules_path.to_path_buf(),
// path needs to be canonicalized for node resolution
// (node_modules_dir_path above is already canonicalized)
None => canonicalize_path_maybe_not_exists(cli_options.initial_cwd())?
.join("node_modules"),
}),
})
} else {
CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions {
snapshot: match cli_options.resolve_npm_resolution_snapshot()? {
Some(snapshot) => {
CliNpmResolverManagedSnapshotOption::Specified(Some(snapshot))
}
None => match cli_options.maybe_lockfile() {
Some(lockfile) => {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(
lockfile.clone(),
)
}
None => CliNpmResolverManagedSnapshotOption::Specified(None),
.get_or_try_init_async(
async {
let fs = self.fs();
let cli_options = self.cli_options()?;
create_cli_npm_resolver(if cli_options.use_byonm() {
CliNpmResolverCreateOptions::Byonm(
CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(fs.clone()),
pkg_json_resolver: self.pkg_json_resolver().clone(),
root_node_modules_dir: Some(
match cli_options.node_modules_dir_path() {
Some(node_modules_path) => node_modules_path.to_path_buf(),
// path needs to be canonicalized for node resolution
// (node_modules_dir_path above is already canonicalized)
None => canonicalize_path_maybe_not_exists(
cli_options.initial_cwd(),
)?
.join("node_modules"),
},
),
},
},
maybe_lockfile: cli_options.maybe_lockfile().cloned(),
fs: fs.clone(),
http_client_provider: self.http_client_provider().clone(),
npm_global_cache_dir: self.deno_dir()?.npm_folder_path(),
cache_setting: cli_options.cache_setting(),
text_only_progress_bar: self.text_only_progress_bar().clone(),
maybe_node_modules_path: cli_options.node_modules_dir_path().cloned(),
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::from_workspace(cli_options.workspace())),
npm_system_info: cli_options.npm_system_info(),
npmrc: cli_options.npmrc().clone(),
lifecycle_scripts: cli_options.lifecycle_scripts_config(),
)
} else {
CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
snapshot: match cli_options.resolve_npm_resolution_snapshot()? {
Some(snapshot) => {
CliNpmResolverManagedSnapshotOption::Specified(Some(
snapshot,
))
}
None => match cli_options.maybe_lockfile() {
Some(lockfile) => {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(
lockfile.clone(),
)
}
None => {
CliNpmResolverManagedSnapshotOption::Specified(None)
}
},
},
maybe_lockfile: cli_options.maybe_lockfile().cloned(),
fs: fs.clone(),
http_client_provider: self.http_client_provider().clone(),
npm_cache_dir: self.npm_cache_dir()?.clone(),
cache_setting: cli_options.cache_setting(),
text_only_progress_bar: self.text_only_progress_bar().clone(),
maybe_node_modules_path: cli_options
.node_modules_dir_path()
.cloned(),
npm_install_deps_provider: Arc::new(
NpmInstallDepsProvider::from_workspace(
cli_options.workspace(),
),
),
npm_system_info: cli_options.npm_system_info(),
npmrc: cli_options.npmrc().clone(),
lifecycle_scripts: cli_options.lifecycle_scripts_config(),
},
)
})
}).await
}.boxed_local())
.await
}
.boxed_local(),
)
.await
}
@ -505,6 +573,7 @@ impl CliFactory {
self.services.module_info_cache.get_or_try_init(|| {
Ok(Arc::new(ModuleInfoCache::new(
self.caches()?.dep_analysis_db(),
self.parsed_source_cache().clone(),
)))
})
}
@ -533,6 +602,7 @@ impl CliFactory {
ts_config_result.ts_config,
)?;
Ok(Arc::new(Emitter::new(
self.cjs_tracker()?.clone(),
self.emit_cache()?.clone(),
self.parsed_source_cache().clone(),
transpile_options,
@ -556,7 +626,9 @@ impl CliFactory {
async {
Ok(Arc::new(NodeResolver::new(
DenoFsNodeResolverEnv::new(self.fs().clone()),
self.in_npm_pkg_checker()?.clone(),
self.npm_resolver().await?.clone().into_npm_resolver(),
self.pkg_json_resolver().clone(),
)))
}
.boxed_local(),
@ -574,23 +646,35 @@ impl CliFactory {
let caches = self.caches()?;
let node_analysis_cache =
NodeAnalysisCache::new(caches.node_analysis_db());
let node_resolver = self.cli_node_resolver().await?.clone();
let node_resolver = self.node_resolver().await?.clone();
let cjs_esm_analyzer = CliCjsCodeAnalyzer::new(
node_analysis_cache,
self.cjs_tracker()?.clone(),
self.fs().clone(),
node_resolver,
Some(self.parsed_source_cache().clone()),
self.cli_options()?.is_npm_main(),
);
Ok(Arc::new(NodeCodeTranslator::new(
cjs_esm_analyzer,
DenoFsNodeResolverEnv::new(self.fs().clone()),
self.node_resolver().await?.clone(),
self.in_npm_pkg_checker()?.clone(),
node_resolver,
self.npm_resolver().await?.clone().into_npm_resolver(),
self.pkg_json_resolver().clone(),
)))
})
.await
}
pub fn pkg_json_resolver(&self) -> &Arc<PackageJsonResolver> {
self.services.pkg_json_resolver.get_or_init(|| {
Arc::new(PackageJsonResolver::new(DenoFsNodeResolverEnv::new(
self.fs().clone(),
)))
})
}
pub async fn type_checker(&self) -> Result<&Arc<TypeChecker>, AnyError> {
self
.services
@ -599,6 +683,10 @@ impl CliFactory {
let cli_options = self.cli_options()?;
Ok(Arc::new(TypeChecker::new(
self.caches()?.clone(),
Arc::new(TypeCheckingCjsTracker::new(
self.cjs_tracker()?.clone(),
self.module_info_cache()?.clone(),
)),
cli_options.clone(),
self.module_graph_builder().await?.clone(),
self.node_resolver().await?.clone(),
@ -617,17 +705,18 @@ impl CliFactory {
.get_or_try_init_async(async {
let cli_options = self.cli_options()?;
Ok(Arc::new(ModuleGraphBuilder::new(
cli_options.clone(),
self.caches()?.clone(),
cli_options.clone(),
self.file_fetcher()?.clone(),
self.fs().clone(),
self.resolver().await?.clone(),
self.npm_resolver().await?.clone(),
self.module_info_cache()?.clone(),
self.parsed_source_cache().clone(),
self.global_http_cache()?.clone(),
self.in_npm_pkg_checker()?.clone(),
cli_options.maybe_lockfile().cloned(),
self.maybe_file_watcher_reporter().clone(),
self.file_fetcher()?.clone(),
self.global_http_cache()?.clone(),
self.module_info_cache()?.clone(),
self.npm_resolver().await?.clone(),
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
self.root_permissions_container()?.clone(),
)))
})
@ -699,8 +788,17 @@ impl CliFactory {
.await
}
pub fn cjs_resolutions(&self) -> &Arc<CjsResolutionStore> {
self.services.cjs_resolutions.get_or_init(Default::default)
pub fn cjs_tracker(&self) -> Result<&Arc<CjsTracker>, AnyError> {
self.services.cjs_tracker.get_or_try_init(|| {
let options = self.cli_options()?;
Ok(Arc::new(CjsTracker::new(
self.in_npm_pkg_checker()?.clone(),
self.pkg_json_resolver().clone(),
CjsTrackerOptions {
unstable_detect_cjs: options.unstable_detect_cjs(),
},
)))
})
}
pub async fn cli_node_resolver(
@ -711,8 +809,9 @@ impl CliFactory {
.cli_node_resolver
.get_or_try_init_async(async {
Ok(Arc::new(CliNodeResolver::new(
self.cjs_resolutions().clone(),
self.cjs_tracker()?.clone(),
self.fs().clone(),
self.in_npm_pkg_checker()?.clone(),
self.node_resolver().await?.clone(),
self.npm_resolver().await?.clone(),
)))
@ -750,7 +849,9 @@ impl CliFactory {
) -> Result<DenoCompileBinaryWriter, AnyError> {
let cli_options = self.cli_options()?;
Ok(DenoCompileBinaryWriter::new(
self.cjs_tracker()?,
self.deno_dir()?,
self.emitter()?,
self.file_fetcher()?,
self.http_client_provider(),
self.npm_resolver().await?.as_ref(),
@ -779,16 +880,20 @@ impl CliFactory {
&self,
) -> Result<CliMainWorkerFactory, AnyError> {
let cli_options = self.cli_options()?;
let fs = self.fs();
let node_resolver = self.node_resolver().await?;
let npm_resolver = self.npm_resolver().await?;
let fs = self.fs();
let cli_node_resolver = self.cli_node_resolver().await?;
let cli_npm_resolver = self.npm_resolver().await?.clone();
let in_npm_pkg_checker = self.in_npm_pkg_checker()?;
let maybe_file_watcher_communicator = if cli_options.has_hmr() {
Some(self.watcher_communicator.clone().unwrap())
} else {
None
};
let node_code_translator = self.node_code_translator().await?;
let cjs_tracker = self.cjs_tracker()?.clone();
let pkg_json_resolver = self.pkg_json_resolver().clone();
Ok(CliMainWorkerFactory::new(
self.blob_store().clone(),
@ -798,33 +903,37 @@ impl CliFactory {
None
},
self.feature_checker()?.clone(),
self.fs().clone(),
fs.clone(),
maybe_file_watcher_communicator,
self.maybe_inspector_server()?.clone(),
cli_options.maybe_lockfile().cloned(),
Box::new(CliModuleLoaderFactory::new(
cli_options,
cjs_tracker,
if cli_options.code_cache_enabled() {
Some(self.code_cache()?.clone())
} else {
None
},
self.emitter()?.clone(),
fs.clone(),
in_npm_pkg_checker.clone(),
self.main_module_graph_container().await?.clone(),
self.module_load_preparer().await?.clone(),
node_code_translator.clone(),
cli_node_resolver.clone(),
cli_npm_resolver.clone(),
NpmModuleLoader::new(
self.cjs_resolutions().clone(),
self.node_code_translator().await?.clone(),
self.cjs_tracker()?.clone(),
fs.clone(),
cli_node_resolver.clone(),
node_code_translator.clone(),
),
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
)),
node_resolver.clone(),
npm_resolver.clone(),
pkg_json_resolver,
self.root_cert_store_provider().clone(),
self.root_permissions_container()?.clone(),
StorageKeyResolver::from_options(cli_options),
@ -840,8 +949,10 @@ impl CliFactory {
let create_hmr_runner = if cli_options.has_hmr() {
let watcher_communicator = self.watcher_communicator.clone().unwrap();
let emitter = self.emitter()?.clone();
let cjs_tracker = self.cjs_tracker()?.clone();
let fn_: crate::worker::CreateHmrRunnerCb = Box::new(move |session| {
Box::new(HmrRunner::new(
cjs_tracker.clone(),
emitter.clone(),
session,
watcher_communicator.clone(),
@ -878,7 +989,6 @@ impl CliFactory {
inspect_wait: cli_options.inspect_wait().is_some(),
strace_ops: cli_options.strace_ops().clone(),
is_inspecting: cli_options.is_inspecting(),
is_npm_main: cli_options.is_npm_main(),
location: cli_options.location_flag().clone(),
// if the user ran a binary command, we'll need to set process.argv[0]
// to be the name of the binary command instead of deno

32
cli/file_fetcher.rs
View file

@ -24,6 +24,7 @@ use deno_graph::source::LoaderChecksum;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_web::BlobStore;
use http::header;
use log::debug;
use std::borrow::Cow;
use std::collections::HashMap;
@ -181,6 +182,7 @@ pub enum FetchPermissionsOptionRef<'a> {
pub struct FetchOptions<'a> {
pub specifier: &'a ModuleSpecifier,
pub permissions: FetchPermissionsOptionRef<'a>,
pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>,
pub maybe_accept: Option<&'a str>,
pub maybe_cache_setting: Option<&'a CacheSetting>,
}
@ -333,7 +335,7 @@ impl FileFetcher {
)
})?;
let bytes = blob.read_all().await?;
let bytes = blob.read_all().await;
let headers =
HashMap::from([("content-type".to_string(), blob.media_type.clone())]);
@ -350,6 +352,7 @@ impl FileFetcher {
maybe_accept: Option<&str>,
cache_setting: &CacheSetting,
maybe_checksum: Option<&LoaderChecksum>,
maybe_auth: Option<(header::HeaderName, header::HeaderValue)>,
) -> Result<FileOrRedirect, AnyError> {
debug!(
"FileFetcher::fetch_remote_no_follow - specifier: {}",
@ -442,6 +445,7 @@ impl FileFetcher {
.as_ref()
.map(|(_, etag)| etag.clone()),
maybe_auth_token: maybe_auth_token.clone(),
maybe_auth: maybe_auth.clone(),
maybe_progress_guard: maybe_progress_guard.as_ref(),
})
.await?
@ -538,7 +542,18 @@ impl FileFetcher {
specifier: &ModuleSpecifier,
) -> Result<File, AnyError> {
self
.fetch_inner(specifier, FetchPermissionsOptionRef::AllowAll)
.fetch_inner(specifier, None, FetchPermissionsOptionRef::AllowAll)
.await
}
#[inline(always)]
pub async fn fetch_bypass_permissions_with_maybe_auth(
&self,
specifier: &ModuleSpecifier,
maybe_auth: Option<(header::HeaderName, header::HeaderValue)>,
) -> Result<File, AnyError> {
self
.fetch_inner(specifier, maybe_auth, FetchPermissionsOptionRef::AllowAll)
.await
}
@ -552,6 +567,7 @@ impl FileFetcher {
self
.fetch_inner(
specifier,
None,
FetchPermissionsOptionRef::StaticContainer(permissions),
)
.await
@ -560,12 +576,14 @@ impl FileFetcher {
async fn fetch_inner(
&self,
specifier: &ModuleSpecifier,
maybe_auth: Option<(header::HeaderName, header::HeaderValue)>,
permissions: FetchPermissionsOptionRef<'_>,
) -> Result<File, AnyError> {
self
.fetch_with_options(FetchOptions {
specifier,
permissions,
maybe_auth,
maybe_accept: None,
maybe_cache_setting: None,
})
@ -585,12 +603,14 @@ impl FileFetcher {
max_redirect: usize,
) -> Result<File, AnyError> {
let mut specifier = Cow::Borrowed(options.specifier);
let mut maybe_auth = options.maybe_auth.clone();
for _ in 0..=max_redirect {
match self
.fetch_no_follow_with_options(FetchNoFollowOptions {
fetch_options: FetchOptions {
specifier: &specifier,
permissions: options.permissions,
maybe_auth: maybe_auth.clone(),
maybe_accept: options.maybe_accept,
maybe_cache_setting: options.maybe_cache_setting,
},
@ -602,6 +622,10 @@ impl FileFetcher {
return Ok(file);
}
FileOrRedirect::Redirect(redirect_specifier) => {
// If we were redirected to another origin, don't send the auth header anymore.
if redirect_specifier.origin() != specifier.origin() {
maybe_auth = None;
}
specifier = Cow::Owned(redirect_specifier);
}
}
@ -666,6 +690,7 @@ impl FileFetcher {
options.maybe_accept,
options.maybe_cache_setting.unwrap_or(&self.cache_setting),
maybe_checksum,
options.maybe_auth,
)
.await
}
@ -756,6 +781,7 @@ mod tests {
FetchOptions {
specifier,
permissions: FetchPermissionsOptionRef::AllowAll,
maybe_auth: None,
maybe_accept: None,
maybe_cache_setting: Some(&file_fetcher.cache_setting),
},
@ -1255,6 +1281,7 @@ mod tests {
FetchOptions {
specifier: &specifier,
permissions: FetchPermissionsOptionRef::AllowAll,
maybe_auth: None,
maybe_accept: None,
maybe_cache_setting: Some(&file_fetcher.cache_setting),
},
@ -1268,6 +1295,7 @@ mod tests {
FetchOptions {
specifier: &specifier,
permissions: FetchPermissionsOptionRef::AllowAll,
maybe_auth: None,
maybe_accept: None,
maybe_cache_setting: Some(&file_fetcher.cache_setting),
},

75
cli/graph_util.rs
View file

@ -48,6 +48,7 @@ use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use import_map::ImportMapError;
use node_resolver::InNpmPackageChecker;
use std::collections::HashSet;
use std::error::Error;
use std::ops::Deref;
@ -377,48 +378,51 @@ pub struct BuildFastCheckGraphOptions<'a> {
}
pub struct ModuleGraphBuilder {
options: Arc<CliOptions>,
caches: Arc<cache::Caches>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<FileFetcher>,
fs: Arc<dyn FileSystem>,
resolver: Arc<CliGraphResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
module_info_cache: Arc<ModuleInfoCache>,
parsed_source_cache: Arc<ParsedSourceCache>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
lockfile: Option<Arc<CliLockfile>>,
maybe_file_watcher_reporter: Option<FileWatcherReporter>,
file_fetcher: Arc<FileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
module_info_cache: Arc<ModuleInfoCache>,
npm_resolver: Arc<dyn CliNpmResolver>,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliGraphResolver>,
root_permissions_container: PermissionsContainer,
}
impl ModuleGraphBuilder {
#[allow(clippy::too_many_arguments)]
pub fn new(
options: Arc<CliOptions>,
caches: Arc<cache::Caches>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<FileFetcher>,
fs: Arc<dyn FileSystem>,
resolver: Arc<CliGraphResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
module_info_cache: Arc<ModuleInfoCache>,
parsed_source_cache: Arc<ParsedSourceCache>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
lockfile: Option<Arc<CliLockfile>>,
maybe_file_watcher_reporter: Option<FileWatcherReporter>,
file_fetcher: Arc<FileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
module_info_cache: Arc<ModuleInfoCache>,
npm_resolver: Arc<dyn CliNpmResolver>,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliGraphResolver>,
root_permissions_container: PermissionsContainer,
) -> Self {
Self {
options,
caches,
cli_options,
file_fetcher,
fs,
resolver,
npm_resolver,
module_info_cache,
parsed_source_cache,
global_http_cache,
in_npm_pkg_checker,
lockfile,
maybe_file_watcher_reporter,
file_fetcher,
global_http_cache,
module_info_cache,
npm_resolver,
parsed_source_cache,
resolver,
root_permissions_container,
}
}
@ -504,13 +508,11 @@ impl ModuleGraphBuilder {
}
let maybe_imports = if options.graph_kind.include_types() {
self.options.to_compiler_option_types()?
self.cli_options.to_compiler_option_types()?
} else {
Vec::new()
};
let analyzer = self
.module_info_cache
.as_module_analyzer(&self.parsed_source_cache);
let analyzer = self.module_info_cache.as_module_analyzer();
let mut loader = match options.loader {
Some(loader) => MutLoaderRef::Borrowed(loader),
None => MutLoaderRef::Owned(self.create_graph_loader()),
@ -558,7 +560,7 @@ impl ModuleGraphBuilder {
// ensure an "npm install" is done if the user has explicitly
// opted into using a node_modules directory
if self
.options
.cli_options
.node_modules_dir()?
.map(|m| m.uses_node_modules_dir())
.unwrap_or(false)
@ -669,10 +671,10 @@ impl ModuleGraphBuilder {
graph.build_fast_check_type_graph(
deno_graph::BuildFastCheckTypeGraphOptions {
jsr_url_provider: &CliJsrUrlProvider,
es_parser: Some(&parser),
fast_check_cache: fast_check_cache.as_ref().map(|c| c as _),
fast_check_dts: false,
module_parser: Some(&parser),
jsr_url_provider: &CliJsrUrlProvider,
resolver: Some(graph_resolver),
npm_resolver: Some(&graph_npm_resolver),
workspace_fast_check: options.workspace_fast_check,
@ -692,14 +694,15 @@ impl ModuleGraphBuilder {
) -> cache::FetchCacher {
cache::FetchCacher::new(
self.file_fetcher.clone(),
self.fs.clone(),
self.global_http_cache.clone(),
self.npm_resolver.clone(),
self.in_npm_pkg_checker.clone(),
self.module_info_cache.clone(),
cache::FetchCacherOptions {
file_header_overrides: self.options.resolve_file_header_overrides(),
file_header_overrides: self.cli_options.resolve_file_header_overrides(),
permissions,
is_deno_publish: matches!(
self.options.sub_command(),
self.cli_options.sub_command(),
crate::args::DenoSubcommand::Publish { .. }
),
},
@ -726,12 +729,12 @@ impl ModuleGraphBuilder {
&self.fs,
roots,
GraphValidOptions {
kind: if self.options.type_check_mode().is_true() {
kind: if self.cli_options.type_check_mode().is_true() {
GraphKind::All
} else {
GraphKind::CodeOnly
},
check_js: self.options.check_js(),
check_js: self.cli_options.check_js(),
exit_integrity_errors: true,
},
)
@ -998,7 +1001,11 @@ impl deno_graph::source::Reporter for FileWatcherReporter {
) {
let mut file_paths = self.file_paths.lock();
if specifier.scheme() == "file" {
file_paths.push(specifier.to_file_path().unwrap());
// Don't trust that the path is a valid path at this point:
// https://github.com/denoland/deno/issues/26209.
if let Ok(file_path) = specifier.to_file_path() {
file_paths.push(file_path);
}
}
if modules_done == modules_total {

37
cli/http_util.rs
View file

@ -19,6 +19,7 @@ use deno_runtime::deno_fetch;
use deno_runtime::deno_fetch::create_http_client;
use deno_runtime::deno_fetch::CreateHttpClientOptions;
use deno_runtime::deno_tls::RootCertStoreProvider;
use http::header;
use http::header::HeaderName;
use http::header::HeaderValue;
use http::header::ACCEPT;
@ -204,6 +205,7 @@ pub struct FetchOnceArgs<'a> {
pub maybe_accept: Option<String>,
pub maybe_etag: Option<String>,
pub maybe_auth_token: Option<AuthToken>,
pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>,
pub maybe_progress_guard: Option<&'a UpdateGuard>,
}
@ -382,6 +384,8 @@ impl HttpClient {
request
.headers_mut()
.insert(AUTHORIZATION, authorization_val);
} else if let Some((header, value)) = args.maybe_auth {
request.headers_mut().insert(header, value);
}
if let Some(accept) = args.maybe_accept {
let accepts_val = HeaderValue::from_str(&accept)?;
@ -470,15 +474,23 @@ impl HttpClient {
}
}
pub async fn download_with_progress(
pub async fn download_with_progress_and_retries(
&self,
url: Url,
maybe_header: Option<(HeaderName, HeaderValue)>,
progress_guard: &UpdateGuard,
) -> Result<Option<Vec<u8>>, DownloadError> {
self
.download_inner(url, maybe_header, Some(progress_guard))
.await
crate::util::retry::retry(
|| {
self.download_inner(
url.clone(),
maybe_header.clone(),
Some(progress_guard),
)
},
|e| matches!(e, DownloadError::BadResponse(_) | DownloadError::Fetch(_)),
)
.await
}
pub async fn get_redirected_url(
@ -784,6 +796,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result {
@ -810,6 +823,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result {
@ -837,6 +851,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result {
@ -858,6 +873,7 @@ mod test {
maybe_etag: Some("33a64df551425fcc55e".to_string()),
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
@ -877,6 +893,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result {
@ -906,6 +923,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, _)) = result {
@ -931,6 +949,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Redirect(url, _)) = result {
@ -966,6 +985,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result {
@ -1013,6 +1033,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
@ -1075,6 +1096,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
@ -1128,6 +1150,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result {
@ -1169,6 +1192,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result {
@ -1191,6 +1215,7 @@ mod test {
maybe_etag: Some("33a64df551425fcc55e".to_string()),
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
@ -1225,6 +1250,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result {
@ -1254,6 +1280,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
assert!(result.is_err());
@ -1275,6 +1302,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;
@ -1298,6 +1326,7 @@ mod test {
maybe_etag: None,
maybe_auth_token: None,
maybe_progress_guard: None,
maybe_auth: None,
})
.await;

124
cli/lsp/analysis.rs
View file

@ -18,7 +18,6 @@ use deno_lint::diagnostic::LintDiagnosticRange;
use deno_ast::SourceRange;
use deno_ast::SourceRangedForSpanned;
use deno_ast::SourceTextInfo;
use deno_core::anyhow::anyhow;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::serde::Deserialize;
@ -37,9 +36,9 @@ use deno_semver::package::PackageReq;
use deno_semver::package::PackageReqReference;
use deno_semver::Version;
use import_map::ImportMap;
use node_resolver::NpmResolver;
use once_cell::sync::Lazy;
use regex::Regex;
use std::borrow::Cow;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::collections::HashSet;
@ -336,7 +335,12 @@ impl<'a> TsResponseImportMapper<'a> {
.resolver
.maybe_managed_npm_resolver(Some(&self.file_referrer))
{
if npm_resolver.in_npm_package(specifier) {
let in_npm_pkg = self
.resolver
.maybe_node_resolver(Some(&self.file_referrer))
.map(|n| n.in_npm_package(specifier))
.unwrap_or(false);
if in_npm_pkg {
if let Ok(Some(pkg_id)) =
npm_resolver.resolve_pkg_id_from_specifier(specifier)
{
@ -598,68 +602,62 @@ pub fn fix_ts_import_changes(
/// Fix tsc import code actions so that the module specifier is correct for
/// resolution by Deno (includes the extension).
fn fix_ts_import_action(
fn fix_ts_import_action<'a>(
referrer: &ModuleSpecifier,
action: &tsc::CodeFixAction,
action: &'a tsc::CodeFixAction,
import_mapper: &TsResponseImportMapper,
) -> Result<Option<tsc::CodeFixAction>, AnyError> {
if matches!(
) -> Option<Cow<'a, tsc::CodeFixAction>> {
if !matches!(
action.fix_name.as_str(),
"import" | "fixMissingFunctionDeclaration"
) {
let change = action
return Some(Cow::Borrowed(action));
}
let specifier = (|| {
let text_change = action.changes.first()?.text_changes.first()?;
let captures = IMPORT_SPECIFIER_RE.captures(&text_change.new_text)?;
Some(captures.get(1)?.as_str())
})();
let Some(specifier) = specifier else {
return Some(Cow::Borrowed(action));
};
if let Some(new_specifier) =
import_mapper.check_unresolved_specifier(specifier, referrer)
{
let description = action.description.replace(specifier, &new_specifier);
let changes = action
.changes
.first()
.ok_or_else(|| anyhow!("Unexpected action changes."))?;
let text_change = change
.text_changes
.first()
.ok_or_else(|| anyhow!("Missing text change."))?;
if let Some(captures) = IMPORT_SPECIFIER_RE.captures(&text_change.new_text)
{
let specifier = captures
.get(1)
.ok_or_else(|| anyhow!("Missing capture."))?
.as_str();
if let Some(new_specifier) =
import_mapper.check_unresolved_specifier(specifier, referrer)
{
let description = action.description.replace(specifier, &new_specifier);
let changes = action
.changes
.iter()
.map(|c| {
let text_changes = c
.text_changes
.iter()
.map(|c| {
let text_changes = c
.text_changes
.iter()
.map(|tc| tsc::TextChange {
span: tc.span.clone(),
new_text: tc.new_text.replace(specifier, &new_specifier),
})
.collect();
tsc::FileTextChanges {
file_name: c.file_name.clone(),
text_changes,
is_new_file: c.is_new_file,
}
.map(|tc| tsc::TextChange {
span: tc.span.clone(),
new_text: tc.new_text.replace(specifier, &new_specifier),
})
.collect();
tsc::FileTextChanges {
file_name: c.file_name.clone(),
text_changes,
is_new_file: c.is_new_file,
}
})
.collect();
return Ok(Some(tsc::CodeFixAction {
description,
changes,
commands: None,
fix_name: action.fix_name.clone(),
fix_id: None,
fix_all_description: None,
}));
} else if !import_mapper.is_valid_import(specifier, referrer) {
return Ok(None);
}
}
Some(Cow::Owned(tsc::CodeFixAction {
description,
changes,
commands: None,
fix_name: action.fix_name.clone(),
fix_id: None,
fix_all_description: None,
}))
} else if !import_mapper.is_valid_import(specifier, referrer) {
None
} else {
Some(Cow::Borrowed(action))
}
Ok(Some(action.clone()))
}
/// Determines if two TypeScript diagnostic codes are effectively equivalent.
@ -1004,8 +1002,7 @@ impl CodeActionCollection {
specifier,
action,
&language_server.get_ts_response_import_mapper(specifier),
)?
else {
) else {
return Ok(());
};
let edit = ts_changes_to_edit(&action.changes, language_server)?;
@ -1027,7 +1024,7 @@ impl CodeActionCollection {
});
self
.actions
.push(CodeActionKind::Tsc(code_action, action.clone()));
.push(CodeActionKind::Tsc(code_action, action.as_ref().clone()));
if let Some(fix_id) = &action.fix_id {
if let Some(CodeActionKind::Tsc(existing_fix_all, existing_action)) =
@ -1206,14 +1203,11 @@ impl CodeActionCollection {
}),
);
match parsed_source.program_ref() {
deno_ast::swc::ast::Program::Module(module) => module
.body
.iter()
.find(|i| i.range().contains(&specifier_range))
.map(|i| text_info.line_and_column_index(i.range().start)),
deno_ast::swc::ast::Program::Script(_) => None,
}
parsed_source
.program_ref()
.body()
.find(|i| i.range().contains(&specifier_range))
.map(|i| text_info.line_and_column_index(i.range().start))
}
async fn deno_types_for_npm_action(

4
cli/lsp/capabilities.rs
View file

@ -147,11 +147,11 @@ pub fn server_capabilities(
moniker_provider: None,
experimental: Some(json!({
"denoConfigTasks": true,
"testingApi":true,
"testingApi": true,
"didRefreshDenoConfigurationTreeNotifications": true,
})),
inlay_hint_provider: Some(OneOf::Left(true)),
position_encoding: None,
// TODO(nayeemrmn): Support pull-based diagnostics.
diagnostic_provider: None,
inline_value_provider: None,
inline_completion_provider: None,

35
cli/lsp/client.rs
View file

@ -92,6 +92,19 @@ impl Client {
});
}
pub fn send_did_refresh_deno_configuration_tree_notification(
&self,
params: lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams,
) {
// do on a task in case the caller currently is in the lsp lock
let client = self.0.clone();
spawn(async move {
client
.send_did_refresh_deno_configuration_tree_notification(params)
.await;
});
}
pub fn send_did_change_deno_configuration_notification(
&self,
params: lsp_custom::DidChangeDenoConfigurationNotificationParams,
@ -169,6 +182,10 @@ trait ClientTrait: Send + Sync {
params: lsp_custom::DiagnosticBatchNotificationParams,
);
async fn send_test_notification(&self, params: TestingNotification);
async fn send_did_refresh_deno_configuration_tree_notification(
&self,
params: lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams,
);
async fn send_did_change_deno_configuration_notification(
&self,
params: lsp_custom::DidChangeDenoConfigurationNotificationParams,
@ -249,6 +266,18 @@ impl ClientTrait for TowerClient {
}
}
async fn send_did_refresh_deno_configuration_tree_notification(
&self,
params: lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams,
) {
self
.0
.send_notification::<lsp_custom::DidRefreshDenoConfigurationTreeNotification>(
params,
)
.await
}
async fn send_did_change_deno_configuration_notification(
&self,
params: lsp_custom::DidChangeDenoConfigurationNotificationParams,
@ -366,6 +395,12 @@ impl ClientTrait for ReplClient {
async fn send_test_notification(&self, _params: TestingNotification) {}
async fn send_did_refresh_deno_configuration_tree_notification(
&self,
_params: lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams,
) {
}
async fn send_did_change_deno_configuration_notification(
&self,
_params: lsp_custom::DidChangeDenoConfigurationNotificationParams,

4
cli/lsp/code_lens.rs
View file

@ -421,7 +421,7 @@ pub fn collect_test(
) -> Result<Vec<lsp::CodeLens>, AnyError> {
let mut collector =
DenoTestCollector::new(specifier.clone(), parsed_source.clone());
parsed_source.module().visit_with(&mut collector);
parsed_source.program().visit_with(&mut collector);
Ok(collector.take())
}
@ -581,7 +581,7 @@ mod tests {
.unwrap();
let mut collector =
DenoTestCollector::new(specifier, parsed_module.clone());
parsed_module.module().visit_with(&mut collector);
parsed_module.program().visit_with(&mut collector);
assert_eq!(
collector.take(),
vec![

130
cli/lsp/completions.rs
View file

@ -200,15 +200,11 @@ pub async fn get_import_completions(
{
// completions for import map specifiers
Some(lsp::CompletionResponse::List(completion_list))
} else if text.starts_with("./")
|| text.starts_with("../")
|| text.starts_with('/')
} else if let Some(completion_list) =
get_local_completions(specifier, &text, &range, resolver)
{
// completions for local relative modules
Some(lsp::CompletionResponse::List(CompletionList {
is_incomplete: false,
items: get_local_completions(specifier, &text, &range, resolver)?,
}))
Some(lsp::CompletionResponse::List(completion_list))
} else if !text.is_empty() {
// completion of modules from a module registry or cache
check_auto_config_registry(
@ -363,15 +359,15 @@ fn get_local_completions(
text: &str,
range: &lsp::Range,
resolver: &LspResolver,
) -> Option<Vec<lsp::CompletionItem>> {
) -> Option<CompletionList> {
if base.scheme() != "file" {
return None;
}
let parent = base.join(text).ok()?.join(".").ok()?;
let parent = &text[..text.char_indices().rfind(|(_, c)| *c == '/')?.0 + 1];
let resolved_parent = resolver
.as_graph_resolver(Some(base))
.resolve(
parent.as_str(),
parent,
&Range {
specifier: base.clone(),
start: deno_graph::Position::zeroed(),
@ -381,62 +377,62 @@ fn get_local_completions(
)
.ok()?;
let resolved_parent_path = url_to_file_path(&resolved_parent).ok()?;
let raw_parent =
&text[..text.char_indices().rfind(|(_, c)| *c == '/')?.0 + 1];
if resolved_parent_path.is_dir() {
let cwd = std::env::current_dir().ok()?;
let items = std::fs::read_dir(resolved_parent_path).ok()?;
Some(
items
.filter_map(|de| {
let de = de.ok()?;
let label = de.path().file_name()?.to_string_lossy().to_string();
let entry_specifier = resolve_path(de.path().to_str()?, &cwd).ok()?;
if entry_specifier == *base {
return None;
}
let full_text = format!("{raw_parent}{label}");
let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
range: *range,
new_text: full_text.clone(),
}));
let filter_text = Some(full_text);
match de.file_type() {
Ok(file_type) if file_type.is_dir() => Some(lsp::CompletionItem {
label,
kind: Some(lsp::CompletionItemKind::FOLDER),
detail: Some("(local)".to_string()),
filter_text,
sort_text: Some("1".to_string()),
text_edit,
commit_characters: Some(
IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(),
),
..Default::default()
}),
Ok(file_type) if file_type.is_file() => {
if is_importable_ext(&de.path()) {
Some(lsp::CompletionItem {
label,
kind: Some(lsp::CompletionItemKind::FILE),
detail: Some("(local)".to_string()),
filter_text,
sort_text: Some("1".to_string()),
text_edit,
commit_characters: Some(
IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(),
),
..Default::default()
})
} else {
None
}
let entries = std::fs::read_dir(resolved_parent_path).ok()?;
let items = entries
.filter_map(|de| {
let de = de.ok()?;
let label = de.path().file_name()?.to_string_lossy().to_string();
let entry_specifier = resolve_path(de.path().to_str()?, &cwd).ok()?;
if entry_specifier == *base {
return None;
}
let full_text = format!("{parent}{label}");
let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
range: *range,
new_text: full_text.clone(),
}));
let filter_text = Some(full_text);
match de.file_type() {
Ok(file_type) if file_type.is_dir() => Some(lsp::CompletionItem {
label,
kind: Some(lsp::CompletionItemKind::FOLDER),
detail: Some("(local)".to_string()),
filter_text,
sort_text: Some("1".to_string()),
text_edit,
commit_characters: Some(
IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(),
),
..Default::default()
}),
Ok(file_type) if file_type.is_file() => {
if is_importable_ext(&de.path()) {
Some(lsp::CompletionItem {
label,
kind: Some(lsp::CompletionItemKind::FILE),
detail: Some("(local)".to_string()),
filter_text,
sort_text: Some("1".to_string()),
text_edit,
commit_characters: Some(
IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(),
),
..Default::default()
})
} else {
None
}
_ => None,
}
})
.collect(),
)
_ => None,
}
})
.collect();
Some(CompletionList {
is_incomplete: false,
items,
})
} else {
None
}
@ -921,11 +917,11 @@ mod tests {
},
},
&Default::default(),
);
assert!(actual.is_some());
let actual = actual.unwrap();
assert_eq!(actual.len(), 3);
for item in actual {
)
.unwrap();
assert!(!actual.is_incomplete);
assert_eq!(actual.items.len(), 3);
for item in actual.items {
match item.text_edit {
Some(lsp::CompletionTextEdit::Edit(text_edit)) => {
assert!(["./b", "./f.mjs", "./g.json"]

67
cli/lsp/config.rs
View file

@ -41,6 +41,7 @@ use deno_runtime::deno_node::PackageJson;
use indexmap::IndexSet;
use lsp_types::ClientCapabilities;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::collections::HashMap;
use std::ops::Deref;
use std::ops::DerefMut;
@ -50,6 +51,8 @@ use std::sync::Arc;
use tower_lsp::lsp_types as lsp;
use super::logging::lsp_log;
use super::lsp_custom;
use super::urls::url_to_uri;
use crate::args::discover_npmrc_from_workspace;
use crate::args::has_flag_env_var;
use crate::args::CliLockfile;
@ -437,6 +440,8 @@ pub struct LanguagePreferences {
pub use_aliases_for_renames: bool,
#[serde(default)]
pub quote_style: QuoteStyle,
#[serde(default)]
pub prefer_type_only_auto_imports: bool,
}
impl Default for LanguagePreferences {
@ -447,6 +452,7 @@ impl Default for LanguagePreferences {
auto_import_file_exclude_patterns: vec![],
use_aliases_for_renames: true,
quote_style: Default::default(),
prefer_type_only_auto_imports: false,
}
}
}
@ -979,7 +985,7 @@ impl Config {
| MediaType::Tsx => Some(&workspace_settings.typescript),
MediaType::Json
| MediaType::Wasm
| MediaType::TsBuildInfo
| MediaType::Css
| MediaType::SourceMap
| MediaType::Unknown => None,
}
@ -1185,6 +1191,7 @@ pub struct ConfigData {
pub resolver: Arc<WorkspaceResolver>,
pub sloppy_imports_resolver: Option<Arc<CliSloppyImportsResolver>>,
pub import_map_from_settings: Option<ModuleSpecifier>,
pub unstable: BTreeSet<String>,
watched_files: HashMap<ModuleSpecifier, ConfigWatchedFileType>,
}
@ -1582,9 +1589,16 @@ impl ConfigData {
.join("\n")
);
}
let unstable = member_dir
.workspace
.unstable_features()
.iter()
.chain(settings.unstable.as_deref())
.cloned()
.collect::<BTreeSet<_>>();
let unstable_sloppy_imports = std::env::var("DENO_UNSTABLE_SLOPPY_IMPORTS")
.is_ok()
|| member_dir.workspace.has_unstable("sloppy-imports");
|| unstable.contains("sloppy-imports");
let sloppy_imports_resolver = unstable_sloppy_imports.then(|| {
Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new_without_stat_cache(Arc::new(
@ -1625,6 +1639,7 @@ impl ConfigData {
lockfile,
npmrc,
import_map_from_settings,
unstable,
watched_files,
}
}
@ -1716,14 +1731,14 @@ impl ConfigTree {
.unwrap_or_else(|| Arc::new(FmtConfig::new_with_base(PathBuf::from("/"))))
}
/// Returns (scope_uri, type).
/// Returns (scope_url, type).
pub fn watched_file_type(
&self,
specifier: &ModuleSpecifier,
) -> Option<(&ModuleSpecifier, ConfigWatchedFileType)> {
for (scope_uri, data) in self.scopes.iter() {
for (scope_url, data) in self.scopes.iter() {
if let Some(typ) = data.watched_files.get(specifier) {
return Some((scope_uri, *typ));
return Some((scope_url, *typ));
}
}
None
@ -1747,6 +1762,46 @@ impl ConfigTree {
.any(|data| data.watched_files.contains_key(specifier))
}
pub fn to_did_refresh_params(
&self,
) -> lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams {
let data = self
.scopes
.values()
.filter_map(|data| {
let workspace_root_scope_uri =
Some(data.member_dir.workspace.root_dir())
.filter(|s| *s != data.member_dir.dir_url())
.and_then(|s| url_to_uri(s).ok());
Some(lsp_custom::DenoConfigurationData {
scope_uri: url_to_uri(&data.scope).ok()?,
deno_json: data.maybe_deno_json().and_then(|c| {
if workspace_root_scope_uri.is_some()
&& Some(&c.specifier)
== data
.member_dir
.workspace
.root_deno_json()
.map(|c| &c.specifier)
{
return None;
}
Some(lsp::TextDocumentIdentifier {
uri: url_to_uri(&c.specifier).ok()?,
})
}),
package_json: data.maybe_pkg_json().and_then(|p| {
Some(lsp::TextDocumentIdentifier {
uri: url_to_uri(&p.specifier()).ok()?,
})
}),
workspace_root_scope_uri,
})
})
.collect();
lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams { data }
}
pub async fn refresh(
&mut self,
settings: &Settings,
@ -2209,6 +2264,7 @@ mod tests {
auto_import_file_exclude_patterns: vec![],
use_aliases_for_renames: true,
quote_style: QuoteStyle::Auto,
prefer_type_only_auto_imports: false,
},
suggest: CompletionSettings {
complete_function_calls: false,
@ -2254,6 +2310,7 @@ mod tests {
auto_import_file_exclude_patterns: vec![],
use_aliases_for_renames: true,
quote_style: QuoteStyle::Auto,
prefer_type_only_auto_imports: false,
},
suggest: CompletionSettings {
complete_function_calls: false,

6
cli/lsp/diagnostics.rs
View file

@ -1499,7 +1499,11 @@ fn diagnose_dependency(
.data_for_specifier(referrer_doc.file_referrer().unwrap_or(referrer))
.and_then(|d| d.resolver.maybe_import_map());
if let Some(import_map) = import_map {
if let Resolution::Ok(resolved) = &dependency.maybe_code {
let resolved = dependency
.maybe_code
.ok()
.or_else(|| dependency.maybe_type.ok());
if let Some(resolved) = resolved {
if let Some(to) = import_map.lookup(&resolved.specifier, referrer) {
if dependency_key != to {
diagnostics.push(

20
cli/lsp/documents.rs
View file

@ -272,7 +272,7 @@ fn get_maybe_test_module_fut(
parsed_source.specifier().clone(),
parsed_source.text_info_lazy().clone(),
);
parsed_source.module().visit_with(&mut collector);
parsed_source.program().visit_with(&mut collector);
Arc::new(collector.take())
})
.map(Result::ok)
@ -332,12 +332,8 @@ impl Document {
.filter(|s| cache.is_valid_file_referrer(s))
.cloned()
.or(file_referrer);
let media_type = resolve_media_type(
&specifier,
maybe_headers.as_ref(),
maybe_language_id,
&resolver,
);
let media_type =
resolve_media_type(&specifier, maybe_headers.as_ref(), maybe_language_id);
let (maybe_parsed_source, maybe_module) =
if media_type_is_diagnosable(media_type) {
parse_and_analyze_module(
@ -399,7 +395,6 @@ impl Document {
&self.specifier,
self.maybe_headers.as_ref(),
self.maybe_language_id,
&resolver,
);
let dependencies;
let maybe_types_dependency;
@ -764,14 +759,7 @@ fn resolve_media_type(
specifier: &ModuleSpecifier,
maybe_headers: Option<&HashMap<String, String>>,
maybe_language_id: Option<LanguageId>,
resolver: &LspResolver,
) -> MediaType {
if resolver.in_node_modules(specifier) {
if let Some(media_type) = resolver.node_media_type(specifier) {
return media_type;
}
}
if let Some(language_id) = maybe_language_id {
return MediaType::from_specifier_and_content_type(
specifier,
@ -1561,7 +1549,7 @@ fn parse_source(
text: Arc<str>,
media_type: MediaType,
) -> ParsedSourceResult {
deno_ast::parse_module(deno_ast::ParseParams {
deno_ast::parse_program(deno_ast::ParseParams {
specifier,
text,
media_type,

19
cli/lsp/language_server.rs
View file

@ -904,7 +904,7 @@ impl Inner {
| MediaType::Tsx => {}
MediaType::Wasm
| MediaType::SourceMap
| MediaType::TsBuildInfo
| MediaType::Css
| MediaType::Unknown => {
if path.extension().and_then(|s| s.to_str()) != Some("jsonc") {
continue;
@ -963,6 +963,11 @@ impl Inner {
.tree
.refresh(&self.config.settings, &self.workspace_files, &file_fetcher)
.await;
self
.client
.send_did_refresh_deno_configuration_tree_notification(
self.config.tree.to_did_refresh_params(),
);
for config_file in self.config.tree.config_files() {
(|| {
let compiler_options = config_file.to_compiler_options().ok()?.options;
@ -1379,14 +1384,10 @@ impl Inner {
.clone();
fmt_options.use_tabs = Some(!params.options.insert_spaces);
fmt_options.indent_width = Some(params.options.tab_size as u8);
let maybe_workspace = self
.config
.tree
.data_for_specifier(&specifier)
.map(|d| &d.member_dir.workspace);
let config_data = self.config.tree.data_for_specifier(&specifier);
let unstable_options = UnstableFmtOptions {
component: maybe_workspace
.map(|w| w.has_unstable("fmt-component"))
component: config_data
.map(|d| d.unstable.contains("fmt-component"))
.unwrap_or(false),
};
let document = document.clone();
@ -3807,7 +3808,7 @@ impl Inner {
let maybe_inlay_hints = maybe_inlay_hints.map(|hints| {
hints
.iter()
.map(|hint| hint.to_lsp(line_index.clone()))
.map(|hint| hint.to_lsp(line_index.clone(), self))
.collect()
});
self.performance.measure(mark);

30
cli/lsp/lsp_custom.rs
View file

@ -46,6 +46,30 @@ pub struct DiagnosticBatchNotificationParams {
pub messages_len: usize,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DenoConfigurationData {
pub scope_uri: lsp::Uri,
pub workspace_root_scope_uri: Option<lsp::Uri>,
pub deno_json: Option<lsp::TextDocumentIdentifier>,
pub package_json: Option<lsp::TextDocumentIdentifier>,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DidRefreshDenoConfigurationTreeNotificationParams {
pub data: Vec<DenoConfigurationData>,
}
pub enum DidRefreshDenoConfigurationTreeNotification {}
impl lsp::notification::Notification
for DidRefreshDenoConfigurationTreeNotification
{
type Params = DidRefreshDenoConfigurationTreeNotificationParams;
const METHOD: &'static str = "deno/didRefreshDenoConfigurationTree";
}
#[derive(Debug, Eq, Hash, PartialEq, Copy, Clone, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum DenoConfigurationChangeType {
@ -88,13 +112,15 @@ pub struct DidChangeDenoConfigurationNotificationParams {
pub changes: Vec<DenoConfigurationChangeEvent>,
}
// TODO(nayeemrmn): This is being replaced by
// `DidRefreshDenoConfigurationTreeNotification` for Deno > v2.0.0. Remove it
// soon.
pub enum DidChangeDenoConfigurationNotification {}
impl lsp::notification::Notification
for DidChangeDenoConfigurationNotification
{
type Params = DidChangeDenoConfigurationNotificationParams;
const METHOD: &'static str = "deno/didChangeDenoConfiguration";
}
@ -102,7 +128,6 @@ pub enum DidUpgradeCheckNotification {}
impl lsp::notification::Notification for DidUpgradeCheckNotification {
type Params = DidUpgradeCheckNotificationParams;
const METHOD: &'static str = "deno/didUpgradeCheck";
}
@ -125,6 +150,5 @@ pub enum DiagnosticBatchNotification {}
impl lsp::notification::Notification for DiagnosticBatchNotification {
type Params = DiagnosticBatchNotificationParams;
const METHOD: &'static str = "deno/internalTestDiagnosticBatch";
}

6
cli/lsp/npm.rs
View file

@ -4,6 +4,7 @@ use dashmap::DashMap;
use deno_core::anyhow::anyhow;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_npm::npm_rc::NpmRc;
use deno_semver::package::PackageNv;
use deno_semver::Version;
use serde::Deserialize;
@ -25,7 +26,10 @@ pub struct CliNpmSearchApi {
impl CliNpmSearchApi {
pub fn new(file_fetcher: Arc<FileFetcher>) -> Self {
let resolver = NpmFetchResolver::new(file_fetcher.clone());
let resolver = NpmFetchResolver::new(
file_fetcher.clone(),
Arc::new(NpmRc::default().as_resolved(npm_registry_url()).unwrap()),
);
Self {
file_fetcher,
resolver,

1
cli/lsp/registries.rs
View file

@ -482,6 +482,7 @@ impl ModuleRegistry {
.fetch_with_options(FetchOptions {
specifier: &specifier,
permissions: FetchPermissionsOptionRef::AllowAll,
maybe_auth: None,
maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"),
maybe_cache_setting: None,
})

234
cli/lsp/resolver.rs
View file

@ -2,6 +2,8 @@
use dashmap::DashMap;
use deno_ast::MediaType;
use deno_ast::ParsedSource;
use deno_cache_dir::npm::NpmCacheDir;
use deno_cache_dir::HttpCache;
use deno_config::workspace::PackageJsonDepResolution;
use deno_config::workspace::WorkspaceResolver;
@ -14,15 +16,15 @@ use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJson;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use indexmap::IndexMap;
use node_resolver::errors::ClosestPkgJsonError;
use node_resolver::NodeResolution;
use node_resolver::InNpmPackageChecker;
use node_resolver::NodeResolutionMode;
use node_resolver::NpmResolver;
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
@ -36,6 +38,7 @@ use crate::args::create_default_npmrc;
use crate::args::CacheSetting;
use crate::args::CliLockfile;
use crate::args::NpmInstallDepsProvider;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::graph_util::CliJsrUrlProvider;
use crate::http_util::HttpClientProvider;
use crate::lsp::config::Config;
@ -43,26 +46,32 @@ use crate::lsp::config::ConfigData;
use crate::lsp::logging::lsp_warn;
use crate::npm::create_cli_npm_resolver_for_lsp;
use crate::npm::CliByonmNpmResolverCreateOptions;
use crate::npm::CliManagedInNpmPkgCheckerCreateOptions;
use crate::npm::CliManagedNpmResolverCreateOptions;
use crate::npm::CliNpmResolver;
use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::npm::ManagedCliNpmResolver;
use crate::resolver::CjsResolutionStore;
use crate::resolver::CjsTracker;
use crate::resolver::CjsTrackerOptions;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliGraphResolver;
use crate::resolver::CliGraphResolverOptions;
use crate::resolver::CliNodeResolver;
use crate::resolver::WorkerCliNpmGraphResolver;
use crate::tsc::into_specifier_and_media_type;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
#[derive(Debug, Clone)]
struct LspScopeResolver {
cjs_tracker: Option<Arc<LspCjsTracker>>,
graph_resolver: Arc<CliGraphResolver>,
jsr_resolver: Option<Arc<JsrCacheResolver>>,
npm_resolver: Option<Arc<dyn CliNpmResolver>>,
node_resolver: Option<Arc<CliNodeResolver>>,
pkg_json_resolver: Option<Arc<PackageJsonResolver>>,
redirect_resolver: Option<Arc<RedirectResolver>>,
graph_imports: Arc<IndexMap<ModuleSpecifier, GraphImport>>,
config_data: Option<Arc<ConfigData>>,
@ -71,10 +80,12 @@ struct LspScopeResolver {
impl Default for LspScopeResolver {
fn default() -> Self {
Self {
cjs_tracker: None,
graph_resolver: create_graph_resolver(None, None, None),
jsr_resolver: None,
npm_resolver: None,
node_resolver: None,
pkg_json_resolver: None,
redirect_resolver: None,
graph_imports: Default::default(),
config_data: None,
@ -90,14 +101,35 @@ impl LspScopeResolver {
) -> Self {
let mut npm_resolver = None;
let mut node_resolver = None;
let mut lsp_cjs_tracker = None;
let fs = Arc::new(deno_fs::RealFs);
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
));
if let Some(http_client) = http_client_provider {
npm_resolver = create_npm_resolver(
config_data.map(|d| d.as_ref()),
cache,
http_client,
&pkg_json_resolver,
)
.await;
node_resolver = create_node_resolver(npm_resolver.as_ref());
if let Some(npm_resolver) = &npm_resolver {
let in_npm_pkg_checker = create_in_npm_pkg_checker(npm_resolver);
let cjs_tracker = create_cjs_tracker(
in_npm_pkg_checker.clone(),
pkg_json_resolver.clone(),
);
lsp_cjs_tracker =
Some(Arc::new(LspCjsTracker::new(cjs_tracker.clone())));
node_resolver = Some(create_node_resolver(
cjs_tracker,
fs.clone(),
in_npm_pkg_checker,
npm_resolver,
pkg_json_resolver.clone(),
));
}
}
let graph_resolver = create_graph_resolver(
config_data.map(|d| d.as_ref()),
@ -134,10 +166,12 @@ impl LspScopeResolver {
})
.unwrap_or_default();
Self {
cjs_tracker: lsp_cjs_tracker,
graph_resolver,
jsr_resolver,
npm_resolver,
node_resolver,
pkg_json_resolver: Some(pkg_json_resolver),
redirect_resolver,
graph_imports,
config_data: config_data.cloned(),
@ -147,18 +181,40 @@ impl LspScopeResolver {
fn snapshot(&self) -> Arc<Self> {
let npm_resolver =
self.npm_resolver.as_ref().map(|r| r.clone_snapshotted());
let node_resolver = create_node_resolver(npm_resolver.as_ref());
let fs = Arc::new(deno_fs::RealFs);
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
));
let mut node_resolver = None;
let mut lsp_cjs_tracker = None;
if let Some(npm_resolver) = &npm_resolver {
let in_npm_pkg_checker = create_in_npm_pkg_checker(npm_resolver);
let cjs_tracker = create_cjs_tracker(
in_npm_pkg_checker.clone(),
pkg_json_resolver.clone(),
);
lsp_cjs_tracker = Some(Arc::new(LspCjsTracker::new(cjs_tracker.clone())));
node_resolver = Some(create_node_resolver(
cjs_tracker,
fs,
in_npm_pkg_checker,
npm_resolver,
pkg_json_resolver.clone(),
));
}
let graph_resolver = create_graph_resolver(
self.config_data.as_deref(),
npm_resolver.as_ref(),
node_resolver.as_ref(),
);
Arc::new(Self {
cjs_tracker: lsp_cjs_tracker,
graph_resolver,
jsr_resolver: self.jsr_resolver.clone(),
npm_resolver,
node_resolver,
redirect_resolver: self.redirect_resolver.clone(),
pkg_json_resolver: Some(pkg_json_resolver),
graph_imports: self.graph_imports.clone(),
config_data: self.config_data.clone(),
})
@ -261,6 +317,22 @@ impl LspResolver {
resolver.graph_resolver.create_graph_npm_resolver()
}
pub fn maybe_cjs_tracker(
&self,
file_referrer: Option<&ModuleSpecifier>,
) -> Option<&Arc<LspCjsTracker>> {
let resolver = self.get_scope_resolver(file_referrer);
resolver.cjs_tracker.as_ref()
}
pub fn maybe_node_resolver(
&self,
file_referrer: Option<&ModuleSpecifier>,
) -> Option<&Arc<CliNodeResolver>> {
let resolver = self.get_scope_resolver(file_referrer);
resolver.node_resolver.as_ref()
}
pub fn maybe_managed_npm_resolver(
&self,
file_referrer: Option<&ModuleSpecifier>,
@ -328,7 +400,7 @@ impl LspResolver {
) -> Option<(ModuleSpecifier, MediaType)> {
let resolver = self.get_scope_resolver(file_referrer);
let node_resolver = resolver.node_resolver.as_ref()?;
Some(NodeResolution::into_specifier_and_media_type(Some(
Some(into_specifier_and_media_type(Some(
node_resolver
.resolve_req_reference(req_ref, referrer, NodeResolutionMode::Types)
.ok()?,
@ -346,14 +418,10 @@ impl LspResolver {
.contains("/node_modules/")
}
let global_npm_resolver = self
.get_scope_resolver(Some(specifier))
.npm_resolver
.as_ref()
.and_then(|npm_resolver| npm_resolver.as_managed())
.filter(|r| r.root_node_modules_path().is_none());
if let Some(npm_resolver) = &global_npm_resolver {
if npm_resolver.in_npm_package(specifier) {
if let Some(node_resolver) =
&self.get_scope_resolver(Some(specifier)).node_resolver
{
if node_resolver.in_npm_package(specifier) {
return true;
}
}
@ -361,18 +429,6 @@ impl LspResolver {
has_node_modules_dir(specifier)
}
pub fn node_media_type(
&self,
specifier: &ModuleSpecifier,
) -> Option<MediaType> {
let resolver = self.get_scope_resolver(Some(specifier));
let node_resolver = resolver.node_resolver.as_ref()?;
let resolution = node_resolver
.url_to_node_resolution(specifier.clone())
.ok()?;
Some(NodeResolution::into_specifier_and_media_type(Some(resolution)).1)
}
pub fn is_bare_package_json_dep(
&self,
specifier_text: &str,
@ -398,10 +454,10 @@ impl LspResolver {
referrer: &ModuleSpecifier,
) -> Result<Option<Arc<PackageJson>>, ClosestPkgJsonError> {
let resolver = self.get_scope_resolver(Some(referrer));
let Some(node_resolver) = resolver.node_resolver.as_ref() else {
let Some(pkg_json_resolver) = resolver.pkg_json_resolver.as_ref() else {
return Ok(None);
};
node_resolver.get_closest_package_json(referrer)
pkg_json_resolver.get_closest_package_json(referrer)
}
pub fn resolve_redirects(
@ -457,11 +513,13 @@ async fn create_npm_resolver(
config_data: Option<&ConfigData>,
cache: &LspCache,
http_client_provider: &Arc<HttpClientProvider>,
pkg_json_resolver: &Arc<PackageJsonResolver>,
) -> Option<Arc<dyn CliNpmResolver>> {
let enable_byonm = config_data.map(|d| d.byonm).unwrap_or(false);
let options = if enable_byonm {
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(Arc::new(deno_fs::RealFs)),
pkg_json_resolver: pkg_json_resolver.clone(),
root_node_modules_dir: config_data.and_then(|config_data| {
config_data.node_modules_dir.clone().or_else(|| {
url_to_file_path(&config_data.scope)
@ -471,7 +529,15 @@ async fn create_npm_resolver(
}),
})
} else {
CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions {
let npmrc = config_data
.and_then(|d| d.npmrc.clone())
.unwrap_or_else(create_default_npmrc);
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(&deno_fs::RealFs),
cache.deno_dir().npm_folder_path(),
npmrc.get_all_known_registries_urls(),
));
CliNpmResolverCreateOptions::Managed(CliManagedNpmResolverCreateOptions {
http_client_provider: http_client_provider.clone(),
snapshot: match config_data.and_then(|d| d.lockfile.as_ref()) {
Some(lockfile) => {
@ -485,7 +551,7 @@ async fn create_npm_resolver(
// updating it. Only the cache request should update the lockfile.
maybe_lockfile: None,
fs: Arc::new(deno_fs::RealFs),
npm_global_cache_dir: cache.deno_dir().npm_folder_path(),
npm_cache_dir,
// Use an "only" cache setting in order to make the
// user do an explicit "cache" command and prevent
// the cache from being filled with lots of packages while
@ -496,9 +562,7 @@ async fn create_npm_resolver(
.and_then(|d| d.node_modules_dir.clone()),
// only used for top level install, so we can ignore this
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()),
npmrc: config_data
.and_then(|d| d.npmrc.clone())
.unwrap_or_else(create_default_npmrc),
npmrc,
npm_system_info: NpmSystemInfo::default(),
lifecycle_scripts: Default::default(),
})
@ -506,28 +570,59 @@ async fn create_npm_resolver(
Some(create_cli_npm_resolver_for_lsp(options).await)
}
fn create_cjs_tracker(
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
pkg_json_resolver: Arc<PackageJsonResolver>,
) -> Arc<CjsTracker> {
Arc::new(CjsTracker::new(
in_npm_pkg_checker,
pkg_json_resolver,
CjsTrackerOptions {
// todo(dsherret): support in the lsp by stabilizing the feature
// so that we don't have to pipe the config in here
unstable_detect_cjs: false,
},
))
}
fn create_in_npm_pkg_checker(
npm_resolver: &Arc<dyn CliNpmResolver>,
) -> Arc<dyn InNpmPackageChecker> {
crate::npm::create_in_npm_pkg_checker(match npm_resolver.as_inner() {
crate::npm::InnerCliNpmResolverRef::Byonm(_) => {
CreateInNpmPkgCheckerOptions::Byonm
}
crate::npm::InnerCliNpmResolverRef::Managed(m) => {
CreateInNpmPkgCheckerOptions::Managed(
CliManagedInNpmPkgCheckerCreateOptions {
root_cache_dir_url: m.global_cache_root_url(),
maybe_node_modules_path: m.maybe_node_modules_path(),
},
)
}
})
}
fn create_node_resolver(
npm_resolver: Option<&Arc<dyn CliNpmResolver>>,
) -> Option<Arc<CliNodeResolver>> {
use once_cell::sync::Lazy;
// it's not ideal to share this across all scopes and to
// never clear it, but it's fine for the time being
static CJS_RESOLUTIONS: Lazy<Arc<CjsResolutionStore>> =
Lazy::new(Default::default);
let npm_resolver = npm_resolver?;
let fs = Arc::new(deno_fs::RealFs);
cjs_tracker: Arc<CjsTracker>,
fs: Arc<dyn deno_fs::FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
npm_resolver: &Arc<dyn CliNpmResolver>,
pkg_json_resolver: Arc<PackageJsonResolver>,
) -> Arc<CliNodeResolver> {
let node_resolver_inner = Arc::new(NodeResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
in_npm_pkg_checker.clone(),
npm_resolver.clone().into_npm_resolver(),
pkg_json_resolver.clone(),
));
Some(Arc::new(CliNodeResolver::new(
CJS_RESOLUTIONS.clone(),
Arc::new(CliNodeResolver::new(
cjs_tracker.clone(),
fs,
in_npm_pkg_checker,
node_resolver_inner,
npm_resolver.clone(),
)))
))
}
fn create_graph_resolver(
@ -555,8 +650,8 @@ fn create_graph_resolver(
workspace.to_maybe_jsx_import_source_config().ok().flatten()
}),
maybe_vendor_dir: config_data.and_then(|d| d.vendor_dir.as_ref()),
bare_node_builtins_enabled: workspace
.is_some_and(|workspace| workspace.has_unstable("bare-node-builtins")),
bare_node_builtins_enabled: config_data
.is_some_and(|d| d.unstable.contains("bare-node-builtins")),
sloppy_imports_resolver: config_data
.and_then(|d| d.sloppy_imports_resolver.clone()),
}))
@ -702,6 +797,45 @@ impl RedirectResolver {
}
}
#[derive(Debug)]
pub struct LspCjsTracker {
cjs_tracker: Arc<CjsTracker>,
}
impl LspCjsTracker {
pub fn new(cjs_tracker: Arc<CjsTracker>) -> Self {
Self { cjs_tracker }
}
pub fn is_cjs(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
maybe_parsed_source: Option<&ParsedSource>,
) -> bool {
if let Some(module_kind) =
self.cjs_tracker.get_known_kind(specifier, media_type)
{
module_kind.is_cjs()
} else {
let maybe_is_script = maybe_parsed_source.map(|p| p.compute_is_script());
maybe_is_script
.and_then(|is_script| {
self
.cjs_tracker
.is_cjs_with_known_is_script(specifier, media_type, is_script)
.ok()
})
.unwrap_or_else(|| {
self
.cjs_tracker
.is_maybe_cjs(specifier, media_type)
.unwrap_or(false)
})
}
}
}
#[cfg(test)]
mod tests {
use super::*;

2
cli/lsp/testing/collectors.rs
View file

@ -650,7 +650,7 @@ pub mod tests {
.unwrap();
let text_info = parsed_module.text_info_lazy().clone();
let mut collector = TestCollector::new(specifier, text_info);
parsed_module.module().visit_with(&mut collector);
parsed_module.program().visit_with(&mut collector);
collector.take()
}

101
cli/lsp/tsc.rs
View file

@ -2182,6 +2182,50 @@ impl NavigateToItem {
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct InlayHintDisplayPart {
pub text: String,
pub span: Option<TextSpan>,
pub file: Option<String>,
}
impl InlayHintDisplayPart {
pub fn to_lsp(
&self,
language_server: &language_server::Inner,
) -> lsp::InlayHintLabelPart {
let location = self.file.as_ref().map(|f| {
let specifier =
resolve_url(f).unwrap_or_else(|_| INVALID_SPECIFIER.clone());
let file_referrer =
language_server.documents.get_file_referrer(&specifier);
let uri = language_server
.url_map
.specifier_to_uri(&specifier, file_referrer.as_deref())
.unwrap_or_else(|_| INVALID_URI.clone());
let range = self
.span
.as_ref()
.and_then(|s| {
let asset_or_doc =
language_server.get_asset_or_document(&specifier).ok()?;
Some(s.to_range(asset_or_doc.line_index()))
})
.unwrap_or_else(|| {
lsp::Range::new(lsp::Position::new(0, 0), lsp::Position::new(0, 0))
});
lsp::Location { uri, range }
});
lsp::InlayHintLabelPart {
value: self.text.clone(),
tooltip: None,
location,
command: None,
}
}
}
#[derive(Debug, Clone, Deserialize)]
pub enum InlayHintKind {
Type,
@ -2203,6 +2247,7 @@ impl InlayHintKind {
#[serde(rename_all = "camelCase")]
pub struct InlayHint {
pub text: String,
pub display_parts: Option<Vec<InlayHintDisplayPart>>,
pub position: u32,
pub kind: InlayHintKind,
pub whitespace_before: Option<bool>,
@ -2210,10 +2255,23 @@ pub struct InlayHint {
}
impl InlayHint {
pub fn to_lsp(&self, line_index: Arc<LineIndex>) -> lsp::InlayHint {
pub fn to_lsp(
&self,
line_index: Arc<LineIndex>,
language_server: &language_server::Inner,
) -> lsp::InlayHint {
lsp::InlayHint {
position: line_index.position_tsc(self.position.into()),
label: lsp::InlayHintLabel::String(self.text.clone()),
label: if let Some(display_parts) = &self.display_parts {
lsp::InlayHintLabel::LabelParts(
display_parts
.iter()
.map(|p| p.to_lsp(language_server))
.collect(),
)
} else {
lsp::InlayHintLabel::String(self.text.clone())
},
kind: self.kind.to_lsp(),
padding_left: self.whitespace_before,
padding_right: self.whitespace_after,
@ -3939,7 +3997,7 @@ pub struct OutliningSpan {
kind: OutliningSpanKind,
}
const FOLD_END_PAIR_CHARACTERS: &[u8] = &[b'}', b']', b')', b'`'];
const FOLD_END_PAIR_CHARACTERS: &[u8] = b"}])`";
impl OutliningSpan {
pub fn to_folding_range(
@ -4304,14 +4362,25 @@ fn op_load<'s>(
None
} else {
let asset_or_document = state.get_asset_or_document(&specifier);
asset_or_document.map(|doc| LoadResponse {
data: doc.text(),
script_kind: crate::tsc::as_ts_script_kind(doc.media_type()),
version: state.script_version(&specifier),
is_cjs: matches!(
doc.media_type(),
MediaType::Cjs | MediaType::Cts | MediaType::Dcts
),
asset_or_document.map(|doc| {
let maybe_cjs_tracker = state
.state_snapshot
.resolver
.maybe_cjs_tracker(Some(&specifier));
LoadResponse {
data: doc.text(),
script_kind: crate::tsc::as_ts_script_kind(doc.media_type()),
version: state.script_version(&specifier),
is_cjs: maybe_cjs_tracker
.map(|t| {
t.is_cjs(
&specifier,
doc.media_type(),
doc.maybe_parsed_source().and_then(|p| p.as_ref().ok()),
)
})
.unwrap_or(false),
}
})
};
@ -4892,6 +4961,10 @@ pub struct UserPreferences {
pub allow_rename_of_import_path: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub auto_import_file_exclude_patterns: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub interactive_inlay_hints: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub prefer_type_only_auto_imports: Option<bool>,
}
impl UserPreferences {
@ -4909,6 +4982,7 @@ impl UserPreferences {
include_completions_with_snippet_text: Some(
config.snippet_support_capable(),
),
interactive_inlay_hints: Some(true),
provide_refactor_not_applicable_reason: Some(true),
quote_preference: Some(fmt_config.into()),
use_label_details_in_completion_entries: Some(true),
@ -5013,6 +5087,9 @@ impl UserPreferences {
} else {
Some(language_settings.preferences.quote_style)
},
prefer_type_only_auto_imports: Some(
language_settings.preferences.prefer_type_only_auto_imports,
),
..base_preferences
}
}
@ -6154,7 +6231,7 @@ mod tests {
let change = changes.text_changes.first().unwrap();
assert_eq!(
change.new_text,
"import type { someLongVariable } from './b.ts'\n"
"import { someLongVariable } from './b.ts'\n"
);
}

114
cli/main.rs
View file

@ -15,7 +15,6 @@ mod js;
mod jsr;
mod lsp;
mod module_loader;
mod napi;
mod node;
mod npm;
mod ops;
@ -47,8 +46,7 @@ use deno_core::error::JsError;
use deno_core::futures::FutureExt;
use deno_core::unsync::JoinHandle;
use deno_npm::resolution::SnapshotFromLockfileError;
use deno_runtime::fmt_errors::format_js_error_with_suggestions;
use deno_runtime::fmt_errors::FixSuggestion;
use deno_runtime::fmt_errors::format_js_error;
use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics;
use deno_terminal::colors;
use factory::CliFactory;
@ -62,6 +60,10 @@ use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
#[cfg(feature = "dhat-heap")]
#[global_allocator]
static ALLOC: dhat::Alloc = dhat::Alloc;
/// Ensures that all subcommands return an i32 exit code and an [`AnyError`] error type.
trait SubcommandOutput {
fn output(self) -> Result<i32, AnyError>;
@ -133,7 +135,7 @@ async fn run_subcommand(flags: Arc<Flags>) -> Result<i32, AnyError> {
tools::compile::compile(flags, compile_flags).await
}),
DenoSubcommand::Coverage(coverage_flags) => spawn_subcommand(async {
tools::coverage::cover_files(flags, coverage_flags).await
tools::coverage::cover_files(flags, coverage_flags)
}),
DenoSubcommand::Fmt(fmt_flags) => {
spawn_subcommand(
@ -362,104 +364,12 @@ fn exit_with_message(message: &str, code: i32) -> ! {
std::process::exit(code);
}
fn get_suggestions_for_terminal_errors(e: &JsError) -> Vec<FixSuggestion> {
if let Some(msg) = &e.message {
if msg.contains("module is not defined")
|| msg.contains("exports is not defined")
{
return vec![
FixSuggestion::info(
"Deno does not support CommonJS modules without `.cjs` extension.",
),
FixSuggestion::hint(
"Rewrite this module to ESM or change the file extension to `.cjs`.",
),
];
} else if msg.contains("openKv is not a function") {
return vec![
FixSuggestion::info("Deno.openKv() is an unstable API."),
FixSuggestion::hint(
"Run again with `--unstable-kv` flag to enable this API.",
),
];
} else if msg.contains("cron is not a function") {
return vec![
FixSuggestion::info("Deno.cron() is an unstable API."),
FixSuggestion::hint(
"Run again with `--unstable-cron` flag to enable this API.",
),
];
} else if msg.contains("WebSocketStream is not defined") {
return vec![
FixSuggestion::info("new WebSocketStream() is an unstable API."),
FixSuggestion::hint(
"Run again with `--unstable-net` flag to enable this API.",
),
];
} else if msg.contains("Temporal is not defined") {
return vec![
FixSuggestion::info("Temporal is an unstable API."),
FixSuggestion::hint(
"Run again with `--unstable-temporal` flag to enable this API.",
),
];
} else if msg.contains("BroadcastChannel is not defined") {
return vec![
FixSuggestion::info("BroadcastChannel is an unstable API."),
FixSuggestion::hint(
"Run again with `--unstable-broadcast-channel` flag to enable this API.",
),
];
} else if msg.contains("window is not defined") {
return vec![
FixSuggestion::info("window global is not available in Deno 2."),
FixSuggestion::hint("Replace `window` with `globalThis`."),
];
} else if msg.contains("UnsafeWindowSurface is not a constructor") {
return vec![
FixSuggestion::info("Deno.UnsafeWindowSurface is an unstable API."),
FixSuggestion::hint(
"Run again with `--unstable-webgpu` flag to enable this API.",
),
];
// Try to capture errors like:
// ```
// Uncaught Error: Cannot find module '../build/Release/canvas.node'
// Require stack:
// - /.../deno/npm/registry.npmjs.org/canvas/2.11.2/lib/bindings.js
// - /.../.cache/deno/npm/registry.npmjs.org/canvas/2.11.2/lib/canvas.js
// ```
} else if msg.contains("Cannot find module")
&& msg.contains("Require stack")
&& msg.contains(".node'")
{
return vec![
FixSuggestion::info_multiline(
&[
"Trying to execute an npm package using Node-API addons,",
"these packages require local `node_modules` directory to be present."
]
),
FixSuggestion::hint_multiline(
&[
"Add `\"nodeModulesDir\": \"auto\" option to `deno.json`, and then run",
"`deno install --allow-scripts=npm:<package> --entrypoint <script>` to setup `node_modules` directory."
]
)
];
}
}
vec![]
}
fn exit_for_error(error: AnyError) -> ! {
let mut error_string = format!("{error:?}");
let mut error_code = 1;
if let Some(e) = error.downcast_ref::<JsError>() {
let suggestions = get_suggestions_for_terminal_errors(e);
error_string = format_js_error_with_suggestions(e, suggestions);
error_string = format_js_error(e);
} else if let Some(SnapshotFromLockfileError::IntegrityCheckFailed(e)) =
error.downcast_ref::<SnapshotFromLockfileError>()
{
@ -480,6 +390,9 @@ pub(crate) fn unstable_exit_cb(feature: &str, api_name: &str) {
}
pub fn main() {
#[cfg(feature = "dhat-heap")]
let profiler = dhat::Profiler::new_heap();
setup_panic_hook();
util::unix::raise_fd_limit();
@ -500,7 +413,12 @@ pub fn main() {
run_subcommand(Arc::new(flags)).await
};
match create_and_run_current_thread_with_maybe_metrics(future) {
let result = create_and_run_current_thread_with_maybe_metrics(future);
#[cfg(feature = "dhat-heap")]
drop(profiler);
match result {
Ok(exit_code) => std::process::exit(exit_code),
Err(err) => exit_for_error(err),
}

9
cli/mainrt.rs
View file

@ -88,11 +88,10 @@ fn main() {
let standalone = standalone::extract_standalone(Cow::Owned(args));
let future = async move {
match standalone {
Ok(Some(future)) => {
let (metadata, eszip) = future.await?;
util::logger::init(metadata.log_level);
load_env_vars(&metadata.env_vars_from_env_file);
let exit_code = standalone::run(eszip, metadata).await?;
Ok(Some(data)) => {
util::logger::init(data.metadata.log_level);
load_env_vars(&data.metadata.env_vars_from_env_file);
let exit_code = standalone::run(data).await?;
std::process::exit(exit_code);
}
Ok(None) => Ok(()),

377
cli/module_loader.rs
View file

@ -2,6 +2,7 @@
use std::borrow::Cow;
use std::cell::RefCell;
use std::path::Path;
use std::path::PathBuf;
use std::pin::Pin;
use std::rc::Rc;
@ -23,19 +24,23 @@ use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::CreateGraphOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::node;
use crate::node::CliNodeCodeTranslator;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliGraphResolver;
use crate::resolver::CliNodeResolver;
use crate::resolver::ModuleCodeStringSource;
use crate::resolver::NotSupportedKindInNpmError;
use crate::resolver::NpmModuleLoader;
use crate::tools::check;
use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::text_encoding::code_without_source_map;
use crate::util::text_encoding::source_map_from_code;
use crate::worker::ModuleLoaderAndSourceMapGetter;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
@ -63,9 +68,12 @@ use deno_graph::Module;
use deno_graph::ModuleGraph;
use deno_graph::Resolution;
use deno_runtime::code_cache;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::npm::NpmPackageReqReference;
use node_resolver::InNpmPackageChecker;
use node_resolver::NodeResolutionMode;
pub struct ModuleLoadPreparer {
@ -198,11 +206,16 @@ struct SharedCliModuleLoaderState {
lib_worker: TsTypeLib,
initial_cwd: PathBuf,
is_inspecting: bool,
is_npm_main: bool,
is_repl: bool,
cjs_tracker: Arc<CjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
node_code_translator: Arc<CliNodeCodeTranslator>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_module_loader: NpmModuleLoader,
@ -218,10 +231,14 @@ impl CliModuleLoaderFactory {
#[allow(clippy::too_many_arguments)]
pub fn new(
options: &CliOptions,
cjs_tracker: Arc<CjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
node_code_translator: Arc<CliNodeCodeTranslator>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_module_loader: NpmModuleLoader,
@ -235,14 +252,19 @@ impl CliModuleLoaderFactory {
lib_worker: options.ts_type_lib_worker(),
initial_cwd: options.initial_cwd().to_path_buf(),
is_inspecting: options.is_inspecting(),
is_npm_main: options.is_npm_main(),
is_repl: matches!(
options.sub_command(),
DenoSubcommand::Repl(_) | DenoSubcommand::Jupyter(_)
),
cjs_tracker,
code_cache,
emitter,
fs,
in_npm_pkg_checker,
main_module_graph_container,
module_load_preparer,
node_code_translator,
node_resolver,
npm_resolver,
npm_module_loader,
@ -259,19 +281,30 @@ impl CliModuleLoaderFactory {
is_worker: bool,
parent_permissions: PermissionsContainer,
permissions: PermissionsContainer,
) -> ModuleLoaderAndSourceMapGetter {
let loader = Rc::new(CliModuleLoader(Rc::new(CliModuleLoaderInner {
lib,
is_worker,
parent_permissions,
permissions,
) -> CreateModuleLoaderResult {
let module_loader =
Rc::new(CliModuleLoader(Rc::new(CliModuleLoaderInner {
lib,
is_worker,
is_npm_main: self.shared.is_npm_main,
parent_permissions,
permissions,
graph_container: graph_container.clone(),
node_code_translator: self.shared.node_code_translator.clone(),
emitter: self.shared.emitter.clone(),
parsed_source_cache: self.shared.parsed_source_cache.clone(),
shared: self.shared.clone(),
})));
let node_require_loader = Rc::new(CliNodeRequireLoader::new(
self.shared.emitter.clone(),
self.shared.fs.clone(),
graph_container,
emitter: self.shared.emitter.clone(),
parsed_source_cache: self.shared.parsed_source_cache.clone(),
shared: self.shared.clone(),
})));
ModuleLoaderAndSourceMapGetter {
module_loader: loader,
self.shared.in_npm_pkg_checker.clone(),
self.shared.npm_resolver.clone(),
));
CreateModuleLoaderResult {
module_loader,
node_require_loader,
}
}
}
@ -280,7 +313,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory {
fn create_for_main(
&self,
root_permissions: PermissionsContainer,
) -> ModuleLoaderAndSourceMapGetter {
) -> CreateModuleLoaderResult {
self.create_with_lib(
(*self.shared.main_module_graph_container).clone(),
self.shared.lib_window,
@ -294,7 +327,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory {
&self,
parent_permissions: PermissionsContainer,
permissions: PermissionsContainer,
) -> ModuleLoaderAndSourceMapGetter {
) -> CreateModuleLoaderResult {
self.create_with_lib(
// create a fresh module graph for the worker
WorkerModuleGraphContainer::new(Arc::new(ModuleGraph::new(
@ -310,6 +343,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory {
struct CliModuleLoaderInner<TGraphContainer: ModuleGraphContainer> {
lib: TsTypeLib,
is_npm_main: bool,
is_worker: bool,
/// The initial set of permissions used to resolve the static imports in the
/// worker. These are "allow all" for main worker, and parent thread
@ -318,6 +352,7 @@ struct CliModuleLoaderInner<TGraphContainer: ModuleGraphContainer> {
permissions: PermissionsContainer,
shared: Arc<SharedCliModuleLoaderState>,
emitter: Arc<Emitter>,
node_code_translator: Arc<CliNodeCodeTranslator>,
parsed_source_cache: Arc<ParsedSourceCache>,
graph_container: TGraphContainer,
}
@ -331,16 +366,7 @@ impl<TGraphContainer: ModuleGraphContainer>
maybe_referrer: Option<&ModuleSpecifier>,
requested_module_type: RequestedModuleType,
) -> Result<ModuleSource, AnyError> {
let code_source = if let Some(result) = self
.shared
.npm_module_loader
.load_if_in_npm_package(specifier, maybe_referrer)
.await
{
result?
} else {
self.load_prepared_module(specifier, maybe_referrer).await?
};
let code_source = self.load_code_source(specifier, maybe_referrer).await?;
let code = if self.shared.is_inspecting {
// we need the code with the source map in order for
// it to work with --inspect or --inspect-brk
@ -394,6 +420,29 @@ impl<TGraphContainer: ModuleGraphContainer>
))
}
async fn load_code_source(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
) -> Result<ModuleCodeStringSource, AnyError> {
if let Some(code_source) = self.load_prepared_module(specifier).await? {
return Ok(code_source);
}
if self.shared.node_resolver.in_npm_package(specifier) {
return self
.shared
.npm_module_loader
.load(specifier, maybe_referrer)
.await;
}
let mut msg = format!("Loading unprepared module: {specifier}");
if let Some(referrer) = maybe_referrer {
msg = format!("{}, imported from: {}", msg, referrer.as_str());
}
Err(anyhow!(msg))
}
fn resolve_referrer(
&self,
referrer: &str,
@ -466,15 +515,11 @@ impl<TGraphContainer: ModuleGraphContainer>
if self.shared.is_repl {
if let Ok(reference) = NpmPackageReqReference::from_specifier(&specifier)
{
return self
.shared
.node_resolver
.resolve_req_reference(
&reference,
referrer,
NodeResolutionMode::Execution,
)
.map(|res| res.into_url());
return self.shared.node_resolver.resolve_req_reference(
&reference,
referrer,
NodeResolutionMode::Execution,
);
}
}
@ -498,13 +543,15 @@ impl<TGraphContainer: ModuleGraphContainer>
.with_context(|| {
format!("Could not resolve '{}'.", module.nv_reference)
})?
.into_url()
}
Some(Module::Node(module)) => module.specifier.clone(),
Some(Module::Js(module)) => module.specifier.clone(),
Some(Module::Json(module)) => module.specifier.clone(),
Some(Module::External(module)) => {
node::resolve_specifier_into_node_modules(&module.specifier)
node::resolve_specifier_into_node_modules(
&module.specifier,
self.shared.fs.as_ref(),
)
}
None => specifier.into_owned(),
};
@ -514,71 +561,82 @@ impl<TGraphContainer: ModuleGraphContainer>
async fn load_prepared_module(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
) -> Result<ModuleCodeStringSource, AnyError> {
) -> Result<Option<ModuleCodeStringSource>, AnyError> {
// Note: keep this in sync with the sync version below
let graph = self.graph_container.graph();
match self.load_prepared_module_or_defer_emit(
&graph,
specifier,
maybe_referrer,
) {
Ok(CodeOrDeferredEmit::Code(code_source)) => Ok(code_source),
Ok(CodeOrDeferredEmit::DeferredEmit {
match self.load_prepared_module_or_defer_emit(&graph, specifier)? {
Some(CodeOrDeferredEmit::Code(code_source)) => Ok(Some(code_source)),
Some(CodeOrDeferredEmit::DeferredEmit {
specifier,
media_type,
source,
}) => {
let transpile_result = self
.emitter
.emit_parsed_source(specifier, media_type, source)
.emit_parsed_source(specifier, media_type, ModuleKind::Esm, source)
.await?;
// at this point, we no longer need the parsed source in memory, so free it
self.parsed_source_cache.free(specifier);
Ok(ModuleCodeStringSource {
code: ModuleSourceCode::Bytes(transpile_result),
Ok(Some(ModuleCodeStringSource {
// note: it's faster to provide a string if we know it's a string
code: ModuleSourceCode::String(transpile_result.into()),
found_url: specifier.clone(),
media_type,
})
}))
}
Err(err) => Err(err),
Some(CodeOrDeferredEmit::Cjs {
specifier,
media_type,
source,
}) => self
.load_maybe_cjs(specifier, media_type, source)
.await
.map(Some),
None => Ok(None),
}
}
fn load_prepared_module_sync(
fn load_prepared_module_for_source_map_sync(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
) -> Result<ModuleCodeStringSource, AnyError> {
) -> Result<Option<ModuleCodeStringSource>, AnyError> {
// Note: keep this in sync with the async version above
let graph = self.graph_container.graph();
match self.load_prepared_module_or_defer_emit(
&graph,
specifier,
maybe_referrer,
) {
Ok(CodeOrDeferredEmit::Code(code_source)) => Ok(code_source),
Ok(CodeOrDeferredEmit::DeferredEmit {
match self.load_prepared_module_or_defer_emit(&graph, specifier)? {
Some(CodeOrDeferredEmit::Code(code_source)) => Ok(Some(code_source)),
Some(CodeOrDeferredEmit::DeferredEmit {
specifier,
media_type,
source,
}) => {
let transpile_result = self
.emitter
.emit_parsed_source_sync(specifier, media_type, source)?;
let transpile_result = self.emitter.emit_parsed_source_sync(
specifier,
media_type,
ModuleKind::Esm,
source,
)?;
// at this point, we no longer need the parsed source in memory, so free it
self.parsed_source_cache.free(specifier);
Ok(ModuleCodeStringSource {
code: ModuleSourceCode::Bytes(transpile_result),
Ok(Some(ModuleCodeStringSource {
// note: it's faster to provide a string if we know it's a string
code: ModuleSourceCode::String(transpile_result.into()),
found_url: specifier.clone(),
media_type,
})
}))
}
Err(err) => Err(err),
Some(CodeOrDeferredEmit::Cjs { .. }) => {
self.parsed_source_cache.free(specifier);
// todo(dsherret): to make this work, we should probably just
// rely on the CJS export cache. At the moment this is hard because
// cjs export analysis is only async
Ok(None)
}
None => Ok(None),
}
}
@ -586,8 +644,7 @@ impl<TGraphContainer: ModuleGraphContainer>
&self,
graph: &'graph ModuleGraph,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
) -> Result<CodeOrDeferredEmit<'graph>, AnyError> {
) -> Result<Option<CodeOrDeferredEmit<'graph>>, AnyError> {
if specifier.scheme() == "node" {
// Node built-in modules should be handled internally.
unreachable!("Deno bug. {} was misconfigured internally.", specifier);
@ -599,38 +656,58 @@ impl<TGraphContainer: ModuleGraphContainer>
media_type,
specifier,
..
})) => Ok(CodeOrDeferredEmit::Code(ModuleCodeStringSource {
})) => Ok(Some(CodeOrDeferredEmit::Code(ModuleCodeStringSource {
code: ModuleSourceCode::String(source.clone().into()),
found_url: specifier.clone(),
media_type: *media_type,
})),
}))),
Some(deno_graph::Module::Js(JsModule {
source,
media_type,
specifier,
is_script,
..
})) => {
// todo(dsherret): revert in https://github.com/denoland/deno/pull/26439
if self.is_npm_main && *is_script
|| self.shared.cjs_tracker.is_cjs_with_known_is_script(
specifier,
*media_type,
*is_script,
)?
{
return Ok(Some(CodeOrDeferredEmit::Cjs {
specifier,
media_type: *media_type,
source,
}));
}
let code: ModuleCodeString = match media_type {
MediaType::JavaScript
| MediaType::Unknown
| MediaType::Cjs
| MediaType::Mjs
| MediaType::Json => source.clone().into(),
MediaType::Dts | MediaType::Dcts | MediaType::Dmts => {
Default::default()
}
MediaType::TypeScript
| MediaType::Mts
| MediaType::Cts
| MediaType::Jsx
| MediaType::Tsx => {
return Ok(CodeOrDeferredEmit::DeferredEmit {
MediaType::Cjs | MediaType::Cts => {
return Ok(Some(CodeOrDeferredEmit::Cjs {
specifier,
media_type: *media_type,
source,
});
}));
}
MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => {
MediaType::TypeScript
| MediaType::Mts
| MediaType::Jsx
| MediaType::Tsx => {
return Ok(Some(CodeOrDeferredEmit::DeferredEmit {
specifier,
media_type: *media_type,
source,
}));
}
MediaType::Css | MediaType::Wasm | MediaType::SourceMap => {
panic!("Unexpected media type {media_type} for {specifier}")
}
};
@ -638,26 +715,62 @@ impl<TGraphContainer: ModuleGraphContainer>
// at this point, we no longer need the parsed source in memory, so free it
self.parsed_source_cache.free(specifier);
Ok(CodeOrDeferredEmit::Code(ModuleCodeStringSource {
Ok(Some(CodeOrDeferredEmit::Code(ModuleCodeStringSource {
code: ModuleSourceCode::String(code),
found_url: specifier.clone(),
media_type: *media_type,
}))
})))
}
Some(
deno_graph::Module::External(_)
| deno_graph::Module::Node(_)
| deno_graph::Module::Npm(_),
)
| None => {
let mut msg = format!("Loading unprepared module: {specifier}");
if let Some(referrer) = maybe_referrer {
msg = format!("{}, imported from: {}", msg, referrer.as_str());
}
Err(anyhow!(msg))
}
| None => Ok(None),
}
}
async fn load_maybe_cjs(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
original_source: &Arc<str>,
) -> Result<ModuleCodeStringSource, AnyError> {
let js_source = if media_type.is_emittable() {
Cow::Owned(
self
.emitter
.emit_parsed_source(
specifier,
media_type,
ModuleKind::Cjs,
original_source,
)
.await?,
)
} else {
Cow::Borrowed(original_source.as_ref())
};
let text = self
.node_code_translator
.translate_cjs_to_esm(specifier, Some(js_source))
.await?;
// at this point, we no longer need the parsed source in memory, so free it
self.parsed_source_cache.free(specifier);
Ok(ModuleCodeStringSource {
code: match text {
// perf: if the text is borrowed, that means it didn't make any changes
// to the original source, so we can just provide that instead of cloning
// the borrowed text
Cow::Borrowed(_) => {
ModuleSourceCode::String(original_source.clone().into())
}
Cow::Owned(text) => ModuleSourceCode::String(text.into()),
},
found_url: specifier.clone(),
media_type,
})
}
}
enum CodeOrDeferredEmit<'a> {
@ -667,6 +780,11 @@ enum CodeOrDeferredEmit<'a> {
media_type: MediaType,
source: &'a Arc<str>,
},
Cjs {
specifier: &'a ModuleSpecifier,
media_type: MediaType,
source: &'a Arc<str>,
},
}
// todo(dsherret): this double Rc boxing is not ideal
@ -828,7 +946,10 @@ impl<TGraphContainer: ModuleGraphContainer> ModuleLoader
"wasm" | "file" | "http" | "https" | "data" | "blob" => (),
_ => return None,
}
let source = self.0.load_prepared_module_sync(&specifier, None).ok()?;
let source = self
.0
.load_prepared_module_for_source_map_sync(&specifier)
.ok()??;
source_map_from_code(source.code.as_bytes())
}
@ -907,3 +1028,79 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit {
drop(self.permit); // explicit drop for clarity
}
}
#[derive(Debug)]
struct CliNodeRequireLoader<TGraphContainer: ModuleGraphContainer> {
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
graph_container: TGraphContainer,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
npm_resolver: Arc<dyn CliNpmResolver>,
}
impl<TGraphContainer: ModuleGraphContainer>
CliNodeRequireLoader<TGraphContainer>
{
pub fn new(
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
graph_container: TGraphContainer,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
npm_resolver: Arc<dyn CliNpmResolver>,
) -> Self {
Self {
emitter,
fs,
graph_container,
in_npm_pkg_checker,
npm_resolver,
}
}
}
impl<TGraphContainer: ModuleGraphContainer> NodeRequireLoader
for CliNodeRequireLoader<TGraphContainer>
{
fn ensure_read_permission<'a>(
&self,
permissions: &mut dyn deno_runtime::deno_node::NodePermissions,
path: &'a Path,
) -> Result<std::borrow::Cow<'a, Path>, AnyError> {
if let Ok(url) = deno_path_util::url_from_file_path(path) {
// allow reading if it's in the module graph
if self.graph_container.graph().get(&url).is_some() {
return Ok(std::borrow::Cow::Borrowed(path));
}
}
self.npm_resolver.ensure_read_permission(permissions, path)
}
fn load_text_file_lossy(&self, path: &Path) -> Result<String, AnyError> {
// todo(dsherret): use the preloaded module from the graph if available?
let media_type = MediaType::from_path(path);
let text = self.fs.read_text_file_lossy_sync(path, None)?;
if media_type.is_emittable() {
let specifier = deno_path_util::url_from_file_path(path)?;
if self.in_npm_pkg_checker.in_npm_package(&specifier) {
return Err(
NotSupportedKindInNpmError {
media_type,
specifier,
}
.into(),
);
}
self.emitter.emit_parsed_source_sync(
&specifier,
media_type,
// this is probably not super accurate due to require esm, but probably ok.
// If we find this causes a lot of churn in the emit cache then we should
// investigate how we can make this better
ModuleKind::Cjs,
&text.into(),
)
} else {
Ok(text)
}
}
}

114
cli/napi/README.md
View file

@ -1,114 +0,0 @@
# napi
This directory contains source for Deno's Node-API implementation. It depends on
`napi_sym` and `deno_napi`.
Files are generally organized the same as in Node.js's implementation to ease in
ensuring compatibility.
## Adding a new function
Add the symbol name to
[`cli/napi_sym/symbol_exports.json`](../napi_sym/symbol_exports.json).
```diff
{
"symbols": [
...
"napi_get_undefined",
- "napi_get_null"
+ "napi_get_null",
+ "napi_get_boolean"
]
}
```
Determine where to place the implementation. `napi_get_boolean` is related to JS
values so we will place it in `js_native_api.rs`. If something is not clear,
just create a new file module.
See [`napi_sym`](../napi_sym/) for writing the implementation:
```rust
#[napi_sym::napi_sym]
pub fn napi_get_boolean(
env: *mut Env,
value: bool,
result: *mut napi_value,
) -> Result {
// ...
Ok(())
}
```
Update the generated symbol lists using the script:
```
deno run --allow-write tools/napi/generate_symbols_lists.js
```
Add a test in [`/tests/napi`](../../tests/napi/). You can also refer to Node.js
test suite for Node-API.
```js
// tests/napi/boolean_test.js
import { assertEquals, loadTestLibrary } from "./common.js";
const lib = loadTestLibrary();
Deno.test("napi get boolean", function () {
assertEquals(lib.test_get_boolean(true), true);
assertEquals(lib.test_get_boolean(false), false);
});
```
```rust
// tests/napi/src/boolean.rs
use napi_sys::Status::napi_ok;
use napi_sys::ValueType::napi_boolean;
use napi_sys::*;
extern "C" fn test_boolean(
env: napi_env,
info: napi_callback_info,
) -> napi_value {
let (args, argc, _) = crate::get_callback_info!(env, info, 1);
assert_eq!(argc, 1);
let mut ty = -1;
assert!(unsafe { napi_typeof(env, args[0], &mut ty) } == napi_ok);
assert_eq!(ty, napi_boolean);
// Use napi_get_boolean here...
value
}
pub fn init(env: napi_env, exports: napi_value) {
let properties = &[crate::new_property!(env, "test_boolean\0", test_boolean)];
unsafe {
napi_define_properties(env, exports, properties.len(), properties.as_ptr())
};
}
```
```diff
// tests/napi/src/lib.rs
+ mod boolean;
...
#[no_mangle]
unsafe extern "C" fn napi_register_module_v1(
env: napi_env,
exports: napi_value,
) -> napi_value {
...
+ boolean::init(env, exports);
exports
}
```
Run the test using `cargo test -p tests/napi`.

21
cli/napi/mod.rs
View file

@ -1,21 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
#![allow(unused_mut)]
#![allow(non_camel_case_types)]
#![allow(clippy::undocumented_unsafe_blocks)]
//! Symbols to be exported are now defined in this JSON file.
//! The `#[napi_sym]` macro checks for missing entries and panics.
//!
//! `./tools/napi/generate_symbols_list.js` is used to generate the LINK `cli/exports.def` on Windows,
//! which is also checked into git.
//!
//! To add a new napi function:
//! 1. Place `#[napi_sym]` on top of your implementation.
//! 2. Add the function's identifier to this JSON list.
//! 3. Finally, run `tools/napi/generate_symbols_list.js` to update `cli/napi/generated_symbol_exports_list_*.def`.
pub mod js_native_api;
pub mod node_api;
pub mod util;
pub mod uv;

133
cli/node.rs
View file

@ -1,10 +1,14 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::sync::Arc;
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_graph::ParsedSourceStore;
use deno_path_util::url_from_file_path;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis;
@ -16,8 +20,9 @@ use serde::Serialize;
use crate::cache::CacheDBHash;
use crate::cache::NodeAnalysisCache;
use crate::resolver::CliNodeResolver;
use crate::util::fs::canonicalize_path_maybe_not_exists;
use crate::cache::ParsedSourceCache;
use crate::resolver::CjsTracker;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
pub type CliNodeCodeTranslator =
NodeCodeTranslator<CliCjsCodeAnalyzer, DenoFsNodeResolverEnv>;
@ -30,14 +35,14 @@ pub type CliNodeCodeTranslator =
/// because the node_modules folder might not exist at that time.
pub fn resolve_specifier_into_node_modules(
specifier: &ModuleSpecifier,
fs: &dyn deno_fs::FileSystem,
) -> ModuleSpecifier {
specifier
.to_file_path()
url_to_file_path(specifier)
.ok()
// this path might not exist at the time the graph is being created
// because the node_modules folder might not yet exist
.and_then(|path| canonicalize_path_maybe_not_exists(&path).ok())
.and_then(|path| ModuleSpecifier::from_file_path(path).ok())
.and_then(|path| canonicalize_path_maybe_not_exists_with_fs(&path, fs).ok())
.and_then(|path| url_from_file_path(&path).ok())
.unwrap_or_else(|| specifier.clone())
}
@ -54,20 +59,29 @@ pub enum CliCjsAnalysis {
pub struct CliCjsCodeAnalyzer {
cache: NodeAnalysisCache,
cjs_tracker: Arc<CjsTracker>,
fs: deno_fs::FileSystemRc,
node_resolver: Arc<CliNodeResolver>,
parsed_source_cache: Option<Arc<ParsedSourceCache>>,
// todo(dsherret): hack, remove in https://github.com/denoland/deno/pull/26439
// For example, this does not properly handle if cjs analysis was already done
// and has been cached.
is_npm_main: bool,
}
impl CliCjsCodeAnalyzer {
pub fn new(
cache: NodeAnalysisCache,
cjs_tracker: Arc<CjsTracker>,
fs: deno_fs::FileSystemRc,
node_resolver: Arc<CliNodeResolver>,
parsed_source_cache: Option<Arc<ParsedSourceCache>>,
is_npm_main: bool,
) -> Self {
Self {
cache,
cjs_tracker,
fs,
node_resolver,
parsed_source_cache,
is_npm_main,
}
}
@ -83,7 +97,7 @@ impl CliCjsCodeAnalyzer {
return Ok(analysis);
}
let mut media_type = MediaType::from_specifier(specifier);
let media_type = MediaType::from_specifier(specifier);
if media_type == MediaType::Json {
return Ok(CliCjsAnalysis::Cjs {
exports: vec![],
@ -91,54 +105,53 @@ impl CliCjsCodeAnalyzer {
});
}
if media_type == MediaType::JavaScript {
if let Some(package_json) =
self.node_resolver.get_closest_package_json(specifier)?
{
match package_json.typ.as_str() {
"commonjs" => {
media_type = MediaType::Cjs;
}
"module" => {
media_type = MediaType::Mjs;
}
_ => {}
}
}
}
let cjs_tracker = self.cjs_tracker.clone();
let is_npm_main = self.is_npm_main;
let is_maybe_cjs =
cjs_tracker.is_maybe_cjs(specifier, media_type)? || is_npm_main;
let analysis = if is_maybe_cjs {
let maybe_parsed_source = self
.parsed_source_cache
.as_ref()
.and_then(|c| c.remove_parsed_source(specifier));
let analysis = deno_core::unsync::spawn_blocking({
let specifier = specifier.clone();
let source: Arc<str> = source.into();
move || -> Result<_, deno_ast::ParseDiagnostic> {
let parsed_source = deno_ast::parse_program(deno_ast::ParseParams {
specifier,
text: source,
media_type,
capture_tokens: true,
scope_analysis: false,
maybe_syntax: None,
})?;
if parsed_source.is_script() {
let analysis = parsed_source.analyze_cjs();
Ok(CliCjsAnalysis::Cjs {
exports: analysis.exports,
reexports: analysis.reexports,
})
} else if media_type == MediaType::Cjs {
// FIXME: `deno_ast` should internally handle MediaType::Cjs implying that
// the result must never be Esm
Ok(CliCjsAnalysis::Cjs {
exports: vec![],
reexports: vec![],
})
} else {
Ok(CliCjsAnalysis::Esm)
deno_core::unsync::spawn_blocking({
let specifier = specifier.clone();
let source: Arc<str> = source.into();
move || -> Result<_, AnyError> {
let parsed_source =
maybe_parsed_source.map(Ok).unwrap_or_else(|| {
deno_ast::parse_program(deno_ast::ParseParams {
specifier,
text: source,
media_type,
capture_tokens: true,
scope_analysis: false,
maybe_syntax: None,
})
})?;
let is_script = parsed_source.compute_is_script();
let is_cjs = cjs_tracker.is_cjs_with_known_is_script(
parsed_source.specifier(),
media_type,
is_script,
)? || is_script && is_npm_main;
if is_cjs {
let analysis = parsed_source.analyze_cjs();
Ok(CliCjsAnalysis::Cjs {
exports: analysis.exports,
reexports: analysis.reexports,
})
} else {
Ok(CliCjsAnalysis::Esm)
}
}
}
})
.await
.unwrap()?;
})
.await
.unwrap()?
} else {
CliCjsAnalysis::Esm
};
self
.cache
@ -150,11 +163,11 @@ impl CliCjsCodeAnalyzer {
#[async_trait::async_trait(?Send)]
impl CjsCodeAnalyzer for CliCjsCodeAnalyzer {
async fn analyze_cjs(
async fn analyze_cjs<'a>(
&self,
specifier: &ModuleSpecifier,
source: Option<String>,
) -> Result<ExtNodeCjsAnalysis, AnyError> {
source: Option<Cow<'a, str>>,
) -> Result<ExtNodeCjsAnalysis<'a>, AnyError> {
let source = match source {
Some(source) => source,
None => {
@ -162,7 +175,7 @@ impl CjsCodeAnalyzer for CliCjsCodeAnalyzer {
if let Ok(source_from_file) =
self.fs.read_text_file_lossy_async(path, None).await
{
source_from_file
Cow::Owned(source_from_file)
} else {
return Ok(ExtNodeCjsAnalysis::Cjs(CjsAnalysisExports {
exports: vec![],

43
cli/npm/byonm.rs
View file

@ -10,8 +10,8 @@ use deno_core::serde_json;
use deno_core::url::Url;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::ByonmNpmResolverCreateOptions;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeRequireResolver;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageReq;
use node_resolver::NpmResolver;
@ -25,30 +25,14 @@ use super::InnerCliNpmResolverRef;
use super::ResolvePkgFolderFromDenoReqError;
pub type CliByonmNpmResolverCreateOptions =
ByonmNpmResolverCreateOptions<CliDenoResolverFs>;
pub type CliByonmNpmResolver = ByonmNpmResolver<CliDenoResolverFs>;
ByonmNpmResolverCreateOptions<CliDenoResolverFs, DenoFsNodeResolverEnv>;
pub type CliByonmNpmResolver =
ByonmNpmResolver<CliDenoResolverFs, DenoFsNodeResolverEnv>;
// todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple.
#[derive(Debug)]
struct CliByonmWrapper(Arc<CliByonmNpmResolver>);
impl NodeRequireResolver for CliByonmWrapper {
fn ensure_read_permission<'a>(
&self,
permissions: &mut dyn NodePermissions,
path: &'a Path,
) -> Result<Cow<'a, Path>, AnyError> {
if !path
.components()
.any(|c| c.as_os_str().to_ascii_lowercase() == "node_modules")
{
permissions.check_read_path(path)
} else {
Ok(Cow::Borrowed(path))
}
}
}
impl NpmProcessStateProvider for CliByonmWrapper {
fn get_npm_process_state(&self) -> String {
serde_json::to_string(&NpmProcessState {
@ -67,10 +51,6 @@ impl CliNpmResolver for CliByonmNpmResolver {
self
}
fn into_require_resolver(self: Arc<Self>) -> Arc<dyn NodeRequireResolver> {
Arc::new(CliByonmWrapper(self))
}
fn into_process_state_provider(
self: Arc<Self>,
) -> Arc<dyn NpmProcessStateProvider> {
@ -100,6 +80,21 @@ impl CliNpmResolver for CliByonmNpmResolver {
.map_err(ResolvePkgFolderFromDenoReqError::Byonm)
}
fn ensure_read_permission<'a>(
&self,
permissions: &mut dyn NodePermissions,
path: &'a Path,
) -> Result<Cow<'a, Path>, AnyError> {
if !path
.components()
.any(|c| c.as_os_str().to_ascii_lowercase() == "node_modules")
{
permissions.check_read_path(path)
} else {
Ok(Cow::Borrowed(path))
}
}
fn check_state_hash(&self) -> Option<u64> {
// it is very difficult to determine the check state hash for byonm
// so we just return None to signify check caching is not supported

23
cli/npm/common.rs
View file

@ -3,6 +3,7 @@
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_npm::npm_rc::RegistryConfig;
use http::header;
@ -36,17 +37,21 @@ pub fn maybe_auth_header_for_npm_registry(
}
if username.is_some() && password.is_some() {
// The npm client does some double encoding when generating the
// bearer token value, see
// https://github.com/npm/cli/blob/780afc50e3a345feb1871a28e33fa48235bc3bd5/workspaces/config/lib/index.js#L846-L851
let pw_base64 = BASE64_STANDARD
.decode(password.unwrap())
.with_context(|| "The password in npmrc is an invalid base64 string")?;
let bearer = BASE64_STANDARD.encode(format!(
"{}:{}",
username.unwrap(),
String::from_utf8_lossy(&pw_base64)
));
return Ok(Some((
header::AUTHORIZATION,
header::HeaderValue::from_str(&format!(
"Basic {}",
BASE64_STANDARD.encode(&format!(
"{}:{}",
username.unwrap(),
password.unwrap()
))
))
.unwrap(),
header::HeaderValue::from_str(&format!("Basic {}", bearer)).unwrap(),
)));
}

14
cli/npm/managed/cache/mod.rs vendored
View file

@ -26,7 +26,7 @@ use crate::cache::CACHE_PERM;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::hard_link_dir_recursive;
mod registry_info;
pub mod registry_info;
mod tarball;
mod tarball_extract;
@ -36,7 +36,7 @@ pub use tarball::TarballCache;
/// Stores a single copy of npm packages in a cache.
#[derive(Debug)]
pub struct NpmCache {
cache_dir: NpmCacheDir,
cache_dir: Arc<NpmCacheDir>,
cache_setting: CacheSetting,
npmrc: Arc<ResolvedNpmRc>,
/// ensures a package is only downloaded once per run
@ -45,7 +45,7 @@ pub struct NpmCache {
impl NpmCache {
pub fn new(
cache_dir: NpmCacheDir,
cache_dir: Arc<NpmCacheDir>,
cache_setting: CacheSetting,
npmrc: Arc<ResolvedNpmRc>,
) -> Self {
@ -61,6 +61,10 @@ impl NpmCache {
&self.cache_setting
}
pub fn root_dir_path(&self) -> &Path {
self.cache_dir.root_dir()
}
pub fn root_dir_url(&self) -> &Url {
self.cache_dir.root_dir_url()
}
@ -152,10 +156,6 @@ impl NpmCache {
self.cache_dir.package_name_folder(name, registry_url)
}
pub fn root_folder(&self) -> PathBuf {
self.cache_dir.root_dir().to_owned()
}
pub fn resolve_package_folder_id_from_specifier(
&self,
specifier: &ModuleSpecifier,

68
cli/npm/managed/cache/registry_info.rs vendored
View file

@ -84,7 +84,7 @@ impl RegistryInfoDownloader {
self.load_package_info_inner(name).await.with_context(|| {
format!(
"Error getting response at {} for package \"{}\"",
self.get_package_url(name),
get_package_url(&self.npmrc, name),
name
)
})
@ -190,7 +190,7 @@ impl RegistryInfoDownloader {
fn create_load_future(self: &Arc<Self>, name: &str) -> LoadFuture {
let downloader = self.clone();
let package_url = self.get_package_url(name);
let package_url = get_package_url(&self.npmrc, name);
let registry_config = self.npmrc.get_registry_config(name);
let maybe_auth_header =
match maybe_auth_header_for_npm_registry(registry_config) {
@ -202,10 +202,13 @@ impl RegistryInfoDownloader {
let guard = self.progress_bar.update(package_url.as_str());
let name = name.to_string();
async move {
let maybe_bytes = downloader
.http_client_provider
.get_or_create()?
.download_with_progress(package_url, maybe_auth_header, &guard)
let client = downloader.http_client_provider.get_or_create()?;
let maybe_bytes = client
.download_with_progress_and_retries(
package_url,
maybe_auth_header,
&guard,
)
.await?;
match maybe_bytes {
Some(bytes) => {
@ -236,25 +239,36 @@ impl RegistryInfoDownloader {
.map(|r| r.map_err(Arc::new))
.boxed_local()
}
fn get_package_url(&self, name: &str) -> Url {
let registry_url = self.npmrc.get_registry_url(name);
// list of all characters used in npm packages:
// !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~
const ASCII_SET: percent_encoding::AsciiSet =
percent_encoding::NON_ALPHANUMERIC
.remove(b'!')
.remove(b'\'')
.remove(b'(')
.remove(b')')
.remove(b'*')
.remove(b'-')
.remove(b'.')
.remove(b'/')
.remove(b'@')
.remove(b'_')
.remove(b'~');
let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET);
registry_url.join(&name.to_string()).unwrap()
}
}
pub fn get_package_url(npmrc: &ResolvedNpmRc, name: &str) -> Url {
let registry_url = npmrc.get_registry_url(name);
// The '/' character in scoped package names "@scope/name" must be
// encoded for older third party registries. Newer registries and
// npm itself support both ways
// - encoded: https://registry.npmjs.org/@rollup%2fplugin-json
// - non-ecoded: https://registry.npmjs.org/@rollup/plugin-json
// To support as many third party registries as possible we'll
// always encode the '/' character.
// list of all characters used in npm packages:
// !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~
const ASCII_SET: percent_encoding::AsciiSet =
percent_encoding::NON_ALPHANUMERIC
.remove(b'!')
.remove(b'\'')
.remove(b'(')
.remove(b')')
.remove(b'*')
.remove(b'-')
.remove(b'.')
.remove(b'@')
.remove(b'_')
.remove(b'~');
let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET);
registry_url
// Ensure that scoped package name percent encoding is lower cased
// to match npm.
.join(&name.to_string().replace("%2F", "%2f"))
.unwrap()
}

2
cli/npm/managed/cache/tarball.rs vendored
View file

@ -172,7 +172,7 @@ impl TarballCache {
let guard = tarball_cache.progress_bar.update(&dist.tarball);
let result = tarball_cache.http_client_provider
.get_or_create()?
.download_with_progress(tarball_uri, maybe_auth_header, &guard)
.download_with_progress_and_retries(tarball_uri, maybe_auth_header, &guard)
.await;
let maybe_bytes = match result {
Ok(maybe_bytes) => maybe_bytes,

113
cli/npm/managed/mod.rs
View file

@ -12,6 +12,7 @@ use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::registry::NpmRegistryApi;
@ -24,12 +25,12 @@ use deno_npm::NpmSystemInfo;
use deno_runtime::colors;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeRequireResolver;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageFolderResolveIoError;
use node_resolver::InNpmPackageChecker;
use node_resolver::NpmResolver;
use resolution::AddPkgReqsResult;
@ -38,7 +39,7 @@ use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::args::PackageJsonDepValueParseWithLocationError;
use crate::cache::FastInsecureHasher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
@ -55,7 +56,7 @@ use super::CliNpmResolver;
use super::InnerCliNpmResolverRef;
use super::ResolvePkgFolderFromDenoReqError;
mod cache;
pub mod cache;
mod registry;
mod resolution;
mod resolvers;
@ -65,12 +66,12 @@ pub enum CliNpmResolverManagedSnapshotOption {
Specified(Option<ValidSerializedNpmResolutionSnapshot>),
}
pub struct CliNpmResolverManagedCreateOptions {
pub struct CliManagedNpmResolverCreateOptions {
pub snapshot: CliNpmResolverManagedSnapshotOption,
pub maybe_lockfile: Option<Arc<CliLockfile>>,
pub fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
pub http_client_provider: Arc<crate::http_util::HttpClientProvider>,
pub npm_global_cache_dir: PathBuf,
pub npm_cache_dir: Arc<NpmCacheDir>,
pub cache_setting: crate::args::CacheSetting,
pub text_only_progress_bar: crate::util::progress_bar::ProgressBar,
pub maybe_node_modules_path: Option<PathBuf>,
@ -81,7 +82,7 @@ pub struct CliNpmResolverManagedCreateOptions {
}
pub async fn create_managed_npm_resolver_for_lsp(
options: CliNpmResolverManagedCreateOptions,
options: CliManagedNpmResolverCreateOptions,
) -> Arc<dyn CliNpmResolver> {
let npm_cache = create_cache(&options);
let npm_api = create_api(&options, npm_cache.clone());
@ -114,7 +115,7 @@ pub async fn create_managed_npm_resolver_for_lsp(
}
pub async fn create_managed_npm_resolver(
options: CliNpmResolverManagedCreateOptions,
options: CliManagedNpmResolverCreateOptions,
) -> Result<Arc<dyn CliNpmResolver>, AnyError> {
let npm_cache = create_cache(&options);
let npm_api = create_api(&options, npm_cache.clone());
@ -188,20 +189,16 @@ fn create_inner(
))
}
fn create_cache(options: &CliNpmResolverManagedCreateOptions) -> Arc<NpmCache> {
fn create_cache(options: &CliManagedNpmResolverCreateOptions) -> Arc<NpmCache> {
Arc::new(NpmCache::new(
NpmCacheDir::new(
&DenoCacheEnvFsAdapter(options.fs.as_ref()),
options.npm_global_cache_dir.clone(),
options.npmrc.get_all_known_registries_urls(),
),
options.npm_cache_dir.clone(),
options.cache_setting.clone(),
options.npmrc.clone(),
))
}
fn create_api(
options: &CliNpmResolverManagedCreateOptions,
options: &CliManagedNpmResolverCreateOptions,
npm_cache: Arc<NpmCache>,
) -> Arc<CliNpmRegistryApi> {
Arc::new(CliNpmRegistryApi::new(
@ -258,6 +255,35 @@ async fn snapshot_from_lockfile(
Ok(snapshot)
}
#[derive(Debug)]
struct ManagedInNpmPackageChecker {
root_dir: Url,
}
impl InNpmPackageChecker for ManagedInNpmPackageChecker {
fn in_npm_package(&self, specifier: &Url) -> bool {
specifier.as_ref().starts_with(self.root_dir.as_str())
}
}
pub struct CliManagedInNpmPkgCheckerCreateOptions<'a> {
pub root_cache_dir_url: &'a Url,
pub maybe_node_modules_path: Option<&'a Path>,
}
pub fn create_managed_in_npm_pkg_checker(
options: CliManagedInNpmPkgCheckerCreateOptions,
) -> Arc<dyn InNpmPackageChecker> {
let root_dir = match options.maybe_node_modules_path {
Some(node_modules_folder) => {
deno_path_util::url_from_directory_path(node_modules_folder).unwrap()
}
None => options.root_cache_dir_url.clone(),
};
debug_assert!(root_dir.as_str().ends_with('/'));
Arc::new(ManagedInNpmPackageChecker { root_dir })
}
/// An npm resolver where the resolution is managed by Deno rather than
/// the user bringing their own node_modules (BYONM) on the file system.
pub struct ManagedCliNpmResolver {
@ -480,19 +506,24 @@ impl ManagedCliNpmResolver {
self.resolution.resolve_pkg_id_from_pkg_req(req)
}
pub fn ensure_no_pkg_json_dep_errors(&self) -> Result<(), AnyError> {
pub fn ensure_no_pkg_json_dep_errors(
&self,
) -> Result<(), Box<PackageJsonDepValueParseWithLocationError>> {
for err in self.npm_install_deps_provider.pkg_json_dep_errors() {
match err {
match &err.source {
deno_package_json::PackageJsonDepValueParseError::VersionReq(_) => {
return Err(
AnyError::from(err.clone())
.context("Failed to install from package.json"),
);
return Err(Box::new(err.clone()));
}
deno_package_json::PackageJsonDepValueParseError::Unsupported {
..
} => {
log::warn!("{} {} in package.json", colors::yellow("Warning"), err)
// only warn for this one
log::warn!(
"{} {}\n at {}",
colors::yellow("Warning"),
err.source,
err.location,
)
}
}
}
@ -549,8 +580,16 @@ impl ManagedCliNpmResolver {
.map_err(|err| err.into())
}
pub fn global_cache_root_folder(&self) -> PathBuf {
self.npm_cache.root_folder()
pub fn maybe_node_modules_path(&self) -> Option<&Path> {
self.fs_resolver.node_modules_path()
}
pub fn global_cache_root_path(&self) -> &Path {
self.npm_cache.root_dir_path()
}
pub fn global_cache_root_url(&self) -> &Url {
self.npm_cache.root_dir_url()
}
}
@ -585,22 +624,6 @@ impl NpmResolver for ManagedCliNpmResolver {
log::debug!("Resolved {} from {} to {}", name, referrer, path.display());
Ok(path)
}
fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool {
let root_dir_url = self.fs_resolver.root_dir_url();
debug_assert!(root_dir_url.as_str().ends_with('/'));
specifier.as_ref().starts_with(root_dir_url.as_str())
}
}
impl NodeRequireResolver for ManagedCliNpmResolver {
fn ensure_read_permission<'a>(
&self,
permissions: &mut dyn NodePermissions,
path: &'a Path,
) -> Result<Cow<'a, Path>, AnyError> {
self.fs_resolver.ensure_read_permission(permissions, path)
}
}
impl NpmProcessStateProvider for ManagedCliNpmResolver {
@ -617,10 +640,6 @@ impl CliNpmResolver for ManagedCliNpmResolver {
self
}
fn into_require_resolver(self: Arc<Self>) -> Arc<dyn NodeRequireResolver> {
self
}
fn into_process_state_provider(
self: Arc<Self>,
) -> Arc<dyn NpmProcessStateProvider> {
@ -681,6 +700,14 @@ impl CliNpmResolver for ManagedCliNpmResolver {
.map_err(ResolvePkgFolderFromDenoReqError::Managed)
}
fn ensure_read_permission<'a>(
&self,
permissions: &mut dyn NodePermissions,
path: &'a Path,
) -> Result<Cow<'a, Path>, AnyError> {
self.fs_resolver.ensure_read_permission(permissions, path)
}
fn check_state_hash(&self) -> Option<u64> {
// We could go further and check all the individual
// npm packages, but that's probably overkill.

4
cli/npm/managed/resolvers/common.rs
View file

@ -17,7 +17,6 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::futures::StreamExt;
use deno_core::url::Url;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
@ -30,9 +29,6 @@ use crate::npm::managed::cache::TarballCache;
/// Part of the resolution that interacts with the file system.
#[async_trait(?Send)]
pub trait NpmPackageFsResolver: Send + Sync {
/// Specifier for the root directory.
fn root_dir_url(&self) -> &Url;
/// The local node_modules folder if it is applicable to the implementation.
fn node_modules_path(&self) -> Option<&Path>;

37
cli/npm/managed/resolvers/common/lifecycle_scripts.rs
View file

@ -2,6 +2,8 @@
use super::bin_entries::BinEntries;
use crate::args::LifecycleScriptsConfig;
use crate::task_runner::TaskStdio;
use crate::util::progress_bar::ProgressBar;
use deno_core::anyhow::Context;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_runtime::deno_io::FromRawIoHandle;
@ -148,6 +150,7 @@ impl<'a> LifecycleScripts<'a> {
snapshot: &NpmResolutionSnapshot,
packages: &[NpmResolutionPackage],
root_node_modules_dir_path: Option<&Path>,
progress_bar: &ProgressBar,
) -> Result<(), AnyError> {
self.warn_not_run_scripts()?;
let get_package_path =
@ -201,7 +204,15 @@ impl<'a> LifecycleScripts<'a> {
{
continue;
}
let exit_code = crate::task_runner::run_task(
let _guard = progress_bar.update_with_prompt(
crate::util::progress_bar::ProgressMessagePrompt::Initialize,
&format!("{}: running '{script_name}' script", package.id.nv),
);
let crate::task_runner::TaskResult {
exit_code,
stderr,
stdout,
} = crate::task_runner::run_task(
crate::task_runner::RunTaskOptions {
task_name: script_name,
script,
@ -211,15 +222,37 @@ impl<'a> LifecycleScripts<'a> {
init_cwd,
argv: &[],
root_node_modules_dir: root_node_modules_dir_path,
stdio: Some(crate::task_runner::TaskIo {
stderr: TaskStdio::piped(),
stdout: TaskStdio::piped(),
}),
},
)
.await?;
let stdout = stdout.unwrap();
let stderr = stderr.unwrap();
if exit_code != 0 {
log::warn!(
"error: script '{}' in '{}' failed with exit code {}",
"error: script '{}' in '{}' failed with exit code {}{}{}",
script_name,
package.id.nv,
exit_code,
if !stdout.trim_ascii().is_empty() {
format!(
"\nstdout:\n{}\n",
String::from_utf8_lossy(&stdout).trim()
)
} else {
String::new()
},
if !stderr.trim_ascii().is_empty() {
format!(
"\nstderr:\n{}\n",
String::from_utf8_lossy(&stderr).trim()
)
} else {
String::new()
},
);
failed_packages.push(&package.id.nv);
// assume if earlier script fails, later ones will fail too

7
cli/npm/managed/resolvers/global.rs
View file

@ -11,7 +11,6 @@ use crate::colors;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_core::url::Url;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
@ -56,7 +55,7 @@ impl GlobalNpmPackageResolver {
Self {
registry_read_permission_checker: RegistryReadPermissionChecker::new(
fs,
cache.root_folder(),
cache.root_dir_path().to_path_buf(),
),
cache,
tarball_cache,
@ -69,10 +68,6 @@ impl GlobalNpmPackageResolver {
#[async_trait(?Send)]
impl NpmPackageFsResolver for GlobalNpmPackageResolver {
fn root_dir_url(&self) -> &Url {
self.cache.root_dir_url()
}
fn node_modules_path(&self) -> Option<&Path> {
None
}

5
cli/npm/managed/resolvers/local.rs
View file

@ -155,10 +155,6 @@ impl LocalNpmPackageResolver {
#[async_trait(?Send)]
impl NpmPackageFsResolver for LocalNpmPackageResolver {
fn root_dir_url(&self) -> &Url {
&self.root_node_modules_url
}
fn node_modules_path(&self) -> Option<&Path> {
Some(self.root_node_modules_path.as_ref())
}
@ -713,6 +709,7 @@ async fn sync_resolution_with_fs(
snapshot,
&package_partitions.packages,
Some(root_node_modules_dir_path),
progress_bar,
)
.await?;

71
cli/npm/mod.rs
View file

@ -4,30 +4,37 @@ mod byonm;
mod common;
mod managed;
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use common::maybe_auth_header_for_npm_registry;
use dashmap::DashMap;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_resolver::npm::ByonmInNpmPackageChecker;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::ByonmResolvePkgFolderFromDenoReqError;
use deno_runtime::deno_node::NodeRequireResolver;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use managed::cache::registry_info::get_package_url;
use managed::create_managed_in_npm_pkg_checker;
use node_resolver::InNpmPackageChecker;
use node_resolver::NpmResolver;
use thiserror::Error;
use crate::args::npm_registry_url;
use crate::file_fetcher::FileFetcher;
pub use self::byonm::CliByonmNpmResolver;
pub use self::byonm::CliByonmNpmResolverCreateOptions;
pub use self::managed::CliNpmResolverManagedCreateOptions;
pub use self::managed::CliManagedInNpmPkgCheckerCreateOptions;
pub use self::managed::CliManagedNpmResolverCreateOptions;
pub use self::managed::CliNpmResolverManagedSnapshotOption;
pub use self::managed::ManagedCliNpmResolver;
@ -40,7 +47,7 @@ pub enum ResolvePkgFolderFromDenoReqError {
}
pub enum CliNpmResolverCreateOptions {
Managed(CliNpmResolverManagedCreateOptions),
Managed(CliManagedNpmResolverCreateOptions),
Byonm(CliByonmNpmResolverCreateOptions),
}
@ -66,6 +73,22 @@ pub async fn create_cli_npm_resolver(
}
}
pub enum CreateInNpmPkgCheckerOptions<'a> {
Managed(CliManagedInNpmPkgCheckerCreateOptions<'a>),
Byonm,
}
pub fn create_in_npm_pkg_checker(
options: CreateInNpmPkgCheckerOptions,
) -> Arc<dyn InNpmPackageChecker> {
match options {
CreateInNpmPkgCheckerOptions::Managed(options) => {
create_managed_in_npm_pkg_checker(options)
}
CreateInNpmPkgCheckerOptions::Byonm => Arc::new(ByonmInNpmPackageChecker),
}
}
pub enum InnerCliNpmResolverRef<'a> {
Managed(&'a ManagedCliNpmResolver),
#[allow(dead_code)]
@ -74,7 +97,6 @@ pub enum InnerCliNpmResolverRef<'a> {
pub trait CliNpmResolver: NpmResolver {
fn into_npm_resolver(self: Arc<Self>) -> Arc<dyn NpmResolver>;
fn into_require_resolver(self: Arc<Self>) -> Arc<dyn NodeRequireResolver>;
fn into_process_state_provider(
self: Arc<Self>,
) -> Arc<dyn NpmProcessStateProvider>;
@ -105,6 +127,12 @@ pub trait CliNpmResolver: NpmResolver {
referrer: &ModuleSpecifier,
) -> Result<PathBuf, ResolvePkgFolderFromDenoReqError>;
fn ensure_read_permission<'a>(
&self,
permissions: &mut dyn NodePermissions,
path: &'a Path,
) -> Result<Cow<'a, Path>, AnyError>;
/// Returns a hash returning the state of the npm resolver
/// or `None` if the state currently can't be determined.
fn check_state_hash(&self) -> Option<u64>;
@ -115,14 +143,19 @@ pub struct NpmFetchResolver {
nv_by_req: DashMap<PackageReq, Option<PackageNv>>,
info_by_name: DashMap<String, Option<Arc<NpmPackageInfo>>>,
file_fetcher: Arc<FileFetcher>,
npmrc: Arc<ResolvedNpmRc>,
}
impl NpmFetchResolver {
pub fn new(file_fetcher: Arc<FileFetcher>) -> Self {
pub fn new(
file_fetcher: Arc<FileFetcher>,
npmrc: Arc<ResolvedNpmRc>,
) -> Self {
Self {
nv_by_req: Default::default(),
info_by_name: Default::default(),
file_fetcher,
npmrc,
}
}
@ -157,11 +190,21 @@ impl NpmFetchResolver {
return info.value().clone();
}
let fetch_package_info = || async {
let info_url = npm_registry_url().join(name).ok()?;
let info_url = get_package_url(&self.npmrc, name);
let file_fetcher = self.file_fetcher.clone();
let registry_config = self.npmrc.get_registry_config(name);
// TODO(bartlomieju): this should error out, not use `.ok()`.
let maybe_auth_header =
maybe_auth_header_for_npm_registry(registry_config).ok()?;
// spawn due to the lsp's `Send` requirement
let file = deno_core::unsync::spawn(async move {
file_fetcher.fetch_bypass_permissions(&info_url).await.ok()
file_fetcher
.fetch_bypass_permissions_with_maybe_auth(
&info_url,
maybe_auth_header,
)
.await
.ok()
})
.await
.ok()??;
@ -172,3 +215,15 @@ impl NpmFetchResolver {
info
}
}
pub const NPM_CONFIG_USER_AGENT_ENV_VAR: &str = "npm_config_user_agent";
pub fn get_npm_config_user_agent() -> String {
format!(
"deno/{} npm/? deno/{} {} {}",
env!("CARGO_PKG_VERSION"),
env!("CARGO_PKG_VERSION"),
std::env::consts::OS,
std::env::consts::ARCH
)
}

368
cli/resolver.rs
View file

@ -4,6 +4,7 @@ use async_trait::async_trait;
use dashmap::DashMap;
use dashmap::DashSet;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_config::workspace::MappedResolution;
use deno_config::workspace::MappedResolutionDiagnostic;
use deno_config::workspace::MappedResolutionError;
@ -11,6 +12,7 @@ use deno_config::workspace::WorkspaceResolver;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::url::Url;
use deno_core::ModuleSourceCode;
use deno_core::ModuleSpecifier;
use deno_graph::source::ResolutionMode;
@ -29,6 +31,7 @@ use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::is_builtin_node_module;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageReq;
use node_resolver::errors::ClosestPkgJsonError;
@ -38,15 +41,16 @@ use node_resolver::errors::PackageFolderResolveErrorKind;
use node_resolver::errors::PackageFolderResolveIoError;
use node_resolver::errors::PackageNotFoundError;
use node_resolver::errors::PackageResolveErrorKind;
use node_resolver::errors::UrlToNodeResolutionError;
use node_resolver::errors::PackageSubpathResolveError;
use node_resolver::InNpmPackageChecker;
use node_resolver::NodeModuleKind;
use node_resolver::NodeResolution;
use node_resolver::NodeResolutionMode;
use node_resolver::PackageJson;
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use thiserror::Error;
use crate::args::JsxImportSourceConfig;
use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS;
@ -54,6 +58,7 @@ use crate::node::CliNodeCodeTranslator;
use crate::npm::CliNpmResolver;
use crate::npm::InnerCliNpmResolverRef;
use crate::util::sync::AtomicFlag;
use crate::util::text_encoding::from_utf8_lossy_owned;
pub struct ModuleCodeStringSource {
pub code: ModuleSourceCode,
@ -103,36 +108,32 @@ impl deno_resolver::fs::DenoResolverFs for CliDenoResolverFs {
#[derive(Debug)]
pub struct CliNodeResolver {
cjs_resolutions: Arc<CjsResolutionStore>,
cjs_tracker: Arc<CjsTracker>,
fs: Arc<dyn deno_fs::FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
node_resolver: Arc<NodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
}
impl CliNodeResolver {
pub fn new(
cjs_resolutions: Arc<CjsResolutionStore>,
cjs_tracker: Arc<CjsTracker>,
fs: Arc<dyn deno_fs::FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
node_resolver: Arc<NodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
) -> Self {
Self {
cjs_resolutions,
cjs_tracker,
fs,
in_npm_pkg_checker,
node_resolver,
npm_resolver,
}
}
pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool {
self.npm_resolver.in_npm_package(specifier)
}
pub fn get_closest_package_json(
&self,
referrer: &ModuleSpecifier,
) -> Result<Option<Arc<PackageJson>>, ClosestPkgJsonError> {
self.node_resolver.get_closest_package_json(referrer)
self.in_npm_pkg_checker.in_npm_package(specifier)
}
pub fn resolve_if_for_npm_pkg(
@ -152,8 +153,7 @@ impl CliNodeResolver {
| NodeResolveErrorKind::UnsupportedEsmUrlScheme(_)
| NodeResolveErrorKind::DataUrlReferrer(_)
| NodeResolveErrorKind::TypesNotFound(_)
| NodeResolveErrorKind::FinalizeResolution(_)
| NodeResolveErrorKind::UrlToNodeResolution(_) => Err(err.into()),
| NodeResolveErrorKind::FinalizeResolution(_) => Err(err.into()),
NodeResolveErrorKind::PackageResolve(err) => {
let err = err.into_kind();
match err {
@ -215,7 +215,11 @@ impl CliNodeResolver {
referrer: &ModuleSpecifier,
mode: NodeResolutionMode,
) -> Result<NodeResolution, NodeResolveError> {
let referrer_kind = if self.cjs_resolutions.contains(referrer) {
let referrer_kind = if self
.cjs_tracker
.is_maybe_cjs(referrer, MediaType::from_specifier(referrer))
.map_err(|err| NodeResolveErrorKind::PackageResolve(err.into()))?
{
NodeModuleKind::Cjs
} else {
NodeModuleKind::Esm
@ -225,7 +229,7 @@ impl CliNodeResolver {
self
.node_resolver
.resolve(specifier, referrer, referrer_kind, mode)?;
Ok(self.handle_node_resolution(res))
Ok(res)
}
pub fn resolve_req_reference(
@ -233,7 +237,7 @@ impl CliNodeResolver {
req_ref: &NpmPackageReqReference,
referrer: &ModuleSpecifier,
mode: NodeResolutionMode,
) -> Result<NodeResolution, AnyError> {
) -> Result<ModuleSpecifier, AnyError> {
self.resolve_req_with_sub_path(
req_ref.req(),
req_ref.sub_path(),
@ -248,7 +252,7 @@ impl CliNodeResolver {
sub_path: Option<&str>,
referrer: &ModuleSpecifier,
mode: NodeResolutionMode,
) -> Result<NodeResolution, AnyError> {
) -> Result<ModuleSpecifier, AnyError> {
let package_folder = self
.npm_resolver
.resolve_pkg_folder_from_deno_module_req(req, referrer)?;
@ -259,7 +263,7 @@ impl CliNodeResolver {
mode,
);
match resolution_result {
Ok(resolution) => Ok(resolution),
Ok(url) => Ok(url),
Err(err) => {
if self.npm_resolver.as_byonm().is_some() {
let package_json_path = package_folder.join("package.json");
@ -270,7 +274,7 @@ impl CliNodeResolver {
));
}
}
Err(err)
Err(err.into())
}
}
}
@ -281,16 +285,13 @@ impl CliNodeResolver {
sub_path: Option<&str>,
maybe_referrer: Option<&ModuleSpecifier>,
mode: NodeResolutionMode,
) -> Result<NodeResolution, AnyError> {
let res = self
.node_resolver
.resolve_package_subpath_from_deno_module(
package_folder,
sub_path,
maybe_referrer,
mode,
)?;
Ok(self.handle_node_resolution(res))
) -> Result<ModuleSpecifier, PackageSubpathResolveError> {
self.node_resolver.resolve_package_subpath_from_deno_module(
package_folder,
sub_path,
maybe_referrer,
mode,
)
}
pub fn handle_if_in_node_modules(
@ -305,74 +306,42 @@ impl CliNodeResolver {
// so canoncalize then check if it's in the node_modules directory.
// If so, check if we need to store this specifier as being a CJS
// resolution.
let specifier =
crate::node::resolve_specifier_into_node_modules(specifier);
if self.in_npm_package(&specifier) {
let resolution =
self.node_resolver.url_to_node_resolution(specifier)?;
if let NodeResolution::CommonJs(specifier) = &resolution {
self.cjs_resolutions.insert(specifier.clone());
}
return Ok(Some(resolution.into_url()));
}
let specifier = crate::node::resolve_specifier_into_node_modules(
specifier,
self.fs.as_ref(),
);
return Ok(Some(specifier));
}
Ok(None)
}
pub fn url_to_node_resolution(
&self,
specifier: ModuleSpecifier,
) -> Result<NodeResolution, UrlToNodeResolutionError> {
self.node_resolver.url_to_node_resolution(specifier)
}
fn handle_node_resolution(
&self,
resolution: NodeResolution,
) -> NodeResolution {
if let NodeResolution::CommonJs(specifier) = &resolution {
// remember that this was a common js resolution
self.cjs_resolutions.insert(specifier.clone());
}
resolution
}
}
#[derive(Debug, Error)]
#[error("{media_type} files are not supported in npm packages: {specifier}")]
pub struct NotSupportedKindInNpmError {
pub media_type: MediaType,
pub specifier: Url,
}
// todo(dsherret): move to module_loader.rs (it seems to be here due to use in standalone)
#[derive(Clone)]
pub struct NpmModuleLoader {
cjs_resolutions: Arc<CjsResolutionStore>,
node_code_translator: Arc<CliNodeCodeTranslator>,
cjs_tracker: Arc<CjsTracker>,
fs: Arc<dyn deno_fs::FileSystem>,
node_resolver: Arc<CliNodeResolver>,
node_code_translator: Arc<CliNodeCodeTranslator>,
}
impl NpmModuleLoader {
pub fn new(
cjs_resolutions: Arc<CjsResolutionStore>,
node_code_translator: Arc<CliNodeCodeTranslator>,
cjs_tracker: Arc<CjsTracker>,
fs: Arc<dyn deno_fs::FileSystem>,
node_resolver: Arc<CliNodeResolver>,
node_code_translator: Arc<CliNodeCodeTranslator>,
) -> Self {
Self {
cjs_resolutions,
cjs_tracker,
node_code_translator,
fs,
node_resolver,
}
}
pub async fn load_if_in_npm_package(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
) -> Option<Result<ModuleCodeStringSource, AnyError>> {
if self.node_resolver.in_npm_package(specifier)
|| (specifier.scheme() == "file" && specifier.path().ends_with(".cjs"))
{
Some(self.load(specifier, maybe_referrer).await)
} else {
None
}
}
@ -418,27 +387,30 @@ impl NpmModuleLoader {
}
})?;
let code = if self.cjs_resolutions.contains(specifier)
|| (specifier.scheme() == "file" && specifier.path().ends_with(".cjs"))
{
let media_type = MediaType::from_specifier(specifier);
if media_type.is_emittable() {
return Err(AnyError::from(NotSupportedKindInNpmError {
media_type,
specifier: specifier.clone(),
}));
}
let code = if self.cjs_tracker.is_maybe_cjs(specifier, media_type)? {
// translate cjs to esm if it's cjs and inject node globals
let code = match String::from_utf8_lossy(&code) {
Cow::Owned(code) => code,
// SAFETY: `String::from_utf8_lossy` guarantees that the result is valid
// UTF-8 if `Cow::Borrowed` is returned.
Cow::Borrowed(_) => unsafe { String::from_utf8_unchecked(code) },
};
let code = from_utf8_lossy_owned(code);
ModuleSourceCode::String(
self
.node_code_translator
.translate_cjs_to_esm(specifier, Some(code))
.translate_cjs_to_esm(specifier, Some(Cow::Owned(code)))
.await?
.into_owned()
.into(),
)
} else {
// esm and json code is untouched
ModuleSourceCode::Bytes(code.into_boxed_slice().into())
};
Ok(ModuleCodeStringSource {
code,
found_url: specifier.clone(),
@ -447,17 +419,165 @@ impl NpmModuleLoader {
}
}
/// Keeps track of what module specifiers were resolved as CJS.
#[derive(Debug, Default)]
pub struct CjsResolutionStore(DashSet<ModuleSpecifier>);
pub struct CjsTrackerOptions {
pub unstable_detect_cjs: bool,
}
impl CjsResolutionStore {
pub fn contains(&self, specifier: &ModuleSpecifier) -> bool {
self.0.contains(specifier)
/// Keeps track of what module specifiers were resolved as CJS.
///
/// Modules that are `.js` or `.ts` are only known to be CJS or
/// ESM after they're loaded based on their contents. So these files
/// will be "maybe CJS" until they're loaded.
#[derive(Debug)]
pub struct CjsTracker {
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
pkg_json_resolver: Arc<PackageJsonResolver>,
unstable_detect_cjs: bool,
known: DashMap<ModuleSpecifier, ModuleKind>,
}
impl CjsTracker {
pub fn new(
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
pkg_json_resolver: Arc<PackageJsonResolver>,
options: CjsTrackerOptions,
) -> Self {
Self {
in_npm_pkg_checker,
pkg_json_resolver,
unstable_detect_cjs: options.unstable_detect_cjs,
known: Default::default(),
}
}
pub fn insert(&self, specifier: ModuleSpecifier) {
self.0.insert(specifier);
/// Checks whether the file might be treated as CJS, but it's not for sure
/// yet because the source hasn't been loaded to see whether it contains
/// imports or exports.
pub fn is_maybe_cjs(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
) -> Result<bool, ClosestPkgJsonError> {
self.treat_as_cjs_with_is_script(specifier, media_type, None)
}
/// Gets whether the file is CJS. If true, this is for sure
/// cjs because `is_script` is provided.
///
/// `is_script` should be `true` when the contents of the file at the
/// provided specifier are known to be a script and not an ES module.
pub fn is_cjs_with_known_is_script(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
is_script: bool,
) -> Result<bool, ClosestPkgJsonError> {
self.treat_as_cjs_with_is_script(specifier, media_type, Some(is_script))
}
fn treat_as_cjs_with_is_script(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
is_script: Option<bool>,
) -> Result<bool, ClosestPkgJsonError> {
let kind = match self
.get_known_kind_with_is_script(specifier, media_type, is_script)
{
Some(kind) => kind,
None => self.check_based_on_pkg_json(specifier)?,
};
Ok(kind.is_cjs())
}
pub fn get_known_kind(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
) -> Option<ModuleKind> {
self.get_known_kind_with_is_script(specifier, media_type, None)
}
fn get_known_kind_with_is_script(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
is_script: Option<bool>,
) -> Option<ModuleKind> {
if specifier.scheme() != "file" {
return Some(ModuleKind::Esm);
}
match media_type {
MediaType::Mts | MediaType::Mjs | MediaType::Dmts => Some(ModuleKind::Esm),
MediaType::Cjs | MediaType::Cts | MediaType::Dcts => Some(ModuleKind::Cjs),
MediaType::Dts => {
// dts files are always determined based on the package.json because
// they contain imports/exports even when considered CJS
if let Some(value) = self.known.get(specifier).map(|v| *v) {
Some(value)
} else {
let value = self.check_based_on_pkg_json(specifier).ok();
if let Some(value) = value {
self.known.insert(specifier.clone(), value);
}
Some(value.unwrap_or(ModuleKind::Esm))
}
}
MediaType::Wasm |
MediaType::Json => Some(ModuleKind::Esm),
MediaType::JavaScript
| MediaType::Jsx
| MediaType::TypeScript
| MediaType::Tsx
// treat these as unknown
| MediaType::Css
| MediaType::SourceMap
| MediaType::Unknown => {
if let Some(value) = self.known.get(specifier).map(|v| *v) {
if value.is_cjs() && is_script == Some(false) {
// we now know this is actually esm
self.known.insert(specifier.clone(), ModuleKind::Esm);
Some(ModuleKind::Esm)
} else {
Some(value)
}
} else if is_script == Some(false) {
// we know this is esm
self.known.insert(specifier.clone(), ModuleKind::Esm);
Some(ModuleKind::Esm)
} else {
None
}
}
}
}
fn check_based_on_pkg_json(
&self,
specifier: &ModuleSpecifier,
) -> Result<ModuleKind, ClosestPkgJsonError> {
if self.in_npm_pkg_checker.in_npm_package(specifier) {
if let Some(pkg_json) =
self.pkg_json_resolver.get_closest_package_json(specifier)?
{
let is_file_location_cjs = pkg_json.typ != "module";
Ok(ModuleKind::from_is_cjs(is_file_location_cjs))
} else {
Ok(ModuleKind::Cjs)
}
} else if self.unstable_detect_cjs {
if let Some(pkg_json) =
self.pkg_json_resolver.get_closest_package_json(specifier)?
{
let is_cjs_type = pkg_json.typ == "commonjs";
Ok(ModuleKind::from_is_cjs(is_cjs_type))
} else {
Ok(ModuleKind::Esm)
}
} else {
Ok(ModuleKind::Esm)
}
}
}
@ -641,8 +761,7 @@ impl Resolver for CliGraphResolver {
Some(referrer),
to_node_mode(mode),
)
.map_err(ResolveError::Other)
.map(|res| res.into_url()),
.map_err(|e| ResolveError::Other(e.into())),
MappedResolution::PackageJson {
dep_result,
alias,
@ -673,19 +792,17 @@ impl Resolver for CliGraphResolver {
)
.map_err(|e| ResolveError::Other(e.into()))
.and_then(|pkg_folder| {
Ok(
self
.node_resolver
.as_ref()
.unwrap()
.resolve_package_sub_path_from_deno_module(
pkg_folder,
sub_path.as_deref(),
Some(referrer),
to_node_mode(mode),
)?
.into_url(),
)
self
.node_resolver
.as_ref()
.unwrap()
.resolve_package_sub_path_from_deno_module(
pkg_folder,
sub_path.as_deref(),
Some(referrer),
to_node_mode(mode),
)
.map_err(|e| ResolveError::Other(e.into()))
}),
})
}
@ -725,23 +842,20 @@ impl Resolver for CliGraphResolver {
npm_req_ref.req(),
)
{
return Ok(
node_resolver
.resolve_package_sub_path_from_deno_module(
pkg_folder,
npm_req_ref.sub_path(),
Some(referrer),
to_node_mode(mode),
)?
.into_url(),
);
return node_resolver
.resolve_package_sub_path_from_deno_module(
pkg_folder,
npm_req_ref.sub_path(),
Some(referrer),
to_node_mode(mode),
)
.map_err(|e| ResolveError::Other(e.into()));
}
// do npm resolution for byonm
if is_byonm {
return node_resolver
.resolve_req_reference(&npm_req_ref, referrer, to_node_mode(mode))
.map(|res| res.into_url())
.map_err(|err| err.into());
}
}
@ -759,9 +873,7 @@ impl Resolver for CliGraphResolver {
.map_err(ResolveError::Other)?;
if let Some(res) = maybe_resolution {
match res {
NodeResolution::Esm(url) | NodeResolution::CommonJs(url) => {
return Ok(url)
}
NodeResolution::Module(url) => return Ok(url),
NodeResolution::BuiltIn(_) => {
// don't resolve bare specifiers for built-in modules via node resolution
}

8
cli/schemas/config-file.v1.json
View file

@ -291,7 +291,7 @@
"type": "array",
"description": "List of tag names that will be run. Empty list disables all tags and will only use rules from `include`.",
"items": {
"type": "string"
"$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/tags.v1.json"
},
"minItems": 0,
"uniqueItems": true
@ -300,7 +300,7 @@
"type": "array",
"description": "List of rule names that will be excluded from configured tag sets. If the same rule is in `include` it will be run.",
"items": {
"type": "string"
"$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json"
},
"minItems": 0,
"uniqueItems": true
@ -309,7 +309,7 @@
"type": "array",
"description": "List of rule names that will be run. Even if the same rule is in `exclude` it will be run.",
"items": {
"type": "string"
"$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json"
},
"minItems": 0,
"uniqueItems": true
@ -528,8 +528,10 @@
"bare-node-builtins",
"byonm",
"cron",
"detect-cjs",
"ffi",
"fs",
"fmt-component",
"http",
"kv",
"net",

482
cli/standalone/binary.rs
View file

@ -9,14 +9,19 @@ use std::ffi::OsString;
use std::fs;
use std::fs::File;
use std::future::Future;
use std::io::ErrorKind;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::io::Write;
use std::ops::Range;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_ast::ModuleSpecifier;
use deno_config::workspace::PackageJsonDepResolution;
use deno_config::workspace::ResolverWorkspaceJsrPackage;
@ -30,13 +35,22 @@ use deno_core::futures::AsyncReadExt;
use deno_core::futures::AsyncSeekExt;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_graph::source::RealFileSystem;
use deno_graph::ModuleGraph;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_npm::NpmSystemInfo;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_io::fs::FsError;
use deno_runtime::deno_node::PackageJson;
use deno_semver::npm::NpmVersionReqParseError;
use deno_semver::package::PackageReq;
use deno_semver::Version;
use deno_semver::VersionReqSpecifierParseError;
use eszip::EszipRelativeFileBaseUrl;
use indexmap::IndexMap;
use log::Level;
use serde::Deserialize;
@ -49,10 +63,12 @@ use crate::args::NpmInstallDepsProvider;
use crate::args::PermissionFlags;
use crate::args::UnstableConfig;
use crate::cache::DenoDir;
use crate::emit::Emitter;
use crate::file_fetcher::FileFetcher;
use crate::http_util::HttpClientProvider;
use crate::npm::CliNpmResolver;
use crate::npm::InnerCliNpmResolverRef;
use crate::resolver::CjsTracker;
use crate::shared::ReleaseChannel;
use crate::standalone::virtual_fs::VfsEntry;
use crate::util::archive;
@ -60,12 +76,63 @@ use crate::util::fs::canonicalize_path_maybe_not_exists;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use super::file_system::DenoCompileFileSystem;
use super::serialization::deserialize_binary_data_section;
use super::serialization::serialize_binary_data_section;
use super::serialization::DenoCompileModuleData;
use super::serialization::DeserializedDataSection;
use super::serialization::RemoteModulesStore;
use super::serialization::RemoteModulesStoreBuilder;
use super::virtual_fs::FileBackedVfs;
use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VfsRoot;
use super::virtual_fs::VirtualDirectory;
const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd";
/// A URL that can be designated as the base for relative URLs.
///
/// After creation, this URL may be used to get the key for a
/// module in the binary.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url);
impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> {
fn from(url: &'a Url) -> Self {
Self(url)
}
}
impl<'a> StandaloneRelativeFileBaseUrl<'a> {
pub fn new(url: &'a Url) -> Self {
debug_assert_eq!(url.scheme(), "file");
Self(url)
}
/// Gets the module map key of the provided specifier.
///
/// * Descendant file specifiers will be made relative to the base.
/// * Non-descendant file specifiers will stay as-is (absolute).
/// * Non-file specifiers will stay as-is.
pub fn specifier_key<'b>(&self, target: &'b Url) -> Cow<'b, str> {
if target.scheme() != "file" {
return Cow::Borrowed(target.as_str());
}
match self.0.make_relative(target) {
Some(relative) => {
if relative.starts_with("../") {
Cow::Borrowed(target.as_str())
} else {
Cow::Owned(relative)
}
}
None => Cow::Borrowed(target.as_str()),
}
}
pub fn inner(&self) -> &Url {
self.0
}
}
#[derive(Deserialize, Serialize)]
pub enum NodeModules {
@ -120,78 +187,23 @@ pub struct Metadata {
pub unstable_config: UnstableConfig,
}
pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result<FileBackedVfs, AnyError> {
let data = libsui::find_section("d3n0l4nd").unwrap();
// We do the first part sync so it can complete quickly
let trailer: [u8; TRAILER_SIZE] = data[0..TRAILER_SIZE].try_into().unwrap();
let trailer = match Trailer::parse(&trailer)? {
None => panic!("Could not find trailer"),
Some(trailer) => trailer,
};
let data = &data[TRAILER_SIZE..];
let vfs_data =
&data[trailer.npm_vfs_pos as usize..trailer.npm_files_pos as usize];
let mut dir: VirtualDirectory = serde_json::from_slice(vfs_data)?;
// align the name of the directory with the root dir
dir.name = root_dir_path
.file_name()
.unwrap()
.to_string_lossy()
.to_string();
let fs_root = VfsRoot {
dir,
root_path: root_dir_path,
start_file_offset: trailer.npm_files_pos,
};
Ok(FileBackedVfs::new(data.to_vec(), fs_root))
}
fn write_binary_bytes(
mut file_writer: File,
original_bin: Vec<u8>,
metadata: &Metadata,
eszip: eszip::EszipV2,
npm_vfs: Option<&VirtualDirectory>,
npm_files: &Vec<Vec<u8>>,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
vfs: VfsBuilder,
compile_flags: &CompileFlags,
) -> Result<(), AnyError> {
let metadata = serde_json::to_string(metadata)?.as_bytes().to_vec();
let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec();
let eszip_archive = eszip.into_bytes();
let mut writer = Vec::new();
// write the trailer, which includes the positions
// of the data blocks in the file
writer.write_all(&{
let metadata_pos = eszip_archive.len() as u64;
let npm_vfs_pos = metadata_pos + (metadata.len() as u64);
let npm_files_pos = npm_vfs_pos + (npm_vfs.len() as u64);
Trailer {
eszip_pos: 0,
metadata_pos,
npm_vfs_pos,
npm_files_pos,
}
.as_bytes()
})?;
writer.write_all(&eszip_archive)?;
writer.write_all(&metadata)?;
writer.write_all(&npm_vfs)?;
for file in npm_files {
writer.write_all(file)?;
}
let data_section_bytes =
serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs)?;
let target = compile_flags.resolve_target();
if target.contains("linux") {
libsui::Elf::new(&original_bin).append(
"d3n0l4nd",
&writer,
&data_section_bytes,
&mut file_writer,
)?;
} else if target.contains("windows") {
@ -201,11 +213,11 @@ fn write_binary_bytes(
pe = pe.set_icon(&icon)?;
}
pe.write_resource("d3n0l4nd", writer)?
pe.write_resource("d3n0l4nd", data_section_bytes)?
.build(&mut file_writer)?;
} else if target.contains("darwin") {
libsui::Macho::from(original_bin)?
.write_section("d3n0l4nd", writer)?
.write_section("d3n0l4nd", data_section_bytes)?
.build_and_sign(&mut file_writer)?;
}
Ok(())
@ -221,6 +233,67 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool {
|| libsui::utils::is_macho(&data)
}
pub struct StandaloneData {
pub fs: Arc<dyn deno_fs::FileSystem>,
pub metadata: Metadata,
pub modules: StandaloneModules,
pub npm_snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
pub root_path: PathBuf,
pub vfs: Arc<FileBackedVfs>,
}
pub struct StandaloneModules {
remote_modules: RemoteModulesStore,
vfs: Arc<FileBackedVfs>,
}
impl StandaloneModules {
pub fn resolve_specifier<'a>(
&'a self,
specifier: &'a ModuleSpecifier,
) -> Result<Option<&'a ModuleSpecifier>, AnyError> {
if specifier.scheme() == "file" {
Ok(Some(specifier))
} else {
self.remote_modules.resolve_specifier(specifier)
}
}
pub fn has_file(&self, path: &Path) -> bool {
self.vfs.file_entry(path).is_ok()
}
pub fn read<'a>(
&'a self,
specifier: &'a ModuleSpecifier,
) -> Result<Option<DenoCompileModuleData<'a>>, AnyError> {
if specifier.scheme() == "file" {
let path = deno_path_util::url_to_file_path(specifier)?;
let bytes = match self.vfs.file_entry(&path) {
Ok(entry) => self.vfs.read_file_all(entry)?,
Err(err) if err.kind() == ErrorKind::NotFound => {
let bytes = match RealFs.read_file_sync(&path, None) {
Ok(bytes) => bytes,
Err(FsError::Io(err)) if err.kind() == ErrorKind::NotFound => {
return Ok(None)
}
Err(err) => return Err(err.into()),
};
Cow::Owned(bytes)
}
Err(err) => return Err(err.into()),
};
Ok(Some(DenoCompileModuleData {
media_type: MediaType::from_specifier(specifier),
specifier,
data: bytes,
}))
} else {
self.remote_modules.read(specifier)
}
}
}
/// This function will try to run this binary as a standalone binary
/// produced by `deno compile`. It determines if this is a standalone
/// binary by skipping over the trailer width at the end of the file,
@ -228,110 +301,67 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool {
/// the bundle is executed. If not, this function exits with `Ok(None)`.
pub fn extract_standalone(
cli_args: Cow<Vec<OsString>>,
) -> Result<
Option<impl Future<Output = Result<(Metadata, eszip::EszipV2), AnyError>>>,
AnyError,
> {
) -> Result<Option<StandaloneData>, AnyError> {
let Some(data) = libsui::find_section("d3n0l4nd") else {
return Ok(None);
};
// We do the first part sync so it can complete quickly
let trailer = match Trailer::parse(&data[0..TRAILER_SIZE])? {
let DeserializedDataSection {
mut metadata,
npm_snapshot,
remote_modules,
mut vfs_dir,
vfs_files_data,
} = match deserialize_binary_data_section(data)? {
Some(data_section) => data_section,
None => return Ok(None),
Some(trailer) => trailer,
};
let root_path = {
let maybe_current_exe = std::env::current_exe().ok();
let current_exe_name = maybe_current_exe
.as_ref()
.and_then(|p| p.file_name())
.map(|p| p.to_string_lossy())
// should never happen
.unwrap_or_else(|| Cow::Borrowed("binary"));
std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name))
};
let cli_args = cli_args.into_owned();
// If we have an eszip, read it out
Ok(Some(async move {
let bufreader =
deno_core::futures::io::BufReader::new(&data[TRAILER_SIZE..]);
metadata.argv.reserve(cli_args.len() - 1);
for arg in cli_args.into_iter().skip(1) {
metadata.argv.push(arg.into_string().unwrap());
}
let vfs = {
// align the name of the directory with the root dir
vfs_dir.name = root_path.file_name().unwrap().to_string_lossy().to_string();
let (eszip, loader) = eszip::EszipV2::parse(bufreader)
.await
.context("Failed to parse eszip header")?;
let bufreader = loader.await.context("Failed to parse eszip archive")?;
let mut metadata = String::new();
bufreader
.take(trailer.metadata_len())
.read_to_string(&mut metadata)
.await
.context("Failed to read metadata from the current executable")?;
let mut metadata: Metadata = serde_json::from_str(&metadata).unwrap();
metadata.argv.reserve(cli_args.len() - 1);
for arg in cli_args.into_iter().skip(1) {
metadata.argv.push(arg.into_string().unwrap());
}
Ok((metadata, eszip))
let fs_root = VfsRoot {
dir: vfs_dir,
root_path: root_path.clone(),
start_file_offset: 0,
};
Arc::new(FileBackedVfs::new(Cow::Borrowed(vfs_files_data), fs_root))
};
let fs: Arc<dyn deno_fs::FileSystem> =
Arc::new(DenoCompileFileSystem::new(vfs.clone()));
Ok(Some(StandaloneData {
fs,
metadata,
modules: StandaloneModules {
remote_modules,
vfs: vfs.clone(),
},
npm_snapshot,
root_path,
vfs,
}))
}
const TRAILER_SIZE: usize = std::mem::size_of::<Trailer>() + 8; // 8 bytes for the magic trailer string
struct Trailer {
eszip_pos: u64,
metadata_pos: u64,
npm_vfs_pos: u64,
npm_files_pos: u64,
}
impl Trailer {
pub fn parse(trailer: &[u8]) -> Result<Option<Trailer>, AnyError> {
let (magic_trailer, rest) = trailer.split_at(8);
if magic_trailer != MAGIC_TRAILER {
return Ok(None);
}
let (eszip_archive_pos, rest) = rest.split_at(8);
let (metadata_pos, rest) = rest.split_at(8);
let (npm_vfs_pos, npm_files_pos) = rest.split_at(8);
let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?;
let metadata_pos = u64_from_bytes(metadata_pos)?;
let npm_vfs_pos = u64_from_bytes(npm_vfs_pos)?;
let npm_files_pos = u64_from_bytes(npm_files_pos)?;
Ok(Some(Trailer {
eszip_pos: eszip_archive_pos,
metadata_pos,
npm_vfs_pos,
npm_files_pos,
}))
}
pub fn metadata_len(&self) -> u64 {
self.npm_vfs_pos - self.metadata_pos
}
pub fn npm_vfs_len(&self) -> u64 {
self.npm_files_pos - self.npm_vfs_pos
}
pub fn as_bytes(&self) -> Vec<u8> {
let mut trailer = MAGIC_TRAILER.to_vec();
trailer.write_all(&self.eszip_pos.to_be_bytes()).unwrap();
trailer.write_all(&self.metadata_pos.to_be_bytes()).unwrap();
trailer.write_all(&self.npm_vfs_pos.to_be_bytes()).unwrap();
trailer
.write_all(&self.npm_files_pos.to_be_bytes())
.unwrap();
trailer
}
}
fn u64_from_bytes(arr: &[u8]) -> Result<u64, AnyError> {
let fixed_arr: &[u8; 8] = arr
.try_into()
.context("Failed to convert the buffer into a fixed-size array")?;
Ok(u64::from_be_bytes(*fixed_arr))
}
pub struct DenoCompileBinaryWriter<'a> {
cjs_tracker: &'a CjsTracker,
deno_dir: &'a DenoDir,
emitter: &'a Emitter,
file_fetcher: &'a FileFetcher,
http_client_provider: &'a HttpClientProvider,
npm_resolver: &'a dyn CliNpmResolver,
@ -342,7 +372,9 @@ pub struct DenoCompileBinaryWriter<'a> {
impl<'a> DenoCompileBinaryWriter<'a> {
#[allow(clippy::too_many_arguments)]
pub fn new(
cjs_tracker: &'a CjsTracker,
deno_dir: &'a DenoDir,
emitter: &'a Emitter,
file_fetcher: &'a FileFetcher,
http_client_provider: &'a HttpClientProvider,
npm_resolver: &'a dyn CliNpmResolver,
@ -350,7 +382,9 @@ impl<'a> DenoCompileBinaryWriter<'a> {
npm_system_info: NpmSystemInfo,
) -> Self {
Self {
cjs_tracker,
deno_dir,
emitter,
file_fetcher,
http_client_provider,
npm_resolver,
@ -362,8 +396,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
pub async fn write_bin(
&self,
writer: File,
eszip: eszip::EszipV2,
root_dir_url: EszipRelativeFileBaseUrl<'_>,
graph: &ModuleGraph,
root_dir_url: StandaloneRelativeFileBaseUrl<'_>,
entrypoint: &ModuleSpecifier,
compile_flags: &CompileFlags,
cli_options: &CliOptions,
@ -390,15 +424,17 @@ impl<'a> DenoCompileBinaryWriter<'a> {
)
}
}
self.write_standalone_binary(
writer,
original_binary,
eszip,
root_dir_url,
entrypoint,
cli_options,
compile_flags,
)
self
.write_standalone_binary(
writer,
original_binary,
graph,
root_dir_url,
entrypoint,
cli_options,
compile_flags,
)
.await
}
async fn get_base_binary(
@ -468,7 +504,11 @@ impl<'a> DenoCompileBinaryWriter<'a> {
self
.http_client_provider
.get_or_create()?
.download_with_progress(download_url.parse()?, None, &progress)
.download_with_progress_and_retries(
download_url.parse()?,
None,
&progress,
)
.await?
};
let bytes = match maybe_bytes {
@ -489,12 +529,12 @@ impl<'a> DenoCompileBinaryWriter<'a> {
/// This functions creates a standalone deno binary by appending a bundle
/// and magic trailer to the currently executing binary.
#[allow(clippy::too_many_arguments)]
fn write_standalone_binary(
async fn write_standalone_binary(
&self,
writer: File,
original_bin: Vec<u8>,
mut eszip: eszip::EszipV2,
root_dir_url: EszipRelativeFileBaseUrl<'_>,
graph: &ModuleGraph,
root_dir_url: StandaloneRelativeFileBaseUrl<'_>,
entrypoint: &ModuleSpecifier,
cli_options: &CliOptions,
compile_flags: &CompileFlags,
@ -508,19 +548,17 @@ impl<'a> DenoCompileBinaryWriter<'a> {
None => None,
};
let root_path = root_dir_url.inner().to_file_path().unwrap();
let (npm_vfs, npm_files, node_modules) = match self.npm_resolver.as_inner()
let (maybe_npm_vfs, node_modules, npm_snapshot) = match self
.npm_resolver
.as_inner()
{
InnerCliNpmResolverRef::Managed(managed) => {
let snapshot =
managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
if !snapshot.as_serialized().packages.is_empty() {
let (root_dir, files) = self
.build_vfs(&root_path, cli_options)?
.into_dir_and_files();
eszip.add_npm_snapshot(snapshot);
let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?;
(
Some(root_dir),
files,
Some(npm_vfs_builder),
Some(NodeModules::Managed {
node_modules_dir: self.npm_resolver.root_node_modules_path().map(
|path| {
@ -532,18 +570,16 @@ impl<'a> DenoCompileBinaryWriter<'a> {
},
),
}),
Some(snapshot),
)
} else {
(None, Vec::new(), None)
(None, None, None)
}
}
InnerCliNpmResolverRef::Byonm(resolver) => {
let (root_dir, files) = self
.build_vfs(&root_path, cli_options)?
.into_dir_and_files();
let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?;
(
Some(root_dir),
files,
Some(npm_vfs_builder),
Some(NodeModules::Byonm {
root_node_modules_dir: resolver.root_node_modules_path().map(
|node_modules_dir| {
@ -556,9 +592,69 @@ impl<'a> DenoCompileBinaryWriter<'a> {
},
),
}),
None,
)
}
};
let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs {
npm_vfs
} else {
VfsBuilder::new(root_path.clone())?
};
let mut remote_modules_store = RemoteModulesStoreBuilder::default();
for module in graph.modules() {
if module.specifier().scheme() == "data" {
continue; // don't store data urls as an entry as they're in the code
}
let (maybe_source, media_type) = match module {
deno_graph::Module::Js(m) => {
let source = if m.media_type.is_emittable() {
let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script(
&m.specifier,
m.media_type,
m.is_script,
)?;
let module_kind = ModuleKind::from_is_cjs(is_cjs);
let source = self
.emitter
.emit_parsed_source(
&m.specifier,
m.media_type,
module_kind,
&m.source,
)
.await?;
source.into_bytes()
} else {
m.source.as_bytes().to_vec()
};
(Some(source), m.media_type)
}
deno_graph::Module::Json(m) => {
(Some(m.source.as_bytes().to_vec()), m.media_type)
}
deno_graph::Module::Npm(_)
| deno_graph::Module::Node(_)
| deno_graph::Module::External(_) => (None, MediaType::Unknown),
};
if module.specifier().scheme() == "file" {
let file_path = deno_path_util::url_to_file_path(module.specifier())?;
vfs
.add_file_with_data(
&file_path,
match maybe_source {
Some(source) => source,
None => RealFs.read_file_sync(&file_path, None)?,
},
)
.with_context(|| {
format!("Failed adding '{}'", file_path.display())
})?;
} else if let Some(source) = maybe_source {
remote_modules_store.add(module.specifier(), media_type, source);
}
}
remote_modules_store.add_redirects(&graph.redirects);
let env_vars_from_env_file = match cli_options.env_file_name() {
Some(env_filename) => {
@ -622,6 +718,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
unstable_config: UnstableConfig {
legacy_flag_enabled: false,
bare_node_builtins: cli_options.unstable_bare_node_builtins(),
detect_cjs: cli_options.unstable_detect_cjs(),
sloppy_imports: cli_options.unstable_sloppy_imports(),
features: cli_options.unstable_features(),
},
@ -631,14 +728,14 @@ impl<'a> DenoCompileBinaryWriter<'a> {
writer,
original_bin,
&metadata,
eszip,
npm_vfs.as_ref(),
&npm_files,
npm_snapshot.map(|s| s.into_serialized()),
&remote_modules_store,
vfs,
compile_flags,
)
}
fn build_vfs(
fn build_npm_vfs(
&self,
root_path: &Path,
cli_options: &CliOptions,
@ -659,8 +756,9 @@ impl<'a> DenoCompileBinaryWriter<'a> {
} else {
// DO NOT include the user's registry url as it may contain credentials,
// but also don't make this dependent on the registry url
let root_path = npm_resolver.global_cache_root_folder();
let mut builder = VfsBuilder::new(root_path)?;
let global_cache_root_path = npm_resolver.global_cache_root_path();
let mut builder =
VfsBuilder::new(global_cache_root_path.to_path_buf())?;
let mut packages =
npm_resolver.all_system_packages(&self.npm_system_info);
packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism
@ -670,12 +768,12 @@ impl<'a> DenoCompileBinaryWriter<'a> {
builder.add_dir_recursive(&folder)?;
}
// Flatten all the registries folders into a single "node_modules/localhost" folder
// Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder
// that will be used by denort when loading the npm cache. This avoids us exposing
// the user's private registry information and means we don't have to bother
// serializing all the different registry config into the binary.
builder.with_root_dir(|root_dir| {
root_dir.name = "node_modules".to_string();
root_dir.name = ".deno_compile_node_modules".to_string();
let mut new_entries = Vec::with_capacity(root_dir.entries.len());
let mut localhost_entries = IndexMap::new();
for entry in std::mem::take(&mut root_dir.entries) {
@ -710,6 +808,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
root_dir.entries = new_entries;
});
builder.set_new_root_path(root_path.to_path_buf())?;
Ok(builder)
}
}

4
cli/standalone/file_system.rs
View file

@ -22,8 +22,8 @@ use super::virtual_fs::FileBackedVfs;
pub struct DenoCompileFileSystem(Arc<FileBackedVfs>);
impl DenoCompileFileSystem {
pub fn new(vfs: FileBackedVfs) -> Self {
Self(Arc::new(vfs))
pub fn new(vfs: Arc<FileBackedVfs>) -> Self {
Self(vfs)
}
fn error_if_in_vfs(&self, path: &Path) -> FsResult<()> {

449
cli/standalone/mod.rs
View file

@ -5,6 +5,8 @@
#![allow(dead_code)]
#![allow(unused_imports)]
use binary::StandaloneData;
use binary::StandaloneModules;
use deno_ast::MediaType;
use deno_cache_dir::npm::NpmCacheDir;
use deno_config::workspace::MappedResolution;
@ -17,6 +19,7 @@ use deno_core::error::type_error;
use deno_core::error::AnyError;
use deno_core::futures::FutureExt;
use deno_core::v8_set_flags;
use deno_core::FastString;
use deno_core::FeatureChecker;
use deno_core::ModuleLoader;
use deno_core::ModuleSourceCode;
@ -28,7 +31,9 @@ use deno_npm::npm_rc::ResolvedNpmRc;
use deno_package_json::PackageJsonDepValue;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_permissions::Permissions;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::rustls::RootCertStore;
@ -38,10 +43,10 @@ use deno_runtime::permissions::RuntimePermissionDescriptorParser;
use deno_runtime::WorkerExecutionMode;
use deno_runtime::WorkerLogLevel;
use deno_semver::npm::NpmPackageReqReference;
use eszip::EszipRelativeFileBaseUrl;
use import_map::parse_from_json;
use node_resolver::analyze::NodeCodeTranslator;
use node_resolver::NodeResolutionMode;
use serialization::DenoCompileModuleSource;
use std::borrow::Cow;
use std::rc::Rc;
use std::sync::Arc;
@ -54,17 +59,24 @@ use crate::args::CacheSetting;
use crate::args::NpmInstallDepsProvider;
use crate::args::StorageKeyResolver;
use crate::cache::Caches;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::DenoDirProvider;
use crate::cache::NodeAnalysisCache;
use crate::cache::RealDenoCacheEnv;
use crate::http_util::HttpClientProvider;
use crate::node::CliCjsCodeAnalyzer;
use crate::node::CliNodeCodeTranslator;
use crate::npm::create_cli_npm_resolver;
use crate::npm::create_in_npm_pkg_checker;
use crate::npm::CliByonmNpmResolverCreateOptions;
use crate::npm::CliManagedInNpmPkgCheckerCreateOptions;
use crate::npm::CliManagedNpmResolverCreateOptions;
use crate::npm::CliNpmResolver;
use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::resolver::CjsResolutionStore;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::resolver::CjsTracker;
use crate::resolver::CjsTrackerOptions;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNodeResolver;
use crate::resolver::NpmModuleLoader;
@ -73,60 +85,30 @@ use crate::util::progress_bar::ProgressBarStyle;
use crate::util::v8::construct_v8_flags;
use crate::worker::CliMainWorkerFactory;
use crate::worker::CliMainWorkerOptions;
use crate::worker::ModuleLoaderAndSourceMapGetter;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
pub mod binary;
mod file_system;
mod serialization;
mod virtual_fs;
pub use binary::extract_standalone;
pub use binary::is_standalone_binary;
pub use binary::DenoCompileBinaryWriter;
use self::binary::load_npm_vfs;
use self::binary::Metadata;
use self::file_system::DenoCompileFileSystem;
struct WorkspaceEszipModule {
specifier: ModuleSpecifier,
inner: eszip::Module,
}
struct WorkspaceEszip {
eszip: eszip::EszipV2,
root_dir_url: Arc<ModuleSpecifier>,
}
impl WorkspaceEszip {
pub fn get_module(
&self,
specifier: &ModuleSpecifier,
) -> Option<WorkspaceEszipModule> {
if specifier.scheme() == "file" {
let specifier_key = EszipRelativeFileBaseUrl::new(&self.root_dir_url)
.specifier_key(specifier);
let module = self.eszip.get_module(&specifier_key)?;
let specifier = self.root_dir_url.join(&module.specifier).unwrap();
Some(WorkspaceEszipModule {
specifier,
inner: module,
})
} else {
let module = self.eszip.get_module(specifier.as_str())?;
Some(WorkspaceEszipModule {
specifier: ModuleSpecifier::parse(&module.specifier).unwrap(),
inner: module,
})
}
}
}
struct SharedModuleLoaderState {
eszip: WorkspaceEszip,
workspace_resolver: WorkspaceResolver,
cjs_tracker: Arc<CjsTracker>,
fs: Arc<dyn deno_fs::FileSystem>,
modules: StandaloneModules,
node_code_translator: Arc<CliNodeCodeTranslator>,
node_resolver: Arc<CliNodeResolver>,
npm_module_loader: Arc<NpmModuleLoader>,
npm_resolver: Arc<dyn CliNpmResolver>,
workspace_resolver: WorkspaceResolver,
}
#[derive(Clone)]
@ -134,6 +116,12 @@ struct EmbeddedModuleLoader {
shared: Arc<SharedModuleLoaderState>,
}
impl std::fmt::Debug for EmbeddedModuleLoader {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("EmbeddedModuleLoader").finish()
}
}
pub const MODULE_NOT_FOUND: &str = "Module not found";
pub const UNSUPPORTED_SCHEME: &str = "Unsupported scheme";
@ -191,8 +179,7 @@ impl ModuleLoader for EmbeddedModuleLoader {
sub_path.as_deref(),
Some(&referrer),
NodeResolutionMode::Execution,
)?
.into_url(),
)?,
),
Ok(MappedResolution::PackageJson {
dep_result,
@ -200,16 +187,14 @@ impl ModuleLoader for EmbeddedModuleLoader {
alias,
..
}) => match dep_result.as_ref().map_err(|e| AnyError::from(e.clone()))? {
PackageJsonDepValue::Req(req) => self
.shared
.node_resolver
.resolve_req_with_sub_path(
PackageJsonDepValue::Req(req) => {
self.shared.node_resolver.resolve_req_with_sub_path(
req,
sub_path.as_deref(),
&referrer,
NodeResolutionMode::Execution,
)
.map(|res| res.into_url()),
}
PackageJsonDepValue::Workspace(version_req) => {
let pkg_folder = self
.shared
@ -227,8 +212,7 @@ impl ModuleLoader for EmbeddedModuleLoader {
sub_path.as_deref(),
Some(&referrer),
NodeResolutionMode::Execution,
)?
.into_url(),
)?,
)
}
},
@ -237,20 +221,18 @@ impl ModuleLoader for EmbeddedModuleLoader {
if let Ok(reference) =
NpmPackageReqReference::from_specifier(&specifier)
{
return self
.shared
.node_resolver
.resolve_req_reference(
&reference,
&referrer,
NodeResolutionMode::Execution,
)
.map(|res| res.into_url());
return self.shared.node_resolver.resolve_req_reference(
&reference,
&referrer,
NodeResolutionMode::Execution,
);
}
if specifier.scheme() == "jsr" {
if let Some(module) = self.shared.eszip.get_module(&specifier) {
return Ok(module.specifier);
if let Some(specifier) =
self.shared.modules.resolve_specifier(&specifier)?
{
return Ok(specifier.clone());
}
}
@ -345,82 +327,140 @@ impl ModuleLoader for EmbeddedModuleLoader {
);
}
let Some(module) = self.shared.eszip.get_module(original_specifier) else {
return deno_core::ModuleLoadResponse::Sync(Err(type_error(format!(
"{MODULE_NOT_FOUND}: {}",
original_specifier
))));
};
let original_specifier = original_specifier.clone();
deno_core::ModuleLoadResponse::Async(
async move {
let code = module.inner.source().await.ok_or_else(|| {
type_error(format!("Module not found: {}", original_specifier))
})?;
let code = arc_u8_to_arc_str(code)
.map_err(|_| type_error("Module source is not utf-8"))?;
Ok(deno_core::ModuleSource::new_with_redirect(
match module.inner.kind {
eszip::ModuleKind::JavaScript => ModuleType::JavaScript,
eszip::ModuleKind::Json => ModuleType::Json,
eszip::ModuleKind::Jsonc => {
return Err(type_error("jsonc modules not supported"))
match self.shared.modules.read(original_specifier) {
Ok(Some(module)) => {
let media_type = module.media_type;
let (module_specifier, module_type, module_source) =
module.into_parts();
let is_maybe_cjs = match self
.shared
.cjs_tracker
.is_maybe_cjs(original_specifier, media_type)
{
Ok(is_maybe_cjs) => is_maybe_cjs,
Err(err) => {
return deno_core::ModuleLoadResponse::Sync(Err(type_error(
format!("{:?}", err),
)));
}
};
if is_maybe_cjs {
let original_specifier = original_specifier.clone();
let module_specifier = module_specifier.clone();
let shared = self.shared.clone();
deno_core::ModuleLoadResponse::Async(
async move {
let source = match module_source {
DenoCompileModuleSource::String(string) => {
Cow::Borrowed(string)
}
DenoCompileModuleSource::Bytes(module_code_bytes) => {
match module_code_bytes {
Cow::Owned(bytes) => Cow::Owned(
crate::util::text_encoding::from_utf8_lossy_owned(bytes),
),
Cow::Borrowed(bytes) => String::from_utf8_lossy(bytes),
}
}
};
let source = shared
.node_code_translator
.translate_cjs_to_esm(&module_specifier, Some(source))
.await?;
let module_source = match source {
Cow::Owned(source) => ModuleSourceCode::String(source.into()),
Cow::Borrowed(source) => {
ModuleSourceCode::String(FastString::from_static(source))
}
};
Ok(deno_core::ModuleSource::new_with_redirect(
module_type,
module_source,
&original_specifier,
&module_specifier,
None,
))
}
eszip::ModuleKind::OpaqueData => {
unreachable!();
}
},
ModuleSourceCode::String(code.into()),
&original_specifier,
&module.specifier,
None,
))
.boxed_local(),
)
} else {
let module_source = module_source.into_for_v8();
deno_core::ModuleLoadResponse::Sync(Ok(
deno_core::ModuleSource::new_with_redirect(
module_type,
module_source,
original_specifier,
module_specifier,
None,
),
))
}
}
.boxed_local(),
)
Ok(None) => deno_core::ModuleLoadResponse::Sync(Err(type_error(
format!("{MODULE_NOT_FOUND}: {}", original_specifier),
))),
Err(err) => deno_core::ModuleLoadResponse::Sync(Err(type_error(
format!("{:?}", err),
))),
}
}
}
fn arc_u8_to_arc_str(
arc_u8: Arc<[u8]>,
) -> Result<Arc<str>, std::str::Utf8Error> {
// Check that the string is valid UTF-8.
std::str::from_utf8(&arc_u8)?;
// SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as
// Arc<str>. This is proven by the From<Arc<str>> impl for Arc<[u8]> from the
// standard library.
Ok(unsafe {
std::mem::transmute::<std::sync::Arc<[u8]>, std::sync::Arc<str>>(arc_u8)
})
impl NodeRequireLoader for EmbeddedModuleLoader {
fn ensure_read_permission<'a>(
&self,
permissions: &mut dyn deno_runtime::deno_node::NodePermissions,
path: &'a std::path::Path,
) -> Result<Cow<'a, std::path::Path>, AnyError> {
if self.shared.modules.has_file(path) {
// allow reading if the file is in the snapshot
return Ok(Cow::Borrowed(path));
}
self
.shared
.npm_resolver
.ensure_read_permission(permissions, path)
}
fn load_text_file_lossy(
&self,
path: &std::path::Path,
) -> Result<String, AnyError> {
Ok(self.shared.fs.read_text_file_lossy_sync(path, None)?)
}
}
struct StandaloneModuleLoaderFactory {
shared: Arc<SharedModuleLoaderState>,
}
impl StandaloneModuleLoaderFactory {
pub fn create_result(&self) -> CreateModuleLoaderResult {
let loader = Rc::new(EmbeddedModuleLoader {
shared: self.shared.clone(),
});
CreateModuleLoaderResult {
module_loader: loader.clone(),
node_require_loader: loader,
}
}
}
impl ModuleLoaderFactory for StandaloneModuleLoaderFactory {
fn create_for_main(
&self,
_root_permissions: PermissionsContainer,
) -> ModuleLoaderAndSourceMapGetter {
ModuleLoaderAndSourceMapGetter {
module_loader: Rc::new(EmbeddedModuleLoader {
shared: self.shared.clone(),
}),
}
) -> CreateModuleLoaderResult {
self.create_result()
}
fn create_for_worker(
&self,
_parent_permissions: PermissionsContainer,
_permissions: PermissionsContainer,
) -> ModuleLoaderAndSourceMapGetter {
ModuleLoaderAndSourceMapGetter {
module_loader: Rc::new(EmbeddedModuleLoader {
shared: self.shared.clone(),
}),
}
) -> CreateModuleLoaderResult {
self.create_result()
}
}
@ -439,13 +479,15 @@ impl RootCertStoreProvider for StandaloneRootCertStoreProvider {
}
}
pub async fn run(
mut eszip: eszip::EszipV2,
metadata: Metadata,
) -> Result<i32, AnyError> {
let current_exe_path = std::env::current_exe().unwrap();
let current_exe_name =
current_exe_path.file_name().unwrap().to_string_lossy();
pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let StandaloneData {
fs,
metadata,
modules,
npm_snapshot,
root_path,
vfs,
} = data;
let deno_dir_provider = Arc::new(DenoDirProvider::new(None));
let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider {
ca_stores: metadata.ca_stores,
@ -459,44 +501,50 @@ pub async fn run(
));
// use a dummy npm registry url
let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap();
let root_path =
std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name));
let root_dir_url =
Arc::new(ModuleSpecifier::from_directory_path(&root_path).unwrap());
let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap();
let root_node_modules_path = root_path.join("node_modules");
let npm_cache_dir = NpmCacheDir::new(
&RealDenoCacheEnv,
root_node_modules_path.clone(),
vec![npm_registry_url.clone()],
);
let npm_global_cache_dir = npm_cache_dir.get_cache_location();
let npm_global_cache_dir = root_path.join(".deno_compile_node_modules");
let cache_setting = CacheSetting::Only;
let (fs, npm_resolver, maybe_vfs_root) = match metadata.node_modules {
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
));
let (in_npm_pkg_checker, npm_resolver) = match metadata.node_modules {
Some(binary::NodeModules::Managed { node_modules_dir }) => {
// this will always have a snapshot
let snapshot = eszip.take_npm_snapshot().unwrap();
let vfs_root_dir_path = if node_modules_dir.is_some() {
root_path.clone()
} else {
npm_cache_dir.root_dir().to_owned()
};
let vfs = load_npm_vfs(vfs_root_dir_path.clone())
.context("Failed to load npm vfs.")?;
// create an npmrc that uses the fake npm_registry_url to resolve packages
let npmrc = Arc::new(ResolvedNpmRc {
default_config: deno_npm::npm_rc::RegistryConfigWithUrl {
registry_url: npm_registry_url.clone(),
config: Default::default(),
},
scopes: Default::default(),
registry_configs: Default::default(),
});
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
npm_global_cache_dir,
npmrc.get_all_known_registries_urls(),
));
let snapshot = npm_snapshot.unwrap();
let maybe_node_modules_path = node_modules_dir
.map(|node_modules_dir| vfs_root_dir_path.join(node_modules_dir));
let fs = Arc::new(DenoCompileFileSystem::new(vfs))
as Arc<dyn deno_fs::FileSystem>;
.map(|node_modules_dir| root_path.join(node_modules_dir));
let in_npm_pkg_checker =
create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed(
CliManagedInNpmPkgCheckerCreateOptions {
root_cache_dir_url: npm_cache_dir.root_dir_url(),
maybe_node_modules_path: maybe_node_modules_path.as_deref(),
},
));
let npm_resolver =
create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed(
CliNpmResolverManagedCreateOptions {
CliManagedNpmResolverCreateOptions {
snapshot: CliNpmResolverManagedSnapshotOption::Specified(Some(
snapshot,
)),
maybe_lockfile: None,
fs: fs.clone(),
http_client_provider: http_client_provider.clone(),
npm_global_cache_dir,
npm_cache_dir,
cache_setting,
text_only_progress_bar: progress_bar,
maybe_node_modules_path,
@ -505,50 +553,54 @@ pub async fn run(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
// create an npmrc that uses the fake npm_registry_url to resolve packages
npmrc: Arc::new(ResolvedNpmRc {
default_config: deno_npm::npm_rc::RegistryConfigWithUrl {
registry_url: npm_registry_url.clone(),
config: Default::default(),
},
scopes: Default::default(),
registry_configs: Default::default(),
}),
npmrc,
lifecycle_scripts: Default::default(),
},
))
.await?;
(fs, npm_resolver, Some(vfs_root_dir_path))
(in_npm_pkg_checker, npm_resolver)
}
Some(binary::NodeModules::Byonm {
root_node_modules_dir,
}) => {
let vfs_root_dir_path = root_path.clone();
let vfs = load_npm_vfs(vfs_root_dir_path.clone())
.context("Failed to load vfs.")?;
let root_node_modules_dir =
root_node_modules_dir.map(|p| vfs.root().join(p));
let fs = Arc::new(DenoCompileFileSystem::new(vfs))
as Arc<dyn deno_fs::FileSystem>;
let in_npm_pkg_checker =
create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Byonm);
let npm_resolver = create_cli_npm_resolver(
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(fs.clone()),
pkg_json_resolver: pkg_json_resolver.clone(),
root_node_modules_dir,
}),
)
.await?;
(fs, npm_resolver, Some(vfs_root_dir_path))
(in_npm_pkg_checker, npm_resolver)
}
None => {
let fs = Arc::new(deno_fs::RealFs) as Arc<dyn deno_fs::FileSystem>;
// Packages from different registries are already inlined in the binary,
// so no need to create actual `.npmrc` configuration.
let npmrc = create_default_npmrc();
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
npm_global_cache_dir,
npmrc.get_all_known_registries_urls(),
));
let in_npm_pkg_checker =
create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed(
CliManagedInNpmPkgCheckerCreateOptions {
root_cache_dir_url: npm_cache_dir.root_dir_url(),
maybe_node_modules_path: None,
},
));
let npm_resolver =
create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed(
CliNpmResolverManagedCreateOptions {
CliManagedNpmResolverCreateOptions {
snapshot: CliNpmResolverManagedSnapshotOption::Specified(None),
maybe_lockfile: None,
fs: fs.clone(),
http_client_provider: http_client_provider.clone(),
npm_global_cache_dir,
npm_cache_dir,
cache_setting,
text_only_progress_bar: progress_bar,
maybe_node_modules_path: None,
@ -557,41 +609,52 @@ pub async fn run(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
// Packages from different registries are already inlined in the ESZip,
// so no need to create actual `.npmrc` configuration.
npmrc: create_default_npmrc(),
lifecycle_scripts: Default::default(),
},
))
.await?;
(fs, npm_resolver, None)
(in_npm_pkg_checker, npm_resolver)
}
};
let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some();
let node_resolver = Arc::new(NodeResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
in_npm_pkg_checker.clone(),
npm_resolver.clone().into_npm_resolver(),
pkg_json_resolver.clone(),
));
let cjs_tracker = Arc::new(CjsTracker::new(
in_npm_pkg_checker.clone(),
pkg_json_resolver.clone(),
CjsTrackerOptions {
unstable_detect_cjs: metadata.unstable_config.detect_cjs,
},
));
let cjs_resolutions = Arc::new(CjsResolutionStore::default());
let cache_db = Caches::new(deno_dir_provider.clone());
let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db());
let cli_node_resolver = Arc::new(CliNodeResolver::new(
cjs_resolutions.clone(),
cjs_tracker.clone(),
fs.clone(),
in_npm_pkg_checker.clone(),
node_resolver.clone(),
npm_resolver.clone(),
));
let cjs_esm_code_analyzer = CliCjsCodeAnalyzer::new(
node_analysis_cache,
cjs_tracker.clone(),
fs.clone(),
cli_node_resolver.clone(),
None,
false,
);
let node_code_translator = Arc::new(NodeCodeTranslator::new(
cjs_esm_code_analyzer,
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
in_npm_pkg_checker,
node_resolver.clone(),
npm_resolver.clone().into_npm_resolver(),
pkg_json_resolver.clone(),
));
let workspace_resolver = {
let import_map = match metadata.workspace_resolver.import_map {
@ -644,37 +707,35 @@ pub async fn run(
};
let module_loader_factory = StandaloneModuleLoaderFactory {
shared: Arc::new(SharedModuleLoaderState {
eszip: WorkspaceEszip {
eszip,
root_dir_url,
},
workspace_resolver,
cjs_tracker: cjs_tracker.clone(),
fs: fs.clone(),
modules,
node_code_translator: node_code_translator.clone(),
node_resolver: cli_node_resolver.clone(),
npm_module_loader: Arc::new(NpmModuleLoader::new(
cjs_resolutions,
node_code_translator,
cjs_tracker.clone(),
fs.clone(),
cli_node_resolver,
node_code_translator,
)),
npm_resolver: npm_resolver.clone(),
workspace_resolver,
}),
};
let permissions = {
let mut permissions =
metadata.permissions.to_options(/* cli_arg_urls */ &[]);
// if running with an npm vfs, grant read access to it
if let Some(vfs_root) = maybe_vfs_root {
match &mut permissions.allow_read {
Some(vec) if vec.is_empty() => {
// do nothing, already granted
}
Some(vec) => {
vec.push(vfs_root.to_string_lossy().to_string());
}
None => {
permissions.allow_read =
Some(vec![vfs_root.to_string_lossy().to_string()]);
}
// grant read access to the vfs
match &mut permissions.allow_read {
Some(vec) if vec.is_empty() => {
// do nothing, already granted
}
Some(vec) => {
vec.push(root_path.to_string_lossy().to_string());
}
None => {
permissions.allow_read =
Some(vec![root_path.to_string_lossy().to_string()]);
}
}
@ -706,6 +767,7 @@ pub async fn run(
Box::new(module_loader_factory),
node_resolver,
npm_resolver,
pkg_json_resolver,
root_cert_store_provider,
permissions,
StorageKeyResolver::empty(),
@ -721,7 +783,6 @@ pub async fn run(
inspect_wait: false,
strace_ops: None,
is_inspecting: false,
is_npm_main: main_module.scheme() == "npm",
skip_op_registration: true,
location: metadata.location,
argv0: NpmPackageReqReference::from_specifier(&main_module)

661
cli/standalone/serialization.rs Normal file
View file

@ -0,0 +1,661 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::io::Write;
use deno_ast::MediaType;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_core::FastString;
use deno_core::ModuleSourceCode;
use deno_core::ModuleType;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_semver::package::PackageReq;
use crate::standalone::virtual_fs::VirtualDirectory;
use super::binary::Metadata;
use super::virtual_fs::VfsBuilder;
const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd";
/// Binary format:
/// * d3n0l4nd
/// * <metadata_len><metadata>
/// * <npm_snapshot_len><npm_snapshot>
/// * <remote_modules_len><remote_modules>
/// * <vfs_headers_len><vfs_headers>
/// * <vfs_file_data_len><vfs_file_data>
/// * d3n0l4nd
pub fn serialize_binary_data_section(
metadata: &Metadata,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
vfs: VfsBuilder,
) -> Result<Vec<u8>, AnyError> {
fn write_bytes_with_len(bytes: &mut Vec<u8>, data: &[u8]) {
bytes.extend_from_slice(&(data.len() as u64).to_le_bytes());
bytes.extend_from_slice(data);
}
let mut bytes = Vec::new();
bytes.extend_from_slice(MAGIC_BYTES);
// 1. Metadata
{
let metadata = serde_json::to_string(metadata)?;
write_bytes_with_len(&mut bytes, metadata.as_bytes());
}
// 2. Npm snapshot
{
let npm_snapshot =
npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default();
write_bytes_with_len(&mut bytes, &npm_snapshot);
}
// 3. Remote modules
{
let update_index = bytes.len();
bytes.extend_from_slice(&(0_u64).to_le_bytes());
let start_index = bytes.len();
remote_modules.write(&mut bytes)?;
let length = bytes.len() - start_index;
let length_bytes = (length as u64).to_le_bytes();
bytes[update_index..update_index + length_bytes.len()]
.copy_from_slice(&length_bytes);
}
// 4. VFS
{
let (vfs, vfs_files) = vfs.into_dir_and_files();
let vfs = serde_json::to_string(&vfs)?;
write_bytes_with_len(&mut bytes, vfs.as_bytes());
let vfs_bytes_len = vfs_files.iter().map(|f| f.len() as u64).sum::<u64>();
bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes());
for file in &vfs_files {
bytes.extend_from_slice(file);
}
}
// write the magic bytes at the end so we can use it
// to make sure we've deserialized correctly
bytes.extend_from_slice(MAGIC_BYTES);
Ok(bytes)
}
pub struct DeserializedDataSection {
pub metadata: Metadata,
pub npm_snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
pub remote_modules: RemoteModulesStore,
pub vfs_dir: VirtualDirectory,
pub vfs_files_data: &'static [u8],
}
pub fn deserialize_binary_data_section(
data: &'static [u8],
) -> Result<Option<DeserializedDataSection>, AnyError> {
fn read_bytes_with_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> {
let (input, len) = read_u64(input)?;
let (input, data) = read_bytes(input, len as usize)?;
Ok((input, data))
}
fn read_magic_bytes(input: &[u8]) -> Result<(&[u8], bool), AnyError> {
if input.len() < MAGIC_BYTES.len() {
bail!("Unexpected end of data. Could not find magic bytes.");
}
let (magic_bytes, input) = input.split_at(MAGIC_BYTES.len());
if magic_bytes != MAGIC_BYTES {
return Ok((input, false));
}
Ok((input, true))
}
let (input, found) = read_magic_bytes(data)?;
if !found {
return Ok(None);
}
// 1. Metadata
let (input, data) = read_bytes_with_len(input).context("reading metadata")?;
let metadata: Metadata =
serde_json::from_slice(data).context("deserializing metadata")?;
// 2. Npm snapshot
let (input, data) =
read_bytes_with_len(input).context("reading npm snapshot")?;
let npm_snapshot = if data.is_empty() {
None
} else {
Some(deserialize_npm_snapshot(data).context("deserializing npm snapshot")?)
};
// 3. Remote modules
let (input, data) =
read_bytes_with_len(input).context("reading remote modules data")?;
let remote_modules =
RemoteModulesStore::build(data).context("deserializing remote modules")?;
// 4. VFS
let (input, data) = read_bytes_with_len(input).context("vfs")?;
let vfs_dir: VirtualDirectory =
serde_json::from_slice(data).context("deserializing vfs data")?;
let (input, vfs_files_data) =
read_bytes_with_len(input).context("reading vfs files data")?;
// finally ensure we read the magic bytes at the end
let (_input, found) = read_magic_bytes(input)?;
if !found {
bail!("Could not find magic bytes at the end of the data.");
}
Ok(Some(DeserializedDataSection {
metadata,
npm_snapshot,
remote_modules,
vfs_dir,
vfs_files_data,
}))
}
#[derive(Default)]
pub struct RemoteModulesStoreBuilder {
specifiers: Vec<(String, u64)>,
data: Vec<(MediaType, Vec<u8>)>,
data_byte_len: u64,
redirects: Vec<(String, String)>,
redirects_len: u64,
}
impl RemoteModulesStoreBuilder {
pub fn add(&mut self, specifier: &Url, media_type: MediaType, data: Vec<u8>) {
log::debug!("Adding '{}' ({})", specifier, media_type);
let specifier = specifier.to_string();
self.specifiers.push((specifier, self.data_byte_len));
self.data_byte_len += 1 + 8 + data.len() as u64; // media type (1 byte), data length (8 bytes), data
self.data.push((media_type, data));
}
pub fn add_redirects(&mut self, redirects: &BTreeMap<Url, Url>) {
self.redirects.reserve(redirects.len());
for (from, to) in redirects {
log::debug!("Adding redirect '{}' -> '{}'", from, to);
let from = from.to_string();
let to = to.to_string();
self.redirects_len += (4 + from.len() + 4 + to.len()) as u64;
self.redirects.push((from, to));
}
}
fn write(&self, writer: &mut dyn Write) -> Result<(), AnyError> {
writer.write_all(&(self.specifiers.len() as u32).to_le_bytes())?;
writer.write_all(&(self.redirects.len() as u32).to_le_bytes())?;
for (specifier, offset) in &self.specifiers {
writer.write_all(&(specifier.len() as u32).to_le_bytes())?;
writer.write_all(specifier.as_bytes())?;
writer.write_all(&offset.to_le_bytes())?;
}
for (from, to) in &self.redirects {
writer.write_all(&(from.len() as u32).to_le_bytes())?;
writer.write_all(from.as_bytes())?;
writer.write_all(&(to.len() as u32).to_le_bytes())?;
writer.write_all(to.as_bytes())?;
}
for (media_type, data) in &self.data {
writer.write_all(&[serialize_media_type(*media_type)])?;
writer.write_all(&(data.len() as u64).to_le_bytes())?;
writer.write_all(data)?;
}
Ok(())
}
}
pub enum DenoCompileModuleSource {
String(&'static str),
Bytes(Cow<'static, [u8]>),
}
impl DenoCompileModuleSource {
pub fn into_for_v8(self) -> ModuleSourceCode {
fn into_bytes(data: Cow<'static, [u8]>) -> ModuleSourceCode {
ModuleSourceCode::Bytes(match data {
Cow::Borrowed(d) => d.into(),
Cow::Owned(d) => d.into_boxed_slice().into(),
})
}
match self {
// todo(https://github.com/denoland/deno_core/pull/943): store whether
// the string is ascii or not ahead of time so we can avoid the is_ascii()
// check in FastString::from_static
Self::String(s) => ModuleSourceCode::String(FastString::from_static(s)),
Self::Bytes(b) => into_bytes(b),
}
}
}
pub struct DenoCompileModuleData<'a> {
pub specifier: &'a Url,
pub media_type: MediaType,
pub data: Cow<'static, [u8]>,
}
impl<'a> DenoCompileModuleData<'a> {
pub fn into_parts(self) -> (&'a Url, ModuleType, DenoCompileModuleSource) {
fn into_string_unsafe(data: Cow<'static, [u8]>) -> DenoCompileModuleSource {
match data {
Cow::Borrowed(d) => DenoCompileModuleSource::String(
// SAFETY: we know this is a valid utf8 string
unsafe { std::str::from_utf8_unchecked(d) },
),
Cow::Owned(d) => DenoCompileModuleSource::Bytes(Cow::Owned(d)),
}
}
let (media_type, source) = match self.media_type {
MediaType::JavaScript
| MediaType::Jsx
| MediaType::Mjs
| MediaType::Cjs
| MediaType::TypeScript
| MediaType::Mts
| MediaType::Cts
| MediaType::Dts
| MediaType::Dmts
| MediaType::Dcts
| MediaType::Tsx => {
(ModuleType::JavaScript, into_string_unsafe(self.data))
}
MediaType::Json => (ModuleType::Json, into_string_unsafe(self.data)),
MediaType::Wasm => {
(ModuleType::Wasm, DenoCompileModuleSource::Bytes(self.data))
}
// just assume javascript if we made it here
MediaType::Css | MediaType::SourceMap | MediaType::Unknown => (
ModuleType::JavaScript,
DenoCompileModuleSource::Bytes(self.data),
),
};
(self.specifier, media_type, source)
}
}
enum RemoteModulesStoreSpecifierValue {
Data(usize),
Redirect(Url),
}
pub struct RemoteModulesStore {
specifiers: HashMap<Url, RemoteModulesStoreSpecifierValue>,
files_data: &'static [u8],
}
impl RemoteModulesStore {
fn build(data: &'static [u8]) -> Result<Self, AnyError> {
fn read_specifier(input: &[u8]) -> Result<(&[u8], (Url, u64)), AnyError> {
let (input, specifier) = read_string_lossy(input)?;
let specifier = Url::parse(&specifier)?;
let (input, offset) = read_u64(input)?;
Ok((input, (specifier, offset)))
}
fn read_redirect(input: &[u8]) -> Result<(&[u8], (Url, Url)), AnyError> {
let (input, from) = read_string_lossy(input)?;
let from = Url::parse(&from)?;
let (input, to) = read_string_lossy(input)?;
let to = Url::parse(&to)?;
Ok((input, (from, to)))
}
fn read_headers(
input: &[u8],
) -> Result<(&[u8], HashMap<Url, RemoteModulesStoreSpecifierValue>), AnyError>
{
let (input, specifiers_len) = read_u32_as_usize(input)?;
let (mut input, redirects_len) = read_u32_as_usize(input)?;
let mut specifiers =
HashMap::with_capacity(specifiers_len + redirects_len);
for _ in 0..specifiers_len {
let (current_input, (specifier, offset)) =
read_specifier(input).context("reading specifier")?;
input = current_input;
specifiers.insert(
specifier,
RemoteModulesStoreSpecifierValue::Data(offset as usize),
);
}
for _ in 0..redirects_len {
let (current_input, (from, to)) = read_redirect(input)?;
input = current_input;
specifiers.insert(from, RemoteModulesStoreSpecifierValue::Redirect(to));
}
Ok((input, specifiers))
}
let (files_data, specifiers) = read_headers(data)?;
Ok(Self {
specifiers,
files_data,
})
}
pub fn resolve_specifier<'a>(
&'a self,
specifier: &'a Url,
) -> Result<Option<&'a Url>, AnyError> {
let mut count = 0;
let mut current = specifier;
loop {
if count > 10 {
bail!("Too many redirects resolving '{}'", specifier);
}
match self.specifiers.get(current) {
Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => {
current = to;
count += 1;
}
Some(RemoteModulesStoreSpecifierValue::Data(_)) => {
return Ok(Some(current));
}
None => {
return Ok(None);
}
}
}
}
pub fn read<'a>(
&'a self,
original_specifier: &'a Url,
) -> Result<Option<DenoCompileModuleData<'a>>, AnyError> {
let mut count = 0;
let mut specifier = original_specifier;
loop {
if count > 10 {
bail!("Too many redirects resolving '{}'", original_specifier);
}
match self.specifiers.get(specifier) {
Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => {
specifier = to;
count += 1;
}
Some(RemoteModulesStoreSpecifierValue::Data(offset)) => {
let input = &self.files_data[*offset..];
let (input, media_type_byte) = read_bytes(input, 1)?;
let media_type = deserialize_media_type(media_type_byte[0])?;
let (input, len) = read_u64(input)?;
let (_input, data) = read_bytes(input, len as usize)?;
return Ok(Some(DenoCompileModuleData {
specifier,
media_type,
data: Cow::Borrowed(data),
}));
}
None => {
return Ok(None);
}
}
}
}
}
fn serialize_npm_snapshot(
mut snapshot: SerializedNpmResolutionSnapshot,
) -> Vec<u8> {
fn append_string(bytes: &mut Vec<u8>, string: &str) {
let len = string.len() as u32;
bytes.extend_from_slice(&len.to_le_bytes());
bytes.extend_from_slice(string.as_bytes());
}
snapshot.packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism
let ids_to_stored_ids = snapshot
.packages
.iter()
.enumerate()
.map(|(i, pkg)| (&pkg.id, i as u32))
.collect::<HashMap<_, _>>();
let mut root_packages: Vec<_> = snapshot.root_packages.iter().collect();
root_packages.sort();
let mut bytes = Vec::new();
bytes.extend_from_slice(&(snapshot.packages.len() as u32).to_le_bytes());
for pkg in &snapshot.packages {
append_string(&mut bytes, &pkg.id.as_serialized());
}
bytes.extend_from_slice(&(root_packages.len() as u32).to_le_bytes());
for (req, id) in root_packages {
append_string(&mut bytes, &req.to_string());
let id = ids_to_stored_ids.get(&id).unwrap();
bytes.extend_from_slice(&id.to_le_bytes());
}
for pkg in &snapshot.packages {
let deps_len = pkg.dependencies.len() as u32;
bytes.extend_from_slice(&deps_len.to_le_bytes());
let mut deps: Vec<_> = pkg.dependencies.iter().collect();
deps.sort();
for (req, id) in deps {
append_string(&mut bytes, req);
let id = ids_to_stored_ids.get(&id).unwrap();
bytes.extend_from_slice(&id.to_le_bytes());
}
}
bytes
}
fn deserialize_npm_snapshot(
input: &[u8],
) -> Result<ValidSerializedNpmResolutionSnapshot, AnyError> {
fn parse_id(input: &[u8]) -> Result<(&[u8], NpmPackageId), AnyError> {
let (input, id) = read_string_lossy(input)?;
let id = NpmPackageId::from_serialized(&id)?;
Ok((input, id))
}
#[allow(clippy::needless_lifetimes)] // clippy bug
fn parse_root_package<'a>(
id_to_npm_id: &'a impl Fn(usize) -> Result<NpmPackageId, AnyError>,
) -> impl Fn(&[u8]) -> Result<(&[u8], (PackageReq, NpmPackageId)), AnyError> + 'a
{
|input| {
let (input, req) = read_string_lossy(input)?;
let req = PackageReq::from_str(&req)?;
let (input, id) = read_u32_as_usize(input)?;
Ok((input, (req, id_to_npm_id(id)?)))
}
}
#[allow(clippy::needless_lifetimes)] // clippy bug
fn parse_package_dep<'a>(
id_to_npm_id: &'a impl Fn(usize) -> Result<NpmPackageId, AnyError>,
) -> impl Fn(&[u8]) -> Result<(&[u8], (String, NpmPackageId)), AnyError> + 'a
{
|input| {
let (input, req) = read_string_lossy(input)?;
let (input, id) = read_u32_as_usize(input)?;
Ok((input, (req.into_owned(), id_to_npm_id(id)?)))
}
}
fn parse_package<'a>(
input: &'a [u8],
id: NpmPackageId,
id_to_npm_id: &impl Fn(usize) -> Result<NpmPackageId, AnyError>,
) -> Result<(&'a [u8], SerializedNpmResolutionSnapshotPackage), AnyError> {
let (input, deps_len) = read_u32_as_usize(input)?;
let (input, dependencies) =
parse_hashmap_n_times(input, deps_len, parse_package_dep(id_to_npm_id))?;
Ok((
input,
SerializedNpmResolutionSnapshotPackage {
id,
system: Default::default(),
dist: Default::default(),
dependencies,
optional_dependencies: Default::default(),
bin: None,
scripts: Default::default(),
deprecated: Default::default(),
},
))
}
let (input, packages_len) = read_u32_as_usize(input)?;
// get a hashmap of all the npm package ids to their serialized ids
let (input, data_ids_to_npm_ids) =
parse_vec_n_times(input, packages_len, parse_id)
.context("deserializing id")?;
let data_id_to_npm_id = |id: usize| {
data_ids_to_npm_ids
.get(id)
.cloned()
.ok_or_else(|| deno_core::anyhow::anyhow!("Invalid npm package id"))
};
let (input, root_packages_len) = read_u32_as_usize(input)?;
let (input, root_packages) = parse_hashmap_n_times(
input,
root_packages_len,
parse_root_package(&data_id_to_npm_id),
)
.context("deserializing root package")?;
let (input, packages) =
parse_vec_n_times_with_index(input, packages_len, |input, index| {
parse_package(input, data_id_to_npm_id(index)?, &data_id_to_npm_id)
})
.context("deserializing package")?;
if !input.is_empty() {
bail!("Unexpected data left over");
}
Ok(
SerializedNpmResolutionSnapshot {
packages,
root_packages,
}
// this is ok because we have already verified that all the
// identifiers found in the snapshot are valid via the
// npm package id -> npm package id mapping
.into_valid_unsafe(),
)
}
fn serialize_media_type(media_type: MediaType) -> u8 {
match media_type {
MediaType::JavaScript => 0,
MediaType::Jsx => 1,
MediaType::Mjs => 2,
MediaType::Cjs => 3,
MediaType::TypeScript => 4,
MediaType::Mts => 5,
MediaType::Cts => 6,
MediaType::Dts => 7,
MediaType::Dmts => 8,
MediaType::Dcts => 9,
MediaType::Tsx => 10,
MediaType::Json => 11,
MediaType::Wasm => 12,
MediaType::Css => 13,
MediaType::SourceMap => 14,
MediaType::Unknown => 15,
}
}
fn deserialize_media_type(value: u8) -> Result<MediaType, AnyError> {
match value {
0 => Ok(MediaType::JavaScript),
1 => Ok(MediaType::Jsx),
2 => Ok(MediaType::Mjs),
3 => Ok(MediaType::Cjs),
4 => Ok(MediaType::TypeScript),
5 => Ok(MediaType::Mts),
6 => Ok(MediaType::Cts),
7 => Ok(MediaType::Dts),
8 => Ok(MediaType::Dmts),
9 => Ok(MediaType::Dcts),
10 => Ok(MediaType::Tsx),
11 => Ok(MediaType::Json),
12 => Ok(MediaType::Wasm),
13 => Ok(MediaType::Css),
14 => Ok(MediaType::SourceMap),
15 => Ok(MediaType::Unknown),
_ => bail!("Unknown media type value: {}", value),
}
}
fn parse_hashmap_n_times<TKey: std::cmp::Eq + std::hash::Hash, TValue>(
mut input: &[u8],
times: usize,
parse: impl Fn(&[u8]) -> Result<(&[u8], (TKey, TValue)), AnyError>,
) -> Result<(&[u8], HashMap<TKey, TValue>), AnyError> {
let mut results = HashMap::with_capacity(times);
for _ in 0..times {
let result = parse(input);
let (new_input, (key, value)) = result?;
results.insert(key, value);
input = new_input;
}
Ok((input, results))
}
fn parse_vec_n_times<TResult>(
input: &[u8],
times: usize,
parse: impl Fn(&[u8]) -> Result<(&[u8], TResult), AnyError>,
) -> Result<(&[u8], Vec<TResult>), AnyError> {
parse_vec_n_times_with_index(input, times, |input, _index| parse(input))
}
fn parse_vec_n_times_with_index<TResult>(
mut input: &[u8],
times: usize,
parse: impl Fn(&[u8], usize) -> Result<(&[u8], TResult), AnyError>,
) -> Result<(&[u8], Vec<TResult>), AnyError> {
let mut results = Vec::with_capacity(times);
for i in 0..times {
let result = parse(input, i);
let (new_input, result) = result?;
results.push(result);
input = new_input;
}
Ok((input, results))
}
fn read_bytes(input: &[u8], len: usize) -> Result<(&[u8], &[u8]), AnyError> {
if input.len() < len {
bail!("Unexpected end of data.",);
}
let (len_bytes, input) = input.split_at(len);
Ok((input, len_bytes))
}
fn read_string_lossy(input: &[u8]) -> Result<(&[u8], Cow<str>), AnyError> {
let (input, str_len) = read_u32_as_usize(input)?;
let (input, data_bytes) = read_bytes(input, str_len)?;
Ok((input, String::from_utf8_lossy(data_bytes)))
}
fn read_u32_as_usize(input: &[u8]) -> Result<(&[u8], usize), AnyError> {
let (input, len_bytes) = read_bytes(input, 4)?;
let len = u32::from_le_bytes(len_bytes.try_into()?);
Ok((input, len as usize))
}
fn read_u64(input: &[u8]) -> Result<(&[u8], u64), AnyError> {
let (input, len_bytes) = read_bytes(input, 8)?;
let len = u64::from_le_bytes(len_bytes.try_into()?);
Ok((input, len))
}

115
cli/standalone/virtual_fs.rs
View file

@ -7,6 +7,7 @@ use std::fs::File;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::ops::Range;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
@ -67,6 +68,26 @@ impl VfsBuilder {
})
}
pub fn set_new_root_path(
&mut self,
root_path: PathBuf,
) -> Result<(), AnyError> {
let root_path = canonicalize_path(&root_path)?;
self.root_path = root_path;
self.root_dir = VirtualDirectory {
name: self
.root_path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or("root".to_string()),
entries: vec![VfsEntry::Dir(VirtualDirectory {
name: std::mem::take(&mut self.root_dir.name),
entries: std::mem::take(&mut self.root_dir.entries),
})],
};
Ok(())
}
pub fn with_root_dir<R>(
&mut self,
with_root: impl FnOnce(&mut VirtualDirectory) -> R,
@ -119,7 +140,7 @@ impl VfsBuilder {
// inline the symlink and make the target file
let file_bytes = std::fs::read(&target)
.with_context(|| format!("Reading {}", path.display()))?;
self.add_file(&path, file_bytes)?;
self.add_file_with_data_inner(&path, file_bytes)?;
} else {
log::warn!(
"{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.",
@ -191,16 +212,32 @@ impl VfsBuilder {
self.add_file_at_path_not_symlink(&target_path)
}
pub fn add_file_at_path_not_symlink(
fn add_file_at_path_not_symlink(
&mut self,
path: &Path,
) -> Result<(), AnyError> {
let file_bytes = std::fs::read(path)
.with_context(|| format!("Reading {}", path.display()))?;
self.add_file(path, file_bytes)
self.add_file_with_data_inner(path, file_bytes)
}
fn add_file(&mut self, path: &Path, data: Vec<u8>) -> Result<(), AnyError> {
pub fn add_file_with_data(
&mut self,
path: &Path,
data: Vec<u8>,
) -> Result<(), AnyError> {
let target_path = canonicalize_path(path)?;
if target_path != path {
self.add_symlink(path, &target_path)?;
}
self.add_file_with_data_inner(&target_path, data)
}
fn add_file_with_data_inner(
&mut self,
path: &Path,
data: Vec<u8>,
) -> Result<(), AnyError> {
log::debug!("Adding file '{}'", path.display());
let checksum = util::checksum::gen(&[&data]);
let offset = if let Some(offset) = self.file_offsets.get(&checksum) {
@ -249,8 +286,15 @@ impl VfsBuilder {
path.display(),
target.display()
);
let dest = self.path_relative_root(target)?;
if dest == self.path_relative_root(path)? {
let relative_target = self.path_relative_root(target)?;
let relative_path = match self.path_relative_root(path) {
Ok(path) => path,
Err(StripRootError { .. }) => {
// ignore if the original path is outside the root directory
return Ok(());
}
};
if relative_target == relative_path {
// it's the same, ignore
return Ok(());
}
@ -263,7 +307,7 @@ impl VfsBuilder {
insert_index,
VfsEntry::Symlink(VirtualSymlink {
name: name.to_string(),
dest_parts: dest
dest_parts: relative_target
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>(),
@ -751,14 +795,14 @@ impl deno_io::fs::File for FileBackedVfsFile {
#[derive(Debug)]
pub struct FileBackedVfs {
file: Mutex<Vec<u8>>,
vfs_data: Cow<'static, [u8]>,
fs_root: VfsRoot,
}
impl FileBackedVfs {
pub fn new(file: Vec<u8>, fs_root: VfsRoot) -> Self {
pub fn new(data: Cow<'static, [u8]>, fs_root: VfsRoot) -> Self {
Self {
file: Mutex::new(file),
vfs_data: data,
fs_root,
}
}
@ -827,10 +871,15 @@ impl FileBackedVfs {
Ok(path)
}
pub fn read_file_all(&self, file: &VirtualFile) -> std::io::Result<Vec<u8>> {
let mut buf = vec![0; file.len as usize];
self.read_file(file, 0, &mut buf)?;
Ok(buf)
pub fn read_file_all(
&self,
file: &VirtualFile,
) -> std::io::Result<Cow<'static, [u8]>> {
let read_range = self.get_read_range(file, 0, file.len)?;
match &self.vfs_data {
Cow::Borrowed(data) => Ok(Cow::Borrowed(&data[read_range])),
Cow::Owned(data) => Ok(Cow::Owned(data[read_range].to_vec())),
}
}
pub fn read_file(
@ -839,18 +888,27 @@ impl FileBackedVfs {
pos: u64,
buf: &mut [u8],
) -> std::io::Result<usize> {
let data = self.file.lock();
let read_range = self.get_read_range(file, pos, buf.len() as u64)?;
buf.copy_from_slice(&self.vfs_data[read_range]);
Ok(buf.len())
}
fn get_read_range(
&self,
file: &VirtualFile,
pos: u64,
len: u64,
) -> std::io::Result<Range<usize>> {
let data = &self.vfs_data;
let start = self.fs_root.start_file_offset + file.offset + pos;
let end = start + buf.len() as u64;
let end = start + len;
if end > data.len() as u64 {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"unexpected EOF",
));
}
buf.copy_from_slice(&data[start as usize..end as usize]);
Ok(buf.len())
Ok(start as usize..end as usize)
}
pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> {
@ -888,7 +946,7 @@ mod test {
#[track_caller]
fn read_file(vfs: &FileBackedVfs, path: &Path) -> String {
let file = vfs.file_entry(path).unwrap();
String::from_utf8(vfs.read_file_all(file).unwrap()).unwrap()
String::from_utf8(vfs.read_file_all(file).unwrap().into_owned()).unwrap()
}
#[test]
@ -901,20 +959,23 @@ mod test {
let src_path = src_path.to_path_buf();
let mut builder = VfsBuilder::new(src_path.clone()).unwrap();
builder
.add_file(&src_path.join("a.txt"), "data".into())
.add_file_with_data_inner(&src_path.join("a.txt"), "data".into())
.unwrap();
builder
.add_file(&src_path.join("b.txt"), "data".into())
.add_file_with_data_inner(&src_path.join("b.txt"), "data".into())
.unwrap();
assert_eq!(builder.files.len(), 1); // because duplicate data
builder
.add_file(&src_path.join("c.txt"), "c".into())
.add_file_with_data_inner(&src_path.join("c.txt"), "c".into())
.unwrap();
builder
.add_file(&src_path.join("sub_dir").join("d.txt"), "d".into())
.add_file_with_data_inner(
&src_path.join("sub_dir").join("d.txt"),
"d".into(),
)
.unwrap();
builder
.add_file(&src_path.join("e.txt"), "e".into())
.add_file_with_data_inner(&src_path.join("e.txt"), "e".into())
.unwrap();
builder
.add_symlink(
@ -1031,7 +1092,7 @@ mod test {
(
dest_path.to_path_buf(),
FileBackedVfs::new(
data,
Cow::Owned(data),
VfsRoot {
dir: root_dir,
root_path: dest_path.to_path_buf(),
@ -1082,7 +1143,7 @@ mod test {
let temp_path = temp_dir.path().canonicalize();
let mut builder = VfsBuilder::new(temp_path.to_path_buf()).unwrap();
builder
.add_file(
.add_file_with_data_inner(
temp_path.join("a.txt").as_path(),
"0123456789".to_string().into_bytes(),
)

103
cli/task_runner.rs
View file

@ -16,8 +16,11 @@ use deno_task_shell::ExecutableCommand;
use deno_task_shell::ExecuteResult;
use deno_task_shell::ShellCommand;
use deno_task_shell::ShellCommandContext;
use deno_task_shell::ShellPipeReader;
use deno_task_shell::ShellPipeWriter;
use lazy_regex::Lazy;
use regex::Regex;
use tokio::task::JoinHandle;
use tokio::task::LocalSet;
use crate::npm::CliNpmResolver;
@ -36,6 +39,35 @@ pub fn get_script_with_args(script: &str, argv: &[String]) -> String {
script.trim().to_owned()
}
pub struct TaskStdio(Option<ShellPipeReader>, ShellPipeWriter);
impl TaskStdio {
pub fn stdout() -> Self {
Self(None, ShellPipeWriter::stdout())
}
pub fn stderr() -> Self {
Self(None, ShellPipeWriter::stderr())
}
pub fn piped() -> Self {
let (r, w) = deno_task_shell::pipe();
Self(Some(r), w)
}
}
pub struct TaskIo {
pub stdout: TaskStdio,
pub stderr: TaskStdio,
}
impl Default for TaskIo {
fn default() -> Self {
Self {
stderr: TaskStdio::stderr(),
stdout: TaskStdio::stdout(),
}
}
}
pub struct RunTaskOptions<'a> {
pub task_name: &'a str,
pub script: &'a str,
@ -45,24 +77,69 @@ pub struct RunTaskOptions<'a> {
pub argv: &'a [String],
pub custom_commands: HashMap<String, Rc<dyn ShellCommand>>,
pub root_node_modules_dir: Option<&'a Path>,
pub stdio: Option<TaskIo>,
}
pub type TaskCustomCommands = HashMap<String, Rc<dyn ShellCommand>>;
pub async fn run_task(opts: RunTaskOptions<'_>) -> Result<i32, AnyError> {
pub struct TaskResult {
pub exit_code: i32,
pub stdout: Option<Vec<u8>>,
pub stderr: Option<Vec<u8>>,
}
pub async fn run_task(
opts: RunTaskOptions<'_>,
) -> Result<TaskResult, AnyError> {
let script = get_script_with_args(opts.script, opts.argv);
let seq_list = deno_task_shell::parser::parse(&script)
.with_context(|| format!("Error parsing script '{}'.", opts.task_name))?;
let env_vars =
prepare_env_vars(opts.env_vars, opts.init_cwd, opts.root_node_modules_dir);
let state =
deno_task_shell::ShellState::new(env_vars, opts.cwd, opts.custom_commands);
let stdio = opts.stdio.unwrap_or_default();
let (
TaskStdio(stdout_read, stdout_write),
TaskStdio(stderr_read, stderr_write),
) = (stdio.stdout, stdio.stderr);
fn read(reader: ShellPipeReader) -> JoinHandle<Result<Vec<u8>, AnyError>> {
tokio::task::spawn_blocking(move || {
let mut buf = Vec::new();
reader.pipe_to(&mut buf)?;
Ok(buf)
})
}
let stdout = stdout_read.map(read);
let stderr = stderr_read.map(read);
let local = LocalSet::new();
let future = deno_task_shell::execute(
seq_list,
env_vars,
opts.cwd,
opts.custom_commands,
);
Ok(local.run_until(future).await)
let future = async move {
let exit_code = deno_task_shell::execute_with_pipes(
seq_list,
state,
ShellPipeReader::stdin(),
stdout_write,
stderr_write,
)
.await;
Ok::<_, AnyError>(TaskResult {
exit_code,
stdout: if let Some(stdout) = stdout {
Some(stdout.await??)
} else {
None
},
stderr: if let Some(stderr) = stderr {
Some(stderr.await??)
} else {
None
},
})
};
local.run_until(future).await
}
fn prepare_env_vars(
@ -78,6 +155,12 @@ fn prepare_env_vars(
initial_cwd.to_string_lossy().to_string(),
);
}
if !env_vars.contains_key(crate::npm::NPM_CONFIG_USER_AGENT_ENV_VAR) {
env_vars.insert(
crate::npm::NPM_CONFIG_USER_AGENT_ENV_VAR.into(),
crate::npm::get_npm_config_user_agent(),
);
}
if let Some(node_modules_dir) = node_modules_dir {
prepend_to_path(
&mut env_vars,
@ -127,7 +210,7 @@ impl ShellCommand for NpmCommand {
mut context: ShellCommandContext,
) -> LocalBoxFuture<'static, ExecuteResult> {
if context.args.first().map(|s| s.as_str()) == Some("run")
&& context.args.len() > 2
&& context.args.len() >= 2
// for now, don't run any npm scripts that have a flag because
// we don't handle stuff like `--workspaces` properly
&& !context.args.iter().any(|s| s.starts_with('-'))
@ -190,10 +273,12 @@ impl ShellCommand for NodeCommand {
)
.execute(context);
}
args.extend(["run", "-A"].into_iter().map(|s| s.to_string()));
args.extend(context.args.iter().cloned());
let mut state = context.state;
state.apply_env_var(USE_PKG_JSON_HIDDEN_ENV_VAR_NAME, "1");
ExecutableCommand::new("deno".to_string(), std::env::current_exe().unwrap())
.execute(ShellCommandContext {

2
cli/tools/bench/mod.rs
View file

@ -193,7 +193,7 @@ async fn bench_specifier_inner(
.await?;
// We execute the main module as a side module so that import.meta.main is not set.
worker.execute_side_module_possibly_with_npm().await?;
worker.execute_side_module().await?;
let mut worker = worker.into_main_worker();

11
cli/tools/check.rs
View file

@ -32,6 +32,7 @@ use crate::graph_util::ModuleGraphBuilder;
use crate::npm::CliNpmResolver;
use crate::tsc;
use crate::tsc::Diagnostics;
use crate::tsc::TypeCheckingCjsTracker;
use crate::util::extract;
use crate::util::path::to_percent_decoded_str;
@ -99,6 +100,7 @@ pub struct CheckOptions {
pub struct TypeChecker {
caches: Arc<Caches>,
cjs_tracker: Arc<TypeCheckingCjsTracker>,
cli_options: Arc<CliOptions>,
module_graph_builder: Arc<ModuleGraphBuilder>,
node_resolver: Arc<NodeResolver>,
@ -108,6 +110,7 @@ pub struct TypeChecker {
impl TypeChecker {
pub fn new(
caches: Arc<Caches>,
cjs_tracker: Arc<TypeCheckingCjsTracker>,
cli_options: Arc<CliOptions>,
module_graph_builder: Arc<ModuleGraphBuilder>,
node_resolver: Arc<NodeResolver>,
@ -115,6 +118,7 @@ impl TypeChecker {
) -> Self {
Self {
caches,
cjs_tracker,
cli_options,
module_graph_builder,
node_resolver,
@ -244,6 +248,7 @@ impl TypeChecker {
graph: graph.clone(),
hash_data,
maybe_npm: Some(tsc::RequestNpmState {
cjs_tracker: self.cjs_tracker.clone(),
node_resolver: self.node_resolver.clone(),
npm_resolver: self.npm_resolver.clone(),
}),
@ -346,7 +351,7 @@ fn get_check_hash(
}
}
MediaType::Json
| MediaType::TsBuildInfo
| MediaType::Css
| MediaType::SourceMap
| MediaType::Wasm
| MediaType::Unknown => continue,
@ -428,7 +433,7 @@ fn get_tsc_roots(
}
MediaType::Json
| MediaType::Wasm
| MediaType::TsBuildInfo
| MediaType::Css
| MediaType::SourceMap
| MediaType::Unknown => None,
},
@ -536,7 +541,7 @@ fn has_ts_check(media_type: MediaType, file_text: &str) -> bool {
| MediaType::Tsx
| MediaType::Json
| MediaType::Wasm
| MediaType::TsBuildInfo
| MediaType::Css
| MediaType::SourceMap
| MediaType::Unknown => false,
}

30
cli/tools/compile.rs
View file

@ -5,6 +5,7 @@ use crate::args::CompileFlags;
use crate::args::Flags;
use crate::factory::CliFactory;
use crate::http_util::HttpClientProvider;
use crate::standalone::binary::StandaloneRelativeFileBaseUrl;
use crate::standalone::is_standalone_binary;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
@ -14,7 +15,6 @@ use deno_core::error::AnyError;
use deno_core::resolve_url_or_path;
use deno_graph::GraphKind;
use deno_terminal::colors;
use eszip::EszipRelativeFileBaseUrl;
use rand::Rng;
use std::path::Path;
use std::path::PathBuf;
@ -29,7 +29,6 @@ pub async fn compile(
let factory = CliFactory::from_flags(flags);
let cli_options = factory.cli_options()?;
let module_graph_creator = factory.module_graph_creator().await?;
let parsed_source_cache = factory.parsed_source_cache();
let binary_writer = factory.create_compile_binary_writer().await?;
let http_client = factory.http_client_provider();
let module_specifier = cli_options.resolve_main_module()?;
@ -70,7 +69,7 @@ pub async fn compile(
let graph = if cli_options.type_check_mode().is_true() {
// In this case, the previous graph creation did type checking, which will
// create a module graph with types information in it. We don't want to
// store that in the eszip so create a code only module graph from scratch.
// store that in the binary so create a code only module graph from scratch.
module_graph_creator
.create_graph(GraphKind::CodeOnly, module_roots)
.await?
@ -81,11 +80,6 @@ pub async fn compile(
let ts_config_for_emit = cli_options
.resolve_ts_config_for_emit(deno_config::deno_json::TsConfigType::Emit)?;
check_warn_tsconfig(&ts_config_for_emit);
let (transpile_options, emit_options) =
crate::args::ts_config_to_transpile_and_emit_options(
ts_config_for_emit.ts_config,
)?;
let parser = parsed_source_cache.as_capturing_parser();
let root_dir_url = resolve_root_dir_from_specifiers(
cli_options.workspace().root_dir(),
graph.specifiers().map(|(s, _)| s).chain(
@ -96,17 +90,6 @@ pub async fn compile(
),
);
log::debug!("Binary root dir: {}", root_dir_url);
let root_dir_url = EszipRelativeFileBaseUrl::new(&root_dir_url);
let eszip = eszip::EszipV2::from_graph(eszip::FromGraphOptions {
graph,
parser,
transpile_options,
emit_options,
// make all the modules relative to the root folder
relative_file_base: Some(root_dir_url),
npm_packages: None,
})?;
log::info!(
"{} {} to {}",
colors::green("Compile"),
@ -133,15 +116,18 @@ pub async fn compile(
let write_result = binary_writer
.write_bin(
file,
eszip,
root_dir_url,
&graph,
StandaloneRelativeFileBaseUrl::from(&root_dir_url),
module_specifier,
&compile_flags,
cli_options,
)
.await
.with_context(|| {
format!("Writing temporary file '{}'", temp_path.display())
format!(
"Writing deno compile executable to temporary file '{}'",
temp_path.display()
)
});
// set it as executable

47
cli/tools/coverage/mod.rs
View file

@ -6,12 +6,12 @@ use crate::args::FileFlags;
use crate::args::Flags;
use crate::cdp;
use crate::factory::CliFactory;
use crate::npm::CliNpmResolver;
use crate::tools::fmt::format_json;
use crate::tools::test::is_supported_test_path;
use crate::util::text_encoding::source_map_from_code;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_ast::ModuleSpecifier;
use deno_config::glob::FileCollector;
use deno_config::glob::FilePatterns;
@ -25,6 +25,7 @@ use deno_core::serde_json;
use deno_core::sourcemap::SourceMap;
use deno_core::url::Url;
use deno_core::LocalInspectorSession;
use node_resolver::InNpmPackageChecker;
use regex::Regex;
use std::fs;
use std::fs::File;
@ -327,6 +328,7 @@ fn generate_coverage_report(
coverage_report.found_lines =
if let Some(source_map) = maybe_source_map.as_ref() {
let script_source_lines = script_source.lines().collect::<Vec<_>>();
let mut found_lines = line_counts
.iter()
.enumerate()
@ -334,7 +336,23 @@ fn generate_coverage_report(
// get all the mappings from this destination line to a different src line
let mut results = source_map
.tokens()
.filter(move |token| token.get_dst_line() as usize == index)
.filter(|token| {
let dst_line = token.get_dst_line() as usize;
dst_line == index && {
let dst_col = token.get_dst_col() as usize;
let content = script_source_lines
.get(dst_line)
.and_then(|line| {
line.get(dst_col..std::cmp::min(dst_col + 2, line.len()))
})
.unwrap_or("");
!content.is_empty()
&& content != "/*"
&& content != "*/"
&& content != "//"
}
})
.map(move |token| (token.get_src_line() as usize, *count))
.collect::<Vec<_>>();
// only keep the results that point at different src lines
@ -444,7 +462,7 @@ fn filter_coverages(
coverages: Vec<cdp::ScriptCoverage>,
include: Vec<String>,
exclude: Vec<String>,
npm_resolver: &dyn CliNpmResolver,
in_npm_pkg_checker: &dyn InNpmPackageChecker,
) -> Vec<cdp::ScriptCoverage> {
let include: Vec<Regex> =
include.iter().map(|e| Regex::new(e).unwrap()).collect();
@ -468,7 +486,7 @@ fn filter_coverages(
|| doc_test_re.is_match(e.url.as_str())
|| Url::parse(&e.url)
.ok()
.map(|url| npm_resolver.in_npm_package(&url))
.map(|url| in_npm_pkg_checker.in_npm_package(&url))
.unwrap_or(false);
let is_included = include.iter().any(|p| p.is_match(&e.url));
@ -479,7 +497,7 @@ fn filter_coverages(
.collect::<Vec<cdp::ScriptCoverage>>()
}
pub async fn cover_files(
pub fn cover_files(
flags: Arc<Flags>,
coverage_flags: CoverageFlags,
) -> Result<(), AnyError> {
@ -489,9 +507,10 @@ pub async fn cover_files(
let factory = CliFactory::from_flags(flags);
let cli_options = factory.cli_options()?;
let npm_resolver = factory.npm_resolver().await?;
let in_npm_pkg_checker = factory.in_npm_pkg_checker()?;
let file_fetcher = factory.file_fetcher()?;
let emitter = factory.emitter()?;
let cjs_tracker = factory.cjs_tracker()?;
assert!(!coverage_flags.files.include.is_empty());
@ -511,7 +530,7 @@ pub async fn cover_files(
script_coverages,
coverage_flags.include,
coverage_flags.exclude,
npm_resolver.as_ref(),
in_npm_pkg_checker.as_ref(),
);
if script_coverages.is_empty() {
return Err(generic_error("No covered files included in the report"));
@ -568,16 +587,21 @@ pub async fn cover_files(
let transpiled_code = match file.media_type {
MediaType::JavaScript
| MediaType::Unknown
| MediaType::Css
| MediaType::Wasm
| MediaType::Cjs
| MediaType::Mjs
| MediaType::Json => None,
MediaType::Dts | MediaType::Dmts | MediaType::Dcts => Some(Vec::new()),
MediaType::Dts | MediaType::Dmts | MediaType::Dcts => Some(String::new()),
MediaType::TypeScript
| MediaType::Jsx
| MediaType::Mts
| MediaType::Cts
| MediaType::Tsx => {
Some(match emitter.maybe_cached_emit(&file.specifier, &file.source) {
let module_kind = ModuleKind::from_is_cjs(
cjs_tracker.is_maybe_cjs(&file.specifier, file.media_type)?,
);
Some(match emitter.maybe_cached_emit(&file.specifier, module_kind, &file.source) {
Some(code) => code,
None => {
return Err(anyhow!(
@ -588,13 +612,12 @@ pub async fn cover_files(
}
})
}
MediaType::Wasm | MediaType::TsBuildInfo | MediaType::SourceMap => {
MediaType::SourceMap => {
unreachable!()
}
};
let runtime_code: String = match transpiled_code {
Some(code) => String::from_utf8(code)
.with_context(|| format!("Failed decoding {}", file.specifier))?,
Some(code) => code,
None => original_source.to_string(),
};

6
cli/tools/doc.rs
View file

@ -22,9 +22,9 @@ use deno_core::serde_json;
use deno_doc as doc;
use deno_doc::html::UrlResolveKind;
use deno_graph::source::NullFileSystem;
use deno_graph::EsParser;
use deno_graph::GraphKind;
use deno_graph::ModuleAnalyzer;
use deno_graph::ModuleParser;
use deno_graph::ModuleSpecifier;
use doc::html::ShortPath;
use doc::DocDiagnostic;
@ -37,7 +37,7 @@ const JSON_SCHEMA_VERSION: u8 = 1;
async fn generate_doc_nodes_for_builtin_types(
doc_flags: DocFlags,
parser: &dyn ModuleParser,
parser: &dyn EsParser,
analyzer: &dyn ModuleAnalyzer,
) -> Result<IndexMap<ModuleSpecifier, Vec<doc::DocNode>>, AnyError> {
let source_file_specifier =
@ -96,7 +96,7 @@ pub async fn doc(
let module_info_cache = factory.module_info_cache()?;
let parsed_source_cache = factory.parsed_source_cache();
let capturing_parser = parsed_source_cache.as_capturing_parser();
let analyzer = module_info_cache.as_module_analyzer(parsed_source_cache);
let analyzer = module_info_cache.as_module_analyzer();
let doc_nodes_by_url = match doc_flags.source_files {
DocSourceFileFlag::Builtin => {

2
cli/tools/fmt.rs
View file

@ -978,6 +978,7 @@ fn get_resolved_malva_config(
single_line_top_level_declarations: false,
selector_override_comment_directive: "deno-fmt-selector-override".into(),
ignore_comment_directive: "deno-fmt-ignore".into(),
ignore_file_comment_directive: "deno-fmt-ignore-file".into(),
};
FormatOptions {
@ -1036,6 +1037,7 @@ fn get_resolved_markup_fmt_config(
svelte_directive_shorthand: Some(true),
astro_attr_shorthand: Some(true),
ignore_comment_directive: "deno-fmt-ignore".into(),
ignore_file_comment_directive: "deno-fmt-ignore-file".into(),
};
FormatOptions {

96
cli/tools/info.rs
View file

@ -11,12 +11,14 @@ use deno_core::anyhow::bail;
use deno_core::error::AnyError;
use deno_core::resolve_url_or_path;
use deno_core::serde_json;
use deno_core::url;
use deno_graph::Dependency;
use deno_graph::GraphKind;
use deno_graph::Module;
use deno_graph::ModuleError;
use deno_graph::ModuleGraph;
use deno_graph::Resolution;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
@ -47,20 +49,23 @@ pub async fn info(
let module_graph_creator = factory.module_graph_creator().await?;
let npm_resolver = factory.npm_resolver().await?;
let maybe_lockfile = cli_options.maybe_lockfile();
let npmrc = cli_options.npmrc();
let resolver = factory.workspace_resolver().await?;
let maybe_import_specifier =
if let Some(import_map) = resolver.maybe_import_map() {
if let Ok(imports_specifier) =
import_map.resolve(&specifier, import_map.base_url())
{
Some(imports_specifier)
} else {
None
}
let cwd_url =
url::Url::from_directory_path(cli_options.initial_cwd()).unwrap();
let maybe_import_specifier = if let Some(import_map) =
resolver.maybe_import_map()
{
if let Ok(imports_specifier) = import_map.resolve(&specifier, &cwd_url) {
Some(imports_specifier)
} else {
None
};
}
} else {
None
};
let specifier = match maybe_import_specifier {
Some(specifier) => specifier,
@ -88,7 +93,8 @@ pub async fn info(
JSON_SCHEMA_VERSION.into(),
);
}
add_npm_packages_to_json(&mut json_graph, npm_resolver.as_ref());
add_npm_packages_to_json(&mut json_graph, npm_resolver.as_ref(), npmrc);
display::write_json_to_stdout(&json_graph)?;
} else {
let mut output = String::new();
@ -185,6 +191,7 @@ fn print_cache_info(
fn add_npm_packages_to_json(
json: &mut serde_json::Value,
npm_resolver: &dyn CliNpmResolver,
npmrc: &ResolvedNpmRc,
) {
let Some(npm_resolver) = npm_resolver.as_managed() else {
return; // does not include byonm to deno info's output
@ -195,45 +202,28 @@ fn add_npm_packages_to_json(
let json = json.as_object_mut().unwrap();
let modules = json.get_mut("modules").and_then(|m| m.as_array_mut());
if let Some(modules) = modules {
if modules.len() == 1
&& modules[0].get("kind").and_then(|k| k.as_str()) == Some("npm")
{
// If there is only one module and it's "external", then that means
// someone provided an npm specifier as a cli argument. In this case,
// we want to show which npm package the cli argument resolved to.
let module = &mut modules[0];
let maybe_package = module
.get("specifier")
.and_then(|k| k.as_str())
.and_then(|specifier| NpmPackageNvReference::from_str(specifier).ok())
.and_then(|package_ref| {
snapshot
.resolve_package_from_deno_module(package_ref.nv())
.ok()
});
if let Some(pkg) = maybe_package {
if let Some(module) = module.as_object_mut() {
module
.insert("npmPackage".to_string(), pkg.id.as_serialized().into());
}
}
} else {
// Filter out npm package references from the modules and instead
// have them only listed as dependencies. This is done because various
// npm specifiers modules in the graph are really just unresolved
// references. So there could be listed multiple npm specifiers
// that would resolve to a single npm package.
for i in (0..modules.len()).rev() {
if matches!(
modules[i].get("kind").and_then(|k| k.as_str()),
Some("npm") | Some("external")
) {
modules.remove(i);
}
}
}
for module in modules.iter_mut() {
if matches!(module.get("kind").and_then(|k| k.as_str()), Some("npm")) {
// If there is only one module and it's "external", then that means
// someone provided an npm specifier as a cli argument. In this case,
// we want to show which npm package the cli argument resolved to.
let maybe_package = module
.get("specifier")
.and_then(|k| k.as_str())
.and_then(|specifier| NpmPackageNvReference::from_str(specifier).ok())
.and_then(|package_ref| {
snapshot
.resolve_package_from_deno_module(package_ref.nv())
.ok()
});
if let Some(pkg) = maybe_package {
if let Some(module) = module.as_object_mut() {
module
.insert("npmPackage".to_string(), pkg.id.as_serialized().into());
}
}
}
let dependencies = module
.get_mut("dependencies")
.and_then(|d| d.as_array_mut());
@ -265,7 +255,7 @@ fn add_npm_packages_to_json(
let mut json_packages = serde_json::Map::with_capacity(sorted_packages.len());
for pkg in sorted_packages {
let mut kv = serde_json::Map::new();
kv.insert("name".to_string(), pkg.id.nv.name.to_string().into());
kv.insert("name".to_string(), pkg.id.nv.name.clone().into());
kv.insert("version".to_string(), pkg.id.nv.version.to_string().into());
let mut deps = pkg.dependencies.values().collect::<Vec<_>>();
deps.sort();
@ -274,6 +264,8 @@ fn add_npm_packages_to_json(
.map(|id| serde_json::Value::String(id.as_serialized()))
.collect::<Vec<_>>();
kv.insert("dependencies".to_string(), deps.into());
let registry_url = npmrc.get_registry_url(&pkg.id.nv.name);
kv.insert("registryUrl".to_string(), registry_url.to_string().into());
json_packages.insert(pkg.id.as_serialized(), kv.into());
}
@ -538,7 +530,7 @@ impl<'a> GraphDisplayContext<'a> {
fn build_module_info(&mut self, module: &Module, type_dep: bool) -> TreeNode {
enum PackageOrSpecifier {
Package(NpmResolutionPackage),
Package(Box<NpmResolutionPackage>),
Specifier(ModuleSpecifier),
}
@ -546,7 +538,7 @@ impl<'a> GraphDisplayContext<'a> {
let package_or_specifier = match module.npm() {
Some(npm) => match self.npm_info.resolve_package(npm.nv_reference.nv()) {
Some(package) => Package(package.clone()),
Some(package) => Package(Box::new(package.clone())),
None => Specifier(module.specifier().clone()), // should never happen
},
None => Specifier(module.specifier().clone()),

41
cli/tools/init/mod.rs
View file

@ -24,32 +24,29 @@ pub fn init_project(init_flags: InitFlags) -> Result<(), AnyError> {
create_file(
&dir,
"main.ts",
r#"import { type Route, route, serveDir } from "@std/http";
r#"import { serveDir } from "@std/http";
const routes: Route[] = [
{
pattern: new URLPattern({ pathname: "/" }),
handler: () => new Response("Home page"),
},
{
pattern: new URLPattern({ pathname: "/users/:id" }),
handler: (_req, _info, params) => new Response(params?.pathname.groups.id),
},
{
pattern: new URLPattern({ pathname: "/static/*" }),
handler: (req) => serveDir(req),
},
];
function defaultHandler(_req: Request) {
return new Response("Not found", { status: 404 });
}
const handler = route(routes, defaultHandler);
const userPagePattern = new URLPattern({ pathname: "/users/:id" });
const staticPathPattern = new URLPattern({ pathname: "/static/*" });
export default {
fetch(req) {
return handler(req);
const url = new URL(req.url);
if (url.pathname === "/") {
return new Response("Home page");
}
const userPageMatch = userPagePattern.exec(url);
if (userPageMatch) {
return new Response(userPageMatch.pathname.groups.id);
}
if (staticPathPattern.test(url)) {
return serveDir(req);
}
return new Response("Not found", { status: 404 });
},
} satisfies Deno.ServeDefaultExport;
"#,

6
cli/tools/jupyter/install.rs
View file

@ -58,9 +58,9 @@ pub fn install() -> Result<(), AnyError> {
let f = std::fs::File::create(kernel_json_path)?;
serde_json::to_writer_pretty(f, &json_data)?;
install_icon(&user_data_dir, "logo-32x32.png", DENO_ICON_32)?;
install_icon(&user_data_dir, "logo-64x64.png", DENO_ICON_64)?;
install_icon(&user_data_dir, "logo-svg.svg", DENO_ICON_SVG)?;
install_icon(&kernel_dir, "logo-32x32.png", DENO_ICON_32)?;
install_icon(&kernel_dir, "logo-64x64.png", DENO_ICON_64)?;
install_icon(&kernel_dir, "logo-svg.svg", DENO_ICON_SVG)?;
log::info!("✅ Deno kernelspec installed successfully.");
Ok(())

BIN
cli/tools/jupyter/resources/deno-logo-32x32.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1 KiB

After

Width:  |  Height:  |  Size: 1.4 KiB

BIN
cli/tools/jupyter/resources/deno-logo-64x64.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2 KiB

After

Width:  |  Height:  |  Size: 2.8 KiB

18
cli/tools/jupyter/resources/deno-logo-svg.svg
View file

@ -1 +1,17 @@
<svg viewBox="0 0 30 30" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#clip0_29_599)"><path d="M15 0C23.2843 0 30 6.71572 30 15C30 23.2843 23.2843 30 15 30C6.71572 30 0 23.2843 0 15C0 6.71572 6.71572 0 15 0Z" fill="currentColor"></path><path d="M14.6635 22.3394C14.2788 22.2357 13.8831 22.4584 13.7705 22.8381L13.7655 22.8558L12.7694 26.5472L12.7649 26.565C12.6711 26.9498 12.9011 27.3414 13.2858 27.4451C13.6704 27.549 14.0661 27.3263 14.1787 26.9465L14.1837 26.9289L15.1797 23.2375L15.1843 23.2196C15.1911 23.1919 15.1962 23.164 15.1997 23.1362L15.2026 23.1084L15.179 22.9888L15.1445 22.8166L15.1227 22.7091C15.076 22.619 15.0111 22.5396 14.932 22.4759C14.853 22.4123 14.7615 22.3658 14.6635 22.3394ZM7.7224 18.5379C7.70424 18.5741 7.68883 18.6123 7.67658 18.6522L7.66967 18.6763L6.67358 22.3677L6.669 22.3856C6.57525 22.7704 6.80524 23.1619 7.1899 23.2657C7.57451 23.3695 7.97026 23.1469 8.08287 22.7671L8.08779 22.7494L8.99096 19.4023C8.51793 19.1518 8.09336 18.8628 7.7224 18.5379ZM5.34707 14.2929C4.9624 14.1891 4.56666 14.4117 4.4541 14.7915L4.44912 14.8092L3.45303 18.5006L3.44846 18.5184C3.35471 18.9032 3.58469 19.2947 3.96936 19.3985C4.35397 19.5023 4.74971 19.2797 4.86232 18.8999L4.86725 18.8822L5.86334 15.1908L5.86791 15.173C5.96166 14.7882 5.73174 14.3967 5.34707 14.2929ZM27.682 13.4546C27.2973 13.3508 26.9015 13.5734 26.789 13.9532L26.784 13.9709L25.7879 17.6623L25.7833 17.6801C25.6896 18.0649 25.9196 18.4564 26.3042 18.5602C26.6889 18.664 27.0846 18.4414 27.1972 18.0616L27.2021 18.0439L28.1982 14.3525L28.2028 14.3347C28.2965 13.9499 28.0666 13.5584 27.682 13.4546ZM3.17781 8.52527C2.34361 10.0444 1.81243 11.7112 1.61377 13.4329C1.7088 13.5412 1.83381 13.619 1.97301 13.6563C2.35768 13.7602 2.75342 13.5375 2.86598 13.1577L2.87096 13.1401L3.86705 9.44865L3.87162 9.43084C3.96537 9.04599 3.73539 8.65447 3.35072 8.5507C3.2943 8.53547 3.23623 8.52694 3.17781 8.52527ZM25.159 8.5507C24.7744 8.44687 24.3786 8.66953 24.266 9.04933L24.2611 9.06697L23.265 12.7584L23.2604 12.7762C23.1667 13.161 23.3966 13.5526 23.7813 13.6563C24.1659 13.7602 24.5617 13.5375 24.6743 13.1577L24.6792 13.1401L25.6753 9.44865L25.6799 9.43084C25.7736 9.04599 25.5436 8.65447 25.159 8.5507Z" fill="white"></path><path d="M7.51285 5.04065C7.12824 4.93682 6.73249 5.15948 6.61988 5.53929L6.61495 5.55692L5.61886 9.24833L5.61429 9.26614C5.52054 9.65098 5.75052 10.0425 6.13519 10.1463C6.5198 10.2501 6.91554 10.0274 7.02816 9.64764L7.03308 9.63001L8.02917 5.9386L8.03374 5.92079C8.12749 5.53595 7.89751 5.14442 7.51285 5.04065ZM20.3116 5.73845C19.9269 5.63462 19.5312 5.85727 19.4186 6.23708L19.4136 6.25471L18.7443 8.73499C19.1779 8.94915 19.5917 9.20126 19.9809 9.48839L20.0453 9.53643L20.8279 6.63639L20.8324 6.61858C20.9262 6.23374 20.6963 5.84221 20.3116 5.73845ZM13.7968 1.57642C13.3296 1.61771 12.8647 1.68338 12.4043 1.77317L12.3066 1.79263L11.3782 5.23419L11.3736 5.252C11.2799 5.63684 11.5099 6.02837 11.8945 6.13214C12.2792 6.23596 12.6749 6.01331 12.7875 5.6335L12.7924 5.61587L13.7885 1.92446L13.7931 1.90665C13.8196 1.79831 13.8209 1.68533 13.7968 1.57642ZM22.9626 4.1263L22.7669 4.85169L22.7623 4.86944C22.6686 5.25429 22.8986 5.64581 23.2832 5.74958C23.6678 5.85341 24.0636 5.63075 24.1762 5.25095L24.1811 5.23331L24.2025 5.15462C23.8362 4.81205 23.4511 4.49009 23.0491 4.19022L22.9626 4.1263ZM17.1672 1.69677L16.8139 3.00593L16.8094 3.02374C16.7156 3.40858 16.9456 3.80011 17.3303 3.90388C17.7149 4.0077 18.1106 3.78505 18.2233 3.40524L18.2282 3.38761L18.6 2.00966C18.1624 1.88867 17.719 1.79001 17.2714 1.71405L17.1672 1.69677Z" fill="white"></path><path d="M9.69085 24.6253C9.80341 24.2455 10.1992 24.0229 10.5838 24.1266C10.9685 24.2303 11.1984 24.6219 11.1047 25.0068L11.1001 25.0246L10.3872 27.6664L10.2876 27.6297C9.85836 27.4694 9.43765 27.2873 9.0271 27.0839L9.68587 24.6429L9.69085 24.6253Z" fill="white"></path><path d="M14.4141 8.49082C10.0522 8.49082 6.65918 11.2368 6.65918 14.6517C6.65918 17.8769 9.78123 19.9362 14.6211 19.8331C15.0327 19.8243 15.1517 20.1008 15.2856 20.4734C15.4196 20.846 15.7796 22.8097 16.0665 24.3117C16.3233 25.656 16.5842 27.0052 16.7834 28.3596C19.9439 27.9418 22.8663 26.3807 25.0076 24.0261L22.7237 15.5088C22.1544 13.4518 21.489 11.5564 19.7283 10.1794C18.3118 9.07166 16.5122 8.49082 14.4141 8.49082Z" fill="white"></path><path d="M15.3516 10.957C15.8694 10.957 16.2891 11.3767 16.2891 11.8945C16.2891 12.4123 15.8694 12.832 15.3516 12.832C14.8338 12.832 14.4141 12.4123 14.4141 11.8945C14.4141 11.3767 14.8338 10.957 15.3516 10.957Z" fill="currentColor"></path></g><defs><clipPath id="clip0_29_599"><rect width="30" height="30" fill="white"></rect></clipPath></defs></svg>
<svg width="100%" height="100%" viewBox="0 0 441 441" version="1.1" xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/"
style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:2;">
<g transform="matrix(1.02631,-2.08167e-17,2.08167e-17,1.02631,-0.525826,-0.525138)">
<path
d="M37.965,296.635C26.441,271.766 20.009,244.065 20.009,214.873C20.009,207.318 20.439,199.863 21.278,192.531C22.129,185.123 23.39,177.852 25.036,170.742C34.286,130.852 55.801,95.64 85.384,69.301C110.233,47.207 140.674,31.444 174.043,24.299C187.212,21.486 200.872,20.006 214.875,20.006C219.783,20.011 224.727,20.2 229.701,20.579C253.285,22.38 275.571,28.317 295.904,37.625C312.305,45.143 327.486,54.87 341.064,66.426C375.17,95.48 398.957,135.953 406.867,181.369C408.757,192.255 409.742,203.45 409.742,214.873C409.738,219.789 409.548,224.74 409.168,229.721C407.731,248.545 403.659,266.542 397.34,283.379C388.521,306.83 375.308,328.136 358.706,346.294C337.113,368.342 309.673,378.152 286.755,377.744C270.09,377.447 253.784,370.816 242.516,361.114C226.42,347.253 219.918,331.409 217.69,313.729C217.136,309.334 217.461,297.358 219.748,289.066C221.453,282.885 225.777,270.948 232.1,265.727C224.703,262.541 215.183,255.604 212.182,252.274C211.445,251.455 211.54,250.174 212.2,249.292C212.861,248.41 214.02,248.062 215.057,248.435C221.416,250.618 229.161,252.771 237.327,254.137C248.067,255.932 261.424,258.194 274.955,258.859C307.946,260.479 342.407,245.67 353.103,216.207C363.798,186.744 359.649,157.602 321.279,140.121C282.909,122.64 265.185,101.856 234.183,89.32C213.934,81.131 191.396,85.992 168.257,98.78C105.931,133.223 50.092,242.048 75.833,342.873C76.201,344.252 75.58,345.705 74.328,346.392C73.156,347.036 71.713,346.852 70.741,345.962C63.25,337.731 56.454,328.857 50.445,319.433C45.796,312.139 41.623,304.524 37.965,296.635Z" />
</g>
<g transform="matrix(0.0920293,0.00428099,-0.00428099,0.0920293,-28.1272,-500.301)">
<path
d="M3053.7,5296.9C4371.65,5296.9 5441.66,6366.91 5441.66,7684.86C5441.66,9002.81 4371.65,10072.8 3053.7,10072.8C1735.75,10072.8 665.74,9002.81 665.74,7684.86C665.74,6366.91 1735.75,5296.9 3053.7,5296.9ZM3745.03,8143.22C3594.12,8142.82 3444.31,8124.57 3323.87,8110.15C3232.29,8099.18 3144.99,8079.23 3073.1,8058.23C3061.36,8054.62 3048.65,8059.09 3041.75,8069.24C3034.86,8079.4 3034.46,8093.71 3043.09,8102.44C3078.21,8137.94 3187.74,8210.21 3271.7,8241.83C3204.04,8303.2 3162.1,8438.28 3146.33,8507.94C3125.17,8601.4 3127.75,8734.83 3136.19,8783.45C3170.14,8979.04 3250.69,9151.99 3436.99,9297.9C3567.4,9400.03 3752.28,9465.38 3937.88,9460.06C4194.01,9452.71 4495.48,9328.51 4724.65,9070.17C5023.25,8710.58 5208.52,8252.45 5223.47,7749.5C5259.08,6551.9 4315.7,5550.69 3118.1,5515.08C1920.51,5479.47 919.301,6422.86 883.689,7620.45C865.246,8240.66 1109.37,8808.21 1515.43,9216.2C1526.73,9227.39 1544.21,9229.43 1557.78,9221.14C1571.35,9212.85 1577.51,9196.36 1572.7,9181.2C1234.07,8072.55 1799.11,6832.64 2474.84,6417.1C2725.71,6262.82 2973.99,6197.06 3203.56,6277.7C3555.04,6401.15 3763.03,6623.26 4199.06,6797.93C4635.09,6972.59 4696.35,7294.74 4592.58,7628.14C4488.81,7961.54 4113,8144.17 3745.03,8143.22ZM2917.17,6442.51C2777.75,6459.97 2693.93,6637.44 2687.08,6749.42C2680.18,6861.39 2744.03,7042.7 2926.19,7030.63C3139.52,7016.49 3195.89,6830.7 3164.24,6654.94C3140.48,6522.94 3033.73,6427.9 2917.17,6442.51Z"
style="fill:white;" />
</g>
<g transform="matrix(7.12289,0.543899,-0.543899,7.12289,-4867.49,-1040.55)">
<path
d="M721.316,105.751C722.813,105.518 724.225,106.703 724.582,108.395C725.058,110.649 724.402,113.065 721.658,113.329C719.314,113.555 718.422,111.242 718.468,109.796C718.513,108.35 719.525,106.03 721.316,105.751Z" />
</g>
</svg>

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.5 KiB

After

Width:  |  Height:  |  Size: 3.7 KiB

14
cli/tools/jupyter/server.rs
View file

@ -329,7 +329,12 @@ impl JupyterServer {
})
.collect();
(candidates, cursor_pos - prop_name.len())
if prop_name.len() > cursor_pos {
// TODO(bartlomieju): most likely not correct, but better than panicking because of sub with overflow
(candidates, cursor_pos)
} else {
(candidates, cursor_pos - prop_name.len())
}
} else {
// combine results of declarations and globalThis properties
let mut candidates = get_expression_property_names(
@ -349,7 +354,12 @@ impl JupyterServer {
candidates.sort();
candidates.dedup(); // make sure to sort first
(candidates, cursor_pos - expr.len())
if expr.len() > cursor_pos {
// TODO(bartlomieju): most likely not correct, but better than panicking because of sub with overflow
(candidates, cursor_pos)
} else {
(candidates, cursor_pos - expr.len())
}
};
connection

769
cli/tools/registry/pm.rs
View file

File diff suppressed because it is too large Load diff

11
cli/tools/registry/pm/cache_deps.rs
View file

@ -75,6 +75,13 @@ pub async fn cache_top_level_deps(
if entry.key.ends_with('/') && specifier.as_str().ends_with('/') {
continue;
}
if specifier.scheme() == "file" {
if let Ok(path) = specifier.to_file_path() {
if !path.is_file() {
continue;
}
}
}
roots.push(specifier.clone());
}
}
@ -82,10 +89,6 @@ pub async fn cache_top_level_deps(
while let Some(info_future) = info_futures.next().await {
if let Some((specifier, info)) = info_future {
if info.export(".").is_some() {
roots.push(specifier.clone());
continue;
}
let exports = info.exports();
for (k, _) in exports {
if let Ok(spec) = specifier.join(k) {

2
cli/tools/registry/tar.rs
View file

@ -120,7 +120,7 @@ fn resolve_content_maybe_unfurling(
| MediaType::Unknown
| MediaType::Json
| MediaType::Wasm
| MediaType::TsBuildInfo => {
| MediaType::Css => {
// not unfurlable data
return Ok(data);
}

6
cli/tools/repl/session.rs
View file

@ -25,6 +25,7 @@ use deno_ast::swc::visit::noop_visit_type;
use deno_ast::swc::visit::Visit;
use deno_ast::swc::visit::VisitWith;
use deno_ast::ImportsNotUsedAsValues;
use deno_ast::ModuleKind;
use deno_ast::ModuleSpecifier;
use deno_ast::ParseDiagnosticsError;
use deno_ast::ParsedSource;
@ -641,6 +642,10 @@ impl ReplSession {
jsx_fragment_factory: self.jsx.frag_factory.clone(),
jsx_import_source: self.jsx.import_source.clone(),
var_decl_imports: true,
verbatim_module_syntax: false,
},
&deno_ast::TranspileModuleOptions {
module_kind: Some(ModuleKind::Esm),
},
&deno_ast::EmitOptions {
source_map: deno_ast::SourceMapOption::None,
@ -651,7 +656,6 @@ impl ReplSession {
},
)?
.into_source()
.into_string()?
.text;
let value = self

25
cli/tools/run/hmr.rs
View file

@ -1,9 +1,11 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::cdp;
use crate::emit::Emitter;
use crate::util::file_watcher::WatcherCommunicator;
use crate::util::file_watcher::WatcherRestartMode;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_core::error::generic_error;
use deno_core::error::AnyError;
use deno_core::futures::StreamExt;
@ -12,11 +14,14 @@ use deno_core::serde_json::{self};
use deno_core::url::Url;
use deno_core::LocalInspectorSession;
use deno_terminal::colors;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::select;
use crate::cdp;
use crate::emit::Emitter;
use crate::resolver::CjsTracker;
use crate::util::file_watcher::WatcherCommunicator;
use crate::util::file_watcher::WatcherRestartMode;
fn explain(status: &cdp::Status) -> &'static str {
match status {
cdp::Status::Ok => "OK",
@ -58,6 +63,7 @@ pub struct HmrRunner {
session: LocalInspectorSession,
watcher_communicator: Arc<WatcherCommunicator>,
script_ids: HashMap<String, String>,
cjs_tracker: Arc<CjsTracker>,
emitter: Arc<Emitter>,
}
@ -139,7 +145,8 @@ impl crate::worker::HmrRunner for HmrRunner {
};
let source_code = self.emitter.load_and_emit_for_hmr(
&module_url
&module_url,
ModuleKind::from_is_cjs(self.cjs_tracker.is_maybe_cjs(&module_url, MediaType::from_specifier(&module_url))?),
).await?;
let mut tries = 1;
@ -172,12 +179,14 @@ impl crate::worker::HmrRunner for HmrRunner {
impl HmrRunner {
pub fn new(
cjs_tracker: Arc<CjsTracker>,
emitter: Arc<Emitter>,
session: LocalInspectorSession,
watcher_communicator: Arc<WatcherCommunicator>,
) -> Self {
Self {
session,
cjs_tracker,
emitter,
watcher_communicator,
script_ids: HashMap::new(),

18
cli/tools/run/mod.rs
View file

@ -30,6 +30,16 @@ To grant permissions, set them before the script argument. For example:
}
}
fn set_npm_user_agent() {
static ONCE: std::sync::Once = std::sync::Once::new();
ONCE.call_once(|| {
std::env::set_var(
crate::npm::NPM_CONFIG_USER_AGENT_ENV_VAR,
crate::npm::get_npm_config_user_agent(),
);
});
}
pub async fn run_script(
mode: WorkerExecutionMode,
flags: Arc<Flags>,
@ -58,6 +68,10 @@ pub async fn run_script(
let main_module = cli_options.resolve_main_module()?;
if main_module.scheme() == "npm" {
set_npm_user_agent();
}
maybe_npm_install(&factory).await?;
let worker_factory = factory.create_cli_main_worker_factory().await?;
@ -119,6 +133,10 @@ async fn run_with_watch(
let cli_options = factory.cli_options()?;
let main_module = cli_options.resolve_main_module()?;
if main_module.scheme() == "npm" {
set_npm_user_agent();
}
maybe_npm_install(&factory).await?;
let _ = watcher_communicator.watch_paths(cli_options.watch_paths());

11
cli/tools/serve.rs
View file

@ -44,12 +44,15 @@ pub async fn serve(
maybe_npm_install(&factory).await?;
let worker_factory = factory.create_cli_main_worker_factory().await?;
let hmr = serve_flags
.watch
.map(|watch_flags| watch_flags.hmr)
.unwrap_or(false);
do_serve(
worker_factory,
main_module.clone(),
serve_flags.worker_count,
false,
hmr,
)
.await
}
@ -109,8 +112,6 @@ async fn do_serve(
}
}
Ok(exit_code)
// main.await?
}
async fn run_worker(
@ -119,7 +120,7 @@ async fn run_worker(
main_module: ModuleSpecifier,
hmr: bool,
) -> Result<i32, AnyError> {
let mut worker = worker_factory
let mut worker: crate::worker::CliMainWorker = worker_factory
.create_main_worker(
deno_runtime::WorkerExecutionMode::Serve {
is_main: false,

Some files were not shown because too many files have changed in this diff Show more