1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-29 02:29:06 -05:00
denoland-deno/cli/npm/managed/resolvers/local.rs

777 lines
24 KiB
Rust
Raw Normal View History

// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
//! Code for local node_modules resolution.
mod bin_entries;
use std::borrow::Cow;
use std::cell::RefCell;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use crate::cache::CACHE_PERM;
use crate::npm::cache_dir::mixed_case_package_name_decode;
fix: make writing to the deps cache more reliable (#24135) I was able to reproduce this locally. ``` [error] Failed to execute snippet: import { validate } from "@std/uuid"; import { assert, assertFalse } from "@std/assert"; assert(validate("6ec0bd7f-11c0-43da-975e-2a8ad9ebae0b")); assertFalse(validate("not a UUID")); Download https://jsr.io/@std/uuid/meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1_meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/common.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/constants.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v1.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v3.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v4.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v5.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/_common.ts error: Access is denied. (os error 5) (for 'V:\.cache\deno\deps\https\jsr.io\2ae5bb614c7526d0876be0b76da1372fd51304ae27d6202ee94df720b3523d08') at file:///V:/deno_std/uuid/common.ts:43 [error] Failed to execute snippet: import { v5, NAMESPACE_DNS, NIL_UUID } from "@std/uuid"; import { assert, assertFalse } from "@std/assert"; const data = new TextEncoder().encode("deno.land"); const uuid = await v5.generate(NAMESPACE_DNS, data); assert(v5.validate(uuid)); assertFalse(v5.validate(NIL_UUID)); Download https://jsr.io/@std/uuid/meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1_meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/common.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/constants.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v1.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v3.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v4.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v5.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/_common.ts error: Access is denied. (os error 5) (for 'V:\.cache\deno\deps\https\jsr.io\63dd818c5fc1ac39c04df9b42bd9dd4bbc07f7d1b174e405d003731125778da1') at https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts:30:15 at file:///V:/deno_std/uuid/mod.ts:4 [error] Failed to execute snippet: import { isNil } from "@std/uuid"; import { assert, assertFalse } from "@std/assert"; assert(isNil("00000000-0000-0000-0000-000000000000")); assertFalse(isNil(crypto.randomUUID())); Download https://jsr.io/@std/uuid/meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1_meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/common.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/constants.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v1.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v3.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v4.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v5.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/_common.ts error: Access is denied. (os error 5) (for 'V:\.cache\deno\deps\https\jsr.io\fd3a12fc091d16ee29f10fa7a05eeeb8bd6c3cc014642e72478c757f00e7261e') at https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts:34:40 at file:///V:/deno_std/uuid/common.ts:23 [error] Failed to execute snippet: import { version } from "@std/uuid"; import { assertEquals } from "@std/assert/assert-equals"; assertEquals(version("d9428888-122b-11e1-b85c-61cd3cbb3210"), 1); assertEquals(version("6ec0bd7f-11c0-43da-975e-2a8ad9ebae0b"), 4); Download https://jsr.io/@std/uuid/meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1_meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/common.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/constants.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v1.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v3.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v4.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v5.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/_common.ts error: Access is denied. (os error 5) (for 'V:\.cache\deno\deps\https\jsr.io\2ae5bb614c7526d0876be0b76da1372fd51304ae27d6202ee94df720b3523d08') at file:///V:/deno_std/uuid/common.ts:66 4 errors found ``` It occurs when many Deno processes are writing to the deps cache at the same time. Fix is to use `atomic_write_with_retries` which is much more reliable (and the function that helped make the ecosystem tests more reliable too). After this change I no longer have this issue. Closes https://github.com/denoland/deno/issues/24073
2024-06-07 13:06:17 -04:00
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::fs::clone_dir_recursive;
use crate::util::fs::symlink_dir;
use crate::util::fs::LaxSingleProcessFsFlag;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressMessagePrompt;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures::stream::FuturesUnordered;
use deno_core::futures::StreamExt;
use deno_core::url::Url;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::NodePermissions;
use deno_semver::package::PackageNv;
use serde::Deserialize;
use serde::Serialize;
use crate::npm::cache_dir::mixed_case_package_name_encode;
use super::super::cache::NpmCache;
use super::super::cache::TarballCache;
use super::super::resolution::NpmResolution;
use super::common::NpmPackageFsResolver;
use super::common::RegistryReadPermissionChecker;
/// Resolver that creates a local node_modules directory
/// and resolves packages from it.
#[derive(Debug)]
pub struct LocalNpmPackageResolver {
cache: Arc<NpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
root_node_modules_path: PathBuf,
root_node_modules_url: Url,
system_info: NpmSystemInfo,
registry_read_permission_checker: RegistryReadPermissionChecker,
}
impl LocalNpmPackageResolver {
pub fn new(
cache: Arc<NpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
node_modules_folder: PathBuf,
system_info: NpmSystemInfo,
) -> Self {
Self {
cache,
fs: fs.clone(),
progress_bar,
resolution,
tarball_cache,
registry_read_permission_checker: RegistryReadPermissionChecker::new(
fs,
node_modules_folder.clone(),
),
root_node_modules_url: Url::from_directory_path(&node_modules_folder)
.unwrap(),
root_node_modules_path: node_modules_folder,
system_info,
}
}
fn resolve_package_root(&self, path: &Path) -> PathBuf {
let mut last_found = path;
loop {
let parent = last_found.parent().unwrap();
if parent.file_name().unwrap() == "node_modules" {
return last_found.to_path_buf();
} else {
last_found = parent;
}
}
}
fn resolve_folder_for_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<Option<PathBuf>, AnyError> {
let Some(relative_url) =
self.root_node_modules_url.make_relative(specifier)
else {
return Ok(None);
};
if relative_url.starts_with("../") {
return Ok(None);
}
// it's within the directory, so use it
let Some(path) = specifier.to_file_path().ok() else {
return Ok(None);
};
// Canonicalize the path so it's not pointing to the symlinked directory
// in `node_modules` directory of the referrer.
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())
.map(Some)
.map_err(|err| err.into())
}
fn resolve_package_folder_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<Option<PathBuf>, AnyError> {
let Some(local_path) = self.resolve_folder_for_specifier(specifier)? else {
return Ok(None);
};
let package_root_path = self.resolve_package_root(&local_path);
Ok(Some(package_root_path))
}
}
#[async_trait(?Send)]
impl NpmPackageFsResolver for LocalNpmPackageResolver {
fn root_dir_url(&self) -> &Url {
&self.root_node_modules_url
}
fn node_modules_path(&self) -> Option<&PathBuf> {
Some(&self.root_node_modules_path)
}
fn package_folder(&self, id: &NpmPackageId) -> Result<PathBuf, AnyError> {
match self.resolution.resolve_pkg_cache_folder_id_from_pkg_id(id) {
// package is stored at:
// node_modules/.deno/<package_cache_folder_id_folder_name>/node_modules/<package_name>
Some(cache_folder_id) => Ok(
self
.root_node_modules_path
.join(".deno")
.join(get_package_folder_id_folder_name(&cache_folder_id))
.join("node_modules")
.join(&cache_folder_id.nv.name),
),
None => bail!(
"Could not find package information for '{}'",
id.as_serialized()
),
}
}
fn resolve_package_folder_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<PathBuf, AnyError> {
let Some(local_path) = self.resolve_folder_for_specifier(referrer)? else {
bail!("could not find npm package for '{}'", referrer);
};
let package_root_path = self.resolve_package_root(&local_path);
let mut current_folder = package_root_path.as_path();
while let Some(parent_folder) = current_folder.parent() {
current_folder = parent_folder;
let node_modules_folder = if current_folder.ends_with("node_modules") {
Cow::Borrowed(current_folder)
} else {
Cow::Owned(current_folder.join("node_modules"))
};
let sub_dir = join_package_name(&node_modules_folder, name);
if self.fs.is_dir_sync(&sub_dir) {
return Ok(sub_dir);
}
if current_folder == self.root_node_modules_path {
break;
}
}
bail!(
"could not find package '{}' from referrer '{}'.",
name,
referrer
);
}
fn resolve_package_cache_folder_id_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<Option<NpmPackageCacheFolderId>, AnyError> {
let Some(folder_path) =
self.resolve_package_folder_from_specifier(specifier)?
else {
return Ok(None);
};
let folder_name = folder_path.parent().unwrap().to_string_lossy();
Ok(get_package_folder_id_from_folder_name(&folder_name))
}
async fn cache_packages(&self) -> Result<(), AnyError> {
sync_resolution_with_fs(
&self.resolution.snapshot(),
&self.cache,
&self.progress_bar,
&self.tarball_cache,
&self.root_node_modules_path,
&self.system_info,
)
.await
}
fn ensure_read_permission(
&self,
permissions: &mut dyn NodePermissions,
path: &Path,
) -> Result<(), AnyError> {
self
.registry_read_permission_checker
.ensure_registry_read_permission(permissions, path)
}
}
/// Creates a pnpm style folder structure.
async fn sync_resolution_with_fs(
snapshot: &NpmResolutionSnapshot,
cache: &Arc<NpmCache>,
progress_bar: &ProgressBar,
tarball_cache: &Arc<TarballCache>,
root_node_modules_dir_path: &Path,
system_info: &NpmSystemInfo,
) -> Result<(), AnyError> {
if snapshot.is_empty() {
return Ok(()); // don't create the directory
}
let deno_local_registry_dir = root_node_modules_dir_path.join(".deno");
let deno_node_modules_dir = deno_local_registry_dir.join("node_modules");
fs::create_dir_all(&deno_node_modules_dir).with_context(|| {
format!("Creating '{}'", deno_local_registry_dir.display())
})?;
let bin_node_modules_dir_path = root_node_modules_dir_path.join(".bin");
fs::create_dir_all(&bin_node_modules_dir_path).with_context(|| {
format!("Creating '{}'", bin_node_modules_dir_path.display())
})?;
let single_process_lock = LaxSingleProcessFsFlag::lock(
deno_local_registry_dir.join(".deno.lock"),
// similar message used by cargo build
"waiting for file lock on node_modules directory",
)
.await;
// load this after we get the directory lock
let mut setup_cache =
SetupCache::load(deno_local_registry_dir.join(".setup-cache.bin"));
let pb_clear_guard = progress_bar.clear_guard(); // prevent flickering
// 1. Write all the packages out the .deno directory.
//
// Copy (hardlink in future) <global_registry_cache>/<package_id>/ to
// node_modules/.deno/<package_folder_id_folder_name>/node_modules/<package_name>
let package_partitions =
snapshot.all_system_packages_partitioned(system_info);
let mut cache_futures = FuturesUnordered::new();
let mut newest_packages_by_name: HashMap<&String, &NpmResolutionPackage> =
HashMap::with_capacity(package_partitions.packages.len());
let bin_entries = Rc::new(RefCell::new(bin_entries::BinEntries::new()));
for package in &package_partitions.packages {
if let Some(current_pkg) =
newest_packages_by_name.get_mut(&package.id.nv.name)
{
if current_pkg.id.nv.cmp(&package.id.nv) == Ordering::Less {
*current_pkg = package;
}
} else {
newest_packages_by_name.insert(&package.id.nv.name, package);
};
let package_folder_name =
get_package_folder_id_folder_name(&package.get_package_cache_folder_id());
let folder_path = deno_local_registry_dir.join(&package_folder_name);
let initialized_file = folder_path.join(".initialized");
if !cache
.cache_setting()
.should_use_for_npm_package(&package.id.nv.name)
|| !initialized_file.exists()
{
// cache bust the dep from the dep setup cache so the symlinks
// are forced to be recreated
setup_cache.remove_dep(&package_folder_name);
let bin_entries_to_setup = bin_entries.clone();
cache_futures.push(async move {
tarball_cache
.ensure_package(&package.id.nv, &package.dist)
.await?;
let pb_guard = progress_bar.update_with_prompt(
ProgressMessagePrompt::Initialize,
&package.id.nv.to_string(),
);
let sub_node_modules = folder_path.join("node_modules");
let package_path =
join_package_name(&sub_node_modules, &package.id.nv.name);
let cache_folder = cache.package_folder_for_nv(&package.id.nv);
perf(cli): Improve concurrency when setting up `node_modules` and loading cached npm package info (#24018) The same issue in two different places - doing blocking FS work in an async task, limiting the amount of work that happens concurrently. - When setting up node_modules, where we try to set up entries concurrently but were blocking other tasks from actually running. - When loading package info from the npm registry file cache, loading and deserializing is expensive and prevents concurrency. This was especially noticeable when loading an npm resolution snapshot from a lockfile (`snapshot_from_lockfile` in `deno_npm`). Installing deps in `deno-docs`: ``` ❯ hyperfine -i -p 'rm -rf node_modules/' '../d7/deno-main i' '../d7/target/release/deno i' Benchmark 1: ../d7/deno-main i Time (mean ± σ): 2.193 s ± 0.027 s [User: 0.589 s, System: 1.033 s] Range (min … max): 2.151 s … 2.242 s 10 runs Benchmark 2: ../d7/target/release/deno i Time (mean ± σ): 1.597 s ± 0.021 s [User: 0.977 s, System: 1.337 s] Range (min … max): 1.550 s … 1.627 s 10 runs Summary ../d7/target/release/deno i ran 1.37 ± 0.02 times faster than ../d7/deno-main i ``` Caching `npm:@11ty/eleventy`: ``` ❯ hyperfine -i -p 'rm -rf node_modules/' --warmup 5 '../../d7/deno-main cache npm:@11ty/eleventy' '../../d7/target/release/deno cache npm:@11ty/eleventy' Benchmark 1: ../../d7/deno-main cache npm:@11ty/eleventy Time (mean ± σ): 129.9 ms ± 2.2 ms [User: 27.5 ms, System: 101.3 ms] Range (min … max): 127.5 ms … 135.8 ms 10 runs Benchmark 2: ../../d7/target/release/deno cache npm:@11ty/eleventy Time (mean ± σ): 100.6 ms ± 1.3 ms [User: 38.8 ms, System: 233.8 ms] Range (min … max): 99.3 ms … 103.2 ms 10 runs Summary ../../d7/target/release/deno cache npm:@11ty/eleventy ran 1.29 ± 0.03 times faster than ../../d7/deno-main cache npm:@11ty/eleventy ``` --------- Co-authored-by: David Sherret <dsherret@gmail.com>
2024-05-28 17:17:36 -04:00
deno_core::unsync::spawn_blocking({
let package_path = package_path.clone();
move || {
clone_dir_recursive(&cache_folder, &package_path)?;
// write out a file that indicates this folder has been initialized
fs::write(initialized_file, "")?;
Ok::<_, AnyError>(())
}
})
.await??;
if package.bin.is_some() {
bin_entries_to_setup
.borrow_mut()
.add(package.clone(), package_path);
}
// finally stop showing the progress bar
drop(pb_guard); // explicit for clarity
Ok::<_, AnyError>(())
});
}
}
while let Some(result) = cache_futures.next().await {
result?; // surface the first error
}
// 2. Create any "copy" packages, which are used for peer dependencies
for package in &package_partitions.copy_packages {
let package_cache_folder_id = package.get_package_cache_folder_id();
let destination_path = deno_local_registry_dir
2022-12-17 17:20:15 -05:00
.join(get_package_folder_id_folder_name(&package_cache_folder_id));
let initialized_file = destination_path.join(".initialized");
if !initialized_file.exists() {
let sub_node_modules = destination_path.join("node_modules");
let package_path =
join_package_name(&sub_node_modules, &package.id.nv.name);
let source_path = join_package_name(
&deno_local_registry_dir
2022-12-17 17:20:15 -05:00
.join(get_package_folder_id_folder_name(
&package_cache_folder_id.with_no_count(),
))
.join("node_modules"),
&package.id.nv.name,
);
perf(cli): Improve concurrency when setting up `node_modules` and loading cached npm package info (#24018) The same issue in two different places - doing blocking FS work in an async task, limiting the amount of work that happens concurrently. - When setting up node_modules, where we try to set up entries concurrently but were blocking other tasks from actually running. - When loading package info from the npm registry file cache, loading and deserializing is expensive and prevents concurrency. This was especially noticeable when loading an npm resolution snapshot from a lockfile (`snapshot_from_lockfile` in `deno_npm`). Installing deps in `deno-docs`: ``` ❯ hyperfine -i -p 'rm -rf node_modules/' '../d7/deno-main i' '../d7/target/release/deno i' Benchmark 1: ../d7/deno-main i Time (mean ± σ): 2.193 s ± 0.027 s [User: 0.589 s, System: 1.033 s] Range (min … max): 2.151 s … 2.242 s 10 runs Benchmark 2: ../d7/target/release/deno i Time (mean ± σ): 1.597 s ± 0.021 s [User: 0.977 s, System: 1.337 s] Range (min … max): 1.550 s … 1.627 s 10 runs Summary ../d7/target/release/deno i ran 1.37 ± 0.02 times faster than ../d7/deno-main i ``` Caching `npm:@11ty/eleventy`: ``` ❯ hyperfine -i -p 'rm -rf node_modules/' --warmup 5 '../../d7/deno-main cache npm:@11ty/eleventy' '../../d7/target/release/deno cache npm:@11ty/eleventy' Benchmark 1: ../../d7/deno-main cache npm:@11ty/eleventy Time (mean ± σ): 129.9 ms ± 2.2 ms [User: 27.5 ms, System: 101.3 ms] Range (min … max): 127.5 ms … 135.8 ms 10 runs Benchmark 2: ../../d7/target/release/deno cache npm:@11ty/eleventy Time (mean ± σ): 100.6 ms ± 1.3 ms [User: 38.8 ms, System: 233.8 ms] Range (min … max): 99.3 ms … 103.2 ms 10 runs Summary ../../d7/target/release/deno cache npm:@11ty/eleventy ran 1.29 ± 0.03 times faster than ../../d7/deno-main cache npm:@11ty/eleventy ``` --------- Co-authored-by: David Sherret <dsherret@gmail.com>
2024-05-28 17:17:36 -04:00
clone_dir_recursive(&source_path, &package_path)?;
// write out a file that indicates this folder has been initialized
fs::write(initialized_file, "")?;
}
}
// 3. Symlink all the dependencies into the .deno directory.
//
// Symlink node_modules/.deno/<package_id>/node_modules/<dep_name> to
// node_modules/.deno/<dep_id>/node_modules/<dep_package_name>
for package in package_partitions.iter_all() {
let package_folder_name =
get_package_folder_id_folder_name(&package.get_package_cache_folder_id());
let sub_node_modules = deno_local_registry_dir
.join(&package_folder_name)
.join("node_modules");
let mut dep_setup_cache = setup_cache.with_dep(&package_folder_name);
for (name, dep_id) in &package.dependencies {
let dep = snapshot.package_from_id(dep_id).unwrap();
if package.optional_dependencies.contains(name)
&& !dep.system.matches_system(system_info)
{
continue; // this isn't a dependency for the current system
}
let dep_cache_folder_id = dep.get_package_cache_folder_id();
let dep_folder_name =
get_package_folder_id_folder_name(&dep_cache_folder_id);
if dep_setup_cache.insert(name, &dep_folder_name) {
let dep_folder_path = join_package_name(
&deno_local_registry_dir
.join(dep_folder_name)
.join("node_modules"),
&dep_id.nv.name,
);
symlink_package_dir(
&dep_folder_path,
&join_package_name(&sub_node_modules, name),
)?;
}
}
}
// 4. Create all the top level packages in the node_modules folder, which are symlinks.
//
// Symlink node_modules/<package_name> to
// node_modules/.deno/<package_id>/node_modules/<package_name>
let mut found_names = HashSet::new();
let mut ids = snapshot.top_level_packages().collect::<Vec<_>>();
ids.sort_by(|a, b| b.cmp(a)); // create determinism and only include the latest version
for id in ids {
if !found_names.insert(&id.nv.name) {
continue; // skip, already handled
}
let package = snapshot.package_from_id(id).unwrap();
let target_folder_name =
get_package_folder_id_folder_name(&package.get_package_cache_folder_id());
if setup_cache.insert_root_symlink(&id.nv.name, &target_folder_name) {
let local_registry_package_path = join_package_name(
&deno_local_registry_dir
.join(target_folder_name)
.join("node_modules"),
&id.nv.name,
);
symlink_package_dir(
&local_registry_package_path,
&join_package_name(root_node_modules_dir_path, &id.nv.name),
)?;
}
}
// 5. Create a node_modules/.deno/node_modules/<package-name> directory with
// the remaining packages
for package in newest_packages_by_name.values() {
if !found_names.insert(&package.id.nv.name) {
continue; // skip, already handled
}
let target_folder_name =
get_package_folder_id_folder_name(&package.get_package_cache_folder_id());
if setup_cache.insert_deno_symlink(&package.id.nv.name, &target_folder_name)
{
let local_registry_package_path = join_package_name(
&deno_local_registry_dir
.join(target_folder_name)
.join("node_modules"),
&package.id.nv.name,
);
symlink_package_dir(
&local_registry_package_path,
&join_package_name(&deno_node_modules_dir, &package.id.nv.name),
)?;
}
}
// 6. Set up `node_modules/.bin` entries for packages that need it.
{
let bin_entries = std::mem::take(&mut *bin_entries.borrow_mut());
bin_entries.finish(snapshot, &bin_node_modules_dir_path)?;
}
setup_cache.save();
drop(single_process_lock);
drop(pb_clear_guard);
Ok(())
}
/// Represents a dependency at `node_modules/.deno/<package_id>/`
struct SetupCacheDep<'a> {
previous: Option<&'a HashMap<String, String>>,
current: &'a mut HashMap<String, String>,
}
impl<'a> SetupCacheDep<'a> {
pub fn insert(&mut self, name: &str, target_folder_name: &str) -> bool {
self
.current
.insert(name.to_string(), target_folder_name.to_string());
if let Some(previous_target) = self.previous.and_then(|p| p.get(name)) {
previous_target != target_folder_name
} else {
true
}
}
}
#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
struct SetupCacheData {
root_symlinks: HashMap<String, String>,
deno_symlinks: HashMap<String, String>,
dep_symlinks: HashMap<String, HashMap<String, String>>,
}
/// It is very slow to try to re-setup the symlinks each time, so this will
/// cache what we've setup on the last run and only update what is necessary.
/// Obviously this could lead to issues if the cache gets out of date with the
/// file system, such as if the user manually deletes a symlink.
struct SetupCache {
file_path: PathBuf,
previous: Option<SetupCacheData>,
current: SetupCacheData,
}
impl SetupCache {
pub fn load(file_path: PathBuf) -> Self {
let previous = std::fs::read(&file_path)
.ok()
.and_then(|data| bincode::deserialize(&data).ok());
Self {
file_path,
previous,
current: Default::default(),
}
}
pub fn save(&self) -> bool {
if let Some(previous) = &self.previous {
if previous == &self.current {
return false; // nothing to save
}
}
bincode::serialize(&self.current).ok().and_then(|data| {
fix: make writing to the deps cache more reliable (#24135) I was able to reproduce this locally. ``` [error] Failed to execute snippet: import { validate } from "@std/uuid"; import { assert, assertFalse } from "@std/assert"; assert(validate("6ec0bd7f-11c0-43da-975e-2a8ad9ebae0b")); assertFalse(validate("not a UUID")); Download https://jsr.io/@std/uuid/meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1_meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/common.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/constants.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v1.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v3.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v4.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v5.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/_common.ts error: Access is denied. (os error 5) (for 'V:\.cache\deno\deps\https\jsr.io\2ae5bb614c7526d0876be0b76da1372fd51304ae27d6202ee94df720b3523d08') at file:///V:/deno_std/uuid/common.ts:43 [error] Failed to execute snippet: import { v5, NAMESPACE_DNS, NIL_UUID } from "@std/uuid"; import { assert, assertFalse } from "@std/assert"; const data = new TextEncoder().encode("deno.land"); const uuid = await v5.generate(NAMESPACE_DNS, data); assert(v5.validate(uuid)); assertFalse(v5.validate(NIL_UUID)); Download https://jsr.io/@std/uuid/meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1_meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/common.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/constants.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v1.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v3.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v4.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v5.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/_common.ts error: Access is denied. (os error 5) (for 'V:\.cache\deno\deps\https\jsr.io\63dd818c5fc1ac39c04df9b42bd9dd4bbc07f7d1b174e405d003731125778da1') at https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts:30:15 at file:///V:/deno_std/uuid/mod.ts:4 [error] Failed to execute snippet: import { isNil } from "@std/uuid"; import { assert, assertFalse } from "@std/assert"; assert(isNil("00000000-0000-0000-0000-000000000000")); assertFalse(isNil(crypto.randomUUID())); Download https://jsr.io/@std/uuid/meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1_meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/common.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/constants.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v1.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v3.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v4.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v5.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/_common.ts error: Access is denied. (os error 5) (for 'V:\.cache\deno\deps\https\jsr.io\fd3a12fc091d16ee29f10fa7a05eeeb8bd6c3cc014642e72478c757f00e7261e') at https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts:34:40 at file:///V:/deno_std/uuid/common.ts:23 [error] Failed to execute snippet: import { version } from "@std/uuid"; import { assertEquals } from "@std/assert/assert-equals"; assertEquals(version("d9428888-122b-11e1-b85c-61cd3cbb3210"), 1); assertEquals(version("6ec0bd7f-11c0-43da-975e-2a8ad9ebae0b"), 4); Download https://jsr.io/@std/uuid/meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1_meta.json Download https://jsr.io/@std/uuid/1.0.0-rc.1/mod.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/common.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/constants.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v1.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v3.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v4.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/v5.ts Download https://jsr.io/@std/uuid/1.0.0-rc.1/_common.ts error: Access is denied. (os error 5) (for 'V:\.cache\deno\deps\https\jsr.io\2ae5bb614c7526d0876be0b76da1372fd51304ae27d6202ee94df720b3523d08') at file:///V:/deno_std/uuid/common.ts:66 4 errors found ``` It occurs when many Deno processes are writing to the deps cache at the same time. Fix is to use `atomic_write_with_retries` which is much more reliable (and the function that helped make the ecosystem tests more reliable too). After this change I no longer have this issue. Closes https://github.com/denoland/deno/issues/24073
2024-06-07 13:06:17 -04:00
atomic_write_file_with_retries(&self.file_path, data, CACHE_PERM).ok()
});
true
}
/// Inserts and checks for the existence of a root symlink
/// at `node_modules/<package_name>` pointing to
/// `node_modules/.deno/<package_id>/`
pub fn insert_root_symlink(
&mut self,
name: &str,
target_folder_name: &str,
) -> bool {
self
.current
.root_symlinks
.insert(name.to_string(), target_folder_name.to_string());
if let Some(previous_target) = self
.previous
.as_ref()
.and_then(|p| p.root_symlinks.get(name))
{
previous_target != target_folder_name
} else {
true
}
}
/// Inserts and checks for the existence of a symlink at
/// `node_modules/.deno/node_modules/<package_name>` pointing to
/// `node_modules/.deno/<package_id>/`
pub fn insert_deno_symlink(
&mut self,
name: &str,
target_folder_name: &str,
) -> bool {
self
.current
.deno_symlinks
.insert(name.to_string(), target_folder_name.to_string());
if let Some(previous_target) = self
.previous
.as_ref()
.and_then(|p| p.deno_symlinks.get(name))
{
previous_target != target_folder_name
} else {
true
}
}
pub fn remove_dep(&mut self, parent_name: &str) {
if let Some(previous) = &mut self.previous {
previous.dep_symlinks.remove(parent_name);
}
}
pub fn with_dep(&mut self, parent_name: &str) -> SetupCacheDep<'_> {
SetupCacheDep {
previous: self
.previous
.as_ref()
.and_then(|p| p.dep_symlinks.get(parent_name)),
current: self
.current
.dep_symlinks
.entry(parent_name.to_string())
.or_default(),
}
}
}
fn get_package_folder_id_folder_name(
folder_id: &NpmPackageCacheFolderId,
) -> String {
let copy_str = if folder_id.copy_index == 0 {
"".to_string()
} else {
format!("_{}", folder_id.copy_index)
};
let nv = &folder_id.nv;
let name = if nv.name.to_lowercase() == nv.name {
Cow::Borrowed(&nv.name)
} else {
Cow::Owned(format!("_{}", mixed_case_package_name_encode(&nv.name)))
};
format!("{}@{}{}", name, nv.version, copy_str).replace('/', "+")
}
fn get_package_folder_id_from_folder_name(
folder_name: &str,
) -> Option<NpmPackageCacheFolderId> {
let folder_name = folder_name.replace('+', "/");
let (name, ending) = folder_name.rsplit_once('@')?;
let name = if let Some(encoded_name) = name.strip_prefix('_') {
mixed_case_package_name_decode(encoded_name)?
} else {
name.to_string()
};
let (raw_version, copy_index) = match ending.split_once('_') {
Some((raw_version, copy_index)) => {
let copy_index = copy_index.parse::<u8>().ok()?;
(raw_version, copy_index)
}
None => (ending, 0),
};
let version = deno_semver::Version::parse_from_npm(raw_version).ok()?;
Some(NpmPackageCacheFolderId {
nv: PackageNv { name, version },
copy_index,
})
}
fn symlink_package_dir(
old_path: &Path,
new_path: &Path,
) -> Result<(), AnyError> {
let new_parent = new_path.parent().unwrap();
if new_parent.file_name().unwrap() != "node_modules" {
// create the parent folder that will contain the symlink
fs::create_dir_all(new_parent)
.with_context(|| format!("Creating '{}'", new_parent.display()))?;
}
// need to delete the previous symlink before creating a new one
let _ignore = fs::remove_dir_all(new_path);
#[cfg(windows)]
return junction_or_symlink_dir(old_path, new_path);
#[cfg(not(windows))]
symlink_dir(old_path, new_path)
}
#[cfg(windows)]
fn junction_or_symlink_dir(
old_path: &Path,
new_path: &Path,
) -> Result<(), AnyError> {
// Use junctions because they're supported on ntfs file systems without
// needing to elevate privileges on Windows
match junction::create(old_path, new_path) {
Ok(()) => Ok(()),
Err(junction_err) => {
if cfg!(debug) {
// When running the tests, junctions should be created, but if not then
// surface this error.
log::warn!("Error creating junction. {:#}", junction_err);
}
match symlink_dir(old_path, new_path) {
Ok(()) => Ok(()),
Err(symlink_err) => bail!(
concat!(
"Failed creating junction and fallback symlink in node_modules folder.\n\n",
"{:#}\n\n{:#}",
),
junction_err,
symlink_err,
),
}
}
}
}
fn join_package_name(path: &Path, package_name: &str) -> PathBuf {
let mut path = path.to_path_buf();
// ensure backslashes are used on windows
for part in package_name.split('/') {
path = path.join(part);
}
path
}
#[cfg(test)]
mod test {
use deno_npm::NpmPackageCacheFolderId;
use deno_semver::package::PackageNv;
use test_util::TempDir;
use super::*;
#[test]
fn test_get_package_folder_id_folder_name() {
let cases = vec![
(
NpmPackageCacheFolderId {
nv: PackageNv::from_str("@types/foo@1.2.3").unwrap(),
copy_index: 1,
},
"@types+foo@1.2.3_1".to_string(),
),
(
NpmPackageCacheFolderId {
nv: PackageNv::from_str("JSON@3.2.1").unwrap(),
copy_index: 0,
},
"_jjju6tq@3.2.1".to_string(),
),
];
for (input, output) in cases {
assert_eq!(get_package_folder_id_folder_name(&input), output);
let folder_id = get_package_folder_id_from_folder_name(&output).unwrap();
assert_eq!(folder_id, input);
}
}
#[test]
fn test_setup_cache() {
let temp_dir = TempDir::new();
let cache_bin_path = temp_dir.path().join("cache.bin").to_path_buf();
let mut cache = SetupCache::load(cache_bin_path.clone());
assert!(cache.insert_deno_symlink("package-a", "package-a@1.0.0"));
assert!(cache.insert_root_symlink("package-a", "package-a@1.0.0"));
assert!(cache
.with_dep("package-a")
.insert("package-b", "package-b@1.0.0"));
assert!(cache.save());
let mut cache = SetupCache::load(cache_bin_path.clone());
assert!(!cache.insert_deno_symlink("package-a", "package-a@1.0.0"));
assert!(!cache.insert_root_symlink("package-a", "package-a@1.0.0"));
assert!(!cache
.with_dep("package-a")
.insert("package-b", "package-b@1.0.0"));
assert!(!cache.save());
assert!(cache.insert_root_symlink("package-b", "package-b@0.2.0"));
assert!(cache.save());
let mut cache = SetupCache::load(cache_bin_path);
cache.remove_dep("package-a");
assert!(cache
.with_dep("package-a")
.insert("package-b", "package-b@1.0.0"));
}
}