mirror of
https://github.com/denoland/deno.git
synced 2024-12-22 23:34:47 -05:00
1b355d8a87
Introduces a `SyncReadAsyncWriteLock` to make it harder to write to the npm resolution without first waiting async in a queue. For the npm resolution, reading synchronously is fine, but when updating, someone should wait async, clone the data, then write the data at the end back.
254 lines
8.1 KiB
Rust
254 lines
8.1 KiB
Rust
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
|
|
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
|
|
use deno_core::anyhow::anyhow;
|
|
use deno_core::anyhow::bail;
|
|
use deno_core::anyhow::Context;
|
|
use deno_core::error::custom_error;
|
|
use deno_core::error::AnyError;
|
|
use deno_core::futures::future::LocalBoxFuture;
|
|
use deno_core::futures::FutureExt;
|
|
use deno_core::parking_lot::Mutex;
|
|
use deno_core::serde_json;
|
|
use deno_core::url::Url;
|
|
use deno_npm::npm_rc::ResolvedNpmRc;
|
|
use deno_npm::registry::NpmPackageInfo;
|
|
|
|
use crate::args::CacheSetting;
|
|
use crate::http_util::HttpClientProvider;
|
|
use crate::npm::common::maybe_auth_header_for_npm_registry;
|
|
use crate::util::progress_bar::ProgressBar;
|
|
use crate::util::sync::MultiRuntimeAsyncValueCreator;
|
|
|
|
use super::NpmCache;
|
|
|
|
// todo(dsherret): create seams and unit test this
|
|
|
|
type LoadResult = Result<FutureResult, Arc<AnyError>>;
|
|
type LoadFuture = LocalBoxFuture<'static, LoadResult>;
|
|
|
|
#[derive(Debug, Clone)]
|
|
enum FutureResult {
|
|
PackageNotExists,
|
|
SavedFsCache(Arc<NpmPackageInfo>),
|
|
ErroredFsCache(Arc<NpmPackageInfo>),
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
enum MemoryCacheItem {
|
|
/// The cache item hasn't loaded yet.
|
|
Pending(Arc<MultiRuntimeAsyncValueCreator<LoadResult>>),
|
|
/// The item has loaded in the past and was stored in the file system cache.
|
|
/// There is no reason to request this package from the npm registry again
|
|
/// for the duration of execution.
|
|
FsCached,
|
|
/// An item is memory cached when it fails saving to the file system cache
|
|
/// or the package does not exist.
|
|
MemoryCached(Result<Option<Arc<NpmPackageInfo>>, Arc<AnyError>>),
|
|
}
|
|
|
|
/// Downloads packuments from the npm registry.
|
|
///
|
|
/// This is shared amongst all the workers.
|
|
#[derive(Debug)]
|
|
pub struct RegistryInfoDownloader {
|
|
cache: Arc<NpmCache>,
|
|
http_client_provider: Arc<HttpClientProvider>,
|
|
npmrc: Arc<ResolvedNpmRc>,
|
|
progress_bar: ProgressBar,
|
|
memory_cache: Mutex<HashMap<String, MemoryCacheItem>>,
|
|
}
|
|
|
|
impl RegistryInfoDownloader {
|
|
pub fn new(
|
|
cache: Arc<NpmCache>,
|
|
http_client_provider: Arc<HttpClientProvider>,
|
|
npmrc: Arc<ResolvedNpmRc>,
|
|
progress_bar: ProgressBar,
|
|
) -> Self {
|
|
Self {
|
|
cache,
|
|
http_client_provider,
|
|
npmrc,
|
|
progress_bar,
|
|
memory_cache: Default::default(),
|
|
}
|
|
}
|
|
|
|
pub async fn load_package_info(
|
|
self: &Arc<Self>,
|
|
name: &str,
|
|
) -> Result<Option<Arc<NpmPackageInfo>>, AnyError> {
|
|
self.load_package_info_inner(name).await.with_context(|| {
|
|
format!(
|
|
"Error getting response at {} for package \"{}\"",
|
|
self.get_package_url(name),
|
|
name
|
|
)
|
|
})
|
|
}
|
|
|
|
async fn load_package_info_inner(
|
|
self: &Arc<Self>,
|
|
name: &str,
|
|
) -> Result<Option<Arc<NpmPackageInfo>>, AnyError> {
|
|
if *self.cache.cache_setting() == CacheSetting::Only {
|
|
return Err(custom_error(
|
|
"NotCached",
|
|
format!(
|
|
"An npm specifier not found in cache: \"{name}\", --cached-only is specified."
|
|
)
|
|
));
|
|
}
|
|
|
|
let cache_item = {
|
|
let mut mem_cache = self.memory_cache.lock();
|
|
if let Some(cache_item) = mem_cache.get(name) {
|
|
cache_item.clone()
|
|
} else {
|
|
let value_creator = MultiRuntimeAsyncValueCreator::new({
|
|
let downloader = self.clone();
|
|
let name = name.to_string();
|
|
Box::new(move || downloader.create_load_future(&name))
|
|
});
|
|
let cache_item = MemoryCacheItem::Pending(Arc::new(value_creator));
|
|
mem_cache.insert(name.to_string(), cache_item.clone());
|
|
cache_item
|
|
}
|
|
};
|
|
|
|
match cache_item {
|
|
MemoryCacheItem::FsCached => {
|
|
// this struct previously loaded from the registry, so we can load it from the file system cache
|
|
self
|
|
.load_file_cached_package_info(name)
|
|
.await
|
|
.map(|info| Some(Arc::new(info)))
|
|
}
|
|
MemoryCacheItem::MemoryCached(maybe_info) => {
|
|
maybe_info.clone().map_err(|e| anyhow!("{}", e))
|
|
}
|
|
MemoryCacheItem::Pending(value_creator) => {
|
|
match value_creator.get().await {
|
|
Ok(FutureResult::SavedFsCache(info)) => {
|
|
// return back the future and mark this package as having
|
|
// been saved in the cache for next time it's requested
|
|
*self.memory_cache.lock().get_mut(name).unwrap() =
|
|
MemoryCacheItem::FsCached;
|
|
Ok(Some(info))
|
|
}
|
|
Ok(FutureResult::ErroredFsCache(info)) => {
|
|
// since saving to the fs cache failed, keep the package information in memory
|
|
*self.memory_cache.lock().get_mut(name).unwrap() =
|
|
MemoryCacheItem::MemoryCached(Ok(Some(info.clone())));
|
|
Ok(Some(info))
|
|
}
|
|
Ok(FutureResult::PackageNotExists) => {
|
|
*self.memory_cache.lock().get_mut(name).unwrap() =
|
|
MemoryCacheItem::MemoryCached(Ok(None));
|
|
Ok(None)
|
|
}
|
|
Err(err) => {
|
|
let return_err = anyhow!("{}", err);
|
|
*self.memory_cache.lock().get_mut(name).unwrap() =
|
|
MemoryCacheItem::MemoryCached(Err(err));
|
|
Err(return_err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn load_file_cached_package_info(
|
|
&self,
|
|
name: &str,
|
|
) -> Result<NpmPackageInfo, AnyError> {
|
|
// this scenario failing should be exceptionally rare so let's
|
|
// deal with improving it only when anyone runs into an issue
|
|
let maybe_package_info = deno_core::unsync::spawn_blocking({
|
|
let cache = self.cache.clone();
|
|
let name = name.to_string();
|
|
move || cache.load_package_info(&name)
|
|
})
|
|
.await
|
|
.unwrap()
|
|
.with_context(|| {
|
|
format!(
|
|
"Previously saved '{}' from the npm cache, but now it fails to load.",
|
|
name
|
|
)
|
|
})?;
|
|
match maybe_package_info {
|
|
Some(package_info) => Ok(package_info),
|
|
None => {
|
|
bail!("The package '{}' previously saved its registry information to the file system cache, but that file no longer exists.", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
fn create_load_future(self: &Arc<Self>, name: &str) -> LoadFuture {
|
|
let downloader = self.clone();
|
|
let package_url = self.get_package_url(name);
|
|
let registry_config = self.npmrc.get_registry_config(name);
|
|
let maybe_auth_header = maybe_auth_header_for_npm_registry(registry_config);
|
|
let guard = self.progress_bar.update(package_url.as_str());
|
|
let name = name.to_string();
|
|
async move {
|
|
let maybe_bytes = downloader
|
|
.http_client_provider
|
|
.get_or_create()?
|
|
.download_with_progress(package_url, maybe_auth_header, &guard)
|
|
.await?;
|
|
match maybe_bytes {
|
|
Some(bytes) => {
|
|
let future_result = deno_core::unsync::spawn_blocking(
|
|
move || -> Result<FutureResult, AnyError> {
|
|
let package_info = serde_json::from_slice(&bytes)?;
|
|
match downloader.cache.save_package_info(&name, &package_info) {
|
|
Ok(()) => {
|
|
Ok(FutureResult::SavedFsCache(Arc::new(package_info)))
|
|
}
|
|
Err(err) => {
|
|
log::debug!(
|
|
"Error saving package {} to cache: {:#}",
|
|
name,
|
|
err
|
|
);
|
|
Ok(FutureResult::ErroredFsCache(Arc::new(package_info)))
|
|
}
|
|
}
|
|
},
|
|
)
|
|
.await??;
|
|
Ok(future_result)
|
|
}
|
|
None => Ok(FutureResult::PackageNotExists),
|
|
}
|
|
}
|
|
.map(|r| r.map_err(Arc::new))
|
|
.boxed_local()
|
|
}
|
|
|
|
fn get_package_url(&self, name: &str) -> Url {
|
|
let registry_url = self.npmrc.get_registry_url(name);
|
|
// list of all characters used in npm packages:
|
|
// !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~
|
|
const ASCII_SET: percent_encoding::AsciiSet =
|
|
percent_encoding::NON_ALPHANUMERIC
|
|
.remove(b'!')
|
|
.remove(b'\'')
|
|
.remove(b'(')
|
|
.remove(b')')
|
|
.remove(b'*')
|
|
.remove(b'-')
|
|
.remove(b'.')
|
|
.remove(b'/')
|
|
.remove(b'@')
|
|
.remove(b'_')
|
|
.remove(b'~');
|
|
let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET);
|
|
registry_url.join(&name.to_string()).unwrap()
|
|
}
|
|
}
|