foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-21 23:04:45 -05:00
Code Issues Packages 1 Wiki Activity

perf: more efficient deno cache and npm package info usage (#16592)

Browse source 
1. There was a lot of cloning going on with `NpmPackageInfo`. This is
now stored in an `Arc<NpmPackageInfo>` and cloning only happens on the
individual version.
2. The package cache is now cleared from memory after resolution.
3. This surfaced a bug in `deno cache` and I noticed it can be more
efficient if we have multiple root specifiers if we provide all the
specifiers as roots.
This commit is contained in:
David Sherret 2022-11-11 11:33:57 -05:00 committed by GitHub
parent 7f0546a6b7
commit 8dc242f789
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 84 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
cli/main.rs
View file

@ -284,23 +284,23 @@ async fn check_command(
async fn load_and_type_check(
ps: &ProcState,
files: &Vec<String>,
files: &[String],
) -> Result<(), AnyError> {
let lib = ps.options.ts_type_lib_window();
for file in files {
let specifier = resolve_url_or_path(file)?;
ps.prepare_module_load(
vec![specifier],
false,
lib,
Permissions::allow_all(),
Permissions::allow_all(),
false,
)
.await?;
}
let specifiers = files
.iter()
.map(|file| resolve_url_or_path(file))
.collect::<Result<Vec<_>, _>>()?;
ps.prepare_module_load(
specifiers,
false,
lib,
Permissions::allow_all(),
Permissions::allow_all(),
false,
)
.await?;
Ok(())
}

32
cli/npm/registry.rs
View file

@ -185,12 +185,12 @@ pub trait NpmRegistryApi: Clone + Sync + Send + 'static {
fn maybe_package_info(
&self,
name: &str,
) -> BoxFuture<'static, Result<Option<NpmPackageInfo>, AnyError>>;
) -> BoxFuture<'static, Result<Option<Arc<NpmPackageInfo>>, AnyError>>;
fn package_info(
&self,
name: &str,
) -> BoxFuture<'static, Result<NpmPackageInfo, AnyError>> {
) -> BoxFuture<'static, Result<Arc<NpmPackageInfo>, AnyError>> {
let api = self.clone();
let name = name.to_string();
async move {
@ -212,13 +212,14 @@ pub trait NpmRegistryApi: Clone + Sync + Send + 'static {
let name = name.to_string();
let version = version.to_string();
async move {
// todo(dsherret): this could be optimized to not clone the
// entire package info in the case of the RealNpmRegistryApi
let mut package_info = api.package_info(&name).await?;
Ok(package_info.versions.remove(&version))
let package_info = api.package_info(&name).await?;
Ok(package_info.versions.get(&version).cloned())
}
.boxed()
}
/// Clears the internal memory cache.
fn clear_memory_cache(&self);
}
#[derive(Clone)]
@ -268,17 +269,21 @@ impl NpmRegistryApi for RealNpmRegistryApi {
fn maybe_package_info(
&self,
name: &str,
) -> BoxFuture<'static, Result<Option<NpmPackageInfo>, AnyError>> {
) -> BoxFuture<'static, Result<Option<Arc<NpmPackageInfo>>, AnyError>> {
let api = self.clone();
let name = name.to_string();
async move { api.0.maybe_package_info(&name).await }.boxed()
}
fn clear_memory_cache(&self) {
self.0.mem_cache.lock().clear();
}
}
struct RealNpmRegistryApiInner {
base_url: Url,
cache: NpmCache,
mem_cache: Mutex<HashMap<String, Option<NpmPackageInfo>>>,
mem_cache: Mutex<HashMap<String, Option<Arc<NpmPackageInfo>>>>,
cache_setting: CacheSetting,
progress_bar: ProgressBar,
}
@ -287,7 +292,7 @@ impl RealNpmRegistryApiInner {
pub async fn maybe_package_info(
&self,
name: &str,
) -> Result<Option<NpmPackageInfo>, AnyError> {
) -> Result<Option<Arc<NpmPackageInfo>>, AnyError> {
let maybe_info = self.mem_cache.lock().get(name).cloned();
if let Some(info) = maybe_info {
Ok(info)
@ -306,6 +311,7 @@ impl RealNpmRegistryApiInner {
format!("Error getting response at {}", self.get_package_url(name))
})?;
}
let maybe_package_info = maybe_package_info.map(Arc::new);
// Not worth the complexity to ensure multiple in-flight requests
// for the same package only request once because with how this is
@ -548,8 +554,12 @@ impl NpmRegistryApi for TestNpmRegistryApi {
fn maybe_package_info(
&self,
name: &str,
) -> BoxFuture<'static, Result<Option<NpmPackageInfo>, AnyError>> {
) -> BoxFuture<'static, Result<Option<Arc<NpmPackageInfo>>, AnyError>> {
let result = self.package_infos.lock().get(name).cloned();
Box::pin(deno_core::futures::future::ready(Ok(result)))
Box::pin(deno_core::futures::future::ready(Ok(result.map(Arc::new))))
}
fn clear_memory_cache(&self) {
// do nothing for the test api
}
}

30
cli/npm/resolution/graph.rs
View file

@ -415,7 +415,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi>
&self,
name: &str,
version_matcher: &impl NpmVersionMatcher,
package_info: NpmPackageInfo,
package_info: &NpmPackageInfo,
) -> Result<VersionAndInfo, AnyError> {
if let Some(version) =
self.resolve_best_package_version(name, version_matcher)
@ -462,10 +462,14 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi>
maybe_best_version.cloned()
}
pub fn has_package_req(&self, req: &NpmPackageReq) -> bool {
self.graph.has_package_req(req)
}
pub fn add_package_req(
&mut self,
package_req: &NpmPackageReq,
package_info: NpmPackageInfo,
package_info: &NpmPackageInfo,
) -> Result<(), AnyError> {
let node = self.resolve_node_from_info(
&package_req.name,
@ -484,7 +488,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi>
fn analyze_dependency(
&mut self,
entry: &NpmDependencyEntry,
package_info: NpmPackageInfo,
package_info: &NpmPackageInfo,
parent_id: &NpmPackageId,
visited_versions: &Arc<VisitedVersionsPath>,
) -> Result<(), AnyError> {
@ -536,7 +540,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi>
&mut self,
name: &str,
version_matcher: &impl NpmVersionMatcher,
package_info: NpmPackageInfo,
package_info: &NpmPackageInfo,
) -> Result<Arc<Mutex<Node>>, AnyError> {
let version_and_info = self.resolve_best_package_version_and_info(
name,
@ -613,7 +617,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi>
NpmDependencyEntryKind::Dep => {
self.analyze_dependency(
dep,
package_info,
&package_info,
&parent_id,
&visited_versions,
)?;
@ -624,7 +628,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi>
&dep.bare_specifier,
&parent_id,
dep,
package_info,
&package_info,
&visited_versions,
)?;
if let Some(new_parent_id) = maybe_new_parent_id {
@ -647,7 +651,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi>
specifier: &str,
parent_id: &NpmPackageId,
peer_dep: &NpmDependencyEntry,
peer_package_info: NpmPackageInfo,
peer_package_info: &NpmPackageInfo,
visited_ancestor_versions: &Arc<VisitedVersionsPath>,
) -> Result<Option<NpmPackageId>, AnyError> {
fn find_matching_child<'a>(
@ -881,7 +885,7 @@ struct VersionAndInfo {
fn get_resolved_package_version_and_info(
pkg_name: &str,
version_matcher: &impl NpmVersionMatcher,
info: NpmPackageInfo,
info: &NpmPackageInfo,
parent: Option<&NpmPackageId>,
) -> Result<VersionAndInfo, AnyError> {
let mut maybe_best_version: Option<VersionAndInfo> = None;
@ -922,7 +926,7 @@ fn get_resolved_package_version_and_info(
bail!("Could not find dist-tag '{}'.", tag)
}
} else {
for (_, version_info) in info.versions.into_iter() {
for version_info in info.versions.values() {
let version = NpmVersion::parse(&version_info.version)?;
if version_matcher.matches(&version) {
let is_best_version = maybe_best_version
@ -932,7 +936,7 @@ fn get_resolved_package_version_and_info(
if is_best_version {
maybe_best_version = Some(VersionAndInfo {
version,
info: version_info,
info: version_info.clone(),
});
}
}
@ -981,7 +985,7 @@ mod test {
let result = get_resolved_package_version_and_info(
"test",
&package_ref.req,
NpmPackageInfo {
&NpmPackageInfo {
name: "test".to_string(),
versions: HashMap::new(),
dist_tags: HashMap::from([(
@ -1001,7 +1005,7 @@ mod test {
let result = get_resolved_package_version_and_info(
"test",
&package_ref.req,
NpmPackageInfo {
&NpmPackageInfo {
name: "test".to_string(),
versions: HashMap::from([
("0.1.0".to_string(), NpmPackageVersionInfo::default()),
@ -2014,7 +2018,7 @@ mod test {
for req in reqs {
let req = NpmPackageReference::from_str(req).unwrap().req;
resolver
.add_package_req(&req, api.package_info(&req.name).await.unwrap())
.add_package_req(&req, &api.package_info(&req.name).await.unwrap())
.unwrap();
}

50
cli/npm/resolution/mod.rs
View file

@ -422,42 +422,54 @@ impl NpmResolution {
// multiple packages are resolved in alphabetical order
package_reqs.sort_by(|a, b| a.name.cmp(&b.name));
// go over the top level packages first, then down the
// go over the top level package names first, then down the
// tree one level at a time through all the branches
let mut unresolved_tasks = Vec::with_capacity(package_reqs.len());
for package_req in package_reqs {
if graph.has_package_req(&package_req) {
let mut resolving_package_names =
HashSet::with_capacity(package_reqs.len());
for package_req in &package_reqs {
if graph.has_package_req(package_req) {
// skip analyzing this package, as there's already a matching top level package
continue;
}
if !resolving_package_names.insert(package_req.name.clone()) {
continue; // already resolving
}
// no existing best version, so resolve the current packages
let api = self.api.clone();
let maybe_info = if should_sync_download() {
// cache the package info up front in parallel
if should_sync_download() {
// for deterministic test output
Some(api.package_info(&package_req.name).await)
self.api.package_info(&package_req.name).await?;
} else {
None
let api = self.api.clone();
let package_name = package_req.name.clone();
unresolved_tasks.push(tokio::task::spawn(async move {
// This is ok to call because api will internally cache
// the package information in memory.
api.package_info(&package_name).await
}));
};
unresolved_tasks.push(tokio::task::spawn(async move {
let info = match maybe_info {
Some(info) => info?,
None => api.package_info(&package_req.name).await?,
};
Result::<_, AnyError>::Ok((package_req, info))
}));
}
for result in futures::future::join_all(unresolved_tasks).await {
result??; // surface the first error
}
let mut resolver = GraphDependencyResolver::new(&mut graph, &self.api);
for result in futures::future::join_all(unresolved_tasks).await {
let (package_req, info) = result??;
resolver.add_package_req(&package_req, info)?;
for package_req in package_reqs {
// avoid loading the info if this is already in the graph
if !resolver.has_package_req(&package_req) {
let info = self.api.package_info(&package_req.name).await?;
resolver.add_package_req(&package_req, &info)?;
}
}
resolver.resolve_pending().await?;
graph.into_snapshot(&self.api).await
let result = graph.into_snapshot(&self.api).await;
self.api.clear_memory_cache();
result
}
pub fn resolve_package_from_id(

2
cli/tests/testdata/cache/037_fetch_multiple.out vendored
View file

@ -1,5 +1,5 @@
Download http://localhost:4545/subdir/mod2.ts
Download http://localhost:4545/subdir/mt_text_typescript.t1.ts
Download http://localhost:4545/subdir/print_hello.ts
Check [WILDCARD]/fetch/test.ts
Download http://localhost:4545/subdir/mt_text_typescript.t1.ts
Check [WILDCARD]/fetch/other.ts

1
cli/tests/testdata/npm/deno_cache.out vendored
View file

@ -1,5 +1,4 @@
Download http://localhost:4545/npm/registry/chalk
Download http://localhost:4545/npm/registry/chalk/chalk-5.0.1.tgz
Download http://localhost:4545/npm/registry/mkdirp
Download http://localhost:4545/npm/registry/chalk/chalk-5.0.1.tgz
Download http://localhost:4545/npm/registry/mkdirp/mkdirp-1.0.4.tgz