diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 7004f11af7..fc64210562 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -46,6 +46,7 @@ pub use emit::EmitCache; pub use incremental::IncrementalCache; pub use module_info::ModuleInfoCache; pub use node::NodeAnalysisCache; +pub use parsed_source::LazyGraphSourceParser; pub use parsed_source::ParsedSourceCache; /// Permissions used to save a file in the disk caches. diff --git a/cli/cache/parsed_source.rs b/cli/cache/parsed_source.rs index 7bb1a72a72..75170aaf96 100644 --- a/cli/cache/parsed_source.rs +++ b/cli/cache/parsed_source.rs @@ -11,6 +11,39 @@ use deno_graph::CapturingModuleParser; use deno_graph::ModuleParser; use deno_graph::ParseOptions; +/// Lazily parses JS/TS sources from a `deno_graph::ModuleGraph` given +/// a `ParsedSourceCache`. Note that deno_graph doesn't necessarily cause +/// files to end up in the `ParsedSourceCache` because it might have all +/// the information it needs via caching in order to skip parsing. +#[derive(Clone, Copy)] +pub struct LazyGraphSourceParser<'a> { + cache: &'a ParsedSourceCache, + graph: &'a deno_graph::ModuleGraph, +} + +impl<'a> LazyGraphSourceParser<'a> { + pub fn new( + cache: &'a ParsedSourceCache, + graph: &'a deno_graph::ModuleGraph, + ) -> Self { + Self { cache, graph } + } + + pub fn get_or_parse_source( + &self, + module_specifier: &ModuleSpecifier, + ) -> Result, deno_ast::Diagnostic> { + let Some(deno_graph::Module::Js(module)) = self.graph.get(module_specifier) + else { + return Ok(None); + }; + self + .cache + .get_parsed_source_from_js_module(module) + .map(Some) + } +} + #[derive(Default)] pub struct ParsedSourceCache { sources: Mutex>, diff --git a/cli/diagnostics.rs b/cli/diagnostics.rs index 846cd7751f..0a674a2c48 100644 --- a/cli/diagnostics.rs +++ b/cli/diagnostics.rs @@ -11,10 +11,11 @@ use deno_ast::SourcePos; use deno_ast::SourceRange; use deno_ast::SourceRanged; use deno_ast::SourceTextInfo; -use deno_graph::ParsedSourceStore; use deno_runtime::colors; use unicode_width::UnicodeWidthStr; +use crate::cache::LazyGraphSourceParser; + pub trait SourceTextStore { fn get_source_text<'a>( &'a self, @@ -22,14 +23,14 @@ pub trait SourceTextStore { ) -> Option>; } -pub struct SourceTextParsedSourceStore<'a>(pub &'a dyn ParsedSourceStore); +pub struct SourceTextParsedSourceStore<'a>(pub LazyGraphSourceParser<'a>); impl SourceTextStore for SourceTextParsedSourceStore<'_> { fn get_source_text<'a>( &'a self, specifier: &ModuleSpecifier, ) -> Option> { - let parsed_source = self.0.get_parsed_source(specifier)?; + let parsed_source = self.0.get_or_parse_source(specifier).ok()??; Some(Cow::Owned(parsed_source.text_info().clone())) } } diff --git a/cli/tests/integration/publish_tests.rs b/cli/tests/integration/publish_tests.rs index dbbbe19120..b15ca5f89f 100644 --- a/cli/tests/integration/publish_tests.rs +++ b/cli/tests/integration/publish_tests.rs @@ -66,6 +66,37 @@ itest!(invalid_import { http_server: true, }); +#[test] +fn publish_non_exported_files_using_import_map() { + let context = publish_context_builder().build(); + let temp_dir = context.temp_dir().path(); + temp_dir.join("deno.json").write_json(&json!({ + "name": "@foo/bar", + "version": "1.0.0", + "exports": "./mod.ts", + "imports": { + "@denotest/add": "jsr:@denotest/add@1" + } + })); + // file not in the graph + let other_ts = temp_dir.join("_other.ts"); + other_ts + .write("import { add } from '@denotest/add'; console.log(add(1, 3));"); + let mod_ts = temp_dir.join("mod.ts"); + mod_ts.write("import { add } from '@denotest/add'; console.log(add(1, 2));"); + let output = context + .new_command() + .args("publish --log-level=debug --token 'sadfasdf'") + .run(); + let lines = output.combined_output().split('\n').collect::>(); + assert!(lines + .iter() + .any(|l| l.contains("Unfurling") && l.ends_with("mod.ts"))); + assert!(lines + .iter() + .any(|l| l.contains("Unfurling") && l.ends_with("other.ts"))); +} + itest!(javascript_missing_decl_file { args: "publish --token 'sadfasdf'", output: "publish/javascript_missing_decl_file.out", diff --git a/cli/tests/testdata/jsr/registry/@denotest/add/1.0.0/mod.ts b/cli/tests/testdata/jsr/registry/@denotest/add/1.0.0/mod.ts new file mode 100644 index 0000000000..8d9b8a22a1 --- /dev/null +++ b/cli/tests/testdata/jsr/registry/@denotest/add/1.0.0/mod.ts @@ -0,0 +1,3 @@ +export function add(a: number, b: number): number { + return a + b; +} diff --git a/cli/tests/testdata/jsr/registry/@denotest/add/1.0.0_meta.json b/cli/tests/testdata/jsr/registry/@denotest/add/1.0.0_meta.json new file mode 100644 index 0000000000..6eebe21985 --- /dev/null +++ b/cli/tests/testdata/jsr/registry/@denotest/add/1.0.0_meta.json @@ -0,0 +1,8 @@ +{ + "exports": { + ".": "./mod.ts" + }, + "moduleGraph1": { + "/mod.ts": {} + } +} diff --git a/cli/tests/testdata/jsr/registry/@denotest/add/meta.json b/cli/tests/testdata/jsr/registry/@denotest/add/meta.json new file mode 100644 index 0000000000..02601e4d0d --- /dev/null +++ b/cli/tests/testdata/jsr/registry/@denotest/add/meta.json @@ -0,0 +1,5 @@ +{ + "versions": { + "1.0.0": {} + } +} diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index acea65062b..729ee05fcb 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -4,6 +4,7 @@ use crate::args::DocFlags; use crate::args::DocHtmlFlag; use crate::args::DocSourceFileFlag; use crate::args::Flags; +use crate::cache::LazyGraphSourceParser; use crate::colors; use crate::diagnostics::Diagnostic; use crate::diagnostics::DiagnosticLevel; @@ -142,7 +143,10 @@ pub async fn doc(flags: Flags, doc_flags: DocFlags) -> Result<(), AnyError> { if doc_flags.lint { let diagnostics = doc_parser.take_diagnostics(); - check_diagnostics(&**parsed_source_cache, &diagnostics)?; + check_diagnostics( + LazyGraphSourceParser::new(parsed_source_cache, &graph), + &diagnostics, + )?; } doc_nodes_by_url @@ -413,7 +417,7 @@ impl Diagnostic for DocDiagnostic { } fn check_diagnostics( - parsed_source_cache: &dyn deno_graph::ParsedSourceStore, + source_parser: LazyGraphSourceParser, diagnostics: &[DocDiagnostic], ) -> Result<(), AnyError> { if diagnostics.is_empty() { @@ -437,8 +441,8 @@ fn check_diagnostics( for (_, diagnostics_by_col) in diagnostics_by_lc { for (_, diagnostics) in diagnostics_by_col { for diagnostic in diagnostics { - let sources = SourceTextParsedSourceStore(parsed_source_cache); - eprintln!("{}", diagnostic.display(&sources)); + let sources = SourceTextParsedSourceStore(source_parser); + log::error!("{}", diagnostic.display(&sources)); } } } diff --git a/cli/tools/registry/diagnostics.rs b/cli/tools/registry/diagnostics.rs index 643a4fb2d1..e7f9473038 100644 --- a/cli/tools/registry/diagnostics.rs +++ b/cli/tools/registry/diagnostics.rs @@ -10,9 +10,9 @@ use deno_ast::swc::common::util::take::Take; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; use deno_graph::FastCheckDiagnostic; -use deno_graph::ParsedSourceStore; use lsp_types::Url; +use crate::cache::LazyGraphSourceParser; use crate::diagnostics::Diagnostic; use crate::diagnostics::DiagnosticLevel; use crate::diagnostics::DiagnosticLocation; @@ -33,7 +33,7 @@ pub struct PublishDiagnosticsCollector { impl PublishDiagnosticsCollector { pub fn print_and_error( &self, - sources: &dyn ParsedSourceStore, + sources: LazyGraphSourceParser, ) -> Result<(), AnyError> { let mut errors = 0; let mut has_zap_errors = false; diff --git a/cli/tools/registry/mod.rs b/cli/tools/registry/mod.rs index 5b3b70c4bc..22f53dab41 100644 --- a/cli/tools/registry/mod.rs +++ b/cli/tools/registry/mod.rs @@ -27,6 +27,7 @@ use crate::args::deno_registry_url; use crate::args::CliOptions; use crate::args::Flags; use crate::args::PublishFlags; +use crate::cache::LazyGraphSourceParser; use crate::cache::ParsedSourceCache; use crate::factory::CliFactory; use crate::graph_util::ModuleGraphBuilder; @@ -90,6 +91,7 @@ fn get_deno_json_package_name( async fn prepare_publish( deno_json: &ConfigFile, source_cache: Arc, + graph: Arc, import_map: Arc, diagnostics_collector: &PublishDiagnosticsCollector, ) -> Result, AnyError> { @@ -140,7 +142,7 @@ async fn prepare_publish( let unfurler = ImportMapUnfurler::new(&import_map); tar::create_gzipped_tarball( &dir_path, - &*source_cache, + LazyGraphSourceParser::new(&source_cache, &graph), &diagnostics_collector, &unfurler, file_patterns, @@ -639,19 +641,19 @@ async fn publish_package( Ok(()) } +struct PreparePackagesData { + publish_order_graph: PublishOrderGraph, + graph: Arc, + package_by_name: HashMap>, +} + async fn prepare_packages_for_publishing( cli_factory: &CliFactory, no_zap: bool, diagnostics_collector: &PublishDiagnosticsCollector, deno_json: ConfigFile, import_map: Arc, -) -> Result< - ( - PublishOrderGraph, - HashMap>, - ), - AnyError, -> { +) -> Result { let maybe_workspace_config = deno_json.to_workspace_config()?; let module_graph_builder = cli_factory.module_graph_builder().await?.as_ref(); let source_cache = cli_factory.parsed_source_cache(); @@ -660,7 +662,7 @@ async fn prepare_packages_for_publishing( let Some(workspace_config) = maybe_workspace_config else { let roots = resolve_config_file_roots_from_exports(&deno_json)?; - build_and_check_graph_for_publish( + let graph = build_and_check_graph_for_publish( module_graph_builder, type_checker, cli_options, @@ -673,10 +675,10 @@ async fn prepare_packages_for_publishing( }], ) .await?; - let mut prepared_package_by_name = HashMap::with_capacity(1); let package = prepare_publish( &deno_json, source_cache.clone(), + graph.clone(), import_map, diagnostics_collector, ) @@ -684,8 +686,12 @@ async fn prepare_packages_for_publishing( let package_name = format!("@{}/{}", package.scope, package.package); let publish_order_graph = PublishOrderGraph::new_single(package_name.clone()); - prepared_package_by_name.insert(package_name, package); - return Ok((publish_order_graph, prepared_package_by_name)); + let package_by_name = HashMap::from([(package_name, package)]); + return Ok(PreparePackagesData { + publish_order_graph, + graph, + package_by_name, + }); }; println!("Publishing a workspace..."); @@ -701,7 +707,7 @@ async fn prepare_packages_for_publishing( ) .await?; - let mut prepared_package_by_name = + let mut package_by_name = HashMap::with_capacity(workspace_config.members.len()); let publish_order_graph = publish_order::build_publish_order_graph(&graph, &roots)?; @@ -712,11 +718,13 @@ async fn prepare_packages_for_publishing( .cloned() .map(|member| { let import_map = import_map.clone(); + let graph = graph.clone(); async move { let package = prepare_publish( &member.config_file, source_cache.clone(), - import_map.clone(), + graph, + import_map, diagnostics_collector, ) .await @@ -731,9 +739,13 @@ async fn prepare_packages_for_publishing( let results = deno_core::futures::future::join_all(results).await; for result in results { let (package_name, package) = result?; - prepared_package_by_name.insert(package_name, package); + package_by_name.insert(package_name, package); } - Ok((publish_order_graph, prepared_package_by_name)) + Ok(PreparePackagesData { + publish_order_graph, + graph, + package_by_name, + }) } async fn build_and_check_graph_for_publish( @@ -828,20 +840,22 @@ pub async fn publish( let diagnostics_collector = PublishDiagnosticsCollector::default(); - let (publish_order_graph, prepared_package_by_name) = - prepare_packages_for_publishing( - &cli_factory, - publish_flags.no_zap, - &diagnostics_collector, - config_file.clone(), - import_map, - ) - .await?; + let prepared_data = prepare_packages_for_publishing( + &cli_factory, + publish_flags.no_zap, + &diagnostics_collector, + config_file.clone(), + import_map, + ) + .await?; - diagnostics_collector - .print_and_error(&**cli_factory.parsed_source_cache())?; + let source_parser = LazyGraphSourceParser::new( + cli_factory.parsed_source_cache(), + &prepared_data.graph, + ); + diagnostics_collector.print_and_error(source_parser)?; - if prepared_package_by_name.is_empty() { + if prepared_data.package_by_name.is_empty() { bail!("No packages to publish"); } @@ -855,8 +869,8 @@ pub async fn publish( perform_publish( cli_factory.http_client(), - publish_order_graph, - prepared_package_by_name, + prepared_data.publish_order_graph, + prepared_data.package_by_name, auth_method, ) .await diff --git a/cli/tools/registry/tar.rs b/cli/tools/registry/tar.rs index 1dcfe2949b..e63a76516f 100644 --- a/cli/tools/registry/tar.rs +++ b/cli/tools/registry/tar.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use bytes::Bytes; +use deno_ast::MediaType; use deno_config::glob::FilePatterns; use deno_core::anyhow::Context; use deno_core::error::AnyError; @@ -13,6 +14,7 @@ use std::io::Write; use std::path::Path; use tar::Header; +use crate::cache::LazyGraphSourceParser; use crate::tools::registry::paths::PackagePath; use crate::util::import_map::ImportMapUnfurler; @@ -34,7 +36,7 @@ pub struct PublishableTarball { pub fn create_gzipped_tarball( dir: &Path, - source_cache: &dyn deno_graph::ParsedSourceStore, + source_parser: LazyGraphSourceParser, diagnostics_collector: &PublishDiagnosticsCollector, unfurler: &ImportMapUnfurler, file_patterns: Option, @@ -122,25 +124,17 @@ pub fn create_gzipped_tarball( } } - let data = std::fs::read(path).with_context(|| { - format!("Unable to read file '{}'", entry.path().display()) - })?; + let content = resolve_content_maybe_unfurling( + path, + &specifier, + unfurler, + source_parser, + diagnostics_collector, + )?; files.push(PublishableTarballFile { specifier: specifier.clone(), - size: data.len(), + size: content.len(), }); - let content = match source_cache.get_parsed_source(&specifier) { - Some(parsed_source) => { - let mut reporter = |diagnostic| { - diagnostics_collector - .push(PublishDiagnostic::ImportMapUnfurl(diagnostic)); - }; - let content = - unfurler.unfurl(&specifier, &parsed_source, &mut reporter); - content.into_bytes() - } - None => data, - }; tar .add_file(format!(".{}", path_str), &content) .with_context(|| { @@ -172,6 +166,64 @@ pub fn create_gzipped_tarball( }) } +fn resolve_content_maybe_unfurling( + path: &Path, + specifier: &Url, + unfurler: &ImportMapUnfurler, + source_parser: LazyGraphSourceParser, + diagnostics_collector: &PublishDiagnosticsCollector, +) -> Result, AnyError> { + let parsed_source = match source_parser.get_or_parse_source(specifier)? { + Some(parsed_source) => parsed_source, + None => { + let data = std::fs::read(path) + .with_context(|| format!("Unable to read file '{}'", path.display()))?; + let media_type = MediaType::from_specifier(specifier); + + match media_type { + MediaType::JavaScript + | MediaType::Jsx + | MediaType::Mjs + | MediaType::Cjs + | MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Dts + | MediaType::Dmts + | MediaType::Dcts + | MediaType::Tsx => { + // continue + } + MediaType::SourceMap + | MediaType::Unknown + | MediaType::Json + | MediaType::Wasm + | MediaType::TsBuildInfo => { + // not unfurlable data + return Ok(data); + } + } + + let text = String::from_utf8(data)?; + deno_ast::parse_module(deno_ast::ParseParams { + specifier: specifier.to_string(), + text_info: deno_ast::SourceTextInfo::from_string(text), + media_type, + capture_tokens: false, + maybe_syntax: None, + scope_analysis: false, + })? + } + }; + + log::debug!("Unfurling {}", specifier); + let mut reporter = |diagnostic| { + diagnostics_collector.push(PublishDiagnostic::ImportMapUnfurl(diagnostic)); + }; + let content = unfurler.unfurl(specifier, &parsed_source, &mut reporter); + Ok(content.into_bytes()) +} + struct TarGzArchive { builder: tar::Builder>, } diff --git a/test_util/src/servers/registry.rs b/test_util/src/servers/registry.rs index c88045a0d0..69728f706e 100644 --- a/test_util/src/servers/registry.rs +++ b/test_util/src/servers/registry.rs @@ -1,5 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use crate::testdata_path; + use super::run_server; use super::ServerKind; use super::ServerOptions; @@ -59,6 +61,16 @@ async fn registry_server_handler( return Ok(res); } + // serve the registry package files + let mut file_path = + testdata_path().to_path_buf().join("jsr").join("registry"); + file_path.push(&req.uri().path()[1..].replace("%2f", "/")); + if let Ok(body) = tokio::fs::read(&file_path).await { + return Ok(Response::new(UnsyncBoxBody::new( + http_body_util::Full::new(Bytes::from(body)), + ))); + } + let empty_body = UnsyncBoxBody::new(Empty::new()); let res = Response::builder() .status(StatusCode::NOT_FOUND)