1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-24 08:09:08 -05:00

feat(unstable): fast subset type checking of JSR dependencies (#21873)

This commit is contained in:
David Sherret 2024-01-10 17:40:30 -05:00 committed by GitHub
parent 515a34b4de
commit 70ac06138c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
42 changed files with 714 additions and 149 deletions

8
Cargo.lock generated
View file

@ -1228,9 +1228,9 @@ dependencies = [
[[package]] [[package]]
name = "deno_doc" name = "deno_doc"
version = "0.89.0" version = "0.89.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00f16c99bcba738fce91decb9ac44213aa0d6b6a03dfd40af2ff2bca0d687fc6" checksum = "1a7eff3f43da69cecfb9a352bbcef2871cea9c188828154f01d5cd2448c75ce7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"cfg-if", "cfg-if",
@ -1322,9 +1322,9 @@ dependencies = [
[[package]] [[package]]
name = "deno_graph" name = "deno_graph"
version = "0.63.0" version = "0.63.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9a7517284a929f0f2f4db8b241c840bccd4debd6635ea0bc7a906c0254a0231" checksum = "1b2fce4e5ae279e181e53a3fa0e9018956356d6b977cc741b665eebe5398a01c"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-trait", "async-trait",

View file

@ -57,9 +57,9 @@ deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "dep_gra
deno_cache_dir = "=0.6.1" deno_cache_dir = "=0.6.1"
deno_config = "=0.6.5" deno_config = "=0.6.5"
deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
deno_doc = { version = "=0.89.0", features = ["html"] } deno_doc = { version = "=0.89.1", features = ["html"] }
deno_emit = "=0.33.0" deno_emit = "=0.33.0"
deno_graph = "=0.63.0" deno_graph = "=0.63.2"
deno_lint = { version = "=0.53.0", features = ["docs"] } deno_lint = { version = "=0.53.0", features = ["docs"] }
deno_lockfile.workspace = true deno_lockfile.workspace = true
deno_npm = "=0.15.3" deno_npm = "=0.15.3"

View file

@ -327,7 +327,6 @@ pub struct VendorFlags {
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
pub struct PublishFlags { pub struct PublishFlags {
pub directory: String,
pub token: Option<String>, pub token: Option<String>,
} }
@ -2372,19 +2371,10 @@ Remote modules and multiple modules may also be specified:
fn publish_subcommand() -> Command { fn publish_subcommand() -> Command {
Command::new("publish") Command::new("publish")
.hide(true) .hide(true)
.about("Unstable preview feature: Publish a package") .about("Unstable preview feature: Publish the current working directory's package or workspace")
// TODO: .long_about() // TODO: .long_about()
.defer(|cmd| { .defer(|cmd| {
cmd.arg( cmd.arg(
Arg::new("directory")
.help(
"The directory to the package, or workspace of packages to publish",
)
.default_missing_value(".")
.value_hint(ValueHint::DirPath)
.required(true),
)
.arg(
Arg::new("token") Arg::new("token")
.long("token") .long("token")
.help("The API token to use when publishing. If unset, interactive authentication is be used") .help("The API token to use when publishing. If unset, interactive authentication is be used")
@ -3821,7 +3811,6 @@ fn vendor_parse(flags: &mut Flags, matches: &mut ArgMatches) {
fn publish_parse(flags: &mut Flags, matches: &mut ArgMatches) { fn publish_parse(flags: &mut Flags, matches: &mut ArgMatches) {
flags.subcommand = DenoSubcommand::Publish(PublishFlags { flags.subcommand = DenoSubcommand::Publish(PublishFlags {
directory: matches.remove_one::<String>("directory").unwrap(),
token: matches.remove_one("token"), token: matches.remove_one("token"),
}); });
} }

View file

@ -54,6 +54,14 @@ itest!(deps_info {
http_server: true, http_server: true,
}); });
itest!(subset_type_graph {
args: "check --all jsr/subset_type_graph/main.ts",
output: "jsr/subset_type_graph/main.check.out",
envs: env_vars_for_jsr_tests(),
http_server: true,
exit_code: 1,
});
itest!(version_not_found { itest!(version_not_found {
args: "run jsr/version_not_found/main.ts", args: "run jsr/version_not_found/main.ts",
output: "jsr/version_not_found/main.out", output: "jsr/version_not_found/main.out",

View file

@ -15,21 +15,75 @@ pub fn env_vars_for_registry() -> Vec<(String, String)> {
} }
itest!(no_token { itest!(no_token {
args: "publish publish/missing_deno_json", args: "publish",
cwd: Some("publish/missing_deno_json"),
output: "publish/no_token.out", output: "publish/no_token.out",
exit_code: 1, exit_code: 1,
}); });
itest!(missing_deno_json { itest!(missing_deno_json {
args: "publish --token 'sadfasdf' $TESTDATA/publish/missing_deno_json", args: "publish --token 'sadfasdf'",
output: "publish/missing_deno_json.out", output: "publish/missing_deno_json.out",
cwd: Some("publish/missing_deno_json"),
copy_temp_dir: Some("publish/missing_deno_json"),
exit_code: 1, exit_code: 1,
temp_cwd: true, temp_cwd: true,
}); });
itest!(invalid_fast_check {
args: "publish --token 'sadfasdf'",
output: "publish/invalid_fast_check.out",
cwd: Some("publish/invalid_fast_check"),
copy_temp_dir: Some("publish/invalid_fast_check"),
exit_code: 1,
temp_cwd: true,
});
itest!(javascript_missing_decl_file {
args: "publish --token 'sadfasdf'",
output: "publish/javascript_missing_decl_file.out",
cwd: Some("publish/javascript_missing_decl_file"),
copy_temp_dir: Some("publish/javascript_missing_decl_file"),
envs: env_vars_for_registry(),
exit_code: 0,
temp_cwd: true,
});
itest!(javascript_decl_file {
args: "publish --token 'sadfasdf'",
output: "publish/javascript_decl_file.out",
cwd: Some("publish/javascript_decl_file"),
copy_temp_dir: Some("publish/javascript_decl_file"),
envs: env_vars_for_registry(),
exit_code: 0,
temp_cwd: true,
});
itest!(successful { itest!(successful {
args: "publish --token 'sadfasdf' $TESTDATA/publish/successful", args: "publish --token 'sadfasdf'",
output: "publish/successful.out", output: "publish/successful.out",
cwd: Some("publish/successful"),
copy_temp_dir: Some("publish/successful"),
envs: env_vars_for_registry(),
http_server: true,
temp_cwd: true,
});
itest!(workspace_all {
args: "publish --unstable-workspaces --token 'sadfasdf'",
output: "publish/workspace.out",
cwd: Some("publish/workspace"),
copy_temp_dir: Some("publish/workspace"),
envs: env_vars_for_registry(),
http_server: true,
temp_cwd: true,
});
itest!(workspace_individual {
args: "publish --unstable-workspaces --token 'sadfasdf'",
output: "publish/workspace_individual.out",
cwd: Some("publish/workspace/bar"),
copy_temp_dir: Some("publish/workspace"),
envs: env_vars_for_registry(), envs: env_vars_for_registry(),
http_server: true, http_server: true,
temp_cwd: true, temp_cwd: true,
@ -43,7 +97,7 @@ fn ignores_directories() {
"name": "@foo/bar", "name": "@foo/bar",
"version": "1.0.0", "version": "1.0.0",
"exclude": [ "ignore" ], "exclude": [ "ignore" ],
"exports": "main_included.ts" "exports": "./main_included.ts"
})); }));
let ignored_dirs = vec![ let ignored_dirs = vec![
@ -68,7 +122,6 @@ fn ignores_directories() {
.arg("--log-level=debug") .arg("--log-level=debug")
.arg("--token") .arg("--token")
.arg("sadfasdf") .arg("sadfasdf")
.arg(temp_dir)
.run(); .run();
output.assert_exit_code(0); output.assert_exit_code(0);
let output = output.combined_output(); let output = output.combined_output();
@ -81,4 +134,5 @@ fn publish_context_builder() -> TestContextBuilder {
TestContextBuilder::new() TestContextBuilder::new()
.use_http_server() .use_http_server()
.envs(env_vars_for_registry()) .envs(env_vars_for_registry())
.use_temp_cwd()
} }

View file

@ -0,0 +1,17 @@
// add some statements that will be removed by the subset
// type graph so that we can test that the source map works
console.log(1);
console.log(2);
console.log(3);
export class Foo {
method(): number {
return Math.random();
}
}
// this won't be type checked against because the subset
// type graph omit this code because it's not part of the
// public API.
const invalidTypeCheck: number = "";
console.log(invalidTypeCheck);

View file

@ -0,0 +1,5 @@
{
"exports": {
".": "./mod.ts"
}
}

View file

@ -0,0 +1,5 @@
{
"versions": {
"0.1.0": {}
}
}

View file

@ -0,0 +1,12 @@
export class Foo {
method() {
return Math.random();
}
}
// This will be analyzed because the method above is missing an
// explicit type which is required for the subset type graph to take
// effect. So the entire source file will be type checked against,
// causing a type error here.
const invalidTypeCheck: number = "";
console.log(invalidTypeCheck);

View file

@ -0,0 +1,5 @@
{
"exports": {
".": "./mod.ts"
}
}

View file

@ -0,0 +1,5 @@
{
"versions": {
"0.1.0": {}
}
}

View file

@ -0,0 +1,45 @@
Download http://localhost:4545/jsr/registry/@denotest/subset_type_graph/meta.json
Download http://localhost:4545/jsr/registry/@denotest/subset_type_graph_invalid/meta.json
Download http://localhost:4545/jsr/registry/@denotest/subset_type_graph/0.1.0_meta.json
Download http://localhost:4545/jsr/registry/@denotest/subset_type_graph_invalid/0.1.0_meta.json
[UNORDERED_START]
Download http://localhost:4545/jsr/registry/@denotest/subset_type_graph/0.1.0/mod.ts
Download http://localhost:4545/jsr/registry/@denotest/subset_type_graph_invalid/0.1.0/mod.ts
[UNORDERED_END]
Check file:///[WILDCARD]/subset_type_graph/main.ts
error: TS2322 [ERROR]: Type 'string' is not assignable to type 'number'.
const invalidTypeCheck: number = "";
~~~~~~~~~~~~~~~~
at http://localhost:4545/jsr/registry/@denotest/subset_type_graph_invalid/0.1.0/mod.ts:11:7
TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
const error1: string = new Foo1().method();
~~~~~~
at file:///[WILDCARD]/subset_type_graph/main.ts:5:7
TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
const error2: string = new Foo2().method();
~~~~~~
at file:///[WILDCARD]/subset_type_graph/main.ts:6:7
TS2551 [ERROR]: Property 'method2' does not exist on type 'Foo'. Did you mean 'method'?
new Foo1().method2();
~~~~~~~
at file:///[WILDCARD]/subset_type_graph/main.ts:12:12
'method' is declared here.
method(): number {
~~~~~~
at http://localhost:4545/jsr/registry/@denotest/subset_type_graph/0.1.0/mod.ts:8:3
TS2551 [ERROR]: Property 'method2' does not exist on type 'Foo'. Did you mean 'method'?
new Foo2().method2();
~~~~~~~
at file:///[WILDCARD]/subset_type_graph/main.ts:13:12
'method' is declared here.
method() {
~~~~~~
at http://localhost:4545/jsr/registry/@denotest/subset_type_graph_invalid/0.1.0/mod.ts:2:3
Found 5 errors.

View file

@ -0,0 +1,13 @@
import { Foo as Foo1 } from "jsr:@denotest/subset_type_graph@0.1.0";
import { Foo as Foo2 } from "jsr:@denotest/subset_type_graph_invalid@0.1.0";
// these will both raise type checking errors
const error1: string = new Foo1().method();
const error2: string = new Foo2().method();
console.log(error1);
console.log(error2);
// now raise some errors that will show the original code and
// these should source map to the original
new Foo1().method2();
new Foo2().method2();

View file

@ -0,0 +1,8 @@
Checking fast check type graph for errors...
Missing explicit return type in the public API.
at file:///[WILDCARD]/publish/invalid_fast_check/mod.ts:2:17
Fixing these fast check errors is required to make the code fast check compatible which enables type checking your package's TypeScript code with the same performance as if you had distributed declaration files. Do any of these errors seem too restrictive or incorrect? Please open an issue if so to help us improve: https://github.com/denoland/deno/issues
error: Had 1 fast check error.

View file

@ -0,0 +1,7 @@
{
"name": "@foo/bar",
"version": "1.1.0",
"exports": {
".": "./mod.ts"
}
}

View file

@ -0,0 +1,4 @@
// requires an explicit type annotation of `number`
export function getRandom() {
return Math.random();
}

View file

@ -0,0 +1,6 @@
Checking fast check type graph for errors...
Ensuring type checks...
Check file:///[WILDCARD]/javascript_decl_file/mod.js
Publishing @foo/bar@1.0.0 ...
Successfully published @foo/bar@1.0.0
Visit http://127.0.0.1:4250/@foo/bar@1.0.0 for details

View file

@ -0,0 +1,7 @@
{
"name": "@foo/bar",
"version": "1.0.0",
"exports": {
".": "./mod.js"
}
}

View file

@ -0,0 +1 @@
export function getRandom(): number;

View file

@ -0,0 +1,5 @@
/// <reference types="./mod.d.ts" />
export function getRandom() {
return Math.random();
}

View file

@ -0,0 +1,6 @@
Checking fast check type graph for errors...
Warning Package '@foo/bar' is a JavaScript package without a corresponding declaration file. This may lead to a non-optimal experience for users of your package. For performance reasons, it's recommended to ship a corresponding TypeScript declaration file or to convert to TypeScript.
Ensuring type checks...
Publishing @foo/bar@1.0.0 ...
Successfully published @foo/bar@1.0.0
Visit http://127.0.0.1:4250/@foo/bar@1.0.0 for details

View file

@ -0,0 +1,8 @@
{
"name": "@foo/bar",
"version": "1.0.0",
"exports": {
".": "./mod.js",
"./other": "./other.js"
}
}

View file

@ -0,0 +1,3 @@
export function getRandom() {
return Math.random();
}

View file

@ -0,0 +1,3 @@
export function other() {
return Math.random();
}

View file

@ -0,0 +1,3 @@
export function add(a: number, b: number): number {
return a + b;
}

View file

@ -1,3 +1,6 @@
Checking fast check type graph for errors...
Ensuring type checks...
Check file:///[WILDCARD]/publish/successful/mod.ts
Publishing @foo/bar@1.0.0 ... Publishing @foo/bar@1.0.0 ...
Successfully published @foo/bar@1.0.0 Successfully published @foo/bar@1.0.0
Visit http://127.0.0.1:4250/@foo/bar@1.0.0 for details Visit http://127.0.0.1:4250/@foo/bar@1.0.0 for details

View file

@ -5,6 +5,6 @@
".": "./mod.ts" ".": "./mod.ts"
}, },
"imports": { "imports": {
"@std/http": "jsr:@std/http@1" "@std/http": "./std_http.ts"
} }
} }

View file

@ -1,5 +1,5 @@
import http from "@std/http"; import http from "@std/http";
export function foobar() { export function foobar(): { fileServer(): void } {
return http.fileServer; return http.fileServer;
} }

View file

@ -0,0 +1,6 @@
// temp until we get jsr:@std/http in the test server
export default {
fileServer() {
console.log("Hi");
},
};

View file

@ -0,0 +1,11 @@
Publishing a workspace...
Checking fast check type graph for errors...
Ensuring type checks...
Check file:///[WILDCARD]/workspace/foo/mod.ts
Check file:///[WILDCARD]/workspace/bar/mod.ts
Publishing @foo/bar@1.0.0 ...
Successfully published @foo/bar@1.0.0
Visit http://127.0.0.1:4250/@foo/bar@1.0.0 for details
Publishing @foo/foo@1.0.0 ...
Successfully published @foo/foo@1.0.0
Visit http://127.0.0.1:4250/@foo/foo@1.0.0 for details

View file

@ -0,0 +1,7 @@
{
"name": "@foo/bar",
"version": "1.0.0",
"exports": {
".": "./mod.ts"
}
}

View file

@ -0,0 +1,3 @@
export function add(a: number, b: number): number {
return a + b;
}

View file

@ -0,0 +1,6 @@
{
"workspaces": [
"foo",
"bar"
]
}

View file

@ -0,0 +1,10 @@
{
"name": "@foo/foo",
"version": "1.0.0",
"exports": {
".": "./mod.ts"
},
"imports": {
"bar": "jsr:@foo/bar@1"
}
}

View file

@ -0,0 +1,5 @@
import * as bar from "bar";
export function add(a: number, b: number): number {
return bar.add(a, b);
}

View file

@ -0,0 +1,6 @@
Checking fast check type graph for errors...
Ensuring type checks...
Check file:///[WILDCARD]/workspace/bar/mod.ts
Publishing @foo/bar@1.0.0 ...
Successfully published @foo/bar@1.0.0
Visit http://127.0.0.1:4250/@foo/bar@1.0.0 for details

View file

@ -1,6 +1,7 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashSet; use std::collections::HashSet;
use std::collections::VecDeque;
use std::sync::Arc; use std::sync::Arc;
use deno_ast::MediaType; use deno_ast::MediaType;
@ -23,6 +24,7 @@ use crate::cache::FastInsecureHasher;
use crate::cache::TypeCheckCache; use crate::cache::TypeCheckCache;
use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolver;
use crate::tsc; use crate::tsc;
use crate::tsc::Diagnostics;
use crate::version; use crate::version;
/// Options for performing a check of a module graph. Note that the decision to /// Options for performing a check of a module graph. Note that the decision to
@ -68,6 +70,23 @@ impl TypeChecker {
graph: Arc<ModuleGraph>, graph: Arc<ModuleGraph>,
options: CheckOptions, options: CheckOptions,
) -> Result<(), AnyError> { ) -> Result<(), AnyError> {
let diagnostics = self.check_diagnostics(graph, options).await?;
if diagnostics.is_empty() {
Ok(())
} else {
Err(diagnostics.into())
}
}
/// Type check the module graph returning its diagnostics.
///
/// It is expected that it is determined if a check and/or emit is validated
/// before the function is called.
pub async fn check_diagnostics(
&self,
graph: Arc<ModuleGraph>,
options: CheckOptions,
) -> Result<Diagnostics, AnyError> {
// node built-in specifiers use the @types/node package to determine // node built-in specifiers use the @types/node package to determine
// types, so inject that now (the caller should do this after the lockfile // types, so inject that now (the caller should do this after the lockfile
// has been written) // has been written)
@ -100,7 +119,7 @@ impl TypeChecker {
type_check_mode, type_check_mode,
&ts_config, &ts_config,
) { ) {
CheckHashResult::NoFiles => return Ok(()), CheckHashResult::NoFiles => return Ok(Default::default()),
CheckHashResult::Hash(hash) => Some(hash), CheckHashResult::Hash(hash) => Some(hash),
} }
} }
@ -111,7 +130,7 @@ impl TypeChecker {
if !options.reload { if !options.reload {
if let Some(check_hash) = maybe_check_hash { if let Some(check_hash) = maybe_check_hash {
if cache.has_check_hash(check_hash) { if cache.has_check_hash(check_hash) {
return Ok(()); return Ok(Default::default());
} }
} }
} }
@ -152,7 +171,7 @@ impl TypeChecker {
check_mode: type_check_mode, check_mode: type_check_mode,
})?; })?;
let diagnostics = if type_check_mode == TypeCheckMode::Local { let mut diagnostics = if type_check_mode == TypeCheckMode::Local {
response.diagnostics.filter(|d| { response.diagnostics.filter(|d| {
if let Some(file_name) = &d.file_name { if let Some(file_name) = &d.file_name {
if !file_name.starts_with("http") { if !file_name.starts_with("http") {
@ -175,6 +194,8 @@ impl TypeChecker {
response.diagnostics response.diagnostics
}; };
diagnostics.apply_fast_check_source_maps(&graph);
if let Some(tsbuildinfo) = response.maybe_tsbuildinfo { if let Some(tsbuildinfo) = response.maybe_tsbuildinfo {
cache.set_tsbuildinfo(&graph.roots[0], &tsbuildinfo); cache.set_tsbuildinfo(&graph.roots[0], &tsbuildinfo);
} }
@ -187,11 +208,7 @@ impl TypeChecker {
log::debug!("{}", response.stats); log::debug!("{}", response.stats);
if diagnostics.is_empty() { Ok(diagnostics)
Ok(())
} else {
Err(diagnostics.into())
}
} }
} }
@ -256,7 +273,12 @@ fn get_check_hash(
} }
hasher.write_str(module.specifier.as_str()); hasher.write_str(module.specifier.as_str());
hasher.write_str(&module.source); hasher.write_str(
module
.fast_check_module()
.map(|s| s.source.as_ref())
.unwrap_or(&module.source),
);
} }
Module::Node(_) => { Module::Node(_) => {
// the @types/node package will be in the resolved // the @types/node package will be in the resolved
@ -345,21 +367,18 @@ fn get_tsc_roots(
)); ));
} }
let mut seen_roots = let mut seen =
HashSet::with_capacity(graph.imports.len() + graph.roots.len()); HashSet::with_capacity(graph.imports.len() + graph.specifiers_count());
let mut pending = VecDeque::new();
// put in the global types first so that they're resolved before anything else // put in the global types first so that they're resolved before anything else
for import in graph.imports.values() { for import in graph.imports.values() {
for dep in import.dependencies.values() { for dep in import.dependencies.values() {
let specifier = dep.get_type().or_else(|| dep.get_code()); let specifier = dep.get_type().or_else(|| dep.get_code());
if let Some(specifier) = &specifier { if let Some(specifier) = &specifier {
if seen_roots.insert(*specifier) { let specifier = graph.resolve(specifier);
let maybe_entry = graph if seen.insert(specifier.clone()) {
.get(specifier) pending.push_back(specifier);
.and_then(|m| maybe_get_check_entry(m, check_js));
if let Some(entry) = maybe_entry {
result.push(entry);
}
} }
} }
} }
@ -367,24 +386,51 @@ fn get_tsc_roots(
// then the roots // then the roots
for root in &graph.roots { for root in &graph.roots {
if let Some(module) = graph.get(root) { let specifier = graph.resolve(root);
if seen_roots.insert(root) { if seen.insert(specifier.clone()) {
if let Some(entry) = maybe_get_check_entry(module, check_js) { pending.push_back(specifier);
result.push(entry); }
}
// now walk the graph that only includes the fast check dependencies
while let Some(specifier) = pending.pop_front() {
let Some(module) = graph.get(&specifier) else {
continue;
};
if let Some(entry) = maybe_get_check_entry(module, check_js) {
result.push(entry);
}
if let Some(module) = module.esm() {
let deps = module.dependencies_prefer_fast_check();
for dep in deps.values() {
// walk both the code and type dependencies
if let Some(specifier) = dep.get_code() {
let specifier = graph.resolve(specifier);
if seen.insert(specifier.clone()) {
pending.push_back(specifier);
}
}
if let Some(specifier) = dep.get_type() {
let specifier = graph.resolve(specifier);
if seen.insert(specifier.clone()) {
pending.push_back(specifier);
}
}
}
if let Some(dep) = module
.maybe_types_dependency
.as_ref()
.and_then(|d| d.dependency.ok())
{
let specifier = graph.resolve(&dep.specifier);
if seen.insert(specifier.clone()) {
pending.push_back(specifier);
} }
} }
} }
} }
// now the rest
result.extend(graph.modules().filter_map(|module| {
if seen_roots.contains(module.specifier()) {
None
} else {
maybe_get_check_entry(module, check_js)
}
}));
result result
} }

154
cli/tools/registry/graph.rs Normal file
View file

@ -0,0 +1,154 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashSet;
use std::collections::VecDeque;
use deno_ast::ModuleSpecifier;
use deno_config::ConfigFile;
use deno_config::WorkspaceConfig;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_graph::FastCheckDiagnostic;
use deno_graph::ModuleGraph;
#[derive(Debug)]
pub struct MemberRoots {
pub name: String,
pub dir_url: ModuleSpecifier,
pub exports: Vec<ModuleSpecifier>,
}
pub fn get_workspace_member_roots(
config: &WorkspaceConfig,
) -> Result<Vec<MemberRoots>, AnyError> {
let mut members = Vec::with_capacity(config.members.len());
let mut seen_names = HashSet::with_capacity(config.members.len());
for member in &config.members {
if !seen_names.insert(&member.package_name) {
bail!(
"Cannot have two workspace packages with the same name ('{}' at {})",
member.package_name,
member.path.display(),
);
}
members.push(MemberRoots {
name: member.package_name.clone(),
dir_url: member.config_file.specifier.join("./").unwrap().clone(),
exports: resolve_config_file_roots_from_exports(&member.config_file)?,
});
}
Ok(members)
}
pub fn resolve_config_file_roots_from_exports(
config_file: &ConfigFile,
) -> Result<Vec<ModuleSpecifier>, AnyError> {
let exports_config = config_file
.to_exports_config()
.with_context(|| {
format!("Failed to parse exports at {}", config_file.specifier)
})?
.into_map();
let mut exports = Vec::with_capacity(exports_config.len());
for (_, value) in exports_config {
let entry_point =
config_file.specifier.join(&value).with_context(|| {
format!("Failed to join {} with {}", config_file.specifier, value)
})?;
exports.push(entry_point);
}
Ok(exports)
}
pub fn surface_fast_check_type_graph_errors(
graph: &ModuleGraph,
packages: &[MemberRoots],
) -> Result<(), AnyError> {
let mut diagnostic_count = 0;
let mut seen_diagnostics = HashSet::new();
let mut seen_modules = HashSet::with_capacity(graph.specifiers_count());
for package in packages {
let mut pending = VecDeque::new();
for export in &package.exports {
if seen_modules.insert(export.clone()) {
pending.push_back(export.clone());
}
}
'analyze_package: while let Some(specifier) = pending.pop_front() {
let Ok(Some(module)) = graph.try_get_prefer_types(&specifier) else {
continue;
};
let Some(esm_module) = module.esm() else {
continue;
};
if let Some(diagnostic) = esm_module.fast_check_diagnostic() {
for diagnostic in diagnostic.flatten_multiple() {
if matches!(
diagnostic,
FastCheckDiagnostic::UnsupportedJavaScriptEntrypoint { .. }
) {
// ignore JS packages for fast check
log::warn!(
concat!(
"{} Package '{}' is a JavaScript package without a corresponding ",
"declaration file. This may lead to a non-optimal experience for ",
"users of your package. For performance reasons, it's recommended ",
"to ship a corresponding TypeScript declaration file or to ",
"convert to TypeScript.",
),
deno_runtime::colors::yellow("Warning"),
package.name,
);
break 'analyze_package; // no need to keep analyzing this package
} else {
let message = diagnostic.message_with_range();
if !seen_diagnostics.insert(message.clone()) {
continue;
}
log::error!("\n{}", message);
diagnostic_count += 1;
}
}
}
// analyze the next dependencies
for dep in esm_module.dependencies_prefer_fast_check().values() {
let Some(specifier) = graph.resolve_dependency_from_dep(dep, true)
else {
continue;
};
let dep_in_same_package =
specifier.as_str().starts_with(package.dir_url.as_str());
if dep_in_same_package {
let is_new = seen_modules.insert(specifier.clone());
if is_new {
pending.push_back(specifier.clone());
}
}
}
}
}
if diagnostic_count > 0 {
// for the time being, tell the user why we have these errors and the benefit they bring
log::error!(
concat!(
"\nFixing these fast check errors is required to make the code fast check compatible ",
"which enables type checking your package's TypeScript code with the same ",
"performance as if you had distributed declaration files. Do any of these ",
"errors seem too restrictive or incorrect? Please open an issue if so to ",
"help us improve: https://github.com/denoland/deno/issues\n",
)
);
bail!(
"Had {} fast check error{}.",
diagnostic_count,
if diagnostic_count == 1 { "" } else { "s" }
)
}
Ok(())
}

View file

@ -24,16 +24,24 @@ use sha2::Digest;
use crate::args::deno_registry_api_url; use crate::args::deno_registry_api_url;
use crate::args::deno_registry_url; use crate::args::deno_registry_url;
use crate::args::CliOptions;
use crate::args::Flags; use crate::args::Flags;
use crate::args::PublishFlags; use crate::args::PublishFlags;
use crate::factory::CliFactory; use crate::factory::CliFactory;
use crate::graph_util::ModuleGraphBuilder;
use crate::http_util::HttpClient; use crate::http_util::HttpClient;
use crate::tools::check::CheckOptions;
use crate::tools::registry::graph::get_workspace_member_roots;
use crate::tools::registry::graph::resolve_config_file_roots_from_exports;
use crate::tools::registry::graph::surface_fast_check_type_graph_errors;
use crate::tools::registry::graph::MemberRoots;
use crate::util::display::human_size; use crate::util::display::human_size;
use crate::util::glob::PathOrPatternSet; use crate::util::glob::PathOrPatternSet;
use crate::util::import_map::ImportMapUnfurler; use crate::util::import_map::ImportMapUnfurler;
mod api; mod api;
mod auth; mod auth;
mod graph;
mod publish_order; mod publish_order;
mod tar; mod tar;
@ -41,6 +49,8 @@ use auth::get_auth_method;
use auth::AuthMethod; use auth::AuthMethod;
use publish_order::PublishOrderGraph; use publish_order::PublishOrderGraph;
use super::check::TypeChecker;
use self::tar::PublishableTarball; use self::tar::PublishableTarball;
fn ring_bell() { fn ring_bell() {
@ -64,6 +74,15 @@ impl PreparedPublishPackage {
static SUGGESTED_ENTRYPOINTS: [&str; 4] = static SUGGESTED_ENTRYPOINTS: [&str; 4] =
["mod.ts", "mod.js", "index.ts", "index.js"]; ["mod.ts", "mod.js", "index.ts", "index.js"];
fn get_deno_json_package_name(
deno_json: &ConfigFile,
) -> Result<String, AnyError> {
match deno_json.json.name.clone() {
Some(name) => Ok(name),
None => bail!("{} is missing 'name' field", deno_json.specifier),
}
}
async fn prepare_publish( async fn prepare_publish(
deno_json: &ConfigFile, deno_json: &ConfigFile,
import_map: Arc<ImportMap>, import_map: Arc<ImportMap>,
@ -73,9 +92,7 @@ async fn prepare_publish(
let Some(version) = deno_json.json.version.clone() else { let Some(version) = deno_json.json.version.clone() else {
bail!("{} is missing 'version' field", deno_json.specifier); bail!("{} is missing 'version' field", deno_json.specifier);
}; };
let Some(name) = deno_json.json.name.clone() else { let name = get_deno_json_package_name(deno_json)?;
bail!("{} is missing 'name' field", deno_json.specifier);
};
if deno_json.json.exports.is_none() { if deno_json.json.exports.is_none() {
let mut suggested_entrypoint = None; let mut suggested_entrypoint = None;
@ -122,6 +139,8 @@ async fn prepare_publish(
}) })
.await??; .await??;
log::debug!("Tarball size ({}): {}", name, tarball.bytes.len());
Ok(Rc::new(PreparedPublishPackage { Ok(Rc::new(PreparedPublishPackage {
scope: scope.to_string(), scope: scope.to_string(),
package: package_name.to_string(), package: package_name.to_string(),
@ -665,11 +684,26 @@ async fn prepare_packages_for_publishing(
AnyError, AnyError,
> { > {
let maybe_workspace_config = deno_json.to_workspace_config()?; let maybe_workspace_config = deno_json.to_workspace_config()?;
let module_graph_builder = cli_factory.module_graph_builder().await?.as_ref();
let type_checker = cli_factory.type_checker().await?;
let cli_options = cli_factory.cli_options();
let Some(workspace_config) = maybe_workspace_config else { let Some(workspace_config) = maybe_workspace_config else {
let roots = resolve_config_file_roots_from_exports(&deno_json)?;
build_and_check_graph_for_publish(
module_graph_builder,
type_checker,
cli_options,
&[MemberRoots {
name: get_deno_json_package_name(&deno_json)?,
dir_url: deno_json.specifier.join("./").unwrap().clone(),
exports: roots,
}],
)
.await?;
let mut prepared_package_by_name = HashMap::with_capacity(1); let mut prepared_package_by_name = HashMap::with_capacity(1);
let package = prepare_publish(&deno_json, import_map).await?; let package = prepare_publish(&deno_json, import_map).await?;
let package_name = package.package.clone(); let package_name = format!("@{}/{}", package.scope, package.package);
let publish_order_graph = let publish_order_graph =
PublishOrderGraph::new_single(package_name.clone()); PublishOrderGraph::new_single(package_name.clone());
prepared_package_by_name.insert(package_name, package); prepared_package_by_name.insert(package_name, package);
@ -677,14 +711,21 @@ async fn prepare_packages_for_publishing(
}; };
println!("Publishing a workspace..."); println!("Publishing a workspace...");
let mut prepared_package_by_name = // create the module graph
HashMap::with_capacity(workspace_config.members.len()); let roots = get_workspace_member_roots(&workspace_config)?;
let publish_order_graph = publish_order::build_publish_graph( let graph = build_and_check_graph_for_publish(
&workspace_config, module_graph_builder,
cli_factory.module_graph_builder().await?.as_ref(), type_checker,
cli_options,
&roots,
) )
.await?; .await?;
let mut prepared_package_by_name =
HashMap::with_capacity(workspace_config.members.len());
let publish_order_graph =
publish_order::build_publish_order_graph(&graph, &roots)?;
let results = let results =
workspace_config workspace_config
.members .members
@ -712,6 +753,55 @@ async fn prepare_packages_for_publishing(
Ok((publish_order_graph, prepared_package_by_name)) Ok((publish_order_graph, prepared_package_by_name))
} }
async fn build_and_check_graph_for_publish(
module_graph_builder: &ModuleGraphBuilder,
type_checker: &TypeChecker,
cli_options: &CliOptions,
packages: &[MemberRoots],
) -> Result<Arc<deno_graph::ModuleGraph>, deno_core::anyhow::Error> {
let graph = Arc::new(
module_graph_builder
.create_graph_with_options(crate::graph_util::CreateGraphOptions {
// All because we're going to use this same graph to determine the publish order later
graph_kind: deno_graph::GraphKind::All,
roots: packages
.iter()
.flat_map(|r| r.exports.iter())
.cloned()
.collect(),
workspace_fast_check: true,
loader: None,
})
.await?,
);
graph.valid()?;
log::info!("Checking fast check type graph for errors...");
surface_fast_check_type_graph_errors(&graph, packages)?;
log::info!("Ensuring type checks...");
let diagnostics = type_checker
.check_diagnostics(
graph.clone(),
CheckOptions {
lib: cli_options.ts_type_lib_window(),
log_ignored_options: false,
reload: cli_options.reload_flag(),
},
)
.await?;
if !diagnostics.is_empty() {
bail!(
concat!(
"{:#}\n\n",
"You may have discovered a bug in Deno's fast check implementation. ",
"Fast check is still early days and we would appreciate if you log a ",
"bug if you believe this is one: https://github.com/denoland/deno/issues/"
),
diagnostics
);
}
Ok(graph)
}
pub async fn publish( pub async fn publish(
flags: Flags, flags: Flags,
publish_flags: PublishFlags, publish_flags: PublishFlags,
@ -728,10 +818,7 @@ pub async fn publish(
Arc::new(ImportMap::new(Url::parse("file:///dev/null").unwrap())) Arc::new(ImportMap::new(Url::parse("file:///dev/null").unwrap()))
}); });
let initial_cwd = let directory_path = cli_factory.cli_options().initial_cwd();
std::env::current_dir().with_context(|| "Failed getting cwd.")?;
let directory_path = initial_cwd.join(publish_flags.directory);
// TODO: doesn't handle jsonc // TODO: doesn't handle jsonc
let deno_json_path = directory_path.join("deno.json"); let deno_json_path = directory_path.join("deno.json");
let deno_json = ConfigFile::read(&deno_json_path).with_context(|| { let deno_json = ConfigFile::read(&deno_json_path).with_context(|| {

View file

@ -4,13 +4,11 @@ use std::collections::HashMap;
use std::collections::HashSet; use std::collections::HashSet;
use std::collections::VecDeque; use std::collections::VecDeque;
use deno_ast::ModuleSpecifier;
use deno_config::WorkspaceConfig;
use deno_core::anyhow::bail; use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError; use deno_core::error::AnyError;
use deno_graph::ModuleGraph;
use crate::graph_util::ModuleGraphBuilder; use super::graph::MemberRoots;
pub struct PublishOrderGraph { pub struct PublishOrderGraph {
packages: HashMap<String, HashSet<String>>, packages: HashMap<String, HashSet<String>>,
@ -122,80 +120,21 @@ impl PublishOrderGraph {
} }
} }
pub async fn build_publish_graph( pub fn build_publish_order_graph(
workspace_config: &WorkspaceConfig, graph: &ModuleGraph,
module_graph_builder: &ModuleGraphBuilder, roots: &[MemberRoots],
) -> Result<PublishOrderGraph, AnyError> { ) -> Result<PublishOrderGraph, AnyError> {
let roots = get_workspace_roots(workspace_config)?;
let graph = module_graph_builder
.create_graph(
deno_graph::GraphKind::All,
roots.iter().flat_map(|r| r.exports.clone()).collect(),
)
.await?;
graph.valid()?;
let packages = build_pkg_deps(graph, roots); let packages = build_pkg_deps(graph, roots);
Ok(build_graph(packages)) Ok(build_publish_order_graph_from_pkgs_deps(packages))
}
#[derive(Debug)]
struct MemberRoots {
name: String,
dir_url: ModuleSpecifier,
exports: Vec<ModuleSpecifier>,
}
fn get_workspace_roots(
config: &WorkspaceConfig,
) -> Result<Vec<MemberRoots>, AnyError> {
let mut members = Vec::with_capacity(config.members.len());
let mut seen_names = HashSet::with_capacity(config.members.len());
for member in &config.members {
let exports_config = member
.config_file
.to_exports_config()
.with_context(|| {
format!(
"Failed to parse exports at {}",
member.config_file.specifier
)
})?
.into_map();
if !seen_names.insert(&member.package_name) {
bail!(
"Cannot have two workspace packages with the same name ('{}' at {})",
member.package_name,
member.path.display(),
);
}
let mut member_root = MemberRoots {
name: member.package_name.clone(),
dir_url: member.config_file.specifier.join("./").unwrap().clone(),
exports: Vec::with_capacity(exports_config.len()),
};
for (_, value) in exports_config {
let entry_point =
member.config_file.specifier.join(&value).with_context(|| {
format!(
"Failed to join {} with {}",
member.config_file.specifier, value
)
})?;
member_root.exports.push(entry_point);
}
members.push(member_root);
}
Ok(members)
} }
fn build_pkg_deps( fn build_pkg_deps(
graph: deno_graph::ModuleGraph, graph: &deno_graph::ModuleGraph,
roots: Vec<MemberRoots>, roots: &[MemberRoots],
) -> HashMap<String, HashSet<String>> { ) -> HashMap<String, HashSet<String>> {
let mut members = HashMap::with_capacity(roots.len()); let mut members = HashMap::with_capacity(roots.len());
let mut seen_modules = HashSet::with_capacity(graph.modules().count()); let mut seen_modules = HashSet::with_capacity(graph.modules().count());
for root in &roots { for root in roots {
let mut deps = HashSet::new(); let mut deps = HashSet::new();
let mut pending = VecDeque::new(); let mut pending = VecDeque::new();
pending.extend(root.exports.clone()); pending.extend(root.exports.clone());
@ -243,7 +182,7 @@ fn build_pkg_deps(
members members
} }
fn build_graph( fn build_publish_order_graph_from_pkgs_deps(
packages: HashMap<String, HashSet<String>>, packages: HashMap<String, HashSet<String>>,
) -> PublishOrderGraph { ) -> PublishOrderGraph {
let mut in_degree = HashMap::new(); let mut in_degree = HashMap::new();
@ -273,7 +212,7 @@ mod test {
#[test] #[test]
fn test_graph_no_deps() { fn test_graph_no_deps() {
let mut graph = build_graph(HashMap::from([ let mut graph = build_publish_order_graph_from_pkgs_deps(HashMap::from([
("a".to_string(), HashSet::new()), ("a".to_string(), HashSet::new()),
("b".to_string(), HashSet::new()), ("b".to_string(), HashSet::new()),
("c".to_string(), HashSet::new()), ("c".to_string(), HashSet::new()),
@ -293,7 +232,7 @@ mod test {
#[test] #[test]
fn test_graph_single_dep() { fn test_graph_single_dep() {
let mut graph = build_graph(HashMap::from([ let mut graph = build_publish_order_graph_from_pkgs_deps(HashMap::from([
("a".to_string(), HashSet::from(["b".to_string()])), ("a".to_string(), HashSet::from(["b".to_string()])),
("b".to_string(), HashSet::from(["c".to_string()])), ("b".to_string(), HashSet::from(["c".to_string()])),
("c".to_string(), HashSet::new()), ("c".to_string(), HashSet::new()),
@ -310,7 +249,7 @@ mod test {
#[test] #[test]
fn test_graph_multiple_dep() { fn test_graph_multiple_dep() {
let mut graph = build_graph(HashMap::from([ let mut graph = build_publish_order_graph_from_pkgs_deps(HashMap::from([
( (
"a".to_string(), "a".to_string(),
HashSet::from(["b".to_string(), "c".to_string()]), HashSet::from(["b".to_string(), "c".to_string()]),
@ -342,7 +281,7 @@ mod test {
#[test] #[test]
fn test_graph_circular_dep() { fn test_graph_circular_dep() {
let mut graph = build_graph(HashMap::from([ let mut graph = build_publish_order_graph_from_pkgs_deps(HashMap::from([
("a".to_string(), HashSet::from(["b".to_string()])), ("a".to_string(), HashSet::from(["b".to_string()])),
("b".to_string(), HashSet::from(["c".to_string()])), ("b".to_string(), HashSet::from(["c".to_string()])),
("c".to_string(), HashSet::from(["a".to_string()])), ("c".to_string(), HashSet::from(["a".to_string()])),

View file

@ -1,11 +1,14 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_ast::ModuleSpecifier;
use deno_graph::ModuleGraph;
use deno_runtime::colors; use deno_runtime::colors;
use deno_core::serde::Deserialize; use deno_core::serde::Deserialize;
use deno_core::serde::Deserializer; use deno_core::serde::Deserializer;
use deno_core::serde::Serialize; use deno_core::serde::Serialize;
use deno_core::serde::Serializer; use deno_core::serde::Serializer;
use deno_core::sourcemap::SourceMap;
use std::error::Error; use std::error::Error;
use std::fmt; use std::fmt;
@ -101,7 +104,9 @@ impl DiagnosticMessageChain {
#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)] #[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct Position { pub struct Position {
/// 0-indexed line number
pub line: u64, pub line: u64,
/// 0-indexed character number
pub character: u64, pub character: u64,
} }
@ -112,6 +117,13 @@ pub struct Diagnostic {
pub code: u64, pub code: u64,
pub start: Option<Position>, pub start: Option<Position>,
pub end: Option<Position>, pub end: Option<Position>,
/// Position of this diagnostic in the original non-mapped source.
///
/// This will exist and be different from the `start` for fast
/// checked modules where the TypeScript source will differ
/// from the original source.
#[serde(skip_serializing)]
pub original_source_start: Option<Position>,
pub message_text: Option<String>, pub message_text: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub message_chain: Option<DiagnosticMessageChain>, pub message_chain: Option<DiagnosticMessageChain>,
@ -145,9 +157,10 @@ impl Diagnostic {
} }
fn fmt_frame(&self, f: &mut fmt::Formatter, level: usize) -> fmt::Result { fn fmt_frame(&self, f: &mut fmt::Formatter, level: usize) -> fmt::Result {
if let (Some(file_name), Some(start)) = if let (Some(file_name), Some(start)) = (
(self.file_name.as_ref(), self.start.as_ref()) self.file_name.as_ref(),
{ self.original_source_start.as_ref().or(self.start.as_ref()),
) {
write!( write!(
f, f,
"\n{:indent$} at {}:{}:{}", "\n{:indent$} at {}:{}:{}",
@ -273,6 +286,51 @@ impl Diagnostics {
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
self.0.is_empty() self.0.is_empty()
} }
/// Modifies all the diagnostics to have their display positions
/// modified to point at the original source.
pub fn apply_fast_check_source_maps(&mut self, graph: &ModuleGraph) {
fn visit_diagnostic(d: &mut Diagnostic, graph: &ModuleGraph) {
if let Some(specifier) = d
.file_name
.as_ref()
.and_then(|n| ModuleSpecifier::parse(n).ok())
{
if let Ok(Some(module)) = graph.try_get_prefer_types(&specifier) {
if let Some(fast_check_module) =
module.esm().and_then(|m| m.fast_check_module())
{
// todo(dsherret): use a short lived cache to prevent parsing
// source maps so often
if let Ok(source_map) =
SourceMap::from_slice(&fast_check_module.source_map)
{
if let Some(start) = d.start.as_mut() {
let maybe_token = source_map
.lookup_token(start.line as u32, start.character as u32);
if let Some(token) = maybe_token {
d.original_source_start = Some(Position {
line: token.get_src_line() as u64,
character: token.get_src_col() as u64,
});
}
}
}
}
}
}
if let Some(related) = &mut d.related_information {
for d in related.iter_mut() {
visit_diagnostic(d, graph);
}
}
}
for d in &mut self.0 {
visit_diagnostic(d, graph);
}
}
} }
impl<'de> Deserialize<'de> for Diagnostics { impl<'de> Deserialize<'de> for Diagnostics {

View file

@ -486,7 +486,11 @@ fn op_load(
match module { match module {
Module::Esm(module) => { Module::Esm(module) => {
media_type = module.media_type; media_type = module.media_type;
Some(Cow::Borrowed(&*module.source)) let source = module
.fast_check_module()
.map(|m| &*m.source)
.unwrap_or(&*module.source);
Some(Cow::Borrowed(source))
} }
Module::Json(module) => { Module::Json(module) => {
media_type = MediaType::Json; media_type = MediaType::Json;
@ -586,7 +590,7 @@ fn op_resolve(
let resolved_dep = graph let resolved_dep = graph
.get(&referrer) .get(&referrer)
.and_then(|m| m.esm()) .and_then(|m| m.esm())
.and_then(|m| m.dependencies.get(&specifier)) .and_then(|m| m.dependencies_prefer_fast_check().get(&specifier))
.and_then(|d| d.maybe_type.ok().or_else(|| d.maybe_code.ok())); .and_then(|d| d.maybe_type.ok().or_else(|| d.maybe_code.ok()));
let maybe_result = match resolved_dep { let maybe_result = match resolved_dep {
@ -1182,6 +1186,7 @@ mod tests {
code: 5023, code: 5023,
start: None, start: None,
end: None, end: None,
original_source_start: None,
message_text: Some( message_text: Some(
"Unknown compiler option \'invalid\'.".to_string() "Unknown compiler option \'invalid\'.".to_string()
), ),