foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-21 15:04:11 -05:00
Code Issues Packages 1 Wiki Activity

refactor(cli): rewrite Deno.transpileOnly() to use SWC (#8090)

Browse source 
Co-authored-by: Kitson Kelly <me@kitsonkelly.com>
This commit is contained in:
Bartek Iwańczuk 2020-10-26 14:03:03 +01:00 committed by GitHub
parent aebbdd5cc2
commit 57cad53945
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 178 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
cli/ast.rs
View file

@ -225,7 +225,7 @@ impl From<tsc_config::TsConfig> for EmitOptions {
EmitOptions {
check_js: options.check_js,
emit_metadata: options.emit_decorator_metadata,
inline_source_map: true,
inline_source_map: options.inline_source_map,
jsx_factory: options.jsx_factory,
jsx_fragment_factory: options.jsx_fragment_factory,
transform_jsx: options.jsx == "react",
@ -356,8 +356,11 @@ impl ParsedModule {
/// - `source` - The source code for the module.
/// - `media_type` - The media type for the module.
///
// NOTE(bartlomieju): `specifier` has `&str` type instead of
// `&ModuleSpecifier` because runtime compiler APIs don't
// require valid module specifiers
pub fn parse(
specifier: &ModuleSpecifier,
specifier: &str,
source: &str,
media_type: &MediaType,
) -> Result<ParsedModule, AnyError> {
@ -505,8 +508,9 @@ mod tests {
let source = r#"import * as bar from "./test.ts";
const foo = await import("./foo.ts");
"#;
let parsed_module = parse(&specifier, source, &MediaType::JavaScript)
.expect("could not parse module");
let parsed_module =
parse(specifier.as_str(), source, &MediaType::JavaScript)
.expect("could not parse module");
let actual = parsed_module.analyze_dependencies();
assert_eq!(
actual,
@ -553,7 +557,7 @@ mod tests {
}
}
"#;
let module = parse(&specifier, source, &MediaType::TypeScript)
let module = parse(specifier.as_str(), source, &MediaType::TypeScript)
.expect("could not parse module");
let (code, maybe_map) = module
.transpile(&EmitOptions::default())
@ -577,7 +581,7 @@ mod tests {
}
}
"#;
let module = parse(&specifier, source, &MediaType::TSX)
let module = parse(specifier.as_str(), source, &MediaType::TSX)
.expect("could not parse module");
let (code, _) = module
.transpile(&EmitOptions::default())
@ -608,7 +612,7 @@ mod tests {
}
}
"#;
let module = parse(&specifier, source, &MediaType::TypeScript)
let module = parse(specifier.as_str(), source, &MediaType::TypeScript)
.expect("could not parse module");
let (code, _) = module
.transpile(&EmitOptions::default())

3
cli/dts/lib.deno.unstable.d.ts vendored
View file

@ -480,8 +480,7 @@ declare namespace Deno {
* source map.
* @param options An option object of options to send to the compiler. This is
* a subset of ts.CompilerOptions which can be supported by Deno.
* Many of the options related to type checking and emitting
* type declaration files will have no impact on the output.
* If unsupported option is passed then the API will throw an error.
*/
export function transpileOnly(
sources: Record<string, string>,

11
cli/module_graph2.rs
View file

@ -320,7 +320,8 @@ impl Module {
/// Parse a module, populating the structure with data retrieved from the
/// source of the module.
pub fn parse(&mut self) -> Result<(), AnyError> {
let parsed_module = parse(&self.specifier, &self.source, &self.media_type)?;
let parsed_module =
parse(self.specifier.as_str(), &self.source, &self.media_type)?;
// parse out any triple slash references
for comment in parsed_module.get_leading_comments().iter() {
@ -639,12 +640,13 @@ impl Graph2 {
let mut ts_config = TsConfig::new(json!({
"checkJs": false,
"emitDecoratorMetadata": false,
"inlineSourceMap": true,
"jsx": "react",
"jsxFactory": "React.createElement",
"jsxFragmentFactory": "React.Fragment",
}));
let maybe_ignored_options =
ts_config.merge_user_config(options.maybe_config_path)?;
ts_config.merge_tsconfig(options.maybe_config_path)?;
let emit_options: EmitOptions = ts_config.into();
let cm = Rc::new(swc_common::SourceMap::new(
swc_common::FilePathMapping::empty(),
@ -730,7 +732,7 @@ impl Graph2 {
}));
}
let maybe_ignored_options =
config.merge_user_config(options.maybe_config_path)?;
config.merge_tsconfig(options.maybe_config_path)?;
// Short circuit if none of the modules require an emit, or all of the
// modules that require an emit have a valid emit. There is also an edge
@ -1187,13 +1189,14 @@ impl Graph2 {
let mut ts_config = TsConfig::new(json!({
"checkJs": false,
"emitDecoratorMetadata": false,
"inlineSourceMap": true,
"jsx": "react",
"jsxFactory": "React.createElement",
"jsxFragmentFactory": "React.Fragment",
}));
let maybe_ignored_options =
ts_config.merge_user_config(options.maybe_config_path)?;
ts_config.merge_tsconfig(options.maybe_config_path)?;
let emit_options: EmitOptions = ts_config.clone().into();

59
cli/ops/runtime_compiler.rs
View file

@ -1,12 +1,17 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
use crate::ast;
use crate::colors;
use crate::media_type::MediaType;
use crate::permissions::Permissions;
use crate::tsc::runtime_bundle;
use crate::tsc::runtime_compile;
use crate::tsc::runtime_transpile;
use crate::tsc_config;
use deno_core::error::AnyError;
use deno_core::futures::FutureExt;
use deno_core::serde::Serialize;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::serde_json::Value;
use deno_core::BufVec;
use deno_core::OpState;
@ -71,16 +76,58 @@ struct TranspileArgs {
options: Option<String>,
}
#[derive(Debug, Serialize)]
struct RuntimeTranspileEmit {
source: String,
map: Option<String>,
}
async fn op_transpile(
state: Rc<RefCell<OpState>>,
args: Value,
_data: BufVec,
) -> Result<Value, AnyError> {
super::check_unstable2(&state, "Deno.transpile");
super::check_unstable2(&state, "Deno.transpileOnly");
let args: TranspileArgs = serde_json::from_value(args)?;
let cli_state = super::global_state2(&state);
let program_state = cli_state.clone();
let result =
runtime_transpile(program_state, &args.sources, &args.options).await?;
let mut compiler_options = tsc_config::TsConfig::new(json!({
"checkJs": true,
"emitDecoratorMetadata": false,
"jsx": "react",
"jsxFactory": "React.createElement",
"jsxFragmentFactory": "React.Fragment",
"inlineSourceMap": false,
}));
let user_options: HashMap<String, Value> = if let Some(options) = args.options
{
serde_json::from_str(&options)?
} else {
HashMap::new()
};
let maybe_ignored_options =
compiler_options.merge_user_config(&user_options)?;
// TODO(@kitsonk) these really should just be passed back to the caller
if let Some(ignored_options) = maybe_ignored_options {
info!("{}: {}", colors::yellow("warning"), ignored_options);
}
let emit_options: ast::EmitOptions = compiler_options.into();
let mut emit_map = HashMap::new();
for (specifier, source) in args.sources {
let media_type = MediaType::from(&specifier);
let parsed_module = ast::parse(&specifier, &source, &media_type)?;
let (source, maybe_source_map) = parsed_module.transpile(&emit_options)?;
emit_map.insert(
specifier.to_string(),
RuntimeTranspileEmit {
source,
map: maybe_source_map,
},
);
}
let result = serde_json::to_value(emit_map)?;
Ok(result)
}

9
cli/tests/compiler_api_test.ts
View file

@ -129,17 +129,16 @@ Deno.test({
async fn() {
const actual = await Deno.transpileOnly(
{
"foo.ts": `export enum Foo { Foo, Bar, Baz };\n`,
"foo.ts": `/** This is JSDoc */\nexport enum Foo { Foo, Bar, Baz };\n`,
},
{
sourceMap: false,
module: "amd",
removeComments: true,
},
);
assert(actual);
assertEquals(Object.keys(actual), ["foo.ts"]);
assert(actual["foo.ts"].source.startsWith("define("));
assert(actual["foo.ts"].map == null);
assert(!actual["foo.ts"].source.includes("This is JSDoc"));
assert(actual["foo.ts"].map);
},
});

39
cli/tsc.rs
View file

@ -725,41 +725,6 @@ pub async fn runtime_bundle(
Ok(serde_json::from_str::<Value>(&json_str).unwrap())
}
/// This function is used by `Deno.transpileOnly()` API.
pub async fn runtime_transpile(
program_state: Arc<ProgramState>,
sources: &HashMap<String, String>,
maybe_options: &Option<String>,
) -> Result<Value, AnyError> {
let user_options = if let Some(options) = maybe_options {
tsc_config::parse_raw_config(options)?
} else {
json!({})
};
let mut compiler_options = json!({
"esModuleInterop": true,
"module": "esnext",
"sourceMap": true,
"scriptComments": true,
"target": "esnext",
});
tsc_config::json_merge(&mut compiler_options, &user_options);
let req_msg = json!({
"type": CompilerRequestType::RuntimeTranspile,
"sources": sources,
"compilerOptions": compiler_options,
})
.to_string();
let json_str =
execute_in_tsc(program_state, req_msg).map_err(extract_js_error)?;
let v = serde_json::from_str::<Value>(&json_str)
.expect("Error decoding JSON string.");
Ok(v)
}
#[derive(Clone, Debug, PartialEq)]
pub struct ImportDesc {
pub specifier: String,
@ -793,7 +758,7 @@ pub fn pre_process_file(
analyze_dynamic_imports: bool,
) -> Result<(Vec<ImportDesc>, Vec<TsReferenceDesc>), AnyError> {
let specifier = ModuleSpecifier::resolve_url_or_path(file_name)?;
let module = parse(&specifier, source_code, &media_type)?;
let module = parse(specifier.as_str(), source_code, &media_type)?;
let dependency_descriptors = module.analyze_dependencies();
@ -894,7 +859,6 @@ fn parse_deno_types(comment: &str) -> Option<String> {
pub enum CompilerRequestType {
RuntimeCompile = 2,
RuntimeBundle = 3,
RuntimeTranspile = 4,
}
impl Serialize for CompilerRequestType {
@ -905,7 +869,6 @@ impl Serialize for CompilerRequestType {
let value: i32 = match self {
CompilerRequestType::RuntimeCompile => 2 as i32,
CompilerRequestType::RuntimeBundle => 3 as i32,
CompilerRequestType::RuntimeTranspile => 4 as i32,
};
Serialize::serialize(&value, serializer)
}

31
cli/tsc/99_main_compiler.js
View file

@ -599,7 +599,6 @@ delete Object.prototype.__proto__;
const CompilerRequestType = {
RuntimeCompile: 2,
RuntimeBundle: 3,
RuntimeTranspile: 4,
};
function createBundleWriteFile(state) {
@ -999,31 +998,6 @@ delete Object.prototype.__proto__;
};
}
function runtimeTranspile(request) {
const result = {};
const { sources, compilerOptions } = request;
const parseResult = parseCompilerOptions(
compilerOptions,
);
const options = parseResult.options;
// TODO(bartlomieju): this options is excluded by `ts.convertCompilerOptionsFromJson`
// however stuff breaks if it's not passed (type_directives_js_main.js, compiler_js_error.ts)
options.allowNonTsExtensions = true;
for (const [fileName, inputText] of Object.entries(sources)) {
const { outputText: source, sourceMapText: map } = ts.transpileModule(
inputText,
{
fileName,
compilerOptions: options,
},
);
result[fileName] = { source, map };
}
return result;
}
function opCompilerRespond(msg) {
core.jsonOpSync("op_compiler_respond", msg);
}
@ -1041,11 +1015,6 @@ delete Object.prototype.__proto__;
opCompilerRespond(result);
break;
}
case CompilerRequestType.RuntimeTranspile: {
const result = runtimeTranspile(request);
opCompilerRespond(result);
break;
}
default:
throw new Error(
`!!! unhandled CompilerRequestType: ${request.type} (${

139
cli/tsc_config.rs
View file

@ -2,6 +2,7 @@
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::serde_json::Value;
use jsonc_parser::JsonValue;
use serde::Deserialize;
@ -20,6 +21,7 @@ use std::str::FromStr;
pub struct EmitConfigOptions {
pub check_js: bool,
pub emit_decorator_metadata: bool,
pub inline_source_map: bool,
pub jsx: String,
pub jsx_factory: String,
pub jsx_fragment_factory: String,
@ -30,77 +32,82 @@ pub struct EmitConfigOptions {
#[derive(Debug, Clone, PartialEq)]
pub struct IgnoredCompilerOptions {
pub items: Vec<String>,
pub path: PathBuf,
pub maybe_path: Option<PathBuf>,
}
impl fmt::Display for IgnoredCompilerOptions {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let mut codes = self.items.clone();
codes.sort();
write!(f, "Unsupported compiler options in \"{}\".\n The following options were ignored:\n {}", self.path.to_string_lossy(), codes.join(", "))
if let Some(path) = &self.maybe_path {
write!(f, "Unsupported compiler options in \"{}\".\n The following options were ignored:\n {}", path.to_string_lossy(), codes.join(", "))
} else {
write!(f, "Unsupported compiler options provided.\n The following options were ignored:\n {}", codes.join(", "))
}
}
}
/// A static slice of all the compiler options that should be ignored that
/// either have no effect on the compilation or would cause the emit to not work
/// in Deno.
const IGNORED_COMPILER_OPTIONS: [&str; 61] = [
const IGNORED_COMPILER_OPTIONS: [&str; 10] = [
"allowSyntheticDefaultImports",
"esModuleInterop",
"inlineSourceMap",
"inlineSources",
// TODO(nayeemrmn): Add "isolatedModules" here for 1.6.0.
"module",
"noLib",
"preserveConstEnums",
"reactNamespace",
"sourceMap",
"target",
];
const IGNORED_RUNTIME_COMPILER_OPTIONS: [&str; 50] = [
"allowUmdGlobalAccess",
"assumeChangesOnlyAffectDirectDependencies",
"baseUrl",
"build",
"composite",
"declaration",
"declarationDir",
"declarationMap",
"diagnostics",
"downlevelIteration",
"emitBOM",
"emitDeclarationOnly",
"esModuleInterop",
"extendedDiagnostics",
"forceConsistentCasingInFileNames",
"generateCpuProfile",
"help",
"importHelpers",
"incremental",
"inlineSourceMap",
"inlineSources",
"init",
// TODO(nayeemrmn): Add "isolatedModules" here for 1.6.0.
"listEmittedFiles",
"listFiles",
"mapRoot",
"maxNodeModuleJsDepth",
"module",
"moduleResolution",
"newLine",
"noEmit",
"noEmitHelpers",
"noEmitOnError",
"noLib",
"noResolve",
"out",
"outDir",
"outFile",
"paths",
"preserveConstEnums",
"preserveSymlinks",
"preserveWatchOutput",
"pretty",
"reactNamespace",
"resolveJsonModule",
"rootDir",
"rootDirs",
"showConfig",
"skipDefaultLibCheck",
"skipLibCheck",
"sourceMap",
"sourceRoot",
"stripInternal",
"target",
"traceResolution",
"tsBuildInfoFile",
"types",
@ -167,6 +174,34 @@ pub fn parse_raw_config(config_text: &str) -> Result<Value, AnyError> {
Ok(jsonc_to_serde(jsonc))
}
fn parse_compiler_options(
compiler_options: &HashMap<String, Value>,
maybe_path: Option<PathBuf>,
is_runtime: bool,
) -> Result<(Value, Option<IgnoredCompilerOptions>), AnyError> {
let mut filtered: HashMap<String, Value> = HashMap::new();
let mut items: Vec<String> = Vec::new();
for (key, value) in compiler_options.iter() {
let key = key.as_str();
if (!is_runtime && IGNORED_COMPILER_OPTIONS.contains(&key))
|| IGNORED_RUNTIME_COMPILER_OPTIONS.contains(&key)
{
items.push(key.to_string());
} else {
filtered.insert(key.to_string(), value.to_owned());
}
}
let value = serde_json::to_value(filtered)?;
let maybe_ignored_options = if !items.is_empty() {
Some(IgnoredCompilerOptions { items, maybe_path })
} else {
None
};
Ok((value, maybe_ignored_options))
}
/// Take a string of JSONC, parse it and return a serde `Value` of the text.
/// The result also contains any options that were ignored.
pub fn parse_config(
@ -176,29 +211,12 @@ pub fn parse_config(
assert!(!config_text.is_empty());
let jsonc = jsonc_parser::parse_to_value(config_text)?.unwrap();
let config: TSConfigJson = serde_json::from_value(jsonc_to_serde(jsonc))?;
let mut compiler_options: HashMap<String, Value> = HashMap::new();
let mut items: Vec<String> = Vec::new();
if let Some(in_compiler_options) = config.compiler_options {
for (key, value) in in_compiler_options.iter() {
if IGNORED_COMPILER_OPTIONS.contains(&key.as_str()) {
items.push(key.to_owned());
} else {
compiler_options.insert(key.to_owned(), value.to_owned());
}
}
}
let options_value = serde_json::to_value(compiler_options)?;
let ignored_options = if !items.is_empty() {
Some(IgnoredCompilerOptions {
items,
path: path.to_path_buf(),
})
if let Some(compiler_options) = config.compiler_options {
parse_compiler_options(&compiler_options, Some(path.to_owned()), false)
} else {
None
};
Ok((options_value, ignored_options))
Ok((json!({}), None))
}
}
/// A structure for managing the configuration of TypeScript
@ -237,7 +255,7 @@ impl TsConfig {
///
/// When there are options ignored out of the file, a warning will be written
/// to stderr regarding the options that were ignored.
pub fn merge_user_config(
pub fn merge_tsconfig(
&mut self,
maybe_path: Option<String>,
) -> Result<Option<IgnoredCompilerOptions>, AnyError> {
@ -263,6 +281,19 @@ impl TsConfig {
Ok(None)
}
}
/// Take a map of compiler options, filtering out any that are ignored, then
/// merge it with the current configuration, returning any options that might
/// have been ignored.
pub fn merge_user_config(
&mut self,
user_options: &HashMap<String, Value>,
) -> Result<Option<IgnoredCompilerOptions>, AnyError> {
let (value, maybe_ignored_options) =
parse_compiler_options(user_options, None, true)?;
json_merge(&mut self.0, &value);
Ok(maybe_ignored_options)
}
}
impl Serialize for TsConfig {
@ -321,11 +352,43 @@ mod tests {
ignored,
Some(IgnoredCompilerOptions {
items: vec!["build".to_string()],
path: config_path,
maybe_path: Some(config_path),
}),
);
}
#[test]
fn test_tsconfig_merge_user_options() {
let mut tsconfig = TsConfig::new(json!({
"target": "esnext",
"module": "esnext",
}));
let user_options = serde_json::from_value(json!({
"target": "es6",
"build": true,
"strict": false,
}))
.expect("could not convert to hashmap");
let maybe_ignored_options = tsconfig
.merge_user_config(&user_options)
.expect("could not merge options");
assert_eq!(
tsconfig.0,
json!({
"module": "esnext",
"target": "es6",
"strict": false,
})
);
assert_eq!(
maybe_ignored_options,
Some(IgnoredCompilerOptions {
items: vec!["build".to_string()],
maybe_path: None
})
);
}
#[test]
fn test_parse_raw_config() {
let invalid_config_text = r#"{