diff --git a/cli/compilers/ts.rs b/cli/compilers/ts.rs index 8a02efeeae..77297713ed 100644 --- a/cli/compilers/ts.rs +++ b/cli/compilers/ts.rs @@ -1,16 +1,16 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. use super::compiler_worker::CompilerWorker; use crate::colors; -use crate::compilers::CompilationResultFuture; use crate::compilers::CompiledModule; use crate::diagnostics::Diagnostic; +use crate::diagnostics::DiagnosticItem; use crate::disk_cache::DiskCache; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; +use crate::fs as deno_fs; use crate::global_state::GlobalState; use crate::msg; use crate::op_error::OpError; -use crate::ops::JsonResult; use crate::source_maps::SourceMapGetter; use crate::startup_data; use crate::state::*; @@ -21,10 +21,11 @@ use crate::worker::WorkerEvent; use deno_core::Buf; use deno_core::ErrBox; use deno_core::ModuleSpecifier; -use futures::future::FutureExt; use log::info; use regex::Regex; +use serde::Deserialize; use serde_json::json; +use serde_json::Value; use std::collections::HashMap; use std::collections::HashSet; use std::fs; @@ -32,7 +33,6 @@ use std::hash::BuildHasher; use std::io; use std::ops::Deref; use std::path::PathBuf; -use std::pin::Pin; use std::str; use std::sync::atomic::Ordering; use std::sync::Arc; @@ -172,7 +172,6 @@ fn req( request_type: msg::CompilerRequestType, root_names: Vec, compiler_config: CompilerConfig, - out_file: Option, target: &str, bundle: bool, unstable: bool, @@ -183,7 +182,6 @@ fn req( "type": request_type as i32, "target": target, "rootNames": root_names, - "outFile": out_file, "bundle": bundle, "unstable": unstable, "configPath": config_path, @@ -194,7 +192,6 @@ fn req( "type": request_type as i32, "target": target, "rootNames": root_names, - "outFile": out_file, "bundle": bundle, "unstable": unstable, "cwd": cwd, @@ -238,6 +235,43 @@ impl Deref for TsCompiler { } } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct EmittedSource { + filename: String, + contents: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct BundleResponse { + diagnostics: Diagnostic, + bundle_output: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct CompileResponse { + diagnostics: Diagnostic, + emit_map: HashMap, +} + +// TODO(bartlomieju): possible deduplicate once TS refactor is stabilized +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +#[allow(unused)] +struct RuntimeBundleResponse { + diagnostics: Vec, + output: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct RuntimeCompileResponse { + diagnostics: Vec, + emit_map: HashMap, +} + impl TsCompiler { pub fn new( file_fetcher: SourceFileFetcher, @@ -287,13 +321,13 @@ impl TsCompiler { "Invoking the compiler to bundle. module_name: {}", module_name ); + eprintln!("Bundling {}", module_name); let root_names = vec![module_name]; let req_msg = req( msg::CompilerRequestType::Compile, root_names, self.config.clone(), - out_file, "main", true, global_state.flags.unstable, @@ -302,9 +336,26 @@ impl TsCompiler { let msg = execute_in_thread(global_state.clone(), req_msg).await?; let json_str = std::str::from_utf8(&msg).unwrap(); debug!("Message: {}", json_str); - if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) { - return Err(ErrBox::from(diagnostics)); + + let bundle_response: BundleResponse = serde_json::from_str(json_str)?; + + if !bundle_response.diagnostics.items.is_empty() { + return Err(ErrBox::from(bundle_response.diagnostics)); } + + if let Some(out_file_) = out_file.as_ref() { + eprintln!("Emitting bundle to {:?}", out_file_); + + let output_bytes = bundle_response.bundle_output.as_bytes(); + let output_len = output_bytes.len(); + + deno_fs::write_file(out_file_, output_bytes, 0o666)?; + // TODO(bartlomieju): add "humanFileSize" method + eprintln!("{} bytes emmited.", output_len); + } else { + println!("{}", bundle_response.bundle_output); + } + Ok(()) } @@ -375,7 +426,6 @@ impl TsCompiler { msg::CompilerRequestType::Compile, root_names, self.config.clone(), - None, target, false, global_state.flags.unstable, @@ -390,11 +440,15 @@ impl TsCompiler { ); let msg = execute_in_thread(global_state.clone(), req_msg).await?; - let json_str = std::str::from_utf8(&msg).unwrap(); - if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) { - return Err(ErrBox::from(diagnostics)); + + let compile_response: CompileResponse = serde_json::from_str(json_str)?; + + if !compile_response.diagnostics.items.is_empty() { + return Err(ErrBox::from(compile_response.diagnostics)); } + + self.cache_emitted_files(compile_response.emit_map)?; ts_compiler.get_compiled_module(&source_file_.url) } @@ -418,6 +472,26 @@ impl TsCompiler { None } + fn cache_emitted_files( + &self, + emit_map: HashMap, + ) -> std::io::Result<()> { + for (emitted_name, source) in emit_map.iter() { + let specifier = ModuleSpecifier::resolve_url(&source.filename) + .expect("Should be a valid module specifier"); + + if emitted_name.ends_with(".map") { + self.cache_source_map(&specifier, &source.contents)?; + } else if emitted_name.ends_with(".js") { + self.cache_compiled_file(&specifier, &source.contents)?; + } else { + panic!("Trying to cache unknown file type {}", emitted_name); + } + } + + Ok(()) + } + pub fn get_compiled_module( &self, module_url: &Url, @@ -468,15 +542,23 @@ impl TsCompiler { module_specifier: &ModuleSpecifier, contents: &str, ) -> std::io::Result<()> { + let source_file = self + .file_fetcher + .fetch_cached_source_file(&module_specifier) + .expect("Source file not found"); + + // NOTE: JavaScript files are only cached to disk if `checkJs` + // option in on + if source_file.media_type == msg::MediaType::JavaScript && !self.compile_js + { + return Ok(()); + } + let js_key = self .disk_cache .get_cache_filename_with_extension(module_specifier.as_url(), "js"); self.disk_cache.set(&js_key, contents.as_bytes())?; self.mark_compiled(module_specifier.as_url()); - let source_file = self - .file_fetcher - .fetch_cached_source_file(&module_specifier) - .expect("Source file not found"); let version_hash = source_code_version_hash( &source_file.source_code, @@ -528,25 +610,23 @@ impl TsCompiler { module_specifier: &ModuleSpecifier, contents: &str, ) -> std::io::Result<()> { + let source_file = self + .file_fetcher + .fetch_cached_source_file(&module_specifier) + .expect("Source file not found"); + + // NOTE: JavaScript files are only cached to disk if `checkJs` + // option in on + if source_file.media_type == msg::MediaType::JavaScript && !self.compile_js + { + return Ok(()); + } + let source_map_key = self .disk_cache .get_cache_filename_with_extension(module_specifier.as_url(), "js.map"); self.disk_cache.set(&source_map_key, contents.as_bytes()) } - - /// This method is called by TS compiler via an "op". - pub fn cache_compiler_output( - &self, - module_specifier: &ModuleSpecifier, - extension: &str, - contents: &str, - ) -> std::io::Result<()> { - match extension { - ".map" => self.cache_source_map(module_specifier, contents), - ".js" => self.cache_compiled_file(module_specifier, contents), - _ => unreachable!(), - } - } } impl SourceMapGetter for TsCompiler { @@ -638,24 +718,14 @@ async fn execute_in_thread( Ok(buf) } -async fn execute_in_thread_json( - req_msg: Buf, - global_state: GlobalState, -) -> JsonResult { - let msg = execute_in_thread(global_state, req_msg) - .await - .map_err(|e| OpError::other(e.to_string()))?; - let json_str = std::str::from_utf8(&msg).unwrap(); - Ok(json!(json_str)) -} - -pub fn runtime_compile( +/// This function is used by `Deno.compile()` and `Deno.bundle()` APIs. +pub async fn runtime_compile( global_state: GlobalState, root_name: &str, sources: &Option>, bundle: bool, options: &Option, -) -> Pin> { +) -> Result { let req_msg = json!({ "type": msg::CompilerRequestType::RuntimeCompile as i32, "target": "runtime", @@ -669,14 +739,35 @@ pub fn runtime_compile( .into_boxed_str() .into_boxed_bytes(); - execute_in_thread_json(req_msg, global_state).boxed_local() + let compiler = global_state.ts_compiler.clone(); + + let msg = execute_in_thread(global_state, req_msg).await?; + let json_str = std::str::from_utf8(&msg).unwrap(); + + // TODO(bartlomieju): factor `bundle` path into separate function `runtime_bundle` + if bundle { + let _response: RuntimeBundleResponse = serde_json::from_str(json_str)?; + return Ok(serde_json::from_str::(json_str).unwrap()); + } + + let response: RuntimeCompileResponse = serde_json::from_str(json_str)?; + + if response.diagnostics.is_empty() && sources.is_none() { + compiler.cache_emitted_files(response.emit_map)?; + } + + // We're returning `Ok()` instead of `Err()` because it's not runtime + // error if there were diagnostics produces; we want to let user handle + // diagnostics in the runtime. + Ok(serde_json::from_str::(json_str).unwrap()) } -pub fn runtime_transpile( +/// This function is used by `Deno.transpileOnly()` API. +pub async fn runtime_transpile( global_state: GlobalState, sources: &HashMap, options: &Option, -) -> Pin> { +) -> Result { let req_msg = json!({ "type": msg::CompilerRequestType::RuntimeTranspile as i32, "sources": sources, @@ -686,7 +777,11 @@ pub fn runtime_transpile( .into_boxed_str() .into_boxed_bytes(); - execute_in_thread_json(req_msg, global_state).boxed_local() + let msg = execute_in_thread(global_state, req_msg).await?; + let json_str = std::str::from_utf8(&msg).unwrap(); + let v = serde_json::from_str::(json_str) + .expect("Error decoding JSON string."); + Ok(v) } #[cfg(test)] @@ -745,11 +840,7 @@ mod tests { let result = state .ts_compiler - .bundle( - state.clone(), - module_name, - Some(PathBuf::from("$deno$/bundle.js")), - ) + .bundle(state.clone(), module_name, None) .await; assert!(result.is_ok()); } diff --git a/cli/diagnostics.rs b/cli/diagnostics.rs index aca86913e9..a21ba654f2 100644 --- a/cli/diagnostics.rs +++ b/cli/diagnostics.rs @@ -7,48 +7,17 @@ use crate::colors; use crate::fmt_errors::format_stack; -use serde_json::value::Value; +use serde::Deserialize; +use serde::Deserializer; use std::error::Error; use std::fmt; -#[derive(Debug, PartialEq, Clone)] +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] pub struct Diagnostic { pub items: Vec, } -impl Diagnostic { - /// Take a JSON value and attempt to map it to a - pub fn from_json_value(v: &serde_json::Value) -> Option { - if !v.is_object() { - return None; - } - let obj = v.as_object().unwrap(); - - let mut items = Vec::::new(); - let items_v = &obj["items"]; - if items_v.is_array() { - let items_values = items_v.as_array().unwrap(); - - for item_v in items_values { - items.push(DiagnosticItem::from_json_value(item_v)?); - } - } - - Some(Self { items }) - } - - pub fn from_emit_result(json_str: &str) -> Option { - let v = serde_json::from_str::(json_str) - .expect("Error decoding JSON string."); - let diagnostics_o = v.get("diagnostics"); - if let Some(diagnostics_v) = diagnostics_o { - return Self::from_json_value(diagnostics_v); - } - - None - } -} - impl fmt::Display for Diagnostic { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut i = 0; @@ -74,7 +43,8 @@ impl Error for Diagnostic { } } -#[derive(Debug, PartialEq, Clone)] +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] pub struct DiagnosticItem { /// The top level message relating to the diagnostic item. pub message: String, @@ -114,71 +84,6 @@ pub struct DiagnosticItem { pub end_column: Option, } -impl DiagnosticItem { - pub fn from_json_value(v: &serde_json::Value) -> Option { - let obj = v.as_object().unwrap(); - - // required attributes - let message = obj - .get("message") - .and_then(|v| v.as_str().map(String::from))?; - let category = DiagnosticCategory::from( - obj.get("category").and_then(Value::as_i64).unwrap(), - ); - let code = obj.get("code").and_then(Value::as_i64).unwrap(); - - // optional attributes - let source_line = obj - .get("sourceLine") - .and_then(|v| v.as_str().map(String::from)); - let script_resource_name = obj - .get("scriptResourceName") - .and_then(|v| v.as_str().map(String::from)); - let line_number = obj.get("lineNumber").and_then(Value::as_i64); - let start_position = obj.get("startPosition").and_then(Value::as_i64); - let end_position = obj.get("endPosition").and_then(Value::as_i64); - let start_column = obj.get("startColumn").and_then(Value::as_i64); - let end_column = obj.get("endColumn").and_then(Value::as_i64); - - let message_chain_v = obj.get("messageChain"); - let message_chain = match message_chain_v { - Some(v) => DiagnosticMessageChain::from_json_value(v), - _ => None, - }; - - let related_information_v = obj.get("relatedInformation"); - let related_information = match related_information_v { - Some(r) => { - let mut related_information = Vec::::new(); - let related_info_values = r.as_array().unwrap(); - - for related_info_v in related_info_values { - related_information - .push(DiagnosticItem::from_json_value(related_info_v)?); - } - - Some(related_information) - } - _ => None, - }; - - Some(Self { - message, - message_chain, - related_information, - code, - source_line, - script_resource_name, - line_number, - start_position, - end_position, - category, - start_column, - end_column, - }) - } -} - fn format_category_and_code( category: &DiagnosticCategory, code: i64, @@ -303,7 +208,8 @@ impl fmt::Display for DiagnosticItem { } } -#[derive(Debug, PartialEq, Clone)] +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] pub struct DiagnosticMessageChain { pub message: String, pub code: i64, @@ -312,54 +218,6 @@ pub struct DiagnosticMessageChain { } impl DiagnosticMessageChain { - fn from_value(v: &serde_json::Value) -> Self { - let obj = v.as_object().unwrap(); - let message = obj - .get("message") - .and_then(|v| v.as_str().map(String::from)) - .unwrap(); - let code = obj.get("code").and_then(Value::as_i64).unwrap(); - let category = DiagnosticCategory::from( - obj.get("category").and_then(Value::as_i64).unwrap(), - ); - - let next_v = obj.get("next"); - let next = match next_v { - Some(n) => DiagnosticMessageChain::from_next_array(n), - _ => None, - }; - - Self { - message, - code, - category, - next, - } - } - - fn from_next_array(v: &serde_json::Value) -> Option> { - if !v.is_array() { - return None; - } - - let vec = v - .as_array() - .unwrap() - .iter() - .map(|item| Self::from_value(&item)) - .collect::>(); - - Some(vec) - } - - pub fn from_json_value(v: &serde_json::Value) -> Option { - if !v.is_object() { - return None; - } - - Some(Self::from_value(v)) - } - pub fn format_message(&self, level: usize) -> String { let mut s = String::new(); @@ -377,7 +235,7 @@ impl DiagnosticMessageChain { } } -#[derive(Debug, PartialEq, Clone)] +#[derive(Clone, Debug, PartialEq)] pub enum DiagnosticCategory { Log, // 0 Debug, // 1 @@ -387,6 +245,16 @@ pub enum DiagnosticCategory { Suggestion, // 5 } +impl<'de> Deserialize<'de> for DiagnosticCategory { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s: i64 = Deserialize::deserialize(deserializer)?; + Ok(DiagnosticCategory::from(s)) + } +} + impl From for DiagnosticCategory { fn from(value: i64) -> Self { match value { @@ -496,7 +364,7 @@ mod tests { #[test] fn from_json() { - let v = serde_json::from_str::( + let r = serde_json::from_str::( &r#"{ "items": [ { @@ -526,8 +394,7 @@ mod tests { ] }"#, ).unwrap(); - let r = Diagnostic::from_json_value(&v); - let expected = Some( + let expected = Diagnostic { items: vec![ DiagnosticItem { @@ -559,52 +426,10 @@ mod tests { end_column: Some(1) } ] - } - ); + }; assert_eq!(expected, r); } - #[test] - fn from_emit_result() { - let r = Diagnostic::from_emit_result( - &r#"{ - "emitSkipped": false, - "diagnostics": { - "items": [ - { - "message": "foo bar", - "code": 9999, - "category": 3 - } - ] - } - }"#, - ); - let expected = Some(Diagnostic { - items: vec![DiagnosticItem { - message: "foo bar".to_string(), - message_chain: None, - related_information: None, - source_line: None, - line_number: None, - script_resource_name: None, - start_position: None, - end_position: None, - category: DiagnosticCategory::Error, - code: 9999, - start_column: None, - end_column: None, - }], - }); - assert_eq!(expected, r); - } - - #[test] - fn from_emit_result_none() { - let r = &r#"{"emitSkipped":false}"#; - assert!(Diagnostic::from_emit_result(r).is_none()); - } - #[test] fn diagnostic_to_string1() { let d = diagnostic1(); diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index d3cd67119d..f1d3393168 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -29,7 +29,10 @@ import { resolveModules, } from "./compiler/imports.ts"; import { - createWriteFile, + EmmitedSource, + WriteFileCallback, + createCompileWriteFile, + createBundleWriteFile, CompilerRequestType, convertCompilerOptions, ignoredDiagnostics, @@ -53,7 +56,6 @@ interface CompilerRequestCompile { config?: string; unstable: boolean; bundle: boolean; - outFile?: string; cwd: string; } @@ -79,16 +81,20 @@ type CompilerRequest = | CompilerRequestRuntimeTranspile; interface CompileResult { - emitSkipped: boolean; - diagnostics?: Diagnostic; + emitMap?: Record; + bundleOutput?: string; + diagnostics: Diagnostic; } -type RuntimeCompileResult = [ - undefined | DiagnosticItem[], - Record -]; +interface RuntimeCompileResult { + emitMap: Record; + diagnostics: DiagnosticItem[]; +} -type RuntimeBundleResult = [undefined | DiagnosticItem[], string]; +interface RuntimeBundleResult { + output: string; + diagnostics: DiagnosticItem[]; +} async function compile( request: CompilerRequestCompile @@ -97,7 +103,6 @@ async function compile( bundle, config, configPath, - outFile, rootNames, target, unstable, @@ -111,31 +116,32 @@ async function compile( // When a programme is emitted, TypeScript will call `writeFile` with // each file that needs to be emitted. The Deno compiler host delegates // this, to make it easier to perform the right actions, which vary - // based a lot on the request. For a `Compile` request, we need to - // cache all the files in the privileged side if we aren't bundling, - // and if we are bundling we need to enrich the bundle and either write - // out the bundle or log it to the console. + // based a lot on the request. const state: WriteFileState = { type: request.type, + emitMap: {}, bundle, host: undefined, - outFile, rootNames, }; - const writeFile = createWriteFile(state); - + let writeFile: WriteFileCallback; + if (bundle) { + writeFile = createBundleWriteFile(state); + } else { + writeFile = createCompileWriteFile(state); + } const host = (state.host = new Host({ bundle, target, writeFile, unstable, })); - let diagnostics: readonly ts.Diagnostic[] | undefined; + let diagnostics: readonly ts.Diagnostic[] = []; // if there is a configuration supplied, we need to parse that if (config && config.length && configPath) { const configResult = host.configure(cwd, configPath, config); - diagnostics = processConfigureResponse(configResult, configPath); + diagnostics = processConfigureResponse(configResult, configPath) || []; } // This will recursively analyse all the code for other imports, @@ -147,10 +153,9 @@ async function compile( bundle || host.getCompilationSettings().checkJs ); - let emitSkipped = true; // if there was a configuration and no diagnostics with it, we will continue // to generate the program and possibly emit it. - if (!diagnostics || (diagnostics && diagnostics.length === 0)) { + if (diagnostics.length === 0) { const options = host.getCompilationSettings(); const program = ts.createProgram({ rootNames, @@ -168,23 +173,28 @@ async function compile( if (bundle) { // we only support a single root module when bundling assert(resolvedRootModules.length === 1); - // warning so it goes to stderr instead of stdout - console.warn(`Bundling "${resolvedRootModules[0]}"`); setRootExports(program, resolvedRootModules[0]); } const emitResult = program.emit(); - emitSkipped = emitResult.emitSkipped; + assert(emitResult.emitSkipped === false, "Unexpected skip of the emit."); // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned // without casting. diagnostics = emitResult.diagnostics; } } + let bundleOutput = undefined; + + if (bundle) { + assert(state.bundleOutput); + bundleOutput = state.bundleOutput; + } + + assert(state.emitMap); const result: CompileResult = { - emitSkipped, - diagnostics: diagnostics.length - ? fromTypeScriptDiagnostic(diagnostics) - : undefined, + emitMap: state.emitMap, + bundleOutput, + diagnostics: fromTypeScriptDiagnostic(diagnostics), }; util.log("<<< compile end", { @@ -259,9 +269,14 @@ async function runtimeCompile( rootNames, sources, emitMap: {}, - emitBundle: undefined, + bundleOutput: undefined, }; - const writeFile = createWriteFile(state); + let writeFile: WriteFileCallback; + if (bundle) { + writeFile = createBundleWriteFile(state); + } else { + writeFile = createCompileWriteFile(state); + } const host = (state.host = new Host({ bundle, @@ -314,12 +329,18 @@ async function runtimeCompile( const maybeDiagnostics = diagnostics.length ? fromTypeScriptDiagnostic(diagnostics).items - : undefined; + : []; if (bundle) { - return [maybeDiagnostics, state.emitBundle] as RuntimeBundleResult; + return { + diagnostics: maybeDiagnostics, + output: state.bundleOutput, + } as RuntimeBundleResult; } else { - return [maybeDiagnostics, state.emitMap] as RuntimeCompileResult; + return { + diagnostics: maybeDiagnostics, + emitMap: state.emitMap, + } as RuntimeCompileResult; } } diff --git a/cli/js/compiler/api.ts b/cli/js/compiler/api.ts index b7c57b528f..a7d1e57a8f 100644 --- a/cli/js/compiler/api.ts +++ b/cli/js/compiler/api.ts @@ -6,6 +6,8 @@ import { DiagnosticItem } from "../diagnostics.ts"; import * as util from "../util.ts"; import * as runtimeCompilerOps from "../ops/runtime_compiler.ts"; +import { TranspileOnlyResult } from "../ops/runtime_compiler.ts"; +export { TranspileOnlyResult } from "../ops/runtime_compiler.ts"; export interface CompilerOptions { allowJs?: boolean; @@ -145,12 +147,8 @@ function checkRelative(specifier: string): string { : `./${specifier}`; } -export interface TranspileOnlyResult { - source: string; - map?: string; -} - -export async function transpileOnly( +// TODO(bartlomieju): change return type to interface? +export function transpileOnly( sources: Record, options: CompilerOptions = {} ): Promise> { @@ -159,10 +157,10 @@ export async function transpileOnly( sources, options: JSON.stringify(options), }; - const result = await runtimeCompilerOps.transpile(payload); - return JSON.parse(result); + return runtimeCompilerOps.transpile(payload); } +// TODO(bartlomieju): change return type to interface? export async function compile( rootName: string, sources?: Record, @@ -180,9 +178,20 @@ export async function compile( options, }); const result = await runtimeCompilerOps.compile(payload); - return JSON.parse(result); + util.assert(result.emitMap); + const maybeDiagnostics = + result.diagnostics.length === 0 ? undefined : result.diagnostics; + + const emitMap: Record = {}; + + for (const [key, emmitedSource] of Object.entries(result.emitMap)) { + emitMap[key] = emmitedSource.contents; + } + + return [maybeDiagnostics, emitMap]; } +// TODO(bartlomieju): change return type to interface? export async function bundle( rootName: string, sources?: Record, @@ -200,5 +209,8 @@ export async function bundle( options, }); const result = await runtimeCompilerOps.compile(payload); - return JSON.parse(result); + util.assert(result.output); + const maybeDiagnostics = + result.diagnostics.length === 0 ? undefined : result.diagnostics; + return [maybeDiagnostics, result.output]; } diff --git a/cli/js/compiler/host.ts b/cli/js/compiler/host.ts index de2eacfa9c..64b5e0245f 100644 --- a/cli/js/compiler/host.ts +++ b/cli/js/compiler/host.ts @@ -255,16 +255,12 @@ export class Host implements ts.CompilerHost { assert(sourceFile != null); if (!sourceFile.tsSourceFile) { assert(sourceFile.sourceCode != null); - // even though we assert the extension for JSON modules to the compiler - // is TypeScript, TypeScript internally analyses the filename for its - // extension and tries to parse it as JSON instead of TS. We have to - // change the filename to the TypeScript file. + const tsSourceFileName = fileName.startsWith(ASSETS) + ? sourceFile.filename + : fileName; + sourceFile.tsSourceFile = ts.createSourceFile( - fileName.startsWith(ASSETS) - ? sourceFile.filename - : fileName.toLowerCase().endsWith(".json") - ? `${fileName}.ts` - : fileName, + tsSourceFileName, sourceFile.sourceCode, languageVersion ); @@ -294,15 +290,20 @@ export class Host implements ts.CompilerHost { containingFile, }); return moduleNames.map((specifier) => { - const url = SourceFile.getUrl(specifier, containingFile); - const sourceFile = specifier.startsWith(ASSETS) - ? getAssetInternal(specifier) - : url - ? SourceFile.get(url) - : undefined; + const maybeUrl = SourceFile.getUrl(specifier, containingFile); + + let sourceFile: SourceFile | undefined = undefined; + + if (specifier.startsWith(ASSETS)) { + sourceFile = getAssetInternal(specifier); + } else if (typeof maybeUrl !== "undefined") { + sourceFile = SourceFile.get(maybeUrl); + } + if (!sourceFile) { return undefined; } + return { resolvedFileName: sourceFile.url, isExternalLibraryImport: specifier.startsWith(ASSETS), diff --git a/cli/js/compiler/imports.ts b/cli/js/compiler/imports.ts index de44027586..a811075bef 100644 --- a/cli/js/compiler/imports.ts +++ b/cli/js/compiler/imports.ts @@ -122,11 +122,8 @@ export async function processImports( SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson); sourceFile.cache(specifiers[i][0], referrer); if (!sourceFile.processed) { - await processImports( - sourceFile.imports(processJsImports), - sourceFile.url, - processJsImports - ); + const sourceFileImports = sourceFile.imports(processJsImports); + await processImports(sourceFileImports, sourceFile.url, processJsImports); } } return resolvedSources; diff --git a/cli/js/compiler/sourcefile.ts b/cli/js/compiler/sourcefile.ts index 3d547551fc..d390c3f562 100644 --- a/cli/js/compiler/sourcefile.ts +++ b/cli/js/compiler/sourcefile.ts @@ -54,8 +54,6 @@ export class SourceFile { extension!: ts.Extension; filename!: string; - importedFiles?: Array<[string, string]>; - mediaType!: MediaType; processed = false; sourceCode?: string; @@ -93,14 +91,18 @@ export class SourceFile { return []; } + const readImportFiles = true; + const detectJsImports = + this.mediaType === MediaType.JavaScript || + this.mediaType === MediaType.JSX; + const preProcessedFileInfo = ts.preProcessFile( this.sourceCode, - true, - this.mediaType === MediaType.JavaScript || - this.mediaType === MediaType.JSX + readImportFiles, + detectJsImports ); this.processed = true; - const files = (this.importedFiles = [] as Array<[string, string]>); + const files: Array<[string, string]> = []; function process(references: Array<{ fileName: string }>): void { for (const { fileName } of references) { @@ -160,8 +162,4 @@ export class SourceFile { static get(url: string): SourceFile | undefined { return moduleCache.get(url); } - - static has(url: string): boolean { - return moduleCache.has(url); - } } diff --git a/cli/js/compiler/util.ts b/cli/js/compiler/util.ts index 35ce2e837e..f3cbe5566e 100644 --- a/cli/js/compiler/util.ts +++ b/cli/js/compiler/util.ts @@ -4,12 +4,16 @@ import { bold, cyan, yellow } from "../colors.ts"; import { CompilerOptions } from "./api.ts"; import { buildBundle } from "./bundler.ts"; import { ConfigureResponse, Host } from "./host.ts"; -import { MediaType, SourceFile } from "./sourcefile.ts"; -import { atob, TextEncoder } from "../web/text_encoding.ts"; +import { atob } from "../web/text_encoding.ts"; import * as compilerOps from "../ops/compiler.ts"; -import * as util from "../util.ts"; import { assert } from "../util.ts"; -import { writeFileSync } from "../write_file.ts"; + +export interface EmmitedSource { + // original filename + filename: string; + // compiled contents + contents: string; +} export type WriteFileCallback = ( fileName: string, @@ -20,11 +24,10 @@ export type WriteFileCallback = ( export interface WriteFileState { type: CompilerRequestType; bundle?: boolean; + bundleOutput?: string; host?: Host; - outFile?: string; rootNames: string[]; - emitMap?: Record; - emitBundle?: string; + emitMap?: Record; sources?: Record; } @@ -38,87 +41,33 @@ export enum CompilerRequestType { export const OUT_DIR = "$deno$"; -function cache( - moduleId: string, - emittedFileName: string, - contents: string, - checkJs = false -): void { - util.log("compiler::cache", { moduleId, emittedFileName, checkJs }); - const sf = SourceFile.get(moduleId); - - if (sf) { - // NOTE: JavaScript files are only cached to disk if `checkJs` - // option in on - if (sf.mediaType === MediaType.JavaScript && !checkJs) { - return; - } - } - - if (emittedFileName.endsWith(".map")) { - // Source Map - compilerOps.cache(".map", moduleId, contents); - } else if (emittedFileName.endsWith(".js")) { - // Compiled JavaScript - compilerOps.cache(".js", moduleId, contents); - } else { - assert(false, `Trying to cache unhandled file type "${emittedFileName}"`); - } -} - export function getAsset(name: string): string { return compilerOps.getAsset(name); } -export function createWriteFile(state: WriteFileState): WriteFileCallback { - const encoder = new TextEncoder(); - if (state.type === CompilerRequestType.Compile) { - return function writeFile( - fileName: string, - data: string, - sourceFiles?: readonly ts.SourceFile[] - ): void { - assert( - sourceFiles != null, - `Unexpected emit of "${fileName}" which isn't part of a program.` - ); - assert(state.host); - if (!state.bundle) { - assert(sourceFiles.length === 1); - cache( - sourceFiles[0].fileName, - fileName, - data, - state.host.getCompilationSettings().checkJs - ); - } else { - // if the fileName is set to an internal value, just noop, this is - // used in the Rust unit tests. - if (state.outFile && state.outFile.startsWith(OUT_DIR)) { - return; - } - // we only support single root names for bundles - assert( - state.rootNames.length === 1, - `Only one root name supported. Got "${JSON.stringify( - state.rootNames - )}"` - ); - // this enriches the string with the loader and re-exports the - // exports of the root module - const content = buildBundle(state.rootNames[0], data, sourceFiles); - if (state.outFile) { - const encodedData = encoder.encode(content); - console.warn(`Emitting bundle to "${state.outFile}"`); - writeFileSync(state.outFile, encodedData); - console.warn(`${humanFileSize(encodedData.length)} emitted.`); - } else { - console.log(content); - } - } - }; - } +// TODO(bartlomieju): probably could be defined inline? +export function createBundleWriteFile( + state: WriteFileState +): WriteFileCallback { + return function writeFile( + _fileName: string, + data: string, + sourceFiles?: readonly ts.SourceFile[] + ): void { + assert(sourceFiles != null); + assert(state.host); + assert(state.emitMap); + assert(state.bundle); + // we only support single root names for bundles + assert(state.rootNames.length === 1); + state.bundleOutput = buildBundle(state.rootNames[0], data, sourceFiles); + }; +} +// TODO(bartlomieju): probably could be defined inline? +export function createCompileWriteFile( + state: WriteFileState +): WriteFileCallback { return function writeFile( fileName: string, data: string, @@ -127,24 +76,12 @@ export function createWriteFile(state: WriteFileState): WriteFileCallback { assert(sourceFiles != null); assert(state.host); assert(state.emitMap); - if (!state.bundle) { - assert(sourceFiles.length === 1); - state.emitMap[fileName] = data; - // we only want to cache the compiler output if we are resolving - // modules externally - if (!state.sources) { - cache( - sourceFiles[0].fileName, - fileName, - data, - state.host.getCompilationSettings().checkJs - ); - } - } else { - // we only support single root names for bundles - assert(state.rootNames.length === 1); - state.emitBundle = buildBundle(state.rootNames[0], data, sourceFiles); - } + assert(!state.bundle); + assert(sourceFiles.length === 1); + state.emitMap[fileName] = { + filename: sourceFiles[0].fileName, + contents: data, + }; }; } @@ -380,20 +317,6 @@ export function commonPath(paths: string[], sep = "/"): string { return prefix.endsWith(sep) ? prefix : `${prefix}${sep}`; } -function humanFileSize(bytes: number): string { - const thresh = 1000; - if (Math.abs(bytes) < thresh) { - return bytes + " B"; - } - const units = ["kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; - let u = -1; - do { - bytes /= thresh; - ++u; - } while (Math.abs(bytes) >= thresh && u < units.length - 1); - return `${bytes.toFixed(1)} ${units[u]}`; -} - // @internal export function base64ToUint8Array(data: string): Uint8Array { const binString = atob(data); diff --git a/cli/js/ops/compiler.ts b/cli/js/ops/compiler.ts index 825cadc16b..60f8147417 100644 --- a/cli/js/ops/compiler.ts +++ b/cli/js/ops/compiler.ts @@ -38,15 +38,3 @@ export function getAsset(name: string): string { const sourceCodeBytes = core.dispatch(opId, encoder.encode(name)); return decoder.decode(sourceCodeBytes!); } - -export function cache( - extension: string, - moduleId: string, - contents: string -): void { - sendSync("op_cache", { - extension, - moduleId, - contents, - }); -} diff --git a/cli/js/ops/runtime_compiler.ts b/cli/js/ops/runtime_compiler.ts index b46670acef..5a89983ee7 100644 --- a/cli/js/ops/runtime_compiler.ts +++ b/cli/js/ops/runtime_compiler.ts @@ -1,6 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import { sendAsync } from "./dispatch_json.ts"; +import { DiagnosticItem } from "../diagnostics.ts"; interface CompileRequest { rootName: string; @@ -9,7 +10,13 @@ interface CompileRequest { bundle: boolean; } -export function compile(request: CompileRequest): Promise { +interface CompileResponse { + diagnostics: DiagnosticItem[]; + output?: string; + emitMap?: Record>; +} + +export function compile(request: CompileRequest): Promise { return sendAsync("op_compile", request); } @@ -18,6 +25,13 @@ interface TranspileRequest { options?: string; } -export function transpile(request: TranspileRequest): Promise { +export interface TranspileOnlyResult { + source: string; + map?: string; +} + +export function transpile( + request: TranspileRequest +): Promise> { return sendAsync("op_transpile", request); } diff --git a/cli/js/tests/format_error_test.ts b/cli/js/tests/format_error_test.ts index 0cb963ae69..ae7559c82e 100644 --- a/cli/js/tests/format_error_test.ts +++ b/cli/js/tests/format_error_test.ts @@ -26,7 +26,7 @@ unitTest(function formatDiagnosticError() { try { Deno.formatDiagnostics(bad); } catch (e) { - assert(e instanceof TypeError); + assert(e instanceof Deno.errors.InvalidData); thrown = true; } assert(thrown); diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index c66b56d43c..d93a0edf40 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -13,7 +13,6 @@ use deno_core::ZeroCopyBuf; use futures::future::FutureExt; pub fn init(i: &mut CoreIsolate, s: &State) { - i.register_op("op_cache", s.stateful_json_op(op_cache)); i.register_op("op_resolve_modules", s.stateful_json_op(op_resolve_modules)); i.register_op( "op_fetch_source_files", @@ -26,35 +25,6 @@ pub fn init(i: &mut CoreIsolate, s: &State) { ); } -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -struct CacheArgs { - module_id: String, - contents: String, - extension: String, -} - -fn op_cache( - state: &State, - args: Value, - _zero_copy: Option, -) -> Result { - let args: CacheArgs = serde_json::from_value(args)?; - - let module_specifier = ModuleSpecifier::resolve_url(&args.module_id) - .expect("Should be valid module specifier"); - - let state_ = &state.borrow().global_state; - let ts_compiler = state_.ts_compiler.clone(); - ts_compiler.cache_compiler_output( - &module_specifier, - &args.extension, - &args.contents, - )?; - - Ok(JsonOp::Sync(json!({}))) -} - #[derive(Deserialize, Debug)] struct SpecifiersReferrerArgs { specifiers: Vec, diff --git a/cli/ops/errors.rs b/cli/ops/errors.rs index 3e48fd0076..766c130e2c 100644 --- a/cli/ops/errors.rs +++ b/cli/ops/errors.rs @@ -57,9 +57,6 @@ fn op_format_diagnostic( args: Value, _zero_copy: Option, ) -> Result { - if let Some(diagnostic) = Diagnostic::from_json_value(&args) { - Ok(JsonOp::Sync(json!(diagnostic.to_string()))) - } else { - Err(OpError::type_error("bad diagnostic".to_string())) - } + let diagnostic = serde_json::from_value::(args)?; + Ok(JsonOp::Sync(json!(diagnostic.to_string()))) } diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs index 4c4110e1e8..c7225b9441 100644 --- a/cli/ops/runtime_compiler.rs +++ b/cli/ops/runtime_compiler.rs @@ -2,6 +2,7 @@ use super::dispatch_json::{Deserialize, JsonOp, Value}; use crate::compilers::runtime_compile; use crate::compilers::runtime_transpile; +use crate::futures::FutureExt; use crate::op_error::OpError; use crate::state::State; use deno_core::CoreIsolate; @@ -29,13 +30,19 @@ fn op_compile( ) -> Result { state.check_unstable("Deno.compile"); let args: CompileArgs = serde_json::from_value(args)?; - Ok(JsonOp::Async(runtime_compile( - state.borrow().global_state.clone(), - &args.root_name, - &args.sources, - args.bundle, - &args.options, - ))) + let global_state = state.borrow().global_state.clone(); + let fut = async move { + runtime_compile( + global_state, + &args.root_name, + &args.sources, + args.bundle, + &args.options, + ) + .await + } + .boxed_local(); + Ok(JsonOp::Async(fut)) } #[derive(Deserialize, Debug)] @@ -51,9 +58,10 @@ fn op_transpile( ) -> Result { state.check_unstable("Deno.transpile"); let args: TranspileArgs = serde_json::from_value(args)?; - Ok(JsonOp::Async(runtime_transpile( - state.borrow().global_state.clone(), - &args.sources, - &args.options, - ))) + let global_state = state.borrow().global_state.clone(); + let fut = async move { + runtime_transpile(global_state, &args.sources, &args.options).await + } + .boxed_local(); + Ok(JsonOp::Async(fut)) } diff --git a/cli/tests/lib_ref.ts.out b/cli/tests/lib_ref.ts.out index 6ff3616b70..9f8c62d8af 100644 --- a/cli/tests/lib_ref.ts.out +++ b/cli/tests/lib_ref.ts.out @@ -1,2 +1,2 @@ -null +undefined [ "main.js.map", "main.js" ] diff --git a/cli/tests/lib_runtime_api.ts.out b/cli/tests/lib_runtime_api.ts.out index 6ff3616b70..9f8c62d8af 100644 --- a/cli/tests/lib_runtime_api.ts.out +++ b/cli/tests/lib_runtime_api.ts.out @@ -1,2 +1,2 @@ -null +undefined [ "main.js.map", "main.js" ]