foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-22 07:14:47 -05:00
Code Issues Packages 1 Wiki Activity

refactor(ts): remove op_cache (#5071)

Browse source 
This PR removes op_cache and refactors how Deno interacts with TS compiler.

Ultimate goal is to completely sandbox TS compiler worker; it should operate on
simple request -> response basis. With this commit TS compiler no longer
caches compiled sources as they are generated but rather collects all sources
and sends them back to Rust when compilation is done.

Additionally "Diagnostic" and its children got refactored to use "Deserialize" trait
instead of manually implementing JSON deserialization.
This commit is contained in:
Bartek Iwańczuk 2020-05-05 18:23:15 +02:00 committed by GitHub
parent e574437922
commit cf5a39a361
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 348 additions and 503 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

199
cli/compilers/ts.rs
View file

@ -1,16 +1,16 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
use super::compiler_worker::CompilerWorker;
use crate::colors;
use crate::compilers::CompilationResultFuture;
use crate::compilers::CompiledModule;
use crate::diagnostics::Diagnostic;
use crate::diagnostics::DiagnosticItem;
use crate::disk_cache::DiskCache;
use crate::file_fetcher::SourceFile;
use crate::file_fetcher::SourceFileFetcher;
use crate::fs as deno_fs;
use crate::global_state::GlobalState;
use crate::msg;
use crate::op_error::OpError;
use crate::ops::JsonResult;
use crate::source_maps::SourceMapGetter;
use crate::startup_data;
use crate::state::*;
@ -21,10 +21,11 @@ use crate::worker::WorkerEvent;
use deno_core::Buf;
use deno_core::ErrBox;
use deno_core::ModuleSpecifier;
use futures::future::FutureExt;
use log::info;
use regex::Regex;
use serde::Deserialize;
use serde_json::json;
use serde_json::Value;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fs;
@ -32,7 +33,6 @@ use std::hash::BuildHasher;
use std::io;
use std::ops::Deref;
use std::path::PathBuf;
use std::pin::Pin;
use std::str;
use std::sync::atomic::Ordering;
use std::sync::Arc;
@ -172,7 +172,6 @@ fn req(
request_type: msg::CompilerRequestType,
root_names: Vec<String>,
compiler_config: CompilerConfig,
out_file: Option<PathBuf>,
target: &str,
bundle: bool,
unstable: bool,
@ -183,7 +182,6 @@ fn req(
"type": request_type as i32,
"target": target,
"rootNames": root_names,
"outFile": out_file,
"bundle": bundle,
"unstable": unstable,
"configPath": config_path,
@ -194,7 +192,6 @@ fn req(
"type": request_type as i32,
"target": target,
"rootNames": root_names,
"outFile": out_file,
"bundle": bundle,
"unstable": unstable,
"cwd": cwd,
@ -238,6 +235,43 @@ impl Deref for TsCompiler {
}
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct EmittedSource {
filename: String,
contents: String,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct BundleResponse {
diagnostics: Diagnostic,
bundle_output: String,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct CompileResponse {
diagnostics: Diagnostic,
emit_map: HashMap<String, EmittedSource>,
}
// TODO(bartlomieju): possible deduplicate once TS refactor is stabilized
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
#[allow(unused)]
struct RuntimeBundleResponse {
diagnostics: Vec<DiagnosticItem>,
output: String,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct RuntimeCompileResponse {
diagnostics: Vec<DiagnosticItem>,
emit_map: HashMap<String, EmittedSource>,
}
impl TsCompiler {
pub fn new(
file_fetcher: SourceFileFetcher,
@ -287,13 +321,13 @@ impl TsCompiler {
"Invoking the compiler to bundle. module_name: {}",
module_name
);
eprintln!("Bundling {}", module_name);
let root_names = vec![module_name];
let req_msg = req(
msg::CompilerRequestType::Compile,
root_names,
self.config.clone(),
out_file,
"main",
true,
global_state.flags.unstable,
@ -302,9 +336,26 @@ impl TsCompiler {
let msg = execute_in_thread(global_state.clone(), req_msg).await?;
let json_str = std::str::from_utf8(&msg).unwrap();
debug!("Message: {}", json_str);
if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) {
return Err(ErrBox::from(diagnostics));
let bundle_response: BundleResponse = serde_json::from_str(json_str)?;
if !bundle_response.diagnostics.items.is_empty() {
return Err(ErrBox::from(bundle_response.diagnostics));
}
if let Some(out_file_) = out_file.as_ref() {
eprintln!("Emitting bundle to {:?}", out_file_);
let output_bytes = bundle_response.bundle_output.as_bytes();
let output_len = output_bytes.len();
deno_fs::write_file(out_file_, output_bytes, 0o666)?;
// TODO(bartlomieju): add "humanFileSize" method
eprintln!("{} bytes emmited.", output_len);
} else {
println!("{}", bundle_response.bundle_output);
}
Ok(())
}
@ -375,7 +426,6 @@ impl TsCompiler {
msg::CompilerRequestType::Compile,
root_names,
self.config.clone(),
None,
target,
false,
global_state.flags.unstable,
@ -390,11 +440,15 @@ impl TsCompiler {
);
let msg = execute_in_thread(global_state.clone(), req_msg).await?;
let json_str = std::str::from_utf8(&msg).unwrap();
if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) {
return Err(ErrBox::from(diagnostics));
let compile_response: CompileResponse = serde_json::from_str(json_str)?;
if !compile_response.diagnostics.items.is_empty() {
return Err(ErrBox::from(compile_response.diagnostics));
}
self.cache_emitted_files(compile_response.emit_map)?;
ts_compiler.get_compiled_module(&source_file_.url)
}
@ -418,6 +472,26 @@ impl TsCompiler {
None
}
fn cache_emitted_files(
&self,
emit_map: HashMap<String, EmittedSource>,
) -> std::io::Result<()> {
for (emitted_name, source) in emit_map.iter() {
let specifier = ModuleSpecifier::resolve_url(&source.filename)
.expect("Should be a valid module specifier");
if emitted_name.ends_with(".map") {
self.cache_source_map(&specifier, &source.contents)?;
} else if emitted_name.ends_with(".js") {
self.cache_compiled_file(&specifier, &source.contents)?;
} else {
panic!("Trying to cache unknown file type {}", emitted_name);
}
}
Ok(())
}
pub fn get_compiled_module(
&self,
module_url: &Url,
@ -468,15 +542,23 @@ impl TsCompiler {
module_specifier: &ModuleSpecifier,
contents: &str,
) -> std::io::Result<()> {
let source_file = self
.file_fetcher
.fetch_cached_source_file(&module_specifier)
.expect("Source file not found");
// NOTE: JavaScript files are only cached to disk if `checkJs`
// option in on
if source_file.media_type == msg::MediaType::JavaScript && !self.compile_js
{
return Ok(());
}
let js_key = self
.disk_cache
.get_cache_filename_with_extension(module_specifier.as_url(), "js");
self.disk_cache.set(&js_key, contents.as_bytes())?;
self.mark_compiled(module_specifier.as_url());
let source_file = self
.file_fetcher
.fetch_cached_source_file(&module_specifier)
.expect("Source file not found");
let version_hash = source_code_version_hash(
&source_file.source_code,
@ -528,25 +610,23 @@ impl TsCompiler {
module_specifier: &ModuleSpecifier,
contents: &str,
) -> std::io::Result<()> {
let source_file = self
.file_fetcher
.fetch_cached_source_file(&module_specifier)
.expect("Source file not found");
// NOTE: JavaScript files are only cached to disk if `checkJs`
// option in on
if source_file.media_type == msg::MediaType::JavaScript && !self.compile_js
{
return Ok(());
}
let source_map_key = self
.disk_cache
.get_cache_filename_with_extension(module_specifier.as_url(), "js.map");
self.disk_cache.set(&source_map_key, contents.as_bytes())
}
/// This method is called by TS compiler via an "op".
pub fn cache_compiler_output(
&self,
module_specifier: &ModuleSpecifier,
extension: &str,
contents: &str,
) -> std::io::Result<()> {
match extension {
".map" => self.cache_source_map(module_specifier, contents),
".js" => self.cache_compiled_file(module_specifier, contents),
_ => unreachable!(),
}
}
}
impl SourceMapGetter for TsCompiler {
@ -638,24 +718,14 @@ async fn execute_in_thread(
Ok(buf)
}
async fn execute_in_thread_json(
req_msg: Buf,
global_state: GlobalState,
) -> JsonResult {
let msg = execute_in_thread(global_state, req_msg)
.await
.map_err(|e| OpError::other(e.to_string()))?;
let json_str = std::str::from_utf8(&msg).unwrap();
Ok(json!(json_str))
}
pub fn runtime_compile<S: BuildHasher>(
/// This function is used by `Deno.compile()` and `Deno.bundle()` APIs.
pub async fn runtime_compile<S: BuildHasher>(
global_state: GlobalState,
root_name: &str,
sources: &Option<HashMap<String, String, S>>,
bundle: bool,
options: &Option<String>,
) -> Pin<Box<CompilationResultFuture>> {
) -> Result<Value, OpError> {
let req_msg = json!({
"type": msg::CompilerRequestType::RuntimeCompile as i32,
"target": "runtime",
@ -669,14 +739,35 @@ pub fn runtime_compile<S: BuildHasher>(
.into_boxed_str()
.into_boxed_bytes();
execute_in_thread_json(req_msg, global_state).boxed_local()
let compiler = global_state.ts_compiler.clone();
let msg = execute_in_thread(global_state, req_msg).await?;
let json_str = std::str::from_utf8(&msg).unwrap();
// TODO(bartlomieju): factor `bundle` path into separate function `runtime_bundle`
if bundle {
let _response: RuntimeBundleResponse = serde_json::from_str(json_str)?;
return Ok(serde_json::from_str::<Value>(json_str).unwrap());
}
pub fn runtime_transpile<S: BuildHasher>(
let response: RuntimeCompileResponse = serde_json::from_str(json_str)?;
if response.diagnostics.is_empty() && sources.is_none() {
compiler.cache_emitted_files(response.emit_map)?;
}
// We're returning `Ok()` instead of `Err()` because it's not runtime
// error if there were diagnostics produces; we want to let user handle
// diagnostics in the runtime.
Ok(serde_json::from_str::<Value>(json_str).unwrap())
}
/// This function is used by `Deno.transpileOnly()` API.
pub async fn runtime_transpile<S: BuildHasher>(
global_state: GlobalState,
sources: &HashMap<String, String, S>,
options: &Option<String>,
) -> Pin<Box<CompilationResultFuture>> {
) -> Result<Value, OpError> {
let req_msg = json!({
"type": msg::CompilerRequestType::RuntimeTranspile as i32,
"sources": sources,
@ -686,7 +777,11 @@ pub fn runtime_transpile<S: BuildHasher>(
.into_boxed_str()
.into_boxed_bytes();
execute_in_thread_json(req_msg, global_state).boxed_local()
let msg = execute_in_thread(global_state, req_msg).await?;
let json_str = std::str::from_utf8(&msg).unwrap();
let v = serde_json::from_str::<serde_json::Value>(json_str)
.expect("Error decoding JSON string.");
Ok(v)
}
#[cfg(test)]
@ -745,11 +840,7 @@ mod tests {
let result = state
.ts_compiler
.bundle(
state.clone(),
module_name,
Some(PathBuf::from("$deno$/bundle.js")),
)
.bundle(state.clone(), module_name, None)
.await;
assert!(result.is_ok());
}

219
cli/diagnostics.rs
View file

@ -7,48 +7,17 @@
use crate::colors;
use crate::fmt_errors::format_stack;
use serde_json::value::Value;
use serde::Deserialize;
use serde::Deserializer;
use std::error::Error;
use std::fmt;
#[derive(Debug, PartialEq, Clone)]
#[derive(Clone, Debug, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct Diagnostic {
pub items: Vec<DiagnosticItem>,
}
impl Diagnostic {
/// Take a JSON value and attempt to map it to a
pub fn from_json_value(v: &serde_json::Value) -> Option<Self> {
if !v.is_object() {
return None;
}
let obj = v.as_object().unwrap();
let mut items = Vec::<DiagnosticItem>::new();
let items_v = &obj["items"];
if items_v.is_array() {
let items_values = items_v.as_array().unwrap();
for item_v in items_values {
items.push(DiagnosticItem::from_json_value(item_v)?);
}
}
Some(Self { items })
}
pub fn from_emit_result(json_str: &str) -> Option<Self> {
let v = serde_json::from_str::<serde_json::Value>(json_str)
.expect("Error decoding JSON string.");
let diagnostics_o = v.get("diagnostics");
if let Some(diagnostics_v) = diagnostics_o {
return Self::from_json_value(diagnostics_v);
}
None
}
}
impl fmt::Display for Diagnostic {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let mut i = 0;
@ -74,7 +43,8 @@ impl Error for Diagnostic {
}
}
#[derive(Debug, PartialEq, Clone)]
#[derive(Clone, Debug, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct DiagnosticItem {
/// The top level message relating to the diagnostic item.
pub message: String,
@ -114,71 +84,6 @@ pub struct DiagnosticItem {
pub end_column: Option<i64>,
}
impl DiagnosticItem {
pub fn from_json_value(v: &serde_json::Value) -> Option<Self> {
let obj = v.as_object().unwrap();
// required attributes
let message = obj
.get("message")
.and_then(|v| v.as_str().map(String::from))?;
let category = DiagnosticCategory::from(
obj.get("category").and_then(Value::as_i64).unwrap(),
);
let code = obj.get("code").and_then(Value::as_i64).unwrap();
// optional attributes
let source_line = obj
.get("sourceLine")
.and_then(|v| v.as_str().map(String::from));
let script_resource_name = obj
.get("scriptResourceName")
.and_then(|v| v.as_str().map(String::from));
let line_number = obj.get("lineNumber").and_then(Value::as_i64);
let start_position = obj.get("startPosition").and_then(Value::as_i64);
let end_position = obj.get("endPosition").and_then(Value::as_i64);
let start_column = obj.get("startColumn").and_then(Value::as_i64);
let end_column = obj.get("endColumn").and_then(Value::as_i64);
let message_chain_v = obj.get("messageChain");
let message_chain = match message_chain_v {
Some(v) => DiagnosticMessageChain::from_json_value(v),
_ => None,
};
let related_information_v = obj.get("relatedInformation");
let related_information = match related_information_v {
Some(r) => {
let mut related_information = Vec::<DiagnosticItem>::new();
let related_info_values = r.as_array().unwrap();
for related_info_v in related_info_values {
related_information
.push(DiagnosticItem::from_json_value(related_info_v)?);
}
Some(related_information)
}
_ => None,
};
Some(Self {
message,
message_chain,
related_information,
code,
source_line,
script_resource_name,
line_number,
start_position,
end_position,
category,
start_column,
end_column,
})
}
}
fn format_category_and_code(
category: &DiagnosticCategory,
code: i64,
@ -303,7 +208,8 @@ impl fmt::Display for DiagnosticItem {
}
}
#[derive(Debug, PartialEq, Clone)]
#[derive(Clone, Debug, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct DiagnosticMessageChain {
pub message: String,
pub code: i64,
@ -312,54 +218,6 @@ pub struct DiagnosticMessageChain {
}
impl DiagnosticMessageChain {
fn from_value(v: &serde_json::Value) -> Self {
let obj = v.as_object().unwrap();
let message = obj
.get("message")
.and_then(|v| v.as_str().map(String::from))
.unwrap();
let code = obj.get("code").and_then(Value::as_i64).unwrap();
let category = DiagnosticCategory::from(
obj.get("category").and_then(Value::as_i64).unwrap(),
);
let next_v = obj.get("next");
let next = match next_v {
Some(n) => DiagnosticMessageChain::from_next_array(n),
_ => None,
};
Self {
message,
code,
category,
next,
}
}
fn from_next_array(v: &serde_json::Value) -> Option<Vec<Self>> {
if !v.is_array() {
return None;
}
let vec = v
.as_array()
.unwrap()
.iter()
.map(|item| Self::from_value(&item))
.collect::<Vec<Self>>();
Some(vec)
}
pub fn from_json_value(v: &serde_json::Value) -> Option<Self> {
if !v.is_object() {
return None;
}
Some(Self::from_value(v))
}
pub fn format_message(&self, level: usize) -> String {
let mut s = String::new();
@ -377,7 +235,7 @@ impl DiagnosticMessageChain {
}
}
#[derive(Debug, PartialEq, Clone)]
#[derive(Clone, Debug, PartialEq)]
pub enum DiagnosticCategory {
Log, // 0
Debug, // 1
@ -387,6 +245,16 @@ pub enum DiagnosticCategory {
Suggestion, // 5
}
impl<'de> Deserialize<'de> for DiagnosticCategory {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s: i64 = Deserialize::deserialize(deserializer)?;
Ok(DiagnosticCategory::from(s))
}
}
impl From<i64> for DiagnosticCategory {
fn from(value: i64) -> Self {
match value {
@ -496,7 +364,7 @@ mod tests {
#[test]
fn from_json() {
let v = serde_json::from_str::<serde_json::Value>(
let r = serde_json::from_str::<Diagnostic>(
&r#"{
"items": [
{
@ -526,8 +394,7 @@ mod tests {
]
}"#,
).unwrap();
let r = Diagnostic::from_json_value(&v);
let expected = Some(
let expected =
Diagnostic {
items: vec![
DiagnosticItem {
@ -559,52 +426,10 @@ mod tests {
end_column: Some(1)
}
]
}
);
};
assert_eq!(expected, r);
}
#[test]
fn from_emit_result() {
let r = Diagnostic::from_emit_result(
&r#"{
"emitSkipped": false,
"diagnostics": {
"items": [
{
"message": "foo bar",
"code": 9999,
"category": 3
}
]
}
}"#,
);
let expected = Some(Diagnostic {
items: vec![DiagnosticItem {
message: "foo bar".to_string(),
message_chain: None,
related_information: None,
source_line: None,
line_number: None,
script_resource_name: None,
start_position: None,
end_position: None,
category: DiagnosticCategory::Error,
code: 9999,
start_column: None,
end_column: None,
}],
});
assert_eq!(expected, r);
}
#[test]
fn from_emit_result_none() {
let r = &r#"{"emitSkipped":false}"#;
assert!(Diagnostic::from_emit_result(r).is_none());
}
#[test]
fn diagnostic_to_string1() {
let d = diagnostic1();

87
cli/js/compiler.ts
View file

@ -29,7 +29,10 @@ import {
resolveModules,
} from "./compiler/imports.ts";
import {
createWriteFile,
EmmitedSource,
WriteFileCallback,
createCompileWriteFile,
createBundleWriteFile,
CompilerRequestType,
convertCompilerOptions,
ignoredDiagnostics,
@ -53,7 +56,6 @@ interface CompilerRequestCompile {
config?: string;
unstable: boolean;
bundle: boolean;
outFile?: string;
cwd: string;
}
@ -79,16 +81,20 @@ type CompilerRequest =
| CompilerRequestRuntimeTranspile;
interface CompileResult {
emitSkipped: boolean;
diagnostics?: Diagnostic;
emitMap?: Record<string, EmmitedSource>;
bundleOutput?: string;
diagnostics: Diagnostic;
}
type RuntimeCompileResult = [
undefined | DiagnosticItem[],
Record<string, string>
];
interface RuntimeCompileResult {
emitMap: Record<string, EmmitedSource>;
diagnostics: DiagnosticItem[];
}
type RuntimeBundleResult = [undefined | DiagnosticItem[], string];
interface RuntimeBundleResult {
output: string;
diagnostics: DiagnosticItem[];
}
async function compile(
request: CompilerRequestCompile
@ -97,7 +103,6 @@ async function compile(
bundle,
config,
configPath,
outFile,
rootNames,
target,
unstable,
@ -111,31 +116,32 @@ async function compile(
// When a programme is emitted, TypeScript will call `writeFile` with
// each file that needs to be emitted. The Deno compiler host delegates
// this, to make it easier to perform the right actions, which vary
// based a lot on the request. For a `Compile` request, we need to
// cache all the files in the privileged side if we aren't bundling,
// and if we are bundling we need to enrich the bundle and either write
// out the bundle or log it to the console.
// based a lot on the request.
const state: WriteFileState = {
type: request.type,
emitMap: {},
bundle,
host: undefined,
outFile,
rootNames,
};
const writeFile = createWriteFile(state);
let writeFile: WriteFileCallback;
if (bundle) {
writeFile = createBundleWriteFile(state);
} else {
writeFile = createCompileWriteFile(state);
}
const host = (state.host = new Host({
bundle,
target,
writeFile,
unstable,
}));
let diagnostics: readonly ts.Diagnostic[] | undefined;
let diagnostics: readonly ts.Diagnostic[] = [];
// if there is a configuration supplied, we need to parse that
if (config && config.length && configPath) {
const configResult = host.configure(cwd, configPath, config);
diagnostics = processConfigureResponse(configResult, configPath);
diagnostics = processConfigureResponse(configResult, configPath) || [];
}
// This will recursively analyse all the code for other imports,
@ -147,10 +153,9 @@ async function compile(
bundle || host.getCompilationSettings().checkJs
);
let emitSkipped = true;
// if there was a configuration and no diagnostics with it, we will continue
// to generate the program and possibly emit it.
if (!diagnostics || (diagnostics && diagnostics.length === 0)) {
if (diagnostics.length === 0) {
const options = host.getCompilationSettings();
const program = ts.createProgram({
rootNames,
@ -168,23 +173,28 @@ async function compile(
if (bundle) {
// we only support a single root module when bundling
assert(resolvedRootModules.length === 1);
// warning so it goes to stderr instead of stdout
console.warn(`Bundling "${resolvedRootModules[0]}"`);
setRootExports(program, resolvedRootModules[0]);
}
const emitResult = program.emit();
emitSkipped = emitResult.emitSkipped;
assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
// emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned
// without casting.
diagnostics = emitResult.diagnostics;
}
}
let bundleOutput = undefined;
if (bundle) {
assert(state.bundleOutput);
bundleOutput = state.bundleOutput;
}
assert(state.emitMap);
const result: CompileResult = {
emitSkipped,
diagnostics: diagnostics.length
? fromTypeScriptDiagnostic(diagnostics)
: undefined,
emitMap: state.emitMap,
bundleOutput,
diagnostics: fromTypeScriptDiagnostic(diagnostics),
};
util.log("<<< compile end", {
@ -259,9 +269,14 @@ async function runtimeCompile(
rootNames,
sources,
emitMap: {},
emitBundle: undefined,
bundleOutput: undefined,
};
const writeFile = createWriteFile(state);
let writeFile: WriteFileCallback;
if (bundle) {
writeFile = createBundleWriteFile(state);
} else {
writeFile = createCompileWriteFile(state);
}
const host = (state.host = new Host({
bundle,
@ -314,12 +329,18 @@ async function runtimeCompile(
const maybeDiagnostics = diagnostics.length
? fromTypeScriptDiagnostic(diagnostics).items
: undefined;
: [];
if (bundle) {
return [maybeDiagnostics, state.emitBundle] as RuntimeBundleResult;
return {
diagnostics: maybeDiagnostics,
output: state.bundleOutput,
} as RuntimeBundleResult;
} else {
return [maybeDiagnostics, state.emitMap] as RuntimeCompileResult;
return {
diagnostics: maybeDiagnostics,
emitMap: state.emitMap,
} as RuntimeCompileResult;
}
}

32
cli/js/compiler/api.ts
View file

@ -6,6 +6,8 @@
import { DiagnosticItem } from "../diagnostics.ts";
import * as util from "../util.ts";
import * as runtimeCompilerOps from "../ops/runtime_compiler.ts";
import { TranspileOnlyResult } from "../ops/runtime_compiler.ts";
export { TranspileOnlyResult } from "../ops/runtime_compiler.ts";
export interface CompilerOptions {
allowJs?: boolean;
@ -145,12 +147,8 @@ function checkRelative(specifier: string): string {
: `./${specifier}`;
}
export interface TranspileOnlyResult {
source: string;
map?: string;
}
export async function transpileOnly(
// TODO(bartlomieju): change return type to interface?
export function transpileOnly(
sources: Record<string, string>,
options: CompilerOptions = {}
): Promise<Record<string, TranspileOnlyResult>> {
@ -159,10 +157,10 @@ export async function transpileOnly(
sources,
options: JSON.stringify(options),
};
const result = await runtimeCompilerOps.transpile(payload);
return JSON.parse(result);
return runtimeCompilerOps.transpile(payload);
}
// TODO(bartlomieju): change return type to interface?
export async function compile(
rootName: string,
sources?: Record<string, string>,
@ -180,9 +178,20 @@ export async function compile(
options,
});
const result = await runtimeCompilerOps.compile(payload);
return JSON.parse(result);
util.assert(result.emitMap);
const maybeDiagnostics =
result.diagnostics.length === 0 ? undefined : result.diagnostics;
const emitMap: Record<string, string> = {};
for (const [key, emmitedSource] of Object.entries(result.emitMap)) {
emitMap[key] = emmitedSource.contents;
}
return [maybeDiagnostics, emitMap];
}
// TODO(bartlomieju): change return type to interface?
export async function bundle(
rootName: string,
sources?: Record<string, string>,
@ -200,5 +209,8 @@ export async function bundle(
options,
});
const result = await runtimeCompilerOps.compile(payload);
return JSON.parse(result);
util.assert(result.output);
const maybeDiagnostics =
result.diagnostics.length === 0 ? undefined : result.diagnostics;
return [maybeDiagnostics, result.output];
}

31
cli/js/compiler/host.ts
View file

@ -255,16 +255,12 @@ export class Host implements ts.CompilerHost {
assert(sourceFile != null);
if (!sourceFile.tsSourceFile) {
assert(sourceFile.sourceCode != null);
// even though we assert the extension for JSON modules to the compiler
// is TypeScript, TypeScript internally analyses the filename for its
// extension and tries to parse it as JSON instead of TS. We have to
// change the filename to the TypeScript file.
sourceFile.tsSourceFile = ts.createSourceFile(
fileName.startsWith(ASSETS)
const tsSourceFileName = fileName.startsWith(ASSETS)
? sourceFile.filename
: fileName.toLowerCase().endsWith(".json")
? `${fileName}.ts`
: fileName,
: fileName;
sourceFile.tsSourceFile = ts.createSourceFile(
tsSourceFileName,
sourceFile.sourceCode,
languageVersion
);
@ -294,15 +290,20 @@ export class Host implements ts.CompilerHost {
containingFile,
});
return moduleNames.map((specifier) => {
const url = SourceFile.getUrl(specifier, containingFile);
const sourceFile = specifier.startsWith(ASSETS)
? getAssetInternal(specifier)
: url
? SourceFile.get(url)
: undefined;
const maybeUrl = SourceFile.getUrl(specifier, containingFile);
let sourceFile: SourceFile | undefined = undefined;
if (specifier.startsWith(ASSETS)) {
sourceFile = getAssetInternal(specifier);
} else if (typeof maybeUrl !== "undefined") {
sourceFile = SourceFile.get(maybeUrl);
}
if (!sourceFile) {
return undefined;
}
return {
resolvedFileName: sourceFile.url,
isExternalLibraryImport: specifier.startsWith(ASSETS),

7
cli/js/compiler/imports.ts
View file

@ -122,11 +122,8 @@ export async function processImports(
SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson);
sourceFile.cache(specifiers[i][0], referrer);
if (!sourceFile.processed) {
await processImports(
sourceFile.imports(processJsImports),
sourceFile.url,
processJsImports
);
const sourceFileImports = sourceFile.imports(processJsImports);
await processImports(sourceFileImports, sourceFile.url, processJsImports);
}
}
return resolvedSources;

18
cli/js/compiler/sourcefile.ts
View file

@ -54,8 +54,6 @@ export class SourceFile {
extension!: ts.Extension;
filename!: string;
importedFiles?: Array<[string, string]>;
mediaType!: MediaType;
processed = false;
sourceCode?: string;
@ -93,14 +91,18 @@ export class SourceFile {
return [];
}
const readImportFiles = true;
const detectJsImports =
this.mediaType === MediaType.JavaScript ||
this.mediaType === MediaType.JSX;
const preProcessedFileInfo = ts.preProcessFile(
this.sourceCode,
true,
this.mediaType === MediaType.JavaScript ||
this.mediaType === MediaType.JSX
readImportFiles,
detectJsImports
);
this.processed = true;
const files = (this.importedFiles = [] as Array<[string, string]>);
const files: Array<[string, string]> = [];
function process(references: Array<{ fileName: string }>): void {
for (const { fileName } of references) {
@ -160,8 +162,4 @@ export class SourceFile {
static get(url: string): SourceFile | undefined {
return moduleCache.get(url);
}
static has(url: string): boolean {
return moduleCache.has(url);
}
}

135
cli/js/compiler/util.ts
View file

@ -4,12 +4,16 @@ import { bold, cyan, yellow } from "../colors.ts";
import { CompilerOptions } from "./api.ts";
import { buildBundle } from "./bundler.ts";
import { ConfigureResponse, Host } from "./host.ts";
import { MediaType, SourceFile } from "./sourcefile.ts";
import { atob, TextEncoder } from "../web/text_encoding.ts";
import { atob } from "../web/text_encoding.ts";
import * as compilerOps from "../ops/compiler.ts";
import * as util from "../util.ts";
import { assert } from "../util.ts";
import { writeFileSync } from "../write_file.ts";
export interface EmmitedSource {
// original filename
filename: string;
// compiled contents
contents: string;
}
export type WriteFileCallback = (
fileName: string,
@ -20,11 +24,10 @@ export type WriteFileCallback = (
export interface WriteFileState {
type: CompilerRequestType;
bundle?: boolean;
bundleOutput?: string;
host?: Host;
outFile?: string;
rootNames: string[];
emitMap?: Record<string, string>;
emitBundle?: string;
emitMap?: Record<string, EmmitedSource>;
sources?: Record<string, string>;
}
@ -38,87 +41,33 @@ export enum CompilerRequestType {
export const OUT_DIR = "$deno$";
function cache(
moduleId: string,
emittedFileName: string,
contents: string,
checkJs = false
): void {
util.log("compiler::cache", { moduleId, emittedFileName, checkJs });
const sf = SourceFile.get(moduleId);
if (sf) {
// NOTE: JavaScript files are only cached to disk if `checkJs`
// option in on
if (sf.mediaType === MediaType.JavaScript && !checkJs) {
return;
}
}
if (emittedFileName.endsWith(".map")) {
// Source Map
compilerOps.cache(".map", moduleId, contents);
} else if (emittedFileName.endsWith(".js")) {
// Compiled JavaScript
compilerOps.cache(".js", moduleId, contents);
} else {
assert(false, `Trying to cache unhandled file type "${emittedFileName}"`);
}
}
export function getAsset(name: string): string {
return compilerOps.getAsset(name);
}
export function createWriteFile(state: WriteFileState): WriteFileCallback {
const encoder = new TextEncoder();
if (state.type === CompilerRequestType.Compile) {
// TODO(bartlomieju): probably could be defined inline?
export function createBundleWriteFile(
state: WriteFileState
): WriteFileCallback {
return function writeFile(
fileName: string,
_fileName: string,
data: string,
sourceFiles?: readonly ts.SourceFile[]
): void {
assert(
sourceFiles != null,
`Unexpected emit of "${fileName}" which isn't part of a program.`
);
assert(sourceFiles != null);
assert(state.host);
if (!state.bundle) {
assert(sourceFiles.length === 1);
cache(
sourceFiles[0].fileName,
fileName,
data,
state.host.getCompilationSettings().checkJs
);
} else {
// if the fileName is set to an internal value, just noop, this is
// used in the Rust unit tests.
if (state.outFile && state.outFile.startsWith(OUT_DIR)) {
return;
}
assert(state.emitMap);
assert(state.bundle);
// we only support single root names for bundles
assert(
state.rootNames.length === 1,
`Only one root name supported. Got "${JSON.stringify(
state.rootNames
)}"`
);
// this enriches the string with the loader and re-exports the
// exports of the root module
const content = buildBundle(state.rootNames[0], data, sourceFiles);
if (state.outFile) {
const encodedData = encoder.encode(content);
console.warn(`Emitting bundle to "${state.outFile}"`);
writeFileSync(state.outFile, encodedData);
console.warn(`${humanFileSize(encodedData.length)} emitted.`);
} else {
console.log(content);
}
}
assert(state.rootNames.length === 1);
state.bundleOutput = buildBundle(state.rootNames[0], data, sourceFiles);
};
}
// TODO(bartlomieju): probably could be defined inline?
export function createCompileWriteFile(
state: WriteFileState
): WriteFileCallback {
return function writeFile(
fileName: string,
data: string,
@ -127,24 +76,12 @@ export function createWriteFile(state: WriteFileState): WriteFileCallback {
assert(sourceFiles != null);
assert(state.host);
assert(state.emitMap);
if (!state.bundle) {
assert(!state.bundle);
assert(sourceFiles.length === 1);
state.emitMap[fileName] = data;
// we only want to cache the compiler output if we are resolving
// modules externally
if (!state.sources) {
cache(
sourceFiles[0].fileName,
fileName,
data,
state.host.getCompilationSettings().checkJs
);
}
} else {
// we only support single root names for bundles
assert(state.rootNames.length === 1);
state.emitBundle = buildBundle(state.rootNames[0], data, sourceFiles);
}
state.emitMap[fileName] = {
filename: sourceFiles[0].fileName,
contents: data,
};
};
}
@ -380,20 +317,6 @@ export function commonPath(paths: string[], sep = "/"): string {
return prefix.endsWith(sep) ? prefix : `${prefix}${sep}`;
}
function humanFileSize(bytes: number): string {
const thresh = 1000;
if (Math.abs(bytes) < thresh) {
return bytes + " B";
}
const units = ["kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
let u = -1;
do {
bytes /= thresh;
++u;
} while (Math.abs(bytes) >= thresh && u < units.length - 1);
return `${bytes.toFixed(1)} ${units[u]}`;
}
// @internal
export function base64ToUint8Array(data: string): Uint8Array {
const binString = atob(data);

12
cli/js/ops/compiler.ts
View file

@ -38,15 +38,3 @@ export function getAsset(name: string): string {
const sourceCodeBytes = core.dispatch(opId, encoder.encode(name));
return decoder.decode(sourceCodeBytes!);
}
export function cache(
extension: string,
moduleId: string,
contents: string
): void {
sendSync("op_cache", {
extension,
moduleId,
contents,
});
}

18
cli/js/ops/runtime_compiler.ts
View file

@ -1,6 +1,7 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import { sendAsync } from "./dispatch_json.ts";
import { DiagnosticItem } from "../diagnostics.ts";
interface CompileRequest {
rootName: string;
@ -9,7 +10,13 @@ interface CompileRequest {
bundle: boolean;
}
export function compile(request: CompileRequest): Promise<string> {
interface CompileResponse {
diagnostics: DiagnosticItem[];
output?: string;
emitMap?: Record<string, Record<string, string>>;
}
export function compile(request: CompileRequest): Promise<CompileResponse> {
return sendAsync("op_compile", request);
}
@ -18,6 +25,13 @@ interface TranspileRequest {
options?: string;
}
export function transpile(request: TranspileRequest): Promise<string> {
export interface TranspileOnlyResult {
source: string;
map?: string;
}
export function transpile(
request: TranspileRequest
): Promise<Record<string, TranspileOnlyResult>> {
return sendAsync("op_transpile", request);
}

2
cli/js/tests/format_error_test.ts
View file

@ -26,7 +26,7 @@ unitTest(function formatDiagnosticError() {
try {
Deno.formatDiagnostics(bad);
} catch (e) {
assert(e instanceof TypeError);
assert(e instanceof Deno.errors.InvalidData);
thrown = true;
}
assert(thrown);

30
cli/ops/compiler.rs
View file

@ -13,7 +13,6 @@ use deno_core::ZeroCopyBuf;
use futures::future::FutureExt;
pub fn init(i: &mut CoreIsolate, s: &State) {
i.register_op("op_cache", s.stateful_json_op(op_cache));
i.register_op("op_resolve_modules", s.stateful_json_op(op_resolve_modules));
i.register_op(
"op_fetch_source_files",
@ -26,35 +25,6 @@ pub fn init(i: &mut CoreIsolate, s: &State) {
);
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct CacheArgs {
module_id: String,
contents: String,
extension: String,
}
fn op_cache(
state: &State,
args: Value,
_zero_copy: Option<ZeroCopyBuf>,
) -> Result<JsonOp, OpError> {
let args: CacheArgs = serde_json::from_value(args)?;
let module_specifier = ModuleSpecifier::resolve_url(&args.module_id)
.expect("Should be valid module specifier");
let state_ = &state.borrow().global_state;
let ts_compiler = state_.ts_compiler.clone();
ts_compiler.cache_compiler_output(
&module_specifier,
&args.extension,
&args.contents,
)?;
Ok(JsonOp::Sync(json!({})))
}
#[derive(Deserialize, Debug)]
struct SpecifiersReferrerArgs {
specifiers: Vec<String>,

5
cli/ops/errors.rs
View file

@ -57,9 +57,6 @@ fn op_format_diagnostic(
args: Value,
_zero_copy: Option<ZeroCopyBuf>,
) -> Result<JsonOp, OpError> {
if let Some(diagnostic) = Diagnostic::from_json_value(&args) {
let diagnostic = serde_json::from_value::<Diagnostic>(args)?;
Ok(JsonOp::Sync(json!(diagnostic.to_string())))
} else {
Err(OpError::type_error("bad diagnostic".to_string()))
}
}

24
cli/ops/runtime_compiler.rs
View file

@ -2,6 +2,7 @@
use super::dispatch_json::{Deserialize, JsonOp, Value};
use crate::compilers::runtime_compile;
use crate::compilers::runtime_transpile;
use crate::futures::FutureExt;
use crate::op_error::OpError;
use crate::state::State;
use deno_core::CoreIsolate;
@ -29,13 +30,19 @@ fn op_compile(
) -> Result<JsonOp, OpError> {
state.check_unstable("Deno.compile");
let args: CompileArgs = serde_json::from_value(args)?;
Ok(JsonOp::Async(runtime_compile(
state.borrow().global_state.clone(),
let global_state = state.borrow().global_state.clone();
let fut = async move {
runtime_compile(
global_state,
&args.root_name,
&args.sources,
args.bundle,
&args.options,
)))
)
.await
}
.boxed_local();
Ok(JsonOp::Async(fut))
}
#[derive(Deserialize, Debug)]
@ -51,9 +58,10 @@ fn op_transpile(
) -> Result<JsonOp, OpError> {
state.check_unstable("Deno.transpile");
let args: TranspileArgs = serde_json::from_value(args)?;
Ok(JsonOp::Async(runtime_transpile(
state.borrow().global_state.clone(),
&args.sources,
&args.options,
)))
let global_state = state.borrow().global_state.clone();
let fut = async move {
runtime_transpile(global_state, &args.sources, &args.options).await
}
.boxed_local();
Ok(JsonOp::Async(fut))
}

2
cli/tests/lib_ref.ts.out
View file

@ -1,2 +1,2 @@
null
undefined
[ "main.js.map", "main.js" ]

2
cli/tests/lib_runtime_api.ts.out
View file

@ -1,2 +1,2 @@
null
undefined
[ "main.js.map", "main.js" ]