foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-21 15:04:11 -05:00
Code Issues Packages 1 Wiki Activity

Runtime Compiler API (#3442)

Browse source 
Also restructures the compiler TypeScript files to make them easier to
manage and eventually integrate deno_typescript fully.
This commit is contained in:
Kitson Kelly 2020-01-09 01:17:44 +11:00 committed by Ry Dahl
parent cbdf9c5009
commit d325566a7e
26 changed files with 2704 additions and 877 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cli/compilers/mod.rs
View file

@ -1,6 +1,7 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
use deno_core::ErrBox;
use futures::Future;
use serde_json::Value;
mod js;
mod json;
@ -9,9 +10,14 @@ mod wasm;
pub use js::JsCompiler;
pub use json::JsonCompiler;
pub use ts::runtime_compile_async;
pub use ts::runtime_transpile_async;
pub use ts::TsCompiler;
pub use wasm::WasmCompiler;
pub type CompilationResultFuture =
dyn Future<Output = Result<Value, ErrBox>> + Send;
#[derive(Debug, Clone)]
pub struct CompiledModule {
pub code: String,

71
cli/compilers/ts.rs
View file

@ -1,4 +1,5 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
use crate::compilers::CompilationResultFuture;
use crate::compilers::CompiledModule;
use crate::compilers::CompiledModuleFuture;
use crate::diagnostics::Diagnostic;
@ -7,6 +8,7 @@ use crate::file_fetcher::SourceFile;
use crate::file_fetcher::SourceFileFetcher;
use crate::global_state::ThreadSafeGlobalState;
use crate::msg;
use crate::serde_json::json;
use crate::source_maps::SourceMapGetter;
use crate::startup_data;
use crate::state::*;
@ -18,8 +20,10 @@ use deno_core::ModuleSpecifier;
use futures::future::FutureExt;
use futures::Future;
use regex::Regex;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fs;
use std::hash::BuildHasher;
use std::io;
use std::path::PathBuf;
use std::pin::Pin;
@ -156,12 +160,14 @@ fn req(
root_names: Vec<String>,
compiler_config: CompilerConfig,
out_file: Option<String>,
bundle: bool,
) -> Buf {
let j = match (compiler_config.path, compiler_config.content) {
(Some(config_path), Some(config_data)) => json!({
"type": request_type as i32,
"rootNames": root_names,
"outFile": out_file,
"bundle": bundle,
"configPath": config_path,
"config": str::from_utf8(&config_data).unwrap(),
}),
@ -169,6 +175,7 @@ fn req(
"type": request_type as i32,
"rootNames": root_names,
"outFile": out_file,
"bundle": bundle,
}),
};
@ -258,10 +265,11 @@ impl TsCompiler {
let root_names = vec![module_name];
let req_msg = req(
msg::CompilerRequestType::Bundle,
msg::CompilerRequestType::Compile,
root_names,
self.config.clone(),
out_file,
true,
);
let worker = TsCompiler::setup_worker(global_state);
@ -356,6 +364,7 @@ impl TsCompiler {
root_names,
self.config.clone(),
None,
false,
);
let worker = TsCompiler::setup_worker(global_state.clone());
@ -599,6 +608,66 @@ impl TsCompiler {
}
}
pub fn runtime_compile_async<S: BuildHasher>(
global_state: ThreadSafeGlobalState,
root_name: &str,
sources: &Option<HashMap<String, String, S>>,
bundle: bool,
options: &Option<String>,
) -> Pin<Box<CompilationResultFuture>> {
let req_msg = json!({
"type": msg::CompilerRequestType::RuntimeCompile as i32,
"rootName": root_name,
"sources": sources,
"options": options,
"bundle": bundle,
})
.to_string()
.into_boxed_str()
.into_boxed_bytes();
let worker = TsCompiler::setup_worker(global_state);
let worker_ = worker.clone();
async move {
worker.post_message(req_msg).await?;
worker.await?;
debug!("Sent message to worker");
let msg = (worker_.get_message().await?).unwrap();
let json_str = std::str::from_utf8(&msg).unwrap();
Ok(json!(json_str))
}
.boxed()
}
pub fn runtime_transpile_async<S: BuildHasher>(
global_state: ThreadSafeGlobalState,
sources: &HashMap<String, String, S>,
options: &Option<String>,
) -> Pin<Box<CompilationResultFuture>> {
let req_msg = json!({
"type": msg::CompilerRequestType::RuntimeTranspile as i32,
"sources": sources,
"options": options,
})
.to_string()
.into_boxed_str()
.into_boxed_bytes();
let worker = TsCompiler::setup_worker(global_state);
let worker_ = worker.clone();
async move {
worker.post_message(req_msg).await?;
worker.await?;
debug!("Sent message to worker");
let msg = (worker_.get_message().await?).unwrap();
let json_str = std::str::from_utf8(&msg).unwrap();
Ok(json!(json_str))
}
.boxed()
}
#[cfg(test)]
mod tests {
use super::*;

909
cli/js/compiler.ts
View file

@ -1,711 +1,301 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
// TODO(ry) Combine this implementation with //deno_typescript/compiler_main.js
// these are imported for their side effects
import "./globals.ts";
import "./ts_global.d.ts";
import { emitBundle, setRootExports } from "./bundler.ts";
import { bold, cyan, yellow } from "./colors.ts";
import { Console } from "./console.ts";
import { core } from "./core.ts";
import { Diagnostic, fromTypeScriptDiagnostic } from "./diagnostics.ts";
import { cwd } from "./dir.ts";
import * as dispatch from "./dispatch.ts";
import { sendAsync, sendSync } from "./dispatch_json.ts";
import { TranspileOnlyResult } from "./compiler_api.ts";
import { setRootExports } from "./compiler_bundler.ts";
import {
defaultBundlerOptions,
defaultRuntimeCompileOptions,
defaultTranspileOptions,
Host
} from "./compiler_host.ts";
import {
processImports,
processLocalImports,
resolveModules
} from "./compiler_imports.ts";
import {
createWriteFile,
CompilerRequestType,
convertCompilerOptions,
ignoredDiagnostics,
WriteFileState,
processConfigureResponse
} from "./compiler_util.ts";
import { Diagnostic } from "./diagnostics.ts";
import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts";
import * as os from "./os.ts";
import { getMappedModuleName, parseTypeDirectives } from "./type_directives.ts";
import { assert, notImplemented } from "./util.ts";
import { assert } from "./util.ts";
import * as util from "./util.ts";
import { window } from "./window.ts";
import { window as self } from "./window.ts";
import { postMessage, workerClose, workerMain } from "./workers.ts";
// Warning! The values in this enum are duplicated in cli/msg.rs
// Update carefully!
enum MediaType {
JavaScript = 0,
JSX = 1,
TypeScript = 2,
TSX = 3,
Json = 4,
Wasm = 5,
Unknown = 6
}
// Warning! The values in this enum are duplicated in cli/msg.rs
// Update carefully!
enum CompilerRequestType {
Compile = 0,
Bundle = 1
}
// Startup boilerplate. This is necessary because the compiler has its own
// snapshot. (It would be great if we could remove these things or centralize
// them somewhere else.)
const console = new Console(core.print);
window.console = console;
window.workerMain = workerMain;
function denoMain(compilerType?: string): void {
os.start(true, compilerType || "TS");
}
window["denoMain"] = denoMain;
const ASSETS = "$asset$";
const OUT_DIR = "$deno$";
/** The format of the work message payload coming from the privileged side */
type CompilerRequest = {
interface CompilerRequestCompile {
type: CompilerRequestType.Compile;
rootNames: string[];
// TODO(ry) add compiler config to this interface.
// options: ts.CompilerOptions;
configPath?: string;
config?: string;
} & (
| {
type: CompilerRequestType.Compile;
}
| {
type: CompilerRequestType.Bundle;
outFile?: string;
}
);
interface ConfigureResponse {
ignoredOptions?: string[];
diagnostics?: ts.Diagnostic[];
bundle?: boolean;
outFile?: string;
}
/** Options that either do nothing in Deno, or would cause undesired behavior
* if modified. */
const ignoredCompilerOptions: readonly string[] = [
"allowSyntheticDefaultImports",
"baseUrl",
"build",
"composite",
"declaration",
"declarationDir",
"declarationMap",
"diagnostics",
"downlevelIteration",
"emitBOM",
"emitDeclarationOnly",
"esModuleInterop",
"extendedDiagnostics",
"forceConsistentCasingInFileNames",
"help",
"importHelpers",
"incremental",
"inlineSourceMap",
"inlineSources",
"init",
"isolatedModules",
"lib",
"listEmittedFiles",
"listFiles",
"mapRoot",
"maxNodeModuleJsDepth",
"module",
"moduleResolution",
"newLine",
"noEmit",
"noEmitHelpers",
"noEmitOnError",
"noLib",
"noResolve",
"out",
"outDir",
"outFile",
"paths",
"preserveSymlinks",
"preserveWatchOutput",
"pretty",
"rootDir",
"rootDirs",
"showConfig",
"skipDefaultLibCheck",
"skipLibCheck",
"sourceMap",
"sourceRoot",
"stripInternal",
"target",
"traceResolution",
"tsBuildInfoFile",
"types",
"typeRoots",
"version",
"watch"
];
/** The shape of the SourceFile that comes from the privileged side */
interface SourceFileJson {
url: string;
filename: string;
mediaType: MediaType;
sourceCode: string;
interface CompilerRequestRuntimeCompile {
type: CompilerRequestType.RuntimeCompile;
rootName: string;
sources?: Record<string, string>;
bundle?: boolean;
options?: string;
}
/** A self registering abstraction of source files. */
class SourceFile {
extension!: ts.Extension;
filename!: string;
/** An array of tuples which represent the imports for the source file. The
* first element is the one that will be requested at compile time, the
* second is the one that should be actually resolved. This provides the
* feature of type directives for Deno. */
importedFiles?: Array<[string, string]>;
mediaType!: MediaType;
processed = false;
sourceCode!: string;
tsSourceFile?: ts.SourceFile;
url!: string;
constructor(json: SourceFileJson) {
if (SourceFile._moduleCache.has(json.url)) {
throw new TypeError("SourceFile already exists");
}
Object.assign(this, json);
this.extension = getExtension(this.url, this.mediaType);
SourceFile._moduleCache.set(this.url, this);
}
/** Cache the source file to be able to be retrieved by `moduleSpecifier` and
* `containingFile`. */
cache(moduleSpecifier: string, containingFile?: string): void {
containingFile = containingFile || "";
let innerCache = SourceFile._specifierCache.get(containingFile);
if (!innerCache) {
innerCache = new Map();
SourceFile._specifierCache.set(containingFile, innerCache);
}
innerCache.set(moduleSpecifier, this);
}
/** Process the imports for the file and return them. */
imports(): Array<[string, string]> {
if (this.processed) {
throw new Error("SourceFile has already been processed.");
}
assert(this.sourceCode != null);
// we shouldn't process imports for files which contain the nocheck pragma
// (like bundles)
if (this.sourceCode.match(/\/{2}\s+@ts-nocheck/)) {
util.log(`Skipping imports for "${this.filename}"`);
return [];
}
const preProcessedFileInfo = ts.preProcessFile(this.sourceCode, true, true);
this.processed = true;
const files = (this.importedFiles = [] as Array<[string, string]>);
function process(references: ts.FileReference[]): void {
for (const { fileName } of references) {
files.push([fileName, fileName]);
}
}
const {
importedFiles,
referencedFiles,
libReferenceDirectives,
typeReferenceDirectives
} = preProcessedFileInfo;
const typeDirectives = parseTypeDirectives(this.sourceCode);
if (typeDirectives) {
for (const importedFile of importedFiles) {
files.push([
importedFile.fileName,
getMappedModuleName(importedFile, typeDirectives)
]);
}
} else {
process(importedFiles);
}
process(referencedFiles);
process(libReferenceDirectives);
process(typeReferenceDirectives);
return files;
}
/** A cache of all the source files which have been loaded indexed by the
* url. */
private static _moduleCache: Map<string, SourceFile> = new Map();
/** A cache of source files based on module specifiers and containing files
* which is used by the TypeScript compiler to resolve the url */
private static _specifierCache: Map<
string,
Map<string, SourceFile>
> = new Map();
/** Retrieve a `SourceFile` based on a `moduleSpecifier` and `containingFile`
* or return `undefined` if not preset. */
static getUrl(
moduleSpecifier: string,
containingFile: string
): string | undefined {
const containingCache = this._specifierCache.get(containingFile);
if (containingCache) {
const sourceFile = containingCache.get(moduleSpecifier);
return sourceFile && sourceFile.url;
}
return undefined;
}
/** Retrieve a `SourceFile` based on a `url` */
static get(url: string): SourceFile | undefined {
return this._moduleCache.get(url);
}
interface CompilerRequestRuntimeTranspile {
type: CompilerRequestType.RuntimeTranspile;
sources: Record<string, string>;
options?: string;
}
interface EmitResult {
/** The format of the work message payload coming from the privileged side */
type CompilerRequest =
| CompilerRequestCompile
| CompilerRequestRuntimeCompile
| CompilerRequestRuntimeTranspile;
/** The format of the result sent back when doing a compilation. */
interface CompileResult {
emitSkipped: boolean;
diagnostics?: Diagnostic;
}
/** Ops to Rust to resolve special static assets. */
function fetchAsset(name: string): string {
return sendSync(dispatch.OP_FETCH_ASSET, { name });
}
// bootstrap the runtime environment, this gets called as the isolate is setup
self.denoMain = function denoMain(compilerType?: string): void {
os.start(true, compilerType || "TS");
};
/** Ops to Rust to resolve and fetch modules meta data. */
function fetchSourceFiles(
specifiers: string[],
referrer?: string
): Promise<SourceFileJson[]> {
util.log("compiler::fetchSourceFiles", { specifiers, referrer });
return sendAsync(dispatch.OP_FETCH_SOURCE_FILES, {
specifiers,
referrer
});
}
/** Recursively process the imports of modules, generating `SourceFile`s of any
* imported files.
*
* Specifiers are supplied in an array of tupples where the first is the
* specifier that will be requested in the code and the second is the specifier
* that should be actually resolved. */
async function processImports(
specifiers: Array<[string, string]>,
referrer?: string
): Promise<SourceFileJson[]> {
if (!specifiers.length) {
return [];
}
const sources = specifiers.map(([, moduleSpecifier]) => moduleSpecifier);
const sourceFiles = await fetchSourceFiles(sources, referrer);
assert(sourceFiles.length === specifiers.length);
for (let i = 0; i < sourceFiles.length; i++) {
const sourceFileJson = sourceFiles[i];
const sourceFile =
SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson);
sourceFile.cache(specifiers[i][0], referrer);
if (!sourceFile.processed) {
await processImports(sourceFile.imports(), sourceFile.url);
}
}
return sourceFiles;
}
/** Ops to rest for caching source map and compiled js */
function cache(extension: string, moduleId: string, contents: string): void {
util.log("compiler::cache", { extension, moduleId });
sendSync(dispatch.OP_CACHE, { extension, moduleId, contents });
}
/** Returns the TypeScript Extension enum for a given media type. */
function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
switch (mediaType) {
case MediaType.JavaScript:
return ts.Extension.Js;
case MediaType.JSX:
return ts.Extension.Jsx;
case MediaType.TypeScript:
return fileName.endsWith(".d.ts") ? ts.Extension.Dts : ts.Extension.Ts;
case MediaType.TSX:
return ts.Extension.Tsx;
case MediaType.Json:
return ts.Extension.Json;
case MediaType.Wasm:
// Custom marker for Wasm type.
return ts.Extension.Js;
case MediaType.Unknown:
default:
throw TypeError("Cannot resolve extension.");
}
}
class Host implements ts.CompilerHost {
private readonly _options: ts.CompilerOptions = {
allowJs: true,
allowNonTsExtensions: true,
// TODO(#3324) Enable strict mode for user code.
// strict: true,
checkJs: false,
esModuleInterop: true,
module: ts.ModuleKind.ESNext,
outDir: OUT_DIR,
resolveJsonModule: true,
sourceMap: true,
stripComments: true,
target: ts.ScriptTarget.ESNext,
jsx: ts.JsxEmit.React
};
private _getAsset(filename: string): SourceFile {
const sourceFile = SourceFile.get(filename);
if (sourceFile) {
return sourceFile;
}
const url = filename.split("/").pop()!;
const assetName = url.includes(".") ? url : `${url}.d.ts`;
const sourceCode = fetchAsset(assetName);
return new SourceFile({
url,
filename,
mediaType: MediaType.TypeScript,
sourceCode
});
}
/* Deno specific APIs */
/** Provides the `ts.HostCompiler` interface for Deno.
*
* @param _rootNames A set of modules that are the ones that should be
* instantiated first. Used when generating a bundle.
* @param _bundle Set to a string value to configure the host to write out a
* bundle instead of caching individual files.
*/
constructor(
private _requestType: CompilerRequestType,
private _rootNames: string[],
private _outFile?: string
) {
if (this._requestType === CompilerRequestType.Bundle) {
// options we need to change when we are generating a bundle
const bundlerOptions: ts.CompilerOptions = {
module: ts.ModuleKind.AMD,
outDir: undefined,
outFile: `${OUT_DIR}/bundle.js`,
// disabled until we have effective way to modify source maps
sourceMap: false
};
Object.assign(this._options, bundlerOptions);
}
}
/** Take a configuration string, parse it, and use it to merge with the
* compiler's configuration options. The method returns an array of compiler
* options which were ignored, or `undefined`. */
configure(path: string, configurationText: string): ConfigureResponse {
util.log("compiler::host.configure", path);
const { config, error } = ts.parseConfigFileTextToJson(
path,
configurationText
);
if (error) {
return { diagnostics: [error] };
}
const { options, errors } = ts.convertCompilerOptionsFromJson(
config.compilerOptions,
cwd()
);
const ignoredOptions: string[] = [];
for (const key of Object.keys(options)) {
if (
ignoredCompilerOptions.includes(key) &&
(!(key in this._options) || options[key] !== this._options[key])
) {
ignoredOptions.push(key);
delete options[key];
}
}
Object.assign(this._options, options);
return {
ignoredOptions: ignoredOptions.length ? ignoredOptions : undefined,
diagnostics: errors.length ? errors : undefined
};
}
/* TypeScript CompilerHost APIs */
fileExists(_fileName: string): boolean {
return notImplemented();
}
getCanonicalFileName(fileName: string): string {
return fileName;
}
getCompilationSettings(): ts.CompilerOptions {
util.log("compiler::host.getCompilationSettings()");
return this._options;
}
getCurrentDirectory(): string {
return "";
}
getDefaultLibFileName(_options: ts.CompilerOptions): string {
return ASSETS + "/lib.deno_runtime.d.ts";
}
getNewLine(): string {
return "\n";
}
getSourceFile(
fileName: string,
languageVersion: ts.ScriptTarget,
onError?: (message: string) => void,
shouldCreateNewSourceFile?: boolean
): ts.SourceFile | undefined {
util.log("compiler::host.getSourceFile", fileName);
try {
assert(!shouldCreateNewSourceFile);
const sourceFile = fileName.startsWith(ASSETS)
? this._getAsset(fileName)
: SourceFile.get(fileName);
assert(sourceFile != null);
if (!sourceFile.tsSourceFile) {
sourceFile.tsSourceFile = ts.createSourceFile(
fileName,
sourceFile.sourceCode,
languageVersion
);
}
return sourceFile!.tsSourceFile;
} catch (e) {
if (onError) {
onError(String(e));
} else {
throw e;
}
return undefined;
}
}
readFile(_fileName: string): string | undefined {
return notImplemented();
}
resolveModuleNames(
moduleNames: string[],
containingFile: string
): Array<ts.ResolvedModuleFull | undefined> {
util.log("compiler::host.resolveModuleNames", {
moduleNames,
containingFile
});
return moduleNames.map(specifier => {
const url = SourceFile.getUrl(specifier, containingFile);
const sourceFile = specifier.startsWith(ASSETS)
? this._getAsset(specifier)
: url
? SourceFile.get(url)
: undefined;
if (!sourceFile) {
return undefined;
}
return {
resolvedFileName: sourceFile.url,
isExternalLibraryImport: specifier.startsWith(ASSETS),
extension: sourceFile.extension
};
});
}
useCaseSensitiveFileNames(): boolean {
return true;
}
writeFile(
fileName: string,
data: string,
_writeByteOrderMark: boolean,
onError?: (message: string) => void,
sourceFiles?: readonly ts.SourceFile[]
): void {
util.log("compiler::host.writeFile", fileName);
try {
assert(sourceFiles != null);
if (this._requestType === CompilerRequestType.Bundle) {
emitBundle(this._rootNames, this._outFile, data, sourceFiles);
} else {
assert(sourceFiles.length == 1);
const url = sourceFiles[0].fileName;
const sourceFile = SourceFile.get(url);
if (sourceFile) {
// NOTE: If it's a `.json` file we don't want to write it to disk.
// JSON files are loaded and used by TS compiler to check types, but we don't want
// to emit them to disk because output file is the same as input file.
if (sourceFile.extension === ts.Extension.Json) {
return;
}
// NOTE: JavaScript files are only emitted to disk if `checkJs` option in on
if (
sourceFile.extension === ts.Extension.Js &&
!this._options.checkJs
) {
return;
}
}
if (fileName.endsWith(".map")) {
// Source Map
cache(".map", url, data);
} else if (fileName.endsWith(".js") || fileName.endsWith(".json")) {
// Compiled JavaScript
cache(".js", url, data);
} else {
assert(false, "Trying to cache unhandled file type " + fileName);
}
}
} catch (e) {
if (onError) {
onError(String(e));
} else {
throw e;
}
}
}
}
// bootstrap the worker environment, this gets called as the isolate is setup
self.workerMain = workerMain;
// provide the "main" function that will be called by the privileged side when
// lazy instantiating the compiler web worker
window.compilerMain = function compilerMain(): void {
self.compilerMain = function compilerMain(): void {
// workerMain should have already been called since a compiler is a worker.
window.onmessage = async ({
self.onmessage = async ({
data: request
}: {
data: CompilerRequest;
}): Promise<void> => {
const { rootNames, configPath, config } = request;
util.log(">>> compile start", {
rootNames,
type: CompilerRequestType[request.type]
});
// This will recursively analyse all the code for other imports, requesting
// those from the privileged side, populating the in memory cache which
// will be used by the host, before resolving.
const resolvedRootModules = (
await processImports(rootNames.map(rootName => [rootName, rootName]))
).map(info => info.url);
const host = new Host(
request.type,
resolvedRootModules,
request.type === CompilerRequestType.Bundle ? request.outFile : undefined
);
let emitSkipped = true;
let diagnostics: ts.Diagnostic[] | undefined;
// if there is a configuration supplied, we need to parse that
if (config && config.length && configPath) {
const configResult = host.configure(configPath, config);
const ignoredOptions = configResult.ignoredOptions;
diagnostics = configResult.diagnostics;
if (ignoredOptions) {
console.warn(
yellow(`Unsupported compiler options in "${configPath}"\n`) +
cyan(` The following options were ignored:\n`) +
` ${ignoredOptions
.map((value): string => bold(value))
.join(", ")}`
);
}
}
// if there was a configuration and no diagnostics with it, we will continue
// to generate the program and possibly emit it.
if (!diagnostics || (diagnostics && diagnostics.length === 0)) {
const options = host.getCompilationSettings();
const program = ts.createProgram(rootNames, options, host);
diagnostics = ts
.getPreEmitDiagnostics(program)
.filter(({ code }): boolean => {
// TS1103: 'for-await-of' statement is only allowed within an async
// function or async generator.
if (code === 1103) return false;
// TS1308: 'await' expression is only allowed within an async
// function.
if (code === 1308) return false;
// TS2691: An import path cannot end with a '.ts' extension. Consider
// importing 'bad-module' instead.
if (code === 2691) return false;
// TS5009: Cannot find the common subdirectory path for the input files.
if (code === 5009) return false;
// TS5055: Cannot write file
// 'http://localhost:4545/tests/subdir/mt_application_x_javascript.j4.js'
// because it would overwrite input file.
if (code === 5055) return false;
// TypeScript is overly opinionated that only CommonJS modules kinds can
// support JSON imports. Allegedly this was fixed in
// Microsoft/TypeScript#26825 but that doesn't seem to be working here,
// so we will ignore complaints about this compiler setting.
if (code === 5070) return false;
return true;
switch (request.type) {
// `Compile` are requests from the internals to Deno, generated by both
// the `run` and `bundle` sub command.
case CompilerRequestType.Compile: {
const { bundle, config, configPath, outFile, rootNames } = request;
util.log(">>> compile start", {
rootNames,
type: CompilerRequestType[request.type]
});
// We will only proceed with the emit if there are no diagnostics.
if (diagnostics && diagnostics.length === 0) {
if (request.type === CompilerRequestType.Bundle) {
// warning so it goes to stderr instead of stdout
console.warn(`Bundling "${resolvedRootModules.join(`", "`)}"`);
// This will recursively analyse all the code for other imports,
// requesting those from the privileged side, populating the in memory
// cache which will be used by the host, before resolving.
const resolvedRootModules = await processImports(
rootNames.map(rootName => [rootName, rootName])
);
// When a programme is emitted, TypeScript will call `writeFile` with
// each file that needs to be emitted. The Deno compiler host delegates
// this, to make it easier to perform the right actions, which vary
// based a lot on the request. For a `Compile` request, we need to
// cache all the files in the privileged side if we aren't bundling,
// and if we are bundling we need to enrich the bundle and either write
// out the bundle or log it to the console.
const state: WriteFileState = {
type: request.type,
bundle,
host: undefined,
outFile,
rootNames
};
const writeFile = createWriteFile(state);
const host = (state.host = new Host({ bundle, writeFile }));
let diagnostics: readonly ts.Diagnostic[] | undefined;
// if there is a configuration supplied, we need to parse that
if (config && config.length && configPath) {
const configResult = host.configure(configPath, config);
diagnostics = processConfigureResponse(configResult, configPath);
}
if (request.type === CompilerRequestType.Bundle) {
setRootExports(program, resolvedRootModules);
let emitSkipped = true;
// if there was a configuration and no diagnostics with it, we will continue
// to generate the program and possibly emit it.
if (!diagnostics || (diagnostics && diagnostics.length === 0)) {
const options = host.getCompilationSettings();
const program = ts.createProgram(rootNames, options, host);
diagnostics = ts
.getPreEmitDiagnostics(program)
.filter(({ code }) => !ignoredDiagnostics.includes(code));
// We will only proceed with the emit if there are no diagnostics.
if (diagnostics && diagnostics.length === 0) {
if (bundle) {
// we only support a single root module when bundling
assert(resolvedRootModules.length === 1);
// warning so it goes to stderr instead of stdout
console.warn(`Bundling "${resolvedRootModules[0]}"`);
setRootExports(program, resolvedRootModules[0]);
}
const emitResult = program.emit();
emitSkipped = emitResult.emitSkipped;
// emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned
// without casting.
diagnostics = emitResult.diagnostics;
}
}
const emitResult = program.emit();
emitSkipped = emitResult.emitSkipped;
// emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned
// without casting.
diagnostics = emitResult.diagnostics as ts.Diagnostic[];
const result: CompileResult = {
emitSkipped,
diagnostics: diagnostics.length
? fromTypeScriptDiagnostic(diagnostics)
: undefined
};
postMessage(result);
util.log("<<< compile end", {
rootNames,
type: CompilerRequestType[request.type]
});
break;
}
case CompilerRequestType.RuntimeCompile: {
// `RuntimeCompile` are requests from a runtime user, both compiles and
// bundles. The process is similar to a request from the privileged
// side, but also returns the output to the on message.
const { rootName, sources, options, bundle } = request;
util.log(">>> runtime compile start", {
rootName,
bundle,
sources: sources ? Object.keys(sources) : undefined
});
const resolvedRootName = sources
? rootName
: resolveModules([rootName])[0];
const rootNames = sources
? processLocalImports(sources, [[resolvedRootName, resolvedRootName]])
: await processImports([[resolvedRootName, resolvedRootName]]);
const state: WriteFileState = {
type: request.type,
bundle,
host: undefined,
rootNames,
sources,
emitMap: {},
emitBundle: undefined
};
const writeFile = createWriteFile(state);
const host = (state.host = new Host({ bundle, writeFile }));
const compilerOptions = [defaultRuntimeCompileOptions];
if (options) {
compilerOptions.push(convertCompilerOptions(options));
}
if (bundle) {
compilerOptions.push(defaultBundlerOptions);
}
host.mergeOptions(...compilerOptions);
const program = ts.createProgram(
rootNames,
host.getCompilationSettings(),
host
);
if (bundle) {
setRootExports(program, rootNames[0]);
}
const diagnostics = ts
.getPreEmitDiagnostics(program)
.filter(({ code }) => !ignoredDiagnostics.includes(code));
const emitResult = program.emit();
assert(
emitResult.emitSkipped === false,
"Unexpected skip of the emit."
);
const { items } = fromTypeScriptDiagnostic(diagnostics);
const result = [
items && items.length ? items : undefined,
bundle ? state.emitBundle : state.emitMap
];
postMessage(result);
assert(state.emitMap);
util.log("<<< runtime compile finish", {
rootName,
sources: sources ? Object.keys(sources) : undefined,
bundle,
emitMap: Object.keys(state.emitMap)
});
break;
}
case CompilerRequestType.RuntimeTranspile: {
const result: Record<string, TranspileOnlyResult> = {};
const { sources, options } = request;
const compilerOptions = options
? Object.assign(
{},
defaultTranspileOptions,
convertCompilerOptions(options)
)
: defaultTranspileOptions;
for (const [fileName, inputText] of Object.entries(sources)) {
const { outputText: source, sourceMapText: map } = ts.transpileModule(
inputText,
{
fileName,
compilerOptions
}
);
result[fileName] = { source, map };
}
postMessage(result);
break;
}
default:
util.log(
`!!! unhandled CompilerRequestType: ${
(request as CompilerRequest).type
} (${CompilerRequestType[(request as CompilerRequest).type]})`
);
}
const result: EmitResult = {
emitSkipped,
diagnostics: diagnostics.length
? fromTypeScriptDiagnostic(diagnostics)
: undefined
};
postMessage(result);
util.log("<<< compile end", {
rootNames,
type: CompilerRequestType[request.type]
});
// The compiler isolate exits after a single message.
workerClose();
};
};
function base64ToUint8Array(data: string): Uint8Array {
const binString = window.atob(data);
const size = binString.length;
const bytes = new Uint8Array(size);
for (let i = 0; i < size; i++) {
bytes[i] = binString.charCodeAt(i);
}
return bytes;
}
window.wasmCompilerMain = function wasmCompilerMain(): void {
self.wasmCompilerMain = function wasmCompilerMain(): void {
// workerMain should have already been called since a compiler is a worker.
window.onmessage = async ({
self.onmessage = async ({
data: binary
}: {
data: string;
}): Promise<void> => {
const buffer = base64ToUint8Array(binary);
const buffer = util.base64ToUint8Array(binary);
// @ts-ignore
const compiled = await WebAssembly.compile(buffer);
@ -720,10 +310,7 @@ window.wasmCompilerMain = function wasmCompilerMain(): void {
new Set(WebAssembly.Module.exports(compiled).map(({ name }) => name))
);
postMessage({
importList,
exportList
});
postMessage({ importList, exportList });
util.log("<<< WASM compile end");

395
cli/js/compiler_api.ts Normal file
View file

@ -0,0 +1,395 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
// This file contains the runtime APIs which will dispatch work to the internal
// compiler within Deno.
import { Diagnostic } from "./diagnostics.ts";
import * as dispatch from "./dispatch.ts";
import { sendAsync } from "./dispatch_json.ts";
import * as util from "./util.ts";
/** A specific subset TypeScript compiler options that can be supported by
* the Deno TypeScript compiler. */
export interface CompilerOptions {
/** Allow JavaScript files to be compiled. Defaults to `true`. */
allowJs?: boolean;
/** Allow default imports from modules with no default export. This does not
* affect code emit, just typechecking. Defaults to `false`. */
allowSyntheticDefaultImports?: boolean;
/** Allow accessing UMD globals from modules. Defaults to `false`. */
allowUmdGlobalAccess?: boolean;
/** Do not report errors on unreachable code. Defaults to `false`. */
allowUnreachableCode?: boolean;
/** Do not report errors on unused labels. Defaults to `false` */
allowUnusedLabels?: boolean;
/** Parse in strict mode and emit `"use strict"` for each source file.
* Defaults to `true`. */
alwaysStrict?: boolean;
/** Base directory to resolve non-relative module names. Defaults to
* `undefined`. */
baseUrl?: string;
/** Report errors in `.js` files. Use in conjunction with `allowJs`. Defaults
* to `false`. */
checkJs?: boolean;
/** Generates corresponding `.d.ts` file. Defaults to `false`. */
declaration?: boolean;
/** Output directory for generated declaration files. */
declarationDir?: string;
/** Generates a source map for each corresponding `.d.ts` file. Defaults to
* `false`. */
declarationMap?: boolean;
/** Provide full support for iterables in `for..of`, spread and
* destructuring when targeting ES5 or ES3. Defaults to `false`. */
downlevelIteration?: boolean;
/** Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files.
* Defaults to `false`. */
emitBOM?: boolean;
/** Only emit `.d.ts` declaration files. Defaults to `false`. */
emitDeclarationOnly?: boolean;
/** Emit design-type metadata for decorated declarations in source. See issue
* [microsoft/TypeScript#2577](https://github.com/Microsoft/TypeScript/issues/2577)
* for details. Defaults to `false`. */
emitDecoratorMetadata?: boolean;
/** Emit `__importStar` and `__importDefault` helpers for runtime babel
* ecosystem compatibility and enable `allowSyntheticDefaultImports` for type
* system compatibility. Defaults to `true`. */
esModuleInterop?: boolean;
/** Enables experimental support for ES decorators. Defaults to `false`. */
experimentalDecorators?: boolean;
/** Emit a single file with source maps instead of having a separate file.
* Defaults to `false`. */
inlineSourceMap?: boolean;
/** Emit the source alongside the source maps within a single file; requires
* `inlineSourceMap` or `sourceMap` to be set. Defaults to `false`. */
inlineSources?: boolean;
/** Perform additional checks to ensure that transpile only would be safe.
* Defaults to `false`. */
isolatedModules?: boolean;
/** Support JSX in `.tsx` files: `"react"`, `"preserve"`, `"react-native"`.
* Defaults to `"react"`. */
jsx?: "react" | "preserve" | "react-native";
/** Specify the JSX factory function to use when targeting react JSX emit,
* e.g. `React.createElement` or `h`. Defaults to `React.createElement`. */
jsxFactory?: string;
/** Resolve keyof to string valued property names only (no numbers or
* symbols). Defaults to `false`. */
keyofStringsOnly?: string;
/** Emit class fields with ECMAScript-standard semantics. Defaults to `false`.
* Does not apply to `"esnext"` target. */
useDefineForClassFields?: boolean;
/** The locale to use to show error messages. */
locale?: string;
/** Specifies the location where debugger should locate map files instead of
* generated locations. Use this flag if the `.map` files will be located at
* run-time in a different location than the `.js` files. The location
* specified will be embedded in the source map to direct the debugger where
* the map files will be located. Defaults to `undefined`. */
mapRoot?: string;
/** Specify the module format for the emitted code. Defaults to
* `"esnext"`. */
module?:
| "none"
| "commonjs"
| "amd"
| "system"
| "umd"
| "es6"
| "es2015"
| "esnext";
/** Do not generate custom helper functions like `__extends` in compiled
* output. Defaults to `false`. */
noEmitHelpers?: boolean;
/** Report errors for fallthrough cases in switch statement. Defaults to
* `false`. */
noFallthroughCasesInSwitch?: boolean;
/** Raise error on expressions and declarations with an implied any type.
* Defaults to `true`. */
noImplicitAny?: boolean;
/** Report an error when not all code paths in function return a value.
* Defaults to `false`. */
noImplicitReturns?: boolean;
/** Raise error on `this` expressions with an implied `any` type. Defaults to
* `true`. */
noImplicitThis?: boolean;
/** Do not emit `"use strict"` directives in module output. Defaults to
* `false`. */
noImplicitUseStrict?: boolean;
/** Do not add triple-slash references or module import targets to the list of
* compiled files. Defaults to `false`. */
noResolve?: boolean;
/** Disable strict checking of generic signatures in function types. Defaults
* to `false`. */
noStrictGenericChecks?: boolean;
/** Report errors on unused locals. Defaults to `false`. */
noUnusedLocals?: boolean;
/** Report errors on unused parameters. Defaults to `false`. */
noUnusedParameters?: boolean;
/** Redirect output structure to the directory. This only impacts
* `Deno.compile` and only changes the emitted file names. Defaults to
* `undefined`. */
outDir?: string;
/** List of path mapping entries for module names to locations relative to the
* `baseUrl`. Defaults to `undefined`. */
paths?: Record<string, string[]>;
/** Do not erase const enum declarations in generated code. Defaults to
* `false`. */
preserveConstEnums?: boolean;
/** Remove all comments except copy-right header comments beginning with
* `/*!`. Defaults to `true`. */
removeComments?: boolean;
/** Include modules imported with `.json` extension. Defaults to `true`. */
resolveJsonModule?: boolean;
/** Specifies the root directory of input files. Only use to control the
* output directory structure with `outDir`. Defaults to `undefined`. */
rootDir?: string;
/** List of _root_ folders whose combined content represent the structure of
* the project at runtime. Defaults to `undefined`. */
rootDirs?: string[];
/** Generates corresponding `.map` file. Defaults to `false`. */
sourceMap?: boolean;
/** Specifies the location where debugger should locate TypeScript files
* instead of source locations. Use this flag if the sources will be located
* at run-time in a different location than that at design-time. The location
* specified will be embedded in the sourceMap to direct the debugger where
* the source files will be located. Defaults to `undefined`. */
sourceRoot?: string;
/** Enable all strict type checking options. Enabling `strict` enables
* `noImplicitAny`, `noImplicitThis`, `alwaysStrict`, `strictBindCallApply`,
* `strictNullChecks`, `strictFunctionTypes` and
* `strictPropertyInitialization`. Defaults to `true`. */
strict?: boolean;
/** Enable stricter checking of the `bind`, `call`, and `apply` methods on
* functions. Defaults to `true`. */
strictBindCallApply?: boolean;
/** Disable bivariant parameter checking for function types. Defaults to
* `true`. */
strictFunctionTypes?: boolean;
/** Ensure non-undefined class properties are initialized in the constructor.
* This option requires `strictNullChecks` be enabled in order to take effect.
* Defaults to `true`. */
strictPropertyInitialization?: boolean;
/** In strict null checking mode, the `null` and `undefined` values are not in
* the domain of every type and are only assignable to themselves and `any`
* (the one exception being that `undefined` is also assignable to `void`). */
strictNullChecks?: boolean;
/** Suppress excess property checks for object literals. Defaults to
* `false`. */
suppressExcessPropertyErrors?: boolean;
/** Suppress `noImplicitAny` errors for indexing objects lacking index
* signatures. */
suppressImplicitAnyIndexErrors?: boolean;
/** Specify ECMAScript target version. Defaults to `esnext`. */
target?:
| "es3"
| "es5"
| "es6"
| "es2015"
| "es2016"
| "es2017"
| "es2018"
| "es2019"
| "es2020"
| "esnext";
/** List of names of type definitions to include. Defaults to `undefined`. */
types?: string[];
}
/** Internal function to just validate that the specifier looks relative, that
* it starts with `./`. */
function checkRelative(specifier: string): string {
return specifier.match(/^([\.\/\\]|https?:\/{2}|file:\/{2})/)
? specifier
: `./${specifier}`;
}
/** The results of a transpile only command, where the `source` contains the
* emitted source, and `map` optionally contains the source map.
*/
export interface TranspileOnlyResult {
source: string;
map?: string;
}
/** Takes a set of TypeScript sources and resolves with a map where the key was
* the original file name provided in sources and the result contains the
* `source` and optionally the `map` from the transpile operation. This does no
* type checking and validation, it effectively "strips" the types from the
* file.
*
* const results = await Deno.transpileOnly({
* "foo.ts": `const foo: string = "foo";`
* });
*
* @param sources A map where the key is the filename and the value is the text
* to transpile. The filename is only used in the transpile and
* not resolved, for example to fill in the source name in the
* source map.
* @param options An option object of options to send to the compiler. This is
* a subset of ts.CompilerOptions which can be supported by Deno.
* Many of the options related to type checking and emitting
* type declaration files will have no impact on the output.
*/
export function transpileOnly(
sources: Record<string, string>,
options?: CompilerOptions
): Promise<Record<string, TranspileOnlyResult>> {
util.log("Deno.transpileOnly", { sources: Object.keys(sources), options });
const payload = {
sources,
options: options ? JSON.stringify(options) : undefined
};
return sendAsync(dispatch.OP_TRANSPILE, payload).then(result =>
JSON.parse(result)
);
}
/** Takes a root module name, any optionally a record set of sources. Resolves
* with a compiled set of modules. If just a root name is provided, the modules
* will be resolved as if the root module had been passed on the command line.
*
* If sources are passed, all modules will be resolved out of this object, where
* the key is the module name and the value is the content. The extension of
* the module name will be used to determine the media type of the module.
*
* const [ maybeDiagnostics1, output1 ] = await Deno.compile("foo.ts");
*
* const [ maybeDiagnostics2, output2 ] = await Deno.compile("/foo.ts", {
* "/foo.ts": `export * from "./bar.ts";`,
* "/bar.ts": `export const bar = "bar";`
* });
*
* @param rootName The root name of the module which will be used as the
* "starting point". If no `sources` is specified, Deno will
* resolve the module externally as if the `rootName` had been
* specified on the command line.
* @param sources An optional key/value map of sources to be used when resolving
* modules, where the key is the module name, and the value is
* the source content. The extension of the key will determine
* the media type of the file when processing. If supplied,
* Deno will not attempt to resolve any modules externally.
* @param options An optional object of options to send to the compiler. This is
* a subset of ts.CompilerOptions which can be supported by Deno.
*/
export function compile(
rootName: string,
sources?: Record<string, string>,
options?: CompilerOptions
): Promise<[Diagnostic[] | undefined, Record<string, string>]> {
const payload = {
rootName: sources ? rootName : checkRelative(rootName),
sources,
options: options ? JSON.stringify(options) : undefined,
bundle: false
};
util.log("Deno.compile", {
rootName: payload.rootName,
sources: !!sources,
options
});
return sendAsync(dispatch.OP_COMPILE, payload).then(result =>
JSON.parse(result)
);
}
/** Takes a root module name, and optionally a record set of sources. Resolves
* with a single JavaScript string that is like the output of a `deno bundle`
* command. If just a root name is provided, the modules will be resolved as if
* the root module had been passed on the command line.
*
* If sources are passed, all modules will be resolved out of this object, where
* the key is the module name and the value is the content. The extension of the
* module name will be used to determine the media type of the module.
*
* const [ maybeDiagnostics1, output1 ] = await Deno.bundle("foo.ts");
*
* const [ maybeDiagnostics2, output2 ] = await Deno.bundle("/foo.ts", {
* "/foo.ts": `export * from "./bar.ts";`,
* "/bar.ts": `export const bar = "bar";`
* });
*
* @param rootName The root name of the module which will be used as the
* "starting point". If no `sources` is specified, Deno will
* resolve the module externally as if the `rootName` had been
* specified on the command line.
* @param sources An optional key/value map of sources to be used when resolving
* modules, where the key is the module name, and the value is
* the source content. The extension of the key will determine
* the media type of the file when processing. If supplied,
* Deno will not attempt to resolve any modules externally.
* @param options An optional object of options to send to the compiler. This is
* a subset of ts.CompilerOptions which can be supported by Deno.
*/
export function bundle(
rootName: string,
sources?: Record<string, string>,
options?: CompilerOptions
): Promise<[Diagnostic[] | undefined, string]> {
const payload = {
rootName: sources ? rootName : checkRelative(rootName),
sources,
options: options ? JSON.stringify(options) : undefined,
bundle: true
};
util.log("Deno.bundle", {
rootName: payload.rootName,
sources: !!sources,
options
});
return sendAsync(dispatch.OP_COMPILE, payload).then(result =>
JSON.parse(result)
);
}

105
cli/js/compiler_api_test.ts Normal file
View file

@ -0,0 +1,105 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import { assert, assertEquals, test } from "./test_util.ts";
const { compile, transpileOnly, bundle } = Deno;
test(async function compilerApiCompileSources() {
const [diagnostics, actual] = await compile("/foo.ts", {
"/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`,
"/bar.ts": `export const bar = "bar";\n`
});
assert(diagnostics == null);
assert(actual);
assertEquals(Object.keys(actual), [
"/bar.js.map",
"/bar.js",
"/foo.js.map",
"/foo.js"
]);
});
test(async function compilerApiCompileNoSources() {
const [diagnostics, actual] = await compile("./cli/tests/subdir/mod1.ts");
assert(diagnostics == null);
assert(actual);
const keys = Object.keys(actual);
assertEquals(keys.length, 6);
assert(keys[0].endsWith("print_hello.js.map"));
assert(keys[1].endsWith("print_hello.js"));
});
test(async function compilerApiCompileOptions() {
const [diagnostics, actual] = await compile(
"/foo.ts",
{
"/foo.ts": `export const foo = "foo";`
},
{
module: "amd",
sourceMap: false
}
);
assert(diagnostics == null);
assert(actual);
assertEquals(Object.keys(actual), ["/foo.js"]);
assert(actual["/foo.js"].startsWith("define("));
});
test(async function transpileOnlyApi() {
const actual = await transpileOnly({
"foo.ts": `export enum Foo { Foo, Bar, Baz };\n`
});
assert(actual);
assertEquals(Object.keys(actual), ["foo.ts"]);
assert(actual["foo.ts"].source.startsWith("export var Foo;"));
assert(actual["foo.ts"].map);
});
test(async function transpileOnlyApiConfig() {
const actual = await transpileOnly(
{
"foo.ts": `export enum Foo { Foo, Bar, Baz };\n`
},
{
sourceMap: false,
module: "amd"
}
);
assert(actual);
assertEquals(Object.keys(actual), ["foo.ts"]);
assert(actual["foo.ts"].source.startsWith("define("));
assert(actual["foo.ts"].map == null);
});
test(async function bundleApiSources() {
const [diagnostics, actual] = await bundle("/foo.ts", {
"/foo.ts": `export * from "./bar.ts";\n`,
"/bar.ts": `export const bar = "bar";\n`
});
assert(diagnostics == null);
assert(actual.includes(`instantiate("foo")`));
assert(actual.includes(`__rootExports["bar"]`));
});
test(async function bundleApiNoSources() {
const [diagnostics, actual] = await bundle("./cli/tests/subdir/mod1.ts");
assert(diagnostics == null);
assert(actual.includes(`instantiate("mod1")`));
assert(actual.includes(`__rootExports["printHello3"]`));
});
test(async function bundleApiConfig() {
const [diagnostics, actual] = await bundle(
"/foo.ts",
{
"/foo.ts": `// random comment\nexport * from "./bar.ts";\n`,
"/bar.ts": `export const bar = "bar";\n`
},
{
removeComments: true
}
);
assert(diagnostics == null);
assert(!actual.includes(`random`));
});

78
cli/js/bundler.ts → cli/js/compiler_bundler.ts
View file

@ -1,41 +1,47 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import { Console } from "./console.ts";
import * as dispatch from "./dispatch.ts";
import { sendSync } from "./dispatch_json.ts";
import { TextEncoder } from "./text_encoding.ts";
import { assert, commonPath, humanFileSize } from "./util.ts";
import { writeFileSync } from "./write_file.ts";
declare global {
const console: Console;
}
import {
assert,
commonPath,
normalizeString,
CHAR_FORWARD_SLASH
} from "./util.ts";
const BUNDLE_LOADER = "bundle_loader.js";
const encoder = new TextEncoder();
/** A loader of bundled modules that we will inline into any bundle outputs. */
let bundleLoader: string;
/** Local state of what the root exports are of a root module. */
let rootExports: string[] | undefined;
/** Given a fileName and the data, emit the file to the file system. */
export function emitBundle(
rootNames: string[],
fileName: string | undefined,
/** Take a URL and normalize it, resolving relative path parts. */
function normalizeUrl(rootName: string): string {
const match = /^(\S+:\/{2,3})(.+)$/.exec(rootName);
if (match) {
const [, protocol, path] = match;
return `${protocol}${normalizeString(
path,
false,
"/",
code => code === CHAR_FORWARD_SLASH
)}`;
} else {
return rootName;
}
}
/** Given a root name, contents, and source files, enrich the data of the
* bundle with a loader and re-export the exports of the root name. */
export function buildBundle(
rootName: string,
data: string,
sourceFiles: readonly ts.SourceFile[]
): void {
// if the fileName is set to an internal value, just noop
if (fileName && fileName.startsWith("$deno$")) {
return;
}
// This should never happen at the moment, but this code can't currently
// support it
assert(
rootNames.length === 1,
"Only single root modules supported for bundling."
);
): string {
// we can only do this once we are bootstrapped and easiest way to do it is
// inline here
if (!bundleLoader) {
bundleLoader = sendSync(dispatch.OP_FETCH_ASSET, { name: BUNDLE_LOADER });
}
@ -45,7 +51,9 @@ export function emitBundle(
// publicly, so we have to try to replicate
const sources = sourceFiles.map(sf => sf.fileName);
const sharedPath = commonPath(sources);
const rootName = rootNames[0].replace(sharedPath, "").replace(/\.\w+$/i, "");
rootName = normalizeUrl(rootName)
.replace(sharedPath, "")
.replace(/\.\w+$/i, "");
let instantiate: string;
if (rootExports && rootExports.length) {
instantiate = `const __rootExports = instantiate("${rootName}");\n`;
@ -59,28 +67,16 @@ export function emitBundle(
} else {
instantiate = `instantiate("${rootName}");\n`;
}
const bundle = `${bundleLoader}\n${data}\n${instantiate}`;
if (fileName) {
const encodedData = encoder.encode(bundle);
console.warn(`Emitting bundle to "${fileName}"`);
writeFileSync(fileName, encodedData);
console.warn(`${humanFileSize(encodedData.length)} emitted.`);
} else {
console.log(bundle);
}
return `${bundleLoader}\n${data}\n${instantiate}`;
}
/** Set the rootExports which will by the `emitBundle()` */
export function setRootExports(
program: ts.Program,
rootModules: string[]
): void {
export function setRootExports(program: ts.Program, rootModule: string): void {
// get a reference to the type checker, this will let us find symbols from
// the AST.
const checker = program.getTypeChecker();
assert(rootModules.length === 1);
// get a reference to the main source file for the bundle
const mainSourceFile = program.getSourceFile(rootModules[0]);
const mainSourceFile = program.getSourceFile(rootModule);
assert(mainSourceFile);
// retrieve the internal TypeScript symbol for this AST node
const mainSymbol = checker.getSymbolAtLocation(mainSourceFile);

302
cli/js/compiler_host.ts Normal file
View file

@ -0,0 +1,302 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import { MediaType, SourceFile } from "./compiler_sourcefile.ts";
import { OUT_DIR, WriteFileCallback } from "./compiler_util.ts";
import { cwd } from "./dir.ts";
import { sendSync } from "./dispatch_json.ts";
import * as dispatch from "./dispatch.ts";
import { assert, notImplemented } from "./util.ts";
import * as util from "./util.ts";
export interface CompilerHostOptions {
bundle?: boolean;
writeFile: WriteFileCallback;
}
export interface ConfigureResponse {
ignoredOptions?: string[];
diagnostics?: ts.Diagnostic[];
}
const ASSETS = "$asset$";
/** Options that need to be used when generating a bundle (either trusted or
* runtime). */
export const defaultBundlerOptions: ts.CompilerOptions = {
inlineSourceMap: false,
module: ts.ModuleKind.AMD,
outDir: undefined,
outFile: `${OUT_DIR}/bundle.js`,
// disabled until we have effective way to modify source maps
sourceMap: false
};
/** Default options used by the compiler Host when compiling. */
export const defaultCompileOptions: ts.CompilerOptions = {
allowJs: true,
allowNonTsExtensions: true,
// TODO(#3324) Enable strict mode for user code.
// strict: true,
checkJs: false,
esModuleInterop: true,
module: ts.ModuleKind.ESNext,
outDir: OUT_DIR,
resolveJsonModule: true,
sourceMap: true,
stripComments: true,
target: ts.ScriptTarget.ESNext,
jsx: ts.JsxEmit.React
};
/** Options that need to be used when doing a runtime (non bundled) compilation */
export const defaultRuntimeCompileOptions: ts.CompilerOptions = {
outDir: undefined
};
/** Default options used when doing a transpile only. */
export const defaultTranspileOptions: ts.CompilerOptions = {
esModuleInterop: true,
module: ts.ModuleKind.ESNext,
sourceMap: true,
scriptComments: true,
target: ts.ScriptTarget.ESNext
};
/** Options that either do nothing in Deno, or would cause undesired behavior
* if modified. */
const ignoredCompilerOptions: readonly string[] = [
"allowSyntheticDefaultImports",
"baseUrl",
"build",
"composite",
"declaration",
"declarationDir",
"declarationMap",
"diagnostics",
"downlevelIteration",
"emitBOM",
"emitDeclarationOnly",
"esModuleInterop",
"extendedDiagnostics",
"forceConsistentCasingInFileNames",
"help",
"importHelpers",
"incremental",
"inlineSourceMap",
"inlineSources",
"init",
"isolatedModules",
"lib",
"listEmittedFiles",
"listFiles",
"mapRoot",
"maxNodeModuleJsDepth",
"module",
"moduleResolution",
"newLine",
"noEmit",
"noEmitHelpers",
"noEmitOnError",
"noLib",
"noResolve",
"out",
"outDir",
"outFile",
"paths",
"preserveSymlinks",
"preserveWatchOutput",
"pretty",
"rootDir",
"rootDirs",
"showConfig",
"skipDefaultLibCheck",
"skipLibCheck",
"sourceMap",
"sourceRoot",
"stripInternal",
"target",
"traceResolution",
"tsBuildInfoFile",
"types",
"typeRoots",
"version",
"watch"
];
export class Host implements ts.CompilerHost {
private readonly _options = defaultCompileOptions;
private _writeFile: WriteFileCallback;
private _getAsset(filename: string): SourceFile {
const sourceFile = SourceFile.get(filename);
if (sourceFile) {
return sourceFile;
}
const url = filename.split("/").pop()!;
const name = url.includes(".") ? url : `${url}.d.ts`;
const sourceCode = sendSync(dispatch.OP_FETCH_ASSET, { name });
return new SourceFile({
url,
filename,
mediaType: MediaType.TypeScript,
sourceCode
});
}
/* Deno specific APIs */
/** Provides the `ts.HostCompiler` interface for Deno. */
constructor(options: CompilerHostOptions) {
const { bundle = false, writeFile } = options;
this._writeFile = writeFile;
if (bundle) {
// options we need to change when we are generating a bundle
Object.assign(this._options, defaultBundlerOptions);
}
}
/** Take a configuration string, parse it, and use it to merge with the
* compiler's configuration options. The method returns an array of compiler
* options which were ignored, or `undefined`. */
configure(path: string, configurationText: string): ConfigureResponse {
util.log("compiler::host.configure", path);
assert(configurationText);
const { config, error } = ts.parseConfigFileTextToJson(
path,
configurationText
);
if (error) {
return { diagnostics: [error] };
}
const { options, errors } = ts.convertCompilerOptionsFromJson(
config.compilerOptions,
cwd()
);
const ignoredOptions: string[] = [];
for (const key of Object.keys(options)) {
if (
ignoredCompilerOptions.includes(key) &&
(!(key in this._options) || options[key] !== this._options[key])
) {
ignoredOptions.push(key);
delete options[key];
}
}
Object.assign(this._options, options);
return {
ignoredOptions: ignoredOptions.length ? ignoredOptions : undefined,
diagnostics: errors.length ? errors : undefined
};
}
/** Merge options into the host's current set of compiler options and return
* the merged set. */
mergeOptions(...options: ts.CompilerOptions[]): ts.CompilerOptions {
Object.assign(this._options, ...options);
return Object.assign({}, this._options);
}
/* TypeScript CompilerHost APIs */
fileExists(_fileName: string): boolean {
return notImplemented();
}
getCanonicalFileName(fileName: string): string {
return fileName;
}
getCompilationSettings(): ts.CompilerOptions {
util.log("compiler::host.getCompilationSettings()");
return this._options;
}
getCurrentDirectory(): string {
return "";
}
getDefaultLibFileName(_options: ts.CompilerOptions): string {
return ASSETS + "/lib.deno_runtime.d.ts";
}
getNewLine(): string {
return "\n";
}
getSourceFile(
fileName: string,
languageVersion: ts.ScriptTarget,
onError?: (message: string) => void,
shouldCreateNewSourceFile?: boolean
): ts.SourceFile | undefined {
util.log("compiler::host.getSourceFile", fileName);
try {
assert(!shouldCreateNewSourceFile);
const sourceFile = fileName.startsWith(ASSETS)
? this._getAsset(fileName)
: SourceFile.get(fileName);
assert(sourceFile != null);
if (!sourceFile.tsSourceFile) {
sourceFile.tsSourceFile = ts.createSourceFile(
fileName,
sourceFile.sourceCode,
languageVersion
);
}
return sourceFile!.tsSourceFile;
} catch (e) {
if (onError) {
onError(String(e));
} else {
throw e;
}
return undefined;
}
}
readFile(_fileName: string): string | undefined {
return notImplemented();
}
resolveModuleNames(
moduleNames: string[],
containingFile: string
): Array<ts.ResolvedModuleFull | undefined> {
util.log("compiler::host.resolveModuleNames", {
moduleNames,
containingFile
});
return moduleNames.map(specifier => {
const url = SourceFile.getUrl(specifier, containingFile);
const sourceFile = specifier.startsWith(ASSETS)
? this._getAsset(specifier)
: url
? SourceFile.get(url)
: undefined;
if (!sourceFile) {
return undefined;
}
return {
resolvedFileName: sourceFile.url,
isExternalLibraryImport: specifier.startsWith(ASSETS),
extension: sourceFile.extension
};
});
}
useCaseSensitiveFileNames(): boolean {
return true;
}
writeFile(
fileName: string,
data: string,
_writeByteOrderMark: boolean,
_onError?: (message: string) => void,
sourceFiles?: readonly ts.SourceFile[]
): void {
util.log("compiler::host.writeFile", fileName);
this._writeFile(fileName, data, sourceFiles);
}
}

179
cli/js/compiler_imports.ts Normal file
View file

@ -0,0 +1,179 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import {
MediaType,
SourceFile,
SourceFileJson
} from "./compiler_sourcefile.ts";
import { cwd } from "./dir.ts";
import * as dispatch from "./dispatch.ts";
import { sendAsync, sendSync } from "./dispatch_json.ts";
import { assert } from "./util.ts";
import * as util from "./util.ts";
/** Resolve a path to the final path segment passed. */
function resolvePath(...pathSegments: string[]): string {
let resolvedPath = "";
let resolvedAbsolute = false;
for (let i = pathSegments.length - 1; i >= -1 && !resolvedAbsolute; i--) {
let path: string;
if (i >= 0) path = pathSegments[i];
else path = cwd();
// Skip empty entries
if (path.length === 0) {
continue;
}
resolvedPath = `${path}/${resolvedPath}`;
resolvedAbsolute = path.charCodeAt(0) === util.CHAR_FORWARD_SLASH;
}
// At this point the path should be resolved to a full absolute path, but
// handle relative paths to be safe (might happen when cwd() fails)
// Normalize the path
resolvedPath = util.normalizeString(
resolvedPath,
!resolvedAbsolute,
"/",
code => code === util.CHAR_FORWARD_SLASH
);
if (resolvedAbsolute) {
if (resolvedPath.length > 0) return `/${resolvedPath}`;
else return "/";
} else if (resolvedPath.length > 0) return resolvedPath;
else return ".";
}
/** Resolve a relative specifier based on the referrer. Used when resolving
* modules internally within the runtime compiler API. */
function resolveSpecifier(specifier: string, referrer: string): string {
if (!specifier.startsWith(".")) {
return specifier;
}
const pathParts = referrer.split("/");
pathParts.pop();
let path = pathParts.join("/");
path = path.endsWith("/") ? path : `${path}/`;
return resolvePath(path, specifier);
}
/** Ops to Rust to resolve modules' URLs. */
export function resolveModules(
specifiers: string[],
referrer?: string
): string[] {
util.log("compiler_imports::resolveModules", { specifiers, referrer });
return sendSync(dispatch.OP_RESOLVE_MODULES, { specifiers, referrer });
}
/** Ops to Rust to fetch modules meta data. */
function fetchSourceFiles(
specifiers: string[],
referrer?: string
): Promise<SourceFileJson[]> {
util.log("compiler_imports::fetchSourceFiles", { specifiers, referrer });
return sendAsync(dispatch.OP_FETCH_SOURCE_FILES, {
specifiers,
referrer
});
}
/** Given a filename, determine the media type based on extension. Used when
* resolving modules internally in a runtime compile. */
function getMediaType(filename: string): MediaType {
const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename);
if (!maybeExtension) {
util.log(`!!! Could not identify valid extension: "${filename}"`);
return MediaType.Unknown;
}
const [, extension] = maybeExtension;
switch (extension.toLowerCase()) {
case "js":
return MediaType.JavaScript;
case "jsx":
return MediaType.JSX;
case "json":
return MediaType.Json;
case "ts":
return MediaType.TypeScript;
case "tsx":
return MediaType.TSX;
case "wasm":
return MediaType.Wasm;
default:
util.log(`!!! Unknown extension: "${extension}"`);
return MediaType.Unknown;
}
}
/** Recursively process the imports of modules from within the supplied sources,
* generating `SourceFile`s of any imported files.
*
* Specifiers are supplied in an array of tuples where the first is the
* specifier that will be requested in the code and the second is the specifier
* that should be actually resolved. */
export function processLocalImports(
sources: Record<string, string>,
specifiers: Array<[string, string]>,
referrer?: string
): string[] {
if (!specifiers.length) {
return [];
}
const moduleNames = specifiers.map(
referrer
? ([, specifier]): string => resolveSpecifier(specifier, referrer)
: ([, specifier]): string => specifier
);
for (let i = 0; i < moduleNames.length; i++) {
const moduleName = moduleNames[i];
assert(moduleName in sources, `Missing module in sources: "${moduleName}"`);
const sourceFile =
SourceFile.get(moduleName) ||
new SourceFile({
url: moduleName,
filename: moduleName,
sourceCode: sources[moduleName],
mediaType: getMediaType(moduleName)
});
sourceFile.cache(specifiers[i][0], referrer);
if (!sourceFile.processed) {
processLocalImports(sources, sourceFile.imports(), sourceFile.url);
}
}
return moduleNames;
}
/** Recursively process the imports of modules, generating `SourceFile`s of any
* imported files.
*
* Specifiers are supplied in an array of tuples where the first is the
* specifier that will be requested in the code and the second is the specifier
* that should be actually resolved. */
export async function processImports(
specifiers: Array<[string, string]>,
referrer?: string
): Promise<string[]> {
if (!specifiers.length) {
return [];
}
const sources = specifiers.map(([, moduleSpecifier]) => moduleSpecifier);
const resolvedSources = resolveModules(sources, referrer);
const sourceFiles = await fetchSourceFiles(resolvedSources, referrer);
assert(sourceFiles.length === specifiers.length);
for (let i = 0; i < sourceFiles.length; i++) {
const sourceFileJson = sourceFiles[i];
const sourceFile =
SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson);
sourceFile.cache(specifiers[i][0], referrer);
if (!sourceFile.processed) {
await processImports(sourceFile.imports(), sourceFile.url);
}
}
return resolvedSources;
}

168
cli/js/compiler_sourcefile.ts Normal file
View file

@ -0,0 +1,168 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import {
getMappedModuleName,
parseTypeDirectives
} from "./compiler_type_directives.ts";
import { assert, log } from "./util.ts";
// Warning! The values in this enum are duplicated in `cli/msg.rs`
// Update carefully!
export enum MediaType {
JavaScript = 0,
JSX = 1,
TypeScript = 2,
TSX = 3,
Json = 4,
Wasm = 5,
Unknown = 6
}
/** The shape of the SourceFile that comes from the privileged side */
export interface SourceFileJson {
url: string;
filename: string;
mediaType: MediaType;
sourceCode: string;
}
/** Returns the TypeScript Extension enum for a given media type. */
function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
switch (mediaType) {
case MediaType.JavaScript:
return ts.Extension.Js;
case MediaType.JSX:
return ts.Extension.Jsx;
case MediaType.TypeScript:
return fileName.endsWith(".d.ts") ? ts.Extension.Dts : ts.Extension.Ts;
case MediaType.TSX:
return ts.Extension.Tsx;
case MediaType.Json:
return ts.Extension.Json;
case MediaType.Wasm:
// Custom marker for Wasm type.
return ts.Extension.Js;
case MediaType.Unknown:
default:
throw TypeError("Cannot resolve extension.");
}
}
/** A self registering abstraction of source files. */
export class SourceFile {
extension!: ts.Extension;
filename!: string;
/** An array of tuples which represent the imports for the source file. The
* first element is the one that will be requested at compile time, the
* second is the one that should be actually resolved. This provides the
* feature of type directives for Deno. */
importedFiles?: Array<[string, string]>;
mediaType!: MediaType;
processed = false;
sourceCode!: string;
tsSourceFile?: ts.SourceFile;
url!: string;
constructor(json: SourceFileJson) {
if (SourceFile._moduleCache.has(json.url)) {
throw new TypeError("SourceFile already exists");
}
Object.assign(this, json);
this.extension = getExtension(this.url, this.mediaType);
SourceFile._moduleCache.set(this.url, this);
}
/** Cache the source file to be able to be retrieved by `moduleSpecifier` and
* `containingFile`. */
cache(moduleSpecifier: string, containingFile?: string): void {
containingFile = containingFile || "";
let innerCache = SourceFile._specifierCache.get(containingFile);
if (!innerCache) {
innerCache = new Map();
SourceFile._specifierCache.set(containingFile, innerCache);
}
innerCache.set(moduleSpecifier, this);
}
/** Process the imports for the file and return them. */
imports(): Array<[string, string]> {
if (this.processed) {
throw new Error("SourceFile has already been processed.");
}
assert(this.sourceCode != null);
// we shouldn't process imports for files which contain the nocheck pragma
// (like bundles)
if (this.sourceCode.match(/\/{2}\s+@ts-nocheck/)) {
log(`Skipping imports for "${this.filename}"`);
return [];
}
const preProcessedFileInfo = ts.preProcessFile(this.sourceCode, true, true);
this.processed = true;
const files = (this.importedFiles = [] as Array<[string, string]>);
function process(references: ts.FileReference[]): void {
for (const { fileName } of references) {
files.push([fileName, fileName]);
}
}
const {
importedFiles,
referencedFiles,
libReferenceDirectives,
typeReferenceDirectives
} = preProcessedFileInfo;
const typeDirectives = parseTypeDirectives(this.sourceCode);
if (typeDirectives) {
for (const importedFile of importedFiles) {
files.push([
importedFile.fileName,
getMappedModuleName(importedFile, typeDirectives)
]);
}
} else {
process(importedFiles);
}
process(referencedFiles);
process(libReferenceDirectives);
process(typeReferenceDirectives);
return files;
}
/** A cache of all the source files which have been loaded indexed by the
* url. */
private static _moduleCache: Map<string, SourceFile> = new Map();
/** A cache of source files based on module specifiers and containing files
* which is used by the TypeScript compiler to resolve the url */
private static _specifierCache: Map<
string,
Map<string, SourceFile>
> = new Map();
/** Retrieve a `SourceFile` based on a `moduleSpecifier` and `containingFile`
* or return `undefined` if not preset. */
static getUrl(
moduleSpecifier: string,
containingFile: string
): string | undefined {
const containingCache = this._specifierCache.get(containingFile);
if (containingCache) {
const sourceFile = containingCache.get(moduleSpecifier);
return sourceFile && sourceFile.url;
}
return undefined;
}
/** Retrieve a `SourceFile` based on a `url` */
static get(url: string): SourceFile | undefined {
return this._moduleCache.get(url);
}
/** Determine if a source file exists or not */
static has(url: string): boolean {
return this._moduleCache.has(url);
}
}

0
cli/js/type_directives.ts → cli/js/compiler_type_directives.ts
View file

298
cli/js/compiler_util.ts Normal file
View file

@ -0,0 +1,298 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import { bold, cyan, yellow } from "./colors.ts";
import { CompilerOptions } from "./compiler_api.ts";
import { buildBundle } from "./compiler_bundler.ts";
import { ConfigureResponse, Host } from "./compiler_host.ts";
import { SourceFile } from "./compiler_sourcefile.ts";
import { sendSync } from "./dispatch_json.ts";
import * as dispatch from "./dispatch.ts";
import { TextEncoder } from "./text_encoding.ts";
import * as util from "./util.ts";
import { assert } from "./util.ts";
import { writeFileSync } from "./write_file.ts";
/** Type for the write fall callback that allows delegation from the compiler
* host on writing files. */
export type WriteFileCallback = (
fileName: string,
data: string,
sourceFiles?: readonly ts.SourceFile[]
) => void;
/** An object which is passed to `createWriteFile` to be used to read and set
* state related to the emit of a program. */
export interface WriteFileState {
type: CompilerRequestType;
bundle?: boolean;
host?: Host;
outFile?: string;
rootNames: string[];
emitMap?: Record<string, string>;
emitBundle?: string;
sources?: Record<string, string>;
}
// Warning! The values in this enum are duplicated in `cli/msg.rs`
// Update carefully!
export enum CompilerRequestType {
Compile = 0,
RuntimeCompile = 1,
RuntimeTranspile = 2
}
export const OUT_DIR = "$deno$";
/** Cache the contents of a file on the trusted side. */
function cache(
moduleId: string,
emittedFileName: string,
contents: string,
checkJs = false
): void {
util.log("compiler::cache", { moduleId, emittedFileName, checkJs });
const sf = SourceFile.get(moduleId);
if (sf) {
// NOTE: If it's a `.json` file we don't want to write it to disk.
// JSON files are loaded and used by TS compiler to check types, but we don't want
// to emit them to disk because output file is the same as input file.
if (sf.extension === ts.Extension.Json) {
return;
}
// NOTE: JavaScript files are only cached to disk if `checkJs`
// option in on
if (sf.extension === ts.Extension.Js && !checkJs) {
return;
}
}
if (emittedFileName.endsWith(".map")) {
// Source Map
sendSync(dispatch.OP_CACHE, {
extension: ".map",
moduleId,
contents
});
} else if (
emittedFileName.endsWith(".js") ||
emittedFileName.endsWith(".json")
) {
// Compiled JavaScript
sendSync(dispatch.OP_CACHE, {
extension: ".js",
moduleId,
contents
});
} else {
assert(false, `Trying to cache unhandled file type "${emittedFileName}"`);
}
}
const encoder = new TextEncoder();
/** Generates a `writeFile` function which can be passed to the compiler `Host`
* to use when emitting files. */
export function createWriteFile(state: WriteFileState): WriteFileCallback {
if (state.type === CompilerRequestType.Compile) {
return function writeFile(
fileName: string,
data: string,
sourceFiles?: readonly ts.SourceFile[]
): void {
assert(
sourceFiles != null,
`Unexpected emit of "${fileName}" which isn't part of a program.`
);
assert(state.host);
if (!state.bundle) {
assert(sourceFiles.length === 1);
cache(
sourceFiles[0].fileName,
fileName,
data,
state.host.getCompilationSettings().checkJs
);
} else {
// if the fileName is set to an internal value, just noop, this is
// used in the Rust unit tests.
if (state.outFile && state.outFile.startsWith(OUT_DIR)) {
return;
}
// we only support single root names for bundles
assert(
state.rootNames.length === 1,
`Only one root name supported. Got "${JSON.stringify(
state.rootNames
)}"`
);
// this enriches the string with the loader and re-exports the
// exports of the root module
const content = buildBundle(state.rootNames[0], data, sourceFiles);
if (state.outFile) {
const encodedData = encoder.encode(content);
console.warn(`Emitting bundle to "${state.outFile}"`);
writeFileSync(state.outFile, encodedData);
console.warn(`${util.humanFileSize(encodedData.length)} emitted.`);
} else {
console.log(content);
}
}
};
}
return function writeFile(
fileName: string,
data: string,
sourceFiles?: readonly ts.SourceFile[]
): void {
assert(sourceFiles != null);
assert(state.host);
assert(state.emitMap);
if (!state.bundle) {
assert(sourceFiles.length === 1);
state.emitMap[fileName] = data;
// we only want to cache the compiler output if we are resolving
// modules externally
if (!state.sources) {
cache(
sourceFiles[0].fileName,
fileName,
data,
state.host.getCompilationSettings().checkJs
);
}
} else {
// we only support single root names for bundles
assert(state.rootNames.length === 1);
state.emitBundle = buildBundle(state.rootNames[0], data, sourceFiles);
}
};
}
/** Take a runtime set of compiler options as stringified JSON and convert it
* to a set of TypeScript compiler options. */
export function convertCompilerOptions(str: string): ts.CompilerOptions {
const options: CompilerOptions = JSON.parse(str);
const out: Record<string, unknown> = {};
const keys = Object.keys(options) as Array<keyof CompilerOptions>;
for (const key of keys) {
switch (key) {
case "jsx":
const value = options[key];
if (value === "preserve") {
out[key] = ts.JsxEmit.Preserve;
} else if (value === "react") {
out[key] = ts.JsxEmit.React;
} else {
out[key] = ts.JsxEmit.ReactNative;
}
break;
case "module":
switch (options[key]) {
case "amd":
out[key] = ts.ModuleKind.AMD;
break;
case "commonjs":
out[key] = ts.ModuleKind.CommonJS;
break;
case "es2015":
case "es6":
out[key] = ts.ModuleKind.ES2015;
break;
case "esnext":
out[key] = ts.ModuleKind.ESNext;
break;
case "none":
out[key] = ts.ModuleKind.None;
break;
case "system":
out[key] = ts.ModuleKind.System;
break;
case "umd":
out[key] = ts.ModuleKind.UMD;
break;
default:
throw new TypeError("Unexpected module type");
}
break;
case "target":
switch (options[key]) {
case "es3":
out[key] = ts.ScriptTarget.ES3;
break;
case "es5":
out[key] = ts.ScriptTarget.ES5;
break;
case "es6":
case "es2015":
out[key] = ts.ScriptTarget.ES2015;
break;
case "es2016":
out[key] = ts.ScriptTarget.ES2016;
break;
case "es2017":
out[key] = ts.ScriptTarget.ES2017;
break;
case "es2018":
out[key] = ts.ScriptTarget.ES2018;
break;
case "es2019":
out[key] = ts.ScriptTarget.ES2019;
break;
case "es2020":
out[key] = ts.ScriptTarget.ES2020;
break;
case "esnext":
out[key] = ts.ScriptTarget.ESNext;
break;
default:
throw new TypeError("Unexpected emit target.");
}
default:
out[key] = options[key];
}
}
return out as ts.CompilerOptions;
}
/** An array of TypeScript diagnostic types we ignore. */
export const ignoredDiagnostics = [
// TS1103: 'for-await-of' statement is only allowed within an async function
// or async generator.
1103,
// TS1308: 'await' expression is only allowed within an async function.
1308,
// TS2691: An import path cannot end with a '.ts' extension. Consider
// importing 'bad-module' instead.
2691,
// TS5009: Cannot find the common subdirectory path for the input files.
5009,
// TS5055: Cannot write file
// 'http://localhost:4545/tests/subdir/mt_application_x_javascript.j4.js'
// because it would overwrite input file.
5055,
// TypeScript is overly opinionated that only CommonJS modules kinds can
// support JSON imports. Allegedly this was fixed in
// Microsoft/TypeScript#26825 but that doesn't seem to be working here,
// so we will ignore complaints about this compiler setting.
5070
];
/** When doing a host configuration, processing the response and logging out
* and options which were ignored. */
export function processConfigureResponse(
configResult: ConfigureResponse,
configPath: string
): ts.Diagnostic[] | undefined {
const { ignoredOptions, diagnostics } = configResult;
if (ignoredOptions) {
console.warn(
yellow(`Unsupported compiler options in "${configPath}"\n`) +
cyan(` The following options were ignored:\n`) +
` ${ignoredOptions.map((value): string => bold(value)).join(", ")}`
);
}
return diagnostics;
}

1
cli/js/deno.ts
View file

@ -97,6 +97,7 @@ export {
ProcessStatus,
Signal
} from "./process.ts";
export { transpileOnly, compile, bundle } from "./compiler_api.ts";
export { inspect, customInspect } from "./console.ts";
export { build, OperatingSystem, Arch } from "./build.ts";
export { version } from "./version.ts";

149
cli/js/diagnostics.ts
View file

@ -64,152 +64,3 @@ export interface Diagnostic {
/** An array of diagnostic items. */
items: DiagnosticItem[];
}
interface SourceInformation {
sourceLine: string;
lineNumber: number;
scriptResourceName: string;
startColumn: number;
endColumn: number;
}
function fromDiagnosticCategory(
category: ts.DiagnosticCategory
): DiagnosticCategory {
switch (category) {
case ts.DiagnosticCategory.Error:
return DiagnosticCategory.Error;
case ts.DiagnosticCategory.Message:
return DiagnosticCategory.Info;
case ts.DiagnosticCategory.Suggestion:
return DiagnosticCategory.Suggestion;
case ts.DiagnosticCategory.Warning:
return DiagnosticCategory.Warning;
default:
throw new Error(
`Unexpected DiagnosticCategory: "${category}"/"${ts.DiagnosticCategory[category]}"`
);
}
}
function getSourceInformation(
sourceFile: ts.SourceFile,
start: number,
length: number
): SourceInformation {
const scriptResourceName = sourceFile.fileName;
const {
line: lineNumber,
character: startColumn
} = sourceFile.getLineAndCharacterOfPosition(start);
const endPosition = sourceFile.getLineAndCharacterOfPosition(start + length);
const endColumn =
lineNumber === endPosition.line ? endPosition.character : startColumn;
const lastLineInFile = sourceFile.getLineAndCharacterOfPosition(
sourceFile.text.length
).line;
const lineStart = sourceFile.getPositionOfLineAndCharacter(lineNumber, 0);
const lineEnd =
lineNumber < lastLineInFile
? sourceFile.getPositionOfLineAndCharacter(lineNumber + 1, 0)
: sourceFile.text.length;
const sourceLine = sourceFile.text
.slice(lineStart, lineEnd)
.replace(/\s+$/g, "")
.replace("\t", " ");
return {
sourceLine,
lineNumber,
scriptResourceName,
startColumn,
endColumn
};
}
/** Converts a TypeScript diagnostic message chain to a Deno one. */
function fromDiagnosticMessageChain(
messageChain: ts.DiagnosticMessageChain[] | undefined
): DiagnosticMessageChain[] | undefined {
if (!messageChain) {
return undefined;
}
return messageChain.map(({ messageText: message, code, category, next }) => {
return {
message,
code,
category: fromDiagnosticCategory(category),
next: fromDiagnosticMessageChain(next)
};
});
}
/** Parse out information from a TypeScript diagnostic structure. */
function parseDiagnostic(
item: ts.Diagnostic | ts.DiagnosticRelatedInformation
): DiagnosticItem {
const {
messageText,
category: sourceCategory,
code,
file,
start: startPosition,
length
} = item;
const sourceInfo =
file && startPosition && length
? getSourceInformation(file, startPosition, length)
: undefined;
const endPosition =
startPosition && length ? startPosition + length : undefined;
const category = fromDiagnosticCategory(sourceCategory);
let message: string;
let messageChain: DiagnosticMessageChain | undefined;
if (typeof messageText === "string") {
message = messageText;
} else {
message = messageText.messageText;
messageChain = fromDiagnosticMessageChain([messageText])![0];
}
const base = {
message,
messageChain,
code,
category,
startPosition,
endPosition
};
return sourceInfo ? { ...base, ...sourceInfo } : base;
}
/** Convert a diagnostic related information array into a Deno diagnostic
* array. */
function parseRelatedInformation(
relatedInformation: readonly ts.DiagnosticRelatedInformation[]
): DiagnosticItem[] {
const result: DiagnosticItem[] = [];
for (const item of relatedInformation) {
result.push(parseDiagnostic(item));
}
return result;
}
/** Convert TypeScript diagnostics to Deno diagnostics. */
export function fromTypeScriptDiagnostic(
diagnostics: readonly ts.Diagnostic[]
): Diagnostic {
const items: DiagnosticItem[] = [];
for (const sourceDiagnostic of diagnostics) {
const item: DiagnosticItem = parseDiagnostic(sourceDiagnostic);
if (sourceDiagnostic.relatedInformation) {
item.relatedInformation = parseRelatedInformation(
sourceDiagnostic.relatedInformation
);
}
items.push(item);
}
return { items };
}

160
cli/js/diagnostics_util.ts Normal file
View file

@ -0,0 +1,160 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
// These utilities are used by compiler.ts to format TypeScript diagnostics
// into Deno Diagnostics.
import {
Diagnostic,
DiagnosticCategory,
DiagnosticMessageChain,
DiagnosticItem
} from "./diagnostics.ts";
interface SourceInformation {
sourceLine: string;
lineNumber: number;
scriptResourceName: string;
startColumn: number;
endColumn: number;
}
function fromDiagnosticCategory(
category: ts.DiagnosticCategory
): DiagnosticCategory {
switch (category) {
case ts.DiagnosticCategory.Error:
return DiagnosticCategory.Error;
case ts.DiagnosticCategory.Message:
return DiagnosticCategory.Info;
case ts.DiagnosticCategory.Suggestion:
return DiagnosticCategory.Suggestion;
case ts.DiagnosticCategory.Warning:
return DiagnosticCategory.Warning;
default:
throw new Error(
`Unexpected DiagnosticCategory: "${category}"/"${ts.DiagnosticCategory[category]}"`
);
}
}
function getSourceInformation(
sourceFile: ts.SourceFile,
start: number,
length: number
): SourceInformation {
const scriptResourceName = sourceFile.fileName;
const {
line: lineNumber,
character: startColumn
} = sourceFile.getLineAndCharacterOfPosition(start);
const endPosition = sourceFile.getLineAndCharacterOfPosition(start + length);
const endColumn =
lineNumber === endPosition.line ? endPosition.character : startColumn;
const lastLineInFile = sourceFile.getLineAndCharacterOfPosition(
sourceFile.text.length
).line;
const lineStart = sourceFile.getPositionOfLineAndCharacter(lineNumber, 0);
const lineEnd =
lineNumber < lastLineInFile
? sourceFile.getPositionOfLineAndCharacter(lineNumber + 1, 0)
: sourceFile.text.length;
const sourceLine = sourceFile.text
.slice(lineStart, lineEnd)
.replace(/\s+$/g, "")
.replace("\t", " ");
return {
sourceLine,
lineNumber,
scriptResourceName,
startColumn,
endColumn
};
}
/** Converts a TypeScript diagnostic message chain to a Deno one. */
function fromDiagnosticMessageChain(
messageChain: ts.DiagnosticMessageChain[] | undefined
): DiagnosticMessageChain[] | undefined {
if (!messageChain) {
return undefined;
}
return messageChain.map(({ messageText: message, code, category, next }) => {
return {
message,
code,
category: fromDiagnosticCategory(category),
next: fromDiagnosticMessageChain(next)
};
});
}
/** Parse out information from a TypeScript diagnostic structure. */
function parseDiagnostic(
item: ts.Diagnostic | ts.DiagnosticRelatedInformation
): DiagnosticItem {
const {
messageText,
category: sourceCategory,
code,
file,
start: startPosition,
length
} = item;
const sourceInfo =
file && startPosition && length
? getSourceInformation(file, startPosition, length)
: undefined;
const endPosition =
startPosition && length ? startPosition + length : undefined;
const category = fromDiagnosticCategory(sourceCategory);
let message: string;
let messageChain: DiagnosticMessageChain | undefined;
if (typeof messageText === "string") {
message = messageText;
} else {
message = messageText.messageText;
messageChain = fromDiagnosticMessageChain([messageText])![0];
}
const base = {
message,
messageChain,
code,
category,
startPosition,
endPosition
};
return sourceInfo ? { ...base, ...sourceInfo } : base;
}
/** Convert a diagnostic related information array into a Deno diagnostic
* array. */
function parseRelatedInformation(
relatedInformation: readonly ts.DiagnosticRelatedInformation[]
): DiagnosticItem[] {
const result: DiagnosticItem[] = [];
for (const item of relatedInformation) {
result.push(parseDiagnostic(item));
}
return result;
}
/** Convert TypeScript diagnostics to Deno diagnostics. */
export function fromTypeScriptDiagnostic(
diagnostics: readonly ts.Diagnostic[]
): Diagnostic {
const items: DiagnosticItem[] = [];
for (const sourceDiagnostic of diagnostics) {
const item: DiagnosticItem = parseDiagnostic(sourceDiagnostic);
if (sourceDiagnostic.relatedInformation) {
item.relatedInformation = parseRelatedInformation(
sourceDiagnostic.relatedInformation
);
}
items.push(item);
}
return { items };
}

5
cli/js/dispatch.ts
View file

@ -18,6 +18,7 @@ export let OP_START: number;
export let OP_APPLY_SOURCE_MAP: number;
export let OP_FORMAT_ERROR: number;
export let OP_CACHE: number;
export let OP_RESOLVE_MODULES: number;
export let OP_FETCH_SOURCE_FILES: number;
export let OP_OPEN: number;
export let OP_CLOSE: number;
@ -69,6 +70,8 @@ export let OP_FETCH_ASSET: number;
export let OP_DIAL_TLS: number;
export let OP_HOSTNAME: number;
export let OP_OPEN_PLUGIN: number;
export let OP_COMPILE: number;
export let OP_TRANSPILE: number;
const PLUGIN_ASYNC_HANDLER_MAP: Map<number, AsyncHandler> = new Map();
@ -120,6 +123,8 @@ export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void {
case OP_MAKE_TEMP_DIR:
case OP_DIAL_TLS:
case OP_FETCH_SOURCE_FILES:
case OP_COMPILE:
case OP_TRANSPILE:
json.asyncMsgFromRust(opId, ui8);
break;
default:

2
cli/js/globals.ts
View file

@ -62,6 +62,8 @@ declare global {
interface Object {
[consoleTypes.customInspect]?(): string;
}
const console: consoleTypes.Console;
}
// A self reference to the global object.

404
cli/js/lib.deno_runtime.d.ts vendored
View file

@ -1505,6 +1505,410 @@ declare namespace Deno {
export const version: Version;
export {};
// @url js/diagnostics.d.ts
/** The log category for a diagnostic message */
export enum DiagnosticCategory {
Log = 0,
Debug = 1,
Info = 2,
Error = 3,
Warning = 4,
Suggestion = 5
}
export interface DiagnosticMessageChain {
message: string;
category: DiagnosticCategory;
code: number;
next?: DiagnosticMessageChain[];
}
export interface DiagnosticItem {
/** A string message summarizing the diagnostic. */
message: string;
/** An ordered array of further diagnostics. */
messageChain?: DiagnosticMessageChain;
/** Information related to the diagnostic. This is present when there is a
* suggestion or other additional diagnostic information */
relatedInformation?: DiagnosticItem[];
/** The text of the source line related to the diagnostic */
sourceLine?: string;
/** The line number that is related to the diagnostic */
lineNumber?: number;
/** The name of the script resource related to the diagnostic */
scriptResourceName?: string;
/** The start position related to the diagnostic */
startPosition?: number;
/** The end position related to the diagnostic */
endPosition?: number;
/** The category of the diagnostic */
category: DiagnosticCategory;
/** A number identifier */
code: number;
/** The the start column of the sourceLine related to the diagnostic */
startColumn?: number;
/** The end column of the sourceLine related to the diagnostic */
endColumn?: number;
}
export interface Diagnostic {
/** An array of diagnostic items. */
items: DiagnosticItem[];
}
// @url js/compiler_api.ts
/** A specific subset TypeScript compiler options that can be supported by
* the Deno TypeScript compiler. */
export interface CompilerOptions {
/** Allow JavaScript files to be compiled. Defaults to `true`. */
allowJs?: boolean;
/** Allow default imports from modules with no default export. This does not
* affect code emit, just typechecking. Defaults to `false`. */
allowSyntheticDefaultImports?: boolean;
/** Allow accessing UMD globals from modules. Defaults to `false`. */
allowUmdGlobalAccess?: boolean;
/** Do not report errors on unreachable code. Defaults to `false`. */
allowUnreachableCode?: boolean;
/** Do not report errors on unused labels. Defaults to `false` */
allowUnusedLabels?: boolean;
/** Parse in strict mode and emit `"use strict"` for each source file.
* Defaults to `true`. */
alwaysStrict?: boolean;
/** Base directory to resolve non-relative module names. Defaults to
* `undefined`. */
baseUrl?: string;
/** Report errors in `.js` files. Use in conjunction with `allowJs`. Defaults
* to `false`. */
checkJs?: boolean;
/** Generates corresponding `.d.ts` file. Defaults to `false`. */
declaration?: boolean;
/** Output directory for generated declaration files. */
declarationDir?: string;
/** Generates a source map for each corresponding `.d.ts` file. Defaults to
* `false`. */
declarationMap?: boolean;
/** Provide full support for iterables in `for..of`, spread and
* destructuring when targeting ES5 or ES3. Defaults to `false`. */
downlevelIteration?: boolean;
/** Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files.
* Defaults to `false`. */
emitBOM?: boolean;
/** Only emit `.d.ts` declaration files. Defaults to `false`. */
emitDeclarationOnly?: boolean;
/** Emit design-type metadata for decorated declarations in source. See issue
* [microsoft/TypeScript#2577](https://github.com/Microsoft/TypeScript/issues/2577)
* for details. Defaults to `false`. */
emitDecoratorMetadata?: boolean;
/** Emit `__importStar` and `__importDefault` helpers for runtime babel
* ecosystem compatibility and enable `allowSyntheticDefaultImports` for type
* system compatibility. Defaults to `true`. */
esModuleInterop?: boolean;
/** Enables experimental support for ES decorators. Defaults to `false`. */
experimentalDecorators?: boolean;
/** Emit a single file with source maps instead of having a separate file.
* Defaults to `false`. */
inlineSourceMap?: boolean;
/** Emit the source alongside the source maps within a single file; requires
* `inlineSourceMap` or `sourceMap` to be set. Defaults to `false`. */
inlineSources?: boolean;
/** Perform additional checks to ensure that transpile only would be safe.
* Defaults to `false`. */
isolatedModules?: boolean;
/** Support JSX in `.tsx` files: `"react"`, `"preserve"`, `"react-native"`.
* Defaults to `"react"`. */
jsx?: "react" | "preserve" | "react-native";
/** Specify the JSX factory function to use when targeting react JSX emit,
* e.g. `React.createElement` or `h`. Defaults to `React.createElement`. */
jsxFactory?: string;
/** Resolve keyof to string valued property names only (no numbers or
* symbols). Defaults to `false`. */
keyofStringsOnly?: string;
/** Emit class fields with ECMAScript-standard semantics. Defaults to `false`.
* Does not apply to `"esnext"` target. */
useDefineForClassFields?: boolean;
/** The locale to use to show error messages. */
locale?: string;
/** Specifies the location where debugger should locate map files instead of
* generated locations. Use this flag if the `.map` files will be located at
* run-time in a different location than the `.js` files. The location
* specified will be embedded in the source map to direct the debugger where
* the map files will be located. Defaults to `undefined`. */
mapRoot?: string;
/** Specify the module format for the emitted code. Defaults to
* `"esnext"`. */
module?:
| "none"
| "commonjs"
| "amd"
| "system"
| "umd"
| "es6"
| "es2015"
| "esnext";
/** Do not generate custom helper functions like `__extends` in compiled
* output. Defaults to `false`. */
noEmitHelpers?: boolean;
/** Report errors for fallthrough cases in switch statement. Defaults to
* `false`. */
noFallthroughCasesInSwitch?: boolean;
/** Raise error on expressions and declarations with an implied any type.
* Defaults to `true`. */
noImplicitAny?: boolean;
/** Report an error when not all code paths in function return a value.
* Defaults to `false`. */
noImplicitReturns?: boolean;
/** Raise error on `this` expressions with an implied `any` type. Defaults to
* `true`. */
noImplicitThis?: boolean;
/** Do not emit `"use strict"` directives in module output. Defaults to
* `false`. */
noImplicitUseStrict?: boolean;
/** Do not add triple-slash references or module import targets to the list of
* compiled files. Defaults to `false`. */
noResolve?: boolean;
/** Disable strict checking of generic signatures in function types. Defaults
* to `false`. */
noStrictGenericChecks?: boolean;
/** Report errors on unused locals. Defaults to `false`. */
noUnusedLocals?: boolean;
/** Report errors on unused parameters. Defaults to `false`. */
noUnusedParameters?: boolean;
/** Redirect output structure to the directory. This only impacts
* `Deno.compile` and only changes the emitted file names. Defaults to
* `undefined`. */
outDir?: string;
/** List of path mapping entries for module names to locations relative to the
* `baseUrl`. Defaults to `undefined`. */
paths?: Record<string, string[]>;
/** Do not erase const enum declarations in generated code. Defaults to
* `false`. */
preserveConstEnums?: boolean;
/** Remove all comments except copy-right header comments beginning with
* `/*!`. Defaults to `true`. */
removeComments?: boolean;
/** Include modules imported with `.json` extension. Defaults to `true`. */
resolveJsonModule?: boolean;
/** Specifies the root directory of input files. Only use to control the
* output directory structure with `outDir`. Defaults to `undefined`. */
rootDir?: string;
/** List of _root_ folders whose combined content represent the structure of
* the project at runtime. Defaults to `undefined`. */
rootDirs?: string[];
/** Generates corresponding `.map` file. Defaults to `false`. */
sourceMap?: boolean;
/** Specifies the location where debugger should locate TypeScript files
* instead of source locations. Use this flag if the sources will be located
* at run-time in a different location than that at design-time. The location
* specified will be embedded in the sourceMap to direct the debugger where
* the source files will be located. Defaults to `undefined`. */
sourceRoot?: string;
/** Enable all strict type checking options. Enabling `strict` enables
* `noImplicitAny`, `noImplicitThis`, `alwaysStrict`, `strictBindCallApply`,
* `strictNullChecks`, `strictFunctionTypes` and
* `strictPropertyInitialization`. Defaults to `true`. */
strict?: boolean;
/** Enable stricter checking of the `bind`, `call`, and `apply` methods on
* functions. Defaults to `true`. */
strictBindCallApply?: boolean;
/** Disable bivariant parameter checking for function types. Defaults to
* `true`. */
strictFunctionTypes?: boolean;
/** Ensure non-undefined class properties are initialized in the constructor.
* This option requires `strictNullChecks` be enabled in order to take effect.
* Defaults to `true`. */
strictPropertyInitialization?: boolean;
/** In strict null checking mode, the `null` and `undefined` values are not in
* the domain of every type and are only assignable to themselves and `any`
* (the one exception being that `undefined` is also assignable to `void`). */
strictNullChecks?: boolean;
/** Suppress excess property checks for object literals. Defaults to
* `false`. */
suppressExcessPropertyErrors?: boolean;
/** Suppress `noImplicitAny` errors for indexing objects lacking index
* signatures. */
suppressImplicitAnyIndexErrors?: boolean;
/** Specify ECMAScript target version. Defaults to `esnext`. */
target?:
| "es3"
| "es5"
| "es6"
| "es2015"
| "es2016"
| "es2017"
| "es2018"
| "es2019"
| "es2020"
| "esnext";
/** List of names of type definitions to include. Defaults to `undefined`. */
types?: string[];
}
/** The results of a transpile only command, where the `source` contains the
* emitted source, and `map` optionally contains the source map.
*/
export interface TranspileOnlyResult {
source: string;
map?: string;
}
/** Takes a set of TypeScript sources and resolves with a map where the key was
* the original file name provided in sources and the result contains the
* `source` and optionally the `map` from the transpile operation. This does no
* type checking and validation, it effectively "strips" the types from the
* file.
*
* const results = await Deno.transpileOnly({
* "foo.ts": `const foo: string = "foo";`
* });
*
* @param sources A map where the key is the filename and the value is the text
* to transpile. The filename is only used in the transpile and
* not resolved, for example to fill in the source name in the
* source map.
* @param options An option object of options to send to the compiler. This is
* a subset of ts.CompilerOptions which can be supported by Deno.
* Many of the options related to type checking and emitting
* type declaration files will have no impact on the output.
*/
export function transpileOnly(
sources: Record<string, string>,
options?: CompilerOptions
): Promise<Record<string, TranspileOnlyResult>>;
/** Takes a root module name, any optionally a record set of sources. Resolves
* with a compiled set of modules. If just a root name is provided, the modules
* will be resolved as if the root module had been passed on the command line.
*
* If sources are passed, all modules will be resolved out of this object, where
* the key is the module name and the value is the content. The extension of
* the module name will be used to determine the media type of the module.
*
* const [ maybeDiagnostics1, output1 ] = await Deno.compile("foo.ts");
*
* const [ maybeDiagnostics2, output2 ] = await Deno.compile("/foo.ts", {
* "/foo.ts": `export * from "./bar.ts";`,
* "/bar.ts": `export const bar = "bar";`
* });
*
* @param rootName The root name of the module which will be used as the
* "starting point". If no `sources` is specified, Deno will
* resolve the module externally as if the `rootName` had been
* specified on the command line.
* @param sources An optional key/value map of sources to be used when resolving
* modules, where the key is the module name, and the value is
* the source content. The extension of the key will determine
* the media type of the file when processing. If supplied,
* Deno will not attempt to resolve any modules externally.
* @param options An optional object of options to send to the compiler. This is
* a subset of ts.CompilerOptions which can be supported by Deno.
*/
export function compile(
rootName: string,
sources?: Record<string, string>,
options?: CompilerOptions
): Promise<[Diagnostic[] | undefined, Record<string, string>]>;
/** Takes a root module name, and optionally a record set of sources. Resolves
* with a single JavaScript string that is like the output of a `deno bundle`
* command. If just a root name is provided, the modules will be resolved as if
* the root module had been passed on the command line.
*
* If sources are passed, all modules will be resolved out of this object, where
* the key is the module name and the value is the content. The extension of the
* module name will be used to determine the media type of the module.
*
* const [ maybeDiagnostics1, output1 ] = await Deno.bundle("foo.ts");
*
* const [ maybeDiagnostics2, output2 ] = await Deno.bundle("/foo.ts", {
* "/foo.ts": `export * from "./bar.ts";`,
* "/bar.ts": `export const bar = "bar";`
* });
*
* @param rootName The root name of the module which will be used as the
* "starting point". If no `sources` is specified, Deno will
* resolve the module externally as if the `rootName` had been
* specified on the command line.
* @param sources An optional key/value map of sources to be used when resolving
* modules, where the key is the module name, and the value is
* the source content. The extension of the key will determine
* the media type of the file when processing. If supplied,
* Deno will not attempt to resolve any modules externally.
* @param options An optional object of options to send to the compiler. This is
* a subset of ts.CompilerOptions which can be supported by Deno.
*/
export function bundle(
rootName: string,
sources?: Record<string, string>,
options?: CompilerOptions
): Promise<[Diagnostic[] | undefined, string]>;
// @url js/deno.d.ts
export const args: string[];

6
cli/js/unit_test_runner.ts
View file

@ -9,14 +9,14 @@ import {
interface TestResult {
perms: string;
output: string;
output?: string;
result: number;
}
function permsToCliFlags(perms: Permissions): string[] {
return Object.keys(perms)
.map((key): string => {
if (!perms[key]) return "";
.map(key => {
if (!perms[key as keyof Permissions]) return "";
const cliFlag = key.replace(
/\.?([A-Z])/g,

1
cli/js/unit_tests.ts
View file

@ -9,6 +9,7 @@ import "./buffer_test.ts";
import "./build_test.ts";
import "./chmod_test.ts";
import "./chown_test.ts";
import "./compiler_api_test.ts";
import "./console_test.ts";
import "./copy_file_test.ts";
import "./custom_event_test.ts";

84
cli/js/util.ts
View file

@ -126,6 +126,7 @@ export function isObject(o: unknown): o is object {
}
// Returns whether o is iterable.
// @internal
export function isIterable<T, P extends keyof T, K extends T[P]>(
o: T
): o is T & Iterable<[P, K]> {
@ -224,6 +225,78 @@ export function splitNumberToParts(n: number): number[] {
return [lower, higher];
}
// Constants used by `normalizeString` and `resolvePath`
export const CHAR_DOT = 46; /* . */
export const CHAR_FORWARD_SLASH = 47; /* / */
/** Resolves `.` and `..` elements in a path with directory names */
export function normalizeString(
path: string,
allowAboveRoot: boolean,
separator: string,
isPathSeparator: (code: number) => boolean
): string {
let res = "";
let lastSegmentLength = 0;
let lastSlash = -1;
let dots = 0;
let code: number;
for (let i = 0, len = path.length; i <= len; ++i) {
if (i < len) code = path.charCodeAt(i);
else if (isPathSeparator(code!)) break;
else code = CHAR_FORWARD_SLASH;
if (isPathSeparator(code)) {
if (lastSlash === i - 1 || dots === 1) {
// NOOP
} else if (lastSlash !== i - 1 && dots === 2) {
if (
res.length < 2 ||
lastSegmentLength !== 2 ||
res.charCodeAt(res.length - 1) !== CHAR_DOT ||
res.charCodeAt(res.length - 2) !== CHAR_DOT
) {
if (res.length > 2) {
const lastSlashIndex = res.lastIndexOf(separator);
if (lastSlashIndex === -1) {
res = "";
lastSegmentLength = 0;
} else {
res = res.slice(0, lastSlashIndex);
lastSegmentLength = res.length - 1 - res.lastIndexOf(separator);
}
lastSlash = i;
dots = 0;
continue;
} else if (res.length === 2 || res.length === 1) {
res = "";
lastSegmentLength = 0;
lastSlash = i;
dots = 0;
continue;
}
}
if (allowAboveRoot) {
if (res.length > 0) res += `${separator}..`;
else res = "..";
lastSegmentLength = 2;
}
} else {
if (res.length > 0) res += separator + path.slice(lastSlash + 1, i);
else res = path.slice(lastSlash + 1, i);
lastSegmentLength = i - lastSlash - 1;
}
lastSlash = i;
dots = 0;
} else if (code === CHAR_DOT && dots !== -1) {
++dots;
} else {
dots = -1;
}
}
return res;
}
/** Return the common path shared by the `paths`.
*
* @param paths The set of paths to compare.
@ -269,3 +342,14 @@ export function humanFileSize(bytes: number): string {
} while (Math.abs(bytes) >= thresh && u < units.length - 1);
return `${bytes.toFixed(1)} ${units[u]}`;
}
// @internal
export function base64ToUint8Array(data: string): Uint8Array {
const binString = window.atob(data);
const size = binString.length;
const bytes = new Uint8Array(size);
for (let i = 0; i < size; i++) {
bytes[i] = binString.charCodeAt(i);
}
return bytes;
}

3
cli/msg.rs
View file

@ -97,5 +97,6 @@ pub fn enum_name_media_type(mt: MediaType) -> &'static str {
#[derive(Clone, Copy, PartialEq, Debug)]
pub enum CompilerRequestType {
Compile = 0,
Bundle = 1,
RuntimeCompile = 1,
RuntimeTranspile = 2,
}

102
cli/ops/compiler.rs
View file

@ -1,14 +1,21 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
use super::dispatch_json::{Deserialize, JsonOp, Value};
use crate::compilers::runtime_compile_async;
use crate::compilers::runtime_transpile_async;
use crate::futures::future::try_join_all;
use crate::msg;
use crate::ops::json_op;
use crate::state::ThreadSafeState;
use deno_core::Loader;
use deno_core::*;
use std::collections::HashMap;
pub fn init(i: &mut Isolate, s: &ThreadSafeState) {
i.register_op("cache", s.core_op(json_op(s.stateful_op(op_cache))));
i.register_op(
"resolve_modules",
s.core_op(json_op(s.stateful_op(op_resolve_modules))),
);
i.register_op(
"fetch_source_files",
s.core_op(json_op(s.stateful_op(op_fetch_source_files))),
@ -17,6 +24,8 @@ pub fn init(i: &mut Isolate, s: &ThreadSafeState) {
"fetch_asset",
s.core_op(json_op(s.stateful_op(op_fetch_asset))),
);
i.register_op("compile", s.core_op(json_op(s.stateful_op(op_compile))));
i.register_op("transpile", s.core_op(json_op(s.stateful_op(op_transpile))));
}
#[derive(Deserialize)]
@ -46,36 +55,62 @@ fn op_cache(
Ok(JsonOp::Sync(json!({})))
}
#[derive(Deserialize)]
struct FetchSourceFilesArgs {
#[derive(Deserialize, Debug)]
struct SpecifiersReferrerArgs {
specifiers: Vec<String>,
referrer: Option<String>,
}
fn op_resolve_modules(
state: &ThreadSafeState,
args: Value,
_data: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: SpecifiersReferrerArgs = serde_json::from_value(args)?;
// TODO(ry) Maybe a security hole. Only the compiler worker should have access
// to this. Need a test to demonstrate the hole.
let is_dyn_import = false;
let (referrer, is_main) = if let Some(referrer) = args.referrer {
(referrer, false)
} else {
("<unknown>".to_owned(), true)
};
let mut specifiers = vec![];
for specifier in &args.specifiers {
let resolved_specifier =
state.resolve(specifier, &referrer, is_main, is_dyn_import);
match resolved_specifier {
Ok(ms) => specifiers.push(ms.as_str().to_owned()),
Err(err) => return Err(err),
}
}
Ok(JsonOp::Sync(json!(specifiers)))
}
fn op_fetch_source_files(
state: &ThreadSafeState,
args: Value,
_data: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: FetchSourceFilesArgs = serde_json::from_value(args)?;
let args: SpecifiersReferrerArgs = serde_json::from_value(args)?;
// TODO(ry) Maybe a security hole. Only the compiler worker should have access
// to this. Need a test to demonstrate the hole.
let is_dyn_import = false;
let (referrer, ref_specifier) = if let Some(referrer) = args.referrer {
let ref_specifier = if let Some(referrer) = args.referrer {
let specifier = ModuleSpecifier::resolve_url(&referrer)
.expect("Referrer is not a valid specifier");
(referrer, Some(specifier))
Some(specifier)
} else {
// main script import
(".".to_string(), None)
None
};
let mut futures = vec![];
for specifier in &args.specifiers {
let resolved_specifier =
state.resolve(specifier, &referrer, false, is_dyn_import)?;
ModuleSpecifier::resolve_url(&specifier).expect("Invalid specifier");
let fut = state
.global_state
.file_fetcher
@ -137,3 +172,46 @@ fn op_fetch_asset(
panic!("op_fetch_asset bad asset {}", args.name)
}
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct CompileArgs {
root_name: String,
sources: Option<HashMap<String, String>>,
bundle: bool,
options: Option<String>,
}
fn op_compile(
state: &ThreadSafeState,
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: CompileArgs = serde_json::from_value(args)?;
Ok(JsonOp::Async(runtime_compile_async(
state.global_state.clone(),
&args.root_name,
&args.sources,
args.bundle,
&args.options,
)))
}
#[derive(Deserialize, Debug)]
struct TranspileArgs {
sources: HashMap<String, String>,
options: Option<String>,
}
fn op_transpile(
state: &ThreadSafeState,
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: TranspileArgs = serde_json::from_value(args)?;
Ok(JsonOp::Async(runtime_transpile_async(
state.global_state.clone(),
&args.sources,
&args.options,
)))
}

9
cli/tests/error_011_bad_module_specifier.ts.out
View file

@ -1,5 +1,8 @@
[WILDCARD]error: Uncaught ImportPrefixMissing: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_011_bad_module_specifier.ts"
[WILDCARD]dispatch_json.ts:[WILDCARD]
at DenoError ([WILDCARD]errors.ts:[WILDCARD])
at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
at sendAsync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD])
at DenoError ($deno$/errors.ts:[WILDCARD])
at unwrapResponse ($deno$/dispatch_json.ts:[WILDCARD])
at sendSync ($deno$/dispatch_json.ts:[WILDCARD])
at resolveModules ($deno$/compiler_imports.ts:[WILDCARD])
at processImports ($deno$/compiler_imports.ts:[WILDCARD])
at processImports ($deno$/compiler_imports.ts:[WILDCARD])

9
cli/tests/error_012_bad_dynamic_import_specifier.ts.out
View file

@ -1,5 +1,8 @@
[WILDCARD]error: Uncaught ImportPrefixMissing: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts"
[WILDCARD]dispatch_json.ts:[WILDCARD]
at DenoError ([WILDCARD]errors.ts:[WILDCARD])
at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
at sendAsync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD])
at DenoError ($deno$/errors.ts:[WILDCARD])
at unwrapResponse ($deno$/dispatch_json.ts:[WILDCARD])
at sendSync ($deno$/dispatch_json.ts:[WILDCARD])
at resolveModules ($deno$/compiler_imports.ts:[WILDCARD])
at processImports ($deno$/compiler_imports.ts:[WILDCARD])
at processImports ($deno$/compiler_imports.ts:[WILDCARD])

9
cli/tests/error_type_definitions.ts.out
View file

@ -1,5 +1,8 @@
[WILDCARD]error: Uncaught ImportPrefixMissing: relative import path "baz" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/type_definitions/bar.d.ts"
[WILDCARD]dispatch_json.ts:[WILDCARD]
at DenoError ([WILDCARD]errors.ts:[WILDCARD])
at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
at sendAsync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD])
at DenoError ($deno$/errors.ts:[WILDCARD])
at unwrapResponse ($deno$/dispatch_json.ts:[WILDCARD])
at sendSync ($deno$/dispatch_json.ts:[WILDCARD])
at resolveModules ($deno$/compiler_imports.ts:[WILDCARD])
at processImports ($deno$/compiler_imports.ts:[WILDCARD])
at processImports ($deno$/compiler_imports.ts:[WILDCARD])

126
std/manual.md
View file

@ -897,6 +897,132 @@ import { fib } from "./fib.wasm";
console.log(fib(20));
```
## Compiler API
Deno supports runtime access to the built in TypeScript compiler. There are
three methods in the `Deno` namespace that provide this access.
### `Deno.compile()`
This works similar to `deno fetch` in that it can fetch code, compile it, but
not run it. It takes up to three arguments, the `rootName`, optionally
`sources`, and optionally `options`. The `rootName` is the root module which
will be used to generate the resulting program. This is like module name you
would pass on the command line in `deno --reload run example.ts`. The `sources`
is a hash where the key is the fully qualified module name, and the value is the
text source of the module. If `sources` is passed, Deno will resolve all the
modules from within that hash and not attempt to resolve them outside of Deno.
If `sources` are not provided, Deno will resolve modules as if the root module
had been passed on the command line. Deno will also cache any of these
resources. The `options` argument is a set of options of type
`Deno.CompilerOptions`, which is a subset of the TypeScript compiler options
which can be supported by Deno.
The method resolves with a tuple where the first argument is any diagnostics
(syntax or type errors) related to the code, and a map of the code, where the
key would be the output filename and the value would be the content.
An example of providing sources:
```ts
const [diagnostics, emitMap] = await Deno.compile("/foo.ts", {
"/foo.ts": `import * as bar from "./bar.ts";\nconsole.log(bar);\n`,
"/bar.ts": `export const bar = "bar";\n`
});
assert(diagnostics == null); // ensuring no diagnostics are returned
console.log(emitMap);
```
We would expect map to contain 4 "files", named `/foo.js.map`, `/foo.js`,
`/bar.js.map`, and `/bar.js`.
When not supplying resources, you can use local or remote modules, just like you
could do on the command line. So you could do something like this:
```ts
const [diagnostics, emitMap] = await Deno.compile(
"https://deno.land/std/examples/welcome.ts"
);
```
We should get back in the `emitMap` a simple `console.log()` statement.
### `Deno.bundle()`
This works a lot like `deno bundle` does on the command line. It is also like
`Deno.compile()`, except instead of returning a map of files, it returns a
single string, which is a self-contained JavaScript ES module which will include
all of the code that was provided or resolved as well as exports of all the
exports of the root module that was provided. It takes up to three arguments,
the `rootName`, optionally `sources`, and optionally `options`. The `rootName`
is the root module which will be used to generate the resulting program. This is
like module name you would pass on the command line in `deno bundle example.ts`.
The `sources` is a hash where the key is the fully qualified module name, and
the value is the text source of the module. If `sources` is passed, Deno will
resolve all the modules from within that hash and not attempt to resolve them
outside of Deno. If `sources` are not provided, Deno will resolve modules as if
the root module had been passed on the command line. Deno will also cache any of
these resources. The `options` argument is a set of options of type
`Deno.CompilerOptions`, which is a subset of the TypeScript compiler options
which can be supported by Deno.
An example of providing sources:
```ts
const [diagnostics, emit] = await Deno.compile("/foo.ts", {
"/foo.ts": `import * as bar from "./bar.ts";\nconsole.log(bar);\n`,
"/bar.ts": `export const bar = "bar";\n`
});
assert(diagnostics == null); // ensuring no diagnostics are returned
console.log(emit);
```
We would expect `emit` to be the text for an ES module, which would contain the
output sources for both modules.
When not supplying resources, you can use local or remote modules, just like you
could do on the command line. So you could do something like this:
```ts
const [diagnostics, emit] = await Deno.compile(
"https://deno.land/std/http/server.ts"
);
```
We should get back in `emit` a self contained JavaScript ES module with all of
its dependencies resolved and exporting the same exports as the source module.
### `Deno.transpileOnly()`
This is based off of the TypeScript function `transpileModule()`. All this does
is "erase" any types from the modules and emit JavaScript. There is no type
checking and no resolution of dependencies. It accepts up to two arguments, the
first is a hash where the key is the module name and the value is the contents.
The only purpose of the module name is when putting information into a source
map, of what the source file name was. The second is optionally `options` which
is of type `Deno.CompilerOptions`. This is a subset of options which can be
supported by Deno. It resolves with a map where the key is the source module
name supplied, and the value is an object with a property of `source` which is
the output contents of the module, and optionally `map` which would be the
source map. By default, source maps are output, but can be turned off via the
`options` argument.
An example:
```ts
const result = await Deno.transpileOnly({
"/foo.ts": `enum Foo { Foo, Bar, Baz };\n`
});
console.log(result["/foo.ts"].source);
console.log(result["/foo.ts"].map);
```
We would expect the `enum` would be rewritten to an IIFE which constructs the
enumerable, and the map to be defined.
## Program lifecycle
Deno supports browser compatible lifecycle events: `load` and `unload`. You can