// Copyright 2018 the Deno authors. All rights reserved. MIT license.
///
import * as ts from "typescript";
import { assetSourceCode } from "./assets";
import * as deno from "./deno";
import { globalEval } from "./global-eval";
import { libdeno } from "./libdeno";
import { window } from "./globals";
import * as os from "./os";
import { RawSourceMap } from "./types";
import { assert, log, notImplemented } from "./util";
import * as sourceMaps from "./v8_source_maps";
const EOL = "\n";
const ASSETS = "$asset$";
// tslint:disable:no-any
type AmdCallback = (...args: any[]) => void;
type AmdErrback = (err: any) => void;
export type AmdFactory = (...args: any[]) => object | void;
// tslint:enable:no-any
export type AmdDefine = (deps: ModuleSpecifier[], factory: AmdFactory) => void;
type AMDRequire = (
deps: ModuleSpecifier[],
callback: AmdCallback,
errback: AmdErrback
) => void;
/**
* The location that a module is being loaded from. This could be a directory,
* like `.`, or it could be a module specifier like
* `http://gist.github.com/somefile.ts`
*/
type ContainingFile = string;
/**
* The internal local filename of a compiled module. It will often be something
* like `/home/ry/.deno/gen/f7b4605dfbc4d3bb356e98fda6ceb1481e4a8df5.js`
*/
type ModuleFileName = string;
/**
* The original resolved resource name.
* Path to cached module file or URL from which dependency was retrieved
*/
type ModuleId = string;
/**
* The external name of a module - could be a URL or could be a relative path.
* Examples `http://gist.github.com/somefile.ts` or `./somefile.ts`
*/
type ModuleSpecifier = string;
/**
* The compiled source code which is cached in `.deno/gen/`
*/
type OutputCode = string;
/**
* The original source code
*/
type SourceCode = string;
/**
* Abstraction of the APIs required from the `os` module so they can be
* easily mocked.
* @internal
*/
export interface Os {
codeCache: typeof os.codeCache;
codeFetch: typeof os.codeFetch;
exit: typeof os.exit;
}
/**
* Abstraction of the APIs required from the `typescript` module so they can
* be easily mocked.
* @internal
*/
export interface Ts {
createLanguageService: typeof ts.createLanguageService;
/* tslint:disable-next-line:max-line-length */
formatDiagnosticsWithColorAndContext: typeof ts.formatDiagnosticsWithColorAndContext;
}
/**
* A simple object structure for caching resolved modules and their contents.
*
* Named `ModuleMetaData` to clarify it is just a representation of meta data of
* the module, not the actual module instance.
*/
export class ModuleMetaData implements ts.IScriptSnapshot {
public deps?: ModuleFileName[];
public readonly exports = {};
public factory?: AmdFactory;
public gatheringDeps = false;
public hasRun = false;
public scriptVersion = "";
constructor(
public readonly moduleId: ModuleId,
public readonly fileName: ModuleFileName,
public readonly sourceCode: SourceCode = "",
public outputCode: OutputCode = ""
) {
if (outputCode !== "" || fileName.endsWith(".d.ts")) {
this.scriptVersion = "1";
}
}
public getText(start: number, end: number): string {
return this.sourceCode.substring(start, end);
}
public getLength(): number {
return this.sourceCode.length;
}
public getChangeRange(): undefined {
// Required `IScriptSnapshot` API, but not implemented/needed in deno
return undefined;
}
}
/**
* A singleton class that combines the TypeScript Language Service host API
* with Deno specific APIs to provide an interface for compiling and running
* TypeScript and JavaScript modules.
*/
export class DenoCompiler
implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
// Modules are usually referenced by their ModuleSpecifier and ContainingFile,
// and keeping a map of the resolved module file name allows more efficient
// future resolution
private readonly _fileNamesMap = new Map<
ContainingFile,
Map
>();
// A reference to global eval, so it can be monkey patched during testing
private _globalEval = globalEval;
// A reference to the log utility, so it can be monkey patched during testing
private _log = log;
// A map of module file names to module meta data
private readonly _moduleMetaDataMap = new Map<
ModuleFileName,
ModuleMetaData
>();
// TODO ideally this are not static and can be influenced by command line
// arguments
private readonly _options: Readonly = {
allowJs: true,
module: ts.ModuleKind.AMD,
outDir: "$deno$",
// TODO https://github.com/denoland/deno/issues/23
inlineSourceMap: true,
inlineSources: true,
stripComments: true,
target: ts.ScriptTarget.ESNext
};
// A reference to the `./os.ts` module, so it can be monkey patched during
// testing
private _os: Os = os;
// Contains a queue of modules that have been resolved, but not yet
// run
private _runQueue: ModuleMetaData[] = [];
// Used to contain the script file we are currently running
private _scriptFileNames: string[] = [];
// A reference to the TypeScript LanguageService instance so it can be
// monkey patched during testing
private _service: ts.LanguageService;
// A reference to `typescript` module so it can be monkey patched during
// testing
private _ts: Ts = ts;
// A reference to the global scope so it can be monkey patched during
// testing
private _window = window;
// Flags forcing recompilation of TS code
public recompile = false;
/**
* Drain the run queue, retrieving the arguments for the module
* factory and calling the module's factory.
*/
private _drainRunQueue(): void {
this._log(
"compiler._drainRunQueue",
this._runQueue.map(metaData => metaData.fileName)
);
let moduleMetaData: ModuleMetaData | undefined;
while ((moduleMetaData = this._runQueue.shift())) {
assert(
moduleMetaData.factory != null,
"Cannot run module without factory."
);
assert(moduleMetaData.hasRun === false, "Module has already been run.");
// asserts not tracked by TypeScripts, so using not null operator
moduleMetaData.factory!(...this._getFactoryArguments(moduleMetaData));
moduleMetaData.hasRun = true;
}
}
/**
* Get the dependencies for a given module, but don't run the module,
* just add the module factory to the run queue.
*/
private _gatherDependencies(moduleMetaData: ModuleMetaData): void {
this._log("compiler._resolveDependencies", moduleMetaData.fileName);
// if the module has already run, we can short circuit.
// it is intentional though that if we have already resolved dependencies,
// we won't short circuit, as something may have changed, or we might have
// only collected the dependencies to be able to able to obtain the graph of
// dependencies
if (moduleMetaData.hasRun) {
return;
}
this._window.define = this._makeDefine(moduleMetaData);
this._globalEval(this.compile(moduleMetaData));
this._window.define = undefined;
}
/**
* Retrieve the arguments to pass a module's factory function.
*/
// tslint:disable-next-line:no-any
private _getFactoryArguments(moduleMetaData: ModuleMetaData): any[] {
if (!moduleMetaData.deps) {
throw new Error("Cannot get arguments until dependencies resolved.");
}
return moduleMetaData.deps.map(dep => {
if (dep === "require") {
return this._makeLocalRequire(moduleMetaData);
}
if (dep === "exports") {
return moduleMetaData.exports;
}
if (dep in DenoCompiler._builtins) {
return DenoCompiler._builtins[dep];
}
const dependencyMetaData = this._getModuleMetaData(dep);
assert(dependencyMetaData != null, `Missing dependency "${dep}".`);
// TypeScript does not track assert, therefore using not null operator
return dependencyMetaData!.exports;
});
}
/**
* The TypeScript language service often refers to the resolved fileName of
* a module, this is a shortcut to avoid unnecessary module resolution logic
* for modules that may have been initially resolved by a `moduleSpecifier`
* and `containingFile`. Also, `resolveModule()` throws when the module
* cannot be resolved, which isn't always valid when dealing with the
* TypeScript compiler, but the TypeScript compiler shouldn't be asking about
* external modules that we haven't told it about yet.
*/
private _getModuleMetaData(
fileName: ModuleFileName
): ModuleMetaData | undefined {
return this._moduleMetaDataMap.has(fileName)
? this._moduleMetaDataMap.get(fileName)
: fileName.startsWith(ASSETS)
? this.resolveModule(fileName, "")
: undefined;
}
/**
* Create a localized AMD `define` function and return it.
*/
private _makeDefine(moduleMetaData: ModuleMetaData): AmdDefine {
return (deps: ModuleSpecifier[], factory: AmdFactory): void => {
this._log("compiler.localDefine", moduleMetaData.fileName);
moduleMetaData.factory = factory;
// when there are circular dependencies, we need to skip recursing the
// dependencies
moduleMetaData.gatheringDeps = true;
// we will recursively resolve the dependencies for any modules
moduleMetaData.deps = deps.map(dep => {
if (
dep === "require" ||
dep === "exports" ||
dep in DenoCompiler._builtins
) {
return dep;
}
const dependencyMetaData = this.resolveModule(
dep,
moduleMetaData.fileName
);
if (!dependencyMetaData.gatheringDeps) {
this._gatherDependencies(dependencyMetaData);
}
return dependencyMetaData.fileName;
});
moduleMetaData.gatheringDeps = false;
if (!this._runQueue.includes(moduleMetaData)) {
this._runQueue.push(moduleMetaData);
}
};
}
/**
* Returns a require that specifically handles the resolution of a transpiled
* emit of a dynamic ES `import()` from TypeScript.
*/
private _makeLocalRequire(moduleMetaData: ModuleMetaData): AMDRequire {
return (
deps: ModuleSpecifier[],
callback: AmdCallback,
errback: AmdErrback
): void => {
log("localRequire", deps);
assert(
deps.length === 1,
"Local require requires exactly one dependency."
);
const [moduleSpecifier] = deps;
try {
const requiredMetaData = this.run(
moduleSpecifier,
moduleMetaData.fileName
);
callback(requiredMetaData.exports);
} catch (e) {
errback(e);
}
};
}
/**
* Given a `moduleSpecifier` and `containingFile` retrieve the cached
* `fileName` for a given module. If the module has yet to be resolved
* this will return `undefined`.
*/
private _resolveFileName(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleFileName | undefined {
this._log("compiler.resolveFileName", { moduleSpecifier, containingFile });
const innerMap = this._fileNamesMap.get(containingFile);
if (innerMap) {
return innerMap.get(moduleSpecifier);
}
return undefined;
}
/**
* Resolve the `fileName` for a given `moduleSpecifier` and `containingFile`
*/
private _resolveModuleName(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleFileName | undefined {
const moduleMetaData = this.resolveModule(moduleSpecifier, containingFile);
return moduleMetaData ? moduleMetaData.fileName : undefined;
}
/**
* Caches the resolved `fileName` in relationship to the `moduleSpecifier`
* and `containingFile` in order to reduce calls to the privileged side
* to retrieve the contents of a module.
*/
private _setFileName(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile,
fileName: ModuleFileName
): void {
this._log("compiler.setFileName", { moduleSpecifier, containingFile });
let innerMap = this._fileNamesMap.get(containingFile);
if (!innerMap) {
innerMap = new Map();
this._fileNamesMap.set(containingFile, innerMap);
}
innerMap.set(moduleSpecifier, fileName);
}
/**
* Setup being able to map back source references back to their source
*
* TODO is this the best place for this? It is tightly coupled to how the
* compiler works, but it is also tightly coupled to how the whole runtime
* environment is bootstrapped. It also needs efficient access to the
* `outputCode` of the module information, which exists inside of the
* compiler instance.
*/
private _setupSourceMaps(): void {
sourceMaps.install({
installPrepareStackTrace: true,
getGeneratedContents: (fileName: string): string | RawSourceMap => {
this._log("compiler.getGeneratedContents", fileName);
if (fileName === "gen/bundle/main.js") {
assert(libdeno.mainSource.length > 0);
return libdeno.mainSource;
} else if (fileName === "main.js.map") {
return libdeno.mainSourceMap;
} else if (fileName === "deno_main.js") {
return "";
} else {
const moduleMetaData = this._moduleMetaDataMap.get(fileName);
if (!moduleMetaData) {
this._log("compiler.getGeneratedContents cannot find", fileName);
return "";
}
return moduleMetaData.outputCode;
}
}
});
}
private constructor() {
if (DenoCompiler._instance) {
throw new TypeError("Attempt to create an additional compiler.");
}
this._service = this._ts.createLanguageService(this);
this._setupSourceMaps();
}
// Deno specific compiler API
/**
* Retrieve the output of the TypeScript compiler for a given module and
* cache the result. Re-compilation can be forced using '--recompile' flag.
*/
compile(moduleMetaData: ModuleMetaData): OutputCode {
const recompile = !!this.recompile;
if (!recompile && moduleMetaData.outputCode) {
return moduleMetaData.outputCode;
}
const { fileName, sourceCode, moduleId } = moduleMetaData;
console.warn("Compiling", moduleId);
const service = this._service;
const output = service.getEmitOutput(fileName);
// Get the relevant diagnostics - this is 3x faster than
// `getPreEmitDiagnostics`.
const diagnostics = [
...service.getCompilerOptionsDiagnostics(),
...service.getSyntacticDiagnostics(fileName),
...service.getSemanticDiagnostics(fileName)
];
if (diagnostics.length > 0) {
const errMsg = this._ts.formatDiagnosticsWithColorAndContext(
diagnostics,
this
);
console.log(errMsg);
// All TypeScript errors are terminal for deno
this._os.exit(1);
}
assert(!output.emitSkipped, "The emit was skipped for an unknown reason.");
// Currently we are inlining source maps, there should be only 1 output file
// See: https://github.com/denoland/deno/issues/23
assert(
output.outputFiles.length === 1,
"Only single file should be output."
);
const [outputFile] = output.outputFiles;
const outputCode = (moduleMetaData.outputCode = `${
outputFile.text
}\n//# sourceURL=${fileName}`);
moduleMetaData.scriptVersion = "1";
this._os.codeCache(fileName, sourceCode, outputCode);
return moduleMetaData.outputCode;
}
/**
* For a given module specifier and containing file, return a list of absolute
* identifiers for dependent modules that are required by this module.
*/
getModuleDependencies(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleFileName[] {
assert(
this._runQueue.length === 0,
"Cannot get dependencies with modules queued to be run."
);
const moduleMetaData = this.resolveModule(moduleSpecifier, containingFile);
assert(
!moduleMetaData.hasRun,
"Cannot get dependencies for a module that has already been run."
);
this._gatherDependencies(moduleMetaData);
const dependencies = this._runQueue.map(
moduleMetaData => moduleMetaData.moduleId
);
// empty the run queue, to free up references to factories we have collected
// and to ensure that if there is a further invocation of `.run()` the
// factories don't get called
this._runQueue = [];
return dependencies;
}
/**
* Given a `moduleSpecifier` and `containingFile`, resolve the module and
* return the `ModuleMetaData`.
*/
resolveModule(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleMetaData {
this._log("compiler.resolveModule", { moduleSpecifier, containingFile });
assert(moduleSpecifier != null && moduleSpecifier.length > 0);
let fileName = this._resolveFileName(moduleSpecifier, containingFile);
if (fileName && this._moduleMetaDataMap.has(fileName)) {
return this._moduleMetaDataMap.get(fileName)!;
}
let moduleId: ModuleId;
let sourceCode: SourceCode;
let outputCode: OutputCode | null;
if (
moduleSpecifier.startsWith(ASSETS) ||
containingFile.startsWith(ASSETS)
) {
// Assets are compiled into the runtime javascript bundle.
// we _know_ `.pop()` will return a string, but TypeScript doesn't so
// not null assertion
moduleId = moduleSpecifier.split("/").pop()!;
const assetName = moduleId.includes(".") ? moduleId : `${moduleId}.d.ts`;
assert(assetName in assetSourceCode, `No such asset "${assetName}"`);
sourceCode = assetSourceCode[assetName];
fileName = `${ASSETS}/${assetName}`;
outputCode = "";
} else {
// We query Rust with a CodeFetch message. It will load the sourceCode,
// and if there is any outputCode cached, will return that as well.
const fetchResponse = this._os.codeFetch(moduleSpecifier, containingFile);
moduleId = fetchResponse.moduleName!;
fileName = fetchResponse.filename!;
sourceCode = fetchResponse.sourceCode!;
outputCode = fetchResponse.outputCode!;
}
assert(sourceCode!.length > 0);
this._log("resolveModule sourceCode length:", sourceCode.length);
this._log("resolveModule has outputCode:", outputCode! != null);
this._setFileName(moduleSpecifier, containingFile, fileName);
if (fileName && this._moduleMetaDataMap.has(fileName)) {
return this._moduleMetaDataMap.get(fileName)!;
}
const moduleMetaData = new ModuleMetaData(
moduleId,
fileName,
sourceCode,
outputCode
);
this._moduleMetaDataMap.set(fileName, moduleMetaData);
return moduleMetaData;
}
/**
* Load and run a module and all of its dependencies based on a module
* specifier and a containing file
*/
run(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleMetaData {
this._log("compiler.run", { moduleSpecifier, containingFile });
const moduleMetaData = this.resolveModule(moduleSpecifier, containingFile);
this._scriptFileNames = [moduleMetaData.fileName];
if (!moduleMetaData.deps) {
this._gatherDependencies(moduleMetaData);
}
this._drainRunQueue();
return moduleMetaData;
}
// TypeScript Language Service and Format Diagnostic Host API
getCanonicalFileName(fileName: string): string {
this._log("getCanonicalFileName", fileName);
return fileName;
}
getCompilationSettings(): ts.CompilerOptions {
this._log("getCompilationSettings()");
return this._options;
}
getNewLine(): string {
return EOL;
}
getScriptFileNames(): string[] {
// This is equal to `"files"` in the `tsconfig.json`, therefore we only need
// to include the actual base source files we are evaluating at the moment,
// which would be what is set during the `.run()`
return this._scriptFileNames;
}
getScriptKind(fileName: ModuleFileName): ts.ScriptKind {
this._log("getScriptKind()", fileName);
const suffix = fileName.substr(fileName.lastIndexOf(".") + 1);
switch (suffix) {
case "ts":
return ts.ScriptKind.TS;
case "js":
return ts.ScriptKind.JS;
case "json":
return ts.ScriptKind.JSON;
default:
return this._options.allowJs ? ts.ScriptKind.JS : ts.ScriptKind.TS;
}
}
getScriptVersion(fileName: ModuleFileName): string {
this._log("getScriptVersion()", fileName);
const moduleMetaData = this._getModuleMetaData(fileName);
return (moduleMetaData && moduleMetaData.scriptVersion) || "";
}
getScriptSnapshot(fileName: ModuleFileName): ts.IScriptSnapshot | undefined {
this._log("getScriptSnapshot()", fileName);
return this._getModuleMetaData(fileName);
}
getCurrentDirectory(): string {
this._log("getCurrentDirectory()");
return "";
}
getDefaultLibFileName(): string {
this._log("getDefaultLibFileName()");
const moduleSpecifier = "globals.d.ts";
const moduleMetaData = this.resolveModule(moduleSpecifier, ASSETS);
return moduleMetaData.fileName;
}
useCaseSensitiveFileNames(): boolean {
this._log("useCaseSensitiveFileNames()");
return true;
}
readFile(path: string): string | undefined {
this._log("readFile()", path);
return notImplemented();
}
fileExists(fileName: string): boolean {
const moduleMetaData = this._getModuleMetaData(fileName);
const exists = moduleMetaData != null;
this._log("fileExists()", fileName, exists);
return exists;
}
resolveModuleNames(
moduleNames: ModuleSpecifier[],
containingFile: ContainingFile
): ts.ResolvedModule[] {
this._log("resolveModuleNames()", { moduleNames, containingFile });
return moduleNames.map(name => {
let resolvedFileName;
if (name === "deno" || name === "compiler") {
// builtin modules are part of `globals.d.ts`
resolvedFileName = this._resolveModuleName("globals.d.ts", ASSETS);
} else if (name === "typescript") {
resolvedFileName = this._resolveModuleName("typescript.d.ts", ASSETS);
} else {
resolvedFileName = this._resolveModuleName(name, containingFile);
}
// According to the interface we shouldn't return `undefined` but if we
// fail to return the same length of modules to those we cannot resolve
// then TypeScript fails on an assertion that the lengths can't be
// different, so we have to return an "empty" resolved module
// TODO: all this does is push the problem downstream, and TypeScript
// will complain it can't identify the type of the file and throw
// a runtime exception, so we need to handle missing modules better
resolvedFileName = resolvedFileName || "";
// This flags to the compiler to not go looking to transpile functional
// code, anything that is in `/$asset$/` is just library code
const isExternalLibraryImport = resolvedFileName.startsWith(ASSETS);
// TODO: we should be returning a ts.ResolveModuleFull
return { resolvedFileName, isExternalLibraryImport };
});
}
// Deno specific static properties and methods
/**
* Built in modules which can be returned to external modules
*
* Placed as a private static otherwise we get use before
* declared with the `DenoCompiler`
*/
// tslint:disable-next-line:no-any
private static _builtins: { [mid: string]: any } = {
typescript: ts,
deno,
compiler: { DenoCompiler, ModuleMetaData }
};
private static _instance: DenoCompiler | undefined;
/**
* Returns the instance of `DenoCompiler` or creates a new instance.
*/
static instance(): DenoCompiler {
return (
DenoCompiler._instance || (DenoCompiler._instance = new DenoCompiler())
);
}
}