// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import * as ts from "typescript"; import * as msg from "gen/cli/msg_generated"; import { window } from "./window"; import { assetSourceCode } from "./assets"; import { bold, cyan, yellow } from "./colors"; import { Console } from "./console"; import { core } from "./core"; import { cwd } from "./dir"; import { sendSync } from "./dispatch"; import * as flatbuffers from "./flatbuffers"; import * as os from "./os"; import { TextDecoder, TextEncoder } from "./text_encoding"; import { clearTimer, setTimeout } from "./timers"; import { postMessage, workerClose, workerMain } from "./workers"; import { assert, log, notImplemented } from "./util"; const EOL = "\n"; const ASSETS = "$asset$"; const LIB_RUNTIME = `${ASSETS}/lib.deno_runtime.d.ts`; // An instance of console const console = new Console(core.print); /** The location that a module is being loaded from. This could be a directory, * like `.`, or it could be a module specifier like * `http://gist.github.com/somefile.ts` */ type ContainingFile = string; /** The internal local filename of a compiled module. It will often be something * like `/home/ry/.deno/gen/f7b4605dfbc4d3bb356e98fda6ceb1481e4a8df5.js` */ type ModuleFileName = string; /** The original resolved resource name. * Path to cached module file or URL from which dependency was retrieved */ type ModuleId = string; /** The external name of a module - could be a URL or could be a relative path. * Examples `http://gist.github.com/somefile.ts` or `./somefile.ts` */ type ModuleSpecifier = string; /** The compiled source code which is cached in `.deno/gen/` */ type OutputCode = string; /** The original source code */ type SourceCode = string; /** The output source map */ type SourceMap = string; /** The format of the work message payload coming from the privileged side */ interface CompilerLookup { specifier: ModuleSpecifier; referrer: ContainingFile; cmdId: number; } /** Abstraction of the APIs required from the `os` module so they can be * easily mocked. */ interface Os { fetchModuleMetaData: typeof os.fetchModuleMetaData; exit: typeof os.exit; noColor: typeof os.noColor; } /** Abstraction of the APIs required from the `typescript` module so they can * be easily mocked. */ interface Ts { convertCompilerOptionsFromJson: typeof ts.convertCompilerOptionsFromJson; createLanguageService: typeof ts.createLanguageService; formatDiagnosticsWithColorAndContext: typeof ts.formatDiagnosticsWithColorAndContext; formatDiagnostics: typeof ts.formatDiagnostics; parseConfigFileTextToJson: typeof ts.parseConfigFileTextToJson; } /** Options that either do nothing in Deno, or would cause undesired behavior * if modified. */ const ignoredCompilerOptions: ReadonlyArray = [ "allowSyntheticDefaultImports", "baseUrl", "build", "composite", "declaration", "declarationDir", "declarationMap", "diagnostics", "downlevelIteration", "emitBOM", "emitDeclarationOnly", "esModuleInterop", "extendedDiagnostics", "forceConsistentCasingInFileNames", "help", "importHelpers", "incremental", "inlineSourceMap", "inlineSources", "init", "isolatedModules", "lib", "listEmittedFiles", "listFiles", "mapRoot", "maxNodeModuleJsDepth", "module", "moduleResolution", "newLine", "noEmit", "noEmitHelpers", "noEmitOnError", "noLib", "noResolve", "out", "outDir", "outFile", "paths", "preserveSymlinks", "preserveWatchOutput", "pretty", "rootDir", "rootDirs", "showConfig", "skipDefaultLibCheck", "skipLibCheck", "sourceMap", "sourceRoot", "stripInternal", "target", "traceResolution", "tsBuildInfoFile", "types", "typeRoots", "version", "watch" ]; /** A simple object structure for caching resolved modules and their contents. * * Named `ModuleMetaData` to clarify it is just a representation of meta data of * the module, not the actual module instance. */ class ModuleMetaData implements ts.IScriptSnapshot { public scriptVersion = ""; constructor( public readonly moduleId: ModuleId, public readonly fileName: ModuleFileName, public readonly mediaType: msg.MediaType, public readonly sourceCode: SourceCode = "", public outputCode: OutputCode = "", public sourceMap: SourceMap = "" ) { if (outputCode !== "" || fileName.endsWith(".d.ts")) { this.scriptVersion = "1"; } } /** TypeScript IScriptSnapshot Interface */ public getText(start: number, end: number): string { return start === 0 && end === this.sourceCode.length ? this.sourceCode : this.sourceCode.substring(start, end); } public getLength(): number { return this.sourceCode.length; } public getChangeRange(): undefined { // Required `IScriptSnapshot` API, but not implemented/needed in deno return undefined; } } /** Returns the TypeScript Extension enum for a given media type. */ function getExtension( fileName: ModuleFileName, mediaType: msg.MediaType ): ts.Extension { switch (mediaType) { case msg.MediaType.JavaScript: return ts.Extension.Js; case msg.MediaType.TypeScript: return fileName.endsWith(".d.ts") ? ts.Extension.Dts : ts.Extension.Ts; case msg.MediaType.Json: return ts.Extension.Json; case msg.MediaType.Unknown: default: throw TypeError("Cannot resolve extension."); } } /** Generate output code for a provided JSON string along with its source. */ function jsonEsmTemplate( jsonString: string, sourceFileName: string ): OutputCode { return ( `const _json = JSON.parse(\`${jsonString}\`);\n` + `export default _json;\n` + `//# sourceURL=${sourceFileName}\n` ); } /** A singleton class that combines the TypeScript Language Service host API * with Deno specific APIs to provide an interface for compiling and running * TypeScript and JavaScript modules. */ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost { // Modules are usually referenced by their ModuleSpecifier and ContainingFile, // and keeping a map of the resolved module file name allows more efficient // future resolution private readonly _fileNamesMap = new Map< ContainingFile, Map >(); // A reference to the log utility, so it can be monkey patched during testing private _log = log; // A map of module file names to module meta data private readonly _moduleMetaDataMap = new Map< ModuleFileName, ModuleMetaData >(); // TODO ideally this are not static and can be influenced by command line // arguments private readonly _options: ts.CompilerOptions = { allowJs: true, allowNonTsExtensions: true, checkJs: true, esModuleInterop: true, module: ts.ModuleKind.ESNext, outDir: "$deno$", resolveJsonModule: true, sourceMap: true, stripComments: true, target: ts.ScriptTarget.ESNext }; // A reference to the `./os.ts` module, so it can be monkey patched during // testing private _os: Os = os; // Used to contain the script file we are currently running private _scriptFileNames: string[] = []; // A reference to the TypeScript LanguageService instance so it can be // monkey patched during testing private _service: ts.LanguageService; // A reference to `typescript` module so it can be monkey patched during // testing private _ts: Ts = ts; private readonly _assetsSourceCode: { [key: string]: string }; /** The TypeScript language service often refers to the resolved fileName of * a module, this is a shortcut to avoid unnecessary module resolution logic * for modules that may have been initially resolved by a `moduleSpecifier` * and `containingFile`. Also, `resolveModule()` throws when the module * cannot be resolved, which isn't always valid when dealing with the * TypeScript compiler, but the TypeScript compiler shouldn't be asking about * external modules that we haven't told it about yet. */ private _getModuleMetaData( fileName: ModuleFileName ): ModuleMetaData | undefined { return ( this._moduleMetaDataMap.get(fileName) || (fileName.startsWith(ASSETS) ? this._resolveModule(fileName, "") : undefined) ); } /** Log TypeScript diagnostics to the console and exit */ private _logDiagnostics(diagnostics: ts.Diagnostic[]): never { const errMsg = this._os.noColor ? this._ts.formatDiagnostics(diagnostics, this) : this._ts.formatDiagnosticsWithColorAndContext(diagnostics, this); console.log(errMsg); // TODO The compiler isolate shouldn't exit. Errors should be forwarded to // to the caller and the caller exit. return this._os.exit(1); } /** Given a `moduleSpecifier` and `containingFile` retrieve the cached * `fileName` for a given module. If the module has yet to be resolved * this will return `undefined`. */ private _resolveFileName( moduleSpecifier: ModuleSpecifier, containingFile: ContainingFile ): ModuleFileName | undefined { this._log("compiler._resolveFileName", { moduleSpecifier, containingFile }); const innerMap = this._fileNamesMap.get(containingFile); if (innerMap) { return innerMap.get(moduleSpecifier); } return undefined; } /** Given a `moduleSpecifier` and `containingFile`, resolve the module and * return the `ModuleMetaData`. */ private _resolveModule( moduleSpecifier: ModuleSpecifier, containingFile: ContainingFile ): ModuleMetaData { this._log("compiler._resolveModule", { moduleSpecifier, containingFile }); assert(moduleSpecifier != null && moduleSpecifier.length > 0); let fileName = this._resolveFileName(moduleSpecifier, containingFile); if (fileName && this._moduleMetaDataMap.has(fileName)) { return this._moduleMetaDataMap.get(fileName)!; } let moduleId: ModuleId | undefined; let mediaType = msg.MediaType.Unknown; let sourceCode: SourceCode | undefined; if ( moduleSpecifier.startsWith(ASSETS) || containingFile.startsWith(ASSETS) ) { // Assets are compiled into the runtime javascript bundle. // we _know_ `.pop()` will return a string, but TypeScript doesn't so // not null assertion moduleId = moduleSpecifier.split("/").pop()!; const assetName = moduleId.includes(".") ? moduleId : `${moduleId}.d.ts`; assert( assetName in this._assetsSourceCode, `No such asset "${assetName}"` ); mediaType = msg.MediaType.TypeScript; sourceCode = this._assetsSourceCode[assetName]; fileName = `${ASSETS}/${assetName}`; } else { // We query Rust with a CodeFetch message. It will load the sourceCode, // and if there is any outputCode cached, will return that as well. const fetchResponse = this._os.fetchModuleMetaData( moduleSpecifier, containingFile ); moduleId = fetchResponse.moduleName; fileName = fetchResponse.filename; mediaType = fetchResponse.mediaType; sourceCode = fetchResponse.sourceCode; } assert(moduleId != null, "No module ID."); assert(fileName != null, "No file name."); assert( mediaType !== msg.MediaType.Unknown, `Unknown media type for: "${moduleSpecifier}" from "${containingFile}".` ); this._log( "resolveModule sourceCode length:", sourceCode && sourceCode.length ); this._log("resolveModule has media type:", msg.MediaType[mediaType]); // fileName is asserted above, but TypeScript does not track so not null this._setFileName(moduleSpecifier, containingFile, fileName!); if (fileName && this._moduleMetaDataMap.has(fileName)) { return this._moduleMetaDataMap.get(fileName)!; } const moduleMetaData = new ModuleMetaData( moduleId!, fileName!, mediaType, sourceCode ); this._moduleMetaDataMap.set(fileName!, moduleMetaData); return moduleMetaData; } /** Caches the resolved `fileName` in relationship to the `moduleSpecifier` * and `containingFile` in order to reduce calls to the privileged side * to retrieve the contents of a module. */ private _setFileName( moduleSpecifier: ModuleSpecifier, containingFile: ContainingFile, fileName: ModuleFileName ): void { this._log("compiler._setFileName", { moduleSpecifier, containingFile }); let innerMap = this._fileNamesMap.get(containingFile); if (!innerMap) { innerMap = new Map(); this._fileNamesMap.set(containingFile, innerMap); } innerMap.set(moduleSpecifier, fileName); } constructor(assetsSourceCode: { [key: string]: string }) { this._assetsSourceCode = assetsSourceCode; this._service = this._ts.createLanguageService(this); } // Deno specific compiler API /** Retrieve the output of the TypeScript compiler for a given module. */ compile( moduleSpecifier: ModuleSpecifier, containingFile: ContainingFile ): { outputCode: OutputCode; sourceMap: SourceMap } { this._log("compiler.compile", { moduleSpecifier, containingFile }); const moduleMetaData = this._resolveModule(moduleSpecifier, containingFile); const { fileName, mediaType, sourceCode } = moduleMetaData; this._scriptFileNames = [fileName]; let outputCode: string; let sourceMap = ""; // Instead of using TypeScript to transpile JSON modules, we will just do // it directly. if (mediaType === msg.MediaType.Json) { outputCode = moduleMetaData.outputCode = jsonEsmTemplate( sourceCode, fileName ); } else { const service = this._service; assert( mediaType === msg.MediaType.TypeScript || mediaType === msg.MediaType.JavaScript ); const output = service.getEmitOutput(fileName); // Get the relevant diagnostics - this is 3x faster than // `getPreEmitDiagnostics`. const diagnostics = [ // TypeScript is overly opinionated that only CommonJS modules kinds can // support JSON imports. Allegedly this was fixed in // Microsoft/TypeScript#26825 but that doesn't seem to be working here, // so we will ignore complaints about this compiler setting. ...service .getCompilerOptionsDiagnostics() .filter((diagnostic): boolean => diagnostic.code !== 5070), ...service.getSyntacticDiagnostics(fileName), ...service.getSemanticDiagnostics(fileName) ]; if (diagnostics.length > 0) { this._logDiagnostics(diagnostics); } assert( !output.emitSkipped, "The emit was skipped for an unknown reason." ); assert( output.outputFiles.length === 2, `Expected 2 files to be emitted, got ${output.outputFiles.length}.` ); const [sourceMapFile, outputFile] = output.outputFiles; assert( sourceMapFile.name.endsWith(".map"), "Expected first emitted file to be a source map" ); assert( outputFile.name.endsWith(".js"), "Expected second emitted file to be JavaScript" ); outputCode = moduleMetaData.outputCode = `${ outputFile.text }\n//# sourceURL=${fileName}`; sourceMap = moduleMetaData.sourceMap = sourceMapFile.text; } moduleMetaData.scriptVersion = "1"; return { outputCode, sourceMap }; } /** Take a configuration string, parse it, and use it to merge with the * compiler's configuration options. The method returns an array of compiler * options which were ignored, or `undefined`. */ configure(path: string, configurationText: string): string[] | undefined { this._log("compile.configure", path); const { config, error } = this._ts.parseConfigFileTextToJson( path, configurationText ); if (error) { this._logDiagnostics([error]); } const { options, errors } = this._ts.convertCompilerOptionsFromJson( config.compilerOptions, cwd() ); if (errors.length) { this._logDiagnostics(errors); } const ignoredOptions: string[] = []; for (const key of Object.keys(options)) { if ( ignoredCompilerOptions.includes(key) && (!(key in this._options) || options[key] !== this._options[key]) ) { ignoredOptions.push(key); delete options[key]; } } Object.assign(this._options, options); return ignoredOptions.length ? ignoredOptions : undefined; } // TypeScript Language Service and Format Diagnostic Host API getCanonicalFileName(fileName: string): string { this._log("getCanonicalFileName", fileName); return fileName; } getCompilationSettings(): ts.CompilerOptions { this._log("getCompilationSettings()"); return this._options; } getNewLine(): string { return EOL; } getScriptFileNames(): string[] { // This is equal to `"files"` in the `tsconfig.json`, therefore we only need // to include the actual base source files we are evaluating at the moment, // which would be what is set during the `.compile()` return this._scriptFileNames; } getScriptKind(fileName: ModuleFileName): ts.ScriptKind { this._log("getScriptKind()", fileName); const moduleMetaData = this._getModuleMetaData(fileName); if (moduleMetaData) { switch (moduleMetaData.mediaType) { case msg.MediaType.TypeScript: return ts.ScriptKind.TS; case msg.MediaType.JavaScript: return ts.ScriptKind.JS; case msg.MediaType.Json: return ts.ScriptKind.JSON; default: return this._options.allowJs ? ts.ScriptKind.JS : ts.ScriptKind.TS; } } else { return this._options.allowJs ? ts.ScriptKind.JS : ts.ScriptKind.TS; } } getScriptVersion(fileName: ModuleFileName): string { const moduleMetaData = this._getModuleMetaData(fileName); const version = (moduleMetaData && moduleMetaData.scriptVersion) || ""; this._log("getScriptVersion()", fileName, version); return version; } getScriptSnapshot(fileName: ModuleFileName): ts.IScriptSnapshot | undefined { this._log("getScriptSnapshot()", fileName); return this._getModuleMetaData(fileName); } getCurrentDirectory(): string { this._log("getCurrentDirectory()"); return ""; } getDefaultLibFileName(): string { this._log("getDefaultLibFileName()"); const moduleSpecifier = LIB_RUNTIME; const moduleMetaData = this._getModuleMetaData(moduleSpecifier); assert(moduleMetaData != null); return moduleMetaData!.fileName; } useCaseSensitiveFileNames(): boolean { this._log("useCaseSensitiveFileNames()"); return true; } readFile(path: string): string | undefined { this._log("readFile()", path); return notImplemented(); } fileExists(fileName: string): boolean { const moduleMetaData = this._getModuleMetaData(fileName); const exists = moduleMetaData != null; this._log("fileExists()", fileName, exists); return exists; } resolveModuleNames( moduleNames: ModuleSpecifier[], containingFile: ContainingFile ): Array { this._log("resolveModuleNames()", { moduleNames, containingFile }); const resolvedModuleNames: ts.ResolvedModuleFull[] = []; for (const moduleName of moduleNames) { const moduleMetaData = this._resolveModule(moduleName, containingFile); // According to the interface we shouldn't return `undefined` but if we // fail to return the same length of modules to those we cannot resolve // then TypeScript fails on an assertion that the lengths can't be // different, so we have to return an "empty" resolved module // TODO: all this does is push the problem downstream, and TypeScript // will complain it can't identify the type of the file and throw // a runtime exception, so we need to handle missing modules better const resolvedFileName = moduleMetaData.fileName || ""; // This flags to the compiler to not go looking to transpile functional // code, anything that is in `/$asset$/` is just library code const isExternalLibraryImport = resolvedFileName.startsWith(ASSETS); resolvedModuleNames.push({ resolvedFileName, isExternalLibraryImport, extension: getExtension(resolvedFileName, moduleMetaData.mediaType) }); } return resolvedModuleNames; } } const compiler = new Compiler(assetSourceCode); // set global objects for compiler web worker window.clearTimeout = clearTimer; window.console = console; window.postMessage = postMessage; window.setTimeout = setTimeout; window.workerMain = workerMain; window.close = workerClose; window.TextDecoder = TextDecoder; window.TextEncoder = TextEncoder; // provide the "main" function that will be called by the privileged side when // lazy instantiating the compiler web worker window.compilerMain = function compilerMain(): void { // workerMain should have already been called since a compiler is a worker. window.onmessage = ({ data }: { data: CompilerLookup }): void => { const { specifier, referrer, cmdId } = data; try { const result = compiler.compile(specifier, referrer); postMessage({ success: true, cmdId, data: result }); } catch (e) { postMessage({ success: false, cmdId, data: JSON.parse(core.errorToJSON(e)) }); } }; }; const decoder = new TextDecoder(); // Perform the op to retrieve the compiler configuration if there was any // provided on startup. function getCompilerConfig( compilerType: string ): { path: string; data: string } { const builder = flatbuffers.createBuilder(); const compilerType_ = builder.createString(compilerType); msg.CompilerConfig.startCompilerConfig(builder); msg.CompilerConfig.addCompilerType(builder, compilerType_); const inner = msg.CompilerConfig.endCompilerConfig(builder); const baseRes = sendSync(builder, msg.Any.CompilerConfig, inner); assert(baseRes != null); assert(msg.Any.CompilerConfigRes === baseRes!.innerType()); const res = new msg.CompilerConfigRes(); assert(baseRes!.inner(res) != null); // the privileged side does not normalize path separators in windows, so we // will normalize them here const path = res.path()!.replace(/\\/g, "/"); assert(path != null); const dataArray = res.dataArray()!; assert(dataArray != null); const data = decoder.decode(dataArray); return { path, data }; } export default function denoMain(): void { os.start("TS"); const { path, data } = getCompilerConfig("typescript"); if (data.length) { const ignoredOptions = compiler.configure(path, data); if (ignoredOptions) { console.warn( yellow(`Unsupported compiler options in "${path}"\n`) + cyan(` The following options were ignored:\n`) + ` ${ignoredOptions.map((value): string => bold(value)).join(", ")}` ); } } }