// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. // This module is the entry point for "compiler" isolate, ie. the one // that is created when Deno needs to compile TS/WASM to JS. // // It provides two functions that should be called by Rust: // - `startup` // This functions must be called when creating isolate // to properly setup runtime. // - `tsCompilerOnMessage` // This function must be called when sending a request // to the compiler. // Removes the `__proto__` for security reasons. This intentionally makes // Deno non compliant with ECMA-262 Annex B.2.2.1 // delete Object.prototype.__proto__; ((window) => { const core = window.Deno.core; let logDebug = false; let logSource = "JS"; /** Instructs the host to behave in a legacy fashion, with the legacy * pipeline for handling code. Setting the value to `true` will cause the * host to behave in the modern way. */ let legacy = true; function setLogDebug(debug, source) { logDebug = debug; if (source) { logSource = source; } } function debug(...args) { if (logDebug) { const stringifiedArgs = args.map((arg) => JSON.stringify(arg)).join(" "); core.print(`DEBUG ${logSource} - ${stringifiedArgs}\n`); } } class AssertionError extends Error { constructor(msg) { super(msg); this.name = "AssertionError"; } } function assert(cond, msg = "Assertion failed.") { if (!cond) { throw new AssertionError(msg); } } /** @type {Map} */ const sourceFileCache = new Map(); /** * @param {import("../dts/typescript").DiagnosticRelatedInformation} diagnostic */ function fromRelatedInformation({ start, length, file, messageText: msgText, ...ri }) { let messageText; let messageChain; if (typeof msgText === "object") { messageChain = msgText; } else { messageText = msgText; } if (start !== undefined && length !== undefined && file) { const startPos = file.getLineAndCharacterOfPosition(start); const sourceLine = file.getFullText().split("\n")[startPos.line]; const fileName = file.fileName; return { start: startPos, end: file.getLineAndCharacterOfPosition(start + length), fileName, messageChain, messageText, sourceLine, ...ri, }; } else { return { messageChain, messageText, ...ri, }; } } /** * @param {import("../dts/typescript").Diagnostic[]} diagnostics */ function fromTypeScriptDiagnostic(diagnostics) { return diagnostics.map(({ relatedInformation: ri, source, ...diag }) => { const value = fromRelatedInformation(diag); value.relatedInformation = ri ? ri.map(fromRelatedInformation) : undefined; value.source = source; return value; }); } // We really don't want to depend on JSON dispatch during snapshotting, so // this op exchanges strings with Rust as raw byte arrays. function getAsset(name) { const opId = core.ops()["op_fetch_asset"]; const sourceCodeBytes = core.dispatch(opId, core.encode(name)); return core.decode(sourceCodeBytes); } // Using incremental compile APIs requires that all // paths must be either relative or absolute. Since // analysis in Rust operates on fully resolved URLs, // it makes sense to use the same scheme here. const ASSETS = "asset:///"; const OUT_DIR = "deno://"; const CACHE = "cache:///"; // This constant is passed to compiler settings when // doing incremental compiles. Contents of this // file are passed back to Rust and saved to $DENO_DIR. const TS_BUILD_INFO = "cache:///tsbuildinfo.json"; const DEFAULT_COMPILE_OPTIONS = { allowJs: false, allowNonTsExtensions: true, checkJs: false, esModuleInterop: true, jsx: ts.JsxEmit.React, module: ts.ModuleKind.ESNext, outDir: OUT_DIR, sourceMap: true, strict: true, removeComments: true, target: ts.ScriptTarget.ESNext, }; const CompilerHostTarget = { Main: "main", Runtime: "runtime", Worker: "worker", }; // Warning! The values in this enum are duplicated in `cli/msg.rs` // Update carefully! const MediaType = { 0: "JavaScript", 1: "JSX", 2: "TypeScript", 3: "Dts", 4: "TSX", 5: "Json", 6: "Wasm", 7: "TsBuildInfo", 8: "SourceMap", 9: "Unknown", JavaScript: 0, JSX: 1, TypeScript: 2, Dts: 3, TSX: 4, Json: 5, Wasm: 6, TsBuildInfo: 7, SourceMap: 8, Unknown: 9, }; function getExtension(fileName, mediaType) { switch (mediaType) { case MediaType.JavaScript: return ts.Extension.Js; case MediaType.JSX: return ts.Extension.Jsx; case MediaType.TypeScript: return ts.Extension.Ts; case MediaType.Dts: return ts.Extension.Dts; case MediaType.TSX: return ts.Extension.Tsx; case MediaType.Wasm: // Custom marker for Wasm type. return ts.Extension.Js; case MediaType.Unknown: default: throw TypeError( `Cannot resolve extension for "${fileName}" with mediaType "${ MediaType[mediaType] }".`, ); } } /** A global cache of module source files that have been loaded. * This cache will be rewritten to be populated on compiler startup * with files provided from Rust in request message. */ const SOURCE_FILE_CACHE = new Map(); /** A map of maps which cache resolved specifier for each import in a file. * This cache is used so `resolveModuleNames` ops is called as few times * as possible. * * First map's key is "referrer" URL ("file://a/b/c/mod.ts") * Second map's key is "raw" import specifier ("./foo.ts") * Second map's value is resolved import URL ("file:///a/b/c/foo.ts") */ const RESOLVED_SPECIFIER_CACHE = new Map(); class SourceFile { constructor(json) { this.processed = false; Object.assign(this, json); this.extension = getExtension(this.url, this.mediaType); } static addToCache(json) { if (SOURCE_FILE_CACHE.has(json.url)) { throw new TypeError("SourceFile already exists"); } const sf = new SourceFile(json); SOURCE_FILE_CACHE.set(sf.url, sf); return sf; } static getCached(url) { return SOURCE_FILE_CACHE.get(url); } static cacheResolvedUrl(resolvedUrl, rawModuleSpecifier, containingFile) { containingFile = containingFile || ""; let innerCache = RESOLVED_SPECIFIER_CACHE.get(containingFile); if (!innerCache) { innerCache = new Map(); RESOLVED_SPECIFIER_CACHE.set(containingFile, innerCache); } innerCache.set(rawModuleSpecifier, resolvedUrl); } static getResolvedUrl(moduleSpecifier, containingFile) { const containingCache = RESOLVED_SPECIFIER_CACHE.get(containingFile); if (containingCache) { return containingCache.get(moduleSpecifier); } return undefined; } } function getAssetInternal(filename) { const lastSegment = filename.split("/").pop(); const url = ts.libMap.has(lastSegment) ? ts.libMap.get(lastSegment) : lastSegment; const sourceFile = SourceFile.getCached(url); if (sourceFile) { return sourceFile; } const name = url.includes(".") ? url : `${url}.d.ts`; const sourceCode = getAsset(name); return SourceFile.addToCache({ url, filename: `${ASSETS}/${name}`, mediaType: MediaType.TypeScript, versionHash: "1", sourceCode, }); } /** There was some private state in the legacy host, that is moved out to * here which can then be refactored out later. */ const legacyHostState = { buildInfo: "", target: CompilerHostTarget.Main, writeFile: (_fileName, _data, _sourceFiles) => {}, }; /** @type {import("../dts/typescript").CompilerHost} */ const host = { fileExists(fileName) { debug(`host.fileExists("${fileName}")`); return false; }, readFile(specifier) { debug(`host.readFile("${specifier}")`); if (legacy) { if (specifier == TS_BUILD_INFO) { return legacyHostState.buildInfo; } return unreachable(); } else { return core.jsonOpSync("op_load", { specifier }).data; } }, getSourceFile( specifier, languageVersion, onError, shouldCreateNewSourceFile, ) { debug( `host.getSourceFile("${specifier}", ${ ts.ScriptTarget[languageVersion] })`, ); if (legacy) { try { assert(!shouldCreateNewSourceFile); const sourceFile = specifier.startsWith(ASSETS) ? getAssetInternal(specifier) : SourceFile.getCached(specifier); assert(sourceFile != null); if (!sourceFile.tsSourceFile) { assert(sourceFile.sourceCode != null); const tsSourceFileName = specifier.startsWith(ASSETS) ? sourceFile.filename : specifier; sourceFile.tsSourceFile = ts.createSourceFile( tsSourceFileName, sourceFile.sourceCode, languageVersion, ); sourceFile.tsSourceFile.version = sourceFile.versionHash; delete sourceFile.sourceCode; // This code is to support transition from the "legacy" compiler // to the new one, by populating the new source file cache. if ( !sourceFileCache.has(specifier) && specifier.startsWith(ASSETS) ) { sourceFileCache.set(specifier, sourceFile.tsSourceFile); } } return sourceFile.tsSourceFile; } catch (e) { if (onError) { onError(String(e)); } else { throw e; } return undefined; } } else { let sourceFile = sourceFileCache.get(specifier); if (sourceFile) { return sourceFile; } /** @type {{ data: string; hash?: string; scriptKind: ts.ScriptKind }} */ const { data, hash, scriptKind } = core.jsonOpSync( "op_load", { specifier }, ); assert( data != null, `"data" is unexpectedly null for "${specifier}".`, ); sourceFile = ts.createSourceFile( specifier, data, languageVersion, false, scriptKind, ); sourceFile.moduleName = specifier; sourceFile.version = hash; sourceFileCache.set(specifier, sourceFile); return sourceFile; } }, getDefaultLibFileName() { if (legacy) { switch (legacyHostState.target) { case CompilerHostTarget.Main: case CompilerHostTarget.Runtime: return `${ASSETS}/lib.deno.window.d.ts`; case CompilerHostTarget.Worker: return `${ASSETS}/lib.deno.worker.d.ts`; } } else { return `${ASSETS}/lib.esnext.d.ts`; } }, getDefaultLibLocation() { return ASSETS; }, writeFile(fileName, data, _writeByteOrderMark, _onError, sourceFiles) { debug(`host.writeFile("${fileName}")`); if (legacy) { legacyHostState.writeFile(fileName, data, sourceFiles); } else { let maybeSpecifiers; if (sourceFiles) { maybeSpecifiers = sourceFiles.map((sf) => sf.moduleName); } return core.jsonOpSync( "op_emit", { maybeSpecifiers, fileName, data }, ); } }, getCurrentDirectory() { return CACHE; }, getCanonicalFileName(fileName) { return fileName; }, useCaseSensitiveFileNames() { return true; }, getNewLine() { return "\n"; }, resolveModuleNames(specifiers, base) { debug(`host.resolveModuleNames()`); debug(` base: ${base}`); debug(` specifiers: ${specifiers.join(", ")}`); if (legacy) { const resolved = specifiers.map((specifier) => { const maybeUrl = SourceFile.getResolvedUrl(specifier, base); debug("compiler::host.resolveModuleNames maybeUrl", { specifier, maybeUrl, }); let sourceFile = undefined; if (specifier.startsWith(ASSETS)) { sourceFile = getAssetInternal(specifier); } else if (typeof maybeUrl !== "undefined") { sourceFile = SourceFile.getCached(maybeUrl); } if (!sourceFile) { return undefined; } return { resolvedFileName: sourceFile.url, isExternalLibraryImport: specifier.startsWith(ASSETS), extension: sourceFile.extension, }; }); debug(resolved); return resolved; } else { /** @type {Array<[string, import("../dts/typescript").Extension]>} */ const resolved = core.jsonOpSync("op_resolve", { specifiers, base, }); let r = resolved.map(([resolvedFileName, extension]) => ({ resolvedFileName, extension, isExternalLibraryImport: false, })); return r; } }, createHash(data) { return core.jsonOpSync("op_create_hash", { data }).hash; }, }; // This is a hacky way of adding our libs to the libs available in TypeScript() // as these are internal APIs of TypeScript which maintain valid libs ts.libs.push("deno.ns", "deno.window", "deno.worker", "deno.shared_globals"); ts.libMap.set("deno.ns", "lib.deno.ns.d.ts"); ts.libMap.set("deno.web", "lib.deno.web.d.ts"); ts.libMap.set("deno.fetch", "lib.deno.fetch.d.ts"); ts.libMap.set("deno.window", "lib.deno.window.d.ts"); ts.libMap.set("deno.worker", "lib.deno.worker.d.ts"); ts.libMap.set("deno.shared_globals", "lib.deno.shared_globals.d.ts"); ts.libMap.set("deno.unstable", "lib.deno.unstable.d.ts"); // TODO(@kitsonk) remove once added to TypeScript ts.libs.push("esnext.weakref"); ts.libMap.set("esnext.weakref", "lib.esnext.weakref.d.ts"); // this pre-populates the cache at snapshot time of our library files, so they // are available in the future when needed. host.getSourceFile( `${ASSETS}lib.deno.ns.d.ts`, ts.ScriptTarget.ESNext, ); host.getSourceFile( `${ASSETS}lib.deno.web.d.ts`, ts.ScriptTarget.ESNext, ); host.getSourceFile( `${ASSETS}lib.deno.fetch.d.ts`, ts.ScriptTarget.ESNext, ); host.getSourceFile( `${ASSETS}lib.deno.window.d.ts`, ts.ScriptTarget.ESNext, ); host.getSourceFile( `${ASSETS}lib.deno.worker.d.ts`, ts.ScriptTarget.ESNext, ); host.getSourceFile( `${ASSETS}lib.deno.shared_globals.d.ts`, ts.ScriptTarget.ESNext, ); host.getSourceFile( `${ASSETS}lib.deno.unstable.d.ts`, ts.ScriptTarget.ESNext, ); // We never use this program; it's only created // during snapshotting to hydrate and populate // source file cache with lib declaration files. const _TS_SNAPSHOT_PROGRAM = ts.createProgram({ rootNames: [`${ASSETS}bootstrap.ts`], options: DEFAULT_COMPILE_OPTIONS, host, }); const IGNORED_DIAGNOSTICS = [ // TS2306: File 'file:///Users/rld/src/deno/cli/tests/subdir/amd_like.js' is // not a module. 2306, // TS1375: 'await' expressions are only allowed at the top level of a file // when that file is a module, but this file has no imports or exports. // Consider adding an empty 'export {}' to make this file a module. 1375, // TS1103: 'for-await-of' statement is only allowed within an async function // or async generator. 1103, // TS2691: An import path cannot end with a '.ts' extension. Consider // importing 'bad-module' instead. 2691, // TS5009: Cannot find the common subdirectory path for the input files. 5009, // TS5055: Cannot write file // 'http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js' // because it would overwrite input file. 5055, // TypeScript is overly opinionated that only CommonJS modules kinds can // support JSON imports. Allegedly this was fixed in // Microsoft/TypeScript#26825 but that doesn't seem to be working here, // so we will ignore complaints about this compiler setting. 5070, // TS7016: Could not find a declaration file for module '...'. '...' // implicitly has an 'any' type. This is due to `allowJs` being off by // default but importing of a JavaScript module. 7016, ]; const IGNORED_COMPILE_DIAGNOSTICS = [ // TS1208: All files must be modules when the '--isolatedModules' flag is // provided. We can ignore because we guarantuee that all files are // modules. 1208, ]; /** @type {Array<{ key: string, value: number }>} */ const stats = []; let statsStart = 0; function performanceStart() { stats.length = 0; statsStart = new Date(); ts.performance.enable(); } function performanceProgram({ program, fileCount }) { if (program) { if ("getProgram" in program) { program = program.getProgram(); } stats.push({ key: "Files", value: program.getSourceFiles().length }); stats.push({ key: "Nodes", value: program.getNodeCount() }); stats.push({ key: "Identifiers", value: program.getIdentifierCount() }); stats.push({ key: "Symbols", value: program.getSymbolCount() }); stats.push({ key: "Types", value: program.getTypeCount() }); stats.push({ key: "Instantiations", value: program.getInstantiationCount(), }); } else if (fileCount != null) { stats.push({ key: "Files", value: fileCount }); } const programTime = ts.performance.getDuration("Program"); const bindTime = ts.performance.getDuration("Bind"); const checkTime = ts.performance.getDuration("Check"); const emitTime = ts.performance.getDuration("Emit"); stats.push({ key: "Parse time", value: programTime }); stats.push({ key: "Bind time", value: bindTime }); stats.push({ key: "Check time", value: checkTime }); stats.push({ key: "Emit time", value: emitTime }); stats.push({ key: "Total TS time", value: programTime + bindTime + checkTime + emitTime, }); } function performanceEnd() { const duration = new Date() - statsStart; stats.push({ key: "Compile time", value: duration }); return stats; } /** * @typedef {object} Request * @property {Record} config * @property {boolean} debug * @property {string[]} rootNames */ /** The API that is called by Rust when executing a request. * @param {Request} request */ function exec({ config, debug: debugFlag, rootNames }) { setLogDebug(debugFlag, "TS"); performanceStart(); debug(">>> exec start", { rootNames }); debug(config); const { options, errors: configFileParsingDiagnostics } = ts .convertCompilerOptionsFromJson(config, "", "tsconfig.json"); const program = ts.createIncrementalProgram({ rootNames, options, host, configFileParsingDiagnostics, }); const { diagnostics: emitDiagnostics } = program.emit(); const diagnostics = [ ...program.getConfigFileParsingDiagnostics(), ...program.getSyntacticDiagnostics(), ...program.getOptionsDiagnostics(), ...program.getGlobalDiagnostics(), ...program.getSemanticDiagnostics(), ...emitDiagnostics, ].filter(({ code }) => !IGNORED_DIAGNOSTICS.includes(code) && !IGNORED_COMPILE_DIAGNOSTICS.includes(code) ); performanceProgram({ program }); // TODO(@kitsonk) when legacy stats are removed, convert to just tuples let stats = performanceEnd().map(({ key, value }) => [key, value]); core.jsonOpSync("op_respond", { diagnostics: fromTypeScriptDiagnostic(diagnostics), stats, }); debug("<<< exec stop"); } let hasStarted = false; /** Startup the runtime environment, setting various flags. * @param {{ debugFlag?: boolean; legacyFlag?: boolean; }} msg */ function startup({ debugFlag = false, legacyFlag = true }) { if (hasStarted) { throw new Error("The compiler runtime already started."); } hasStarted = true; core.ops(); core.registerErrorClass("Error", Error); setLogDebug(!!debugFlag, "TS"); legacy = legacyFlag; } globalThis.startup = startup; globalThis.exec = exec; })(this);