import { writeFileSync } from "fs"; import { join } from "path"; import * as prettier from "prettier"; import { InterfaceDeclaration, ExpressionStatement, NamespaceDeclarationKind, Project, SourceFile, ts, Type, TypeGuards } from "ts-morph"; import { addInterfaceDeclaration, addInterfaceProperty, addSourceComment, addTypeAlias, addVariableDeclaration, checkDiagnostics, flattenNamespace, getSourceComment, inlineFiles, loadDtsFiles, loadFiles, log, logDiagnostics, namespaceSourceFile, normalizeSlashes, setSilent } from "./ast_util"; export interface BuildLibraryOptions { /** * The path to the root of the deno repository */ basePath: string; /** * The path to the current build path */ buildPath: string; /** * Denotes if the library should be built with debug information (comments * that indicate the source of the types) */ debug?: boolean; /** * An array of files that should be inlined into the library */ inline?: string[]; /** An array of input files to be provided to the input project, relative to * the basePath. */ inputs?: string[]; /** * Path to globals file to be used I.E. `js/globals.ts` */ additionalGlobals?: string[]; /** * List of global variables to define as let instead of the default const. */ declareAsLet?: string[]; /** * The path to the output library */ outFile: string; /** * Execute in silent mode or not */ silent?: boolean; } const { ModuleKind, ModuleResolutionKind, ScriptTarget } = ts; /** * A preamble which is appended to the start of the library. */ const libPreamble = `// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. /// /// `; // The path to the msg_generated file relative to the build path const MSG_GENERATED_PATH = "/gen/cli/msg_generated.ts"; // An array of enums we want to expose pub const MSG_GENERATED_ENUMS = ["ErrorKind"]; /** Extracts enums from a source file */ function extract(sourceFile: SourceFile, enumNames: string[]): string { // Copy specified enums from msg_generated let output = ""; for (const enumName of enumNames) { const enumDeclaration = sourceFile.getEnumOrThrow(enumName); enumDeclaration.setHasDeclareKeyword(false); // we are not copying JSDocs or other trivia here because msg_generated only // contains some non-useful JSDocs and comments that are not ideal to copy // over output += enumDeclaration.getText(); } return output; } interface FlattenOptions { basePath: string; customSources: { [filePath: string]: string }; filePath: string; debug?: boolean; declarationProject: Project; globalInterfaceName?: string; moduleName?: string; namespaceName?: string; targetSourceFile: SourceFile; } /** Flatten a module */ export function flatten({ basePath, customSources, filePath, debug, declarationProject, globalInterfaceName, moduleName, namespaceName, targetSourceFile }: FlattenOptions): void { // Flatten the source file into a single set of statements const statements = flattenNamespace({ sourceFile: declarationProject.getSourceFileOrThrow(filePath), rootPath: basePath, customSources, debug }); // If a module name is specified create the module in the target file if (moduleName) { const namespace = targetSourceFile.addNamespace({ name: moduleName, hasDeclareKeyword: true, declarationKind: NamespaceDeclarationKind.Module }); // Add the output of the flattening to the namespace namespace.addStatements(statements); } if (namespaceName) { const namespace = targetSourceFile.insertNamespace(0, { name: namespaceName, hasDeclareKeyword: true, declarationKind: NamespaceDeclarationKind.Namespace }); // Add the output of the flattening to the namespace namespace.addStatements(statements); if (globalInterfaceName) { // Retrieve the global interface const interfaceDeclaration = targetSourceFile.getInterfaceOrThrow( globalInterfaceName ); // Add the namespace to the global interface addInterfaceProperty( interfaceDeclaration, namespaceName, `typeof ${namespaceName}` ); } } } interface PrepareFileForMergeOptions { globalVarName: string; interfaceName: string; targetSourceFile: SourceFile; } interface PrepareFileForMergeReturn { interfaceDeclaration: InterfaceDeclaration; } export function prepareFileForMerge({ globalVarName, interfaceName, targetSourceFile }: PrepareFileForMergeOptions): PrepareFileForMergeReturn { // Add the global object interface const interfaceDeclaration = targetSourceFile.addInterface({ name: interfaceName, hasDeclareKeyword: true }); // Declare the global variable addVariableDeclaration( targetSourceFile, globalVarName, interfaceName, true, true ); // `globalThis` accesses the global scope and is defined here: // https://github.com/tc39/proposal-global addVariableDeclaration( targetSourceFile, "globalThis", interfaceName, true, true ); // Add self reference to the global variable addInterfaceProperty(interfaceDeclaration, globalVarName, interfaceName); return { interfaceDeclaration }; } interface MergeGlobalOptions extends PrepareFileForMergeOptions { basePath: string; debug?: boolean; declarationProject: Project; filePath: string; ignore?: string[]; inputProject: Project; prepareReturn: PrepareFileForMergeReturn; declareAsLet?: string[]; } /** Take a module and merge it into the global scope */ export function mergeGlobals({ basePath, debug, declarationProject, filePath, globalVarName, ignore, inputProject, targetSourceFile, declareAsLet, prepareReturn: { interfaceDeclaration } }: MergeGlobalOptions): void { // Retrieve source file from the input project const sourceFile = inputProject.getSourceFileOrThrow(filePath); // we are going to create a map of variables const globalVariables = new Map< string, { type: Type; node: ExpressionStatement; } >(); const globalInterfaces: InterfaceDeclaration[] = []; // For every augmentation of the global variable in source file, we want // to extract the type and add it to the global variable map sourceFile.forEachChild(node => { if (TypeGuards.isExpressionStatement(node)) { const firstChild = node.getFirstChild(); if (!firstChild) { return; } if (TypeGuards.isBinaryExpression(firstChild)) { const leftExpression = firstChild.getLeft(); if ( TypeGuards.isPropertyAccessExpression(leftExpression) && leftExpression.getExpression().getText() === globalVarName ) { const globalVarProperty = leftExpression.getName(); if (globalVarProperty !== globalVarName) { globalVariables.set(globalVarProperty, { type: firstChild.getType(), node }); } } } } else if (TypeGuards.isInterfaceDeclaration(node) && node.isExported()) { globalInterfaces.push(node); } }); // A set of source files that the types we are using are dependent on us // importing const dependentSourceFiles = new Set(); // Create a global variable and add the property to the `Window` interface // for each mutation of the `window` variable we observed in `globals.ts` for (const [property, info] of globalVariables) { if (!(ignore && ignore.includes(property))) { const type = info.type.getText(info.node); const typeSymbol = info.type.getSymbol(); if (typeSymbol) { const valueDeclaration = typeSymbol.getValueDeclaration(); if (valueDeclaration) { dependentSourceFiles.add(valueDeclaration.getSourceFile()); } } const isConst = !(declareAsLet && declareAsLet.includes(property)); addVariableDeclaration(targetSourceFile, property, type, isConst, true); addInterfaceProperty(interfaceDeclaration, property, type); } } // We need to copy over any type aliases for (const typeAlias of sourceFile.getTypeAliases()) { addTypeAlias( targetSourceFile, typeAlias.getName(), typeAlias.getType().getText(sourceFile), true ); } // We need to copy over any interfaces for (const interfaceDeclaration of globalInterfaces) { addInterfaceDeclaration(targetSourceFile, interfaceDeclaration); } // We need to ensure that we only namespace each source file once, so we // will use this map for tracking that. const sourceFileMap = new Map(); // For each import declaration in source file we will want to convert the // declaration source file into a namespace that exists within the merged // namespace const importDeclarations = sourceFile.getImportDeclarations(); const namespaces = new Set(); for (const declaration of importDeclarations) { const namespaceImport = declaration.getNamespaceImport(); if (namespaceImport) { const declarationSourceFile = declaration.getModuleSpecifierSourceFile(); if ( declarationSourceFile && dependentSourceFiles.has(declarationSourceFile) ) { // the source file will resolve to the original `.ts` file, but the // information we really want is in the emitted `.d.ts` file, so we will // resolve to that file const dtsFilePath = declarationSourceFile .getFilePath() .replace(/\.ts$/, ".d.ts"); const dtsSourceFile = declarationProject.getSourceFileOrThrow( dtsFilePath ); targetSourceFile.addStatements( namespaceSourceFile(dtsSourceFile, { debug, namespace: namespaceImport.getText(), namespaces, rootPath: basePath, sourceFileMap }) ); } } } if (debug) { addSourceComment(targetSourceFile, sourceFile, basePath); } } /** * Generate the runtime library for Deno and write it to the supplied out file * name. */ export function main({ basePath, buildPath, inline, inputs, additionalGlobals, declareAsLet, debug, outFile, silent }: BuildLibraryOptions): void { setSilent(silent); log("-----"); log("build_lib"); log(); log(`basePath: "${basePath}"`); log(`buildPath: "${buildPath}"`); if (inline && inline.length) { log("inline:"); for (const filename of inline) { log(` "${filename}"`); } } if (inputs && inputs.length) { log("inputs:"); for (const input of inputs) { log(` "${input}"`); } } log(`debug: ${!!debug}`); log(`outFile: "${outFile}"`); log(); // the inputProject will take in the TypeScript files that are internal // to Deno to be used to generate the library const inputProject = new Project({ compilerOptions: { baseUrl: basePath, declaration: true, emitDeclarationOnly: true, lib: ["esnext"], module: ModuleKind.ESNext, moduleResolution: ModuleResolutionKind.NodeJs, paths: { "*": ["*", `${buildPath}/*`] }, preserveConstEnums: true, strict: true, stripInternal: true, target: ScriptTarget.ESNext } }); // Add the input files we will need to generate the declarations, `globals` // plus any modules that are importable in the runtime need to be added here // plus the `lib.esnext` which is used as the base library if (inputs) { inputProject.addExistingSourceFiles( inputs.map(input => join(basePath, input)) ); } // emit the project, which will be only the declaration files const inputEmitResult = inputProject.emitToMemory(); log("Emitted input project."); const inputDiagnostics = inputEmitResult .getDiagnostics() .map(d => d.compilerObject); logDiagnostics(inputDiagnostics); if (inputDiagnostics.length) { console.error("\nDiagnostics present during input project emit.\n"); process.exit(1); } // the declaration project will be the target for the emitted files from // the input project, these will be used to transfer information over to // the final library file const declarationProject = new Project({ compilerOptions: { baseUrl: basePath, lib: ["esnext"], moduleResolution: ModuleResolutionKind.NodeJs, paths: { "*": ["*", `${buildPath}/*`] }, strict: true, target: ScriptTarget.ESNext }, useVirtualFileSystem: true }); // we don't want to add to the declaration project any of the original // `.ts` source files, so we need to filter those out const jsPath = normalizeSlashes(`${basePath}/js`); const inputProjectFiles = inputProject .getSourceFiles() .map(sourceFile => sourceFile.getFilePath()) .filter(filePath => !filePath.startsWith(jsPath)); loadFiles(declarationProject, inputProjectFiles); // now we add the emitted declaration files from the input project for (const { filePath, text } of inputEmitResult.getFiles()) { declarationProject.createSourceFile(filePath, text); } // the outputProject will contain the final library file we are looking to // build const outputProjectCompilerOptions: ts.CompilerOptions = { baseUrl: buildPath, lib: ["esnext"], moduleResolution: ModuleResolutionKind.NodeJs, strict: true, target: ScriptTarget.ESNext }; const outputProject = new Project({ compilerOptions: outputProjectCompilerOptions, useVirtualFileSystem: true }); // There are files we need to load into memory, so that the project "compiles" loadDtsFiles(outputProject, outputProjectCompilerOptions); // libDts is the final output file we are looking to build and we are not // actually creating it, only in memory at this stage. const libDTs = outputProject.createSourceFile(outFile); // Deal with `js/deno.ts` // `gen/msg_generated.d.ts` contains too much exported information that is not // part of the public API surface of Deno, so we are going to extract just the // information we need. const msgGeneratedDts = inputProject.getSourceFileOrThrow( `${buildPath}${MSG_GENERATED_PATH}` ); const msgGeneratedDtsText = extract(msgGeneratedDts, MSG_GENERATED_ENUMS); // Generate a object hash of substitutions of modules to use when flattening const customSources = { [msgGeneratedDts.getFilePath().replace(/(\.d)?\.ts$/, "")]: `${ debug ? getSourceComment(msgGeneratedDts, basePath) : "" }${msgGeneratedDtsText}\n` }; const prepareForMergeOpts: PrepareFileForMergeOptions = { globalVarName: "window", interfaceName: "Window", targetSourceFile: libDTs }; const prepareReturn = prepareFileForMerge(prepareForMergeOpts); mergeGlobals({ basePath, debug, declarationProject, filePath: `${basePath}/js/globals.ts`, inputProject, ignore: ["Deno"], declareAsLet, ...prepareForMergeOpts, prepareReturn }); log(`Merged "globals" into global scope.`); if (additionalGlobals) { for (const additionalGlobal of additionalGlobals) { mergeGlobals({ basePath, debug, declarationProject, filePath: `${basePath}/${additionalGlobal}`, inputProject, ignore: ["Deno"], declareAsLet, ...prepareForMergeOpts, prepareReturn }); } log(`Added additional "globals" into global scope.`); } flatten({ basePath, customSources, debug, declarationProject, filePath: `${basePath}/js/deno.d.ts`, globalInterfaceName: "Window", namespaceName: "Deno", targetSourceFile: libDTs }); log(`Created module "deno" and namespace Deno.`); // Inline any files that were passed in, to be used to add additional libs // which are not part of TypeScript. if (inline && inline.length) { inlineFiles({ basePath, debug, inline, targetSourceFile: libDTs }); } // Add the preamble libDTs.insertStatements(0, libPreamble); // Check diagnostics checkDiagnostics(outputProject); // Output the final library file libDTs.saveSync(); const libDTsText = prettier.format( outputProject.getFileSystem().readFileSync(outFile, "utf8"), { parser: "typescript" } ); if (!silent) { console.log(`Outputting library to: "${outFile}"`); console.log(` Length: ${libDTsText.length}`); } writeFileSync(outFile, libDTsText, { encoding: "utf8" }); if (!silent) { console.log("-----"); console.log(); } }