foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-21 15:04:11 -05:00
Code Issues Packages 1 Wiki Activity

Replace globals.d.ts with lib.deno_runtime.d.ts

Browse source
This commit is contained in:
Kitson Kelly 2018-10-12 08:23:13 +11:00 committed by Ryan Dahl
parent fda7aaa10e
commit ec402c6932
23 changed files with 1183 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
BUILD.gn
View file

@ -107,7 +107,6 @@ ts_sources = [
"js/v8_source_maps.ts",
"js/write_file.ts",
"js/tsconfig.declarations.json",
"tsconfig.json",
# Listing package.json and yarn.lock as sources ensures the bundle is rebuilt
@ -246,26 +245,33 @@ executable("snapshot_creator") {
configs += [ ":deno_config" ]
}
# Generates type declarations for files that need to be included
# in the runtime bundle
run_node("gen_declarations") {
# Generates the core TypeScript type library for deno that will be
# included in the runtime bundle
run_node("deno_runtime_declaration") {
out_dir = target_gen_dir
sources = ts_sources
outputs = [
"$out_dir/types/globals.d.ts",
"$out_dir/lib/lib.deno_runtime.d.ts",
]
deps = [
":msg_ts",
]
args = [
"./node_modules/typescript/bin/tsc",
"-p",
rebase_path("js/tsconfig.declarations.json", root_build_dir),
"--baseUrl",
rebase_path("node_modules/.bin/ts-node", root_build_dir),
"--project",
rebase_path("tools/ts_library_builder/tsconfig.json"),
rebase_path("tools/ts_library_builder/main.ts", root_build_dir),
"--basePath",
rebase_path(".", root_build_dir),
"--buildPath",
rebase_path(root_build_dir, root_build_dir),
"--outFile",
rebase_path("$out_dir/types/globals.js", root_build_dir),
rebase_path("$out_dir/lib/lib.deno_runtime.d.ts", root_build_dir),
"--silent",
]
if (is_debug) {
args += [ "--debug" ]
}
}
run_node("bundle") {
@ -276,7 +282,7 @@ run_node("bundle") {
out_dir + "main.js.map",
]
deps = [
":gen_declarations",
":deno_runtime_declaration",
":msg_ts",
]
args = [

6
js/assets.ts
View file

@ -7,7 +7,7 @@
// tslint:disable:max-line-length
// Generated default library
import globalsDts from "gen/types/globals.d.ts!string";
import libDts from "gen/lib/lib.deno_runtime.d.ts!string";
// Static libraries
import libEs2015Dts from "/third_party/node_modules/typescript/lib/lib.es2015.d.ts!string";
@ -40,7 +40,6 @@ import libEsnextIntlDts from "/third_party/node_modules/typescript/lib/lib.esnex
import libEsnextSymbolDts from "/third_party/node_modules/typescript/lib/lib.esnext.symbol.d.ts!string";
// Static definitions
import flatbuffersDts from "/third_party/node_modules/@types/flatbuffers/index.d.ts!string";
import textEncodingDts from "/third_party/node_modules/@types/text-encoding/index.d.ts!string";
import typescriptDts from "/third_party/node_modules/typescript/lib/typescript.d.ts!string";
// tslint:enable:max-line-length
@ -48,7 +47,7 @@ import typescriptDts from "/third_party/node_modules/typescript/lib/typescript.d
// @internal
export const assetSourceCode: { [key: string]: string } = {
// Generated library
"globals.d.ts": globalsDts,
"lib.deno_runtime.d.ts": libDts,
// Static libraries
"lib.es2015.collection.d.ts": libEs2015CollectionDts,
@ -81,7 +80,6 @@ export const assetSourceCode: { [key: string]: string } = {
"lib.esnext.symbol.d.ts": libEsnextSymbolDts,
// Static definitions
"flatbuffers.d.ts": flatbuffersDts,
"text-encoding.d.ts": textEncodingDts,
"typescript.d.ts": typescriptDts
};

7
js/compiler.ts
View file

@ -13,6 +13,7 @@ import * as sourceMaps from "./v8_source_maps";
const EOL = "\n";
const ASSETS = "$asset$";
const LIB_RUNTIME = "lib.deno_runtime.d.ts";
// tslint:disable:no-any
type AmdCallback = (...args: any[]) => void;
@ -619,7 +620,7 @@ export class DenoCompiler
getDefaultLibFileName(): string {
this._log("getDefaultLibFileName()");
const moduleSpecifier = "globals.d.ts";
const moduleSpecifier = LIB_RUNTIME;
const moduleMetaData = this.resolveModule(moduleSpecifier, ASSETS);
return moduleMetaData.fileName;
}
@ -649,8 +650,8 @@ export class DenoCompiler
return moduleNames.map(name => {
let resolvedFileName;
if (name === "deno") {
// builtin modules are part of `globals.d.ts`
resolvedFileName = this._resolveModuleName("globals.d.ts", ASSETS);
// builtin modules are part of the runtime lib
resolvedFileName = this._resolveModuleName(LIB_RUNTIME, ASSETS);
} else if (name === "typescript") {
resolvedFileName = this._resolveModuleName("typescript.d.ts", ASSETS);
} else {

9
js/compiler_test.ts
View file

@ -546,7 +546,10 @@ test(function compilerGetCurrentDirectory() {
test(function compilerGetDefaultLibFileName() {
setup();
assertEqual(compilerInstance.getDefaultLibFileName(), "$asset$/globals.d.ts");
assertEqual(
compilerInstance.getDefaultLibFileName(),
"$asset$/lib.deno_runtime.d.ts"
);
teardown();
});
@ -572,7 +575,7 @@ test(function compilerFileExists() {
"/root/project"
);
assert(compilerInstance.fileExists(moduleMetaData.fileName));
assert(compilerInstance.fileExists("$asset$/globals.d.ts"));
assert(compilerInstance.fileExists("$asset$/lib.deno_runtime.d.ts"));
assertEqual(
compilerInstance.fileExists("/root/project/unknown-module.ts"),
false
@ -590,7 +593,7 @@ test(function compilerResolveModuleNames() {
const fixtures: Array<[string, boolean]> = [
["/root/project/foo/bar.ts", false],
["/root/project/foo/baz.ts", false],
["$asset$/globals.d.ts", true]
["$asset$/lib.deno_runtime.d.ts", true]
];
for (let i = 0; i < results.length; i++) {
const result = results[i];

61
js/globals.ts
View file

@ -1,55 +1,22 @@
// Copyright 2018 the Deno authors. All rights reserved. MIT license.
import { Console } from "./console";
import * as timers from "./timers";
import * as textEncoding from "./text_encoding";
import * as blob from "./blob";
import * as console from "./console";
import * as fetch_ from "./fetch";
import { libdeno } from "./libdeno";
import { globalEval } from "./global_eval";
import { DenoHeaders } from "./fetch";
import { DenoBlob } from "./blob";
import { libdeno } from "./libdeno";
import * as textEncoding from "./text_encoding";
import * as timers from "./timers";
// During the build process, augmentations to the variable `window` in this
// file are tracked and created as part of default library that is built into
// deno, we only need to declare the enough to compile deno.
declare global {
interface Window {
define: Readonly<unknown>;
clearTimeout: typeof clearTimeout;
clearInterval: typeof clearInterval;
setTimeout: typeof setTimeout;
setInterval: typeof setInterval;
console: typeof console;
window: typeof window;
fetch: typeof fetch;
TextEncoder: typeof TextEncoder;
TextDecoder: typeof TextDecoder;
atob: typeof atob;
btoa: typeof btoa;
Headers: typeof Headers;
Blob: typeof Blob;
}
const clearTimeout: typeof timers.clearTimer;
const clearInterval: typeof timers.clearTimer;
const console: console.Console;
const setTimeout: typeof timers.setTimeout;
const setInterval: typeof timers.setInterval;
const console: Console;
const window: Window;
const fetch: typeof fetch_.fetch;
// tslint:disable:variable-name
// tslint:disable-next-line:variable-name
const TextEncoder: typeof textEncoding.TextEncoder;
const TextDecoder: typeof textEncoding.TextDecoder;
const atob: typeof textEncoding.atob;
const btoa: typeof textEncoding.btoa;
const Headers: typeof DenoHeaders;
const Blob: typeof DenoBlob;
// tslint:enable:variable-name
}
// A reference to the global object.
@ -61,7 +28,7 @@ window.setInterval = timers.setInterval;
window.clearTimeout = timers.clearTimer;
window.clearInterval = timers.clearTimer;
window.console = new Console(libdeno.print);
window.console = new console.Console(libdeno.print);
window.TextEncoder = textEncoding.TextEncoder;
window.TextDecoder = textEncoding.TextDecoder;
window.atob = textEncoding.atob;
@ -69,5 +36,5 @@ window.btoa = textEncoding.btoa;
window.fetch = fetch_.fetch;
window.Headers = DenoHeaders;
window.Blob = DenoBlob;
window.Headers = fetch_.DenoHeaders;
window.Blob = blob.DenoBlob;

2
js/testing/testing.ts
View file

@ -13,7 +13,7 @@
limitations under the License.
*/
export { assert, assertEqual, equal } from "./util.ts";
export { assert, assertEqual, equal } from "./util";
export type TestFunction = () => void | Promise<void>;

19
js/tsconfig.declarations.json
View file

@ -1,19 +0,0 @@
{
// This configuration file provides the tsc configuration for generating
// definitions for the runtime, which are then inlined via the `js/assets.ts`
// module into the bundle to be available for type checking at runtime
// See also gen_declarations in //BUILD.gn
"extends": "../tsconfig.json",
"compilerOptions": {
"declaration": true,
"emitDeclarationOnly": true,
"module": "amd",
"removeComments": false,
"stripInternal": true
},
"files": [
"../node_modules/typescript/lib/lib.esnext.d.ts",
"./deno.ts",
"./globals.ts"
]
}

3
package.json
View file

@ -3,6 +3,7 @@
"devDependencies": {
"@types/base64-js": "^1.2.5",
"@types/flatbuffers": "^1.9.0",
"@types/prettier": "^1.13.2",
"@types/source-map-support": "^0.4.1",
"@types/text-encoding": "0.0.33",
"base64-js": "^1.3.0",
@ -20,6 +21,8 @@
"rollup-pluginutils": "^2.3.0",
"source-map-support": "^0.5.6",
"text-encoding": "0.6.4",
"ts-node": "^7.0.1",
"ts-simple-ast": "^16.0.4",
"tslint": "^5.10.0",
"tslint-eslint-rules": "^5.3.1",
"tslint-no-circular-imports": "^0.5.0",

10
rollup.config.js
View file

@ -26,12 +26,6 @@ const tsconfigOverride = {
}
};
// this is a preamble for the `globals.d.ts` file to allow it to be the default
// lib for deno.
const libPreamble = `/// <reference no-default-lib="true"/>
/// <reference lib="esnext" />
`;
// this is a rollup plugin which will look for imports ending with `!string` and resolve
// them with a module that will inline the contents of the file as a string. Needed to
// support `js/assets.ts`.
@ -70,9 +64,7 @@ function strings({ include, exclude } = {}) {
transform(code, id) {
if (filter(id)) {
return {
code: `export default ${JSON.stringify(
id.endsWith("globals.d.ts") ? libPreamble + code : code
)};`,
code: `export default ${JSON.stringify(code)};`,
map: { mappings: "" }
};
}

4
tests/error_003_typescript.ts.out
View file

@ -3,8 +3,8 @@
[WILDCARD][0m consol.log("hello world!");
  ~~~~~~
$asset$/globals.d.tsILDCARD]
[WILDCARD]const console: Console;
$asset$/lib.deno_runtime.d.tsILDCARD]
[WILDCARD]const console: console.Console;
[WILDCARD]~~~~~~~
[WILDCARD]'console' is declared here.

2
third_party

@ -1 +1 @@
Subproject commit 8401dc99b953c61c2a79c2f85294d8c7f3c55c9d
Subproject commit a133fa714b960d8f88c55188ccc1a41882961e6e

1
tools/format.py
View file

@ -34,6 +34,7 @@ run(["node", prettier, "--write"] +
find_exts(".github/", ".md") +
find_exts("js/", ".js", ".ts", ".md") +
find_exts("tests/", ".js", ".ts", ".md") +
find_exts("tools/", ".js", ".json", ".ts", ".md") +
find_exts("website/", ".js", ".ts", ".md"))
# yapf: enable

95
tools/ts_library_builder/README.md Normal file
View file

@ -0,0 +1,95 @@
# ts_library_builder
This tool allows us to produce a single TypeScript declaration file that
describes the complete Deno runtime, including global variables and the built-in
`deno` module. The output of this tool, `lib.deno_runtime.d.ts`, serves several
purposes:
1. It is passed to the TypeScript compiler `js/compiler.ts`, so that TypeScript
knows what types to expect and can validate code against the runtime
environment.
2. It is outputted to stdout by `deno --types`, so that users can easily have
access to the complete declaration file. Editors can use this in the future
to perform type checking.
3. Because JSDocs are maintained, this serves as a simple documentation page for
Deno. We will use this file to generate HTML docs in the future.
The tool depends upon a couple libraries:
- [`ts-node`](https://www.npmjs.com/package/ts-node) to provide just in time
transpiling of TypeScript for the tool itself.
- [`ts-simple-ast`](https://www.npmjs.com/package/ts-simple-ast) which provides
a more rational and functional interface to the TypeScript AST to make
manipulations easier.
- [`prettier`](https://www.npmjs.com/package/prettier) and
[`@types/prettier`](https://www.npmjs.com/package/@types/prettier) to format
the output.
## Design
Ideally we wouldn't have to build this tool at all, and could simply use `tsc`
to output this declaration file. While, `--emitDeclarationsOnly`, `--outFile`
and `--module AMD` generates a single declaration file, it isn't clean. It was
never designed for a library generation, where what is available in a runtime
environment significantly differs from the code that creates that environment's
structure.
Therefore this tool injects some of the knowledge of what occurs in the Deno
runtime environment as well as ensures that the output file is more clean and
logical for an end user. In the deno runtime, code runs in a global scope that
is defined in `js/global.ts`. This contains global scope items that one
reasonably expects in a JavaScript runtime, like `console`. It also defines the
global scope on a self-reflective `window` variable. There is currently only one
module of Deno specific APIs which is available to the user. This is defined in
`js/deno.ts`.
This tool takes advantage of an experimental feature of TypeScript that items
that are not really intended to be part of the public API are marked with a
comment pragma of `@internal` and then are not emitted when generating type
definitions. In addition TypeScript will _tree-shake_ any dependencies tied to
that "hidden" API and elide them as well. This really helps keep the public API
clean and as minimal as needed.
In order to create the default type library, the process at a high-level looks
like this:
- We read in all of the runtime environment definition code into TypeScript AST
parser "project".
- We emit the TypeScript type definitions only into another AST parser
"project".
- We process the `deno` namespace/module, by "flattening" the type definition
file.
- We determine the exported symbols for `js/deno.ts`.
- We create a custom extraction of the `gen/msg_generated.ts` which is
generated during the build process and contains the type information related
to flatbuffer structures that communicate between the privileged part of
deno and the user land. Currently, the tool doesn't do full complex
dependency analysis to be able to determine what is required out of this
file, so we explicitly extract the type information we need.
- We recurse over all imports/exports of the modules, only exporting those
symbols which are finally exported by `js/deno.ts`.
- We replace the import/export with the type information from the source file.
- This process assumes that all the modules that feed `js/deno.ts` will have a
public type API that does not have name conflicts.
- We process the `js/globals.ts` file to generate the global namespace.
- Currently we create a `"globals"` module which will contain the type
definitions.
- We create a `Window` interface and a `global` scope augmentation namespace.
- We iterate over augmentations to the `window` variable declared in the file,
extract the type information and apply it to both a global variable
declaration and a property on the `Window` interface.
- We take each namespace import to `js/globals.ts`, we resolve the emitted
declaration `.d.ts` file and create it as its own namespace withing the
`"globals"` module. It is unsafe to just flatten these, because there is a
high risk of collisions, but also, it makes authoring the types easier within
the generated interface and variable declarations.
- We then validate the resulting definition file and write it out to the
appropriate build path.
## TODO
- The tool does not _tree-shake_ when flattening imports. This means there are
extraneous types that get included that are not really needed and it means
that `gen/msg_generated.ts` has to be explicitly carved down.
- Complete the tests... we have some coverage, but not a lot of what is in
`ast_util_test` which is being tested implicitly.

331
tools/ts_library_builder/ast_util.ts Normal file
View file

@ -0,0 +1,331 @@
import { relative } from "path";
import { readFileSync } from "fs";
import { EOL } from "os";
import {
ExportDeclaration,
ImportDeclaration,
InterfaceDeclaration,
JSDoc,
Project,
PropertySignature,
SourceFile,
StatementedNode,
ts,
TypeGuards,
VariableStatement,
VariableDeclarationKind
} from "ts-simple-ast";
/** Add a property to an interface */
export function addInterfaceProperty(
interfaceDeclaration: InterfaceDeclaration,
name: string,
type: string,
jsdocs?: JSDoc[]
): PropertySignature {
return interfaceDeclaration.addProperty({
name,
type,
docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText())
});
}
/** Add `@url` comment to node. */
export function addSourceComment(
node: StatementedNode,
sourceFile: SourceFile,
rootPath: string
): void {
node.insertStatements(
0,
`// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n`
);
}
/** Add a declaration of a variable to a node */
export function addVariableDeclaration(
node: StatementedNode,
name: string,
type: string,
jsdocs?: JSDoc[]
): VariableStatement {
return node.addVariableStatement({
declarationKind: VariableDeclarationKind.Const,
declarations: [{ name, type }],
docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText())
});
}
/** Check diagnostics, and if any exist, exit the process */
export function checkDiagnostics(project: Project, onlyFor?: string[]) {
const program = project.getProgram();
const diagnostics = [
...program.getGlobalDiagnostics(),
...program.getSyntacticDiagnostics(),
...program.getSemanticDiagnostics(),
...program.getDeclarationDiagnostics()
]
.filter(diagnostic => {
const sourceFile = diagnostic.getSourceFile();
return onlyFor && sourceFile
? onlyFor.includes(sourceFile.getFilePath())
: true;
})
.map(diagnostic => diagnostic.compilerObject);
if (diagnostics.length) {
console.log(
ts.formatDiagnosticsWithColorAndContext(diagnostics, formatDiagnosticHost)
);
process.exit(1);
}
}
export interface FlattenNamespaceOptions {
customSources?: { [sourceFilePath: string]: string };
debug?: boolean;
rootPath: string;
sourceFile: SourceFile;
}
/** Take a namespace and flatten all exports. */
export function flattenNamespace({
customSources,
debug,
rootPath,
sourceFile
}: FlattenNamespaceOptions): string {
const sourceFiles = new Set<SourceFile>();
let output = "";
const exportedSymbols = getExportedSymbols(sourceFile);
function flattenDeclarations(
declaration: ImportDeclaration | ExportDeclaration
) {
const declarationSourceFile = declaration.getModuleSpecifierSourceFile();
if (declarationSourceFile) {
processSourceFile(declarationSourceFile);
declaration.remove();
}
}
function rectifyNodes(currentSourceFile: SourceFile) {
currentSourceFile.forEachChild(node => {
if (TypeGuards.isAmbientableNode(node)) {
node.setHasDeclareKeyword(false);
}
if (TypeGuards.isExportableNode(node)) {
const nodeSymbol = node.getSymbol();
if (
nodeSymbol &&
!exportedSymbols.has(nodeSymbol.getFullyQualifiedName())
) {
node.setIsExported(false);
}
}
});
}
function processSourceFile(currentSourceFile: SourceFile) {
if (sourceFiles.has(currentSourceFile)) {
return;
}
sourceFiles.add(currentSourceFile);
const currentSourceFilePath = currentSourceFile.getFilePath();
if (customSources && currentSourceFilePath in customSources) {
output += customSources[currentSourceFilePath];
return;
}
currentSourceFile.getImportDeclarations().forEach(flattenDeclarations);
currentSourceFile.getExportDeclarations().forEach(flattenDeclarations);
rectifyNodes(currentSourceFile);
output +=
(debug ? getSourceComment(currentSourceFile, rootPath) : "") +
currentSourceFile.print();
}
sourceFile.getExportDeclarations().forEach(exportDeclaration => {
processSourceFile(exportDeclaration.getModuleSpecifierSourceFileOrThrow());
exportDeclaration.remove();
});
rectifyNodes(sourceFile);
return (
output +
(debug ? getSourceComment(sourceFile, rootPath) : "") +
sourceFile.print()
);
}
/** Used when formatting diagnostics */
const formatDiagnosticHost: ts.FormatDiagnosticsHost = {
getCurrentDirectory() {
return process.cwd();
},
getCanonicalFileName(path: string) {
return path;
},
getNewLine() {
return EOL;
}
};
/** Return a set of fully qualified symbol names for the files exports */
function getExportedSymbols(sourceFile: SourceFile): Set<string> {
const exportedSymbols = new Set<string>();
const exportDeclarations = sourceFile.getExportDeclarations();
for (const exportDeclaration of exportDeclarations) {
const exportSpecifiers = exportDeclaration.getNamedExports();
for (const exportSpecifier of exportSpecifiers) {
const aliasedSymbol = exportSpecifier
.getSymbolOrThrow()
.getAliasedSymbol();
if (aliasedSymbol) {
exportedSymbols.add(aliasedSymbol.getFullyQualifiedName());
}
}
}
return exportedSymbols;
}
/** Returns a string which indicates the source file as the source */
export function getSourceComment(
sourceFile: SourceFile,
rootPath: string
): string {
return `\n// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n`;
}
/**
* Load and write to a virtual file system all the default libs needed to
* resolve types on project.
*/
export function loadDtsFiles(project: Project) {
loadFiles(
project,
[
"lib.es2015.collection.d.ts",
"lib.es2015.core.d.ts",
"lib.es2015.d.ts",
"lib.es2015.generator.d.ts",
"lib.es2015.iterable.d.ts",
"lib.es2015.promise.d.ts",
"lib.es2015.proxy.d.ts",
"lib.es2015.reflect.d.ts",
"lib.es2015.symbol.d.ts",
"lib.es2015.symbol.wellknown.d.ts",
"lib.es2016.array.include.d.ts",
"lib.es2016.d.ts",
"lib.es2017.d.ts",
"lib.es2017.intl.d.ts",
"lib.es2017.object.d.ts",
"lib.es2017.sharedmemory.d.ts",
"lib.es2017.string.d.ts",
"lib.es2017.typedarrays.d.ts",
"lib.es2018.d.ts",
"lib.es2018.intl.d.ts",
"lib.es2018.promise.d.ts",
"lib.es2018.regexp.d.ts",
"lib.es5.d.ts",
"lib.esnext.d.ts",
"lib.esnext.array.d.ts",
"lib.esnext.asynciterable.d.ts",
"lib.esnext.intl.d.ts",
"lib.esnext.symbol.d.ts"
].map(fileName => `node_modules/typescript/lib/${fileName}`)
);
}
/** Load a set of files into a file system host. */
export function loadFiles(project: Project, filePaths: string[]) {
const fileSystem = project.getFileSystem();
for (const filePath of filePaths) {
const fileText = readFileSync(filePath, {
encoding: "utf8"
});
fileSystem.writeFileSync(filePath, fileText);
}
}
export interface NamespaceSourceFileOptions {
debug?: boolean;
namespace?: string;
rootPath: string;
sourceFileMap: Map<SourceFile, string>;
}
/**
* Take a source file (`.d.ts`) and convert it to a namespace, resolving any
* imports as their own namespaces.
*/
export function namespaceSourceFile(
sourceFile: SourceFile,
{ debug, namespace, rootPath, sourceFileMap }: NamespaceSourceFileOptions
): string {
if (sourceFileMap.has(sourceFile)) {
return "";
}
if (!namespace) {
namespace = sourceFile.getBaseNameWithoutExtension();
}
sourceFileMap.set(sourceFile, namespace);
sourceFile.forEachChild(node => {
if (TypeGuards.isAmbientableNode(node)) {
node.setHasDeclareKeyword(false);
}
});
const globalNamespace = sourceFile.getNamespace("global");
const globalNamespaceText = globalNamespace && globalNamespace.print();
if (globalNamespace) {
globalNamespace.remove();
}
const output = sourceFile
.getImportDeclarations()
.map(declaration => {
if (
declaration.getNamedImports().length ||
!declaration.getNamespaceImport()
) {
throw new Error(
"Unsupported import clause.\n" +
` In: "${declaration.getSourceFile().getFilePath()}"\n` +
` Text: "${declaration.getText()}"`
);
}
const text = namespaceSourceFile(
declaration.getModuleSpecifierSourceFileOrThrow(),
{
debug,
namespace: declaration.getNamespaceImportOrThrow().getText(),
rootPath,
sourceFileMap
}
);
declaration.remove();
return text;
})
.join("\n");
sourceFile
.getExportDeclarations()
.forEach(declaration => declaration.remove());
return `${output}
${globalNamespaceText || ""}
namespace ${namespace} {
${debug ? getSourceComment(sourceFile, rootPath) : ""}
${sourceFile.getText()}
}`;
}
/** Mirrors TypeScript's handling of paths */
export function normalizeSlashes(path: string): string {
return path.replace(/\\/g, "/");
}

453
tools/ts_library_builder/build_library.ts Normal file
View file

@ -0,0 +1,453 @@
import { writeFileSync } from "fs";
import * as prettier from "prettier";
import {
ExpressionStatement,
ModuleKind,
ModuleResolutionKind,
NamespaceDeclarationKind,
Project,
ScriptTarget,
SourceFile,
Type,
TypeGuards
} from "ts-simple-ast";
import {
addInterfaceProperty,
addSourceComment,
addVariableDeclaration,
checkDiagnostics,
flattenNamespace,
getSourceComment,
loadDtsFiles,
loadFiles,
namespaceSourceFile,
normalizeSlashes
} from "./ast_util";
export interface BuildLibraryOptions {
/**
* The path to the root of the deno repository
*/
basePath: string;
/**
* The path to the current build path
*/
buildPath: string;
/**
* Denotes if the library should be built with debug information (comments
* that indicate the source of the types)
*/
debug?: boolean;
/**
* The path to the output library
*/
outFile: string;
/**
* Execute in silent mode or not
*/
silent?: boolean;
}
/**
* A preamble which is appended to the start of the library.
*/
// tslint:disable-next-line:max-line-length
const libPreamble = `// Copyright 2018 the Deno authors. All rights reserved. MIT license.
/// <reference no-default-lib="true" />
/// <reference lib="esnext" />
`;
// The path to the msg_generated file relative to the build path
const MSG_GENERATED_PATH = "/gen/msg_generated.ts";
// An array of enums we want to expose pub
const MSG_GENERATED_ENUMS = ["ErrorKind"];
/** Extracts enums from a source file */
function extract(sourceFile: SourceFile, enumNames: string[]): string {
// Copy specified enums from msg_generated
let output = "";
for (const enumName of enumNames) {
const enumDeclaration = sourceFile.getEnumOrThrow(enumName);
enumDeclaration.setHasDeclareKeyword(false);
// we are not copying JSDocs or other trivia here because msg_generated only
// contains some non-useful JSDocs and comments that are not ideal to copy
// over
output += enumDeclaration.getText();
}
return output;
}
interface FlattenOptions {
basePath: string;
customSources: { [filePath: string]: string };
filePath: string;
debug?: boolean;
declarationProject: Project;
namespaceName: string;
targetSourceFile: SourceFile;
}
/** Flatten a module */
export function flatten({
basePath,
customSources,
filePath,
debug,
declarationProject,
namespaceName,
targetSourceFile
}: FlattenOptions): void {
// Flatten the source file into a single module declaration
const statements = flattenNamespace({
sourceFile: declarationProject.getSourceFileOrThrow(filePath),
rootPath: basePath,
customSources,
debug
});
// Create the module in the target file
const namespace = targetSourceFile.addNamespace({
name: namespaceName,
hasDeclareKeyword: true,
declarationKind: NamespaceDeclarationKind.Module
});
// Add the output of the flattening to the namespace
namespace.addStatements(statements);
}
interface MergeOptions {
basePath: string;
declarationProject: Project;
debug?: boolean;
globalVarName: string;
filePath: string;
inputProject: Project;
interfaceName: string;
namespaceName: string;
targetSourceFile: SourceFile;
}
/** Take a module and merge into into a single namespace */
export function merge({
basePath,
declarationProject,
debug,
globalVarName,
filePath,
inputProject,
interfaceName,
namespaceName,
targetSourceFile
}: MergeOptions) {
// We have to build the module/namespace in small pieces which will reflect
// how the global runtime environment will be for Deno
// We need to add a module named `"globals"` which will contain all the global
// runtime context
const mergedModule = targetSourceFile.addNamespace({
name: namespaceName,
hasDeclareKeyword: true,
declarationKind: NamespaceDeclarationKind.Module
});
// Add the global Window interface
const interfaceDeclaration = mergedModule.addInterface({
name: interfaceName
});
// Add the global scope augmentation module of the "globals" module
const mergedGlobalNamespace = mergedModule.addNamespace({
name: "global",
declarationKind: NamespaceDeclarationKind.Global
});
// Declare the global variable
addVariableDeclaration(mergedGlobalNamespace, globalVarName, interfaceName);
// Add self reference to the global variable
addInterfaceProperty(interfaceDeclaration, globalVarName, interfaceName);
// Retrieve source file from the input project
const sourceFile = inputProject.getSourceFileOrThrow(filePath);
// we are going to create a map of variables
const globalVariables = new Map<
string,
{
type: Type;
node: ExpressionStatement;
}
>();
// For every augmentation of the global variable in source file, we want
// to extract the type and add it to the global variable map
sourceFile.forEachChild(node => {
if (TypeGuards.isExpressionStatement(node)) {
const firstChild = node.getFirstChild();
if (!firstChild) {
return;
}
if (TypeGuards.isBinaryExpression(firstChild)) {
const leftExpression = firstChild.getLeft();
if (
TypeGuards.isPropertyAccessExpression(leftExpression) &&
leftExpression.getExpression().getText() === globalVarName
) {
const windowProperty = leftExpression.getName();
if (windowProperty !== globalVarName) {
globalVariables.set(windowProperty, {
type: firstChild.getType(),
node
});
}
}
}
}
});
// A set of source files that the types we are using are dependent on us
// importing
const dependentSourceFiles = new Set<SourceFile>();
// Create a global variable and add the property to the `Window` interface
// for each mutation of the `window` variable we observed in `globals.ts`
for (const [property, info] of globalVariables) {
const type = info.type.getText(info.node);
const typeSymbol = info.type.getSymbol();
if (typeSymbol) {
const valueDeclaration = typeSymbol.getValueDeclaration();
if (valueDeclaration) {
dependentSourceFiles.add(valueDeclaration.getSourceFile());
}
}
addVariableDeclaration(mergedGlobalNamespace, property, type);
addInterfaceProperty(interfaceDeclaration, property, type);
}
// We need to ensure that we only namespace each source file once, so we
// will use this map for tracking that.
const sourceFileMap = new Map<SourceFile, string>();
// For each import declaration in source file we will want to convert the
// declaration source file into a namespace that exists within the merged
// namespace
const importDeclarations = sourceFile.getImportDeclarations();
for (const declaration of importDeclarations) {
const declarationSourceFile = declaration.getModuleSpecifierSourceFile();
if (
declarationSourceFile &&
dependentSourceFiles.has(declarationSourceFile)
) {
// the source file will resolve to the original `.ts` file, but the
// information we really want is in the emitted `.d.ts` file, so we will
// resolve to that file
const dtsFilePath = declarationSourceFile
.getFilePath()
.replace(/\.ts$/, ".d.ts");
const dtsSourceFile = declarationProject.getSourceFileOrThrow(
dtsFilePath
);
mergedModule.addStatements(
namespaceSourceFile(dtsSourceFile, {
debug,
namespace: declaration.getNamespaceImportOrThrow().getText(),
rootPath: basePath,
sourceFileMap
})
);
}
}
if (debug) {
addSourceComment(mergedModule, sourceFile, basePath);
}
}
/**
* Generate the runtime library for Deno and write it to the supplied out file
* name.
*/
export function main({
basePath,
buildPath,
debug,
outFile,
silent
}: BuildLibraryOptions) {
if (!silent) {
console.log("-----");
console.log("build_lib");
console.log();
console.log(`basePath: "${basePath}"`);
console.log(`buildPath: "${buildPath}"`);
console.log(`debug: ${!!debug}`);
console.log(`outFile: "${outFile}"`);
console.log();
}
// the inputProject will take in the TypeScript files that are internal
// to Deno to be used to generate the library
const inputProject = new Project({
compilerOptions: {
baseUrl: basePath,
declaration: true,
emitDeclarationOnly: true,
lib: [],
module: ModuleKind.AMD,
moduleResolution: ModuleResolutionKind.NodeJs,
noLib: true,
paths: {
"*": ["*", `${buildPath}/*`]
},
preserveConstEnums: true,
strict: true,
stripInternal: true,
target: ScriptTarget.ESNext
}
});
// Add the input files we will need to generate the declarations, `globals`
// plus any modules that are importable in the runtime need to be added here
// plus the `lib.esnext` which is used as the base library
inputProject.addExistingSourceFiles([
`${basePath}/node_modules/typescript/lib/lib.esnext.d.ts`,
`${basePath}/js/deno.ts`,
`${basePath}/js/globals.ts`
]);
// emit the project, which will be only the declaration files
const inputEmitResult = inputProject.emitToMemory();
// the declaration project will be the target for the emitted files from
// the input project, these will be used to transfer information over to
// the final library file
const declarationProject = new Project({
compilerOptions: {
baseUrl: basePath,
moduleResolution: ModuleResolutionKind.NodeJs,
noLib: true,
paths: {
"*": ["*", `${buildPath}/*`]
},
strict: true,
target: ScriptTarget.ESNext
},
useVirtualFileSystem: true
});
// we don't want to add to the declaration project any of the original
// `.ts` source files, so we need to filter those out
const jsPath = normalizeSlashes(`${basePath}/js`);
const inputProjectFiles = inputProject
.getSourceFiles()
.map(sourceFile => sourceFile.getFilePath())
.filter(filePath => !filePath.startsWith(jsPath));
loadFiles(declarationProject, inputProjectFiles);
// now we add the emitted declaration files from the input project
for (const { filePath, text } of inputEmitResult.getFiles()) {
declarationProject.createSourceFile(filePath, text);
}
// the outputProject will contain the final library file we are looking to
// build
const outputProject = new Project({
compilerOptions: {
baseUrl: buildPath,
moduleResolution: ModuleResolutionKind.NodeJs,
noLib: true,
strict: true,
target: ScriptTarget.ESNext,
types: ["text-encoding"]
},
useVirtualFileSystem: true
});
// There are files we need to load into memory, so that the project "compiles"
loadDtsFiles(outputProject);
// tslint:disable-next-line:max-line-length
const textEncodingFilePath = `${buildPath}/node_modules/@types/text-encoding/index.d.ts`;
loadFiles(outputProject, [textEncodingFilePath]);
outputProject.addExistingSourceFileIfExists(textEncodingFilePath);
// libDts is the final output file we are looking to build and we are not
// actually creating it, only in memory at this stage.
const libDTs = outputProject.createSourceFile(outFile);
// Deal with `js/deno.ts`
// `gen/msg_generated.d.ts` contains too much exported information that is not
// part of the public API surface of Deno, so we are going to extract just the
// information we need.
const msgGeneratedDts = inputProject.getSourceFileOrThrow(
`${buildPath}${MSG_GENERATED_PATH}`
);
const msgGeneratedDtsText = extract(msgGeneratedDts, MSG_GENERATED_ENUMS);
// Generate a object hash of substitutions of modules to use when flattening
const customSources = {
[msgGeneratedDts.getFilePath()]: `${
debug ? getSourceComment(msgGeneratedDts, basePath) : ""
}${msgGeneratedDtsText}\n`
};
flatten({
basePath,
customSources,
debug,
declarationProject,
filePath: `${basePath}/js/deno.d.ts`,
namespaceName: `"deno"`,
targetSourceFile: libDTs
});
if (!silent) {
console.log(`Created module "deno".`);
}
merge({
basePath,
declarationProject,
debug,
globalVarName: "window",
filePath: `${basePath}/js/globals.ts`,
inputProject,
interfaceName: "Window",
namespaceName: `"globals"`,
targetSourceFile: libDTs
});
if (!silent) {
console.log(`Created module "globals".`);
}
// Add the preamble
libDTs.insertStatements(0, libPreamble);
// Check diagnostics
checkDiagnostics(outputProject);
// Output the final library file
libDTs.saveSync();
const libDTsText = prettier.format(
outputProject.getFileSystem().readFileSync(outFile, "utf8"),
{ parser: "typescript" }
);
if (!silent) {
console.log(`Outputting library to: "${outFile}"`);
console.log(` Length: ${libDTsText.length}`);
}
writeFileSync(outFile, libDTsText, { encoding: "utf8" });
if (!silent) {
console.log("-----");
console.log();
}
}

39
tools/ts_library_builder/main.ts Normal file
View file

@ -0,0 +1,39 @@
import * as path from "path";
import { main as buildRuntimeLib } from "./build_library";
// this is very simplistic argument parsing, just enough to integrate into
// the build scripts, versus being very robust
let basePath = process.cwd();
let buildPath = path.join(basePath, "out", "debug");
let outFile = path.join(buildPath, "gen", "lib", "lib.d.ts");
let debug = false;
let silent = false;
process.argv.forEach((arg, i, argv) => {
// tslint:disable-next-line:switch-default
switch (arg) {
case "--basePath":
basePath = path.resolve(argv[i + 1]);
break;
case "--buildPath":
buildPath = path.resolve(argv[i + 1]);
break;
case "--outFile":
outFile = path.resolve(argv[i + 1]);
break;
case "--debug":
debug = true;
break;
case "--silent":
silent = true;
break;
}
});
buildRuntimeLib({
basePath,
buildPath,
debug,
outFile,
silent
});

159
tools/ts_library_builder/test.ts Normal file
View file

@ -0,0 +1,159 @@
// Run this manually with:
//
// ./node_modules/.bin/ts-node --project tools/ts_library_builder/tsconfig.json tools/ts_library_builder/test.ts
import {
ModuleKind,
ModuleResolutionKind,
Project,
ScriptTarget
} from "ts-simple-ast";
import { assert, assertEqual, test } from "../../js/testing/testing";
import { flatten, merge } from "./build_library";
import { loadDtsFiles } from "./ast_util";
/** setups and returns the fixtures for testing */
function setupFixtures() {
const basePath = process.cwd();
const buildPath = `${basePath}/tools/ts_library_builder/testdata`;
const outputFile = `${buildPath}/lib.output.d.ts`;
const inputProject = new Project({
compilerOptions: {
baseUrl: basePath,
declaration: true,
emitDeclarationOnly: true,
module: ModuleKind.AMD,
moduleResolution: ModuleResolutionKind.NodeJs,
strict: true,
stripInternal: true,
target: ScriptTarget.ESNext
}
});
inputProject.addExistingSourceFiles([
`${buildPath}/globals.ts`,
`${buildPath}/api.ts`
]);
const declarationProject = new Project({
compilerOptions: {},
useVirtualFileSystem: true
});
loadDtsFiles(declarationProject);
for (const { filePath, text } of inputProject.emitToMemory().getFiles()) {
declarationProject.createSourceFile(filePath, text);
}
const outputProject = new Project({
compilerOptions: {},
useVirtualFileSystem: true
});
loadDtsFiles(outputProject);
const outputSourceFile = outputProject.createSourceFile(outputFile);
const debug = true;
return {
basePath,
buildPath,
inputProject,
outputFile,
declarationProject,
outputProject,
outputSourceFile,
debug
};
}
test(function buildLibraryFlatten() {
const {
basePath,
buildPath,
debug,
declarationProject,
outputSourceFile: targetSourceFile
} = setupFixtures();
flatten({
basePath,
customSources: {},
debug,
declarationProject,
filePath: `${buildPath}/api.d.ts`,
namespaceName: `"api"`,
targetSourceFile
});
assert(targetSourceFile.getNamespace(`"api"`) != null);
assertEqual(targetSourceFile.getNamespaces().length, 1);
const namespaceApi = targetSourceFile.getNamespaceOrThrow(`"api"`);
const functions = namespaceApi.getFunctions();
assertEqual(functions[0].getName(), "foo");
assertEqual(
functions[0]
.getJsDocs()
.map(jsdoc => jsdoc.getInnerText())
.join("\n"),
"jsdoc for foo"
);
assertEqual(functions[1].getName(), "bar");
assertEqual(
functions[1]
.getJsDocs()
.map(jsdoc => jsdoc.getInnerText())
.join("\n"),
""
);
assertEqual(functions.length, 2);
const classes = namespaceApi.getClasses();
assertEqual(classes[0].getName(), "Foo");
assertEqual(classes.length, 1);
const variableDeclarations = namespaceApi.getVariableDeclarations();
assertEqual(variableDeclarations[0].getName(), "arr");
assertEqual(variableDeclarations.length, 1);
});
test(function buildLibraryMerge() {
const {
basePath,
buildPath,
declarationProject,
debug,
inputProject,
outputSourceFile: targetSourceFile
} = setupFixtures();
merge({
basePath,
declarationProject,
debug,
globalVarName: "foobarbaz",
filePath: `${buildPath}/globals.ts`,
inputProject,
interfaceName: "FooBar",
namespaceName: `"bazqat"`,
targetSourceFile
});
assert(targetSourceFile.getNamespace(`"bazqat"`) != null);
assertEqual(targetSourceFile.getNamespaces().length, 1);
const namespaceBazqat = targetSourceFile.getNamespaceOrThrow(`"bazqat"`);
assert(namespaceBazqat.getNamespace("global") != null);
assert(namespaceBazqat.getNamespace("moduleC") != null);
assertEqual(namespaceBazqat.getNamespaces().length, 2);
assert(namespaceBazqat.getInterface("FooBar") != null);
assertEqual(namespaceBazqat.getInterfaces().length, 1);
const globalNamespace = namespaceBazqat.getNamespaceOrThrow("global");
const variableDeclarations = globalNamespace.getVariableDeclarations();
assertEqual(
variableDeclarations[0].getType().getText(),
`import("bazqat").FooBar`
);
assertEqual(
variableDeclarations[1].getType().getText(),
`import("bazqat").moduleC.Bar`
);
assertEqual(
variableDeclarations[2].getType().getText(),
`typeof import("bazqat").moduleC.qat`
);
assertEqual(variableDeclarations.length, 3);
});
// TODO author unit tests for `ast_util.ts`

4
tools/ts_library_builder/testdata/api.ts vendored Normal file
View file

@ -0,0 +1,4 @@
export { foo, bar } from "./moduleA";
export { Foo } from "./moduleB";
/** jsdoc for arr */
export const arr: string[] = [];

6
tools/ts_library_builder/testdata/globals.ts vendored Normal file
View file

@ -0,0 +1,6 @@
import * as moduleC from "./moduleC";
// tslint:disable-next-line:no-any
const foobarbaz: any = {};
foobarbaz.bar = new moduleC.Bar();
foobarbaz.qat = moduleC.qat;

9
tools/ts_library_builder/testdata/moduleA.ts vendored Normal file
View file

@ -0,0 +1,9 @@
/** jsdoc for foo */
export function foo(a: string, b: string) {
console.log(a, b);
}
// no jsdoc for bar
export async function bar(promise: Promise<void>): Promise<void> {
return promise.then(() => {});
}

9
tools/ts_library_builder/testdata/moduleB.ts vendored Normal file
View file

@ -0,0 +1,9 @@
/** jsdoc about Foo */
export class Foo {
private _foo = "foo";
/** jsdoc about Foo.log() */
log() {
console.log(this._foo);
return this._foo;
}
}

18
tools/ts_library_builder/testdata/moduleC.ts vendored Normal file
View file

@ -0,0 +1,18 @@
/** jsdoc for Bar */
export class Bar {
private _bar: string;
/** jsdoc for Bar.log() */
log() {
console.log(this._bar);
return this.log;
}
}
/**
* jsdoc for qat
* @param a jsdoc for qat(a)
* @param b jsdoc for qat(b)
*/
export function qat(a: string, b: string) {
return a + b;
}

8
tools/ts_library_builder/tsconfig.json Normal file
View file

@ -0,0 +1,8 @@
{
"compilerOptions": {
"moduleResolution": "node",
"strict": true,
"target": "esnext"
},
"files": ["./build_library.ts"]
}