foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-25 00:29:09 -05:00
Code Issues Packages 1 Wiki Activity

Remove ts_library_builder, maintain lib.deno_runtime.d.ts by hand (#2827)

Browse source
This commit is contained in:
Ryan Dahl 2019-08-30 11:11:33 -04:00 committed by GitHub
parent 5401cb7630
commit c370f749b2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 2869 additions and 1753 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
cli/BUILD.gn
View file

@ -61,7 +61,6 @@ if (is_posix) {
}
ts_sources = [
"../js/assets.ts",
"../js/base64.ts",
"../js/blob.ts",
"../js/body.ts",
@ -98,6 +97,7 @@ ts_sources = [
"../js/globals.ts",
"../js/headers.ts",
"../js/io.ts",
"../js/lib.deno_runtime.d.ts",
"../js/lib.web_assembly.d.ts",
"../js/link.ts",
"../js/location.ts",
@ -200,55 +200,16 @@ rust_test("cli_test") {
env = [ "CARGO_PKG_VERSION=${deno_cargo_info.version}" ]
}
# Generates the core TypeScript type library for deno that will be
# included in the runtime bundle
run_node("deno_runtime_declaration") {
out_dir = target_gen_dir
sources = ts_sources
outputs = [
"$out_dir/lib/lib.deno_runtime.d.ts",
]
inputs = ts_sources + [
"//tools/ts_library_builder/tsconfig.json",
"//tools/ts_library_builder/main.ts",
"//tools/ts_library_builder/build_library.ts",
"//tools/ts_library_builder/ast_util.ts",
]
args = [
rebase_path("//node_modules/ts-node/dist/bin.js", root_build_dir),
"--project",
rebase_path("//tools/ts_library_builder/tsconfig.json"),
"--skip-ignore",
rebase_path("//tools/ts_library_builder/main.ts", root_build_dir),
"--basePath",
rebase_path("//", root_build_dir),
"--inline",
rebase_path("//js/lib.web_assembly.d.ts", root_build_dir),
"--buildPath",
rebase_path(root_build_dir, root_build_dir),
"--outFile",
rebase_path(outputs[0], root_build_dir),
"--silent",
]
if (is_debug) {
args += [ "--debug" ]
}
}
bundle("main_bundle") {
sources = ts_sources
out_dir = "$target_gen_dir/bundle/"
out_name = "main"
deps = [
":deno_runtime_declaration",
]
}
bundle("compiler_bundle") {
sources = ts_sources
out_dir = "$target_gen_dir/bundle/"
out_name = "compiler"
deps = [
":deno_runtime_declaration",
]
}
# Generates $target_gen_dir/snapshot_deno.bin

59
cli/assets.rs Normal file
View file

@ -0,0 +1,59 @@
static DENO_RUNTIME: &str = include_str!("../js/lib.deno_runtime.d.ts");
macro_rules! inc {
($e:expr) => {
Some(include_str!(concat!(
"../third_party/node_modules/typescript/lib/",
$e
)))
};
}
pub fn get_source_code(name: &str) -> Option<&'static str> {
match name {
"lib.deno_runtime.d.ts" => Some(DENO_RUNTIME),
"lib.es2015.collection.d.ts" => inc!("lib.es2015.collection.d.ts"),
"lib.es2015.core.d.ts" => inc!("lib.es2015.core.d.ts"),
"lib.es2015.d.ts" => inc!("lib.es2015.d.ts"),
"lib.es2015.generator.d.ts" => inc!("lib.es2015.generator.d.ts"),
"lib.es2015.iterable.d.ts" => inc!("lib.es2015.iterable.d.ts"),
"lib.es2015.promise.d.ts" => inc!("lib.es2015.promise.d.ts"),
"lib.es2015.proxy.d.ts" => inc!("lib.es2015.proxy.d.ts"),
"lib.es2015.reflect.d.ts" => inc!("lib.es2015.reflect.d.ts"),
"lib.es2015.symbol.d.ts" => inc!("lib.es2015.symbol.d.ts"),
"lib.es2015.symbol.wellknown.d.ts" => {
inc!("lib.es2015.symbol.wellknown.d.ts")
}
"lib.es2016.array.include.d.ts" => inc!("lib.es2016.array.include.d.ts"),
"lib.es2016.d.ts" => inc!("lib.es2016.d.ts"),
"lib.es2017.d.ts" => inc!("lib.es2017.d.ts"),
"lib.es2017.intl.d.ts" => inc!("lib.es2017.intl.d.ts"),
"lib.es2017.object.d.ts" => inc!("lib.es2017.object.d.ts"),
"lib.es2017.sharedmemory.d.ts" => inc!("lib.es2017.sharedmemory.d.ts"),
"lib.es2017.string.d.ts" => inc!("lib.es2017.string.d.ts"),
"lib.es2017.typedarrays.d.ts" => inc!("lib.es2017.typedarrays.d.ts"),
"lib.es2018.d.ts" => inc!("lib.es2018.d.ts"),
"lib.es2018.asynciterable.d.ts" => inc!("lib.es2018.asynciterable.d.ts"),
"lib.es2018.intl.d.ts" => inc!("lib.es2018.intl.d.ts"),
"lib.es2018.promise.d.ts" => inc!("lib.es2018.promise.d.ts"),
"lib.es2018.regexp.d.ts" => inc!("lib.es2018.regexp.d.ts"),
"lib.es2019.d.ts" => inc!("lib.es2019.d.ts"),
"lib.es2019.array.d.ts" => inc!("lib.es2019.array.d.ts"),
"lib.es2019.object.d.ts" => inc!("lib.es2019.object.d.ts"),
"lib.es2019.string.d.ts" => inc!("lib.es2019.string.d.ts"),
"lib.es2019.symbol.d.ts" => inc!("lib.es2019.symbol.d.ts"),
"lib.es2020.d.ts" => inc!("lib.es2020.d.ts"),
"lib.es2020.string.d.ts" => inc!("lib.es2020.string.d.ts"),
"lib.es2020.symbol.wellknown.d.ts" => {
inc!("lib.es2020.symbol.wellknown.d.ts")
}
"lib.es5.d.ts" => inc!("lib.es5.d.ts"),
"lib.esnext.d.ts" => inc!("lib.esnext.d.ts"),
"lib.esnext.array.d.ts" => inc!("lib.esnext.array.d.ts"),
"lib.esnext.asynciterable.d.ts" => inc!("lib.esnext.asynciterable.d.ts"),
"lib.esnext.bigint.d.ts" => inc!("lib.esnext.bigint.d.ts"),
"lib.esnext.intl.d.ts" => inc!("lib.esnext.intl.d.ts"),
"lib.esnext.symbol.d.ts" => inc!("lib.esnext.symbol.d.ts"),
_ => None,
}
}

6
cli/main.rs
View file

@ -18,6 +18,7 @@ extern crate serde_derive;
extern crate url;
mod ansi;
mod assets;
pub mod compilers;
pub mod deno_dir;
pub mod deno_error;
@ -127,10 +128,7 @@ fn create_worker_and_state(
}
fn types_command() {
let content = include_str!(concat!(
env!("GN_OUT_DIR"),
"/gen/cli/lib/lib.deno_runtime.d.ts"
));
let content = assets::get_source_code("lib.deno_runtime.d.ts").unwrap();
println!("{}", content);
}

19
cli/ops/compiler.rs
View file

@ -1,5 +1,6 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
use super::dispatch_json::{Deserialize, JsonOp, Value};
use crate::assets;
use crate::state::ThreadSafeState;
use crate::tokio_util;
use deno::*;
@ -66,3 +67,21 @@ pub fn op_fetch_source_file(
"sourceCode": String::from_utf8(out.source_code).unwrap(),
})))
}
#[derive(Deserialize)]
struct FetchAssetArgs {
name: String,
}
pub fn op_fetch_asset(
_state: &ThreadSafeState,
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: FetchAssetArgs = serde_json::from_value(args)?;
if let Some(source_code) = assets::get_source_code(&args.name) {
Ok(JsonOp::Sync(json!(source_code)))
} else {
panic!("op_fetch_asset bad asset {}", args.name)
}
}

7
cli/ops/mod.rs
View file

@ -80,6 +80,7 @@ pub const OP_READ_LINK: OpId = 53;
pub const OP_TRUNCATE: OpId = 54;
pub const OP_MAKE_TEMP_DIR: OpId = 55;
pub const OP_CWD: OpId = 56;
pub const OP_FETCH_ASSET: OpId = 57;
pub fn dispatch(
state: &ThreadSafeState,
@ -293,6 +294,12 @@ pub fn dispatch(
dispatch_json::dispatch(fs::op_make_temp_dir, state, control, zero_copy)
}
OP_CWD => dispatch_json::dispatch(fs::op_cwd, state, control, zero_copy),
OP_FETCH_ASSET => dispatch_json::dispatch(
compiler::op_fetch_asset,
state,
control,
zero_copy,
),
_ => panic!("bad op_id"),
};

99
js/assets.ts
View file

@ -1,99 +0,0 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
// eslint-disable-next-line @typescript-eslint/no-triple-slash-reference
/// <reference path="./plugins.d.ts" />
// There is a rollup plugin that will inline any module ending with `!string`
// Generated default library
import libDts from "gen/cli/lib/lib.deno_runtime.d.ts!string";
// Static libraries
import libEs2015Dts from "/third_party/node_modules/typescript/lib/lib.es2015.d.ts!string";
import libEs2015CollectionDts from "/third_party/node_modules/typescript/lib/lib.es2015.collection.d.ts!string";
import libEs2015CoreDts from "/third_party/node_modules/typescript/lib/lib.es2015.core.d.ts!string";
import libEs2015GeneratorDts from "/third_party/node_modules/typescript/lib/lib.es2015.generator.d.ts!string";
import libEs2015IterableDts from "/third_party/node_modules/typescript/lib/lib.es2015.iterable.d.ts!string";
import libEs2015PromiseDts from "/third_party/node_modules/typescript/lib/lib.es2015.promise.d.ts!string";
import libEs2015ProxyDts from "/third_party/node_modules/typescript/lib/lib.es2015.proxy.d.ts!string";
import libEs2015ReflectDts from "/third_party/node_modules/typescript/lib/lib.es2015.reflect.d.ts!string";
import libEs2015SymbolDts from "/third_party/node_modules/typescript/lib/lib.es2015.symbol.d.ts!string";
import libEs2015SymbolWellknownDts from "/third_party/node_modules/typescript/lib/lib.es2015.symbol.wellknown.d.ts!string";
import libEs2016Dts from "/third_party/node_modules/typescript/lib/lib.es2016.d.ts!string";
import libEs2016ArrayIncludeDts from "/third_party/node_modules/typescript/lib/lib.es2016.array.include.d.ts!string";
import libEs2017Dts from "/third_party/node_modules/typescript/lib/lib.es2017.d.ts!string";
import libEs2017IntlDts from "/third_party/node_modules/typescript/lib/lib.es2017.intl.d.ts!string";
import libEs2017ObjectDts from "/third_party/node_modules/typescript/lib/lib.es2017.object.d.ts!string";
import libEs2017SharedmemoryDts from "/third_party/node_modules/typescript/lib/lib.es2017.sharedmemory.d.ts!string";
import libEs2017StringDts from "/third_party/node_modules/typescript/lib/lib.es2017.string.d.ts!string";
import libEs2017TypedarraysDts from "/third_party/node_modules/typescript/lib/lib.es2017.typedarrays.d.ts!string";
import libEs2018Dts from "/third_party/node_modules/typescript/lib/lib.es2018.d.ts!string";
import libEs2018AsyncIterableDts from "/third_party/node_modules/typescript/lib/lib.es2018.asynciterable.d.ts!string";
import libEs2018IntlDts from "/third_party/node_modules/typescript/lib/lib.es2018.intl.d.ts!string";
import libEs2018PromiseDts from "/third_party/node_modules/typescript/lib/lib.es2018.promise.d.ts!string";
import libEs2018RegexpDts from "/third_party/node_modules/typescript/lib/lib.es2018.regexp.d.ts!string";
import libEs2019Dts from "/third_party/node_modules/typescript/lib/lib.es2019.d.ts!string";
import libEs2019ArrayDts from "/third_party/node_modules/typescript/lib/lib.es2019.array.d.ts!string";
import libEs2019ObjectDts from "/third_party/node_modules/typescript/lib/lib.es2019.object.d.ts!string";
import libEs2019StringDts from "/third_party/node_modules/typescript/lib/lib.es2019.string.d.ts!string";
import libEs2019SymbolDts from "/third_party/node_modules/typescript/lib/lib.es2019.symbol.d.ts!string";
import libEs2020Dts from "/third_party/node_modules/typescript/lib/lib.es2020.d.ts!string";
import libEs2020String from "/third_party/node_modules/typescript/lib/lib.es2020.string.d.ts!string";
import libEs2020SymbolWellknownDts from "/third_party/node_modules/typescript/lib/lib.es2020.symbol.wellknown.d.ts!string";
import libEs5Dts from "/third_party/node_modules/typescript/lib/lib.es5.d.ts!string";
import libEsnextArrayDts from "/third_party/node_modules/typescript/lib/lib.esnext.array.d.ts!string";
import libEsnextAsynciterablesDts from "/third_party/node_modules/typescript/lib/lib.esnext.asynciterable.d.ts!string";
import libEsnextBigintDts from "/third_party/node_modules/typescript/lib/lib.esnext.bigint.d.ts!string";
import libEsnextDts from "/third_party/node_modules/typescript/lib/lib.esnext.d.ts!string";
import libEsnextIntlDts from "/third_party/node_modules/typescript/lib/lib.esnext.intl.d.ts!string";
import libEsnextSymbolDts from "/third_party/node_modules/typescript/lib/lib.esnext.symbol.d.ts!string";
// Default static libraries for all compile jobs
const defaultAssets: { [key: string]: string } = {
"lib.es2015.collection.d.ts": libEs2015CollectionDts,
"lib.es2015.core.d.ts": libEs2015CoreDts,
"lib.es2015.d.ts": libEs2015Dts,
"lib.es2015.generator.d.ts": libEs2015GeneratorDts,
"lib.es2015.iterable.d.ts": libEs2015IterableDts,
"lib.es2015.promise.d.ts": libEs2015PromiseDts,
"lib.es2015.proxy.d.ts": libEs2015ProxyDts,
"lib.es2015.reflect.d.ts": libEs2015ReflectDts,
"lib.es2015.symbol.d.ts": libEs2015SymbolDts,
"lib.es2015.symbol.wellknown.d.ts": libEs2015SymbolWellknownDts,
"lib.es2016.array.include.d.ts": libEs2016ArrayIncludeDts,
"lib.es2016.d.ts": libEs2016Dts,
"lib.es2017.d.ts": libEs2017Dts,
"lib.es2017.intl.d.ts": libEs2017IntlDts,
"lib.es2017.object.d.ts": libEs2017ObjectDts,
"lib.es2017.sharedmemory.d.ts": libEs2017SharedmemoryDts,
"lib.es2017.string.d.ts": libEs2017StringDts,
"lib.es2017.typedarrays.d.ts": libEs2017TypedarraysDts,
"lib.es2018.d.ts": libEs2018Dts,
"lib.es2018.asynciterable.d.ts": libEs2018AsyncIterableDts,
"lib.es2018.intl.d.ts": libEs2018IntlDts,
"lib.es2018.promise.d.ts": libEs2018PromiseDts,
"lib.es2018.regexp.d.ts": libEs2018RegexpDts,
"lib.es2019.d.ts": libEs2019Dts,
"lib.es2019.array.d.ts": libEs2019ArrayDts,
"lib.es2019.object.d.ts": libEs2019ObjectDts,
"lib.es2019.string.d.ts": libEs2019StringDts,
"lib.es2019.symbol.d.ts": libEs2019SymbolDts,
"lib.es2020.d.ts": libEs2020Dts,
"lib.es2020.string.d.ts": libEs2020String,
"lib.es2020.symbol.wellknown.d.ts": libEs2020SymbolWellknownDts,
"lib.es5.d.ts": libEs5Dts,
"lib.esnext.d.ts": libEsnextDts,
"lib.esnext.array.d.ts": libEsnextArrayDts,
"lib.esnext.asynciterable.d.ts": libEsnextAsynciterablesDts,
"lib.esnext.bigint.d.ts": libEsnextBigintDts,
"lib.esnext.intl.d.ts": libEsnextIntlDts,
"lib.esnext.symbol.d.ts": libEsnextSymbolDts
};
// assests for normal compile jobs
// @internal
export const assetSourceCode: { [key: string]: string } = {
// Generated library
"lib.deno_runtime.d.ts": libDts,
...defaultAssets
};

8
js/compiler.ts
View file

@ -1,6 +1,5 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import * as ts from "typescript";
import { assetSourceCode } from "./assets";
import { bold, cyan, yellow } from "./colors";
import { Console } from "./console";
import { core } from "./core";
@ -128,6 +127,10 @@ interface EmitResult {
diagnostics?: Diagnostic;
}
function fetchAsset(name: string): string {
return sendSync(dispatch.OP_FETCH_ASSET, { name });
}
/** Ops to Rust to resolve and fetch a modules meta data. */
function fetchSourceFile(specifier: string, referrer: string): SourceFile {
util.log("compiler.fetchSourceFile", { specifier, referrer });
@ -222,8 +225,7 @@ class Host implements ts.CompilerHost {
const assetName = moduleName.includes(".")
? moduleName
: `${moduleName}.d.ts`;
assert(assetName in assetSourceCode, `No such asset "${assetName}"`);
const sourceCode = assetSourceCode[assetName];
const sourceCode = fetchAsset(assetName);
const sourceFile = {
moduleName,
filename: specifier,

1
js/dispatch.ts
View file

@ -59,6 +59,7 @@ export const OP_READ_LINK = 53;
export const OP_TRUNCATE = 54;
export const OP_MAKE_TEMP_DIR = 55;
export const OP_CWD = 56;
export const OP_FETCH_ASSET = 57;
export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void {
switch (opId) {

3
js/globals.ts
View file

@ -37,9 +37,6 @@ import { core } from "./core";
// file are tracked and created as part of default library that is built into
// Deno, we only need to declare the enough to compile Deno.
declare global {
const console: consoleTypes.Console;
const setTimeout: typeof timers.setTimeout;
interface CallSite {
getThis(): unknown;
getTypeName(): string;

2771
js/lib.deno_runtime.d.ts vendored Normal file
View file

File diff suppressed because it is too large Load diff

59
rollup.config.js
View file

@ -29,56 +29,6 @@ const tsconfigOverride = {
}
};
/** this is a rollup plugin which will look for imports ending with `!string` and resolve
* them with a module that will inline the contents of the file as a string. Needed to
* support `js/assets.ts`.
* @param {any} param0
*/
function strings(
{ include, exclude } = { include: undefined, exclude: undefined }
) {
if (!include) {
throw new Error("include option must be passed");
}
const filter = createFilter(include, exclude);
return {
name: "strings",
/**
* @param {string} importee
*/
resolveId(importee) {
if (importee.endsWith("!string")) {
// strip the `!string` from `importee`
importee = importee.slice(0, importee.lastIndexOf("!string"));
if (!importee.startsWith("gen/")) {
// this is a static asset which is located relative to the root of
// the source project
return path.resolve(path.join(__dirname, importee));
}
// this is an asset which has been generated, therefore it will be
// located within the build path
return path.resolve(path.join(process.cwd(), importee));
}
},
/**
* @param {any} code
* @param {string} id
*/
transform(code, id) {
if (filter(id)) {
return {
code: `export default ${JSON.stringify(code)};`,
map: { mappings: "" }
};
}
}
};
}
const archNodeToDeno = {
x64: "x64"
};
@ -191,15 +141,6 @@ export default function makeConfig(commandOptions) {
module: mockPath
}),
// Provides inlining of file contents for `js/assets.ts`
strings({
include: [
"*.d.ts",
`${__dirname}/**/*.d.ts`,
`${process.cwd()}/**/*.d.ts`
]
}),
// Allows rollup to resolve modules based on Node.js resolution
nodeResolve(),

4
tests/types.out
View file

@ -1,7 +1,5 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
/// <reference no-default-lib="true" />
/// <reference lib="esnext" />
[WILDCARD]
declare namespace Deno {
[WILDCARD]

10
tools/target_test.py
View file

@ -34,16 +34,6 @@ class TestTarget(DenoTestCase):
def test_core_http_benchmark(self):
self._test("deno_core_http_bench_test")
def test_ts_library_builder(self):
result = run_output([
"node", "./node_modules/.bin/ts-node", "--project",
"tools/ts_library_builder/tsconfig.json",
"tools/ts_library_builder/test.ts"
],
quiet=True)
self.assertEqual(result.code, 0)
assert "ts_library_builder ok" in result.out
def test_no_color(self):
t = os.path.join(tests_path, "no_color.js")
result = run_output([self.deno_exe, "run", t],

88
tools/ts_library_builder/README.md
View file

@ -1,88 +0,0 @@
# ts_library_builder
This tool allows us to produce a single TypeScript declaration file that
describes the complete Deno runtime, including global variables and the built-in
`Deno` global object. The output of this tool, `lib.deno_runtime.d.ts`, serves
several purposes:
1. It is passed to the TypeScript compiler `js/compiler.ts`, so that TypeScript
knows what types to expect and can validate code against the runtime
environment.
2. It is outputted to stdout by `deno types`, so that users can easily have
access to the complete declaration file. Editors can use this in the future
to perform type checking.
3. Because JSDocs are maintained, this serves as a simple documentation page for
Deno. We will use this file to generate HTML docs in the future.
The tool depends upon a couple libraries:
- [`ts-node`](https://www.npmjs.com/package/ts-node) to provide just in time
transpiling of TypeScript for the tool itself.
- [`ts-morph`](https://www.npmjs.com/package/ts-morph) which provides a more
rational and functional interface to the TypeScript AST to make manipulations
easier.
- [`prettier`](https://www.npmjs.com/package/prettier) and
[`@types/prettier`](https://www.npmjs.com/package/@types/prettier) to format
the output.
## Design
Ideally we wouldn't have to build this tool at all, and could simply use `tsc`
to output this declaration file. While, `--emitDeclarationsOnly`, `--outFile`
and `--module AMD` generates a single declaration file, it isn't clean. It was
never designed for a library generation, where what is available in a runtime
environment significantly differs from the code that creates that environment's
structure.
Therefore this tool injects some of the knowledge of what occurs in the Deno
runtime environment as well as ensures that the output file is more clean and
logical for an end user. In the deno runtime, code runs in a global scope that
is defined in `js/global.ts`. This contains global scope items that one
reasonably expects in a JavaScript runtime, like `console`. It also defines the
global scope on a self-reflective `window` variable. There is currently only one
module of Deno specific APIs which is available to the user. This is defined in
`js/deno.ts`.
This tool takes advantage of an experimental feature of TypeScript that items
that are not really intended to be part of the public API are marked with a
comment pragma of `@internal` and then are not emitted when generating type
definitions. In addition TypeScript will _tree-shake_ any dependencies tied to
that "hidden" API and elide them as well. This really helps keep the public API
clean and as minimal as needed.
In order to create the default type library, the process at a high-level looks
like this:
- We read in all of the runtime environment definition code into TypeScript AST
parser "project".
- We emit the TypeScript type definitions only into another AST parser
"project".
- We process the `deno` namespace/module, by "flattening" the type definition
file.
- We determine the exported symbols for `js/deno.ts`.
- We recurse over all imports/exports of the modules, only exporting those
symbols which are finally exported by `js/deno.ts`.
- We replace the import/export with the type information from the source file.
- This process assumes that all the modules that feed `js/deno.ts` will have a
public type API that does not have name conflicts.
- We process the `js/globals.ts` file to generate the global namespace.
- We create a `Window` interface and a `global` scope augmentation namespace.
- We iterate over augmentations to the `window` variable declared in the file,
extract the type information and apply it to both a global variable
declaration and a property on the `Window` interface.
- We identify any type aliases in the module and declare them globally.
- We take each namespace import to `js/globals.ts`, we resolve the emitted
declaration `.d.ts` file and create it as its own namespace within the global
scope. It is unsafe to just flatten these, because there is a high risk of
collisions, but also, it makes authoring the types easier within the generated
interface and variable declarations.
- We then validate the resulting definition file and write it out to the
appropriate build path.
## TODO
- The tool does not _tree-shake_ when flattening imports. This means there are
extraneous types that get included that are not really needed and it means
that `gen/msg_generated.ts` has to be explicitly carved down.
- Complete the tests... we have some coverage, but not a lot of what is in
`ast_util_test` which is being tested implicitly.

499
tools/ts_library_builder/ast_util.ts
View file

@ -1,499 +0,0 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import { basename, dirname, join, relative } from "path";
import { readFileSync } from "fs";
import { EOL } from "os";
import {
ExportDeclaration,
ImportDeclaration,
InterfaceDeclaration,
JSDoc,
Project,
PropertySignature,
SourceFile,
StatementedNode,
ts,
TypeAliasDeclaration,
TypeGuards,
VariableStatement,
VariableDeclarationKind
} from "ts-morph";
let silent = false;
/** Logs a message to the console. */
export function log(message: any = "", ...args: any[]): void {
if (!silent) {
console.log(message, ...args);
}
}
/** Sets the silent flag which impacts logging to the console. */
export function setSilent(value = false): void {
silent = value;
}
/** Add a property to an interface */
export function addInterfaceProperty(
interfaceDeclaration: InterfaceDeclaration,
name: string,
type: string,
jsdocs?: JSDoc[]
): PropertySignature {
return interfaceDeclaration.addProperty({
name,
type,
docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText())
});
}
/** Add `@url` comment to node. */
export function addSourceComment(
node: StatementedNode,
sourceFile: SourceFile,
rootPath: string
): void {
node.insertStatements(
0,
`// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n`
);
}
/** Add a declaration of a type alias to a node */
export function addTypeAlias(
node: StatementedNode,
name: string,
type: string,
hasDeclareKeyword = false,
jsdocs?: JSDoc[]
): TypeAliasDeclaration {
return node.addTypeAlias({
name,
type,
docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText()),
hasDeclareKeyword
});
}
/** Add a declaration of an interface to a node */
export function addInterfaceDeclaration(
node: StatementedNode,
interfaceDeclaration: InterfaceDeclaration
) {
const interfaceStructure = interfaceDeclaration.getStructure();
return node.addInterface({
name: interfaceStructure.name,
properties: interfaceStructure.properties,
docs: interfaceStructure.docs,
hasDeclareKeyword: true
});
}
/** Add a declaration of a variable to a node */
export function addVariableDeclaration(
node: StatementedNode,
name: string,
type: string,
isConst: boolean,
hasDeclareKeyword?: boolean,
jsdocs?: JSDoc[]
): VariableStatement {
return node.addVariableStatement({
declarationKind: isConst
? VariableDeclarationKind.Const
: VariableDeclarationKind.Let,
declarations: [{ name, type }],
docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText()),
hasDeclareKeyword
});
}
/** Copy one source file to the end of another source file. */
export function appendSourceFile(
sourceFile: SourceFile,
targetSourceFile: SourceFile
): void {
targetSourceFile.addStatements(`\n${sourceFile.print()}`);
}
/** Used when formatting diagnostics */
const formatDiagnosticHost: ts.FormatDiagnosticsHost = {
getCurrentDirectory() {
return process.cwd();
},
getCanonicalFileName(path: string) {
return path;
},
getNewLine() {
return EOL;
}
};
/** Log diagnostics to the console with colour. */
export function logDiagnostics(diagnostics: ts.Diagnostic[]): void {
if (diagnostics.length) {
console.log(
ts.formatDiagnosticsWithColorAndContext(diagnostics, formatDiagnosticHost)
);
}
}
/** Check diagnostics, and if any exist, exit the process */
export function checkDiagnostics(project: Project, onlyFor?: string[]): void {
const program = project.getProgram();
const diagnostics = [
...program.getGlobalDiagnostics(),
...program.getSyntacticDiagnostics(),
...program.getSemanticDiagnostics(),
...program.getDeclarationDiagnostics()
]
.filter(diagnostic => {
const sourceFile = diagnostic.getSourceFile();
return onlyFor && sourceFile
? onlyFor.includes(sourceFile.getFilePath())
: true;
})
.map(diagnostic => diagnostic.compilerObject);
logDiagnostics(diagnostics);
if (diagnostics.length) {
process.exit(1);
}
}
function createDeclarationError(
msg: string,
declaration: ImportDeclaration | ExportDeclaration
): Error {
return new Error(
`${msg}\n` +
` In: "${declaration.getSourceFile().getFilePath()}"\n` +
` Text: "${declaration.getText()}"`
);
}
export interface FlattenNamespaceOptions {
customSources?: { [sourceFilePath: string]: string };
debug?: boolean;
rootPath: string;
sourceFile: SourceFile;
}
/** Returns a string which indicates the source file as the source */
export function getSourceComment(
sourceFile: SourceFile,
rootPath: string
): string {
return `\n// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n`;
}
/** Return a set of fully qualified symbol names for the files exports */
function getExportedSymbols(sourceFile: SourceFile): Set<string> {
const exportedSymbols = new Set<string>();
const exportDeclarations = sourceFile.getExportDeclarations();
for (const exportDeclaration of exportDeclarations) {
const exportSpecifiers = exportDeclaration.getNamedExports();
for (const exportSpecifier of exportSpecifiers) {
const aliasedSymbol = exportSpecifier
.getSymbolOrThrow()
.getAliasedSymbol();
if (aliasedSymbol) {
exportedSymbols.add(aliasedSymbol.getFullyQualifiedName());
}
}
}
return exportedSymbols;
}
/** Take a namespace and flatten all exports. */
export function flattenNamespace({
customSources,
debug,
rootPath,
sourceFile
}: FlattenNamespaceOptions): string {
const sourceFiles = new Set<SourceFile>();
let output = "";
const exportedSymbols = getExportedSymbols(sourceFile);
function flattenDeclarations(
declaration: ImportDeclaration | ExportDeclaration
): void {
const declarationSourceFile = declaration.getModuleSpecifierSourceFile();
if (declarationSourceFile) {
// eslint-disable-next-line @typescript-eslint/no-use-before-define
processSourceFile(declarationSourceFile);
declaration.remove();
}
}
function rectifyNodes(currentSourceFile: SourceFile): void {
currentSourceFile.forEachChild(node => {
if (TypeGuards.isAmbientableNode(node)) {
node.setHasDeclareKeyword(false);
}
if (TypeGuards.isExportableNode(node)) {
const nodeSymbol = node.getSymbol();
if (
nodeSymbol &&
!exportedSymbols.has(nodeSymbol.getFullyQualifiedName())
) {
node.setIsExported(false);
}
}
});
}
function processSourceFile(
currentSourceFile: SourceFile
): string | undefined {
if (sourceFiles.has(currentSourceFile)) {
return;
}
sourceFiles.add(currentSourceFile);
const currentSourceFilePath = currentSourceFile
.getFilePath()
.replace(/(\.d)?\.ts$/, "");
log("Process source file:", currentSourceFilePath);
if (customSources && currentSourceFilePath in customSources) {
log(" Using custom source.");
output += customSources[currentSourceFilePath];
return;
}
currentSourceFile.getImportDeclarations().forEach(flattenDeclarations);
currentSourceFile.getExportDeclarations().forEach(flattenDeclarations);
rectifyNodes(currentSourceFile);
output +=
(debug ? getSourceComment(currentSourceFile, rootPath) : "") +
currentSourceFile.print();
}
sourceFile.getExportDeclarations().forEach(exportDeclaration => {
const exportedSourceFile = exportDeclaration.getModuleSpecifierSourceFile();
if (exportedSourceFile) {
processSourceFile(exportedSourceFile);
} else {
throw createDeclarationError("Missing source file.", exportDeclaration);
}
exportDeclaration.remove();
});
rectifyNodes(sourceFile);
return (
output +
(debug ? getSourceComment(sourceFile, rootPath) : "") +
sourceFile.print()
);
}
interface InlineFilesOptions {
basePath: string;
debug?: boolean;
inline: string[];
targetSourceFile: SourceFile;
}
/** Inline files into the target source file. */
export function inlineFiles({
basePath,
debug,
inline,
targetSourceFile
}: InlineFilesOptions): void {
for (const filename of inline) {
const text = readFileSync(filename, {
encoding: "utf8"
});
targetSourceFile.addStatements(
debug
? `\n// @url ${relative(basePath, filename)}\n\n${text}`
: `\n${text}`
);
}
}
/** Load a set of files into a file system host. */
export function loadFiles(
project: Project,
filePaths: string[],
rebase?: string
): void {
const fileSystem = project.getFileSystem();
for (const filePath of filePaths) {
const fileText = readFileSync(filePath, {
encoding: "utf8"
});
fileSystem.writeFileSync(
rebase ? join(rebase, basename(filePath)) : filePath,
fileText
);
}
}
/**
* Load and write to a virtual file system all the default libs needed to
* resolve types on project.
*/
export function loadDtsFiles(
project: Project,
compilerOptions: ts.CompilerOptions
): void {
const libSourcePath = dirname(ts.getDefaultLibFilePath(compilerOptions));
// TODO (@kitsonk) Add missing libs when ts-morph supports TypeScript 3.4
loadFiles(
project,
[
"lib.es2015.collection.d.ts",
"lib.es2015.core.d.ts",
"lib.es2015.d.ts",
"lib.es2015.generator.d.ts",
"lib.es2015.iterable.d.ts",
"lib.es2015.promise.d.ts",
"lib.es2015.proxy.d.ts",
"lib.es2015.reflect.d.ts",
"lib.es2015.symbol.d.ts",
"lib.es2015.symbol.wellknown.d.ts",
"lib.es2016.array.include.d.ts",
"lib.es2016.d.ts",
"lib.es2017.d.ts",
"lib.es2017.intl.d.ts",
"lib.es2017.object.d.ts",
"lib.es2017.sharedmemory.d.ts",
"lib.es2017.string.d.ts",
"lib.es2017.typedarrays.d.ts",
"lib.es2018.d.ts",
"lib.es2018.intl.d.ts",
"lib.es2018.promise.d.ts",
"lib.es5.d.ts",
"lib.esnext.d.ts",
"lib.esnext.array.d.ts",
"lib.esnext.asynciterable.d.ts",
"lib.esnext.intl.d.ts",
"lib.esnext.symbol.d.ts"
].map(fileName => join(libSourcePath, fileName)),
"node_modules/typescript/lib/"
);
}
export interface NamespaceSourceFileOptions {
debug?: boolean;
namespace?: string;
namespaces: Set<string>;
rootPath: string;
sourceFileMap: Map<SourceFile, string>;
}
/**
* Take a source file (`.d.ts`) and convert it to a namespace, resolving any
* imports as their own namespaces.
*/
export function namespaceSourceFile(
sourceFile: SourceFile,
{
debug,
namespace,
namespaces,
rootPath,
sourceFileMap
}: NamespaceSourceFileOptions
): string {
if (sourceFileMap.has(sourceFile)) {
return "";
}
if (!namespace) {
namespace = sourceFile.getBaseNameWithoutExtension();
}
sourceFileMap.set(sourceFile, namespace);
sourceFile.forEachChild(node => {
if (TypeGuards.isAmbientableNode(node)) {
node.setHasDeclareKeyword(false);
}
});
// TODO need to properly unwrap this
const globalNamespace = sourceFile.getNamespace("global");
let globalNamespaceText = "";
if (globalNamespace) {
const structure = globalNamespace.getStructure();
if (structure.bodyText && typeof structure.bodyText === "string") {
globalNamespaceText = structure.bodyText;
} else {
throw new TypeError("Unexpected global declaration structure.");
}
}
if (globalNamespace) {
globalNamespace.remove();
}
const output = sourceFile
.getImportDeclarations()
.filter(declaration => {
const dsf = declaration.getModuleSpecifierSourceFile();
if (dsf == null) {
try {
const namespaceName = declaration
.getNamespaceImportOrThrow()
.getText();
if (!namespaces.has(namespaceName)) {
throw createDeclarationError(
"Already defined source file under different namespace.",
declaration
);
}
} catch (e) {
throw createDeclarationError(
`Unsupported import clause: ${e}`,
declaration
);
}
declaration.remove();
}
return dsf;
})
.map(declaration => {
if (
declaration.getNamedImports().length ||
!declaration.getNamespaceImport()
) {
throw createDeclarationError("Unsupported import clause.", declaration);
}
const text = namespaceSourceFile(
declaration.getModuleSpecifierSourceFileOrThrow(),
{
debug,
namespace: declaration.getNamespaceImportOrThrow().getText(),
namespaces,
rootPath,
sourceFileMap
}
);
declaration.remove();
return text;
})
.join("\n");
sourceFile
.getExportDeclarations()
.forEach(declaration => declaration.remove());
namespaces.add(namespace);
return `${output}
${globalNamespaceText || ""}
declare namespace ${namespace} {
${debug ? getSourceComment(sourceFile, rootPath) : ""}
${sourceFile.getText()}
}`;
}
/** Mirrors TypeScript's handling of paths */
export function normalizeSlashes(path: string): string {
return path.replace(/\\/g, "/");
}

569
tools/ts_library_builder/build_library.ts
View file

@ -1,569 +0,0 @@
import { writeFileSync } from "fs";
import { join } from "path";
import * as prettier from "prettier";
import {
InterfaceDeclaration,
ExpressionStatement,
NamespaceDeclarationKind,
Project,
SourceFile,
ts,
Type,
TypeGuards
} from "ts-morph";
import {
addInterfaceDeclaration,
addInterfaceProperty,
addSourceComment,
addTypeAlias,
addVariableDeclaration,
checkDiagnostics,
flattenNamespace,
getSourceComment,
inlineFiles,
loadDtsFiles,
loadFiles,
log,
logDiagnostics,
namespaceSourceFile,
normalizeSlashes,
setSilent
} from "./ast_util";
export interface BuildLibraryOptions {
/**
* The path to the root of the deno repository
*/
basePath: string;
/**
* The path to the current build path
*/
buildPath: string;
/**
* Denotes if the library should be built with debug information (comments
* that indicate the source of the types)
*/
debug?: boolean;
/**
* An array of files that should be inlined into the library
*/
inline?: string[];
/** An array of input files to be provided to the input project, relative to
* the basePath. */
inputs?: string[];
/**
* Path to globals file to be used I.E. `js/globals.ts`
*/
additionalGlobals?: string[];
/**
* List of global variables to define as let instead of the default const.
*/
declareAsLet?: string[];
/**
* The path to the output library
*/
outFile: string;
/**
* Execute in silent mode or not
*/
silent?: boolean;
}
const { ModuleKind, ModuleResolutionKind, ScriptTarget } = ts;
/**
* A preamble which is appended to the start of the library.
*/
const libPreamble = `// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
/// <reference no-default-lib="true" />
/// <reference lib="esnext" />
`;
interface FlattenOptions {
basePath: string;
customSources: { [filePath: string]: string };
filePath: string;
debug?: boolean;
declarationProject: Project;
globalInterfaceName?: string;
moduleName?: string;
namespaceName?: string;
targetSourceFile: SourceFile;
}
/** Flatten a module */
export function flatten({
basePath,
customSources,
filePath,
debug,
declarationProject,
globalInterfaceName,
moduleName,
namespaceName,
targetSourceFile
}: FlattenOptions): void {
// Flatten the source file into a single set of statements
const statements = flattenNamespace({
sourceFile: declarationProject.getSourceFileOrThrow(filePath),
rootPath: basePath,
customSources,
debug
});
// If a module name is specified create the module in the target file
if (moduleName) {
const namespace = targetSourceFile.addNamespace({
name: moduleName,
hasDeclareKeyword: true,
declarationKind: NamespaceDeclarationKind.Module
});
// Add the output of the flattening to the namespace
namespace.addStatements(statements);
}
if (namespaceName) {
const namespace = targetSourceFile.insertNamespace(0, {
name: namespaceName,
hasDeclareKeyword: true,
declarationKind: NamespaceDeclarationKind.Namespace
});
// Add the output of the flattening to the namespace
namespace.addStatements(statements);
if (globalInterfaceName) {
// Retrieve the global interface
const interfaceDeclaration = targetSourceFile.getInterfaceOrThrow(
globalInterfaceName
);
// Add the namespace to the global interface
addInterfaceProperty(
interfaceDeclaration,
namespaceName,
`typeof ${namespaceName}`
);
}
}
}
interface PrepareFileForMergeOptions {
globalVarName: string;
interfaceName: string;
targetSourceFile: SourceFile;
}
interface PrepareFileForMergeReturn {
interfaceDeclaration: InterfaceDeclaration;
}
export function prepareFileForMerge({
globalVarName,
interfaceName,
targetSourceFile
}: PrepareFileForMergeOptions): PrepareFileForMergeReturn {
// Add the global object interface
const interfaceDeclaration = targetSourceFile.addInterface({
name: interfaceName,
hasDeclareKeyword: true
});
// Declare the global variable
addVariableDeclaration(
targetSourceFile,
globalVarName,
interfaceName,
true,
true
);
// Add self reference to the global variable
addInterfaceProperty(interfaceDeclaration, globalVarName, interfaceName);
return {
interfaceDeclaration
};
}
interface MergeGlobalOptions extends PrepareFileForMergeOptions {
basePath: string;
debug?: boolean;
declarationProject: Project;
filePath: string;
ignore?: string[];
inputProject: Project;
prepareReturn: PrepareFileForMergeReturn;
declareAsLet?: string[];
}
/** Take a module and merge it into the global scope */
export function mergeGlobals({
basePath,
debug,
declarationProject,
filePath,
globalVarName,
ignore,
inputProject,
targetSourceFile,
declareAsLet,
prepareReturn: { interfaceDeclaration }
}: MergeGlobalOptions): void {
// Retrieve source file from the input project
const sourceFile = inputProject.getSourceFileOrThrow(filePath);
// we are going to create a map of variables
const globalVariables = new Map<
string,
{
type: Type;
node: ExpressionStatement;
}
>();
const globalInterfaces: InterfaceDeclaration[] = [];
// For every augmentation of the global variable in source file, we want
// to extract the type and add it to the global variable map
sourceFile.forEachChild(node => {
if (TypeGuards.isExpressionStatement(node)) {
const firstChild = node.getFirstChild();
if (!firstChild) {
return;
}
if (TypeGuards.isBinaryExpression(firstChild)) {
const leftExpression = firstChild.getLeft();
if (
TypeGuards.isPropertyAccessExpression(leftExpression) &&
leftExpression.getExpression().getText() === globalVarName
) {
const globalVarProperty = leftExpression.getName();
if (globalVarProperty !== globalVarName) {
globalVariables.set(globalVarProperty, {
type: firstChild.getType(),
node
});
}
}
}
} else if (TypeGuards.isInterfaceDeclaration(node) && node.isExported()) {
globalInterfaces.push(node);
}
});
// A set of source files that the types we are using are dependent on us
// importing
const dependentSourceFiles = new Set<SourceFile>();
// Create a global variable and add the property to the `Window` interface
// for each mutation of the `window` variable we observed in `globals.ts`
for (const [property, info] of globalVariables) {
if (!(ignore && ignore.includes(property))) {
const type = info.type.getText(info.node);
const typeSymbol = info.type.getSymbol();
if (typeSymbol) {
const valueDeclaration = typeSymbol.getValueDeclaration();
if (valueDeclaration) {
dependentSourceFiles.add(valueDeclaration.getSourceFile());
}
}
const isConst = !(declareAsLet && declareAsLet.includes(property));
addVariableDeclaration(targetSourceFile, property, type, isConst, true);
addInterfaceProperty(interfaceDeclaration, property, type);
}
}
// We need to copy over any type aliases
for (const typeAlias of sourceFile.getTypeAliases()) {
addTypeAlias(
targetSourceFile,
typeAlias.getName(),
typeAlias.getType().getText(sourceFile),
true
);
}
// We need to copy over any interfaces
for (const interfaceDeclaration of globalInterfaces) {
addInterfaceDeclaration(targetSourceFile, interfaceDeclaration);
}
// We need to ensure that we only namespace each source file once, so we
// will use this map for tracking that.
const sourceFileMap = new Map<SourceFile, string>();
// For each import declaration in source file we will want to convert the
// declaration source file into a namespace that exists within the merged
// namespace
const importDeclarations = sourceFile.getImportDeclarations();
const namespaces = new Set<string>();
for (const declaration of importDeclarations) {
const namespaceImport = declaration.getNamespaceImport();
if (namespaceImport) {
const declarationSourceFile = declaration.getModuleSpecifierSourceFile();
if (
declarationSourceFile &&
dependentSourceFiles.has(declarationSourceFile)
) {
// the source file will resolve to the original `.ts` file, but the
// information we really want is in the emitted `.d.ts` file, so we will
// resolve to that file
const dtsFilePath = declarationSourceFile
.getFilePath()
.replace(/\.ts$/, ".d.ts");
const dtsSourceFile = declarationProject.getSourceFileOrThrow(
dtsFilePath
);
targetSourceFile.addStatements(
namespaceSourceFile(dtsSourceFile, {
debug,
namespace: namespaceImport.getText(),
namespaces,
rootPath: basePath,
sourceFileMap
})
);
}
}
}
if (debug) {
addSourceComment(targetSourceFile, sourceFile, basePath);
}
}
/**
* Generate the runtime library for Deno and write it to the supplied out file
* name.
*/
export function main({
basePath,
buildPath,
inline,
inputs,
additionalGlobals,
declareAsLet,
debug,
outFile,
silent
}: BuildLibraryOptions): void {
setSilent(silent);
log("-----");
log("build_lib");
log();
log(`basePath: "${basePath}"`);
log(`buildPath: "${buildPath}"`);
if (inline && inline.length) {
log("inline:");
for (const filename of inline) {
log(` "${filename}"`);
}
}
if (inputs && inputs.length) {
log("inputs:");
for (const input of inputs) {
log(` "${input}"`);
}
}
log(`debug: ${!!debug}`);
log(`outFile: "${outFile}"`);
log();
// the inputProject will take in the TypeScript files that are internal
// to Deno to be used to generate the library
const inputProject = new Project({
compilerOptions: {
baseUrl: basePath,
declaration: true,
emitDeclarationOnly: true,
lib: ["esnext"],
module: ModuleKind.ESNext,
moduleResolution: ModuleResolutionKind.NodeJs,
paths: {
"*": ["*", `${buildPath}/*`]
},
preserveConstEnums: true,
strict: true,
stripInternal: true,
target: ScriptTarget.ESNext
}
});
// Add the input files we will need to generate the declarations, `globals`
// plus any modules that are importable in the runtime need to be added here
// plus the `lib.esnext` which is used as the base library
if (inputs) {
inputProject.addExistingSourceFiles(
inputs.map(input => join(basePath, input))
);
}
// emit the project, which will be only the declaration files
const inputEmitResult = inputProject.emitToMemory();
log("Emitted input project.");
const inputDiagnostics = inputEmitResult
.getDiagnostics()
.map(d => d.compilerObject);
logDiagnostics(inputDiagnostics);
if (inputDiagnostics.length) {
console.error("\nDiagnostics present during input project emit.\n");
process.exit(1);
}
// the declaration project will be the target for the emitted files from
// the input project, these will be used to transfer information over to
// the final library file
const declarationProject = new Project({
compilerOptions: {
baseUrl: basePath,
lib: ["esnext"],
moduleResolution: ModuleResolutionKind.NodeJs,
paths: {
"*": ["*", `${buildPath}/*`]
},
strict: true,
target: ScriptTarget.ESNext
},
useVirtualFileSystem: true
});
// we don't want to add to the declaration project any of the original
// `.ts` source files, so we need to filter those out
const jsPath = normalizeSlashes(`${basePath}/js`);
const inputProjectFiles = inputProject
.getSourceFiles()
.map(sourceFile => sourceFile.getFilePath())
.filter(filePath => !filePath.startsWith(jsPath));
loadFiles(declarationProject, inputProjectFiles);
// now we add the emitted declaration files from the input project
for (const { filePath, text } of inputEmitResult.getFiles()) {
declarationProject.createSourceFile(filePath, text);
}
// the outputProject will contain the final library file we are looking to
// build
const outputProjectCompilerOptions: ts.CompilerOptions = {
baseUrl: buildPath,
lib: ["esnext"],
moduleResolution: ModuleResolutionKind.NodeJs,
strict: true,
target: ScriptTarget.ESNext
};
const outputProject = new Project({
compilerOptions: outputProjectCompilerOptions,
useVirtualFileSystem: true
});
// There are files we need to load into memory, so that the project "compiles"
loadDtsFiles(outputProject, outputProjectCompilerOptions);
// libDts is the final output file we are looking to build and we are not
// actually creating it, only in memory at this stage.
const libDTs = outputProject.createSourceFile(outFile);
// Deal with `js/deno.ts`
// Generate a object hash of substitutions of modules to use when flattening
const customSources = {};
const prepareForMergeOpts: PrepareFileForMergeOptions = {
globalVarName: "window",
interfaceName: "Window",
targetSourceFile: libDTs
};
const prepareReturn = prepareFileForMerge(prepareForMergeOpts);
mergeGlobals({
basePath,
debug,
declarationProject,
filePath: `${basePath}/js/globals.ts`,
inputProject,
ignore: ["Deno"],
declareAsLet,
...prepareForMergeOpts,
prepareReturn
});
log(`Merged "globals" into global scope.`);
if (additionalGlobals) {
for (const additionalGlobal of additionalGlobals) {
mergeGlobals({
basePath,
debug,
declarationProject,
filePath: `${basePath}/${additionalGlobal}`,
inputProject,
ignore: ["Deno"],
declareAsLet,
...prepareForMergeOpts,
prepareReturn
});
}
log(`Added additional "globals" into global scope.`);
}
flatten({
basePath,
customSources,
debug,
declarationProject,
filePath: `${basePath}/js/deno.d.ts`,
globalInterfaceName: "Window",
namespaceName: "Deno",
targetSourceFile: libDTs
});
log(`Created module "deno" and namespace Deno.`);
// Inline any files that were passed in, to be used to add additional libs
// which are not part of TypeScript.
if (inline && inline.length) {
inlineFiles({
basePath,
debug,
inline,
targetSourceFile: libDTs
});
}
// Add the preamble
libDTs.insertStatements(0, libPreamble);
// Check diagnostics
checkDiagnostics(outputProject);
// Output the final library file
libDTs.saveSync();
const libDTsText = prettier.format(
outputProject.getFileSystem().readFileSync(outFile, "utf8"),
{ parser: "typescript" }
);
if (!silent) {
console.log(`Outputting library to: "${outFile}"`);
console.log(` Length: ${libDTsText.length}`);
}
writeFileSync(outFile, libDTsText, { encoding: "utf8" });
if (!silent) {
console.log("-----");
console.log();
}
}

52
tools/ts_library_builder/main.ts
View file

@ -1,52 +0,0 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import * as path from "path";
import { main as buildRuntimeLib } from "./build_library";
// this is very simplistic argument parsing, just enough to integrate into
// the build scripts, versus being very robust
let basePath = process.cwd();
let buildPath = path.join(basePath, "target", "debug");
let outFile = path.join(buildPath, "gen", "lib", "lib.d.ts");
let inline: string[] = [];
let debug = false;
let silent = false;
process.argv.forEach((arg, i, argv) => {
switch (arg) {
case "--basePath":
basePath = path.resolve(argv[i + 1]);
break;
case "--buildPath":
buildPath = path.resolve(argv[i + 1]);
break;
case "--inline":
inline = argv[i + 1].split(",").map(filename => {
return path.resolve(filename);
});
break;
case "--outFile":
outFile = path.resolve(argv[i + 1]);
break;
case "--debug":
debug = true;
break;
case "--silent":
silent = true;
break;
}
});
buildRuntimeLib({
basePath,
buildPath,
debug,
inline,
inputs: [
"node_modules/typescript/lib/lib.esnext.d.ts",
"js/deno.ts",
"js/globals.ts"
],
declareAsLet: ["onmessage"],
outFile,
silent
});

240
tools/ts_library_builder/test.ts
View file

@ -1,240 +0,0 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
// Run this manually with:
//
// ./node_modules/.bin/ts-node --project tools/ts_library_builder/tsconfig.json tools/ts_library_builder/test.ts
import * as assert from "assert";
import { Project, ts } from "ts-morph";
import { flatten, mergeGlobals, prepareFileForMerge } from "./build_library";
import { inlineFiles, loadDtsFiles } from "./ast_util";
const { ModuleKind, ModuleResolutionKind, ScriptTarget } = ts;
/** setups and returns the fixtures for testing */
// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
function setupFixtures() {
const basePath = process.cwd();
const buildPath = `${basePath}/tools/ts_library_builder/testdata`;
const outputFile = `${buildPath}/lib.output.d.ts`;
const inputProject = new Project({
compilerOptions: {
baseUrl: basePath,
declaration: true,
emitDeclarationOnly: true,
module: ModuleKind.ESNext,
moduleResolution: ModuleResolutionKind.NodeJs,
strict: true,
stripInternal: true,
target: ScriptTarget.ESNext
}
});
inputProject.addExistingSourceFiles([
`${buildPath}/globals.ts`,
`${buildPath}/api.ts`
]);
const declarationProject = new Project({
compilerOptions: {},
useVirtualFileSystem: true
});
loadDtsFiles(declarationProject, {});
for (const { filePath, text } of inputProject.emitToMemory().getFiles()) {
declarationProject.createSourceFile(filePath, text);
}
const outputProject = new Project({
compilerOptions: {},
useVirtualFileSystem: true
});
loadDtsFiles(outputProject, {});
const outputSourceFile = outputProject.createSourceFile(outputFile);
const debug = true;
return {
basePath,
buildPath,
inputProject,
outputFile,
declarationProject,
outputProject,
outputSourceFile,
debug
};
}
function buildLibraryFlatten(): void {
const {
basePath,
buildPath,
debug,
declarationProject,
outputSourceFile: targetSourceFile
} = setupFixtures();
flatten({
basePath,
customSources: {},
debug,
declarationProject,
filePath: `${buildPath}/api.d.ts`,
moduleName: `"api"`,
namespaceName: "Api",
targetSourceFile
});
assert(targetSourceFile.getNamespace(`"api"`) != null);
assert(targetSourceFile.getNamespace("Api") != null);
assert.equal(targetSourceFile.getNamespaces().length, 2);
const moduleApi = targetSourceFile.getNamespaceOrThrow(`"api"`);
const functions = moduleApi.getFunctions();
assert.equal(functions[0].getName(), "foo");
assert.equal(
functions[0]
.getJsDocs()
.map(jsdoc => jsdoc.getInnerText())
.join("\n"),
"jsdoc for foo"
);
assert.equal(functions[1].getName(), "bar");
assert.equal(
functions[1]
.getJsDocs()
.map(jsdoc => jsdoc.getInnerText())
.join("\n"),
""
);
assert.equal(functions.length, 2);
const classes = moduleApi.getClasses();
assert.equal(classes[0].getName(), "Foo");
assert.equal(classes.length, 1);
const variableDeclarations = moduleApi.getVariableDeclarations();
assert.equal(variableDeclarations[0].getName(), "arr");
assert.equal(variableDeclarations.length, 1);
const namespaceApi = targetSourceFile.getNamespaceOrThrow(`"api"`);
const functionsNs = namespaceApi.getFunctions();
assert.equal(functionsNs[0].getName(), "foo");
assert.equal(
functionsNs[0]
.getJsDocs()
.map(jsdoc => jsdoc.getInnerText())
.join("\n"),
"jsdoc for foo"
);
assert.equal(functionsNs[1].getName(), "bar");
assert.equal(
functionsNs[1]
.getJsDocs()
.map(jsdoc => jsdoc.getInnerText())
.join("\n"),
""
);
assert.equal(functionsNs.length, 2);
const classesNs = namespaceApi.getClasses();
assert.equal(classesNs[0].getName(), "Foo");
assert.equal(classesNs.length, 1);
const variableDeclarationsNs = namespaceApi.getVariableDeclarations();
assert.equal(variableDeclarationsNs[0].getName(), "arr");
assert.equal(variableDeclarationsNs.length, 1);
}
function buildLibraryMerge(): void {
const {
basePath,
buildPath,
declarationProject,
debug,
inputProject,
outputSourceFile: targetSourceFile
} = setupFixtures();
const prepareForMergeOpts = {
globalVarName: "foobarbaz",
interfaceName: "FooBar",
targetSourceFile
};
const prepareReturn = prepareFileForMerge(prepareForMergeOpts);
mergeGlobals({
basePath,
declarationProject,
debug,
filePath: `${buildPath}/globals.ts`,
inputProject,
...prepareForMergeOpts,
prepareReturn
});
assert(targetSourceFile.getNamespace("moduleC") != null);
assert(targetSourceFile.getNamespace("moduleD") != null);
assert(targetSourceFile.getNamespace("moduleE") != null);
assert(targetSourceFile.getNamespace("moduleF") != null);
assert.equal(targetSourceFile.getNamespaces().length, 4);
assert(targetSourceFile.getInterface("FooBar") != null);
assert.equal(targetSourceFile.getInterfaces().length, 2);
const variableDeclarations = targetSourceFile.getVariableDeclarations();
assert.equal(variableDeclarations[0].getType().getText(), `FooBar`);
assert.equal(variableDeclarations[1].getType().getText(), `moduleC.Bar`);
assert.equal(
variableDeclarations[2].getType().getText(),
`typeof moduleC.qat`
);
assert.equal(
variableDeclarations[3].getType().getText(),
`typeof moduleE.process`
);
assert.equal(
variableDeclarations[4].getType().getText(),
`typeof moduleD.reprocess`
);
assert.equal(
variableDeclarations[5].getType().getText(),
`typeof moduleC.Bar`
);
assert.equal(variableDeclarations.length, 6);
const typeAliases = targetSourceFile.getTypeAliases();
assert.equal(typeAliases[0].getName(), "Bar");
assert.equal(typeAliases[0].getType().getText(), "moduleC.Bar");
assert.equal(typeAliases.length, 1);
const exportedInterface = targetSourceFile.getInterfaceOrThrow("FizzBuzz");
const interfaceProperties = exportedInterface.getStructure().properties;
assert(interfaceProperties != null);
assert.equal(interfaceProperties!.length, 2);
assert.equal(interfaceProperties![0].name, "foo");
assert.equal(interfaceProperties![0].type, "string");
assert.equal(interfaceProperties![1].name, "bar");
assert.equal(interfaceProperties![1].type, "number");
}
function testInlineFiles(): void {
const {
basePath,
buildPath,
debug,
outputSourceFile: targetSourceFile
} = setupFixtures();
inlineFiles({
basePath,
debug,
inline: [`${buildPath}/lib.extra.d.ts`],
targetSourceFile
});
assert(targetSourceFile.getNamespace("Qat") != null);
const qatNamespace = targetSourceFile.getNamespaceOrThrow("Qat");
assert(qatNamespace.getClass("Foo") != null);
}
// TODO author unit tests for `ast_util.ts`
function main(): void {
console.log("ts_library_builder buildLibraryFlatten");
buildLibraryFlatten();
console.log("ts_library_builder buildLibraryMerge");
buildLibraryMerge();
console.log("ts_library_builder testInlineFiles");
testInlineFiles();
console.log("ts_library_builder ok");
}
main();

4
tools/ts_library_builder/testdata/api.ts vendored
View file

@ -1,4 +0,0 @@
export { foo, bar } from "./moduleA";
export { Foo } from "./moduleB";
/** jsdoc for arr */
export const arr: string[] = [];

15
tools/ts_library_builder/testdata/globals.ts vendored
View file

@ -1,15 +0,0 @@
import * as moduleC from "./moduleC";
import * as moduleD from "./moduleD";
import * as moduleE from "./moduleE";
const foobarbaz: any = {};
foobarbaz.bar = new moduleC.Bar();
foobarbaz.qat = moduleC.qat;
foobarbaz.process = moduleE.process;
foobarbaz.reprocess = moduleD.reprocess;
foobarbaz.Bar = moduleC.Bar;
export type Bar = moduleC.Bar;
export interface FizzBuzz {
foo: string;
bar: number;
}

7
tools/ts_library_builder/testdata/lib.extra.d.ts vendored
View file

@ -1,7 +0,0 @@
// comment
declare namespace Qat {
class Foo {
bar: string;
}
}

9
tools/ts_library_builder/testdata/moduleA.ts vendored
View file

@ -1,9 +0,0 @@
/** jsdoc for foo */
export function foo(a: string, b: string) {
console.log(a, b);
}
// no jsdoc for bar
export async function bar(promise: Promise<void>): Promise<void> {
return promise.then(() => {});
}

9
tools/ts_library_builder/testdata/moduleB.ts vendored
View file

@ -1,9 +0,0 @@
/** jsdoc about Foo */
export class Foo {
private _foo = "foo";
/** jsdoc about Foo.log() */
log() {
console.log(this._foo);
return this._foo;
}
}

18
tools/ts_library_builder/testdata/moduleC.ts vendored
View file

@ -1,18 +0,0 @@
/** jsdoc for Bar */
export class Bar {
private _bar: string;
/** jsdoc for Bar.log() */
log() {
console.log(this._bar);
return this.log;
}
}
/**
* jsdoc for qat
* @param a jsdoc for qat(a)
* @param b jsdoc for qat(b)
*/
export function qat(a: string, b: string) {
return a + b;
}

5
tools/ts_library_builder/testdata/moduleD.ts vendored
View file

@ -1,5 +0,0 @@
import * as moduleF from "./moduleF";
export function reprocess(value: typeof moduleF.key) {
console.log(value);
}

5
tools/ts_library_builder/testdata/moduleE.ts vendored
View file

@ -1,5 +0,0 @@
import * as moduleF from "./moduleF";
export function process(value: typeof moduleF.key) {
console.log(value);
}

1
tools/ts_library_builder/testdata/moduleF.ts vendored
View file

@ -1 +0,0 @@
export const key = "value";

9
tools/ts_library_builder/tsconfig.json
View file

@ -1,9 +0,0 @@
{
"compilerOptions": {
"lib": ["esnext"],
"moduleResolution": "node",
"strict": true,
"target": "esnext"
},
"files": ["./build_library.ts"]
}

1
tsconfig.json
View file

@ -21,6 +21,7 @@
},
"files": [
"js/lib.web_assembly.d.ts",
"js/lib.deno_runtime.d.ts",
"core/core.d.ts",
"core/libdeno/libdeno.d.ts",
"js/main.ts",