From 6c7d337960b3745a7b614a18150862279ef1c942 Mon Sep 17 00:00:00 2001 From: Kitson Kelly Date: Fri, 23 Aug 2019 02:05:01 +1000 Subject: [PATCH] Support .d.ts files (#2746) Fixes #1432 --- cli/BUILD.gn | 3 +- js/compiler.ts | 75 ++++++++++++++++++------- js/type_directives.ts | 87 +++++++++++++++++++++++++++++ tests/error_type_definitions.test | 4 ++ tests/error_type_definitions.ts | 5 ++ tests/error_type_definitions.ts.out | 4 ++ tests/type_definitions.test | 2 + tests/type_definitions.ts | 4 ++ tests/type_definitions.ts.out | 1 + tests/type_definitions/bar.d.ts | 7 +++ tests/type_definitions/foo.d.ts | 2 + tests/type_definitions/foo.js | 1 + website/manual.md | 42 ++++++++++++++ 13 files changed, 216 insertions(+), 21 deletions(-) create mode 100644 js/type_directives.ts create mode 100644 tests/error_type_definitions.test create mode 100644 tests/error_type_definitions.ts create mode 100644 tests/error_type_definitions.ts.out create mode 100644 tests/type_definitions.test create mode 100644 tests/type_definitions.ts create mode 100644 tests/type_definitions.ts.out create mode 100644 tests/type_definitions/bar.d.ts create mode 100644 tests/type_definitions/foo.d.ts create mode 100644 tests/type_definitions/foo.js diff --git a/cli/BUILD.gn b/cli/BUILD.gn index 3a5912471c..c7a80907ed 100644 --- a/cli/BUILD.gn +++ b/cli/BUILD.gn @@ -111,6 +111,7 @@ ts_sources = [ "../js/mock_builtin.js", "../js/net.ts", "../js/os.ts", + "../js/performance.ts", "../js/permissions.ts", "../js/plugins.d.ts", "../js/process.ts", @@ -127,6 +128,7 @@ ts_sources = [ "../js/text_encoding.ts", "../js/timers.ts", "../js/truncate.ts", + "../js/type_directives.ts", "../js/types.ts", "../js/url.ts", "../js/url_search_params.ts", @@ -135,7 +137,6 @@ ts_sources = [ "../js/window.ts", "../js/workers.ts", "../js/write_file.ts", - "../js/performance.ts", "../js/version.ts", "../js/xeval.ts", "../tsconfig.json", diff --git a/js/compiler.ts b/js/compiler.ts index fd6f0ef58a..7519c51153 100644 --- a/js/compiler.ts +++ b/js/compiler.ts @@ -10,6 +10,7 @@ import { cwd } from "./dir"; import { sendSync, msg, flatbuffers } from "./dispatch_flatbuffers"; import * as os from "./os"; import { TextDecoder, TextEncoder } from "./text_encoding"; +import { getMappedModuleName, parseTypeDirectives } from "./type_directives"; import { assert, notImplemented } from "./util"; import * as util from "./util"; import { window } from "./window"; @@ -110,6 +111,7 @@ interface SourceFile { filename: string | undefined; mediaType: msg.MediaType; sourceCode: string | undefined; + typeDirectives?: Record; } interface EmitResult { @@ -119,7 +121,7 @@ interface EmitResult { /** Ops to Rust to resolve and fetch a modules meta data. */ function fetchSourceFile(specifier: string, referrer: string): SourceFile { - util.log("compiler.fetchSourceFile", { specifier, referrer }); + util.log("fetchSourceFile", { specifier, referrer }); // Send FetchSourceFile message const builder = flatbuffers.createBuilder(); const specifier_ = builder.createString(specifier); @@ -146,7 +148,8 @@ function fetchSourceFile(specifier: string, referrer: string): SourceFile { moduleName: fetchSourceFileRes.moduleName() || undefined, filename: fetchSourceFileRes.filename() || undefined, mediaType: fetchSourceFileRes.mediaType(), - sourceCode + sourceCode, + typeDirectives: parseTypeDirectives(sourceCode) }; } @@ -168,7 +171,7 @@ function humanFileSize(bytes: number): string { /** Ops to rest for caching source map and compiled js */ function cache(extension: string, moduleId: string, contents: string): void { - util.log("compiler.cache", moduleId); + util.log("cache", extension, moduleId); const builder = flatbuffers.createBuilder(); const extension_ = builder.createString(extension); const moduleId_ = builder.createString(moduleId); @@ -189,7 +192,7 @@ const encoder = new TextEncoder(); function emitBundle(fileName: string, data: string): void { // For internal purposes, when trying to emit to `$deno$` just no-op if (fileName.startsWith("$deno$")) { - console.warn("skipping compiler.emitBundle", fileName); + console.warn("skipping emitBundle", fileName); return; } const encodedData = encoder.encode(data); @@ -217,7 +220,7 @@ function getExtension( } class Host implements ts.CompilerHost { - extensionCache: Record = {}; + private _extensionCache: Record = {}; private readonly _options: ts.CompilerOptions = { allowJs: true, @@ -232,23 +235,37 @@ class Host implements ts.CompilerHost { target: ts.ScriptTarget.ESNext }; + private _sourceFileCache: Record = {}; + private _resolveModule(specifier: string, referrer: string): SourceFile { + util.log("host._resolveModule", { specifier, referrer }); // Handle built-in assets specially. if (specifier.startsWith(ASSETS)) { const moduleName = specifier.split("/").pop()!; + if (moduleName in this._sourceFileCache) { + return this._sourceFileCache[moduleName]; + } const assetName = moduleName.includes(".") ? moduleName : `${moduleName}.d.ts`; assert(assetName in assetSourceCode, `No such asset "${assetName}"`); const sourceCode = assetSourceCode[assetName]; - return { + const sourceFile = { moduleName, filename: specifier, mediaType: msg.MediaType.TypeScript, sourceCode }; + this._sourceFileCache[moduleName] = sourceFile; + return sourceFile; } - return fetchSourceFile(specifier, referrer); + const sourceFile = fetchSourceFile(specifier, referrer); + assert(sourceFile.moduleName != null); + const { moduleName } = sourceFile; + if (!(moduleName! in this._sourceFileCache)) { + this._sourceFileCache[moduleName!] = sourceFile; + } + return sourceFile; } /* Deno specific APIs */ @@ -277,7 +294,7 @@ class Host implements ts.CompilerHost { * options which were ignored, or `undefined`. */ configure(path: string, configurationText: string): ConfigureResponse { - util.log("compile.configure", path); + util.log("host.configure", path); const { config, error } = ts.parseConfigFileTextToJson( path, configurationText @@ -308,7 +325,10 @@ class Host implements ts.CompilerHost { /* TypeScript CompilerHost APIs */ - fileExists(_fileName: string): boolean { + fileExists(fileName: string): boolean { + if (fileName.endsWith("package.json")) { + throw new TypeError("Automatic type resolution not supported"); + } return notImplemented(); } @@ -342,13 +362,17 @@ class Host implements ts.CompilerHost { ): ts.SourceFile | undefined { assert(!shouldCreateNewSourceFile); util.log("getSourceFile", fileName); - const SourceFile = this._resolveModule(fileName, "."); - if (!SourceFile || !SourceFile.sourceCode) { + const sourceFile = + fileName in this._sourceFileCache + ? this._sourceFileCache[fileName] + : this._resolveModule(fileName, "."); + assert(sourceFile != null); + if (!sourceFile.sourceCode) { return undefined; } return ts.createSourceFile( fileName, - SourceFile.sourceCode, + sourceFile.sourceCode, languageVersion ); } @@ -362,26 +386,37 @@ class Host implements ts.CompilerHost { containingFile: string ): Array { util.log("resolveModuleNames()", { moduleNames, containingFile }); + const typeDirectives: Record | undefined = + containingFile in this._sourceFileCache + ? this._sourceFileCache[containingFile].typeDirectives + : undefined; return moduleNames.map( (moduleName): ts.ResolvedModuleFull | undefined => { - const SourceFile = this._resolveModule(moduleName, containingFile); - if (SourceFile.moduleName) { - const resolvedFileName = SourceFile.moduleName; + const mappedModuleName = getMappedModuleName( + moduleName, + containingFile, + typeDirectives + ); + const sourceFile = this._resolveModule( + mappedModuleName, + containingFile + ); + if (sourceFile.moduleName) { + const resolvedFileName = sourceFile.moduleName; // This flags to the compiler to not go looking to transpile functional // code, anything that is in `/$asset$/` is just library code const isExternalLibraryImport = moduleName.startsWith(ASSETS); const extension = getExtension( resolvedFileName, - SourceFile.mediaType + sourceFile.mediaType ); - this.extensionCache[resolvedFileName] = extension; + this._extensionCache[resolvedFileName] = extension; - const r = { + return { resolvedFileName, isExternalLibraryImport, extension }; - return r; } else { return undefined; } @@ -407,7 +442,7 @@ class Host implements ts.CompilerHost { } else { assert(sourceFiles != null && sourceFiles.length == 1); const sourceFileName = sourceFiles![0].fileName; - const maybeExtension = this.extensionCache[sourceFileName]; + const maybeExtension = this._extensionCache[sourceFileName]; if (maybeExtension) { // NOTE: If it's a `.json` file we don't want to write it to disk. diff --git a/js/type_directives.ts b/js/type_directives.ts new file mode 100644 index 0000000000..3e903a80b5 --- /dev/null +++ b/js/type_directives.ts @@ -0,0 +1,87 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +interface DirectiveInfo { + path: string; + start: number; + end: number; +} + +/** Remap the module name based on any supplied type directives passed. */ +export function getMappedModuleName( + moduleName: string, + containingFile: string, + typeDirectives?: Record +): string { + if (containingFile.endsWith(".d.ts") && !moduleName.endsWith(".d.ts")) { + moduleName = `${moduleName}.d.ts`; + } + if (!typeDirectives) { + return moduleName; + } + if (moduleName in typeDirectives) { + return typeDirectives[moduleName]; + } + return moduleName; +} + +/** Matches directives that look something like this and parses out the value + * of the directive: + * + * // @deno-types="./foo.d.ts" + * + * [See Diagram](http://bit.ly/31nZPCF) + */ +const typeDirectiveRegEx = /@deno-types\s*=\s*(["'])((?:(?=(\\?))\3.)*?)\1/gi; + +/** Matches `import` or `export from` statements and parses out the value of the + * module specifier in the second capture group: + * + * import * as foo from "./foo.js" + * export { a, b, c } from "./bar.js" + * + * [See Diagram](http://bit.ly/2GSkJlF) + */ +const importExportRegEx = /(?:import|export)\s+[\s\S]*?from\s+(["'])((?:(?=(\\?))\3.)*?)\1/; + +/** Parses out any Deno type directives that are part of the source code, or + * returns `undefined` if there are not any. + */ +export function parseTypeDirectives( + sourceCode: string | undefined +): Record | undefined { + if (!sourceCode) { + return; + } + + // collect all the directives in the file and their start and end positions + const directives: DirectiveInfo[] = []; + let maybeMatch: RegExpExecArray | null = null; + while ((maybeMatch = typeDirectiveRegEx.exec(sourceCode))) { + const [matchString, , path] = maybeMatch; + const { index: start } = maybeMatch; + directives.push({ + path, + start, + end: start + matchString.length + }); + } + if (!directives.length) { + return; + } + + // work from the last directive backwards for the next `import`/`export` + // statement + directives.reverse(); + const directiveRecords: Record = {}; + for (const { path, start, end } of directives) { + const searchString = sourceCode.substring(end); + const maybeMatch = importExportRegEx.exec(searchString); + if (maybeMatch) { + const [, , fromPath] = maybeMatch; + directiveRecords[fromPath] = path; + } + sourceCode = sourceCode.substring(0, start); + } + + return directiveRecords; +} diff --git a/tests/error_type_definitions.test b/tests/error_type_definitions.test new file mode 100644 index 0000000000..367edacc5e --- /dev/null +++ b/tests/error_type_definitions.test @@ -0,0 +1,4 @@ +args: run --reload tests/error_type_definitions.ts +check_stderr: true +exit_code: 1 +output: tests/error_type_definitions.ts.out diff --git a/tests/error_type_definitions.ts b/tests/error_type_definitions.ts new file mode 100644 index 0000000000..ceb11787e5 --- /dev/null +++ b/tests/error_type_definitions.ts @@ -0,0 +1,5 @@ +// @deno-types="./type_definitions/bar.d.ts" +import { Bar } from "./type_definitions/bar.js"; + +const bar = new Bar(); +console.log(bar); diff --git a/tests/error_type_definitions.ts.out b/tests/error_type_definitions.ts.out new file mode 100644 index 0000000000..4b29b5b83a --- /dev/null +++ b/tests/error_type_definitions.ts.out @@ -0,0 +1,4 @@ +[WILDCARD]error: Uncaught TypeError: Automatic type resolution not supported +[WILDCARD]js/compiler.ts:[WILDCARD] + at fileExists (js/compiler.ts:[WILDCARD]) +[WILDCARD] \ No newline at end of file diff --git a/tests/type_definitions.test b/tests/type_definitions.test new file mode 100644 index 0000000000..7ce8d896d6 --- /dev/null +++ b/tests/type_definitions.test @@ -0,0 +1,2 @@ +args: run --reload tests/type_definitions.ts +output: tests/type_definitions.ts.out diff --git a/tests/type_definitions.ts b/tests/type_definitions.ts new file mode 100644 index 0000000000..3725011665 --- /dev/null +++ b/tests/type_definitions.ts @@ -0,0 +1,4 @@ +// @deno-types="./type_definitions/foo.d.ts" +import { foo } from "./type_definitions/foo.js"; + +console.log(foo); diff --git a/tests/type_definitions.ts.out b/tests/type_definitions.ts.out new file mode 100644 index 0000000000..291e85a7be --- /dev/null +++ b/tests/type_definitions.ts.out @@ -0,0 +1 @@ +[WILDCARD]foo diff --git a/tests/type_definitions/bar.d.ts b/tests/type_definitions/bar.d.ts new file mode 100644 index 0000000000..d43335dbb5 --- /dev/null +++ b/tests/type_definitions/bar.d.ts @@ -0,0 +1,7 @@ +/// + +declare namespace bar { + export class Bar { + baz: string; + } +} diff --git a/tests/type_definitions/foo.d.ts b/tests/type_definitions/foo.d.ts new file mode 100644 index 0000000000..ce39201e1b --- /dev/null +++ b/tests/type_definitions/foo.d.ts @@ -0,0 +1,2 @@ +/** An exported value. */ +export const foo: string; diff --git a/tests/type_definitions/foo.js b/tests/type_definitions/foo.js new file mode 100644 index 0000000000..61d366eb25 --- /dev/null +++ b/tests/type_definitions/foo.js @@ -0,0 +1 @@ +export const foo = "foo"; diff --git a/website/manual.md b/website/manual.md index ec007783c8..6cffa7f738 100644 --- a/website/manual.md +++ b/website/manual.md @@ -580,6 +580,48 @@ import { test, assertEquals } from "./deps.ts"; This design circumvents a plethora of complexity spawned by package management software, centralized code repositories, and superfluous file formats. +### Using external type definitions + +Deno supports both JavaScript and TypeScript as first class languages at +runtime. This means it requires fully qualified module names, including the +extension (or a server providing the correct media type). In addition, Deno has +no "magical" module resolution. + +The out of the box TypeScript compiler though relies on both extension-less +modules and the Node.js module resolution logic to apply types to JavaScript +modules. + +In order to bridge this gap, Deno supports compiler hints that inform Deno the +location of `.d.ts` files and the JavaScript code they relate to. A compiler +hint looks like this: + +```ts +// @deno-types="./foo.d.ts" +import * as foo from "./foo.js"; +``` + +Where the hint effects the next `import` statement (or `export ... from` +statement) where the value of the `@deno-types` will be substituted at compile +time instead of the specified module. Like in the above example, the Deno +compiler will load `./foo.d.ts` instead of `./foo.js`. Deno will still load +`./foo.js` when it runs the program. + +**Not all type definitions are supported.** + +Deno will use the compiler hint to load the indicated `.d.ts` files, but some +`.d.ts` files contain unsupported features. Specifically, some `.d.ts` files +expect to be able to load or reference type definitions from other packages +using the module resolution logic. For example a type reference directive to +include `node`, expecting to resolve to some path like +`./node_modules/@types/node/index.d.ts`. Since this depends on non-relative +"magical" resolution, Deno cannot resolve this. + +**Why not use the triple-slash type reference?** + +The TypeScript compiler supports triple-slash directives, including a type +reference directive. If Deno used this, it would interfere with the behavior of +the TypeScript compiler. + ### Testing if current file is the main program To test if the current script has been executed as the main input to the program