diff --git a/Makefile b/Makefile index 0fa0208162..fca1c2f8f7 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,16 @@ TS_FILES = \ - amd.ts \ main.ts \ + msg.pb.d.ts \ msg.pb.js \ - compiler.ts \ - msg.pb.d.ts \ - os.ts \ - util.ts + os.ts \ + runtime.ts \ + util.ts deno: assets.go msg.pb.go main.go go build -o deno assets.go: dist/main.js + cp node_modules/typescript/lib/lib.d.ts dist/ go-bindata -pkg main -o assets.go dist/ msg.pb.go: msg.proto diff --git a/amd.ts b/amd.ts deleted file mode 100644 index 29cee123b5..0000000000 --- a/amd.ts +++ /dev/null @@ -1,80 +0,0 @@ -import * as path from "path"; -import { assert, log } from "./util"; - -namespace ModuleExportsCache { - const cache = new Map(); - export function set(fileName: string, moduleExports: object) { - fileName = normalizeModuleName(fileName); - assert( - fileName.startsWith("/"), - `Normalized modules should start with /\n${fileName}` - ); - log("ModuleExportsCache set", fileName); - cache.set(fileName, moduleExports); - } - export function get(fileName: string): object { - fileName = normalizeModuleName(fileName); - log("ModuleExportsCache get", fileName); - let moduleExports = cache.get(fileName); - if (moduleExports == null) { - moduleExports = {}; - set(fileName, moduleExports); - } - return moduleExports; - } -} - -function normalizeModuleName(fileName: string): string { - // Remove the extension. - return fileName.replace(/\.\w+$/, ""); -} - -function normalizeRelativeModuleName(contextFn: string, depFn: string): string { - if (depFn.startsWith("/")) { - return depFn; - } else { - return path.resolve(path.dirname(contextFn), depFn); - } -} - -const executeQueue: Array<() => void> = []; - -export function executeQueueDrain(): void { - let fn; - while ((fn = executeQueue.shift())) { - fn(); - } -} - -// tslint:disable-next-line:no-any -type AmdFactory = (...args: any[]) => undefined | object; -type AmdDefine = (deps: string[], factory: AmdFactory) => void; - -export function makeDefine(fileName: string): AmdDefine { - const localDefine = (deps: string[], factory: AmdFactory): void => { - const localRequire = (x: string) => { - log("localRequire", x); - }; - const localExports = ModuleExportsCache.get(fileName); - log("localDefine", fileName, deps, localExports); - const args = deps.map(dep => { - if (dep === "require") { - return localRequire; - } else if (dep === "exports") { - return localExports; - } else { - dep = normalizeRelativeModuleName(fileName, dep); - return ModuleExportsCache.get(dep); - } - }); - executeQueue.push(() => { - log("execute", fileName); - const r = factory(...args); - if (r != null) { - ModuleExportsCache.set(fileName, r); - throw Error("x"); - } - }); - }; - return localDefine; -} diff --git a/compiler.ts b/compiler.ts deleted file mode 100644 index 728486a03b..0000000000 --- a/compiler.ts +++ /dev/null @@ -1,221 +0,0 @@ -import * as ts from "typescript"; -import { log, assert, globalEval, _global } from "./util"; -import * as os from "./os"; -import * as path from "path"; -import * as amd from "./amd"; - -/* -export function makeCacheDir(): string { - let cacheDir = path.join(env.HOME, ".deno/cache") - os.mkdirp(cacheDir); - return cacheDir -} -*/ - -export function compile(cwd: string, inputFn: string): void { - const options: ts.CompilerOptions = { - allowJs: true, - module: ts.ModuleKind.AMD, - outDir: "/" // Will be placed in ~/.deno/compile - }; - const host = new CompilerHost(); - - const inputExt = path.extname(inputFn); - if (!EXTENSIONS.includes(inputExt)) { - console.error(`Bad file name extension for input "${inputFn}"`); - os.exit(1); - } - - const program = ts.createProgram([inputFn], options, host); - //let sourceFiles = program.getSourceFiles(); - //log("rootFileNames", program.getRootFileNames()); - - // Print compilation errors, if any. - const diagnostics = getDiagnostics(program); - if (diagnostics.length > 0) { - const errorMessages = diagnostics.map(d => formatDiagnostic(d, cwd)); - for (const msg of errorMessages) { - console.error(msg); - } - os.exit(2); - } - - const emitResult = program.emit(); - assert(!emitResult.emitSkipped); - log("emitResult", emitResult); - - amd.executeQueueDrain(); -} - -/** - * Format a diagnostic object into a string. - * Adapted from TS-Node https://github.com/TypeStrong/ts-node - * which uses the same MIT license as this file but is - * Copyright (c) 2014 Blake Embrey (hello@blakeembrey.com) - */ -export function formatDiagnostic( - diagnostic: ts.Diagnostic, - cwd: string, - lineOffset = 0 -): string { - const messageText = ts.flattenDiagnosticMessageText( - diagnostic.messageText, - "\n" - ); - const { code } = diagnostic; - if (diagnostic.file) { - const fn = path.relative(cwd, diagnostic.file.fileName); - if (diagnostic.start) { - const { line, character } = diagnostic.file.getLineAndCharacterOfPosition( - diagnostic.start - ); - const r = Number(line) + 1 + lineOffset; - const c = Number(character) + 1; - return `${fn} (${r},${c}): ${messageText} (${code})`; - } - return `${fn}: ${messageText} (${code})`; - } - return `${messageText} (${code})`; -} - -function getDiagnostics(program: ts.Program): ReadonlyArray { - return program - .getOptionsDiagnostics() - .concat( - program.getGlobalDiagnostics(), - program.getSyntacticDiagnostics(), - program.getSemanticDiagnostics(), - program.getDeclarationDiagnostics() - ); -} - -const EXTENSIONS = [".ts", ".js"]; - -export class CompilerHost { - constructor() {} - - getSourceFile( - fileName: string, - languageVersion: ts.ScriptTarget, - onError?: (message: string) => void, - shouldCreateNewSourceFile?: boolean - ): ts.SourceFile | undefined { - let sourceText: string; - if (fileName === "lib.d.ts") { - // TODO This should be compiled into the bindata. - sourceText = os.readFileSync("node_modules/typescript/lib/lib.d.ts"); - } else { - sourceText = os.readFileSync(fileName); - } - // fileName = fileName.replace(/\.\w+$/, ""); // Remove extension. - if (sourceText) { - log("getSourceFile", { fileName }); - return ts.createSourceFile(fileName, sourceText, languageVersion); - } else { - log("getSourceFile NOT FOUND", { fileName }); - return undefined; - } - } - - getSourceFileByPath?( - fileName: string, - path: ts.Path, - languageVersion: ts.ScriptTarget, - onError?: (message: string) => void, - shouldCreateNewSourceFile?: boolean - ): ts.SourceFile | undefined { - console.log("getSourceFileByPath", fileName); - return undefined; - } - - // getCancellationToken?(): CancellationToken; - getDefaultLibFileName(options: ts.CompilerOptions): string { - return ts.getDefaultLibFileName(options); - } - - getDefaultLibLocation(): string { - return "/blah/"; - } - - writeFile( - fileName: string, - data: string, - writeByteOrderMark: boolean, - onError: ((message: string) => void) | undefined, - sourceFiles: ReadonlyArray - ): void { - //log("writeFile", { fileName, data }); - - os.compileOutput(data, fileName); - - _global["define"] = amd.makeDefine(fileName); - globalEval(data); - _global["define"] = null; - } - - getCurrentDirectory(): string { - log("getCurrentDirectory", "."); - return "."; - } - - getDirectories(path: string): string[] { - log("getDirectories", path); - return []; - } - - getCanonicalFileName(fileName: string): string { - return fileName; - } - - useCaseSensitiveFileNames(): boolean { - return true; - } - - getNewLine(): string { - return "\n"; - } - - resolveModuleNames( - moduleNames: string[], - containingFile: string, - reusedNames?: string[] - ): Array { - //log("resolveModuleNames", { moduleNames, reusedNames }); - return moduleNames.map((name: string) => { - if ( - name.startsWith("/") || - name.startsWith("http://") || - name.startsWith("https://") - ) { - throw Error("Non-relative imports not yet supported."); - } else { - // Relative import. - const containingDir = path.dirname(containingFile); - const resolvedFileName = path.join(containingDir, name); - //log("relative import", { containingFile, name, resolvedFileName }); - const isExternalLibraryImport = false; - return { resolvedFileName, isExternalLibraryImport }; - } - }); - } - - fileExists(fileName: string): boolean { - log("fileExists", fileName); - return false; - } - - readFile(fileName: string): string | undefined { - log("readFile", fileName); - return undefined; - } - - /** - * This method is a companion for 'resolveModuleNames' and is used to resolve - * 'types' references to actual type declaration files - */ - // resolveTypeReferenceDirectives?(typeReferenceDirectiveNames: string[], - // containingFile: string): (ResolvedTypeReferenceDirective | undefined)[]; - - // getEnvironmentVariable?(name: string): string - // createHash?(data: string): string; -} diff --git a/main.go b/main.go index daadadb905..78428f942d 100644 --- a/main.go +++ b/main.go @@ -1,29 +1,48 @@ package main import ( + "crypto/md5" + "encoding/hex" "github.com/golang/protobuf/proto" "github.com/ry/v8worker2" "io/ioutil" "os" "path" - "path/filepath" "runtime" - "strings" ) -func HandleCompileOutput(source string, filename string) []byte { - // println("compile output from golang", filename) - // Remove any ".." elements. This prevents hacking by trying to move up. - filename, err := filepath.Rel("/", filename) - check(err) - if strings.Contains(filename, "..") { - panic("Assertion error.") +func SourceCodeHash(filename string, sourceCodeBuf []byte) string { + h := md5.New() + h.Write([]byte(filename)) + h.Write(sourceCodeBuf) + return hex.EncodeToString(h.Sum(nil)) +} + +func HandleSourceCodeFetch(filename string) []byte { + res := &Msg{Kind: Msg_SOURCE_CODE_FETCH_RES} + sourceCodeBuf, err := Asset("dist/" + filename) + if err != nil { + sourceCodeBuf, err = ioutil.ReadFile(filename) } - filename = path.Join(CompileDir, filename) - err = os.MkdirAll(path.Dir(filename), 0700) - check(err) - err = ioutil.WriteFile(filename, []byte(source), 0600) + if err != nil { + res.Error = err.Error() + } else { + cacheKey := SourceCodeHash(filename, sourceCodeBuf) + println("cacheKey", filename, cacheKey) + // TODO For now don't do any cache lookups.. + res.Payload = &Msg_SourceCodeFetchRes{ + SourceCodeFetchRes: &SourceCodeFetchResMsg{ + SourceCode: string(sourceCodeBuf), + OutputCode: "", + }, + } + } + out, err := proto.Marshal(res) check(err) + return out +} + +func HandleSourceCodeCache(filename string, sourceCode string, outputCode string) []byte { return nil } @@ -87,9 +106,12 @@ func recv(buf []byte) []byte { return ReadFileSync(msg.Path) case Msg_EXIT: os.Exit(int(msg.Code)) - case Msg_COMPILE_OUTPUT: - payload := msg.GetCompileOutput() - return HandleCompileOutput(payload.Source, payload.Filename) + case Msg_SOURCE_CODE_FETCH: + payload := msg.GetSourceCodeFetch() + return HandleSourceCodeFetch(payload.Filename) + case Msg_SOURCE_CODE_CACHE: + payload := msg.GetSourceCodeCache() + return HandleSourceCodeCache(payload.Filename, payload.SourceCode, payload.OutputCode) default: panic("Unexpected message") } diff --git a/main.ts b/main.ts index ecef8aea77..4b64650f96 100644 --- a/main.ts +++ b/main.ts @@ -1,11 +1,14 @@ import { main as pb } from "./msg.pb"; import "./util"; -import { compile } from "./compiler"; +import * as runtime from "./runtime"; +import * as path from "path"; function start(cwd: string, argv: string[]): void { // TODO parse arguments. const inputFn = argv[1]; - compile(cwd, inputFn); + const fn = path.resolve(cwd, inputFn); + const m = runtime.FileModule.load(fn); + m.compileAndRun(); } V8Worker2.recv((ab: ArrayBuffer) => { diff --git a/msg.proto b/msg.proto index 55ee93a307..fbedf9752b 100644 --- a/msg.proto +++ b/msg.proto @@ -7,13 +7,18 @@ message Msg { READ_FILE_SYNC = 1; DATA_RESPONSE = 2; EXIT = 3; - COMPILE_OUTPUT = 4; + + SOURCE_CODE_FETCH = 4; + SOURCE_CODE_FETCH_RES = 5; + SOURCE_CODE_CACHE = 6; } MsgKind kind = 10; oneof payload { StartMsg start = 90; - CompileOutputMsg compile_output = 100; + SourceCodeFetchMsg source_code_fetch = 91; + SourceCodeFetchResMsg source_code_fetch_res = 92; + SourceCodeCacheMsg source_code_cache = 93; } // READ_FILE_SYNC and MKDIRP @@ -33,8 +38,15 @@ message StartMsg { repeated string argv = 2; } -// WRITE_COMPILE_OUTPUT -message CompileOutputMsg { - string source = 1; - string filename = 2; +message SourceCodeFetchMsg { string filename = 1; } + +message SourceCodeFetchResMsg { + string source_code = 1; + string output_code = 2; +} + +message SourceCodeCacheMsg { + string filename = 1; + string source_code = 2; + string output_code = 3; } diff --git a/os.ts b/os.ts index 1e1ad27c43..4952609f21 100644 --- a/os.ts +++ b/os.ts @@ -11,11 +11,27 @@ export function exit(code = 0): void { }); } -export function compileOutput(source: string, filename: string): void { - sendMsgFromObject({ - kind: pb.Msg.MsgKind.COMPILE_OUTPUT, - compileOutput: { source, filename } +export function sourceCodeFetch( + filename: string +): { sourceCode: string; outputCode: string } { + const res = sendMsgFromObject({ + kind: pb.Msg.MsgKind.SOURCE_CODE_FETCH, + sourceCodeFetch: { filename } }); + const { sourceCode, outputCode } = res.sourceCodeFetchRes; + return { sourceCode, outputCode }; +} + +export function sourceCodeCache( + filename: string, + sourceCode: string, + outputCode: string +): void { + const res = sendMsgFromObject({ + kind: pb.Msg.MsgKind.SOURCE_CODE_CACHE, + sourceCodeCache: { filename, sourceCode, outputCode } + }); + throwOnError(res); } export function readFileSync(filename: string): string { @@ -23,9 +39,6 @@ export function readFileSync(filename: string): string { kind: pb.Msg.MsgKind.READ_FILE_SYNC, path: filename }); - if (res.error != null && res.error.length > 0) { - throw Error(res.error); - } const decoder = new TextDecoder("utf8"); return decoder.decode(res.data); } @@ -41,8 +54,16 @@ function sendMsgFromObject(obj: pb.IMsg): null | pb.Msg { const ab = typedArrayToArrayBuffer(ui8); const resBuf = V8Worker2.send(ab); if (resBuf != null && resBuf.byteLength > 0) { - return pb.Msg.decode(new Uint8Array(resBuf)); + const res = pb.Msg.decode(new Uint8Array(resBuf)); + throwOnError(res); + return res; } else { return null; } } + +function throwOnError(res: pb.Msg) { + if (res != null && res.error != null && res.error.length > 0) { + throw Error(res.error); + } +} diff --git a/runtime.ts b/runtime.ts new file mode 100644 index 0000000000..1e031e7f7e --- /dev/null +++ b/runtime.ts @@ -0,0 +1,244 @@ +// Glossary +// outputCode = generated javascript code +// sourceCode = typescript code (or input javascript code) +// fileName = an unresolved raw fileName. +// moduleName = a resolved module name + +import * as ts from "typescript"; +import * as path from "path"; +import * as util from "./util"; +import { log } from "./util"; +import * as os from "./os"; + +// This class represents a module. We call it FileModule to make it explicit +// that each module represents a single file. +// Access to FileModule instances should only be done thru the static method +// FileModule.load(). FileModules are executed upon first load. +export class FileModule { + scriptVersion: string = undefined; + sourceCode: string; + outputCode: string; + readonly exports = {}; + + private static readonly map = new Map(); + private constructor(readonly fileName: string) { + FileModule.map.set(fileName, this); + + assertValidFileName(this.fileName); + + // Load typescript code (sourceCode) and maybe load compiled javascript + // (outputCode) from cache. If cache is empty, outputCode will be null. + const { sourceCode, outputCode } = os.sourceCodeFetch(this.fileName); + this.sourceCode = sourceCode; + this.outputCode = outputCode; + this.scriptVersion = "1"; + } + + compileAndRun() { + if (!this.outputCode) { + // If there is no cached outputCode, the compile the code. + util.assert(this.sourceCode && this.sourceCode.length > 0); + const compiler = Compiler.instance(); + this.outputCode = compiler.compile(this.fileName); + os.sourceCodeCache(this.fileName, this.sourceCode, this.outputCode); + } + util.log("compileAndRun", this.sourceCode); + execute(this.fileName, this.outputCode); + } + + static load(fileName: string): FileModule { + assertValidFileName(fileName); + let m = this.map.get(fileName); + if (m == null) { + m = new this(fileName); + util.assert(this.map.has(fileName)); + } + return m; + } + + static getScriptsWithSourceCode(): string[] { + const out = []; + for (const fn of this.map.keys()) { + const m = this.map.get(fn); + if (m.sourceCode) { + out.push(fn); + } + } + return out; + } +} + +function assertValidFileName(fileName: string): void { + if (fileName !== "lib.d.ts") { + util.assert(fileName[0] === "/", `fileName must be absolute: ${fileName}`); + } +} + +// tslint:disable-next-line:no-any +type AmdFactory = (...args: any[]) => undefined | object; +type AmdDefine = (deps: string[], factory: AmdFactory) => void; + +export function makeDefine(fileName: string): AmdDefine { + const localDefine = (deps: string[], factory: AmdFactory): void => { + const localRequire = (x: string) => { + log("localRequire", x); + }; + const currentModule = FileModule.load(fileName); + const localExports = currentModule.exports; + log("localDefine", fileName, deps, localExports); + const args = deps.map(dep => { + if (dep === "require") { + return localRequire; + } else if (dep === "exports") { + return localExports; + } else { + dep = resolveModuleName(dep, fileName); + const depModule = FileModule.load(dep); + depModule.compileAndRun(); + return depModule.exports; + } + }); + factory(...args); + }; + return localDefine; +} + +function resolveModuleName(fileName: string, contextFileName: string): string { + return path.resolve(path.dirname(contextFileName), fileName); +} + +function execute(fileName: string, outputCode: string): void { + util.assert(outputCode && outputCode.length > 0); + util._global["define"] = makeDefine(fileName); + util.globalEval(outputCode); + util._global["define"] = null; +} + +// This is a singleton class. Use Compiler.instance() to access. +class Compiler { + options: ts.CompilerOptions = { + allowJs: true, + module: ts.ModuleKind.AMD, + outDir: "$deno$" + }; + /* + allowJs: true, + inlineSourceMap: true, + inlineSources: true, + module: ts.ModuleKind.AMD, + noEmit: false, + outDir: '$deno$', + */ + private service: ts.LanguageService; + + private constructor() { + const host = new TypeScriptHost(this.options); + this.service = ts.createLanguageService(host); + } + + private static _instance: Compiler; + static instance(): Compiler { + return this._instance || (this._instance = new this()); + } + + compile(fileName: string): string { + const output = this.service.getEmitOutput(fileName); + + // Get the relevant diagnostics - this is 3x faster than + // `getPreEmitDiagnostics`. + const diagnostics = this.service + .getCompilerOptionsDiagnostics() + .concat(this.service.getSyntacticDiagnostics(fileName)) + .concat(this.service.getSemanticDiagnostics(fileName)); + if (diagnostics.length > 0) { + throw Error("diagnotics"); + } + + util.log("compile output", output); + util.assert(!output.emitSkipped); + + const outputCode = output.outputFiles[0].text; + // let sourceMapCode = output.outputFiles[0].text; + return outputCode; + } +} + +// Create the compiler host for type checking. +class TypeScriptHost implements ts.LanguageServiceHost { + constructor(readonly options: ts.CompilerOptions) {} + + getScriptFileNames(): string[] { + const keys = FileModule.getScriptsWithSourceCode(); + util.log("getScriptFileNames", keys); + return keys; + } + + getScriptVersion(fileName: string): string { + util.log("getScriptVersion", fileName); + const m = FileModule.load(fileName); + return m.scriptVersion; + } + + getScriptSnapshot(fileName: string): ts.IScriptSnapshot | undefined { + util.log("getScriptSnapshot", fileName); + const m = FileModule.load(fileName); + if (m.sourceCode) { + return ts.ScriptSnapshot.fromString(m.sourceCode); + } else { + return undefined; + } + } + + fileExists(fileName: string): boolean { + throw Error("not implemented"); + } + + readFile(path: string, encoding?: string): string | undefined { + util.log("readFile", path); + throw Error("not implemented"); + } + + getNewLine() { + const EOL = "\n"; + return EOL; + } + + getCurrentDirectory() { + util.log("getCurrentDirectory"); + return "."; + } + + getCompilationSettings() { + util.log("getCompilationSettings"); + return this.options; + } + + getDefaultLibFileName(options: ts.CompilerOptions): string { + util.log("getDefaultLibFileName"); + return ts.getDefaultLibFileName(options); + } + + resolveModuleNames( + moduleNames: string[], + containingFile: string, + reusedNames?: string[] + ): Array { + util.log("resolveModuleNames", { moduleNames, reusedNames }); + return moduleNames.map((name: string) => { + if ( + name.startsWith("/") || + name.startsWith("http://") || + name.startsWith("https://") + ) { + throw Error("Non-relative imports not yet supported."); + } else { + // Relative import. + const containingDir = path.dirname(containingFile); + const resolvedFileName = path.join(containingDir, name); + util.log("relative import", { containingFile, name, resolvedFileName }); + const isExternalLibraryImport = false; + return { resolvedFileName, isExternalLibraryImport }; + } + }); + } +}