runtime.ts - first pass at caching compiler

Makefile

@@ -1,16 +1,16 @@
 TS_FILES = \
-	amd.ts \
 	main.ts \
-	msg.pb.js \
-  compiler.ts \
 	msg.pb.d.ts \
+	msg.pb.js \
 	os.ts \
+	runtime.ts \
 	util.ts
 
 deno: assets.go msg.pb.go main.go
 	go build -o deno
 
 assets.go: dist/main.js
+	cp node_modules/typescript/lib/lib.d.ts dist/
 	go-bindata -pkg main -o assets.go dist/
 
 msg.pb.go: msg.proto

amd.ts

@@ -1,80 +0,0 @@
-import * as path from "path";
-import { assert, log } from "./util";
-
-namespace ModuleExportsCache {
-  const cache = new Map<string, object>();
-  export function set(fileName: string, moduleExports: object) {
-    fileName = normalizeModuleName(fileName);
-    assert(
-      fileName.startsWith("/"),
-      `Normalized modules should start with /\n${fileName}`
-    );
-    log("ModuleExportsCache set", fileName);
-    cache.set(fileName, moduleExports);
-  }
-  export function get(fileName: string): object {
-    fileName = normalizeModuleName(fileName);
-    log("ModuleExportsCache get", fileName);
-    let moduleExports = cache.get(fileName);
-    if (moduleExports == null) {
-      moduleExports = {};
-      set(fileName, moduleExports);
-    }
-    return moduleExports;
-  }
-}
-
-function normalizeModuleName(fileName: string): string {
-  // Remove the extension.
-  return fileName.replace(/\.\w+$/, "");
-}
-
-function normalizeRelativeModuleName(contextFn: string, depFn: string): string {
-  if (depFn.startsWith("/")) {
-    return depFn;
-  } else {
-    return path.resolve(path.dirname(contextFn), depFn);
-  }
-}
-
-const executeQueue: Array<() => void> = [];
-
-export function executeQueueDrain(): void {
-  let fn;
-  while ((fn = executeQueue.shift())) {
-    fn();
-  }
-}
-
-// tslint:disable-next-line:no-any
-type AmdFactory = (...args: any[]) => undefined | object;
-type AmdDefine = (deps: string[], factory: AmdFactory) => void;
-
-export function makeDefine(fileName: string): AmdDefine {
-  const localDefine = (deps: string[], factory: AmdFactory): void => {
-    const localRequire = (x: string) => {
-      log("localRequire", x);
-    };
-    const localExports = ModuleExportsCache.get(fileName);
-    log("localDefine", fileName, deps, localExports);
-    const args = deps.map(dep => {
-      if (dep === "require") {
-        return localRequire;
-      } else if (dep === "exports") {
-        return localExports;
-      } else {
-        dep = normalizeRelativeModuleName(fileName, dep);
-        return ModuleExportsCache.get(dep);
-      }
-    });
-    executeQueue.push(() => {
-      log("execute", fileName);
-      const r = factory(...args);
-      if (r != null) {
-        ModuleExportsCache.set(fileName, r);
-        throw Error("x");
-      }
-    });
-  };
-  return localDefine;
-}

compiler.ts

@@ -1,221 +0,0 @@
-import * as ts from "typescript";
-import { log, assert, globalEval, _global } from "./util";
-import * as os from "./os";
-import * as path from "path";
-import * as amd from "./amd";
-
-/*
-export function makeCacheDir(): string {
-  let cacheDir = path.join(env.HOME, ".deno/cache")
-  os.mkdirp(cacheDir);
-  return cacheDir
-}
-*/
-
-export function compile(cwd: string, inputFn: string): void {
-  const options: ts.CompilerOptions = {
-    allowJs: true,
-    module: ts.ModuleKind.AMD,
-    outDir: "/" // Will be placed in ~/.deno/compile
-  };
-  const host = new CompilerHost();
-
-  const inputExt = path.extname(inputFn);
-  if (!EXTENSIONS.includes(inputExt)) {
-    console.error(`Bad file name extension for input "${inputFn}"`);
-    os.exit(1);
-  }
-
-  const program = ts.createProgram([inputFn], options, host);
-  //let sourceFiles = program.getSourceFiles();
-  //log("rootFileNames", program.getRootFileNames());
-
-  // Print compilation errors, if any.
-  const diagnostics = getDiagnostics(program);
-  if (diagnostics.length > 0) {
-    const errorMessages = diagnostics.map(d => formatDiagnostic(d, cwd));
-    for (const msg of errorMessages) {
-      console.error(msg);
-    }
-    os.exit(2);
-  }
-
-  const emitResult = program.emit();
-  assert(!emitResult.emitSkipped);
-  log("emitResult", emitResult);
-
-  amd.executeQueueDrain();
-}
-
-/**
- * Format a diagnostic object into a string.
- * Adapted from TS-Node https://github.com/TypeStrong/ts-node
- * which uses the same MIT license as this file but is
- * Copyright (c) 2014 Blake Embrey (hello@blakeembrey.com)
- */
-export function formatDiagnostic(
-  diagnostic: ts.Diagnostic,
-  cwd: string,
-  lineOffset = 0
-): string {
-  const messageText = ts.flattenDiagnosticMessageText(
-    diagnostic.messageText,
-    "\n"
-  );
-  const { code } = diagnostic;
-  if (diagnostic.file) {
-    const fn = path.relative(cwd, diagnostic.file.fileName);
-    if (diagnostic.start) {
-      const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(
-        diagnostic.start
-      );
-      const r = Number(line) + 1 + lineOffset;
-      const c = Number(character) + 1;
-      return `${fn} (${r},${c}): ${messageText} (${code})`;
-    }
-    return `${fn}: ${messageText} (${code})`;
-  }
-  return `${messageText} (${code})`;
-}
-
-function getDiagnostics(program: ts.Program): ReadonlyArray<ts.Diagnostic> {
-  return program
-    .getOptionsDiagnostics()
-    .concat(
-      program.getGlobalDiagnostics(),
-      program.getSyntacticDiagnostics(),
-      program.getSemanticDiagnostics(),
-      program.getDeclarationDiagnostics()
-    );
-}
-
-const EXTENSIONS = [".ts", ".js"];
-
-export class CompilerHost {
-  constructor() {}
-
-  getSourceFile(
-    fileName: string,
-    languageVersion: ts.ScriptTarget,
-    onError?: (message: string) => void,
-    shouldCreateNewSourceFile?: boolean
-  ): ts.SourceFile | undefined {
-    let sourceText: string;
-    if (fileName === "lib.d.ts") {
-      // TODO This should be compiled into the bindata.
-      sourceText = os.readFileSync("node_modules/typescript/lib/lib.d.ts");
-    } else {
-      sourceText = os.readFileSync(fileName);
-    }
-    // fileName = fileName.replace(/\.\w+$/, ""); // Remove extension.
-    if (sourceText) {
-      log("getSourceFile", { fileName });
-      return ts.createSourceFile(fileName, sourceText, languageVersion);
-    } else {
-      log("getSourceFile NOT FOUND", { fileName });
-      return undefined;
-    }
-  }
-
-  getSourceFileByPath?(
-    fileName: string,
-    path: ts.Path,
-    languageVersion: ts.ScriptTarget,
-    onError?: (message: string) => void,
-    shouldCreateNewSourceFile?: boolean
-  ): ts.SourceFile | undefined {
-    console.log("getSourceFileByPath", fileName);
-    return undefined;
-  }
-
-  // getCancellationToken?(): CancellationToken;
-  getDefaultLibFileName(options: ts.CompilerOptions): string {
-    return ts.getDefaultLibFileName(options);
-  }
-
-  getDefaultLibLocation(): string {
-    return "/blah/";
-  }
-
-  writeFile(
-    fileName: string,
-    data: string,
-    writeByteOrderMark: boolean,
-    onError: ((message: string) => void) | undefined,
-    sourceFiles: ReadonlyArray<ts.SourceFile>
-  ): void {
-    //log("writeFile", { fileName, data });
-
-    os.compileOutput(data, fileName);
-
-    _global["define"] = amd.makeDefine(fileName);
-    globalEval(data);
-    _global["define"] = null;
-  }
-
-  getCurrentDirectory(): string {
-    log("getCurrentDirectory", ".");
-    return ".";
-  }
-
-  getDirectories(path: string): string[] {
-    log("getDirectories", path);
-    return [];
-  }
-
-  getCanonicalFileName(fileName: string): string {
-    return fileName;
-  }
-
-  useCaseSensitiveFileNames(): boolean {
-    return true;
-  }
-
-  getNewLine(): string {
-    return "\n";
-  }
-
-  resolveModuleNames(
-    moduleNames: string[],
-    containingFile: string,
-    reusedNames?: string[]
-  ): Array<ts.ResolvedModule | undefined> {
-    //log("resolveModuleNames", { moduleNames, reusedNames });
-    return moduleNames.map((name: string) => {
-      if (
-        name.startsWith("/") ||
-        name.startsWith("http://") ||
-        name.startsWith("https://")
-      ) {
-        throw Error("Non-relative imports not yet supported.");
-      } else {
-        // Relative import.
-        const containingDir = path.dirname(containingFile);
-        const resolvedFileName = path.join(containingDir, name);
-        //log("relative import", { containingFile, name, resolvedFileName });
-        const isExternalLibraryImport = false;
-        return { resolvedFileName, isExternalLibraryImport };
-      }
-    });
-  }
-
-  fileExists(fileName: string): boolean {
-    log("fileExists", fileName);
-    return false;
-  }
-
-  readFile(fileName: string): string | undefined {
-    log("readFile", fileName);
-    return undefined;
-  }
-
-  /**
-   * This method is a companion for 'resolveModuleNames' and is used to resolve
-   * 'types' references to actual type declaration files
-   */
-  // resolveTypeReferenceDirectives?(typeReferenceDirectiveNames: string[],
-  // containingFile: string): (ResolvedTypeReferenceDirective | undefined)[];
-
-  // getEnvironmentVariable?(name: string): string
-  // createHash?(data: string): string;
-}

main.go

@@ -1,29 +1,48 @@
 package main
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"github.com/golang/protobuf/proto"
 	"github.com/ry/v8worker2"
 	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
 	"runtime"
-	"strings"
 )
 
-func HandleCompileOutput(source string, filename string) []byte {
+func SourceCodeHash(filename string, sourceCodeBuf []byte) string {
-	// println("compile output from golang", filename)
+	h := md5.New()
-	// Remove any ".." elements. This prevents hacking by trying to move up.
+	h.Write([]byte(filename))
-	filename, err := filepath.Rel("/", filename)
+	h.Write(sourceCodeBuf)
-	check(err)
+	return hex.EncodeToString(h.Sum(nil))
-	if strings.Contains(filename, "..") {
+}
-		panic("Assertion error.")
+
+func HandleSourceCodeFetch(filename string) []byte {
+	res := &Msg{Kind: Msg_SOURCE_CODE_FETCH_RES}
+	sourceCodeBuf, err := Asset("dist/" + filename)
+	if err != nil {
+		sourceCodeBuf, err = ioutil.ReadFile(filename)
 	}
-	filename = path.Join(CompileDir, filename)
+	if err != nil {
-	err = os.MkdirAll(path.Dir(filename), 0700)
+		res.Error = err.Error()
-	check(err)
+	} else {
-	err = ioutil.WriteFile(filename, []byte(source), 0600)
+		cacheKey := SourceCodeHash(filename, sourceCodeBuf)
+		println("cacheKey", filename, cacheKey)
+		// TODO For now don't do any cache lookups..
+		res.Payload = &Msg_SourceCodeFetchRes{
+			SourceCodeFetchRes: &SourceCodeFetchResMsg{
+				SourceCode: string(sourceCodeBuf),
+				OutputCode: "",
+			},
+		}
+	}
+	out, err := proto.Marshal(res)
 	check(err)
+	return out
+}
+
+func HandleSourceCodeCache(filename string, sourceCode string, outputCode string) []byte {
 	return nil
 }
 
@@ -87,9 +106,12 @@ func recv(buf []byte) []byte {
 		return ReadFileSync(msg.Path)
 	case Msg_EXIT:
 		os.Exit(int(msg.Code))
-	case Msg_COMPILE_OUTPUT:
+	case Msg_SOURCE_CODE_FETCH:
-		payload := msg.GetCompileOutput()
+		payload := msg.GetSourceCodeFetch()
-		return HandleCompileOutput(payload.Source, payload.Filename)
+		return HandleSourceCodeFetch(payload.Filename)
+	case Msg_SOURCE_CODE_CACHE:
+		payload := msg.GetSourceCodeCache()
+		return HandleSourceCodeCache(payload.Filename, payload.SourceCode, payload.OutputCode)
 	default:
 		panic("Unexpected message")
 	}

main.ts

@@ -1,11 +1,14 @@
 import { main as pb } from "./msg.pb";
 import "./util";
-import { compile } from "./compiler";
+import * as runtime from "./runtime";
+import * as path from "path";
 
 function start(cwd: string, argv: string[]): void {
   // TODO parse arguments.
   const inputFn = argv[1];
-  compile(cwd, inputFn);
+  const fn = path.resolve(cwd, inputFn);
+  const m = runtime.FileModule.load(fn);
+  m.compileAndRun();
 }
 
 V8Worker2.recv((ab: ArrayBuffer) => {

msg.proto

@@ -7,13 +7,18 @@ message Msg {
     READ_FILE_SYNC = 1;
     DATA_RESPONSE = 2;
     EXIT = 3;
-    COMPILE_OUTPUT = 4;
+
+    SOURCE_CODE_FETCH = 4;
+    SOURCE_CODE_FETCH_RES = 5;
+    SOURCE_CODE_CACHE = 6;
   }
   MsgKind kind = 10;
 
   oneof payload {
     StartMsg start = 90;
-    CompileOutputMsg compile_output = 100;
+    SourceCodeFetchMsg source_code_fetch = 91;
+    SourceCodeFetchResMsg source_code_fetch_res = 92;
+    SourceCodeCacheMsg source_code_cache = 93;
   }
 
   // READ_FILE_SYNC and MKDIRP
@@ -33,8 +38,15 @@ message StartMsg {
   repeated string argv = 2;
 }
 
-// WRITE_COMPILE_OUTPUT
+message SourceCodeFetchMsg { string filename = 1; }
-message CompileOutputMsg {
+
-  string source = 1;
+message SourceCodeFetchResMsg {
-  string filename = 2;
+  string source_code = 1;
+  string output_code = 2;
+}
+
+message SourceCodeCacheMsg {
+  string filename = 1;
+  string source_code = 2;
+  string output_code = 3;
 }

os.ts

@@ -11,11 +11,27 @@ export function exit(code = 0): void {
   });
 }
 
-export function compileOutput(source: string, filename: string): void {
+export function sourceCodeFetch(
-  sendMsgFromObject({
+  filename: string
-    kind: pb.Msg.MsgKind.COMPILE_OUTPUT,
+): { sourceCode: string; outputCode: string } {
-    compileOutput: { source, filename }
+  const res = sendMsgFromObject({
+    kind: pb.Msg.MsgKind.SOURCE_CODE_FETCH,
+    sourceCodeFetch: { filename }
   });
+  const { sourceCode, outputCode } = res.sourceCodeFetchRes;
+  return { sourceCode, outputCode };
+}
+
+export function sourceCodeCache(
+  filename: string,
+  sourceCode: string,
+  outputCode: string
+): void {
+  const res = sendMsgFromObject({
+    kind: pb.Msg.MsgKind.SOURCE_CODE_CACHE,
+    sourceCodeCache: { filename, sourceCode, outputCode }
+  });
+  throwOnError(res);
 }
 
 export function readFileSync(filename: string): string {
@@ -23,9 +39,6 @@ export function readFileSync(filename: string): string {
     kind: pb.Msg.MsgKind.READ_FILE_SYNC,
     path: filename
   });
-  if (res.error != null && res.error.length > 0) {
-    throw Error(res.error);
-  }
   const decoder = new TextDecoder("utf8");
   return decoder.decode(res.data);
 }
@@ -41,8 +54,16 @@ function sendMsgFromObject(obj: pb.IMsg): null | pb.Msg {
   const ab = typedArrayToArrayBuffer(ui8);
   const resBuf = V8Worker2.send(ab);
   if (resBuf != null && resBuf.byteLength > 0) {
-    return pb.Msg.decode(new Uint8Array(resBuf));
+    const res = pb.Msg.decode(new Uint8Array(resBuf));
+    throwOnError(res);
+    return res;
   } else {
     return null;
   }
 }
+
+function throwOnError(res: pb.Msg) {
+  if (res != null && res.error != null && res.error.length > 0) {
+    throw Error(res.error);
+  }
+}

runtime.ts

@@ -0,0 +1,244 @@
+// Glossary
+// outputCode = generated javascript code
+// sourceCode = typescript code (or input javascript code)
+// fileName = an unresolved raw fileName.
+// moduleName = a resolved module name
+
+import * as ts from "typescript";
+import * as path from "path";
+import * as util from "./util";
+import { log } from "./util";
+import * as os from "./os";
+
+// This class represents a module. We call it FileModule to make it explicit
+// that each module represents a single file.
+// Access to FileModule instances should only be done thru the static method
+// FileModule.load(). FileModules are executed upon first load.
+export class FileModule {
+  scriptVersion: string = undefined;
+  sourceCode: string;
+  outputCode: string;
+  readonly exports = {};
+
+  private static readonly map = new Map<string, FileModule>();
+  private constructor(readonly fileName: string) {
+    FileModule.map.set(fileName, this);
+
+    assertValidFileName(this.fileName);
+
+    // Load typescript code (sourceCode) and maybe load compiled javascript
+    // (outputCode) from cache. If cache is empty, outputCode will be null.
+    const { sourceCode, outputCode } = os.sourceCodeFetch(this.fileName);
+    this.sourceCode = sourceCode;
+    this.outputCode = outputCode;
+    this.scriptVersion = "1";
+  }
+
+  compileAndRun() {
+    if (!this.outputCode) {
+      // If there is no cached outputCode, the compile the code.
+      util.assert(this.sourceCode && this.sourceCode.length > 0);
+      const compiler = Compiler.instance();
+      this.outputCode = compiler.compile(this.fileName);
+      os.sourceCodeCache(this.fileName, this.sourceCode, this.outputCode);
+    }
+    util.log("compileAndRun", this.sourceCode);
+    execute(this.fileName, this.outputCode);
+  }
+
+  static load(fileName: string): FileModule {
+    assertValidFileName(fileName);
+    let m = this.map.get(fileName);
+    if (m == null) {
+      m = new this(fileName);
+      util.assert(this.map.has(fileName));
+    }
+    return m;
+  }
+
+  static getScriptsWithSourceCode(): string[] {
+    const out = [];
+    for (const fn of this.map.keys()) {
+      const m = this.map.get(fn);
+      if (m.sourceCode) {
+        out.push(fn);
+      }
+    }
+    return out;
+  }
+}
+
+function assertValidFileName(fileName: string): void {
+  if (fileName !== "lib.d.ts") {
+    util.assert(fileName[0] === "/", `fileName must be absolute: ${fileName}`);
+  }
+}
+
+// tslint:disable-next-line:no-any
+type AmdFactory = (...args: any[]) => undefined | object;
+type AmdDefine = (deps: string[], factory: AmdFactory) => void;
+
+export function makeDefine(fileName: string): AmdDefine {
+  const localDefine = (deps: string[], factory: AmdFactory): void => {
+    const localRequire = (x: string) => {
+      log("localRequire", x);
+    };
+    const currentModule = FileModule.load(fileName);
+    const localExports = currentModule.exports;
+    log("localDefine", fileName, deps, localExports);
+    const args = deps.map(dep => {
+      if (dep === "require") {
+        return localRequire;
+      } else if (dep === "exports") {
+        return localExports;
+      } else {
+        dep = resolveModuleName(dep, fileName);
+        const depModule = FileModule.load(dep);
+        depModule.compileAndRun();
+        return depModule.exports;
+      }
+    });
+    factory(...args);
+  };
+  return localDefine;
+}
+
+function resolveModuleName(fileName: string, contextFileName: string): string {
+  return path.resolve(path.dirname(contextFileName), fileName);
+}
+
+function execute(fileName: string, outputCode: string): void {
+  util.assert(outputCode && outputCode.length > 0);
+  util._global["define"] = makeDefine(fileName);
+  util.globalEval(outputCode);
+  util._global["define"] = null;
+}
+
+// This is a singleton class. Use Compiler.instance() to access.
+class Compiler {
+  options: ts.CompilerOptions = {
+    allowJs: true,
+    module: ts.ModuleKind.AMD,
+    outDir: "$deno$"
+  };
+  /*
+  allowJs: true,
+  inlineSourceMap: true,
+  inlineSources: true,
+  module: ts.ModuleKind.AMD,
+  noEmit: false,
+  outDir: '$deno$',
+  */
+  private service: ts.LanguageService;
+
+  private constructor() {
+    const host = new TypeScriptHost(this.options);
+    this.service = ts.createLanguageService(host);
+  }
+
+  private static _instance: Compiler;
+  static instance(): Compiler {
+    return this._instance || (this._instance = new this());
+  }
+
+  compile(fileName: string): string {
+    const output = this.service.getEmitOutput(fileName);
+
+    // Get the relevant diagnostics - this is 3x faster than
+    // `getPreEmitDiagnostics`.
+    const diagnostics = this.service
+      .getCompilerOptionsDiagnostics()
+      .concat(this.service.getSyntacticDiagnostics(fileName))
+      .concat(this.service.getSemanticDiagnostics(fileName));
+    if (diagnostics.length > 0) {
+      throw Error("diagnotics");
+    }
+
+    util.log("compile output", output);
+    util.assert(!output.emitSkipped);
+
+    const outputCode = output.outputFiles[0].text;
+    // let sourceMapCode = output.outputFiles[0].text;
+    return outputCode;
+  }
+}
+
+// Create the compiler host for type checking.
+class TypeScriptHost implements ts.LanguageServiceHost {
+  constructor(readonly options: ts.CompilerOptions) {}
+
+  getScriptFileNames(): string[] {
+    const keys = FileModule.getScriptsWithSourceCode();
+    util.log("getScriptFileNames", keys);
+    return keys;
+  }
+
+  getScriptVersion(fileName: string): string {
+    util.log("getScriptVersion", fileName);
+    const m = FileModule.load(fileName);
+    return m.scriptVersion;
+  }
+
+  getScriptSnapshot(fileName: string): ts.IScriptSnapshot | undefined {
+    util.log("getScriptSnapshot", fileName);
+    const m = FileModule.load(fileName);
+    if (m.sourceCode) {
+      return ts.ScriptSnapshot.fromString(m.sourceCode);
+    } else {
+      return undefined;
+    }
+  }
+
+  fileExists(fileName: string): boolean {
+    throw Error("not implemented");
+  }
+
+  readFile(path: string, encoding?: string): string | undefined {
+    util.log("readFile", path);
+    throw Error("not implemented");
+  }
+
+  getNewLine() {
+    const EOL = "\n";
+    return EOL;
+  }
+
+  getCurrentDirectory() {
+    util.log("getCurrentDirectory");
+    return ".";
+  }
+
+  getCompilationSettings() {
+    util.log("getCompilationSettings");
+    return this.options;
+  }
+
+  getDefaultLibFileName(options: ts.CompilerOptions): string {
+    util.log("getDefaultLibFileName");
+    return ts.getDefaultLibFileName(options);
+  }
+
+  resolveModuleNames(
+    moduleNames: string[],
+    containingFile: string,
+    reusedNames?: string[]
+  ): Array<ts.ResolvedModule | undefined> {
+    util.log("resolveModuleNames", { moduleNames, reusedNames });
+    return moduleNames.map((name: string) => {
+      if (
+        name.startsWith("/") ||
+        name.startsWith("http://") ||
+        name.startsWith("https://")
+      ) {
+        throw Error("Non-relative imports not yet supported.");
+      } else {
+        // Relative import.
+        const containingDir = path.dirname(containingFile);
+        const resolvedFileName = path.join(containingDir, name);
+        util.log("relative import", { containingFile, name, resolvedFileName });
+        const isExternalLibraryImport = false;
+        return { resolvedFileName, isExternalLibraryImport };
+      }
+    });
+  }
+}