Compile cache and relative imports working.

.travis.yml

@@ -1,8 +1,12 @@
 language: go
 go:
  - 1.9.2
-cache: ccache
+cache:
+  directories:
+   - $HOME/.ccache
+   - `go env GOPATH`/src/github.com/ry/v8worker2/out
 install:
+ - go get github.com/jteeuwen/go-bindata
  - go get -d github.com/ry/v8worker2
  - (cd `go env GOPATH`/src/github.com/ry/v8worker2 && ./tools/build.py)
  - make

Makefile

@@ -1,3 +1,12 @@
+TS_FILES = \
+	amd.ts \
+	main.ts \
+	msg.pb.js \
+  compiler.ts \
+  msg.pb.d.ts \
+  os.ts \
+  util.ts
+
 deno: assets.go msg.pb.go main.go
 	go build -o deno
 
@@ -13,7 +22,7 @@ msg.pb.js: msg.proto node_modules
 msg.pb.d.ts: msg.pb.js node_modules
 	./node_modules/.bin/pbts -o msg.pb.d.ts msg.pb.js
 
-dist/main.js: main.ts compiler.ts os.ts util.ts msg.pb.js msg.pb.d.ts node_modules
+dist/main.js: $(TS_FILES) node_modules
 	./node_modules/.bin/tsc --noEmit # Only for type checking.
 	./node_modules/.bin/parcel build --out-dir=dist/ --no-minify main.ts
 

amd.ts

@@ -0,0 +1,80 @@
+import * as path from "path";
+import { assert, log } from "./util";
+
+namespace ModuleExportsCache {
+  const cache = new Map<string, object>();
+  export function set(fileName: string, moduleExports: object) {
+    fileName = normalizeModuleName(fileName);
+    assert(
+      fileName.startsWith("/"),
+      `Normalized modules should start with /\n${fileName}`
+    );
+    log("ModuleExportsCache set", fileName);
+    cache.set(fileName, moduleExports);
+  }
+  export function get(fileName: string): object {
+    fileName = normalizeModuleName(fileName);
+    log("ModuleExportsCache get", fileName);
+    let moduleExports = cache.get(fileName);
+    if (moduleExports == null) {
+      moduleExports = {};
+      set(fileName, moduleExports);
+    }
+    return moduleExports;
+  }
+}
+
+function normalizeModuleName(fileName: string): string {
+  // Remove the extension.
+  return fileName.replace(/\.\w+$/, "");
+}
+
+function normalizeRelativeModuleName(contextFn: string, depFn: string): string {
+  if (depFn.startsWith("/")) {
+    return depFn;
+  } else {
+    return path.resolve(path.dirname(contextFn), depFn);
+  }
+}
+
+const executeQueue: Array<() => void> = [];
+
+export function executeQueueDrain(): void {
+  let fn;
+  while ((fn = executeQueue.shift())) {
+    fn();
+  }
+}
+
+// tslint:disable-next-line:no-any
+type AmdFactory = (...args: any[]) => undefined | object;
+type AmdDefine = (deps: string[], factory: AmdFactory) => void;
+
+export function makeDefine(fileName: string): AmdDefine {
+  const localDefine = (deps: string[], factory: AmdFactory): void => {
+    const localRequire = (x: string) => {
+      log("localRequire", x);
+    };
+    const localExports = ModuleExportsCache.get(fileName);
+    log("localDefine", fileName, deps, localExports);
+    const args = deps.map(dep => {
+      if (dep === "require") {
+        return localRequire;
+      } else if (dep === "exports") {
+        return localExports;
+      } else {
+        dep = normalizeRelativeModuleName(fileName, dep);
+        return ModuleExportsCache.get(dep);
+      }
+    });
+    executeQueue.push(() => {
+      log("execute", fileName);
+      const r = factory(...args);
+      if (r != null) {
+        ModuleExportsCache.set(fileName, r);
+        throw Error("x");
+      }
+    });
+  };
+  return localDefine;
+}

compiler.ts

@@ -1,19 +1,29 @@
 import * as ts from "typescript";
-import { log, assert, globalEval } from "./util";
-import { exit, readFileSync } from "./os";
+import { log, assert, globalEval, _global } from "./util";
+import * as os from "./os";
 import * as path from "path";
+import * as amd from "./amd";
+
+/*
+export function makeCacheDir(): string {
+  let cacheDir = path.join(env.HOME, ".deno/cache")
+  os.mkdirp(cacheDir);
+  return cacheDir
+}
+*/
 
 export function compile(cwd: string, inputFn: string): void {
   const options: ts.CompilerOptions = {
     allowJs: true,
-    outDir: "_denoCache_/",
+    module: ts.ModuleKind.AMD,
+    outDir: "/" // Will be placed in ~/.deno/compile
   };
-  const host = new CompilerHost(cwd);
+  const host = new CompilerHost();
 
   const inputExt = path.extname(inputFn);
   if (!EXTENSIONS.includes(inputExt)) {
     console.error(`Bad file name extension for input "${inputFn}"`);
-    exit(1);
+    os.exit(1);
   }
 
   const program = ts.createProgram([inputFn], options, host);
@@ -27,12 +37,14 @@ export function compile(cwd: string, inputFn: string): void {
     for (const msg of errorMessages) {
       console.error(msg);
     }
-    exit(2);
+    os.exit(2);
   }
 
   const emitResult = program.emit();
   assert(!emitResult.emitSkipped);
   log("emitResult", emitResult);
+
+  amd.executeQueueDrain();
 }
 
 /**
@@ -80,7 +92,7 @@ function getDiagnostics(program: ts.Program): ReadonlyArray<ts.Diagnostic> {
 const EXTENSIONS = [".ts", ".js"];
 
 export class CompilerHost {
-  constructor(public cwd: string) {}
+  constructor() {}
 
   getSourceFile(
     fileName: string,
@@ -90,16 +102,14 @@ export class CompilerHost {
   ): ts.SourceFile | undefined {
     let sourceText: string;
     if (fileName === "lib.d.ts") {
-      // TODO this should be compiled into the bindata.
-      sourceText = readFileSync("node_modules/typescript/lib/lib.d.ts");
+      // TODO This should be compiled into the bindata.
+      sourceText = os.readFileSync("node_modules/typescript/lib/lib.d.ts");
     } else {
-      sourceText = readFileSync(fileName);
+      sourceText = os.readFileSync(fileName);
     }
+    // fileName = fileName.replace(/\.\w+$/, ""); // Remove extension.
     if (sourceText) {
-      log("getSourceFile", {
-        fileName
-        //sourceText,
-      });
+      log("getSourceFile", { fileName });
       return ts.createSourceFile(fileName, sourceText, languageVersion);
     } else {
       log("getSourceFile NOT FOUND", { fileName });
@@ -134,13 +144,18 @@ export class CompilerHost {
     onError: ((message: string) => void) | undefined,
     sourceFiles: ReadonlyArray<ts.SourceFile>
   ): void {
-    log("writeFile", { fileName, data });
+    //log("writeFile", { fileName, data });
+
+    os.compileOutput(data, fileName);
+
+    _global["define"] = amd.makeDefine(fileName);
     globalEval(data);
+    _global["define"] = null;
   }
 
   getCurrentDirectory(): string {
-    log("getCurrentDirectory", this.cwd);
-    return this.cwd;
+    log("getCurrentDirectory", ".");
+    return ".";
   }
 
   getDirectories(path: string): string[] {
@@ -165,7 +180,7 @@ export class CompilerHost {
     containingFile: string,
     reusedNames?: string[]
   ): Array<ts.ResolvedModule | undefined> {
-    log("resolveModuleNames", { moduleNames, reusedNames });
+    //log("resolveModuleNames", { moduleNames, reusedNames });
     return moduleNames.map((name: string) => {
       if (
         name.startsWith("/") ||
@@ -177,7 +192,7 @@ export class CompilerHost {
         // Relative import.
         const containingDir = path.dirname(containingFile);
         const resolvedFileName = path.join(containingDir, name);
-        log("relative import", { containingFile, name, resolvedFileName });
+        //log("relative import", { containingFile, name, resolvedFileName });
         const isExternalLibraryImport = false;
         return { resolvedFileName, isExternalLibraryImport };
       }

main.go

@@ -5,8 +5,28 @@ import (
 	"github.com/ry/v8worker2"
 	"io/ioutil"
 	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
 )
 
+func HandleCompileOutput(source string, filename string) []byte {
+	// println("compile output from golang", filename)
+	// Remove any ".." elements. This prevents hacking by trying to move up.
+	filename, err := filepath.Rel("/", filename)
+	check(err)
+	if strings.Contains(filename, "..") {
+		panic("Assertion error.")
+	}
+	filename = path.Join(CompileDir, filename)
+	err = os.MkdirAll(path.Dir(filename), 0700)
+	check(err)
+	err = ioutil.WriteFile(filename, []byte(source), 0600)
+	check(err)
+	return nil
+}
+
 func ReadFileSync(filename string) []byte {
 	buf, err := ioutil.ReadFile(filename)
 	msg := &Msg{Kind: Msg_DATA_RESPONSE}
@@ -16,23 +36,60 @@ func ReadFileSync(filename string) []byte {
 		msg.Data = buf
 	}
 	out, err := proto.Marshal(msg)
-	if err != nil {
-		panic(err)
-	}
+	check(err)
 	return out
 }
 
+func UserHomeDir() string {
+	if runtime.GOOS == "windows" {
+		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+		if home == "" {
+			home = os.Getenv("USERPROFILE")
+		}
+		return home
+	}
+	return os.Getenv("HOME")
+}
+
+func loadAsset(w *v8worker2.Worker, path string) {
+	data, err := Asset(path)
+	check(err)
+	err = w.Load(path, string(data))
+	check(err)
+}
+
+var DenoDir string
+var CompileDir string
+var SrcDir string
+
+func createDirs() {
+	DenoDir = path.Join(UserHomeDir(), ".deno")
+	CompileDir = path.Join(DenoDir, "compile")
+	err := os.MkdirAll(CompileDir, 0700)
+	check(err)
+	SrcDir = path.Join(DenoDir, "src")
+	err = os.MkdirAll(SrcDir, 0700)
+	check(err)
+}
+
+func check(e error) {
+	if e != nil {
+		panic(e)
+	}
+}
+
 func recv(buf []byte) []byte {
 	msg := &Msg{}
 	err := proto.Unmarshal(buf, msg)
-	if err != nil {
-		panic(err)
-	}
+	check(err)
 	switch msg.Kind {
 	case Msg_READ_FILE_SYNC:
 		return ReadFileSync(msg.Path)
 	case Msg_EXIT:
 		os.Exit(int(msg.Code))
+	case Msg_COMPILE_OUTPUT:
+		payload := msg.GetCompileOutput()
+		return HandleCompileOutput(payload.Source, payload.Filename)
 	default:
 		panic("Unexpected message")
 	}
@@ -40,33 +97,24 @@ func recv(buf []byte) []byte {
 	return nil
 }
 
-func loadAsset(w *v8worker2.Worker, path string) {
-	data, err := Asset(path)
-	if err != nil {
-		panic("asset not found")
-	}
-	err = w.Load(path, string(data))
-	if err != nil {
-		panic(err)
-	}
-}
-
 func main() {
 	args := v8worker2.SetFlags(os.Args)
+	createDirs()
 	worker := v8worker2.New(recv)
 	loadAsset(worker, "dist/main.js")
 	cwd, err := os.Getwd()
-	if err != nil {
-		panic(err)
-	}
+	check(err)
+
 	out, err := proto.Marshal(&Msg{
 		Kind: Msg_START,
-		Cwd:  cwd,
-		Argv: args,
+		Payload: &Msg_Start{
+			Start: &StartMsg{
+				Cwd:  cwd,
+				Argv: args,
+			},
+		},
 	})
-	if err != nil {
-		panic(err)
-	}
+	check(err)
 	err = worker.SendBytes(out)
 	if err != nil {
 		os.Stderr.WriteString(err.Error())

main.ts

@@ -12,7 +12,7 @@ V8Worker2.recv((ab: ArrayBuffer) => {
   const msg = pb.Msg.decode(new Uint8Array(ab));
   switch (msg.kind) {
     case pb.Msg.MsgKind.START:
-      start(msg.cwd, msg.argv);
+      start(msg.start.cwd, msg.start.argv);
       break;
     default:
       console.log("Unknown message", msg);

msg.proto

@@ -7,14 +7,16 @@ message Msg {
     READ_FILE_SYNC = 1;
     DATA_RESPONSE = 2;
     EXIT = 3;
+    COMPILE_OUTPUT = 4;
   }
   MsgKind kind = 10;
 
-  // START
-  string cwd = 11;
-  repeated string argv = 12;
+  oneof payload {
+    StartMsg start = 90;
+    CompileOutputMsg compile_output = 100;
+  }
 
-  // READ_FILE_SYNC
+  // READ_FILE_SYNC and MKDIRP
   string path = 20;
 
   // DATA_RESPONSE
@@ -24,3 +26,15 @@ message Msg {
   // EXIT
   int32 code = 40;
 }
+
+// START
+message StartMsg {
+  string cwd = 1;
+  repeated string argv = 2;
+}
+
+// WRITE_COMPILE_OUTPUT
+message CompileOutputMsg {
+  string source = 1;
+  string filename = 2;
+}

os.ts

@@ -11,6 +11,13 @@ export function exit(code = 0): void {
   });
 }
 
+export function compileOutput(source: string, filename: string): void {
+  sendMsgFromObject({
+    kind: pb.Msg.MsgKind.COMPILE_OUTPUT,
+    compileOutput: { source, filename }
+  });
+}
+
 export function readFileSync(filename: string): string {
   const res = sendMsgFromObject({
     kind: pb.Msg.MsgKind.READ_FILE_SYNC,

tslint.json

@@ -35,7 +35,7 @@
     "no-debugger": true,
     "no-default-export": true,
     "no-inferrable-types": true,
-    "no-namespace": [true, "allow-declarations"],
+    //"no-namespace": [true, "allow-declarations"],
     "no-reference": true,
     "no-require-imports": true,
     "no-string-throw": true,

util.ts

@@ -6,12 +6,13 @@
 export const globalEval = eval;
 
 // A reference to the global object.
-const _global = globalEval("this");
+// TODO The underscore is because it's conflicting with @types/node.
+export const _global = globalEval("this");
 
 const print = V8Worker2.print;
 
 // To control internal logging output
-const debug = true;
+const debug = false;
 
 // Internal logging for deno. Use the "debug" variable above to control
 // output.
@@ -49,6 +50,6 @@ function stringifyArgs(args: any[]): string {
 
 export function assert(cond: boolean, msg = "") {
   if (!cond) {
-    throw Error("Assertion failed. " + msg);
+    throw Error("Assert fail. " + msg);
   }
 }