1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-28 16:20:57 -05:00

runtime.ts - first pass at caching compiler

This commit is contained in:
Ryan Dahl 2018-05-17 09:47:09 -04:00
parent 6f9c919f41
commit 05672b7e24
8 changed files with 339 additions and 338 deletions

View file

@ -1,16 +1,16 @@
TS_FILES = \
amd.ts \
main.ts \
msg.pb.d.ts \
msg.pb.js \
compiler.ts \
msg.pb.d.ts \
os.ts \
util.ts
os.ts \
runtime.ts \
util.ts
deno: assets.go msg.pb.go main.go
go build -o deno
assets.go: dist/main.js
cp node_modules/typescript/lib/lib.d.ts dist/
go-bindata -pkg main -o assets.go dist/
msg.pb.go: msg.proto

80
amd.ts
View file

@ -1,80 +0,0 @@
import * as path from "path";
import { assert, log } from "./util";
namespace ModuleExportsCache {
const cache = new Map<string, object>();
export function set(fileName: string, moduleExports: object) {
fileName = normalizeModuleName(fileName);
assert(
fileName.startsWith("/"),
`Normalized modules should start with /\n${fileName}`
);
log("ModuleExportsCache set", fileName);
cache.set(fileName, moduleExports);
}
export function get(fileName: string): object {
fileName = normalizeModuleName(fileName);
log("ModuleExportsCache get", fileName);
let moduleExports = cache.get(fileName);
if (moduleExports == null) {
moduleExports = {};
set(fileName, moduleExports);
}
return moduleExports;
}
}
function normalizeModuleName(fileName: string): string {
// Remove the extension.
return fileName.replace(/\.\w+$/, "");
}
function normalizeRelativeModuleName(contextFn: string, depFn: string): string {
if (depFn.startsWith("/")) {
return depFn;
} else {
return path.resolve(path.dirname(contextFn), depFn);
}
}
const executeQueue: Array<() => void> = [];
export function executeQueueDrain(): void {
let fn;
while ((fn = executeQueue.shift())) {
fn();
}
}
// tslint:disable-next-line:no-any
type AmdFactory = (...args: any[]) => undefined | object;
type AmdDefine = (deps: string[], factory: AmdFactory) => void;
export function makeDefine(fileName: string): AmdDefine {
const localDefine = (deps: string[], factory: AmdFactory): void => {
const localRequire = (x: string) => {
log("localRequire", x);
};
const localExports = ModuleExportsCache.get(fileName);
log("localDefine", fileName, deps, localExports);
const args = deps.map(dep => {
if (dep === "require") {
return localRequire;
} else if (dep === "exports") {
return localExports;
} else {
dep = normalizeRelativeModuleName(fileName, dep);
return ModuleExportsCache.get(dep);
}
});
executeQueue.push(() => {
log("execute", fileName);
const r = factory(...args);
if (r != null) {
ModuleExportsCache.set(fileName, r);
throw Error("x");
}
});
};
return localDefine;
}

View file

@ -1,221 +0,0 @@
import * as ts from "typescript";
import { log, assert, globalEval, _global } from "./util";
import * as os from "./os";
import * as path from "path";
import * as amd from "./amd";
/*
export function makeCacheDir(): string {
let cacheDir = path.join(env.HOME, ".deno/cache")
os.mkdirp(cacheDir);
return cacheDir
}
*/
export function compile(cwd: string, inputFn: string): void {
const options: ts.CompilerOptions = {
allowJs: true,
module: ts.ModuleKind.AMD,
outDir: "/" // Will be placed in ~/.deno/compile
};
const host = new CompilerHost();
const inputExt = path.extname(inputFn);
if (!EXTENSIONS.includes(inputExt)) {
console.error(`Bad file name extension for input "${inputFn}"`);
os.exit(1);
}
const program = ts.createProgram([inputFn], options, host);
//let sourceFiles = program.getSourceFiles();
//log("rootFileNames", program.getRootFileNames());
// Print compilation errors, if any.
const diagnostics = getDiagnostics(program);
if (diagnostics.length > 0) {
const errorMessages = diagnostics.map(d => formatDiagnostic(d, cwd));
for (const msg of errorMessages) {
console.error(msg);
}
os.exit(2);
}
const emitResult = program.emit();
assert(!emitResult.emitSkipped);
log("emitResult", emitResult);
amd.executeQueueDrain();
}
/**
* Format a diagnostic object into a string.
* Adapted from TS-Node https://github.com/TypeStrong/ts-node
* which uses the same MIT license as this file but is
* Copyright (c) 2014 Blake Embrey (hello@blakeembrey.com)
*/
export function formatDiagnostic(
diagnostic: ts.Diagnostic,
cwd: string,
lineOffset = 0
): string {
const messageText = ts.flattenDiagnosticMessageText(
diagnostic.messageText,
"\n"
);
const { code } = diagnostic;
if (diagnostic.file) {
const fn = path.relative(cwd, diagnostic.file.fileName);
if (diagnostic.start) {
const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(
diagnostic.start
);
const r = Number(line) + 1 + lineOffset;
const c = Number(character) + 1;
return `${fn} (${r},${c}): ${messageText} (${code})`;
}
return `${fn}: ${messageText} (${code})`;
}
return `${messageText} (${code})`;
}
function getDiagnostics(program: ts.Program): ReadonlyArray<ts.Diagnostic> {
return program
.getOptionsDiagnostics()
.concat(
program.getGlobalDiagnostics(),
program.getSyntacticDiagnostics(),
program.getSemanticDiagnostics(),
program.getDeclarationDiagnostics()
);
}
const EXTENSIONS = [".ts", ".js"];
export class CompilerHost {
constructor() {}
getSourceFile(
fileName: string,
languageVersion: ts.ScriptTarget,
onError?: (message: string) => void,
shouldCreateNewSourceFile?: boolean
): ts.SourceFile | undefined {
let sourceText: string;
if (fileName === "lib.d.ts") {
// TODO This should be compiled into the bindata.
sourceText = os.readFileSync("node_modules/typescript/lib/lib.d.ts");
} else {
sourceText = os.readFileSync(fileName);
}
// fileName = fileName.replace(/\.\w+$/, ""); // Remove extension.
if (sourceText) {
log("getSourceFile", { fileName });
return ts.createSourceFile(fileName, sourceText, languageVersion);
} else {
log("getSourceFile NOT FOUND", { fileName });
return undefined;
}
}
getSourceFileByPath?(
fileName: string,
path: ts.Path,
languageVersion: ts.ScriptTarget,
onError?: (message: string) => void,
shouldCreateNewSourceFile?: boolean
): ts.SourceFile | undefined {
console.log("getSourceFileByPath", fileName);
return undefined;
}
// getCancellationToken?(): CancellationToken;
getDefaultLibFileName(options: ts.CompilerOptions): string {
return ts.getDefaultLibFileName(options);
}
getDefaultLibLocation(): string {
return "/blah/";
}
writeFile(
fileName: string,
data: string,
writeByteOrderMark: boolean,
onError: ((message: string) => void) | undefined,
sourceFiles: ReadonlyArray<ts.SourceFile>
): void {
//log("writeFile", { fileName, data });
os.compileOutput(data, fileName);
_global["define"] = amd.makeDefine(fileName);
globalEval(data);
_global["define"] = null;
}
getCurrentDirectory(): string {
log("getCurrentDirectory", ".");
return ".";
}
getDirectories(path: string): string[] {
log("getDirectories", path);
return [];
}
getCanonicalFileName(fileName: string): string {
return fileName;
}
useCaseSensitiveFileNames(): boolean {
return true;
}
getNewLine(): string {
return "\n";
}
resolveModuleNames(
moduleNames: string[],
containingFile: string,
reusedNames?: string[]
): Array<ts.ResolvedModule | undefined> {
//log("resolveModuleNames", { moduleNames, reusedNames });
return moduleNames.map((name: string) => {
if (
name.startsWith("/") ||
name.startsWith("http://") ||
name.startsWith("https://")
) {
throw Error("Non-relative imports not yet supported.");
} else {
// Relative import.
const containingDir = path.dirname(containingFile);
const resolvedFileName = path.join(containingDir, name);
//log("relative import", { containingFile, name, resolvedFileName });
const isExternalLibraryImport = false;
return { resolvedFileName, isExternalLibraryImport };
}
});
}
fileExists(fileName: string): boolean {
log("fileExists", fileName);
return false;
}
readFile(fileName: string): string | undefined {
log("readFile", fileName);
return undefined;
}
/**
* This method is a companion for 'resolveModuleNames' and is used to resolve
* 'types' references to actual type declaration files
*/
// resolveTypeReferenceDirectives?(typeReferenceDirectiveNames: string[],
// containingFile: string): (ResolvedTypeReferenceDirective | undefined)[];
// getEnvironmentVariable?(name: string): string
// createHash?(data: string): string;
}

54
main.go
View file

@ -1,29 +1,48 @@
package main
import (
"crypto/md5"
"encoding/hex"
"github.com/golang/protobuf/proto"
"github.com/ry/v8worker2"
"io/ioutil"
"os"
"path"
"path/filepath"
"runtime"
"strings"
)
func HandleCompileOutput(source string, filename string) []byte {
// println("compile output from golang", filename)
// Remove any ".." elements. This prevents hacking by trying to move up.
filename, err := filepath.Rel("/", filename)
check(err)
if strings.Contains(filename, "..") {
panic("Assertion error.")
func SourceCodeHash(filename string, sourceCodeBuf []byte) string {
h := md5.New()
h.Write([]byte(filename))
h.Write(sourceCodeBuf)
return hex.EncodeToString(h.Sum(nil))
}
func HandleSourceCodeFetch(filename string) []byte {
res := &Msg{Kind: Msg_SOURCE_CODE_FETCH_RES}
sourceCodeBuf, err := Asset("dist/" + filename)
if err != nil {
sourceCodeBuf, err = ioutil.ReadFile(filename)
}
filename = path.Join(CompileDir, filename)
err = os.MkdirAll(path.Dir(filename), 0700)
check(err)
err = ioutil.WriteFile(filename, []byte(source), 0600)
if err != nil {
res.Error = err.Error()
} else {
cacheKey := SourceCodeHash(filename, sourceCodeBuf)
println("cacheKey", filename, cacheKey)
// TODO For now don't do any cache lookups..
res.Payload = &Msg_SourceCodeFetchRes{
SourceCodeFetchRes: &SourceCodeFetchResMsg{
SourceCode: string(sourceCodeBuf),
OutputCode: "",
},
}
}
out, err := proto.Marshal(res)
check(err)
return out
}
func HandleSourceCodeCache(filename string, sourceCode string, outputCode string) []byte {
return nil
}
@ -87,9 +106,12 @@ func recv(buf []byte) []byte {
return ReadFileSync(msg.Path)
case Msg_EXIT:
os.Exit(int(msg.Code))
case Msg_COMPILE_OUTPUT:
payload := msg.GetCompileOutput()
return HandleCompileOutput(payload.Source, payload.Filename)
case Msg_SOURCE_CODE_FETCH:
payload := msg.GetSourceCodeFetch()
return HandleSourceCodeFetch(payload.Filename)
case Msg_SOURCE_CODE_CACHE:
payload := msg.GetSourceCodeCache()
return HandleSourceCodeCache(payload.Filename, payload.SourceCode, payload.OutputCode)
default:
panic("Unexpected message")
}

View file

@ -1,11 +1,14 @@
import { main as pb } from "./msg.pb";
import "./util";
import { compile } from "./compiler";
import * as runtime from "./runtime";
import * as path from "path";
function start(cwd: string, argv: string[]): void {
// TODO parse arguments.
const inputFn = argv[1];
compile(cwd, inputFn);
const fn = path.resolve(cwd, inputFn);
const m = runtime.FileModule.load(fn);
m.compileAndRun();
}
V8Worker2.recv((ab: ArrayBuffer) => {

View file

@ -7,13 +7,18 @@ message Msg {
READ_FILE_SYNC = 1;
DATA_RESPONSE = 2;
EXIT = 3;
COMPILE_OUTPUT = 4;
SOURCE_CODE_FETCH = 4;
SOURCE_CODE_FETCH_RES = 5;
SOURCE_CODE_CACHE = 6;
}
MsgKind kind = 10;
oneof payload {
StartMsg start = 90;
CompileOutputMsg compile_output = 100;
SourceCodeFetchMsg source_code_fetch = 91;
SourceCodeFetchResMsg source_code_fetch_res = 92;
SourceCodeCacheMsg source_code_cache = 93;
}
// READ_FILE_SYNC and MKDIRP
@ -33,8 +38,15 @@ message StartMsg {
repeated string argv = 2;
}
// WRITE_COMPILE_OUTPUT
message CompileOutputMsg {
string source = 1;
string filename = 2;
message SourceCodeFetchMsg { string filename = 1; }
message SourceCodeFetchResMsg {
string source_code = 1;
string output_code = 2;
}
message SourceCodeCacheMsg {
string filename = 1;
string source_code = 2;
string output_code = 3;
}

37
os.ts
View file

@ -11,11 +11,27 @@ export function exit(code = 0): void {
});
}
export function compileOutput(source: string, filename: string): void {
sendMsgFromObject({
kind: pb.Msg.MsgKind.COMPILE_OUTPUT,
compileOutput: { source, filename }
export function sourceCodeFetch(
filename: string
): { sourceCode: string; outputCode: string } {
const res = sendMsgFromObject({
kind: pb.Msg.MsgKind.SOURCE_CODE_FETCH,
sourceCodeFetch: { filename }
});
const { sourceCode, outputCode } = res.sourceCodeFetchRes;
return { sourceCode, outputCode };
}
export function sourceCodeCache(
filename: string,
sourceCode: string,
outputCode: string
): void {
const res = sendMsgFromObject({
kind: pb.Msg.MsgKind.SOURCE_CODE_CACHE,
sourceCodeCache: { filename, sourceCode, outputCode }
});
throwOnError(res);
}
export function readFileSync(filename: string): string {
@ -23,9 +39,6 @@ export function readFileSync(filename: string): string {
kind: pb.Msg.MsgKind.READ_FILE_SYNC,
path: filename
});
if (res.error != null && res.error.length > 0) {
throw Error(res.error);
}
const decoder = new TextDecoder("utf8");
return decoder.decode(res.data);
}
@ -41,8 +54,16 @@ function sendMsgFromObject(obj: pb.IMsg): null | pb.Msg {
const ab = typedArrayToArrayBuffer(ui8);
const resBuf = V8Worker2.send(ab);
if (resBuf != null && resBuf.byteLength > 0) {
return pb.Msg.decode(new Uint8Array(resBuf));
const res = pb.Msg.decode(new Uint8Array(resBuf));
throwOnError(res);
return res;
} else {
return null;
}
}
function throwOnError(res: pb.Msg) {
if (res != null && res.error != null && res.error.length > 0) {
throw Error(res.error);
}
}

244
runtime.ts Normal file
View file

@ -0,0 +1,244 @@
// Glossary
// outputCode = generated javascript code
// sourceCode = typescript code (or input javascript code)
// fileName = an unresolved raw fileName.
// moduleName = a resolved module name
import * as ts from "typescript";
import * as path from "path";
import * as util from "./util";
import { log } from "./util";
import * as os from "./os";
// This class represents a module. We call it FileModule to make it explicit
// that each module represents a single file.
// Access to FileModule instances should only be done thru the static method
// FileModule.load(). FileModules are executed upon first load.
export class FileModule {
scriptVersion: string = undefined;
sourceCode: string;
outputCode: string;
readonly exports = {};
private static readonly map = new Map<string, FileModule>();
private constructor(readonly fileName: string) {
FileModule.map.set(fileName, this);
assertValidFileName(this.fileName);
// Load typescript code (sourceCode) and maybe load compiled javascript
// (outputCode) from cache. If cache is empty, outputCode will be null.
const { sourceCode, outputCode } = os.sourceCodeFetch(this.fileName);
this.sourceCode = sourceCode;
this.outputCode = outputCode;
this.scriptVersion = "1";
}
compileAndRun() {
if (!this.outputCode) {
// If there is no cached outputCode, the compile the code.
util.assert(this.sourceCode && this.sourceCode.length > 0);
const compiler = Compiler.instance();
this.outputCode = compiler.compile(this.fileName);
os.sourceCodeCache(this.fileName, this.sourceCode, this.outputCode);
}
util.log("compileAndRun", this.sourceCode);
execute(this.fileName, this.outputCode);
}
static load(fileName: string): FileModule {
assertValidFileName(fileName);
let m = this.map.get(fileName);
if (m == null) {
m = new this(fileName);
util.assert(this.map.has(fileName));
}
return m;
}
static getScriptsWithSourceCode(): string[] {
const out = [];
for (const fn of this.map.keys()) {
const m = this.map.get(fn);
if (m.sourceCode) {
out.push(fn);
}
}
return out;
}
}
function assertValidFileName(fileName: string): void {
if (fileName !== "lib.d.ts") {
util.assert(fileName[0] === "/", `fileName must be absolute: ${fileName}`);
}
}
// tslint:disable-next-line:no-any
type AmdFactory = (...args: any[]) => undefined | object;
type AmdDefine = (deps: string[], factory: AmdFactory) => void;
export function makeDefine(fileName: string): AmdDefine {
const localDefine = (deps: string[], factory: AmdFactory): void => {
const localRequire = (x: string) => {
log("localRequire", x);
};
const currentModule = FileModule.load(fileName);
const localExports = currentModule.exports;
log("localDefine", fileName, deps, localExports);
const args = deps.map(dep => {
if (dep === "require") {
return localRequire;
} else if (dep === "exports") {
return localExports;
} else {
dep = resolveModuleName(dep, fileName);
const depModule = FileModule.load(dep);
depModule.compileAndRun();
return depModule.exports;
}
});
factory(...args);
};
return localDefine;
}
function resolveModuleName(fileName: string, contextFileName: string): string {
return path.resolve(path.dirname(contextFileName), fileName);
}
function execute(fileName: string, outputCode: string): void {
util.assert(outputCode && outputCode.length > 0);
util._global["define"] = makeDefine(fileName);
util.globalEval(outputCode);
util._global["define"] = null;
}
// This is a singleton class. Use Compiler.instance() to access.
class Compiler {
options: ts.CompilerOptions = {
allowJs: true,
module: ts.ModuleKind.AMD,
outDir: "$deno$"
};
/*
allowJs: true,
inlineSourceMap: true,
inlineSources: true,
module: ts.ModuleKind.AMD,
noEmit: false,
outDir: '$deno$',
*/
private service: ts.LanguageService;
private constructor() {
const host = new TypeScriptHost(this.options);
this.service = ts.createLanguageService(host);
}
private static _instance: Compiler;
static instance(): Compiler {
return this._instance || (this._instance = new this());
}
compile(fileName: string): string {
const output = this.service.getEmitOutput(fileName);
// Get the relevant diagnostics - this is 3x faster than
// `getPreEmitDiagnostics`.
const diagnostics = this.service
.getCompilerOptionsDiagnostics()
.concat(this.service.getSyntacticDiagnostics(fileName))
.concat(this.service.getSemanticDiagnostics(fileName));
if (diagnostics.length > 0) {
throw Error("diagnotics");
}
util.log("compile output", output);
util.assert(!output.emitSkipped);
const outputCode = output.outputFiles[0].text;
// let sourceMapCode = output.outputFiles[0].text;
return outputCode;
}
}
// Create the compiler host for type checking.
class TypeScriptHost implements ts.LanguageServiceHost {
constructor(readonly options: ts.CompilerOptions) {}
getScriptFileNames(): string[] {
const keys = FileModule.getScriptsWithSourceCode();
util.log("getScriptFileNames", keys);
return keys;
}
getScriptVersion(fileName: string): string {
util.log("getScriptVersion", fileName);
const m = FileModule.load(fileName);
return m.scriptVersion;
}
getScriptSnapshot(fileName: string): ts.IScriptSnapshot | undefined {
util.log("getScriptSnapshot", fileName);
const m = FileModule.load(fileName);
if (m.sourceCode) {
return ts.ScriptSnapshot.fromString(m.sourceCode);
} else {
return undefined;
}
}
fileExists(fileName: string): boolean {
throw Error("not implemented");
}
readFile(path: string, encoding?: string): string | undefined {
util.log("readFile", path);
throw Error("not implemented");
}
getNewLine() {
const EOL = "\n";
return EOL;
}
getCurrentDirectory() {
util.log("getCurrentDirectory");
return ".";
}
getCompilationSettings() {
util.log("getCompilationSettings");
return this.options;
}
getDefaultLibFileName(options: ts.CompilerOptions): string {
util.log("getDefaultLibFileName");
return ts.getDefaultLibFileName(options);
}
resolveModuleNames(
moduleNames: string[],
containingFile: string,
reusedNames?: string[]
): Array<ts.ResolvedModule | undefined> {
util.log("resolveModuleNames", { moduleNames, reusedNames });
return moduleNames.map((name: string) => {
if (
name.startsWith("/") ||
name.startsWith("http://") ||
name.startsWith("https://")
) {
throw Error("Non-relative imports not yet supported.");
} else {
// Relative import.
const containingDir = path.dirname(containingFile);
const resolvedFileName = path.join(containingDir, name);
util.log("relative import", { containingFile, name, resolvedFileName });
const isExternalLibraryImport = false;
return { resolvedFileName, isExternalLibraryImport };
}
});
}
}