foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-21 15:04:11 -05:00
Code Issues Packages 1 Wiki Activity

Remote module caching

Browse source
This commit is contained in:
Ryan Dahl 2018-05-19 04:47:40 -04:00
parent 487bf4e1ac
commit aeb85efdad
6 changed files with 178 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Makefile
View file

@ -49,5 +49,6 @@ fmt: node_modules
test: deno
node test.js
go test
.PHONY: test lint clean distclean

153
main.go
View file

@ -5,8 +5,10 @@ import (
"encoding/hex"
"github.com/golang/protobuf/proto"
"github.com/ry/v8worker2"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"runtime"
@ -15,6 +17,10 @@ import (
"time"
)
var DenoDir string
var CompileDir string
var SrcDir string
var wg sync.WaitGroup
var resChan chan *Msg
@ -30,59 +36,126 @@ func CacheFileName(filename string, sourceCodeBuf []byte) string {
return path.Join(CompileDir, cacheKey+".js")
}
func IsRemotePath(filename string) bool {
return strings.HasPrefix(filename, "/$remote$/")
func IsRemote(filename string) bool {
u, err := url.Parse(filename)
check(err)
return u.IsAbs()
}
func FetchRemoteSource(remotePath string) (buf []byte, err error) {
url := strings.Replace(remotePath, "/$remote$/", "http://", 1)
// println("FetchRemoteSource", url)
res, err := http.Get(url)
// Fetches a remoteUrl but also caches it to the localFilename.
func FetchRemoteSource(remoteUrl string, localFilename string) ([]byte, error) {
Assert(strings.HasPrefix(localFilename, SrcDir), localFilename)
var sourceReader io.Reader
file, err := os.Open(localFilename)
if os.IsNotExist(err) {
// Fetch from HTTP.
res, err := http.Get(remoteUrl)
if err != nil {
return nil, err
}
defer res.Body.Close()
err = os.MkdirAll(path.Dir(localFilename), 0700)
if err != nil {
return nil, err
}
// Write to to file. Need to reopen it for writing.
file, err = os.OpenFile(localFilename, os.O_RDWR|os.O_CREATE, 0700)
if err != nil {
return nil, err
}
sourceReader = io.TeeReader(res.Body, file) // Fancy!
} else if err != nil {
return nil, err
} else {
sourceReader = file
}
defer file.Close()
return ioutil.ReadAll(sourceReader)
}
func ResolveModule(moduleSpecifier string, containingFile string) (
moduleName string, filename string, err error) {
moduleUrl, err := url.Parse(moduleSpecifier)
if err != nil {
return
}
buf, err = ioutil.ReadAll(res.Body)
//println("FetchRemoteSource", err.Error())
res.Body.Close()
baseUrl, err := url.Parse(containingFile)
if err != nil {
return
}
resolved := baseUrl.ResolveReference(moduleUrl)
moduleName = resolved.String()
if moduleUrl.IsAbs() {
filename = path.Join(SrcDir, resolved.Host, resolved.Path)
} else {
filename = resolved.Path
}
return
}
func HandleSourceCodeFetch(filename string) []byte {
const assetPrefix string = "/$asset$/"
func HandleSourceCodeFetch(moduleSpecifier string, containingFile string) (out []byte) {
res := &Msg{}
var sourceCodeBuf []byte
var err error
if IsRemotePath(filename) {
sourceCodeBuf, err = FetchRemoteSource(filename)
} else {
sourceCodeBuf, err = Asset("dist/" + filename)
defer func() {
if err != nil {
sourceCodeBuf, err = ioutil.ReadFile(filename)
res.Error = err.Error()
}
out, err = proto.Marshal(res)
check(err)
}()
moduleName, filename, err := ResolveModule(moduleSpecifier, containingFile)
if err != nil {
return
}
if IsRemote(moduleName) {
sourceCodeBuf, err = FetchRemoteSource(moduleName, filename)
} else if strings.HasPrefix(moduleName, assetPrefix) {
f := strings.TrimPrefix(moduleName, assetPrefix)
sourceCodeBuf, err = Asset("dist/" + f)
} else {
Assert(moduleName == filename,
"if a module isn't remote, it should have the same filename")
sourceCodeBuf, err = ioutil.ReadFile(moduleName)
}
if err != nil {
res.Error = err.Error()
} else {
cacheFn := CacheFileName(filename, sourceCodeBuf)
outputCodeBuf, err := ioutil.ReadFile(cacheFn)
var outputCode string
if os.IsNotExist(err) {
outputCode = ""
} else if err != nil {
res.Error = err.Error()
} else {
outputCode = string(outputCodeBuf)
}
res.Payload = &Msg_SourceCodeFetchRes{
SourceCodeFetchRes: &SourceCodeFetchResMsg{
SourceCode: string(sourceCodeBuf),
OutputCode: outputCode,
},
}
return
}
out, err := proto.Marshal(res)
check(err)
return out
outputCode, err := LoadOutputCodeCache(filename, sourceCodeBuf)
if err != nil {
return
}
res.Payload = &Msg_SourceCodeFetchRes{
SourceCodeFetchRes: &SourceCodeFetchResMsg{
ModuleName: moduleName,
Filename: filename,
SourceCode: string(sourceCodeBuf),
OutputCode: outputCode,
},
}
return
}
func LoadOutputCodeCache(filename string, sourceCodeBuf []byte) (outputCode string, err error) {
cacheFn := CacheFileName(filename, sourceCodeBuf)
outputCodeBuf, err := ioutil.ReadFile(cacheFn)
if os.IsNotExist(err) {
err = nil // Ignore error if we can't load the cache.
} else if err != nil {
outputCode = string(outputCodeBuf)
}
return
}
func HandleSourceCodeCache(filename string, sourceCode string,
@ -134,10 +207,6 @@ func loadAsset(w *v8worker2.Worker, path string) {
check(err)
}
var DenoDir string
var CompileDir string
var SrcDir string
func createDirs() {
DenoDir = path.Join(UserHomeDir(), ".deno")
CompileDir = path.Join(DenoDir, "compile")
@ -164,7 +233,7 @@ func recv(buf []byte) []byte {
os.Exit(int(payload.Code))
case *Msg_SourceCodeFetch:
payload := msg.GetSourceCodeFetch()
return HandleSourceCodeFetch(payload.Filename)
return HandleSourceCodeFetch(payload.ModuleSpecifier, payload.ContainingFile)
case *Msg_SourceCodeCache:
payload := msg.GetSourceCodeCache()
return HandleSourceCodeCache(payload.Filename, payload.SourceCode,

6
main.ts
View file

@ -2,14 +2,12 @@ import { main as pb } from "./msg.pb";
import "./util";
import * as runtime from "./runtime";
import * as timers from "./timers";
import * as path from "path";
function start(cwd: string, argv: string[]): void {
// TODO parse arguments.
const inputFn = argv[1];
const fn = path.resolve(cwd, inputFn);
const m = runtime.FileModule.load(fn);
m.compileAndRun();
const mod = runtime.resolveModule(inputFn, cwd + "/");
mod.compileAndRun();
}
V8Worker2.recv((ab: ArrayBuffer) => {

16
msg.proto
View file

@ -3,7 +3,6 @@ package main;
message Msg {
string error = 1;
oneof payload {
StartMsg start = 10;
SourceCodeFetchMsg source_code_fetch = 11;
@ -15,17 +14,24 @@ message Msg {
}
}
// START
message StartMsg {
string cwd = 1;
repeated string argv = 2;
}
message SourceCodeFetchMsg { string filename = 1; }
message SourceCodeFetchMsg {
string module_specifier = 1;
string containing_file = 2;
}
message SourceCodeFetchResMsg {
string source_code = 1;
string output_code = 2;
// If it's a non-http module, moduleName and filename will be the same.
// For http modules, moduleName is its resolved http URL, and filename
// is the location of the locally downloaded source code.
string moduleName = 1;
string filename = 2;
string source_code = 3;
string output_code = 4; // Non-empty only if cached.
}
message SourceCodeCacheMsg {

14
os.ts
View file

@ -1,7 +1,5 @@
import { main as pb } from "./msg.pb";
// TODO move this to types.ts
type TypedArray = Uint8Array | Float32Array | Int32Array;
import { TypedArray, ModuleInfo } from "./types";
export function exit(code = 0): void {
sendMsgFromObject({
@ -10,13 +8,13 @@ export function exit(code = 0): void {
}
export function sourceCodeFetch(
filename: string
): { sourceCode: string; outputCode: string } {
moduleSpecifier: string,
containingFile: string
): ModuleInfo {
const res = sendMsgFromObject({
sourceCodeFetch: { filename }
sourceCodeFetch: { moduleSpecifier, containingFile }
});
const { sourceCode, outputCode } = res.sourceCodeFetchRes;
return { sourceCode, outputCode };
return res.sourceCodeFetchRes;
}
export function sourceCodeCache(

102
runtime.ts
View file

@ -1,11 +1,12 @@
// Glossary
// outputCode = generated javascript code
// sourceCode = typescript code (or input javascript code)
// fileName = an unresolved raw fileName.
// moduleName = a resolved module name
// fileName = an unresolved raw fileName.
// for http modules , its the path to the locally downloaded
// version.
import * as ts from "typescript";
import * as path from "path";
import * as util from "./util";
import { log } from "./util";
import * as os from "./os";
@ -16,29 +17,31 @@ const EOL = "\n";
// This class represents a module. We call it FileModule to make it explicit
// that each module represents a single file.
// Access to FileModule instances should only be done thru the static method
// FileModule.load(). FileModules are executed upon first load.
// FileModule.load(). FileModules are NOT executed upon first load, only when
// compileAndRun is called.
export class FileModule {
scriptVersion: string = undefined;
sourceCode: string;
outputCode: string;
readonly exports = {};
private static readonly map = new Map<string, FileModule>();
private constructor(readonly fileName: string) {
constructor(
readonly fileName: string,
readonly sourceCode = "",
public outputCode = ""
) {
FileModule.map.set(fileName, this);
// Load typescript code (sourceCode) and maybe load compiled javascript
// (outputCode) from cache. If cache is empty, outputCode will be null.
const { sourceCode, outputCode } = os.sourceCodeFetch(this.fileName);
this.sourceCode = sourceCode;
this.outputCode = outputCode;
this.scriptVersion = "1";
if (outputCode !== "") {
this.scriptVersion = "1";
}
}
compileAndRun() {
compileAndRun(): void {
if (!this.outputCode) {
// If there is no cached outputCode, the compile the code.
util.assert(this.sourceCode && this.sourceCode.length > 0);
util.assert(
this.sourceCode != null && this.sourceCode.length > 0,
`Have no source code from ${this.fileName}`
);
const compiler = Compiler.instance();
this.outputCode = compiler.compile(this.fileName);
os.sourceCodeCache(this.fileName, this.sourceCode, this.outputCode);
@ -48,12 +51,7 @@ export class FileModule {
}
static load(fileName: string): FileModule {
let m = this.map.get(fileName);
if (m == null) {
m = new this(fileName);
util.assert(this.map.has(fileName));
}
return m;
return this.map.get(fileName);
}
static getScriptsWithSourceCode(): string[] {
@ -97,26 +95,26 @@ export function makeDefine(fileName: string): AmdDefine {
return localDefine;
}
function resolveModuleName(moduleName: string, containingFile: string): string {
if (isUrl(moduleName)) {
// Remove the "http://" from the start of the string.
const u = new URL(moduleName);
const withoutProtocol = u.toString().replace(u.protocol + "//", "");
const name2 = "/$remote$/" + withoutProtocol;
return name2;
} else if (moduleName.startsWith("/")) {
throw Error("Absolute paths not supported");
} else {
// Relative import.
const containingDir = path.dirname(containingFile);
const resolvedFileName = path.join(containingDir, moduleName);
util.log("relative import", {
containingFile,
moduleName,
resolvedFileName
});
return resolvedFileName;
}
export function resolveModule(
moduleSpecifier: string,
containingFile: string
): FileModule {
// We ask golang to sourceCodeFetch. It will load the sourceCode and if
// there is any outputCode cached, it will return that as well.
const { filename, sourceCode, outputCode } = os.sourceCodeFetch(
moduleSpecifier,
containingFile
);
util.log("resolveModule", { containingFile, moduleSpecifier, filename });
return new FileModule(filename, sourceCode, outputCode);
}
function resolveModuleName(
moduleSpecifier: string,
containingFile: string
): string {
const mod = resolveModule(moduleSpecifier, containingFile);
return mod.fileName;
}
function execute(fileName: string, outputCode: string): void {
@ -171,7 +169,6 @@ class Compiler {
os.exit(1);
}
util.log("compile output", output);
util.assert(!output.emitSkipped);
const outputCode = output.outputFiles[0].text;
@ -199,15 +196,14 @@ class TypeScriptHost implements ts.LanguageServiceHost {
getScriptSnapshot(fileName: string): ts.IScriptSnapshot | undefined {
util.log("getScriptSnapshot", fileName);
const m = FileModule.load(fileName);
if (m.sourceCode) {
return ts.ScriptSnapshot.fromString(m.sourceCode);
} else {
return undefined;
}
util.assert(m != null);
util.assert(m.sourceCode.length > 0);
return ts.ScriptSnapshot.fromString(m.sourceCode);
}
fileExists(fileName: string): boolean {
throw Error("not implemented");
util.log("fileExist", fileName);
return true;
}
readFile(path: string, encoding?: string): string | undefined {
@ -231,7 +227,9 @@ class TypeScriptHost implements ts.LanguageServiceHost {
getDefaultLibFileName(options: ts.CompilerOptions): string {
util.log("getDefaultLibFileName");
return ts.getDefaultLibFileName(options);
const fn = ts.getDefaultLibFileName(options);
const m = resolveModule(fn, "/$asset$/");
return m.fileName;
}
resolveModuleNames(
@ -248,12 +246,6 @@ class TypeScriptHost implements ts.LanguageServiceHost {
}
}
function isUrl(p: string): boolean {
return (
p.startsWith("//") || p.startsWith("http://") || p.startsWith("https://")
);
}
const formatDiagnosticsHost: ts.FormatDiagnosticsHost = {
getCurrentDirectory(): string {
return ".";