diff --git a/cli/build.rs b/cli/build.rs index 74e35a6323..ce4027d771 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -14,15 +14,29 @@ fn main() { let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let custom_libs = vec![( + "lib.deno_runtime.d.ts".to_string(), + c.join("js/lib.deno_runtime.d.ts"), + )]; + let root_names = vec![c.join("js/main.ts")]; let bundle = o.join("CLI_SNAPSHOT.js"); - let state = deno_typescript::compile_bundle(&bundle, root_names).unwrap(); + let state = + deno_typescript::compile_bundle(&bundle, root_names, Some(custom_libs)) + .unwrap(); assert!(bundle.exists()); deno_typescript::mksnapshot_bundle(&bundle, state).unwrap(); + let custom_libs = vec![( + "lib.deno_runtime.d.ts".to_string(), + c.join("js/lib.deno_runtime.d.ts"), + )]; + let root_names = vec![c.join("js/compiler.ts")]; let bundle = o.join("COMPILER_SNAPSHOT.js"); - let state = deno_typescript::compile_bundle(&bundle, root_names).unwrap(); + let state = + deno_typescript::compile_bundle(&bundle, root_names, Some(custom_libs)) + .unwrap(); assert!(bundle.exists()); deno_typescript::mksnapshot_bundle_ts(&bundle, state).unwrap(); } diff --git a/cli/js.rs b/cli/js.rs index bf13d704d4..8f57adb4db 100644 --- a/cli/js.rs +++ b/cli/js.rs @@ -16,15 +16,7 @@ pub static COMPILER_SNAPSHOT_MAP: &[u8] = pub static COMPILER_SNAPSHOT_DTS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.d.ts")); -static DENO_RUNTIME: &str = include_str!("js/lib.deno_runtime.d.ts"); - -/// Same as deno_typescript::get_asset but also has lib.deno_runtime.d.ts -pub fn get_asset(name: &str) -> Option<&'static str> { - match name { - "lib.deno_runtime.d.ts" => Some(DENO_RUNTIME), - _ => deno_typescript::get_asset(name), - } -} +pub static DENO_RUNTIME: &str = include_str!("js/lib.deno_runtime.d.ts"); #[test] fn cli_snapshot() { diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 3cebf24b26..54861f7138 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -6,6 +6,7 @@ import "./globals.ts"; import "./ts_global.d.ts"; import { TranspileOnlyResult } from "./compiler_api.ts"; +import { oldProgram } from "./compiler_bootstrap.ts"; import { setRootExports } from "./compiler_bundler.ts"; import { defaultBundlerOptions, @@ -142,7 +143,12 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void { // to generate the program and possibly emit it. if (!diagnostics || (diagnostics && diagnostics.length === 0)) { const options = host.getCompilationSettings(); - const program = ts.createProgram(rootNames, options, host); + const program = ts.createProgram({ + rootNames, + options, + host, + oldProgram + }); diagnostics = ts .getPreEmitDiagnostics(program) @@ -220,11 +226,12 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void { } host.mergeOptions(...compilerOptions); - const program = ts.createProgram( + const program = ts.createProgram({ rootNames, - host.getCompilationSettings(), - host - ); + options: host.getCompilationSettings(), + host, + oldProgram + }); if (bundle) { setRootExports(program, rootNames[0]); diff --git a/cli/js/compiler_bootstrap.ts b/cli/js/compiler_bootstrap.ts new file mode 100644 index 0000000000..6de9787505 --- /dev/null +++ b/cli/js/compiler_bootstrap.ts @@ -0,0 +1,34 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +import { ASSETS, Host } from "./compiler_host.ts"; +import { core } from "./core.ts"; +import * as dispatch from "./dispatch.ts"; +import { sendSync } from "./dispatch_json.ts"; + +// This registers ops that are available during the snapshotting process. +const ops = core.ops(); +for (const [name, opId] of Object.entries(ops)) { + const opName = `OP_${name.toUpperCase()}`; + // TODO This type casting is dangerous, and should be improved when the same + // code in `os.ts` is done. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (dispatch as any)[opName] = opId; +} + +const host = new Host({ writeFile(): void {} }); +const options = host.getCompilationSettings(); + +/** Used to generate the foundational AST for all other compilations, so it can + * be cached as part of the snapshot and available to speed up startup */ +export const oldProgram = ts.createProgram({ + rootNames: [`${ASSETS}/bootstrap.ts`], + options, + host +}); + +/** A module loader which is concatenated into bundle files. We read all static + * assets during the snapshotting process, which is why this is located in + * compiler_bootstrap. */ +export const bundleLoader = sendSync(dispatch.OP_FETCH_ASSET, { + name: "bundle_loader.js" +}); diff --git a/cli/js/compiler_bundler.ts b/cli/js/compiler_bundler.ts index a4e4557ca6..8fb68cc7a7 100644 --- a/cli/js/compiler_bundler.ts +++ b/cli/js/compiler_bundler.ts @@ -1,7 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import * as dispatch from "./dispatch.ts"; -import { sendSync } from "./dispatch_json.ts"; +import { bundleLoader } from "./compiler_bootstrap.ts"; import { assert, commonPath, @@ -9,11 +8,6 @@ import { CHAR_FORWARD_SLASH } from "./util.ts"; -const BUNDLE_LOADER = "bundle_loader.js"; - -/** A loader of bundled modules that we will inline into any bundle outputs. */ -let bundleLoader: string; - /** Local state of what the root exports are of a root module. */ let rootExports: string[] | undefined; @@ -40,12 +34,6 @@ export function buildBundle( data: string, sourceFiles: readonly ts.SourceFile[] ): string { - // we can only do this once we are bootstrapped and easiest way to do it is - // inline here - if (!bundleLoader) { - bundleLoader = sendSync(dispatch.OP_FETCH_ASSET, { name: BUNDLE_LOADER }); - } - // when outputting to AMD and a single outfile, TypeScript makes up the module // specifiers which are used to define the modules, and doesn't expose them // publicly, so we have to try to replicate diff --git a/cli/js/compiler_host.ts b/cli/js/compiler_host.ts index f8921a352b..3e6df44856 100644 --- a/cli/js/compiler_host.ts +++ b/cli/js/compiler_host.ts @@ -18,7 +18,7 @@ export interface ConfigureResponse { diagnostics?: ts.Diagnostic[]; } -const ASSETS = "$asset$"; +export const ASSETS = "$asset$"; /** Options that need to be used when generating a bundle (either trusted or * runtime). */ @@ -129,11 +129,11 @@ export class Host implements ts.CompilerHost { private _writeFile: WriteFileCallback; private _getAsset(filename: string): SourceFile { - const sourceFile = SourceFile.get(filename); + const url = filename.split("/").pop()!; + const sourceFile = SourceFile.get(url); if (sourceFile) { return sourceFile; } - const url = filename.split("/").pop()!; const name = url.includes(".") ? url : `${url}.d.ts`; const sourceCode = sendSync(dispatch.OP_FETCH_ASSET, { name }); return new SourceFile({ @@ -238,13 +238,15 @@ export class Host implements ts.CompilerHost { : SourceFile.get(fileName); assert(sourceFile != null); if (!sourceFile.tsSourceFile) { + assert(sourceFile.sourceCode != null); sourceFile.tsSourceFile = ts.createSourceFile( fileName, sourceFile.sourceCode, languageVersion ); + delete sourceFile.sourceCode; } - return sourceFile!.tsSourceFile; + return sourceFile.tsSourceFile; } catch (e) { if (onError) { onError(String(e)); diff --git a/cli/js/compiler_sourcefile.ts b/cli/js/compiler_sourcefile.ts index 087891c99d..8e81cdb456 100644 --- a/cli/js/compiler_sourcefile.ts +++ b/cli/js/compiler_sourcefile.ts @@ -61,7 +61,7 @@ export class SourceFile { mediaType!: MediaType; processed = false; - sourceCode!: string; + sourceCode?: string; tsSourceFile?: ts.SourceFile; url!: string; diff --git a/cli/js/dispatch.ts b/cli/js/dispatch.ts index 073c32f7a8..f5049cca88 100644 --- a/cli/js/dispatch.ts +++ b/cli/js/dispatch.ts @@ -70,12 +70,15 @@ export let OP_TRUNCATE: number; export let OP_MAKE_TEMP_DIR: number; export let OP_CWD: number; export let OP_CONNECT_TLS: number; -export let OP_FETCH_ASSET: number; export let OP_HOSTNAME: number; export let OP_OPEN_PLUGIN: number; export let OP_COMPILE: number; export let OP_TRANSPILE: number; +/** **WARNING:** This is only available during the snapshotting process and is + * unavailable at runtime. */ +export let OP_FETCH_ASSET: number; + const PLUGIN_ASYNC_HANDLER_MAP: Map = new Map(); export function setPluginAsyncHandler( diff --git a/cli/js/dispatch_json.ts b/cli/js/dispatch_json.ts index 104a7e564c..adccb69c6e 100644 --- a/cli/js/dispatch_json.ts +++ b/cli/js/dispatch_json.ts @@ -62,7 +62,7 @@ export function sendSync( const resUi8 = core.dispatch(opId, argsUi8, zeroCopy); util.assert(resUi8 != null); - const res = decode(resUi8!); + const res = decode(resUi8); util.assert(res.promiseId == null); return unwrapResponse(res); } diff --git a/cli/lib.rs b/cli/lib.rs index 263bd8c615..9753e4c4c6 100644 --- a/cli/lib.rs +++ b/cli/lib.rs @@ -146,8 +146,7 @@ fn create_worker_and_state( } fn types_command() { - let content = crate::js::get_asset("lib.deno_runtime.d.ts").unwrap(); - println!("{}", content); + println!("{}", crate::js::DENO_RUNTIME); } fn print_cache_info(worker: MainWorker) { diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index cb2fb01a0e..e515081df4 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -17,10 +17,6 @@ pub fn init(i: &mut Isolate, s: &ThreadSafeState) { "fetch_source_files", s.core_op(json_op(s.stateful_op(op_fetch_source_files))), ); - i.register_op( - "fetch_asset", - s.core_op(json_op(s.stateful_op(op_fetch_asset))), - ); } #[derive(Deserialize)] @@ -149,21 +145,3 @@ fn op_fetch_source_files( Ok(JsonOp::Async(future)) } - -#[derive(Deserialize)] -struct FetchAssetArgs { - name: String, -} - -fn op_fetch_asset( - _state: &ThreadSafeState, - args: Value, - _zero_copy: Option, -) -> Result { - let args: FetchAssetArgs = serde_json::from_value(args)?; - if let Some(source_code) = crate::js::get_asset(&args.name) { - Ok(JsonOp::Sync(json!(source_code))) - } else { - panic!("op_fetch_asset bad asset {}", args.name) - } -} diff --git a/core/isolate.rs b/core/isolate.rs index ebd25b82ad..fc0d5899b1 100644 --- a/core/isolate.rs +++ b/core/isolate.rs @@ -680,6 +680,7 @@ impl Isolate { let mut hs = v8::HandleScope::new(locker.enter()); let scope = hs.enter(); self.global_context.reset(scope); + self.shared_response_buf.reset(scope); let snapshot_creator = self.snapshot_creator.as_mut().unwrap(); let snapshot = snapshot_creator diff --git a/core/shared_queue.js b/core/shared_queue.js index 8ebb19e5e1..093cc223f7 100644 --- a/core/shared_queue.js +++ b/core/shared_queue.js @@ -184,7 +184,6 @@ SharedQueue Binary Layout } function dispatch(opId, control, zeroCopy = null) { - maybeInit(); return Deno.core.send(opId, control, zeroCopy); } diff --git a/deno_typescript/lib.rs b/deno_typescript/lib.rs index 016adcca0c..ccad69a3de 100644 --- a/deno_typescript/lib.rs +++ b/deno_typescript/lib.rs @@ -16,6 +16,7 @@ use deno_core::PinnedBuf; use deno_core::StartupData; pub use ops::EmitResult; use ops::WrittenFile; +use std::collections::HashMap; use std::fs; use std::path::Path; use std::path::PathBuf; @@ -37,6 +38,7 @@ pub struct TSState { bundle: bool, exit_code: i32, emit_result: Option, + custom_assets: HashMap, /// A list of files emitted by typescript. WrittenFile is tuple of the form /// (url, corresponding_module, source_code) written_files: Vec, @@ -76,6 +78,7 @@ impl TSIsolate { let state = Arc::new(Mutex::new(TSState { bundle, + custom_assets: HashMap::new(), exit_code: 0, emit_result: None, written_files: Vec::new(), @@ -119,13 +122,24 @@ impl TSIsolate { self.isolate.execute("", source)?; Ok(self.state) } + + pub fn add_custom_assets(&mut self, custom_assets: Vec<(String, PathBuf)>) { + let mut state = self.state.lock().unwrap(); + for (name, path) in custom_assets { + state.custom_assets.insert(name, path); + } + } } pub fn compile_bundle( bundle: &Path, root_names: Vec, + custom_assets: Option>, ) -> Result>, ErrBox> { - let ts_isolate = TSIsolate::new(true); + let mut ts_isolate = TSIsolate::new(true); + if let Some(assets) = custom_assets { + ts_isolate.add_custom_assets(assets); + } let config_json = serde_json::json!({ "compilerOptions": { @@ -203,6 +217,10 @@ pub fn mksnapshot_bundle_ts( state: Arc>, ) -> Result<(), ErrBox> { let runtime_isolate = &mut Isolate::new(StartupData::None, true); + runtime_isolate.register_op( + "fetch_asset", + compiler_op(state.clone(), ops::json_op(ops::fetch_asset)), + ); let source_code_vec = std::fs::read(bundle)?; let source_code = std::str::from_utf8(&source_code_vec)?; @@ -255,6 +273,7 @@ pub fn get_asset(name: &str) -> Option<&'static str> { } match name { "bundle_loader.js" => Some(include_str!("bundle_loader.js")), + "bootstrap.ts" => Some("console.log(\"hello deno\");"), "typescript.d.ts" => inc!("typescript.d.ts"), "lib.esnext.d.ts" => inc!("lib.esnext.d.ts"), "lib.es2020.d.ts" => inc!("lib.es2020.d.ts"), diff --git a/deno_typescript/ops.rs b/deno_typescript/ops.rs index e45349591c..0680d07b37 100644 --- a/deno_typescript/ops.rs +++ b/deno_typescript/ops.rs @@ -41,11 +41,16 @@ struct ReadFile { should_create_new_source_file: bool, } -pub fn read_file(_s: &mut TSState, v: Value) -> Result { +pub fn read_file(s: &mut TSState, v: Value) -> Result { let v: ReadFile = serde_json::from_value(v)?; let (module_name, source_code) = if v.file_name.starts_with("$asset$/") { let asset = v.file_name.replace("$asset$/", ""); - let source_code = crate::get_asset2(&asset)?.to_string(); + + let source_code = match s.custom_assets.get(&asset) { + Some(asset_path) => std::fs::read_to_string(&asset_path)?, + None => crate::get_asset2(&asset)?.to_string(), + }; + (asset, source_code) } else { assert!(!v.file_name.starts_with("$assets$"), "you meant $asset$"); @@ -110,6 +115,27 @@ pub fn resolve_module_names( Ok(json!(resolved)) } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct FetchAssetArgs { + name: String, +} + +pub fn fetch_asset(s: &mut TSState, v: Value) -> Result { + let args: FetchAssetArgs = serde_json::from_value(v)?; + + if let Some(asset_path) = s.custom_assets.get(&args.name) { + let source_code = std::fs::read_to_string(&asset_path)?; + return Ok(json!(source_code)); + } + + if let Some(source_code) = crate::get_asset(&args.name) { + Ok(json!(source_code)) + } else { + panic!("op_fetch_asset bad asset {}", args.name) + } +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct Exit {