Reland "Create an old program to be used in snapshot." (#3747)

* read CLI assets from disk during snapshotting

cli/build.rs

@@ -14,15 +14,29 @@ fn main() {
   let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
   let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
 
+  let custom_libs = vec![(
+    "lib.deno_runtime.d.ts".to_string(),
+    c.join("js/lib.deno_runtime.d.ts"),
+  )];
+
   let root_names = vec![c.join("js/main.ts")];
   let bundle = o.join("CLI_SNAPSHOT.js");
-  let state = deno_typescript::compile_bundle(&bundle, root_names).unwrap();
+  let state =
+    deno_typescript::compile_bundle(&bundle, root_names, Some(custom_libs))
+      .unwrap();
   assert!(bundle.exists());
   deno_typescript::mksnapshot_bundle(&bundle, state).unwrap();
 
+  let custom_libs = vec![(
+    "lib.deno_runtime.d.ts".to_string(),
+    c.join("js/lib.deno_runtime.d.ts"),
+  )];
+
   let root_names = vec![c.join("js/compiler.ts")];
   let bundle = o.join("COMPILER_SNAPSHOT.js");
-  let state = deno_typescript::compile_bundle(&bundle, root_names).unwrap();
+  let state =
+    deno_typescript::compile_bundle(&bundle, root_names, Some(custom_libs))
+      .unwrap();
   assert!(bundle.exists());
   deno_typescript::mksnapshot_bundle_ts(&bundle, state).unwrap();
 }

cli/js.rs

@@ -16,15 +16,7 @@ pub static COMPILER_SNAPSHOT_MAP: &[u8] =
 pub static COMPILER_SNAPSHOT_DTS: &[u8] =
   include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.d.ts"));
 
-static DENO_RUNTIME: &str = include_str!("js/lib.deno_runtime.d.ts");
-
-/// Same as deno_typescript::get_asset but also has lib.deno_runtime.d.ts
-pub fn get_asset(name: &str) -> Option<&'static str> {
-  match name {
-    "lib.deno_runtime.d.ts" => Some(DENO_RUNTIME),
-    _ => deno_typescript::get_asset(name),
-  }
-}
+pub static DENO_RUNTIME: &str = include_str!("js/lib.deno_runtime.d.ts");
 
 #[test]
 fn cli_snapshot() {

cli/js/compiler.ts

@@ -6,6 +6,7 @@ import "./globals.ts";
 import "./ts_global.d.ts";
 
 import { TranspileOnlyResult } from "./compiler_api.ts";
+import { oldProgram } from "./compiler_bootstrap.ts";
 import { setRootExports } from "./compiler_bundler.ts";
 import {
   defaultBundlerOptions,
@@ -142,7 +143,12 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
         // to generate the program and possibly emit it.
         if (!diagnostics || (diagnostics && diagnostics.length === 0)) {
           const options = host.getCompilationSettings();
-          const program = ts.createProgram(rootNames, options, host);
+          const program = ts.createProgram({
+            rootNames,
+            options,
+            host,
+            oldProgram
+          });
 
           diagnostics = ts
             .getPreEmitDiagnostics(program)
@@ -220,11 +226,12 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
         }
         host.mergeOptions(...compilerOptions);
 
-        const program = ts.createProgram(
+        const program = ts.createProgram({
           rootNames,
-          host.getCompilationSettings(),
-          host
-        );
+          options: host.getCompilationSettings(),
+          host,
+          oldProgram
+        });
 
         if (bundle) {
           setRootExports(program, rootNames[0]);

cli/js/compiler_bootstrap.ts

@@ -0,0 +1,34 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+import { ASSETS, Host } from "./compiler_host.ts";
+import { core } from "./core.ts";
+import * as dispatch from "./dispatch.ts";
+import { sendSync } from "./dispatch_json.ts";
+
+// This registers ops that are available during the snapshotting process.
+const ops = core.ops();
+for (const [name, opId] of Object.entries(ops)) {
+  const opName = `OP_${name.toUpperCase()}`;
+  // TODO This type casting is dangerous, and should be improved when the same
+  // code in `os.ts` is done.
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  (dispatch as any)[opName] = opId;
+}
+
+const host = new Host({ writeFile(): void {} });
+const options = host.getCompilationSettings();
+
+/** Used to generate the foundational AST for all other compilations, so it can
+ * be cached as part of the snapshot and available to speed up startup */
+export const oldProgram = ts.createProgram({
+  rootNames: [`${ASSETS}/bootstrap.ts`],
+  options,
+  host
+});
+
+/** A module loader which is concatenated into bundle files.  We read all static
+ * assets during the snapshotting process, which is why this is located in
+ * compiler_bootstrap. */
+export const bundleLoader = sendSync(dispatch.OP_FETCH_ASSET, {
+  name: "bundle_loader.js"
+});

cli/js/compiler_bundler.ts

@@ -1,7 +1,6 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-import * as dispatch from "./dispatch.ts";
-import { sendSync } from "./dispatch_json.ts";
+import { bundleLoader } from "./compiler_bootstrap.ts";
 import {
   assert,
   commonPath,
@@ -9,11 +8,6 @@ import {
   CHAR_FORWARD_SLASH
 } from "./util.ts";
 
-const BUNDLE_LOADER = "bundle_loader.js";
-
-/** A loader of bundled modules that we will inline into any bundle outputs. */
-let bundleLoader: string;
-
 /** Local state of what the root exports are of a root module. */
 let rootExports: string[] | undefined;
 
@@ -40,12 +34,6 @@ export function buildBundle(
   data: string,
   sourceFiles: readonly ts.SourceFile[]
 ): string {
-  // we can only do this once we are bootstrapped and easiest way to do it is
-  // inline here
-  if (!bundleLoader) {
-    bundleLoader = sendSync(dispatch.OP_FETCH_ASSET, { name: BUNDLE_LOADER });
-  }
-
   // when outputting to AMD and a single outfile, TypeScript makes up the module
   // specifiers which are used to define the modules, and doesn't expose them
   // publicly, so we have to try to replicate

cli/js/compiler_host.ts

@@ -18,7 +18,7 @@ export interface ConfigureResponse {
   diagnostics?: ts.Diagnostic[];
 }
 
-const ASSETS = "$asset$";
+export const ASSETS = "$asset$";
 
 /** Options that need to be used when generating a bundle (either trusted or
  * runtime). */
@@ -129,11 +129,11 @@ export class Host implements ts.CompilerHost {
   private _writeFile: WriteFileCallback;
 
   private _getAsset(filename: string): SourceFile {
-    const sourceFile = SourceFile.get(filename);
+    const url = filename.split("/").pop()!;
+    const sourceFile = SourceFile.get(url);
     if (sourceFile) {
       return sourceFile;
     }
-    const url = filename.split("/").pop()!;
     const name = url.includes(".") ? url : `${url}.d.ts`;
     const sourceCode = sendSync(dispatch.OP_FETCH_ASSET, { name });
     return new SourceFile({
@@ -238,13 +238,15 @@ export class Host implements ts.CompilerHost {
         : SourceFile.get(fileName);
       assert(sourceFile != null);
       if (!sourceFile.tsSourceFile) {
+        assert(sourceFile.sourceCode != null);
         sourceFile.tsSourceFile = ts.createSourceFile(
           fileName,
           sourceFile.sourceCode,
           languageVersion
         );
+        delete sourceFile.sourceCode;
       }
-      return sourceFile!.tsSourceFile;
+      return sourceFile.tsSourceFile;
     } catch (e) {
       if (onError) {
         onError(String(e));

cli/js/compiler_sourcefile.ts

@@ -61,7 +61,7 @@ export class SourceFile {
 
   mediaType!: MediaType;
   processed = false;
-  sourceCode!: string;
+  sourceCode?: string;
   tsSourceFile?: ts.SourceFile;
   url!: string;
 

cli/js/dispatch.ts

@@ -70,12 +70,15 @@ export let OP_TRUNCATE: number;
 export let OP_MAKE_TEMP_DIR: number;
 export let OP_CWD: number;
 export let OP_CONNECT_TLS: number;
-export let OP_FETCH_ASSET: number;
 export let OP_HOSTNAME: number;
 export let OP_OPEN_PLUGIN: number;
 export let OP_COMPILE: number;
 export let OP_TRANSPILE: number;
 
+/** **WARNING:** This is only available during the snapshotting process and is
+ * unavailable at runtime. */
+export let OP_FETCH_ASSET: number;
+
 const PLUGIN_ASYNC_HANDLER_MAP: Map<number, AsyncHandler> = new Map();
 
 export function setPluginAsyncHandler(

cli/js/dispatch_json.ts

@@ -62,7 +62,7 @@ export function sendSync(
   const resUi8 = core.dispatch(opId, argsUi8, zeroCopy);
   util.assert(resUi8 != null);
 
-  const res = decode(resUi8!);
+  const res = decode(resUi8);
   util.assert(res.promiseId == null);
   return unwrapResponse(res);
 }

cli/lib.rs

@@ -146,8 +146,7 @@ fn create_worker_and_state(
 }
 
 fn types_command() {
-  let content = crate::js::get_asset("lib.deno_runtime.d.ts").unwrap();
-  println!("{}", content);
+  println!("{}", crate::js::DENO_RUNTIME);
 }
 
 fn print_cache_info(worker: MainWorker) {

cli/ops/compiler.rs

@@ -17,10 +17,6 @@ pub fn init(i: &mut Isolate, s: &ThreadSafeState) {
     "fetch_source_files",
     s.core_op(json_op(s.stateful_op(op_fetch_source_files))),
   );
-  i.register_op(
-    "fetch_asset",
-    s.core_op(json_op(s.stateful_op(op_fetch_asset))),
-  );
 }
 
 #[derive(Deserialize)]
@@ -149,21 +145,3 @@ fn op_fetch_source_files(
 
   Ok(JsonOp::Async(future))
 }
-
-#[derive(Deserialize)]
-struct FetchAssetArgs {
-  name: String,
-}
-
-fn op_fetch_asset(
-  _state: &ThreadSafeState,
-  args: Value,
-  _zero_copy: Option<PinnedBuf>,
-) -> Result<JsonOp, ErrBox> {
-  let args: FetchAssetArgs = serde_json::from_value(args)?;
-  if let Some(source_code) = crate::js::get_asset(&args.name) {
-    Ok(JsonOp::Sync(json!(source_code)))
-  } else {
-    panic!("op_fetch_asset bad asset {}", args.name)
-  }
-}

core/isolate.rs

@@ -680,6 +680,7 @@ impl Isolate {
     let mut hs = v8::HandleScope::new(locker.enter());
     let scope = hs.enter();
     self.global_context.reset(scope);
+    self.shared_response_buf.reset(scope);
 
     let snapshot_creator = self.snapshot_creator.as_mut().unwrap();
     let snapshot = snapshot_creator

core/shared_queue.js

@@ -184,7 +184,6 @@ SharedQueue Binary Layout
   }
 
   function dispatch(opId, control, zeroCopy = null) {
-    maybeInit();
     return Deno.core.send(opId, control, zeroCopy);
   }
 

deno_typescript/lib.rs

@@ -16,6 +16,7 @@ use deno_core::PinnedBuf;
 use deno_core::StartupData;
 pub use ops::EmitResult;
 use ops::WrittenFile;
+use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
 use std::path::PathBuf;
@@ -37,6 +38,7 @@ pub struct TSState {
   bundle: bool,
   exit_code: i32,
   emit_result: Option<EmitResult>,
+  custom_assets: HashMap<String, PathBuf>,
   /// A list of files emitted by typescript. WrittenFile is tuple of the form
   /// (url, corresponding_module, source_code)
   written_files: Vec<WrittenFile>,
@@ -76,6 +78,7 @@ impl TSIsolate {
 
     let state = Arc::new(Mutex::new(TSState {
       bundle,
+      custom_assets: HashMap::new(),
       exit_code: 0,
       emit_result: None,
       written_files: Vec::new(),
@@ -119,13 +122,24 @@ impl TSIsolate {
     self.isolate.execute("<anon>", source)?;
     Ok(self.state)
   }
+
+  pub fn add_custom_assets(&mut self, custom_assets: Vec<(String, PathBuf)>) {
+    let mut state = self.state.lock().unwrap();
+    for (name, path) in custom_assets {
+      state.custom_assets.insert(name, path);
+    }
+  }
 }
 
 pub fn compile_bundle(
   bundle: &Path,
   root_names: Vec<PathBuf>,
+  custom_assets: Option<Vec<(String, PathBuf)>>,
 ) -> Result<Arc<Mutex<TSState>>, ErrBox> {
-  let ts_isolate = TSIsolate::new(true);
+  let mut ts_isolate = TSIsolate::new(true);
+  if let Some(assets) = custom_assets {
+    ts_isolate.add_custom_assets(assets);
+  }
 
   let config_json = serde_json::json!({
     "compilerOptions": {
@@ -203,6 +217,10 @@ pub fn mksnapshot_bundle_ts(
   state: Arc<Mutex<TSState>>,
 ) -> Result<(), ErrBox> {
   let runtime_isolate = &mut Isolate::new(StartupData::None, true);
+  runtime_isolate.register_op(
+    "fetch_asset",
+    compiler_op(state.clone(), ops::json_op(ops::fetch_asset)),
+  );
   let source_code_vec = std::fs::read(bundle)?;
   let source_code = std::str::from_utf8(&source_code_vec)?;
 
@@ -255,6 +273,7 @@ pub fn get_asset(name: &str) -> Option<&'static str> {
   }
   match name {
     "bundle_loader.js" => Some(include_str!("bundle_loader.js")),
+    "bootstrap.ts" => Some("console.log(\"hello deno\");"),
     "typescript.d.ts" => inc!("typescript.d.ts"),
     "lib.esnext.d.ts" => inc!("lib.esnext.d.ts"),
     "lib.es2020.d.ts" => inc!("lib.es2020.d.ts"),

deno_typescript/ops.rs

@@ -41,11 +41,16 @@ struct ReadFile {
   should_create_new_source_file: bool,
 }
 
-pub fn read_file(_s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
+pub fn read_file(s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
   let v: ReadFile = serde_json::from_value(v)?;
   let (module_name, source_code) = if v.file_name.starts_with("$asset$/") {
     let asset = v.file_name.replace("$asset$/", "");
-    let source_code = crate::get_asset2(&asset)?.to_string();
+
+    let source_code = match s.custom_assets.get(&asset) {
+      Some(asset_path) => std::fs::read_to_string(&asset_path)?,
+      None => crate::get_asset2(&asset)?.to_string(),
+    };
+
     (asset, source_code)
   } else {
     assert!(!v.file_name.starts_with("$assets$"), "you meant $asset$");
@@ -110,6 +115,27 @@ pub fn resolve_module_names(
   Ok(json!(resolved))
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct FetchAssetArgs {
+  name: String,
+}
+
+pub fn fetch_asset(s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
+  let args: FetchAssetArgs = serde_json::from_value(v)?;
+
+  if let Some(asset_path) = s.custom_assets.get(&args.name) {
+    let source_code = std::fs::read_to_string(&asset_path)?;
+    return Ok(json!(source_code));
+  }
+
+  if let Some(source_code) = crate::get_asset(&args.name) {
+    Ok(json!(source_code))
+  } else {
+    panic!("op_fetch_asset bad asset {}", args.name)
+  }
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct Exit {