foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-21 15:04:11 -05:00
Code Issues Packages 1 Wiki Activity

refactor: internal runtime code TS support (#17672)

Browse source 
This is a proof of concept for being able to snapshot TypeScript files.

Currently only a single runtime file is authored in TypeScript -
"runtime/js/01_version.ts".

Not needed infrastructure was removed from "core/snapshot_util.rs".

---------

Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>
This commit is contained in:
Leo Kettmeir 2023-02-08 22:40:18 +01:00 committed by GitHub
parent bef50416b9
commit 286e5d0be9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 261 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -1156,6 +1156,7 @@ name = "deno_runtime"
version = "0.97.0"
dependencies = [
"atty",
"deno_ast",
"deno_broadcast_channel",
"deno_cache",
"deno_console",

2
bench_util/js_runtime.rs
View file

@ -11,7 +11,7 @@ pub fn create_js_runtime(setup: impl FnOnce() -> Vec<Extension>) -> JsRuntime {
JsRuntime::new(RuntimeOptions {
extensions_with_js: setup(),
module_loader: Some(std::rc::Rc::new(
deno_core::InternalModuleLoader::new(None),
deno_core::InternalModuleLoader::default(),
)),
..Default::default()
})

80
cli/build.rs
View file

@ -4,8 +4,10 @@ use std::env;
use std::path::Path;
use std::path::PathBuf;
use deno_core::include_js_files_dir;
use deno_core::snapshot_util::*;
use deno_core::Extension;
use deno_core::ExtensionFileSource;
use deno_runtime::deno_cache::SqliteBackedCache;
use deno_runtime::permissions::PermissionsContainer;
use deno_runtime::*;
@ -15,6 +17,7 @@ mod ts {
use crate::deno_webgpu_get_declaration;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::include_js_files_dir;
use deno_core::op;
use deno_core::OpState;
use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES;
@ -32,11 +35,7 @@ mod ts {
specifier: String,
}
pub fn create_compiler_snapshot(
snapshot_path: PathBuf,
files: Vec<PathBuf>,
cwd: &Path,
) {
pub fn create_compiler_snapshot(snapshot_path: PathBuf, cwd: &Path) {
// libs that are being provided by op crates.
let mut op_crate_libs = HashMap::new();
op_crate_libs.insert("deno.cache", deno_cache::get_declaration());
@ -252,36 +251,42 @@ mod ts {
}
}
let tsc_extension = Extension::builder("deno_tsc")
.ops(vec![
op_build_info::decl(),
op_cwd::decl(),
op_exists::decl(),
op_is_node_file::decl(),
op_load::decl(),
op_script_version::decl(),
])
.js(include_js_files_dir! {
dir "tsc",
"00_typescript.js",
"99_main_compiler.js",
})
.state(move |state| {
state.put(op_crate_libs.clone());
state.put(build_libs.clone());
state.put(path_dts.clone());
Ok(())
})
.build();
create_snapshot(CreateSnapshotOptions {
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
snapshot_path,
startup_snapshot: None,
extensions: vec![Extension::builder("deno_tsc")
.ops(vec![
op_build_info::decl(),
op_cwd::decl(),
op_exists::decl(),
op_is_node_file::decl(),
op_load::decl(),
op_script_version::decl(),
])
.state(move |state| {
state.put(op_crate_libs.clone());
state.put(build_libs.clone());
state.put(path_dts.clone());
Ok(())
})
.build()],
extensions_with_js: vec![],
additional_files: files,
additional_esm_files: vec![],
extensions: vec![],
extensions_with_js: vec![tsc_extension],
compression_cb: Some(Box::new(|vec, snapshot_slice| {
vec.extend_from_slice(
&zstd::bulk::compress(snapshot_slice, 22)
.expect("snapshot compression failed"),
);
})),
snapshot_module_load_cb: None,
});
}
@ -307,7 +312,7 @@ mod ts {
}
}
fn create_cli_snapshot(snapshot_path: PathBuf, esm_files: Vec<PathBuf>) {
fn create_cli_snapshot(snapshot_path: PathBuf) {
let extensions: Vec<Extension> = vec![
deno_webidl::init(),
deno_console::init(),
@ -338,14 +343,23 @@ fn create_cli_snapshot(snapshot_path: PathBuf, esm_files: Vec<PathBuf>) {
deno_flash::init::<PermissionsContainer>(false), // No --unstable
];
let mut esm_files = include_js_files_dir!(
dir "js",
"40_testing.js",
);
esm_files.push(ExtensionFileSource {
specifier: "runtime/js/99_main.js".to_string(),
code: deno_runtime::js::SOURCE_CODE_FOR_99_MAIN_JS,
});
let extensions_with_js =
vec![Extension::builder("cli").esm(esm_files).build()];
create_snapshot(CreateSnapshotOptions {
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
snapshot_path,
startup_snapshot: Some(deno_runtime::js::deno_isolate_init()),
extensions,
extensions_with_js: vec![],
additional_files: vec![],
additional_esm_files: esm_files,
extensions_with_js,
compression_cb: Some(Box::new(|vec, snapshot_slice| {
lzzzz::lz4_hc::compress_to_vec(
snapshot_slice,
@ -354,6 +368,7 @@ fn create_cli_snapshot(snapshot_path: PathBuf, esm_files: Vec<PathBuf>) {
)
.expect("snapshot compression failed");
})),
snapshot_module_load_cb: None,
})
}
@ -450,13 +465,10 @@ fn main() {
let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin");
let js_files = get_js_files(env!("CARGO_MANIFEST_DIR"), "tsc", None);
ts::create_compiler_snapshot(compiler_snapshot_path, js_files, &c);
ts::create_compiler_snapshot(compiler_snapshot_path, &c);
let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin");
let mut esm_files = get_js_files(env!("CARGO_MANIFEST_DIR"), "js", None);
esm_files.push(deno_runtime::js::get_99_main());
create_cli_snapshot(cli_snapshot_path, esm_files);
create_cli_snapshot(cli_snapshot_path);
#[cfg(target_os = "windows")]
{

32
core/extensions.rs
View file

@ -6,6 +6,7 @@ use std::rc::Rc;
use std::task::Context;
use v8::fast_api::FastFunction;
#[derive(Clone, Debug)]
pub struct ExtensionFileSource {
pub specifier: String,
pub code: &'static str,
@ -244,8 +245,9 @@ impl ExtensionBuilder {
}
}
}
/// Helps embed JS files in an extension. Returns Vec<(&'static str, &'static str)>
/// representing the filename and source code.
/// Helps embed JS files in an extension. Returns a vector of
/// `ExtensionFileSource`, that represent the filename and source code.
///
/// Example:
/// ```ignore
@ -265,3 +267,29 @@ macro_rules! include_js_files {
]
};
}
/// Helps embed JS files in an extension. Returns a vector of
/// `ExtensionFileSource`, that represent the filename and source code.
/// Additional "dir" option is required, that specifies which directory in the
/// crate root contains the listed files. "dir" option will be prepended to
/// each file name.
///
/// Example:
/// ```ignore
/// include_js_files_dir!(
/// dir "example",
/// "01_hello.js",
/// "02_goodbye.js",
/// )
/// ```
#[macro_export]
macro_rules! include_js_files_dir {
(dir $dir:literal, $($file:literal,)+) => {
vec![
$($crate::ExtensionFileSource {
specifier: concat!($dir, "/", $file).to_string(),
code: include_str!(concat!($dir, "/", $file)),
},)+
]
};
}

1
core/lib.rs
View file

@ -75,6 +75,7 @@ pub use crate::module_specifier::ModuleSpecifier;
pub use crate::module_specifier::DUMMY_SPECIFIER;
pub use crate::modules::FsModuleLoader;
pub use crate::modules::InternalModuleLoader;
pub use crate::modules::InternalModuleLoaderCb;
pub use crate::modules::ModuleId;
pub use crate::modules::ModuleLoader;
pub use crate::modules::ModuleSource;

83
core/modules.rs
View file

@ -2,6 +2,7 @@
use crate::bindings;
use crate::error::generic_error;
use crate::extensions::ExtensionFileSource;
use crate::module_specifier::ModuleSpecifier;
use crate::resolve_import;
use crate::resolve_url;
@ -312,13 +313,38 @@ pub(crate) fn resolve_helper(
loader.resolve(specifier, referrer, kind)
}
pub struct InternalModuleLoader(Rc<dyn ModuleLoader>);
/// Function that can be passed to the `InternalModuleLoader` that allows to
/// transpile sources before passing to V8.
pub type InternalModuleLoaderCb =
Box<dyn Fn(&ExtensionFileSource) -> Result<String, Error>>;
pub struct InternalModuleLoader {
module_loader: Rc<dyn ModuleLoader>,
esm_sources: Vec<ExtensionFileSource>,
maybe_load_callback: Option<InternalModuleLoaderCb>,
}
impl Default for InternalModuleLoader {
fn default() -> Self {
Self {
module_loader: Rc::new(NoopModuleLoader),
esm_sources: vec![],
maybe_load_callback: None,
}
}
}
impl InternalModuleLoader {
pub fn new(module_loader: Option<Rc<dyn ModuleLoader>>) -> Self {
InternalModuleLoader(
module_loader.unwrap_or_else(|| Rc::new(NoopModuleLoader)),
)
pub fn new(
module_loader: Option<Rc<dyn ModuleLoader>>,
esm_sources: Vec<ExtensionFileSource>,
maybe_load_callback: Option<InternalModuleLoaderCb>,
) -> Self {
InternalModuleLoader {
module_loader: module_loader.unwrap_or_else(|| Rc::new(NoopModuleLoader)),
esm_sources,
maybe_load_callback,
}
}
}
@ -343,7 +369,7 @@ impl ModuleLoader for InternalModuleLoader {
}
}
self.0.resolve(specifier, referrer, kind)
self.module_loader.resolve(specifier, referrer, kind)
}
fn load(
@ -352,7 +378,46 @@ impl ModuleLoader for InternalModuleLoader {
maybe_referrer: Option<ModuleSpecifier>,
is_dyn_import: bool,
) -> Pin<Box<ModuleSourceFuture>> {
self.0.load(module_specifier, maybe_referrer, is_dyn_import)
if module_specifier.scheme() != "internal" {
return self.module_loader.load(
module_specifier,
maybe_referrer,
is_dyn_import,
);
}
let specifier = module_specifier.to_string();
let maybe_file_source = self
.esm_sources
.iter()
.find(|file_source| file_source.specifier == module_specifier.as_str());
if let Some(file_source) = maybe_file_source {
let result = if let Some(load_callback) = &self.maybe_load_callback {
load_callback(file_source)
} else {
Ok(file_source.code.to_string())
};
return async move {
let code = result?;
let source = ModuleSource {
code: code.into_bytes().into_boxed_slice(),
module_type: ModuleType::JavaScript,
module_url_specified: specifier.clone(),
module_url_found: specifier.clone(),
};
Ok(source)
}
.boxed_local();
}
async move {
Err(generic_error(format!(
"Cannot find internal module source for specifier {specifier}"
)))
}
.boxed_local()
}
fn prepare_load(
@ -366,7 +431,7 @@ impl ModuleLoader for InternalModuleLoader {
return async { Ok(()) }.boxed_local();
}
self.0.prepare_load(
self.module_loader.prepare_load(
op_state,
module_specifier,
maybe_referrer,
@ -2639,7 +2704,7 @@ if (import.meta.url != 'file:///main_with_code.js') throw Error();
#[test]
fn internal_module_loader() {
let loader = InternalModuleLoader::new(None);
let loader = InternalModuleLoader::default();
assert!(loader
.resolve("internal:foo", "internal:bar", ResolutionKind::Import)
.is_ok());

21
core/runtime.rs
View file

@ -8,6 +8,7 @@ use crate::extensions::OpDecl;
use crate::extensions::OpEventLoopFn;
use crate::inspector::JsRuntimeInspector;
use crate::module_specifier::ModuleSpecifier;
use crate::modules::InternalModuleLoaderCb;
use crate::modules::ModuleError;
use crate::modules::ModuleId;
use crate::modules::ModuleLoadId;
@ -19,6 +20,7 @@ use crate::ops::*;
use crate::source_map::SourceMapCache;
use crate::source_map::SourceMapGetter;
use crate::Extension;
use crate::ExtensionFileSource;
use crate::NoopModuleLoader;
use crate::OpMiddlewareFn;
use crate::OpResult;
@ -270,6 +272,11 @@ pub struct RuntimeOptions {
/// The snapshot is deterministic and uses predictable random numbers.
pub will_snapshot: bool,
/// An optional callback that will be called for each module that is loaded
/// during snapshotting. This callback can be used to transpile source on the
/// fly, during snapshotting, eg. to transpile TypeScript to JavaScript.
pub snapshot_module_load_cb: Option<InternalModuleLoaderCb>,
/// Isolate creation parameters.
pub create_params: Option<v8::CreateParams>,
@ -607,8 +614,16 @@ impl JsRuntime {
};
let loader = if snapshot_options != SnapshotOptions::Load {
let esm_sources = options
.extensions_with_js
.iter()
.flat_map(|ext| ext.get_esm_sources().to_owned())
.collect::<Vec<ExtensionFileSource>>();
Rc::new(crate::modules::InternalModuleLoader::new(
options.module_loader,
esm_sources,
options.snapshot_module_load_cb,
))
} else {
options
@ -818,13 +833,13 @@ impl JsRuntime {
let extensions = std::mem::take(&mut self.extensions_with_js);
for ext in &extensions {
{
let js_files = ext.get_esm_sources();
for file_source in js_files {
let esm_files = ext.get_esm_sources();
for file_source in esm_files {
futures::executor::block_on(async {
let id = self
.load_side_module(
&ModuleSpecifier::parse(&file_source.specifier)?,
Some(file_source.code.to_string()),
None,
)
.await?;
let receiver = self.mod_evaluate(id);

46
core/snapshot_util.rs
View file

@ -1,12 +1,11 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use anyhow::Context;
use std::path::Path;
use std::path::PathBuf;
use crate::Extension;
use crate::InternalModuleLoaderCb;
use crate::JsRuntime;
use crate::ModuleSpecifier;
use crate::RuntimeOptions;
use crate::Snapshot;
@ -18,57 +17,20 @@ pub struct CreateSnapshotOptions {
pub startup_snapshot: Option<Snapshot>,
pub extensions: Vec<Extension>,
pub extensions_with_js: Vec<Extension>,
pub additional_files: Vec<PathBuf>,
pub additional_esm_files: Vec<PathBuf>,
pub compression_cb: Option<Box<CompressionCb>>,
pub snapshot_module_load_cb: Option<InternalModuleLoaderCb>,
}
pub fn create_snapshot(create_snapshot_options: CreateSnapshotOptions) {
let mut js_runtime = JsRuntime::new(RuntimeOptions {
let js_runtime = JsRuntime::new(RuntimeOptions {
will_snapshot: true,
startup_snapshot: create_snapshot_options.startup_snapshot,
extensions: create_snapshot_options.extensions,
extensions_with_js: create_snapshot_options.extensions_with_js,
snapshot_module_load_cb: create_snapshot_options.snapshot_module_load_cb,
..Default::default()
});
// TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the
// workspace root.
let display_root = Path::new(create_snapshot_options.cargo_manifest_dir)
.parent()
.unwrap();
for file in create_snapshot_options.additional_files {
let display_path = file.strip_prefix(display_root).unwrap_or(&file);
let display_path_str = display_path.display().to_string();
js_runtime
.execute_script(
&("internal:".to_string() + &display_path_str.replace('\\', "/")),
&std::fs::read_to_string(&file).unwrap(),
)
.unwrap();
}
for file in create_snapshot_options.additional_esm_files {
let display_path = file.strip_prefix(display_root).unwrap_or(&file);
let display_path_str = display_path.display().to_string();
let filename =
&("internal:".to_string() + &display_path_str.replace('\\', "/"));
futures::executor::block_on(async {
let id = js_runtime
.load_side_module(
&ModuleSpecifier::parse(filename)?,
Some(std::fs::read_to_string(&file)?),
)
.await?;
let receiver = js_runtime.mod_evaluate(id);
js_runtime.run_event_loop(false).await?;
receiver.await?
})
.with_context(|| format!("Couldn't execute '{}'", file.display()))
.unwrap();
}
let snapshot = js_runtime.snapshot();
let snapshot_slice: &[u8] = &snapshot;
println!("Snapshot size: {}", snapshot_slice.len());

1
runtime/Cargo.toml
View file

@ -24,6 +24,7 @@ name = "hello_runtime"
path = "examples/hello_runtime.rs"
[build-dependencies]
deno_ast.workspace = true
deno_broadcast_channel.workspace = true
deno_cache.workspace = true
deno_console.workspace = true

87
runtime/build.rs
View file

@ -1,7 +1,7 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use deno_core::include_js_files_dir;
use std::env;
use std::path::PathBuf;
// This is a shim that allows to generate documentation on docs.rs
@ -13,6 +13,39 @@ mod not_docs {
use deno_core::snapshot_util::*;
use deno_core::Extension;
use deno_ast::MediaType;
use deno_ast::ParseParams;
use deno_ast::SourceTextInfo;
use deno_core::error::AnyError;
use deno_core::ExtensionFileSource;
fn transpile_ts_for_snapshotting(
file_source: &ExtensionFileSource,
) -> Result<String, AnyError> {
let media_type = MediaType::from(Path::new(&file_source.specifier));
let should_transpile = match media_type {
MediaType::JavaScript => false,
MediaType::TypeScript => true,
_ => panic!("Unsupported media type for snapshotting {media_type:?}"),
};
if !should_transpile {
return Ok(file_source.code.to_string());
}
let parsed = deno_ast::parse_module(ParseParams {
specifier: file_source.specifier.to_string(),
text_info: SourceTextInfo::from_string(file_source.code.to_string()),
media_type,
capture_tokens: false,
scope_analysis: false,
maybe_syntax: None,
})?;
let transpiled_source = parsed.transpile(&Default::default())?;
Ok(transpiled_source.text)
}
struct Permissions;
impl deno_fetch::FetchPermissions for Permissions {
@ -120,7 +153,10 @@ mod not_docs {
}
}
fn create_runtime_snapshot(snapshot_path: PathBuf, esm_files: Vec<PathBuf>) {
fn create_runtime_snapshot(
snapshot_path: PathBuf,
additional_extension: Extension,
) {
let extensions_with_js: Vec<Extension> = vec![
deno_webidl::init(),
deno_console::init(),
@ -149,6 +185,7 @@ mod not_docs {
deno_napi::init::<Permissions>(false),
deno_http::init(),
deno_flash::init::<Permissions>(false), // No --unstable
additional_extension,
];
create_snapshot(CreateSnapshotOptions {
@ -157,8 +194,6 @@ mod not_docs {
startup_snapshot: None,
extensions: vec![],
extensions_with_js,
additional_files: vec![],
additional_esm_files: esm_files,
compression_cb: Some(Box::new(|vec, snapshot_slice| {
lzzzz::lz4_hc::compress_to_vec(
snapshot_slice,
@ -167,24 +202,50 @@ mod not_docs {
)
.expect("snapshot compression failed");
})),
snapshot_module_load_cb: Some(Box::new(transpile_ts_for_snapshotting)),
});
}
pub fn build_snapshot(runtime_snapshot_path: PathBuf) {
#[allow(unused_mut)]
let mut esm_files = get_js_files(
env!("CARGO_MANIFEST_DIR"),
"js",
Some(Box::new(|path| !path.ends_with("99_main.js"))),
#[allow(unused_mut, unused_assignments)]
let mut esm_files = include_js_files_dir!(
dir "js",
"01_build.js",
"01_errors.js",
"01_version.ts",
"06_util.js",
"10_permissions.js",
"11_workers.js",
"12_io.js",
"13_buffer.js",
"30_fs.js",
"30_os.js",
"40_diagnostics.js",
"40_files.js",
"40_fs_events.js",
"40_http.js",
"40_process.js",
"40_read_file.js",
"40_signals.js",
"40_spawn.js",
"40_tty.js",
"40_write_file.js",
"41_prompt.js",
"90_deno_ns.js",
"98_global_scope.js",
);
#[cfg(not(feature = "snapshot_from_snapshot"))]
{
let manifest = env!("CARGO_MANIFEST_DIR");
let path = PathBuf::from(manifest);
esm_files.push(path.join("js").join("99_main.js"));
esm_files.push(ExtensionFileSource {
specifier: "js/99_main.js".to_string(),
code: include_str!("js/99_main.js"),
});
}
create_runtime_snapshot(runtime_snapshot_path, esm_files);
let additional_extension =
Extension::builder("runtime").esm(esm_files).build();
create_runtime_snapshot(runtime_snapshot_path, additional_extension);
}
}

8
runtime/js.rs
View file

@ -2,7 +2,6 @@
use deno_core::Snapshot;
use log::debug;
use once_cell::sync::Lazy;
use std::path::PathBuf;
pub static RUNTIME_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
#[allow(clippy::uninit_vec)]
@ -35,8 +34,5 @@ pub fn deno_isolate_init() -> Snapshot {
Snapshot::Static(&RUNTIME_SNAPSHOT)
}
pub fn get_99_main() -> PathBuf {
let manifest = env!("CARGO_MANIFEST_DIR");
let path = PathBuf::from(manifest);
path.join("js").join("99_main.js")
}
#[cfg(feature = "snapshot_from_snapshot")]
pub static SOURCE_CODE_FOR_99_MAIN_JS: &str = include_str!("js/99_main.js");

8
runtime/js/01_version.js → runtime/js/01_version.ts
View file

@ -3,7 +3,13 @@
const primordials = globalThis.__bootstrap.primordials;
const { ObjectFreeze } = primordials;
const version = {
interface Version {
deno: string;
v8: string;
typescript: string;
}
const version: Version = {
deno: "",
v8: "",
typescript: "",

2
runtime/js/90_deno_ns.js
View file

@ -12,7 +12,7 @@ import * as http from "internal:deno_http/01_http.js";
import * as flash from "internal:deno_flash/01_http.js";
import * as build from "internal:runtime/js/01_build.js";
import * as errors from "internal:runtime/js/01_errors.js";
import * as version from "internal:runtime/js/01_version.js";
import * as version from "internal:runtime/js/01_version.ts";
import * as permissions from "internal:runtime/js/10_permissions.js";
import * as io from "internal:runtime/js/12_io.js";
import * as buffer from "internal:runtime/js/13_buffer.js";

2
runtime/js/99_main.js
View file

@ -43,7 +43,7 @@ import * as util from "internal:runtime/js/06_util.js";
import * as event from "internal:deno_web/02_event.js";
import * as location from "internal:deno_web/12_location.js";
import * as build from "internal:runtime/js/01_build.js";
import * as version from "internal:runtime/js/01_version.js";
import * as version from "internal:runtime/js/01_version.ts";
import * as os from "internal:runtime/js/30_os.js";
import * as timers from "internal:deno_web/02_timers.js";
import * as colors from "internal:deno_console/01_colors.js";