From cbc2108525f3a01f4a944104457939b741c9898b Mon Sep 17 00:00:00 2001 From: Nayeem Rahman Date: Mon, 4 Jan 2021 23:15:52 +0000 Subject: [PATCH] feat(cli/standalone): support runtime flags for deno compile (#8738) --- cli/file_fetcher.rs | 4 +- cli/flags.rs | 146 +++++++++++++++++---- cli/http_util.rs | 70 +++++++--- cli/main.rs | 14 +- cli/program_state.rs | 15 ++- cli/standalone.rs | 177 +++++++++++++++++++++----- cli/tests/integration_tests.rs | 52 +++++++- cli/tests/standalone_runtime_flags.ts | 3 + docs/tools.md | 1 + docs/tools/compiler.md | 24 +++- runtime/examples/hello_runtime.rs | 2 +- runtime/http_util.rs | 10 +- runtime/ops/fetch.rs | 4 +- runtime/ops/websocket.rs | 17 ++- runtime/web_worker.rs | 8 +- runtime/worker.rs | 8 +- 16 files changed, 428 insertions(+), 127 deletions(-) create mode 100644 cli/tests/standalone_runtime_flags.ts diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 55251dbd03..f10574c2d4 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -283,14 +283,14 @@ impl FileFetcher { http_cache: HttpCache, cache_setting: CacheSetting, allow_remote: bool, - maybe_ca_data: Option<&str>, + ca_data: Option>, ) -> Result { Ok(Self { allow_remote, cache: FileCache::default(), cache_setting, http_cache, - http_client: create_http_client(get_user_agent(), maybe_ca_data)?, + http_client: create_http_client(get_user_agent(), ca_data)?, }) } diff --git a/cli/flags.rs b/cli/flags.rs index 95e5617aba..da0f8ad0e4 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -6,14 +6,20 @@ use clap::Arg; use clap::ArgMatches; use clap::ArgSettings; use clap::SubCommand; +use deno_core::serde::de; +use deno_core::serde::Deserialize; +use deno_core::serde::Deserializer; +use deno_core::serde::Serialize; +use deno_core::serde::Serializer; use deno_runtime::permissions::PermissionsOptions; use log::Level; +use std::fmt; use std::net::SocketAddr; use std::path::PathBuf; use std::str::FromStr; use tempfile::TempDir; -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] pub enum DenoSubcommand { Bundle { source_file: String, @@ -25,6 +31,7 @@ pub enum DenoSubcommand { Compile { source_file: String, output: Option, + args: Vec, }, Completions { buf: Box<[u8]>, @@ -92,7 +99,66 @@ impl Default for DenoSubcommand { } } -#[derive(Clone, Debug, PartialEq, Default)] +fn deserialize_maybe_log_level<'de, D>(d: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct OptionalLogLevelVisitor; + impl<'de> de::Visitor<'de> for OptionalLogLevelVisitor { + type Value = Option; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "null or a valid log level string") + } + + fn visit_none(self) -> Result + where + E: de::Error, + { + Ok(None) + } + + fn visit_some(self, d: D) -> Result + where + D: de::Deserializer<'de>, + { + struct LogLevelVisitor; + impl<'de> de::Visitor<'de> for LogLevelVisitor { + type Value = Level; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a valid log level string") + } + + fn visit_str(self, s: &str) -> Result + where + E: de::Error, + { + Level::from_str(s).map_err(|_| { + de::Error::invalid_value(de::Unexpected::Str(s), &self) + }) + } + } + Ok(Some(d.deserialize_str(LogLevelVisitor)?)) + } + } + d.deserialize_option(OptionalLogLevelVisitor) +} + +fn serialize_maybe_log_level( + maybe_level: &Option, + s: S, +) -> Result +where + S: Serializer, +{ + match maybe_level { + None => s.serialize_none(), + Some(level) => s.serialize_str(&level.to_string()), + } +} + +#[derive(Clone, Debug, PartialEq, Default, Deserialize, Serialize)] pub struct Flags { /// Vector of CLI arguments - these are user script arguments, all Deno /// specific flags are removed. @@ -117,6 +183,8 @@ pub struct Flags { pub inspect_brk: Option, pub lock: Option, pub lock_write: bool, + #[serde(deserialize_with = "deserialize_maybe_log_level")] + #[serde(serialize_with = "serialize_maybe_log_level")] pub log_level: Option, pub no_check: bool, pub no_prompts: bool, @@ -407,7 +475,7 @@ fn fmt_parse(flags: &mut Flags, matches: &clap::ArgMatches) { } fn install_parse(flags: &mut Flags, matches: &clap::ArgMatches) { - runtime_args_parse(flags, matches, true); + runtime_args_parse(flags, matches, true, true); let root = if matches.is_present("root") { let install_root = matches.value_of("root").unwrap(); @@ -437,14 +505,22 @@ fn install_parse(flags: &mut Flags, matches: &clap::ArgMatches) { } fn compile_parse(flags: &mut Flags, matches: &clap::ArgMatches) { - compile_args_parse(flags, matches); + runtime_args_parse(flags, matches, true, false); - let source_file = matches.value_of("source_file").unwrap().to_string(); + let mut script: Vec = matches + .values_of("script_arg") + .unwrap() + .map(String::from) + .collect(); + assert!(!script.is_empty()); + let args = script.split_off(1); + let source_file = script[0].to_string(); let output = matches.value_of("output").map(PathBuf::from); flags.subcommand = DenoSubcommand::Compile { source_file, output, + args, }; } @@ -483,7 +559,7 @@ fn completions_parse(flags: &mut Flags, matches: &clap::ArgMatches) { } fn repl_parse(flags: &mut Flags, matches: &clap::ArgMatches) { - runtime_args_parse(flags, matches, false); + runtime_args_parse(flags, matches, false, true); flags.repl = true; flags.subcommand = DenoSubcommand::Repl; flags.allow_net = Some(vec![]); @@ -496,7 +572,7 @@ fn repl_parse(flags: &mut Flags, matches: &clap::ArgMatches) { } fn eval_parse(flags: &mut Flags, matches: &clap::ArgMatches) { - runtime_args_parse(flags, matches, false); + runtime_args_parse(flags, matches, false, true); flags.allow_net = Some(vec![]); flags.allow_env = true; flags.allow_run = true; @@ -577,13 +653,22 @@ fn compile_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) { ca_file_arg_parse(flags, matches); } -fn runtime_args<'a, 'b>(app: App<'a, 'b>, include_perms: bool) -> App<'a, 'b> { - let app = inspect_args(compile_args(app)); +fn runtime_args<'a, 'b>( + app: App<'a, 'b>, + include_perms: bool, + include_inspector: bool, +) -> App<'a, 'b> { + let app = compile_args(app); let app = if include_perms { permission_args(app) } else { app }; + let app = if include_inspector { + inspect_args(app) + } else { + app + }; app .arg(cached_only_arg()) .arg(v8_flags_arg()) @@ -594,19 +679,22 @@ fn runtime_args_parse( flags: &mut Flags, matches: &clap::ArgMatches, include_perms: bool, + include_inspector: bool, ) { compile_args_parse(flags, matches); cached_only_arg_parse(flags, matches); if include_perms { permission_args_parse(flags, matches); } + if include_inspector { + inspect_arg_parse(flags, matches); + } v8_flags_arg_parse(flags, matches); seed_arg_parse(flags, matches); - inspect_arg_parse(flags, matches); } fn run_parse(flags: &mut Flags, matches: &clap::ArgMatches) { - runtime_args_parse(flags, matches, true); + runtime_args_parse(flags, matches, true, true); let mut script: Vec = matches .values_of("script_arg") @@ -625,7 +713,7 @@ fn run_parse(flags: &mut Flags, matches: &clap::ArgMatches) { } fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) { - runtime_args_parse(flags, matches, true); + runtime_args_parse(flags, matches, true, true); let no_run = matches.is_present("no-run"); let fail_fast = matches.is_present("fail-fast"); @@ -799,12 +887,12 @@ Ignore formatting a file by adding an ignore comment at the top of the file: } fn repl_subcommand<'a, 'b>() -> App<'a, 'b> { - runtime_args(SubCommand::with_name("repl"), false) + runtime_args(SubCommand::with_name("repl"), false, true) .about("Read Eval Print Loop") } fn install_subcommand<'a, 'b>() -> App<'a, 'b> { - runtime_args(SubCommand::with_name("install"), true) + runtime_args(SubCommand::with_name("install"), true, true) .setting(AppSettings::TrailingVarArg) .arg( Arg::with_name("cmd") @@ -859,11 +947,10 @@ These must be added to the path manually if required.") } fn compile_subcommand<'a, 'b>() -> App<'a, 'b> { - compile_args(SubCommand::with_name("compile")) + runtime_args(SubCommand::with_name("compile"), true, false) + .setting(AppSettings::TrailingVarArg) .arg( - Arg::with_name("source_file") - .takes_value(true) - .required(true), + script_arg(), ) .arg( Arg::with_name("output") @@ -878,6 +965,10 @@ fn compile_subcommand<'a, 'b>() -> App<'a, 'b> { deno compile --unstable https://deno.land/std/http/file_server.ts deno compile --unstable --output /usr/local/bin/color_util https://deno.land/std/examples/colors.ts +Any flags passed which affect runtime behavior, such as '--unstable', +'--allow-*', '--v8-flags', etc. are encoded into the output executable and used +at runtime as if they were passed to a similar 'deno run' command. + The executable name is inferred by default: - Attempt to take the file stem of the URL path. The above example would become 'file_server'. @@ -926,7 +1017,7 @@ fn completions_subcommand<'a, 'b>() -> App<'a, 'b> { } fn eval_subcommand<'a, 'b>() -> App<'a, 'b> { - runtime_args(SubCommand::with_name("eval"), false) + runtime_args(SubCommand::with_name("eval"), false, true) .about("Eval script") .long_about( "Evaluate JavaScript from the command line. @@ -1246,7 +1337,7 @@ fn permission_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> { } fn run_subcommand<'a, 'b>() -> App<'a, 'b> { - runtime_args(SubCommand::with_name("run"), true) + runtime_args(SubCommand::with_name("run"), true, true) .arg( watch_arg() .conflicts_with("inspect") @@ -1280,7 +1371,7 @@ Deno allows specifying the filename '-' to read the file from stdin. } fn test_subcommand<'a, 'b>() -> App<'a, 'b> { - runtime_args(SubCommand::with_name("test"), true) + runtime_args(SubCommand::with_name("test"), true, true) .setting(AppSettings::TrailingVarArg) .arg( Arg::with_name("no-run") @@ -3306,7 +3397,8 @@ mod tests { Flags { subcommand: DenoSubcommand::Compile { source_file: "https://deno.land/std/examples/colors.ts".to_string(), - output: None + output: None, + args: vec![], }, ..Flags::default() } @@ -3316,13 +3408,14 @@ mod tests { #[test] fn compile_with_flags() { #[rustfmt::skip] - let r = flags_from_vec_safe(svec!["deno", "compile", "--unstable", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--no-check", "--reload", "--lock", "lock.json", "--lock-write", "--cert", "example.crt", "--output", "colors", "https://deno.land/std/examples/colors.ts"]); + let r = flags_from_vec_safe(svec!["deno", "compile", "--unstable", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--no-check", "--reload", "--lock", "lock.json", "--lock-write", "--cert", "example.crt", "--cached-only", "--allow-read", "--allow-net", "--v8-flags=--help", "--seed", "1", "--output", "colors", "https://deno.land/std/examples/colors.ts", "foo", "bar"]); assert_eq!( r.unwrap(), Flags { subcommand: DenoSubcommand::Compile { source_file: "https://deno.land/std/examples/colors.ts".to_string(), - output: Some(PathBuf::from("colors")) + output: Some(PathBuf::from("colors")), + args: svec!["foo", "bar"], }, unstable: true, import_map_path: Some("import_map.json".to_string()), @@ -3333,6 +3426,11 @@ mod tests { lock: Some(PathBuf::from("lock.json")), lock_write: true, ca_file: Some("example.crt".to_string()), + cached_only: true, + allow_read: Some(vec![]), + allow_net: Some(vec![]), + v8_flags: svec!["--help", "--random-seed=1"], + seed: Some(1), ..Flags::default() } ); diff --git a/cli/http_util.rs b/cli/http_util.rs index cedaa1d0e3..df2a7d3a25 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -23,7 +23,7 @@ pub fn get_user_agent() -> String { /// proxies and doesn't follow redirects. pub fn create_http_client( user_agent: String, - ca_data: Option<&str>, + ca_data: Option>, ) -> Result { let mut headers = HeaderMap::new(); headers.insert(USER_AGENT, user_agent.parse().unwrap()); @@ -33,8 +33,7 @@ pub fn create_http_client( .use_rustls_tls(); if let Some(ca_data) = ca_data { - let ca_data_vec = ca_data.as_bytes().to_vec(); - let cert = reqwest::Certificate::from_pem(&ca_data_vec)?; + let cert = reqwest::Certificate::from_pem(&ca_data)?; builder = builder.add_root_certificate(cert); } @@ -156,9 +155,9 @@ pub async fn fetch_once( #[cfg(test)] mod tests { use super::*; - use std::fs::read_to_string; + use std::fs::read; - fn create_test_client(ca_data: Option<&str>) -> Client { + fn create_test_client(ca_data: Option>) -> Client { create_http_client("test_client".to_string(), ca_data).unwrap() } @@ -312,12 +311,20 @@ mod tests { // Relies on external http server. See target/debug/test_server let url = Url::parse("https://localhost:5545/cli/tests/fixture.json").unwrap(); - let ca_data: String = read_to_string( - test_util::root_path().join("std/http/testdata/tls/RootCA.pem"), + + let client = create_http_client( + get_user_agent(), + Some( + read( + test_util::root_path() + .join("std/http/testdata/tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap(), + ), ) .unwrap(); - let client = - create_http_client(get_user_agent(), Some(ca_data.as_str())).unwrap(); let result = fetch_once(client, &url, None).await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); @@ -337,12 +344,19 @@ mod tests { "https://localhost:5545/cli/tests/053_import_compression/gziped", ) .unwrap(); - let ca_data: String = read_to_string( - test_util::root_path().join("std/http/testdata/tls/RootCA.pem"), + let client = create_http_client( + get_user_agent(), + Some( + read( + test_util::root_path() + .join("std/http/testdata/tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap(), + ), ) .unwrap(); - let client = - create_http_client(get_user_agent(), Some(ca_data.as_str())).unwrap(); let result = fetch_once(client, &url, None).await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); @@ -361,12 +375,19 @@ mod tests { async fn test_fetch_with_cafile_with_etag() { let _http_server_guard = test_util::http_server(); let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap(); - let ca_data: String = read_to_string( - test_util::root_path().join("std/http/testdata/tls/RootCA.pem"), + let client = create_http_client( + get_user_agent(), + Some( + read( + test_util::root_path() + .join("std/http/testdata/tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap(), + ), ) .unwrap(); - let client = - create_http_client(get_user_agent(), Some(ca_data.as_str())).unwrap(); let result = fetch_once(client.clone(), &url, None).await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); @@ -394,12 +415,19 @@ mod tests { "https://localhost:5545/cli/tests/053_import_compression/brotli", ) .unwrap(); - let ca_data: String = read_to_string( - test_util::root_path().join("std/http/testdata/tls/RootCA.pem"), + let client = create_http_client( + get_user_agent(), + Some( + read( + test_util::root_path() + .join("std/http/testdata/tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap(), + ), ) .unwrap(); - let client = - create_http_client(get_user_agent(), Some(ca_data.as_str())).unwrap(); let result = fetch_once(client, &url, None).await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); diff --git a/cli/main.rs b/cli/main.rs index ac0d2f5915..932e465c02 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -113,7 +113,7 @@ fn create_web_worker_callback( .log_level .map_or(false, |l| l == log::Level::Debug), unstable: program_state.flags.unstable, - ca_filepath: program_state.flags.ca_file.clone(), + ca_data: program_state.ca_data.clone(), user_agent: http_util::get_user_agent(), seed: program_state.flags.seed, module_loader, @@ -189,7 +189,7 @@ pub fn create_main_worker( .log_level .map_or(false, |l| l == log::Level::Debug), unstable: program_state.flags.unstable, - ca_filepath: program_state.flags.ca_file.clone(), + ca_data: program_state.ca_data.clone(), user_agent: http_util::get_user_agent(), seed: program_state.flags.seed, js_error_create_fn: Some(js_error_create_fn), @@ -295,6 +295,7 @@ async fn compile_command( flags: Flags, source_file: String, output: Option, + args: Vec, ) -> Result<(), AnyError> { if !flags.unstable { exit_unstable("compile"); @@ -302,6 +303,8 @@ async fn compile_command( let debug = flags.log_level == Some(log::Level::Debug); + let run_flags = standalone::compile_to_runtime_flags(flags.clone(), args)?; + let module_specifier = ModuleSpecifier::resolve_url_or_path(&source_file)?; let program_state = ProgramState::new(flags.clone())?; @@ -330,8 +333,7 @@ async fn compile_command( colors::green("Compile"), module_specifier.to_string() ); - create_standalone_binary(bundle_str.as_bytes().to_vec(), output.clone()) - .await?; + create_standalone_binary(bundle_str, run_flags, output.clone()).await?; info!("{} {}", colors::green("Emit"), output.display()); @@ -1069,6 +1071,7 @@ fn init_v8_flags(v8_flags: &[String]) { let v8_flags_includes_help = v8_flags .iter() .any(|flag| flag == "-help" || flag == "--help"); + // Keep in sync with `standalone.rs`. let v8_flags = once("UNUSED_BUT_NECESSARY_ARG0".to_owned()) .chain(v8_flags.iter().cloned()) .collect::>(); @@ -1147,7 +1150,8 @@ fn get_subcommand( DenoSubcommand::Compile { source_file, output, - } => compile_command(flags, source_file, output).boxed_local(), + args, + } => compile_command(flags, source_file, output, args).boxed_local(), DenoSubcommand::Fmt { check, files, diff --git a/cli/program_state.rs b/cli/program_state.rs index 3000b355de..5eda6b3fa1 100644 --- a/cli/program_state.rs +++ b/cli/program_state.rs @@ -20,12 +20,13 @@ use deno_runtime::permissions::Permissions; use deno_core::error::anyhow; use deno_core::error::get_custom_error_class; use deno_core::error::AnyError; +use deno_core::error::Context; use deno_core::url::Url; use deno_core::ModuleSource; use deno_core::ModuleSpecifier; use std::collections::HashMap; use std::env; -use std::fs::read_to_string; +use std::fs::read; use std::sync::Arc; use std::sync::Mutex; @@ -51,6 +52,7 @@ pub struct ProgramState { pub lockfile: Option>>, pub maybe_import_map: Option, pub maybe_inspector_server: Option>, + pub ca_data: Option>, } impl ProgramState { @@ -59,12 +61,10 @@ impl ProgramState { let dir = deno_dir::DenoDir::new(custom_root)?; let deps_cache_location = dir.root.join("deps"); let http_cache = http_cache::HttpCache::new(&deps_cache_location); - let ca_file_path = - flags.ca_file.clone().or_else(|| env::var("DENO_CERT").ok()); - - let ca_data: Option = match ca_file_path.as_ref() { + let ca_file = flags.ca_file.clone().or_else(|| env::var("DENO_CERT").ok()); + let ca_data = match &ca_file { + Some(ca_file) => Some(read(ca_file).context("Failed to open ca file")?), None => None, - Some(ca_file_path) => Some(read_to_string(ca_file_path)?), }; let cache_usage = if flags.cached_only { @@ -81,7 +81,7 @@ impl ProgramState { http_cache, cache_usage, !flags.no_remote, - ca_data.as_deref(), + ca_data.clone(), )?; let lockfile = if let Some(filename) = &flags.lock { @@ -125,6 +125,7 @@ impl ProgramState { lockfile, maybe_import_map, maybe_inspector_server, + ca_data, }; Ok(Arc::new(program_state)) } diff --git a/cli/standalone.rs b/cli/standalone.rs index af38fd4ebb..8379358a02 100644 --- a/cli/standalone.rs +++ b/cli/standalone.rs @@ -1,11 +1,17 @@ use crate::colors; +use crate::flags::DenoSubcommand; use crate::flags::Flags; use crate::tokio_util; use crate::version; use deno_core::error::bail; use deno_core::error::type_error; use deno_core::error::AnyError; +use deno_core::error::Context; use deno_core::futures::FutureExt; +use deno_core::serde::Deserialize; +use deno_core::serde::Serialize; +use deno_core::serde_json; +use deno_core::v8_set_flags; use deno_core::ModuleLoader; use deno_core::ModuleSpecifier; use deno_core::OpState; @@ -15,43 +21,61 @@ use deno_runtime::worker::WorkerOptions; use std::cell::RefCell; use std::convert::TryInto; use std::env::current_exe; +use std::fs::read; use std::fs::File; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; use std::io::Write; +use std::iter::once; use std::path::PathBuf; use std::pin::Pin; use std::rc::Rc; use std::sync::Arc; +#[derive(Deserialize, Serialize)] +struct Metadata { + flags: Flags, + ca_data: Option>, +} + const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd"; /// This function will try to run this binary as a standalone binary /// produced by `deno compile`. It determines if this is a stanalone /// binary by checking for the magic trailer string `D3N0` at EOF-12. -/// After the magic trailer is a u64 pointer to the start of the JS -/// file embedded in the binary. This file is read, and run. If no -/// magic trailer is present, this function exits with Ok(()). +/// The magic trailer is followed by: +/// - a u64 pointer to the JS bundle embedded in the binary +/// - a u64 pointer to JSON metadata (serialized flags) embedded in the binary +/// These are dereferenced, and the bundle is executed under the configuration +/// specified by the metadata. If no magic trailer is present, this function +/// exits with `Ok(())`. pub fn try_run_standalone_binary(args: Vec) -> Result<(), AnyError> { let current_exe_path = current_exe()?; let mut current_exe = File::open(current_exe_path)?; - let trailer_pos = current_exe.seek(SeekFrom::End(-16))?; - let mut trailer = [0; 16]; + let trailer_pos = current_exe.seek(SeekFrom::End(-24))?; + let mut trailer = [0; 24]; current_exe.read_exact(&mut trailer)?; - let (magic_trailer, bundle_pos_arr) = trailer.split_at(8); + let (magic_trailer, rest) = trailer.split_at(8); if magic_trailer == MAGIC_TRAILER { - let bundle_pos_arr: &[u8; 8] = bundle_pos_arr.try_into()?; - let bundle_pos = u64::from_be_bytes(*bundle_pos_arr); + let (bundle_pos, rest) = rest.split_at(8); + let metadata_pos = rest; + let bundle_pos = u64_from_bytes(bundle_pos)?; + let metadata_pos = u64_from_bytes(metadata_pos)?; + let bundle_len = metadata_pos - bundle_pos; + let metadata_len = trailer_pos - metadata_pos; current_exe.seek(SeekFrom::Start(bundle_pos))?; - let bundle_len = trailer_pos - bundle_pos; - let mut bundle = String::new(); - current_exe.take(bundle_len).read_to_string(&mut bundle)?; - // TODO: check amount of bytes read + let bundle = read_string_slice(&mut current_exe, bundle_pos, bundle_len) + .context("Failed to read source bundle from the current executable")?; + let metadata = + read_string_slice(&mut current_exe, metadata_pos, metadata_len) + .context("Failed to read metadata from the current executable")?; - if let Err(err) = tokio_util::run_basic(run(bundle, args)) { + let mut metadata: Metadata = serde_json::from_str(&metadata).unwrap(); + metadata.flags.argv.append(&mut args[1..].to_vec()); + if let Err(err) = tokio_util::run_basic(run(bundle, metadata)) { eprintln!("{}: {}", colors::red_bold("error"), err.to_string()); std::process::exit(1); } @@ -61,6 +85,25 @@ pub fn try_run_standalone_binary(args: Vec) -> Result<(), AnyError> { } } +fn u64_from_bytes(arr: &[u8]) -> Result { + let fixed_arr: &[u8; 8] = arr + .try_into() + .context("Failed to convert the buffer into a fixed-size array")?; + Ok(u64::from_be_bytes(*fixed_arr)) +} + +fn read_string_slice( + file: &mut File, + pos: u64, + len: u64, +) -> Result { + let mut string = String::new(); + file.seek(SeekFrom::Start(pos))?; + file.take(len).read_to_string(&mut string)?; + // TODO: check amount of bytes read + Ok(string) +} + const SPECIFIER: &str = "file://$deno$/bundle.js"; struct EmbeddedModuleLoader(String); @@ -106,28 +149,30 @@ impl ModuleLoader for EmbeddedModuleLoader { } } -async fn run(source_code: String, args: Vec) -> Result<(), AnyError> { - let flags = Flags { - argv: args[1..].to_vec(), - // TODO(lucacasonato): remove once you can specify this correctly through embedded metadata - unstable: true, - ..Default::default() - }; +async fn run(source_code: String, metadata: Metadata) -> Result<(), AnyError> { + let Metadata { flags, ca_data } = metadata; let main_module = ModuleSpecifier::resolve_url(SPECIFIER)?; - let permissions = Permissions::allow_all(); + let permissions = Permissions::from_options(&flags.clone().into()); let module_loader = Rc::new(EmbeddedModuleLoader(source_code)); let create_web_worker_cb = Arc::new(|_| { todo!("Worker are currently not supported in standalone binaries"); }); + // Keep in sync with `main.rs`. + v8_set_flags( + once("UNUSED_BUT_NECESSARY_ARG0".to_owned()) + .chain(flags.v8_flags.iter().cloned()) + .collect::>(), + ); + // TODO(nayeemrmn): Unify this Flags -> WorkerOptions mapping with `deno run`. let options = WorkerOptions { apply_source_maps: false, args: flags.argv.clone(), - debug_flag: false, + debug_flag: flags.log_level.map_or(false, |l| l == log::Level::Debug), user_agent: crate::http_util::get_user_agent(), - unstable: true, - ca_filepath: None, - seed: None, + unstable: flags.unstable, + ca_data, + seed: flags.seed, js_error_create_fn: None, create_web_worker_cb, attach_inspector: false, @@ -152,19 +197,31 @@ async fn run(source_code: String, args: Vec) -> Result<(), AnyError> { /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. pub async fn create_standalone_binary( - mut source_code: Vec, + source_code: String, + flags: Flags, output: PathBuf, ) -> Result<(), AnyError> { + let mut source_code = source_code.as_bytes().to_vec(); + let ca_data = match &flags.ca_file { + Some(ca_file) => Some(read(ca_file)?), + None => None, + }; + let metadata = Metadata { flags, ca_data }; + let mut metadata = serde_json::to_string(&metadata)?.as_bytes().to_vec(); let original_binary_path = std::env::current_exe()?; let mut original_bin = tokio::fs::read(original_binary_path).await?; + let bundle_pos = original_bin.len(); + let metadata_pos = bundle_pos + source_code.len(); let mut trailer = MAGIC_TRAILER.to_vec(); - trailer.write_all(&original_bin.len().to_be_bytes())?; + trailer.write_all(&bundle_pos.to_be_bytes())?; + trailer.write_all(&metadata_pos.to_be_bytes())?; let mut final_bin = Vec::with_capacity(original_bin.len() + source_code.len() + trailer.len()); final_bin.append(&mut original_bin); final_bin.append(&mut source_code); + final_bin.append(&mut metadata); final_bin.append(&mut trailer); let output = @@ -181,13 +238,18 @@ pub async fn create_standalone_binary( } // Make sure we don't overwrite any file not created by Deno compiler. - // Check for magic trailer in last 16 bytes + // Check for magic trailer in last 24 bytes. + let mut has_trailer = false; let mut output_file = File::open(&output)?; - output_file.seek(SeekFrom::End(-16))?; - let mut trailer = [0; 16]; - output_file.read_exact(&mut trailer)?; - let (magic_trailer, _) = trailer.split_at(8); - if magic_trailer != MAGIC_TRAILER { + // This seek may fail because the file is too small to possibly be + // `deno compile` output. + if output_file.seek(SeekFrom::End(-24)).is_ok() { + let mut trailer = [0; 24]; + output_file.read_exact(&mut trailer)?; + let (magic_trailer, _) = trailer.split_at(8); + has_trailer = magic_trailer == MAGIC_TRAILER; + } + if !has_trailer { bail!("Could not compile: cannot overwrite {:?}.", &output); } } @@ -201,3 +263,52 @@ pub async fn create_standalone_binary( Ok(()) } + +/// Transform the flags passed to `deno compile` to flags that would be used at +/// runtime, as if `deno run` were used. +/// - Flags that affect module resolution, loading, type checking, etc. aren't +/// applicable at runtime so are set to their defaults like `false`. +/// - Other flags are inherited. +pub fn compile_to_runtime_flags( + flags: Flags, + baked_args: Vec, +) -> Result { + // IMPORTANT: Don't abbreviate any of this to `..flags` or + // `..Default::default()`. That forces us to explicitly consider how any + // change to `Flags` should be reflected here. + Ok(Flags { + argv: baked_args, + subcommand: DenoSubcommand::Run { + script: "placeholder".to_string(), + }, + allow_env: flags.allow_env, + allow_hrtime: flags.allow_hrtime, + allow_net: flags.allow_net, + allow_plugin: flags.allow_plugin, + allow_read: flags.allow_read, + allow_run: flags.allow_run, + allow_write: flags.allow_write, + cache_blocklist: vec![], + ca_file: flags.ca_file, + cached_only: false, + config_path: None, + coverage_dir: flags.coverage_dir, + ignore: vec![], + import_map_path: None, + inspect: None, + inspect_brk: None, + lock: None, + lock_write: false, + log_level: flags.log_level, + no_check: false, + no_prompts: flags.no_prompts, + no_remote: false, + reload: false, + repl: false, + seed: flags.seed, + unstable: flags.unstable, + v8_flags: flags.v8_flags, + version: false, + watch: false, + }) +} diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 199172e2c2..dd76c5782d 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -4686,6 +4686,8 @@ fn standalone_args() { .arg("--output") .arg(&exe) .arg("./cli/tests/028_args.ts") + .arg("a") + .arg("b") .stdout(std::process::Stdio::piped()) .spawn() .unwrap() @@ -4702,7 +4704,7 @@ fn standalone_args() { .wait_with_output() .unwrap(); assert!(output.status.success()); - assert_eq!(output.stdout, b"foo\n--bar\n--unstable\n"); + assert_eq!(output.stdout, b"a\nb\nfoo\n--bar\n--unstable\n"); } #[test] @@ -4789,9 +4791,9 @@ fn compile_with_directory_exists_error() { .current_dir(util::root_path()) .arg("compile") .arg("--unstable") - .arg("./cli/tests/028_args.ts") .arg("--output") .arg(&exe) + .arg("./cli/tests/028_args.ts") .stderr(std::process::Stdio::piped()) .spawn() .unwrap() @@ -4818,9 +4820,9 @@ fn compile_with_conflict_file_exists_error() { .current_dir(util::root_path()) .arg("compile") .arg("--unstable") - .arg("./cli/tests/028_args.ts") .arg("--output") .arg(&exe) + .arg("./cli/tests/028_args.ts") .stderr(std::process::Stdio::piped()) .spawn() .unwrap() @@ -4830,6 +4832,7 @@ fn compile_with_conflict_file_exists_error() { let expected_stderr = format!("Could not compile: cannot overwrite {:?}.\n", &exe); let stderr = String::from_utf8(output.stderr).unwrap(); + dbg!(&stderr); assert!(stderr.contains(&expected_stderr)); assert!(std::fs::read(&exe) .expect("cannot read file") @@ -4848,9 +4851,9 @@ fn compile_and_overwrite_file() { .current_dir(util::root_path()) .arg("compile") .arg("--unstable") - .arg("./cli/tests/028_args.ts") .arg("--output") .arg(&exe) + .arg("./cli/tests/028_args.ts") .stderr(std::process::Stdio::piped()) .spawn() .unwrap() @@ -4863,9 +4866,9 @@ fn compile_and_overwrite_file() { .current_dir(util::root_path()) .arg("compile") .arg("--unstable") - .arg("./cli/tests/028_args.ts") .arg("--output") .arg(&exe) + .arg("./cli/tests/028_args.ts") .stderr(std::process::Stdio::piped()) .spawn() .unwrap() @@ -4873,3 +4876,42 @@ fn compile_and_overwrite_file() { .unwrap(); assert!(recompile_output.status.success()); } + +#[test] +fn standalone_runtime_flags() { + let dir = TempDir::new().expect("tempdir fail"); + let exe = if cfg!(windows) { + dir.path().join("flags.exe") + } else { + dir.path().join("flags") + }; + let output = util::deno_cmd() + .current_dir(util::root_path()) + .arg("compile") + .arg("--unstable") + .arg("--allow-read") + .arg("--seed") + .arg("1") + .arg("--output") + .arg(&exe) + .arg("./cli/tests/standalone_runtime_flags.ts") + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(output.status.success()); + let output = Command::new(exe) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(!output.status.success()); + let stdout_str = String::from_utf8(output.stdout).unwrap(); + assert_eq!(util::strip_ansi_codes(&stdout_str), "0.147205063401058\n"); + let stderr_str = String::from_utf8(output.stderr).unwrap(); + assert!(util::strip_ansi_codes(&stderr_str) + .contains("PermissionDenied: write access")); +} diff --git a/cli/tests/standalone_runtime_flags.ts b/cli/tests/standalone_runtime_flags.ts new file mode 100644 index 0000000000..0154c7f4e8 --- /dev/null +++ b/cli/tests/standalone_runtime_flags.ts @@ -0,0 +1,3 @@ +console.log(Math.random()); +await Deno.stat("."); +await Deno.create("foo.txt"); diff --git a/docs/tools.md b/docs/tools.md index 87a3c936b3..22a0695127 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -5,6 +5,7 @@ and TypeScript: - [bundler (`deno bundle`)](./tools/bundler.md) - [compiling executables (`deno compile`)](./tools/compiler.md) +- [installer (`deno install`)](./tools/script_installer.md) - [dependency inspector (`deno info`)](./tools/dependency_inspector.md) - [documentation generator (`deno doc`)](./tools/documentation_generator.md) - [formatter (`deno fmt`)](./tools/formatter.md) diff --git a/docs/tools/compiler.md b/docs/tools/compiler.md index 34dbbdcc84..f468a56c62 100644 --- a/docs/tools/compiler.md +++ b/docs/tools/compiler.md @@ -3,16 +3,34 @@ > Since the compile functionality is relatively new, the `--unstable` flag has > to be set in order for the command to work. -`deno compile [SRC] [OUT]` will compile the script into a self contained -executable. +`deno compile [--output ] ` will compile the script into a +self-contained executable. ``` -> deno compile --unstable https://deno.land/std/http/file_server.ts +> deno compile --unstable https://deno.land/std/examples/welcome.ts ``` If you omit the `OUT` parameter, the name of the executable file will be inferred. +### Flags + +As with [`deno install`](./script_installer.md), the runtime flags used to +execute the script must be specified at compilation time. This includes +permission flags. + +``` +> deno compile --unstable --allow-read --allow-net https://deno.land/std/http/file_server.ts +``` + +[Script arguments](../getting_started/command_line_interface.md#script-arguments) +can be partially embedded. + +``` +> deno compile --unstable --allow-read --allow-net https://deno.land/std/http/file_server.ts -p 8080 +> ./file_server --help +``` + ### Cross Compilation Cross compiling binaries for different platforms is not currently possible. diff --git a/runtime/examples/hello_runtime.rs b/runtime/examples/hello_runtime.rs index dbe539281d..c93bbc90b1 100644 --- a/runtime/examples/hello_runtime.rs +++ b/runtime/examples/hello_runtime.rs @@ -26,7 +26,7 @@ async fn main() -> Result<(), AnyError> { args: vec![], debug_flag: false, unstable: false, - ca_filepath: None, + ca_data: None, user_agent: "hello_runtime".to_string(), seed: None, js_error_create_fn: None, diff --git a/runtime/http_util.rs b/runtime/http_util.rs index 67703c214a..4fe4dc7ec0 100644 --- a/runtime/http_util.rs +++ b/runtime/http_util.rs @@ -7,14 +7,12 @@ use deno_fetch::reqwest::header::HeaderMap; use deno_fetch::reqwest::header::USER_AGENT; use deno_fetch::reqwest::redirect::Policy; use deno_fetch::reqwest::Client; -use std::fs::File; -use std::io::Read; /// Create new instance of async reqwest::Client. This client supports /// proxies and doesn't follow redirects. pub fn create_http_client( user_agent: String, - ca_file: Option<&str>, + ca_data: Option>, ) -> Result { let mut headers = HeaderMap::new(); headers.insert(USER_AGENT, user_agent.parse().unwrap()); @@ -23,10 +21,8 @@ pub fn create_http_client( .default_headers(headers) .use_rustls_tls(); - if let Some(ca_file) = ca_file { - let mut buf = Vec::new(); - File::open(ca_file)?.read_to_end(&mut buf)?; - let cert = reqwest::Certificate::from_pem(&buf)?; + if let Some(ca_data) = ca_data { + let cert = reqwest::Certificate::from_pem(&ca_data)?; builder = builder.add_root_certificate(cert); } diff --git a/runtime/ops/fetch.rs b/runtime/ops/fetch.rs index 0ef99f73d5..c2f62d8ee6 100644 --- a/runtime/ops/fetch.rs +++ b/runtime/ops/fetch.rs @@ -6,13 +6,13 @@ use deno_fetch::reqwest; pub fn init( rt: &mut deno_core::JsRuntime, user_agent: String, - maybe_ca_file: Option<&str>, + ca_data: Option>, ) { { let op_state = rt.op_state(); let mut state = op_state.borrow_mut(); state.put::({ - http_util::create_http_client(user_agent, maybe_ca_file).unwrap() + http_util::create_http_client(user_agent, ca_data).unwrap() }); } super::reg_json_async(rt, "op_fetch", deno_fetch::op_fetch::); diff --git a/runtime/ops/websocket.rs b/runtime/ops/websocket.rs index a5681bc527..812844f393 100644 --- a/runtime/ops/websocket.rs +++ b/runtime/ops/websocket.rs @@ -23,8 +23,8 @@ use http::{Method, Request, Uri}; use serde::Deserialize; use std::borrow::Cow; use std::cell::RefCell; -use std::fs::File; use std::io::BufReader; +use std::io::Cursor; use std::rc::Rc; use std::sync::Arc; use tokio::net::TcpStream; @@ -39,20 +39,20 @@ use tokio_tungstenite::{client_async, WebSocketStream}; use webpki::DNSNameRef; #[derive(Clone)] -struct WsCaFile(String); +struct WsCaData(Vec); #[derive(Clone)] struct WsUserAgent(String); pub fn init( rt: &mut deno_core::JsRuntime, - maybe_ca_file: Option<&str>, + ca_data: Option>, user_agent: String, ) { { let op_state = rt.op_state(); let mut state = op_state.borrow_mut(); - if let Some(ca_file) = maybe_ca_file { - state.put::(WsCaFile(ca_file.to_string())); + if let Some(ca_data) = ca_data { + state.put::(WsCaData(ca_data)); } state.put::(WsUserAgent(user_agent)); } @@ -130,7 +130,7 @@ pub async fn op_ws_create( ); } - let maybe_ca_file = state.borrow().try_borrow::().cloned(); + let ws_ca_data = state.borrow().try_borrow::().cloned(); let user_agent = state.borrow().borrow::().0.clone(); let uri: Uri = args.url.parse()?; let mut request = Request::builder().method(Method::GET).uri(&uri); @@ -163,9 +163,8 @@ pub async fn op_ws_create( .root_store .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS); - if let Some(ws_ca_file) = maybe_ca_file { - let key_file = File::open(ws_ca_file.0)?; - let reader = &mut BufReader::new(key_file); + if let Some(ws_ca_data) = ws_ca_data { + let reader = &mut BufReader::new(Cursor::new(ws_ca_data.0)); config.root_store.add_pem_file(reader).unwrap(); } diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index c1713f8150..988845840f 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -137,7 +137,7 @@ pub struct WebWorkerOptions { pub args: Vec, pub debug_flag: bool, pub unstable: bool, - pub ca_filepath: Option, + pub ca_data: Option>, pub user_agent: String, pub seed: Option, pub module_loader: Rc, @@ -219,7 +219,7 @@ impl WebWorker { ops::fetch::init( js_runtime, options.user_agent.clone(), - options.ca_filepath.as_deref(), + options.ca_data.clone(), ); ops::timers::init(js_runtime); ops::worker_host::init( @@ -237,7 +237,7 @@ impl WebWorker { ops::io::init(js_runtime); ops::websocket::init( js_runtime, - options.ca_filepath.as_deref(), + options.ca_data.clone(), options.user_agent.clone(), ); @@ -483,7 +483,7 @@ mod tests { apply_source_maps: false, debug_flag: false, unstable: false, - ca_filepath: None, + ca_data: None, user_agent: "x".to_string(), seed: None, module_loader, diff --git a/runtime/worker.rs b/runtime/worker.rs index 58a35cc950..a05c9f758c 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -45,7 +45,7 @@ pub struct WorkerOptions { pub args: Vec, pub debug_flag: bool, pub unstable: bool, - pub ca_filepath: Option, + pub ca_data: Option>, pub user_agent: String, pub seed: Option, pub module_loader: Rc, @@ -114,7 +114,7 @@ impl MainWorker { ops::fetch::init( js_runtime, options.user_agent.clone(), - options.ca_filepath.as_deref(), + options.ca_data.clone(), ); ops::timers::init(js_runtime); ops::worker_host::init( @@ -143,7 +143,7 @@ impl MainWorker { ops::tty::init(js_runtime); ops::websocket::init( js_runtime, - options.ca_filepath.as_deref(), + options.ca_data.clone(), options.user_agent.clone(), ); } @@ -270,7 +270,7 @@ mod tests { args: vec![], debug_flag: false, unstable: false, - ca_filepath: None, + ca_data: None, seed: None, js_error_create_fn: None, create_web_worker_cb: Arc::new(|_| unreachable!()),