// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. use std::collections::BTreeMap; use std::env::current_exe; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; use std::io::Write; use std::path::Path; use std::path::PathBuf; use deno_ast::ModuleSpecifier; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::io::AllowStdIo; use deno_core::futures::AsyncReadExt; use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; use deno_npm::registry::PackageDepNpmSchemeValueParseError; use deno_npm::NpmSystemInfo; use deno_runtime::permissions::PermissionsOptions; use deno_semver::npm::NpmPackageReq; use deno_semver::npm::NpmVersionReqSpecifierParseError; use log::Level; use serde::Deserialize; use serde::Serialize; use crate::args::package_json::PackageJsonDepValueParseError; use crate::args::package_json::PackageJsonDeps; use crate::args::CaData; use crate::args::CliOptions; use crate::args::CompileFlags; use crate::args::PackageJsonDepsProvider; use crate::cache::DenoDir; use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClient; use crate::npm::CliNpmRegistryApi; use crate::npm::CliNpmResolver; use crate::npm::NpmCache; use crate::npm::NpmResolution; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd"; #[derive(Serialize, Deserialize)] enum SerializablePackageJsonDepValueParseError { SchemeValue(String), Specifier(String), Unsupported { scheme: String }, } impl SerializablePackageJsonDepValueParseError { pub fn from_err(err: PackageJsonDepValueParseError) -> Self { match err { PackageJsonDepValueParseError::SchemeValue(err) => { Self::SchemeValue(err.value) } PackageJsonDepValueParseError::Specifier(err) => { Self::Specifier(err.source.to_string()) } PackageJsonDepValueParseError::Unsupported { scheme } => { Self::Unsupported { scheme } } } } pub fn into_err(self) -> PackageJsonDepValueParseError { match self { SerializablePackageJsonDepValueParseError::SchemeValue(value) => { PackageJsonDepValueParseError::SchemeValue( PackageDepNpmSchemeValueParseError { value }, ) } SerializablePackageJsonDepValueParseError::Specifier(source) => { PackageJsonDepValueParseError::Specifier( NpmVersionReqSpecifierParseError { source: monch::ParseErrorFailureError::new(source), }, ) } SerializablePackageJsonDepValueParseError::Unsupported { scheme } => { PackageJsonDepValueParseError::Unsupported { scheme } } } } } #[derive(Serialize, Deserialize)] pub struct SerializablePackageJsonDeps( BTreeMap< String, Result, >, ); impl SerializablePackageJsonDeps { pub fn from_deps(deps: PackageJsonDeps) -> Self { Self( deps .into_iter() .map(|(name, req)| { let res = req.map_err(SerializablePackageJsonDepValueParseError::from_err); (name, res) }) .collect(), ) } pub fn into_deps(self) -> PackageJsonDeps { self .0 .into_iter() .map(|(name, res)| (name, res.map_err(|err| err.into_err()))) .collect() } } #[derive(Deserialize, Serialize)] pub struct Metadata { pub argv: Vec, pub unstable: bool, pub seed: Option, pub permissions: PermissionsOptions, pub location: Option, pub v8_flags: Vec, pub log_level: Option, pub ca_stores: Option>, pub ca_data: Option>, pub unsafely_ignore_certificate_errors: Option>, pub maybe_import_map: Option<(Url, String)>, pub entrypoint: ModuleSpecifier, /// Whether this uses a node_modules directory (true) or the global cache (false). pub node_modules_dir: bool, pub package_json_deps: Option, } pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result { let file_path = current_exe().unwrap(); let mut file = std::fs::File::open(file_path)?; file.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))?; let mut trailer = [0; TRAILER_SIZE]; file.read_exact(&mut trailer)?; let trailer = Trailer::parse(&trailer)?.unwrap(); file.seek(SeekFrom::Start(trailer.npm_vfs_pos))?; let mut vfs_data = vec![0; trailer.npm_vfs_len() as usize]; file.read_exact(&mut vfs_data)?; let mut dir: VirtualDirectory = serde_json::from_slice(&vfs_data)?; // align the name of the directory with the root dir dir.name = root_dir_path .file_name() .unwrap() .to_string_lossy() .to_string(); let fs_root = VfsRoot { dir, root_path: root_dir_path, start_file_offset: trailer.npm_files_pos, }; Ok(FileBackedVfs::new(file, fs_root)) } fn write_binary_bytes( writer: &mut impl Write, original_bin: Vec, metadata: &Metadata, eszip: eszip::EszipV2, npm_vfs: Option<&VirtualDirectory>, npm_files: &Vec>, ) -> Result<(), AnyError> { let metadata = serde_json::to_string(metadata)?.as_bytes().to_vec(); let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec(); let eszip_archive = eszip.into_bytes(); writer.write_all(&original_bin)?; writer.write_all(&eszip_archive)?; writer.write_all(&metadata)?; writer.write_all(&npm_vfs)?; for file in npm_files { writer.write_all(file)?; } // write the trailer, which includes the positions // of the data blocks in the file writer.write_all(&{ let eszip_pos = original_bin.len() as u64; let metadata_pos = eszip_pos + (eszip_archive.len() as u64); let npm_vfs_pos = metadata_pos + (metadata.len() as u64); let npm_files_pos = npm_vfs_pos + (npm_vfs.len() as u64); Trailer { eszip_pos, metadata_pos, npm_vfs_pos, npm_files_pos, } .as_bytes() })?; Ok(()) } pub fn is_standalone_binary(exe_path: &Path) -> bool { let Ok(mut output_file) = std::fs::File::open(exe_path) else { return false; }; if output_file .seek(SeekFrom::End(-(TRAILER_SIZE as i64))) .is_err() { // This seek may fail because the file is too small to possibly be // `deno compile` output. return false; } let mut trailer = [0; TRAILER_SIZE]; if output_file.read_exact(&mut trailer).is_err() { return false; }; let (magic_trailer, _) = trailer.split_at(8); magic_trailer == MAGIC_TRAILER } /// This function will try to run this binary as a standalone binary /// produced by `deno compile`. It determines if this is a standalone /// binary by skipping over the trailer width at the end of the file, /// then checking for the magic trailer string `d3n0l4nd`. If found, /// the bundle is executed. If not, this function exits with `Ok(None)`. pub async fn extract_standalone( exe_path: &Path, cli_args: Vec, ) -> Result, AnyError> { let file = std::fs::File::open(exe_path)?; let mut bufreader = deno_core::futures::io::BufReader::new(AllowStdIo::new(file)); let _trailer_pos = bufreader .seek(SeekFrom::End(-(TRAILER_SIZE as i64))) .await?; let mut trailer = [0; TRAILER_SIZE]; bufreader.read_exact(&mut trailer).await?; let trailer = match Trailer::parse(&trailer)? { None => return Ok(None), Some(trailer) => trailer, }; bufreader.seek(SeekFrom::Start(trailer.eszip_pos)).await?; let (eszip, loader) = eszip::EszipV2::parse(bufreader) .await .context("Failed to parse eszip header")?; let mut bufreader = loader.await.context("Failed to parse eszip archive")?; bufreader .seek(SeekFrom::Start(trailer.metadata_pos)) .await?; let mut metadata = String::new(); bufreader .take(trailer.metadata_len()) .read_to_string(&mut metadata) .await .context("Failed to read metadata from the current executable")?; let mut metadata: Metadata = serde_json::from_str(&metadata).unwrap(); metadata.argv.append(&mut cli_args[1..].to_vec()); Ok(Some((metadata, eszip))) } const TRAILER_SIZE: usize = std::mem::size_of::() + 8; // 8 bytes for the magic trailer string struct Trailer { eszip_pos: u64, metadata_pos: u64, npm_vfs_pos: u64, npm_files_pos: u64, } impl Trailer { pub fn parse(trailer: &[u8]) -> Result, AnyError> { let (magic_trailer, rest) = trailer.split_at(8); if magic_trailer != MAGIC_TRAILER { return Ok(None); } let (eszip_archive_pos, rest) = rest.split_at(8); let (metadata_pos, rest) = rest.split_at(8); let (npm_vfs_pos, npm_files_pos) = rest.split_at(8); let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?; let metadata_pos = u64_from_bytes(metadata_pos)?; let npm_vfs_pos = u64_from_bytes(npm_vfs_pos)?; let npm_files_pos = u64_from_bytes(npm_files_pos)?; Ok(Some(Trailer { eszip_pos: eszip_archive_pos, metadata_pos, npm_vfs_pos, npm_files_pos, })) } pub fn metadata_len(&self) -> u64 { self.npm_vfs_pos - self.metadata_pos } pub fn npm_vfs_len(&self) -> u64 { self.npm_files_pos - self.npm_vfs_pos } pub fn as_bytes(&self) -> Vec { let mut trailer = MAGIC_TRAILER.to_vec(); trailer.write_all(&self.eszip_pos.to_be_bytes()).unwrap(); trailer.write_all(&self.metadata_pos.to_be_bytes()).unwrap(); trailer.write_all(&self.npm_vfs_pos.to_be_bytes()).unwrap(); trailer .write_all(&self.npm_files_pos.to_be_bytes()) .unwrap(); trailer } } fn u64_from_bytes(arr: &[u8]) -> Result { let fixed_arr: &[u8; 8] = arr .try_into() .context("Failed to convert the buffer into a fixed-size array")?; Ok(u64::from_be_bytes(*fixed_arr)) } pub struct DenoCompileBinaryWriter<'a> { file_fetcher: &'a FileFetcher, client: &'a HttpClient, deno_dir: &'a DenoDir, npm_api: &'a CliNpmRegistryApi, npm_cache: &'a NpmCache, npm_resolution: &'a NpmResolution, npm_resolver: &'a CliNpmResolver, npm_system_info: NpmSystemInfo, package_json_deps_provider: &'a PackageJsonDepsProvider, } impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( file_fetcher: &'a FileFetcher, client: &'a HttpClient, deno_dir: &'a DenoDir, npm_api: &'a CliNpmRegistryApi, npm_cache: &'a NpmCache, npm_resolution: &'a NpmResolution, npm_resolver: &'a CliNpmResolver, npm_system_info: NpmSystemInfo, package_json_deps_provider: &'a PackageJsonDepsProvider, ) -> Self { Self { file_fetcher, client, deno_dir, npm_api, npm_cache, npm_resolver, npm_system_info, npm_resolution, package_json_deps_provider, } } pub async fn write_bin( &self, writer: &mut impl Write, eszip: eszip::EszipV2, module_specifier: &ModuleSpecifier, compile_flags: &CompileFlags, cli_options: &CliOptions, ) -> Result<(), AnyError> { // Select base binary based on target let original_binary = self.get_base_binary(compile_flags.target.clone()).await?; self .write_standalone_binary( writer, original_binary, eszip, module_specifier, cli_options, compile_flags, ) .await } async fn get_base_binary( &self, target: Option, ) -> Result, AnyError> { if target.is_none() { let path = std::env::current_exe()?; return Ok(std::fs::read(path)?); } let target = target.unwrap_or_else(|| env!("TARGET").to_string()); let binary_name = format!("deno-{target}.zip"); let binary_path_suffix = if crate::version::is_canary() { format!("canary/{}/{}", crate::version::GIT_COMMIT_HASH, binary_name) } else { format!("release/v{}/{}", env!("CARGO_PKG_VERSION"), binary_name) }; let download_directory = self.deno_dir.dl_folder_path(); let binary_path = download_directory.join(&binary_path_suffix); if !binary_path.exists() { self .download_base_binary(&download_directory, &binary_path_suffix) .await?; } let archive_data = std::fs::read(binary_path)?; let temp_dir = tempfile::TempDir::new()?; let base_binary_path = crate::tools::upgrade::unpack_into_dir( archive_data, target.contains("windows"), &temp_dir, )?; let base_binary = std::fs::read(base_binary_path)?; drop(temp_dir); // delete the temp dir Ok(base_binary) } async fn download_base_binary( &self, output_directory: &Path, binary_path_suffix: &str, ) -> Result<(), AnyError> { let download_url = format!("https://dl.deno.land/{binary_path_suffix}"); let maybe_bytes = { let progress_bars = ProgressBar::new(ProgressBarStyle::DownloadBars); let progress = progress_bars.update(&download_url); self .client .download_with_progress(download_url, &progress) .await? }; let bytes = match maybe_bytes { Some(bytes) => bytes, None => { log::info!("Download could not be found, aborting"); std::process::exit(1) } }; std::fs::create_dir_all(output_directory)?; let output_path = output_directory.join(binary_path_suffix); std::fs::create_dir_all(output_path.parent().unwrap())?; tokio::fs::write(output_path, bytes).await?; Ok(()) } /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. async fn write_standalone_binary( &self, writer: &mut impl Write, original_bin: Vec, mut eszip: eszip::EszipV2, entrypoint: &ModuleSpecifier, cli_options: &CliOptions, compile_flags: &CompileFlags, ) -> Result<(), AnyError> { let ca_data = match cli_options.ca_data() { Some(CaData::File(ca_file)) => Some( std::fs::read(ca_file) .with_context(|| format!("Reading: {ca_file}"))?, ), Some(CaData::Bytes(bytes)) => Some(bytes.clone()), None => None, }; let maybe_import_map = cli_options .resolve_import_map(self.file_fetcher) .await? .map(|import_map| (import_map.base_url().clone(), import_map.to_json())); let (npm_vfs, npm_files) = if self.npm_resolution.has_packages() { let (root_dir, files) = self.build_vfs()?.into_dir_and_files(); let snapshot = self .npm_resolution .serialized_valid_snapshot_for_system(&self.npm_system_info); eszip.add_npm_snapshot(snapshot); (Some(root_dir), files) } else { (None, Vec::new()) }; let metadata = Metadata { argv: compile_flags.args.clone(), unstable: cli_options.unstable(), seed: cli_options.seed(), location: cli_options.location_flag().clone(), permissions: cli_options.permissions_options(), v8_flags: cli_options.v8_flags().clone(), unsafely_ignore_certificate_errors: cli_options .unsafely_ignore_certificate_errors() .clone(), log_level: cli_options.log_level(), ca_stores: cli_options.ca_stores().clone(), ca_data, entrypoint: entrypoint.clone(), maybe_import_map, node_modules_dir: self.npm_resolver.node_modules_path().is_some(), package_json_deps: self .package_json_deps_provider .deps() .map(|deps| SerializablePackageJsonDeps::from_deps(deps.clone())), }; write_binary_bytes( writer, original_bin, &metadata, eszip, npm_vfs.as_ref(), &npm_files, ) } fn build_vfs(&self) -> Result { if let Some(node_modules_path) = self.npm_resolver.node_modules_path() { let mut builder = VfsBuilder::new(node_modules_path.clone())?; builder.add_dir_recursive(&node_modules_path)?; Ok(builder) } else { // DO NOT include the user's registry url as it may contain credentials, // but also don't make this dependent on the registry url let registry_url = self.npm_api.base_url(); let root_path = self.npm_cache.registry_folder(registry_url); let mut builder = VfsBuilder::new(root_path)?; for package in self .npm_resolution .all_system_packages(&self.npm_system_info) { let folder = self .npm_resolver .resolve_pkg_folder_from_pkg_id(&package.id)?; builder.add_dir_recursive(&folder)?; } // overwrite the root directory's name to obscure the user's registry url builder.set_root_dir_name("node_modules".to_string()); Ok(builder) } } }