foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-23 15:49:44 -05:00
Code Issues Packages 1 Wiki Activity

feat(compile): unstable npm and node specifier support (#19005)

Browse source 
This is the initial support for npm and node specifiers in `deno
compile`. The npm packages are included in the binary and read from it via
a virtual file system. This also supports the `--node-modules-dir` flag,
dependencies specified in a package.json, and npm binary commands (ex.
`deno compile --unstable npm:cowsay`)

Closes #16632
This commit is contained in:
David Sherret 2023-05-10 20:06:59 -04:00 committed by David Sherret
parent 56a9a2a990
commit 75735fbf7f
44 changed files with 2733 additions and 261 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Cargo.lock generated
View file

@ -2966,9 +2966,9 @@ dependencies = [
[[package]]
name = "monch"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1120c1ab92ab8cdacb3b89ac9a214f512d2e78e90e3b57c00d9551ced19f646f"
checksum = "bb73e1dc7d232e1ab47ef27f45fa1d173a0979b370e763a9d0584556011150e0"
[[package]]
name = "napi-build"

2
cli/Cargo.toml
View file

@ -84,7 +84,7 @@ lazy-regex.workspace = true
libc.workspace = true
log = { workspace = true, features = ["serde"] }
lsp-types.workspace = true
monch = "=0.4.1"
monch = "=0.4.2"
notify.workspace = true
once_cell.workspace = true
os_pipe.workspace = true

7
cli/args/flags.rs
View file

@ -527,8 +527,11 @@ impl Flags {
.ok()
}
Task(_) | Check(_) | Coverage(_) | Cache(_) | Info(_) | Eval(_)
| Test(_) | Bench(_) | Repl(_) => std::env::current_dir().ok(),
_ => None,
| Test(_) | Bench(_) | Repl(_) | Compile(_) => {
std::env::current_dir().ok()
}
Bundle(_) | Completions(_) | Doc(_) | Fmt(_) | Init(_) | Install(_)
| Uninstall(_) | Lsp | Lint(_) | Types | Upgrade(_) | Vendor(_) => None,
}
}

13
cli/args/mod.rs
View file

@ -33,6 +33,7 @@ pub use config_file::TsTypeLib;
pub use flags::*;
pub use lockfile::Lockfile;
pub use lockfile::LockfileError;
pub use package_json::PackageJsonDepsProvider;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::anyhow;
@ -556,7 +557,7 @@ struct CliOptionOverrides {
import_map_specifier: Option<Option<ModuleSpecifier>>,
}
/// Holds the resolved options of many sources used by sub commands
/// Holds the resolved options of many sources used by subcommands
/// and provides some helper function for creating common objects.
pub struct CliOptions {
// the source of the options is a detail the rest of the
@ -1303,6 +1304,16 @@ fn has_flag_env_var(name: &str) -> bool {
matches!(value.as_ref().map(|s| s.as_str()), Ok("1"))
}
pub fn npm_pkg_req_ref_to_binary_command(
req_ref: &NpmPackageReqReference,
) -> String {
let binary_name = req_ref
.sub_path
.as_deref()
.unwrap_or(req_ref.req.name.as_str());
binary_name.to_string()
}
#[cfg(test)]
mod test {
use super::*;

27
cli/args/package_json.rs
View file

@ -28,6 +28,33 @@ pub enum PackageJsonDepValueParseError {
pub type PackageJsonDeps =
BTreeMap<String, Result<NpmPackageReq, PackageJsonDepValueParseError>>;
#[derive(Debug, Default)]
pub struct PackageJsonDepsProvider(Option<PackageJsonDeps>);
impl PackageJsonDepsProvider {
pub fn new(deps: Option<PackageJsonDeps>) -> Self {
Self(deps)
}
pub fn deps(&self) -> Option<&PackageJsonDeps> {
self.0.as_ref()
}
pub fn reqs(&self) -> Vec<&NpmPackageReq> {
match &self.0 {
Some(deps) => {
let mut package_reqs = deps
.values()
.filter_map(|r| r.as_ref().ok())
.collect::<Vec<_>>();
package_reqs.sort(); // deterministic resolution
package_reqs
}
None => Vec::new(),
}
}
}
/// Gets an application level package.json's npm package requirements.
///
/// Note that this function is not general purpose. It is specifically for

50
cli/factory.rs
View file

@ -1,9 +1,11 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use crate::args::npm_pkg_req_ref_to_binary_command;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::Flags;
use crate::args::Lockfile;
use crate::args::PackageJsonDepsProvider;
use crate::args::StorageKeyResolver;
use crate::args::TsConfigType;
use crate::cache::Caches;
@ -30,6 +32,7 @@ use crate::npm::NpmCache;
use crate::npm::NpmResolution;
use crate::npm::PackageJsonDepsInstaller;
use crate::resolver::CliGraphResolver;
use crate::standalone::DenoCompileBinaryWriter;
use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
@ -151,6 +154,7 @@ struct CliFactoryServices {
npm_cache: Deferred<Arc<NpmCache>>,
npm_resolver: Deferred<Arc<CliNpmResolver>>,
npm_resolution: Deferred<Arc<NpmResolution>>,
package_json_deps_provider: Deferred<Arc<PackageJsonDepsProvider>>,
package_json_deps_installer: Deferred<Arc<PackageJsonDepsInstaller>>,
text_only_progress_bar: Deferred<ProgressBar>,
type_checker: Deferred<Arc<TypeChecker>>,
@ -301,8 +305,9 @@ impl CliFactory {
.npm_resolver
.get_or_try_init_async(async {
let npm_resolution = self.npm_resolution().await?;
let fs = self.fs().clone();
let npm_fs_resolver = create_npm_fs_resolver(
self.fs().clone(),
fs.clone(),
self.npm_cache()?.clone(),
self.text_only_progress_bar(),
CliNpmRegistryApi::default_url().to_owned(),
@ -310,6 +315,7 @@ impl CliFactory {
self.options.node_modules_dir_path(),
);
Ok(Arc::new(CliNpmResolver::new(
fs.clone(),
npm_resolution.clone(),
npm_fs_resolver,
self.maybe_lockfile().as_ref().cloned(),
@ -318,6 +324,14 @@ impl CliFactory {
.await
}
pub fn package_json_deps_provider(&self) -> &Arc<PackageJsonDepsProvider> {
self.services.package_json_deps_provider.get_or_init(|| {
Arc::new(PackageJsonDepsProvider::new(
self.options.maybe_package_json_deps(),
))
})
}
pub async fn package_json_deps_installer(
&self,
) -> Result<&Arc<PackageJsonDepsInstaller>, AnyError> {
@ -325,12 +339,10 @@ impl CliFactory {
.services
.package_json_deps_installer
.get_or_try_init_async(async {
let npm_api = self.npm_api()?;
let npm_resolution = self.npm_resolution().await?;
Ok(Arc::new(PackageJsonDepsInstaller::new(
npm_api.clone(),
npm_resolution.clone(),
self.options.maybe_package_json_deps(),
self.package_json_deps_provider().clone(),
self.npm_api()?.clone(),
self.npm_resolution().await?.clone(),
)))
})
.await
@ -365,6 +377,7 @@ impl CliFactory {
self.options.no_npm(),
self.npm_api()?.clone(),
self.npm_resolution().await?.clone(),
self.package_json_deps_provider().clone(),
self.package_json_deps_installer().await?.clone(),
)))
})
@ -535,6 +548,21 @@ impl CliFactory {
self.services.cjs_resolutions.get_or_init(Default::default)
}
pub async fn create_compile_binary_writer(
&self,
) -> Result<DenoCompileBinaryWriter, AnyError> {
Ok(DenoCompileBinaryWriter::new(
self.file_fetcher()?,
self.http_client(),
self.deno_dir()?,
self.npm_api()?,
self.npm_cache()?,
self.npm_resolver().await?,
self.npm_resolution().await?,
self.package_json_deps_provider(),
))
}
/// Gets a function that can be used to create a CliMainWorkerFactory
/// for a file watcher.
pub async fn create_cli_main_worker_factory_func(
@ -572,6 +600,7 @@ impl CliFactory {
NpmModuleLoader::new(
cjs_resolutions.clone(),
node_code_translator.clone(),
fs.clone(),
node_resolver.clone(),
),
)),
@ -587,6 +616,7 @@ impl CliFactory {
&self,
) -> Result<CliMainWorkerFactory, AnyError> {
let node_resolver = self.node_resolver().await?;
let fs = self.fs();
Ok(CliMainWorkerFactory::new(
StorageKeyResolver::from_options(&self.options),
self.npm_resolver().await?.clone(),
@ -603,6 +633,7 @@ impl CliFactory {
NpmModuleLoader::new(
self.cjs_resolutions().clone(),
self.node_code_translator().await?.clone(),
fs.clone(),
node_resolver.clone(),
),
)),
@ -637,11 +668,8 @@ impl CliFactory {
if let Ok(pkg_ref) = NpmPackageReqReference::from_str(&flags.script) {
// if the user ran a binary command, we'll need to set process.argv[0]
// to be the name of the binary command instead of deno
let binary_name = pkg_ref
.sub_path
.as_deref()
.unwrap_or(pkg_ref.req.name.as_str());
maybe_binary_command_name = Some(binary_name.to_string());
maybe_binary_command_name =
Some(npm_pkg_req_ref_to_binary_command(&pkg_ref));
}
}
maybe_binary_command_name

4
cli/graph_util.rs
View file

@ -345,10 +345,10 @@ pub fn error_for_any_npm_specifier(
for module in graph.modules() {
match module {
Module::Npm(module) => {
bail!("npm specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: {}", module.specifier)
bail!("npm specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: {}", module.specifier)
}
Module::Node(module) => {
bail!("Node specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: node:{}", module.module_name)
bail!("Node specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: node:{}", module.module_name)
}
Module::Esm(_) | Module::Json(_) | Module::External(_) => {}
}

6
cli/lsp/documents.rs
View file

@ -46,6 +46,7 @@ use deno_semver::npm::NpmPackageReqReference;
use indexmap::IndexMap;
use lsp::Url;
use once_cell::sync::Lazy;
use package_json::PackageJsonDepsProvider;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
@ -1218,10 +1219,12 @@ impl Documents {
maybe_jsx_config.as_ref(),
maybe_package_json_deps.as_ref(),
);
let deps_provider =
Arc::new(PackageJsonDepsProvider::new(maybe_package_json_deps));
let deps_installer = Arc::new(PackageJsonDepsInstaller::new(
deps_provider.clone(),
npm_registry_api.clone(),
npm_resolution.clone(),
maybe_package_json_deps,
));
self.resolver = Arc::new(CliGraphResolver::new(
maybe_jsx_config,
@ -1229,6 +1232,7 @@ impl Documents {
false,
npm_registry_api,
npm_resolution,
deps_provider,
deps_installer,
));
self.imports = Arc::new(

11
cli/lsp/language_server.rs
View file

@ -457,8 +457,9 @@ fn create_lsp_structs(
));
let resolution =
Arc::new(NpmResolution::from_serialized(api.clone(), None, None));
let fs = Arc::new(deno_fs::RealFs);
let fs_resolver = create_npm_fs_resolver(
Arc::new(deno_fs::RealFs),
fs.clone(),
npm_cache.clone(),
&progress_bar,
registry_url.clone(),
@ -468,7 +469,12 @@ fn create_lsp_structs(
(
api,
npm_cache,
Arc::new(CliNpmResolver::new(resolution.clone(), fs_resolver, None)),
Arc::new(CliNpmResolver::new(
fs,
resolution.clone(),
fs_resolver,
None,
)),
resolution,
)
}
@ -711,6 +717,7 @@ impl Inner {
));
let node_fs = Arc::new(deno_fs::RealFs);
let npm_resolver = Arc::new(CliNpmResolver::new(
node_fs.clone(),
npm_resolution.clone(),
create_npm_fs_resolver(
node_fs.clone(),

3
cli/main.rs
View file

@ -36,7 +36,6 @@ static GLOBAL: Jemalloc = Jemalloc;
use crate::args::flags_from_vec;
use crate::args::DenoSubcommand;
use crate::args::Flags;
use crate::resolver::CliGraphResolver;
use crate::util::display;
use crate::util::v8::get_v8_flags_from_env;
use crate::util::v8::init_v8_flags;
@ -97,7 +96,7 @@ async fn run_subcommand(flags: Flags) -> Result<i32, AnyError> {
Ok(0)
}
DenoSubcommand::Compile(compile_flags) => {
tools::standalone::compile(flags, compile_flags).await?;
tools::compile::compile(flags, compile_flags).await?;
Ok(0)
}
DenoSubcommand::Coverage(coverage_flags) => {

76
cli/module_loader.rs
View file

@ -46,11 +46,13 @@ use deno_graph::JsonModule;
use deno_graph::Module;
use deno_graph::Resolution;
use deno_lockfile::Lockfile;
use deno_runtime::deno_fs;
use deno_runtime::deno_node;
use deno_runtime::deno_node::NodeResolution;
use deno_runtime::deno_node::NodeResolutionMode;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::permissions::PermissionsContainer;
use deno_semver::npm::NpmPackageNvReference;
use deno_semver::npm::NpmPackageReqReference;
use std::borrow::Cow;
use std::cell::RefCell;
@ -417,13 +419,12 @@ impl CliModuleLoader {
} else {
&self.root_permissions
};
let code_source = if let Some(code_source) =
self.shared.npm_module_loader.load_sync(
specifier,
maybe_referrer,
permissions,
)? {
code_source
let code_source = if let Some(result) = self
.shared
.npm_module_loader
.load_sync_if_in_npm_package(specifier, maybe_referrer, permissions)
{
result?
} else {
self
.shared
@ -494,7 +495,7 @@ impl ModuleLoader for CliModuleLoader {
Some(Module::Npm(module)) => self
.shared
.npm_module_loader
.resolve_npm_module(module, permissions),
.resolve_nv_ref(&module.nv_reference, permissions),
Some(Module::Node(module)) => {
deno_node::resolve_builtin_node_module(&module.module_name)
}
@ -547,7 +548,7 @@ impl ModuleLoader for CliModuleLoader {
return self
.shared
.npm_module_loader
.resolve_for_repl(&reference, permissions);
.resolve_req_reference(&reference, permissions);
}
}
}
@ -652,6 +653,7 @@ impl SourceMapGetter for CliSourceMapGetter {
pub struct NpmModuleLoader {
cjs_resolutions: Arc<CjsResolutionStore>,
node_code_translator: Arc<CliNodeCodeTranslator>,
fs: Arc<dyn deno_fs::FileSystem>,
node_resolver: Arc<NodeResolver>,
}
@ -659,11 +661,13 @@ impl NpmModuleLoader {
pub fn new(
cjs_resolutions: Arc<CjsResolutionStore>,
node_code_translator: Arc<CliNodeCodeTranslator>,
fs: Arc<dyn deno_fs::FileSystem>,
node_resolver: Arc<NodeResolver>,
) -> Self {
Self {
cjs_resolutions,
node_code_translator,
fs,
node_resolver,
}
}
@ -693,21 +697,21 @@ impl NpmModuleLoader {
}
}
pub fn resolve_npm_module(
pub fn resolve_nv_ref(
&self,
module: &deno_graph::NpmModule,
nv_ref: &NpmPackageNvReference,
permissions: &PermissionsContainer,
) -> Result<ModuleSpecifier, AnyError> {
self
.handle_node_resolve_result(self.node_resolver.resolve_npm_reference(
&module.nv_reference,
nv_ref,
NodeResolutionMode::Execution,
permissions,
))
.with_context(|| format!("Could not resolve '{}'.", module.nv_reference))
.with_context(|| format!("Could not resolve '{}'.", nv_ref))
}
pub fn resolve_for_repl(
pub fn resolve_req_reference(
&self,
reference: &NpmPackageReqReference,
permissions: &PermissionsContainer,
@ -733,25 +737,39 @@ impl NpmModuleLoader {
}
}
pub fn load_sync(
pub fn load_sync_if_in_npm_package(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
permissions: &PermissionsContainer,
) -> Result<Option<ModuleCodeSource>, AnyError> {
if !self.node_resolver.in_npm_package(specifier) {
return Ok(None);
) -> Option<Result<ModuleCodeSource, AnyError>> {
if self.node_resolver.in_npm_package(specifier) {
Some(self.load_sync(specifier, maybe_referrer, permissions))
} else {
None
}
}
fn load_sync(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
permissions: &PermissionsContainer,
) -> Result<ModuleCodeSource, AnyError> {
let file_path = specifier.to_file_path().unwrap();
let code = std::fs::read_to_string(&file_path).with_context(|| {
let mut msg = "Unable to load ".to_string();
msg.push_str(&file_path.to_string_lossy());
if let Some(referrer) = &maybe_referrer {
msg.push_str(" imported from ");
msg.push_str(referrer.as_str());
}
msg
})?;
let code = self
.fs
.read_to_string(&file_path)
.map_err(AnyError::from)
.with_context(|| {
let mut msg = "Unable to load ".to_string();
msg.push_str(&file_path.to_string_lossy());
if let Some(referrer) = &maybe_referrer {
msg.push_str(" imported from ");
msg.push_str(referrer.as_str());
}
msg
})?;
let code = if self.cjs_resolutions.contains(specifier) {
// translate cjs to esm if it's cjs and inject node globals
@ -766,11 +784,11 @@ impl NpmModuleLoader {
.node_code_translator
.esm_code_with_node_globals(specifier, &code)?
};
Ok(Some(ModuleCodeSource {
Ok(ModuleCodeSource {
code: code.into(),
found_url: specifier.clone(),
media_type: MediaType::from_specifier(specifier),
}))
})
}
fn handle_node_resolve_result(

26
cli/npm/installer.rs
View file

@ -10,7 +10,7 @@ use deno_npm::registry::NpmRegistryApi;
use deno_npm::registry::NpmRegistryPackageInfoLoadError;
use deno_semver::npm::NpmPackageReq;
use crate::args::package_json::PackageJsonDeps;
use crate::args::PackageJsonDepsProvider;
use crate::util::sync::AtomicFlag;
use super::CliNpmRegistryApi;
@ -18,23 +18,13 @@ use super::NpmResolution;
#[derive(Debug)]
struct PackageJsonDepsInstallerInner {
deps_provider: Arc<PackageJsonDepsProvider>,
has_installed_flag: AtomicFlag,
npm_registry_api: Arc<CliNpmRegistryApi>,
npm_resolution: Arc<NpmResolution>,
package_deps: PackageJsonDeps,
}
impl PackageJsonDepsInstallerInner {
pub fn reqs(&self) -> Vec<&NpmPackageReq> {
let mut package_reqs = self
.package_deps
.values()
.filter_map(|r| r.as_ref().ok())
.collect::<Vec<_>>();
package_reqs.sort(); // deterministic resolution
package_reqs
}
pub fn reqs_with_info_futures(
&self,
) -> FuturesOrdered<
@ -45,7 +35,7 @@ impl PackageJsonDepsInstallerInner {
>,
>,
> {
let package_reqs = self.reqs();
let package_reqs = self.deps_provider.reqs();
FuturesOrdered::from_iter(package_reqs.into_iter().map(|req| {
let api = self.npm_registry_api.clone();
@ -63,22 +53,18 @@ pub struct PackageJsonDepsInstaller(Option<PackageJsonDepsInstallerInner>);
impl PackageJsonDepsInstaller {
pub fn new(
deps_provider: Arc<PackageJsonDepsProvider>,
npm_registry_api: Arc<CliNpmRegistryApi>,
npm_resolution: Arc<NpmResolution>,
deps: Option<PackageJsonDeps>,
) -> Self {
Self(deps.map(|package_deps| PackageJsonDepsInstallerInner {
Self(Some(PackageJsonDepsInstallerInner {
deps_provider,
has_installed_flag: Default::default(),
npm_registry_api,
npm_resolution,
package_deps,
}))
}
pub fn package_deps(&self) -> Option<&PackageJsonDeps> {
self.0.as_ref().map(|inner| &inner.package_deps)
}
/// Installs the top level dependencies in the package.json file
/// without going through and resolving the descendant dependencies yet.
pub async fn ensure_top_level_install(&self) -> Result<(), AnyError> {

4
cli/npm/resolution.rs
View file

@ -237,6 +237,10 @@ impl NpmResolution {
Ok(nv)
}
pub fn all_packages(&self) -> Vec<NpmResolutionPackage> {
self.snapshot.read().all_packages()
}
pub fn all_packages_partitioned(&self) -> NpmPackagesPartitioned {
self.snapshot.read().all_packages_partitioned()
}

6
cli/npm/resolvers/common.rs
View file

@ -12,6 +12,7 @@ use deno_core::futures;
use deno_core::url::Url;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeResolutionMode;
@ -90,6 +91,7 @@ pub async fn cache_packages(
}
pub fn ensure_registry_read_permission(
fs: &Arc<dyn FileSystem>,
permissions: &dyn NodePermissions,
registry_path: &Path,
path: &Path,
@ -101,8 +103,8 @@ pub fn ensure_registry_read_permission(
.all(|c| !matches!(c, std::path::Component::ParentDir))
{
// todo(dsherret): cache this?
if let Ok(registry_path) = std::fs::canonicalize(registry_path) {
match std::fs::canonicalize(path) {
if let Ok(registry_path) = fs.realpath_sync(registry_path) {
match fs.realpath_sync(path) {
Ok(path) if path.starts_with(registry_path) => {
return Ok(());
}

6
cli/npm/resolvers/global.rs
View file

@ -14,6 +14,7 @@ use deno_npm::resolution::PackageNotFoundFromReferrerError;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeResolutionMode;
@ -28,6 +29,7 @@ use super::common::NpmPackageFsResolver;
/// Resolves packages from the global npm cache.
#[derive(Debug)]
pub struct GlobalNpmPackageResolver {
fs: Arc<dyn FileSystem>,
cache: Arc<NpmCache>,
resolution: Arc<NpmResolution>,
registry_url: Url,
@ -35,11 +37,13 @@ pub struct GlobalNpmPackageResolver {
impl GlobalNpmPackageResolver {
pub fn new(
fs: Arc<dyn FileSystem>,
cache: Arc<NpmCache>,
registry_url: Url,
resolution: Arc<NpmResolution>,
) -> Self {
Self {
fs,
cache,
resolution,
registry_url,
@ -130,7 +134,7 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver {
path: &Path,
) -> Result<(), AnyError> {
let registry_path = self.cache.registry_folder(&self.registry_url);
ensure_registry_read_permission(permissions, &registry_path, path)
ensure_registry_read_permission(&self.fs, permissions, &registry_path, path)
}
}

5
cli/npm/resolvers/local.rs
View file

@ -154,7 +154,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
loop {
current_folder = get_next_node_modules_ancestor(current_folder);
let sub_dir = join_package_name(current_folder, name);
if sub_dir.is_dir() {
if self.fs.is_dir(&sub_dir) {
// if doing types resolution, only resolve the package if it specifies a types property
if mode.is_types() && !name.starts_with("@types/") {
let package_json = PackageJson::load_skip_read_permission(
@ -173,7 +173,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
if mode.is_types() && !name.starts_with("@types/") {
let sub_dir =
join_package_name(current_folder, &types_package_name(name));
if sub_dir.is_dir() {
if self.fs.is_dir(&sub_dir) {
return Ok(sub_dir);
}
}
@ -214,6 +214,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
path: &Path,
) -> Result<(), AnyError> {
ensure_registry_read_permission(
&self.fs,
permissions,
&self.root_node_modules_path,
path,

26
cli/npm/resolvers/mod.rs
View file

@ -18,7 +18,7 @@ use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::resolution::PackageReqNotFoundError;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeResolutionMode;
use deno_runtime::deno_node::NpmResolver;
@ -32,7 +32,7 @@ use serde::Deserialize;
use serde::Serialize;
use crate::args::Lockfile;
use crate::util::fs::canonicalize_path_maybe_not_exists;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::progress_bar::ProgressBar;
use self::common::NpmPackageFsResolver;
@ -49,6 +49,7 @@ pub struct NpmProcessState {
/// Brings together the npm resolution with the file system.
pub struct CliNpmResolver {
fs: Arc<dyn FileSystem>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
resolution: Arc<NpmResolution>,
maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
@ -57,6 +58,7 @@ pub struct CliNpmResolver {
impl std::fmt::Debug for CliNpmResolver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NpmPackageResolver")
.field("fs", &"<omitted>")
.field("fs_resolver", &"<omitted>")
.field("resolution", &"<omitted>")
.field("maybe_lockfile", &"<omitted>")
@ -66,11 +68,13 @@ impl std::fmt::Debug for CliNpmResolver {
impl CliNpmResolver {
pub fn new(
fs: Arc<dyn FileSystem>,
resolution: Arc<NpmResolution>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
) -> Self {
Self {
fs,
fs_resolver,
resolution,
maybe_lockfile,
@ -81,6 +85,10 @@ impl CliNpmResolver {
self.fs_resolver.root_dir_url()
}
pub fn node_modules_path(&self) -> Option<PathBuf> {
self.fs_resolver.node_modules_path()
}
pub fn resolve_pkg_id_from_pkg_req(
&self,
req: &NpmPackageReq,
@ -88,12 +96,17 @@ impl CliNpmResolver {
self.resolution.resolve_pkg_id_from_pkg_req(req)
}
fn resolve_pkg_folder_from_deno_module_at_pkg_id(
pub fn resolve_pkg_folder_from_pkg_id(
&self,
pkg_id: &NpmPackageId,
) -> Result<PathBuf, AnyError> {
let path = self.fs_resolver.package_folder(pkg_id)?;
let path = canonicalize_path_maybe_not_exists(&path)?;
let path = canonicalize_path_maybe_not_exists_with_fs(&path, |path| {
self
.fs
.realpath_sync(path)
.map_err(|err| err.into_io_error())
})?;
log::debug!(
"Resolved package folder of {} to {}",
pkg_id.as_serialized(),
@ -237,7 +250,7 @@ impl NpmResolver for CliNpmResolver {
pkg_nv: &NpmPackageNv,
) -> Result<PathBuf, AnyError> {
let pkg_id = self.resolution.resolve_pkg_id_from_deno_module(pkg_nv)?;
self.resolve_pkg_folder_from_deno_module_at_pkg_id(&pkg_id)
self.resolve_pkg_folder_from_pkg_id(&pkg_id)
}
fn resolve_pkg_id_from_pkg_req(
@ -270,7 +283,7 @@ impl NpmResolver for CliNpmResolver {
}
pub fn create_npm_fs_resolver(
fs: Arc<dyn deno_fs::FileSystem>,
fs: Arc<dyn FileSystem>,
cache: Arc<NpmCache>,
progress_bar: &ProgressBar,
registry_url: Url,
@ -287,6 +300,7 @@ pub fn create_npm_fs_resolver(
resolution,
)),
None => Arc::new(GlobalNpmPackageResolver::new(
fs,
cache,
registry_url,
resolution,

116
cli/resolver.rs
View file

@ -20,16 +20,86 @@ use std::sync::Arc;
use crate::args::package_json::PackageJsonDeps;
use crate::args::JsxImportSourceConfig;
use crate::args::PackageJsonDepsProvider;
use crate::npm::CliNpmRegistryApi;
use crate::npm::NpmResolution;
use crate::npm::PackageJsonDepsInstaller;
use crate::util::sync::AtomicFlag;
/// Result of checking if a specifier is mapped via
/// an import map or package.json.
pub enum MappedResolution {
None,
PackageJson(ModuleSpecifier),
ImportMap(ModuleSpecifier),
}
impl MappedResolution {
pub fn into_specifier(self) -> Option<ModuleSpecifier> {
match self {
MappedResolution::None => Option::None,
MappedResolution::PackageJson(specifier) => Some(specifier),
MappedResolution::ImportMap(specifier) => Some(specifier),
}
}
}
/// Resolver for specifiers that could be mapped via an
/// import map or package.json.
#[derive(Debug)]
pub struct MappedSpecifierResolver {
maybe_import_map: Option<Arc<ImportMap>>,
package_json_deps_provider: Arc<PackageJsonDepsProvider>,
}
impl MappedSpecifierResolver {
pub fn new(
maybe_import_map: Option<Arc<ImportMap>>,
package_json_deps_provider: Arc<PackageJsonDepsProvider>,
) -> Self {
Self {
maybe_import_map,
package_json_deps_provider,
}
}
pub fn resolve(
&self,
specifier: &str,
referrer: &ModuleSpecifier,
) -> Result<MappedResolution, AnyError> {
// attempt to resolve with the import map first
let maybe_import_map_err = match self
.maybe_import_map
.as_ref()
.map(|import_map| import_map.resolve(specifier, referrer))
{
Some(Ok(value)) => return Ok(MappedResolution::ImportMap(value)),
Some(Err(err)) => Some(err),
None => None,
};
// then with package.json
if let Some(deps) = self.package_json_deps_provider.deps() {
if let Some(specifier) = resolve_package_json_dep(specifier, deps)? {
return Ok(MappedResolution::PackageJson(specifier));
}
}
// otherwise, surface the import map error or try resolving when has no import map
if let Some(err) = maybe_import_map_err {
Err(err.into())
} else {
Ok(MappedResolution::None)
}
}
}
/// A resolver that takes care of resolution, taking into account loaded
/// import map, JSX settings.
#[derive(Debug)]
pub struct CliGraphResolver {
maybe_import_map: Option<Arc<ImportMap>>,
mapped_specifier_resolver: MappedSpecifierResolver,
maybe_default_jsx_import_source: Option<String>,
maybe_jsx_import_source_module: Option<String>,
no_npm: bool,
@ -51,7 +121,10 @@ impl Default for CliGraphResolver {
None,
));
Self {
maybe_import_map: Default::default(),
mapped_specifier_resolver: MappedSpecifierResolver {
maybe_import_map: Default::default(),
package_json_deps_provider: Default::default(),
},
maybe_default_jsx_import_source: Default::default(),
maybe_jsx_import_source_module: Default::default(),
no_npm: false,
@ -71,10 +144,14 @@ impl CliGraphResolver {
no_npm: bool,
npm_registry_api: Arc<CliNpmRegistryApi>,
npm_resolution: Arc<NpmResolution>,
package_json_deps_provider: Arc<PackageJsonDepsProvider>,
package_json_deps_installer: Arc<PackageJsonDepsInstaller>,
) -> Self {
Self {
maybe_import_map,
mapped_specifier_resolver: MappedSpecifierResolver {
maybe_import_map,
package_json_deps_provider,
},
maybe_default_jsx_import_source: maybe_jsx_import_source_config
.as_ref()
.and_then(|c| c.default_specifier.clone()),
@ -135,31 +212,20 @@ impl Resolver for CliGraphResolver {
specifier: &str,
referrer: &ModuleSpecifier,
) -> Result<ModuleSpecifier, AnyError> {
// attempt to resolve with the import map first
let maybe_import_map_err = match self
.maybe_import_map
.as_ref()
.map(|import_map| import_map.resolve(specifier, referrer))
use MappedResolution::*;
match self
.mapped_specifier_resolver
.resolve(specifier, referrer)?
{
Some(Ok(value)) => return Ok(value),
Some(Err(err)) => Some(err),
None => None,
};
// then with package.json
if let Some(deps) = self.package_json_deps_installer.package_deps().as_ref()
{
if let Some(specifier) = resolve_package_json_dep(specifier, deps)? {
ImportMap(specifier) => Ok(specifier),
PackageJson(specifier) => {
// found a specifier in the package.json, so mark that
// we need to do an "npm install" later
self.found_package_json_dep_flag.raise();
return Ok(specifier);
Ok(specifier)
}
}
// otherwise, surface the import map error or try resolving when has no import map
if let Some(err) = maybe_import_map_err {
Err(err.into())
} else {
deno_graph::resolve_import(specifier, referrer).map_err(|err| err.into())
None => deno_graph::resolve_import(specifier, referrer)
.map_err(|err| err.into()),
}
}
}

314
cli/standalone/binary.rs
View file

@ -1,10 +1,13 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use std::collections::BTreeMap;
use std::env::current_exe;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::Context;
@ -14,22 +17,112 @@ use deno_core::futures::AsyncReadExt;
use deno_core::futures::AsyncSeekExt;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_npm::registry::PackageDepNpmSchemeValueParseError;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_runtime::permissions::PermissionsOptions;
use deno_semver::npm::NpmPackageReq;
use deno_semver::npm::NpmVersionReqSpecifierParseError;
use log::Level;
use serde::Deserialize;
use serde::Serialize;
use crate::args::package_json::PackageJsonDepValueParseError;
use crate::args::package_json::PackageJsonDeps;
use crate::args::CaData;
use crate::args::CliOptions;
use crate::args::CompileFlags;
use crate::args::PackageJsonDepsProvider;
use crate::cache::DenoDir;
use crate::file_fetcher::FileFetcher;
use crate::http_util::HttpClient;
use crate::npm::CliNpmRegistryApi;
use crate::npm::CliNpmResolver;
use crate::npm::NpmCache;
use crate::npm::NpmResolution;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use super::virtual_fs::FileBackedVfs;
use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VfsRoot;
use super::virtual_fs::VirtualDirectory;
const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd";
#[derive(Serialize, Deserialize)]
enum SerializablePackageJsonDepValueParseError {
SchemeValue(String),
Specifier(String),
Unsupported { scheme: String },
}
impl SerializablePackageJsonDepValueParseError {
pub fn from_err(err: PackageJsonDepValueParseError) -> Self {
match err {
PackageJsonDepValueParseError::SchemeValue(err) => {
Self::SchemeValue(err.value)
}
PackageJsonDepValueParseError::Specifier(err) => {
Self::Specifier(err.source.to_string())
}
PackageJsonDepValueParseError::Unsupported { scheme } => {
Self::Unsupported { scheme }
}
}
}
pub fn into_err(self) -> PackageJsonDepValueParseError {
match self {
SerializablePackageJsonDepValueParseError::SchemeValue(value) => {
PackageJsonDepValueParseError::SchemeValue(
PackageDepNpmSchemeValueParseError { value },
)
}
SerializablePackageJsonDepValueParseError::Specifier(source) => {
PackageJsonDepValueParseError::Specifier(
NpmVersionReqSpecifierParseError {
source: monch::ParseErrorFailureError::new(source),
},
)
}
SerializablePackageJsonDepValueParseError::Unsupported { scheme } => {
PackageJsonDepValueParseError::Unsupported { scheme }
}
}
}
}
#[derive(Serialize, Deserialize)]
pub struct SerializablePackageJsonDeps(
BTreeMap<
String,
Result<NpmPackageReq, SerializablePackageJsonDepValueParseError>,
>,
);
impl SerializablePackageJsonDeps {
pub fn from_deps(deps: PackageJsonDeps) -> Self {
Self(
deps
.into_iter()
.map(|(name, req)| {
let res =
req.map_err(SerializablePackageJsonDepValueParseError::from_err);
(name, res)
})
.collect(),
)
}
pub fn into_deps(self) -> PackageJsonDeps {
self
.0
.into_iter()
.map(|(name, res)| (name, res.map_err(|err| err.into_err())))
.collect()
}
}
#[derive(Deserialize, Serialize)]
pub struct Metadata {
pub argv: Vec<String>,
@ -44,27 +137,74 @@ pub struct Metadata {
pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
pub maybe_import_map: Option<(Url, String)>,
pub entrypoint: ModuleSpecifier,
/// Whether this uses a node_modules directory (true) or the global cache (false).
pub node_modules_dir: bool,
pub npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
pub package_json_deps: Option<SerializablePackageJsonDeps>,
}
pub fn write_binary_bytes(
pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result<FileBackedVfs, AnyError> {
let file_path = current_exe().unwrap();
let mut file = std::fs::File::open(file_path)?;
file.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))?;
let mut trailer = [0; TRAILER_SIZE];
file.read_exact(&mut trailer)?;
let trailer = Trailer::parse(&trailer)?.unwrap();
file.seek(SeekFrom::Start(trailer.npm_vfs_pos))?;
let mut vfs_data = vec![0; trailer.npm_vfs_len() as usize];
file.read_exact(&mut vfs_data)?;
let mut dir: VirtualDirectory = serde_json::from_slice(&vfs_data)?;
// align the name of the directory with the root dir
dir.name = root_dir_path
.file_name()
.unwrap()
.to_string_lossy()
.to_string();
let fs_root = VfsRoot {
dir,
root_path: root_dir_path,
start_file_offset: trailer.npm_files_pos,
};
Ok(FileBackedVfs::new(file, fs_root))
}
fn write_binary_bytes(
writer: &mut impl Write,
original_bin: Vec<u8>,
metadata: &Metadata,
eszip: eszip::EszipV2,
npm_vfs: Option<&VirtualDirectory>,
npm_files: &Vec<Vec<u8>>,
) -> Result<(), AnyError> {
let metadata = serde_json::to_string(metadata)?.as_bytes().to_vec();
let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec();
let eszip_archive = eszip.into_bytes();
let eszip_pos = original_bin.len();
let metadata_pos = eszip_pos + eszip_archive.len();
let mut trailer = MAGIC_TRAILER.to_vec();
trailer.write_all(&eszip_pos.to_be_bytes())?;
trailer.write_all(&metadata_pos.to_be_bytes())?;
writer.write_all(&original_bin)?;
writer.write_all(&eszip_archive)?;
writer.write_all(&metadata)?;
writer.write_all(&trailer)?;
writer.write_all(&npm_vfs)?;
for file in npm_files {
writer.write_all(file)?;
}
// write the trailer, which includes the positions
// of the data blocks in the file
writer.write_all(&{
let eszip_pos = original_bin.len() as u64;
let metadata_pos = eszip_pos + (eszip_archive.len() as u64);
let npm_vfs_pos = metadata_pos + (metadata.len() as u64);
let npm_files_pos = npm_vfs_pos + (npm_vfs.len() as u64);
Trailer {
eszip_pos,
metadata_pos,
npm_vfs_pos,
npm_files_pos,
}
.as_bytes()
})?;
Ok(())
}
@ -73,12 +213,15 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool {
let Ok(mut output_file) = std::fs::File::open(exe_path) else {
return false;
};
if output_file.seek(SeekFrom::End(-24)).is_err() {
if output_file
.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))
.is_err()
{
// This seek may fail because the file is too small to possibly be
// `deno compile` output.
return false;
}
let mut trailer = [0; 24];
let mut trailer = [0; TRAILER_SIZE];
if output_file.read_exact(&mut trailer).is_err() {
return false;
};
@ -88,13 +231,9 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool {
/// This function will try to run this binary as a standalone binary
/// produced by `deno compile`. It determines if this is a standalone
/// binary by checking for the magic trailer string `d3n0l4nd` at EOF-24 (8 bytes * 3).
/// The magic trailer is followed by:
/// - a u64 pointer to the JS bundle embedded in the binary
/// - a u64 pointer to JSON metadata (serialized flags) embedded in the binary
/// These are dereferenced, and the bundle is executed under the configuration
/// specified by the metadata. If no magic trailer is present, this function
/// exits with `Ok(None)`.
/// binary by skipping over the trailer width at the end of the file,
/// then checking for the magic trailer string `d3n0l4nd`. If found,
/// the bundle is executed. If not, this function exits with `Ok(None)`.
pub async fn extract_standalone(
exe_path: &Path,
cli_args: Vec<String>,
@ -104,21 +243,17 @@ pub async fn extract_standalone(
let mut bufreader =
deno_core::futures::io::BufReader::new(AllowStdIo::new(file));
let trailer_pos = bufreader.seek(SeekFrom::End(-24)).await?;
let mut trailer = [0; 24];
let _trailer_pos = bufreader
.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))
.await?;
let mut trailer = [0; TRAILER_SIZE];
bufreader.read_exact(&mut trailer).await?;
let (magic_trailer, rest) = trailer.split_at(8);
if magic_trailer != MAGIC_TRAILER {
return Ok(None);
}
let trailer = match Trailer::parse(&trailer)? {
None => return Ok(None),
Some(trailer) => trailer,
};
let (eszip_archive_pos, rest) = rest.split_at(8);
let metadata_pos = rest;
let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?;
let metadata_pos = u64_from_bytes(metadata_pos)?;
let metadata_len = trailer_pos - metadata_pos;
bufreader.seek(SeekFrom::Start(eszip_archive_pos)).await?;
bufreader.seek(SeekFrom::Start(trailer.eszip_pos)).await?;
let (eszip, loader) = eszip::EszipV2::parse(bufreader)
.await
@ -126,12 +261,14 @@ pub async fn extract_standalone(
let mut bufreader = loader.await.context("Failed to parse eszip archive")?;
bufreader.seek(SeekFrom::Start(metadata_pos)).await?;
bufreader
.seek(SeekFrom::Start(trailer.metadata_pos))
.await?;
let mut metadata = String::new();
bufreader
.take(metadata_len)
.take(trailer.metadata_len())
.read_to_string(&mut metadata)
.await
.context("Failed to read metadata from the current executable")?;
@ -142,6 +279,57 @@ pub async fn extract_standalone(
Ok(Some((metadata, eszip)))
}
const TRAILER_SIZE: usize = std::mem::size_of::<Trailer>() + 8; // 8 bytes for the magic trailer string
struct Trailer {
eszip_pos: u64,
metadata_pos: u64,
npm_vfs_pos: u64,
npm_files_pos: u64,
}
impl Trailer {
pub fn parse(trailer: &[u8]) -> Result<Option<Trailer>, AnyError> {
let (magic_trailer, rest) = trailer.split_at(8);
if magic_trailer != MAGIC_TRAILER {
return Ok(None);
}
let (eszip_archive_pos, rest) = rest.split_at(8);
let (metadata_pos, rest) = rest.split_at(8);
let (npm_vfs_pos, npm_files_pos) = rest.split_at(8);
let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?;
let metadata_pos = u64_from_bytes(metadata_pos)?;
let npm_vfs_pos = u64_from_bytes(npm_vfs_pos)?;
let npm_files_pos = u64_from_bytes(npm_files_pos)?;
Ok(Some(Trailer {
eszip_pos: eszip_archive_pos,
metadata_pos,
npm_vfs_pos,
npm_files_pos,
}))
}
pub fn metadata_len(&self) -> u64 {
self.npm_vfs_pos - self.metadata_pos
}
pub fn npm_vfs_len(&self) -> u64 {
self.npm_files_pos - self.npm_vfs_pos
}
pub fn as_bytes(&self) -> Vec<u8> {
let mut trailer = MAGIC_TRAILER.to_vec();
trailer.write_all(&self.eszip_pos.to_be_bytes()).unwrap();
trailer.write_all(&self.metadata_pos.to_be_bytes()).unwrap();
trailer.write_all(&self.npm_vfs_pos.to_be_bytes()).unwrap();
trailer
.write_all(&self.npm_files_pos.to_be_bytes())
.unwrap();
trailer
}
}
fn u64_from_bytes(arr: &[u8]) -> Result<u64, AnyError> {
let fixed_arr: &[u8; 8] = arr
.try_into()
@ -153,18 +341,34 @@ pub struct DenoCompileBinaryWriter<'a> {
file_fetcher: &'a FileFetcher,
client: &'a HttpClient,
deno_dir: &'a DenoDir,
npm_api: &'a CliNpmRegistryApi,
npm_cache: &'a NpmCache,
npm_resolver: &'a CliNpmResolver,
resolution: &'a NpmResolution,
package_json_deps_provider: &'a PackageJsonDepsProvider,
}
impl<'a> DenoCompileBinaryWriter<'a> {
#[allow(clippy::too_many_arguments)]
pub fn new(
file_fetcher: &'a FileFetcher,
client: &'a HttpClient,
deno_dir: &'a DenoDir,
npm_api: &'a CliNpmRegistryApi,
npm_cache: &'a NpmCache,
npm_resolver: &'a CliNpmResolver,
resolution: &'a NpmResolution,
package_json_deps_provider: &'a PackageJsonDepsProvider,
) -> Self {
Self {
file_fetcher,
client,
deno_dir,
npm_api,
npm_cache,
npm_resolver,
resolution,
package_json_deps_provider,
}
}
@ -284,6 +488,14 @@ impl<'a> DenoCompileBinaryWriter<'a> {
.resolve_import_map(self.file_fetcher)
.await?
.map(|import_map| (import_map.base_url().clone(), import_map.to_json()));
let (npm_snapshot, npm_vfs, npm_files) = if self.resolution.has_packages() {
let (root_dir, files) = self.build_vfs()?.into_dir_and_files();
let snapshot = self.resolution.serialized_snapshot();
(Some(snapshot), Some(root_dir), files)
} else {
(None, None, Vec::new())
};
let metadata = Metadata {
argv: compile_flags.args.clone(),
unstable: cli_options.unstable(),
@ -299,8 +511,44 @@ impl<'a> DenoCompileBinaryWriter<'a> {
ca_data,
entrypoint: entrypoint.clone(),
maybe_import_map,
node_modules_dir: self.npm_resolver.node_modules_path().is_some(),
npm_snapshot,
package_json_deps: self
.package_json_deps_provider
.deps()
.map(|deps| SerializablePackageJsonDeps::from_deps(deps.clone())),
};
write_binary_bytes(writer, original_bin, &metadata, eszip)
write_binary_bytes(
writer,
original_bin,
&metadata,
eszip,
npm_vfs.as_ref(),
&npm_files,
)
}
fn build_vfs(&self) -> Result<VfsBuilder, AnyError> {
if let Some(node_modules_path) = self.npm_resolver.node_modules_path() {
let mut builder = VfsBuilder::new(node_modules_path.clone());
builder.add_dir_recursive(&node_modules_path)?;
Ok(builder)
} else {
// DO NOT include the user's registry url as it may contain credentials,
// but also don't make this dependent on the registry url
let registry_url = self.npm_api.base_url();
let root_path = self.npm_cache.registry_folder(registry_url);
let mut builder = VfsBuilder::new(root_path);
for package in self.resolution.all_packages() {
let folder = self
.npm_resolver
.resolve_pkg_folder_from_pkg_id(&package.pkg_id)?;
builder.add_dir_recursive(&folder)?;
}
// overwrite the root directory's name to obscure the user's registry url
builder.set_root_dir_name("node_modules".to_string());
Ok(builder)
}
}
}

337
cli/standalone/file_system.rs Normal file
View file

@ -0,0 +1,337 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsDirEntry;
use deno_runtime::deno_fs::FsFileType;
use deno_runtime::deno_fs::OpenOptions;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_io::fs::File;
use deno_runtime::deno_io::fs::FsError;
use deno_runtime::deno_io::fs::FsResult;
use deno_runtime::deno_io::fs::FsStat;
use super::virtual_fs::FileBackedVfs;
#[derive(Debug, Clone)]
pub struct DenoCompileFileSystem(Arc<FileBackedVfs>);
impl DenoCompileFileSystem {
pub fn new(vfs: FileBackedVfs) -> Self {
Self(Arc::new(vfs))
}
fn error_if_in_vfs(&self, path: &Path) -> FsResult<()> {
if self.0.is_path_within(path) {
Err(FsError::NotSupported)
} else {
Ok(())
}
}
fn copy_to_real_path(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> {
let old_file = self.0.file_entry(oldpath)?;
let old_file_bytes = self.0.read_file_all(old_file)?;
RealFs.write_file_sync(
newpath,
OpenOptions {
read: false,
write: true,
create: true,
truncate: true,
append: false,
create_new: false,
mode: None,
},
&old_file_bytes,
)
}
}
#[async_trait::async_trait(?Send)]
impl FileSystem for DenoCompileFileSystem {
fn cwd(&self) -> FsResult<PathBuf> {
RealFs.cwd()
}
fn tmp_dir(&self) -> FsResult<PathBuf> {
RealFs.tmp_dir()
}
fn chdir(&self, path: &Path) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.chdir(path)
}
fn umask(&self, mask: Option<u32>) -> FsResult<u32> {
RealFs.umask(mask)
}
fn open_sync(
&self,
path: &Path,
options: OpenOptions,
) -> FsResult<Rc<dyn File>> {
if self.0.is_path_within(path) {
Ok(self.0.open_file(path)?)
} else {
RealFs.open_sync(path, options)
}
}
async fn open_async(
&self,
path: PathBuf,
options: OpenOptions,
) -> FsResult<Rc<dyn File>> {
if self.0.is_path_within(&path) {
Ok(self.0.open_file(&path)?)
} else {
RealFs.open_async(path, options).await
}
}
fn mkdir_sync(
&self,
path: &Path,
recursive: bool,
mode: u32,
) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.mkdir_sync(path, recursive, mode)
}
async fn mkdir_async(
&self,
path: PathBuf,
recursive: bool,
mode: u32,
) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.mkdir_async(path, recursive, mode).await
}
fn chmod_sync(&self, path: &Path, mode: u32) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.chmod_sync(path, mode)
}
async fn chmod_async(&self, path: PathBuf, mode: u32) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.chmod_async(path, mode).await
}
fn chown_sync(
&self,
path: &Path,
uid: Option<u32>,
gid: Option<u32>,
) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.chown_sync(path, uid, gid)
}
async fn chown_async(
&self,
path: PathBuf,
uid: Option<u32>,
gid: Option<u32>,
) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.chown_async(path, uid, gid).await
}
fn remove_sync(&self, path: &Path, recursive: bool) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.remove_sync(path, recursive)
}
async fn remove_async(&self, path: PathBuf, recursive: bool) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.remove_async(path, recursive).await
}
fn copy_file_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> {
self.error_if_in_vfs(newpath)?;
if self.0.is_path_within(oldpath) {
self.copy_to_real_path(oldpath, newpath)
} else {
RealFs.copy_file_sync(oldpath, newpath)
}
}
async fn copy_file_async(
&self,
oldpath: PathBuf,
newpath: PathBuf,
) -> FsResult<()> {
self.error_if_in_vfs(&newpath)?;
if self.0.is_path_within(&oldpath) {
let fs = self.clone();
tokio::task::spawn_blocking(move || {
fs.copy_to_real_path(&oldpath, &newpath)
})
.await?
} else {
RealFs.copy_file_async(oldpath, newpath).await
}
}
fn stat_sync(&self, path: &Path) -> FsResult<FsStat> {
if self.0.is_path_within(path) {
Ok(self.0.stat(path)?)
} else {
RealFs.stat_sync(path)
}
}
async fn stat_async(&self, path: PathBuf) -> FsResult<FsStat> {
if self.0.is_path_within(&path) {
Ok(self.0.stat(&path)?)
} else {
RealFs.stat_async(path).await
}
}
fn lstat_sync(&self, path: &Path) -> FsResult<FsStat> {
if self.0.is_path_within(path) {
Ok(self.0.lstat(path)?)
} else {
RealFs.lstat_sync(path)
}
}
async fn lstat_async(&self, path: PathBuf) -> FsResult<FsStat> {
if self.0.is_path_within(&path) {
Ok(self.0.lstat(&path)?)
} else {
RealFs.lstat_async(path).await
}
}
fn realpath_sync(&self, path: &Path) -> FsResult<PathBuf> {
if self.0.is_path_within(path) {
Ok(self.0.canonicalize(path)?)
} else {
RealFs.realpath_sync(path)
}
}
async fn realpath_async(&self, path: PathBuf) -> FsResult<PathBuf> {
if self.0.is_path_within(&path) {
Ok(self.0.canonicalize(&path)?)
} else {
RealFs.realpath_async(path).await
}
}
fn read_dir_sync(&self, path: &Path) -> FsResult<Vec<FsDirEntry>> {
if self.0.is_path_within(path) {
Ok(self.0.read_dir(path)?)
} else {
RealFs.read_dir_sync(path)
}
}
async fn read_dir_async(&self, path: PathBuf) -> FsResult<Vec<FsDirEntry>> {
if self.0.is_path_within(&path) {
Ok(self.0.read_dir(&path)?)
} else {
RealFs.read_dir_async(path).await
}
}
fn rename_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> {
self.error_if_in_vfs(oldpath)?;
self.error_if_in_vfs(newpath)?;
RealFs.rename_sync(oldpath, newpath)
}
async fn rename_async(
&self,
oldpath: PathBuf,
newpath: PathBuf,
) -> FsResult<()> {
self.error_if_in_vfs(&oldpath)?;
self.error_if_in_vfs(&newpath)?;
RealFs.rename_async(oldpath, newpath).await
}
fn link_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> {
self.error_if_in_vfs(oldpath)?;
self.error_if_in_vfs(newpath)?;
RealFs.link_sync(oldpath, newpath)
}
async fn link_async(
&self,
oldpath: PathBuf,
newpath: PathBuf,
) -> FsResult<()> {
self.error_if_in_vfs(&oldpath)?;
self.error_if_in_vfs(&newpath)?;
RealFs.link_async(oldpath, newpath).await
}
fn symlink_sync(
&self,
oldpath: &Path,
newpath: &Path,
file_type: Option<FsFileType>,
) -> FsResult<()> {
self.error_if_in_vfs(oldpath)?;
self.error_if_in_vfs(newpath)?;
RealFs.symlink_sync(oldpath, newpath, file_type)
}
async fn symlink_async(
&self,
oldpath: PathBuf,
newpath: PathBuf,
file_type: Option<FsFileType>,
) -> FsResult<()> {
self.error_if_in_vfs(&oldpath)?;
self.error_if_in_vfs(&newpath)?;
RealFs.symlink_async(oldpath, newpath, file_type).await
}
fn read_link_sync(&self, path: &Path) -> FsResult<PathBuf> {
if self.0.is_path_within(path) {
Ok(self.0.read_link(path)?)
} else {
RealFs.read_link_sync(path)
}
}
async fn read_link_async(&self, path: PathBuf) -> FsResult<PathBuf> {
if self.0.is_path_within(&path) {
Ok(self.0.read_link(&path)?)
} else {
RealFs.read_link_async(path).await
}
}
fn truncate_sync(&self, path: &Path, len: u64) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.truncate_sync(path, len)
}
async fn truncate_async(&self, path: PathBuf, len: u64) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.truncate_async(path, len).await
}
fn utime_sync(
&self,
path: &Path,
atime_secs: i64,
atime_nanos: u32,
mtime_secs: i64,
mtime_nanos: u32,
) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.utime_sync(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos)
}
async fn utime_async(
&self,
path: PathBuf,
atime_secs: i64,
atime_nanos: u32,
mtime_secs: i64,
mtime_nanos: u32,
) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs
.utime_async(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos)
.await
}
}

255
cli/standalone/mod.rs
View file

@ -1,17 +1,25 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use crate::args::get_root_cert_store;
use crate::args::npm_pkg_req_ref_to_binary_command;
use crate::args::CaData;
use crate::args::CacheSetting;
use crate::args::PackageJsonDepsProvider;
use crate::args::StorageKeyResolver;
use crate::cache::Caches;
use crate::cache::DenoDir;
use crate::cache::NodeAnalysisCache;
use crate::file_fetcher::get_source_from_data_url;
use crate::http_util::HttpClient;
use crate::module_loader::CjsResolutionStore;
use crate::module_loader::NpmModuleLoader;
use crate::node::CliCjsEsmCodeAnalyzer;
use crate::npm::create_npm_fs_resolver;
use crate::npm::CliNpmRegistryApi;
use crate::npm::CliNpmResolver;
use crate::npm::NpmCache;
use crate::npm::NpmResolution;
use crate::resolver::MappedSpecifierResolver;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use crate::util::v8::construct_v8_flags;
@ -19,7 +27,7 @@ use crate::worker::CliMainWorkerFactory;
use crate::worker::CliMainWorkerOptions;
use crate::worker::HasNodeSpecifierChecker;
use crate::worker::ModuleLoaderFactory;
use crate::CliGraphResolver;
use deno_ast::MediaType;
use deno_core::anyhow::Context;
use deno_core::error::type_error;
use deno_core::error::AnyError;
@ -29,31 +37,44 @@ use deno_core::ModuleLoader;
use deno_core::ModuleSpecifier;
use deno_core::ModuleType;
use deno_core::ResolutionKind;
use deno_graph::source::Resolver;
use deno_runtime::deno_fs;
use deno_runtime::deno_node;
use deno_runtime::deno_node::analyze::NodeCodeTranslator;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_tls::rustls::RootCertStore;
use deno_runtime::deno_tls::RootCertStoreProvider;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::permissions::Permissions;
use deno_runtime::permissions::PermissionsContainer;
use deno_semver::npm::NpmPackageReqReference;
use import_map::parse_from_json;
use std::pin::Pin;
use std::rc::Rc;
use std::sync::Arc;
mod binary;
mod file_system;
mod virtual_fs;
pub use binary::extract_standalone;
pub use binary::is_standalone_binary;
pub use binary::DenoCompileBinaryWriter;
use self::binary::load_npm_vfs;
use self::binary::Metadata;
use self::file_system::DenoCompileFileSystem;
struct SharedModuleLoaderState {
eszip: eszip::EszipV2,
mapped_specifier_resolver: MappedSpecifierResolver,
npm_module_loader: Arc<NpmModuleLoader>,
}
#[derive(Clone)]
struct EmbeddedModuleLoader {
eszip: Arc<eszip::EszipV2>,
maybe_import_map_resolver: Option<Arc<CliGraphResolver>>,
shared: Arc<SharedModuleLoaderState>,
root_permissions: PermissionsContainer,
dynamic_permissions: PermissionsContainer,
}
impl ModuleLoader for EmbeddedModuleLoader {
@ -61,10 +82,10 @@ impl ModuleLoader for EmbeddedModuleLoader {
&self,
specifier: &str,
referrer: &str,
_kind: ResolutionKind,
kind: ResolutionKind,
) -> Result<ModuleSpecifier, AnyError> {
// Try to follow redirects when resolving.
let referrer = match self.eszip.get_module(referrer) {
let referrer = match self.shared.eszip.get_module(referrer) {
Some(eszip::Module { ref specifier, .. }) => {
ModuleSpecifier::parse(specifier)?
}
@ -74,27 +95,93 @@ impl ModuleLoader for EmbeddedModuleLoader {
}
};
self
.maybe_import_map_resolver
let permissions = if matches!(kind, ResolutionKind::DynamicImport) {
&self.dynamic_permissions
} else {
&self.root_permissions
};
if let Some(result) = self
.shared
.npm_module_loader
.resolve_if_in_npm_package(specifier, &referrer, permissions)
{
return result;
}
let maybe_mapped = self
.shared
.mapped_specifier_resolver
.resolve(specifier, &referrer)?
.into_specifier();
// npm specifier
let specifier_text = maybe_mapped
.as_ref()
.map(|r| r.resolve(specifier, &referrer))
.unwrap_or_else(|| {
deno_core::resolve_import(specifier, referrer.as_str())
.map_err(|err| err.into())
})
.map(|r| r.as_str())
.unwrap_or(specifier);
if let Ok(reference) = NpmPackageReqReference::from_str(specifier_text) {
return self
.shared
.npm_module_loader
.resolve_req_reference(&reference, permissions);
}
// Built-in Node modules
if let Some(module_name) = specifier_text.strip_prefix("node:") {
return deno_node::resolve_builtin_node_module(module_name);
}
match maybe_mapped {
Some(resolved) => Ok(resolved),
None => deno_core::resolve_import(specifier, referrer.as_str())
.map_err(|err| err.into()),
}
}
fn load(
&self,
module_specifier: &ModuleSpecifier,
_maybe_referrer: Option<&ModuleSpecifier>,
_is_dynamic: bool,
maybe_referrer: Option<&ModuleSpecifier>,
is_dynamic: bool,
) -> Pin<Box<deno_core::ModuleSourceFuture>> {
let is_data_uri = get_source_from_data_url(module_specifier).ok();
let permissions = if is_dynamic {
&self.dynamic_permissions
} else {
&self.root_permissions
};
if let Some(result) =
self.shared.npm_module_loader.load_sync_if_in_npm_package(
module_specifier,
maybe_referrer,
permissions,
)
{
return match result {
Ok(code_source) => Box::pin(deno_core::futures::future::ready(Ok(
deno_core::ModuleSource::new_with_redirect(
match code_source.media_type {
MediaType::Json => ModuleType::Json,
_ => ModuleType::JavaScript,
},
code_source.code,
module_specifier,
&code_source.found_url,
),
))),
Err(err) => Box::pin(deno_core::futures::future::ready(Err(err))),
};
}
let module = self
.shared
.eszip
.get_module(module_specifier.as_str())
.ok_or_else(|| type_error("Module not found"));
.ok_or_else(|| {
type_error(format!("Module not found: {}", module_specifier))
});
// TODO(mmastrac): This clone can probably be removed in the future if ModuleSpecifier is no longer a full-fledged URL
let module_specifier = module_specifier.clone();
@ -128,24 +215,32 @@ impl ModuleLoader for EmbeddedModuleLoader {
}
struct StandaloneModuleLoaderFactory {
loader: EmbeddedModuleLoader,
shared: Arc<SharedModuleLoaderState>,
}
impl ModuleLoaderFactory for StandaloneModuleLoaderFactory {
fn create_for_main(
&self,
_root_permissions: PermissionsContainer,
_dynamic_permissions: PermissionsContainer,
root_permissions: PermissionsContainer,
dynamic_permissions: PermissionsContainer,
) -> Rc<dyn ModuleLoader> {
Rc::new(self.loader.clone())
Rc::new(EmbeddedModuleLoader {
shared: self.shared.clone(),
root_permissions,
dynamic_permissions,
})
}
fn create_for_worker(
&self,
_root_permissions: PermissionsContainer,
_dynamic_permissions: PermissionsContainer,
root_permissions: PermissionsContainer,
dynamic_permissions: PermissionsContainer,
) -> Rc<dyn ModuleLoader> {
Rc::new(self.loader.clone())
Rc::new(EmbeddedModuleLoader {
shared: self.shared.clone(),
root_permissions,
dynamic_permissions,
})
}
fn create_source_map_getter(
@ -183,6 +278,9 @@ pub async fn run(
metadata: Metadata,
) -> Result<(), AnyError> {
let main_module = &metadata.entrypoint;
let current_exe_path = std::env::current_exe().unwrap();
let current_exe_name =
current_exe_path.file_name().unwrap().to_string_lossy();
let dir = DenoDir::new(None)?;
let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider {
ca_stores: metadata.ca_stores,
@ -194,9 +292,14 @@ pub async fn run(
Some(root_cert_store_provider.clone()),
metadata.unsafely_ignore_certificate_errors.clone(),
));
let npm_registry_url = CliNpmRegistryApi::default_url().to_owned();
// use a dummy npm registry url
let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap();
let root_path = std::env::temp_dir()
.join(format!("deno-compile-{}", current_exe_name))
.join("node_modules");
let npm_cache = Arc::new(NpmCache::new(
dir.npm_folder_path(),
root_path.clone(),
CacheSetting::Use,
http_client.clone(),
progress_bar.clone(),
@ -207,44 +310,92 @@ pub async fn run(
http_client.clone(),
progress_bar.clone(),
));
let fs = Arc::new(deno_fs::RealFs);
let npm_resolution =
Arc::new(NpmResolution::from_serialized(npm_api.clone(), None, None));
let (fs, node_modules_path, snapshot) = if let Some(snapshot) =
metadata.npm_snapshot
{
let vfs_root_dir_path = if metadata.node_modules_dir {
root_path
} else {
npm_cache.registry_folder(&npm_registry_url)
};
let vfs =
load_npm_vfs(vfs_root_dir_path).context("Failed to load npm vfs.")?;
let node_modules_path = if metadata.node_modules_dir {
Some(vfs.root().to_path_buf())
} else {
None
};
(
Arc::new(DenoCompileFileSystem::new(vfs)) as Arc<dyn deno_fs::FileSystem>,
node_modules_path,
Some(snapshot.into_valid()?),
)
} else {
(
Arc::new(deno_fs::RealFs) as Arc<dyn deno_fs::FileSystem>,
None,
None,
)
};
let npm_resolution = Arc::new(NpmResolution::from_serialized(
npm_api.clone(),
snapshot,
None,
));
let has_node_modules_dir = node_modules_path.is_some();
let npm_fs_resolver = create_npm_fs_resolver(
fs.clone(),
npm_cache,
&progress_bar,
npm_registry_url,
npm_resolution.clone(),
None,
node_modules_path,
);
let npm_resolver = Arc::new(CliNpmResolver::new(
fs.clone(),
npm_resolution.clone(),
npm_fs_resolver,
None,
));
let node_resolver =
Arc::new(NodeResolver::new(fs.clone(), npm_resolver.clone()));
let cjs_resolutions = Arc::new(CjsResolutionStore::default());
let cache_db = Caches::new(dir.clone());
let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db());
let cjs_esm_code_analyzer = CliCjsEsmCodeAnalyzer::new(node_analysis_cache);
let node_code_translator = Arc::new(NodeCodeTranslator::new(
cjs_esm_code_analyzer,
fs.clone(),
node_resolver.clone(),
npm_resolver.clone(),
));
let package_json_deps_provider = Arc::new(PackageJsonDepsProvider::new(
metadata
.package_json_deps
.map(|serialized| serialized.into_deps()),
));
let maybe_import_map = metadata.maybe_import_map.map(|(base, source)| {
Arc::new(parse_from_json(&base, &source).unwrap().import_map)
});
let module_loader_factory = StandaloneModuleLoaderFactory {
loader: EmbeddedModuleLoader {
eszip: Arc::new(eszip),
maybe_import_map_resolver: metadata.maybe_import_map.map(
|(base, source)| {
Arc::new(CliGraphResolver::new(
None,
Some(Arc::new(
parse_from_json(&base, &source).unwrap().import_map,
)),
false,
npm_api.clone(),
npm_resolution.clone(),
Default::default(),
))
},
shared: Arc::new(SharedModuleLoaderState {
eszip,
mapped_specifier_resolver: MappedSpecifierResolver::new(
maybe_import_map.clone(),
package_json_deps_provider.clone(),
),
},
npm_module_loader: Arc::new(NpmModuleLoader::new(
cjs_resolutions,
node_code_translator,
fs.clone(),
node_resolver.clone(),
)),
}),
};
let permissions = PermissionsContainer::new(Permissions::from_options(
&metadata.permissions,
)?);
let worker_factory = CliMainWorkerFactory::new(
StorageKeyResolver::empty(),
npm_resolver.clone(),
@ -260,14 +411,17 @@ pub async fn run(
debug: false,
coverage_dir: None,
enable_testing_features: false,
has_node_modules_dir: false,
has_node_modules_dir,
inspect_brk: false,
inspect_wait: false,
is_inspecting: false,
is_npm_main: false,
is_npm_main: main_module.scheme() == "npm",
location: metadata.location,
// todo(dsherret): support a binary command being compiled
maybe_binary_npm_command_name: None,
maybe_binary_npm_command_name: NpmPackageReqReference::from_specifier(
main_module,
)
.ok()
.map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref)),
origin_data_folder_path: None,
seed: metadata.seed,
unsafely_ignore_certificate_errors: metadata
@ -278,9 +432,6 @@ pub async fn run(
v8_set_flags(construct_v8_flags(&metadata.v8_flags, vec![]));
let permissions = PermissionsContainer::new(Permissions::from_options(
&metadata.permissions,
)?);
let mut worker = worker_factory
.create_main_worker(main_module.clone(), permissions)
.await?;

983
cli/standalone/virtual_fs.rs Normal file
View file

@ -0,0 +1,983 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fs::File;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::BufMutView;
use deno_core::BufView;
use deno_runtime::deno_fs::FsDirEntry;
use deno_runtime::deno_io;
use deno_runtime::deno_io::fs::FsError;
use deno_runtime::deno_io::fs::FsResult;
use deno_runtime::deno_io::fs::FsStat;
use serde::Deserialize;
use serde::Serialize;
use crate::util;
pub struct VfsBuilder {
root_path: PathBuf,
root_dir: VirtualDirectory,
files: Vec<Vec<u8>>,
current_offset: u64,
file_offsets: HashMap<String, u64>,
}
impl VfsBuilder {
pub fn new(root_path: PathBuf) -> Self {
Self {
root_dir: VirtualDirectory {
name: root_path
.file_stem()
.unwrap()
.to_string_lossy()
.into_owned(),
entries: Vec::new(),
},
root_path,
files: Vec::new(),
current_offset: 0,
file_offsets: Default::default(),
}
}
pub fn set_root_dir_name(&mut self, name: String) {
self.root_dir.name = name;
}
pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> {
self.add_dir(path);
let read_dir = std::fs::read_dir(path)
.with_context(|| format!("Reading {}", path.display()))?;
for entry in read_dir {
let entry = entry?;
let file_type = entry.file_type()?;
let path = entry.path();
if file_type.is_dir() {
self.add_dir_recursive(&path)?;
} else if file_type.is_file() {
let file_bytes = std::fs::read(&path)
.with_context(|| format!("Reading {}", path.display()))?;
self.add_file(&path, file_bytes);
} else if file_type.is_symlink() {
let target = std::fs::read_link(&path)
.with_context(|| format!("Reading symlink {}", path.display()))?;
self.add_symlink(&path, &target);
}
}
Ok(())
}
pub fn add_dir(&mut self, path: &Path) -> &mut VirtualDirectory {
let path = path.strip_prefix(&self.root_path).unwrap();
let mut current_dir = &mut self.root_dir;
for component in path.components() {
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&name))
{
Ok(index) => index,
Err(insert_index) => {
current_dir.entries.insert(
insert_index,
VfsEntry::Dir(VirtualDirectory {
name: name.to_string(),
entries: Vec::new(),
}),
);
insert_index
}
};
match &mut current_dir.entries[index] {
VfsEntry::Dir(dir) => {
current_dir = dir;
}
_ => unreachable!(),
};
}
current_dir
}
pub fn add_file(&mut self, path: &Path, data: Vec<u8>) {
let checksum = util::checksum::gen(&[&data]);
let offset = if let Some(offset) = self.file_offsets.get(&checksum) {
// duplicate file, reuse an old offset
*offset
} else {
self.file_offsets.insert(checksum, self.current_offset);
self.current_offset
};
let dir = self.add_dir(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
let data_len = data.len();
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
Ok(_) => unreachable!(),
Err(insert_index) => {
dir.entries.insert(
insert_index,
VfsEntry::File(VirtualFile {
name: name.to_string(),
offset,
len: data.len() as u64,
}),
);
}
}
// new file, update the list of files
if self.current_offset == offset {
self.files.push(data);
self.current_offset += data_len as u64;
}
}
pub fn add_symlink(&mut self, path: &Path, target: &Path) {
let dest = target.strip_prefix(&self.root_path).unwrap().to_path_buf();
let dir = self.add_dir(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
Ok(_) => unreachable!(),
Err(insert_index) => {
dir.entries.insert(
insert_index,
VfsEntry::Symlink(VirtualSymlink {
name: name.to_string(),
dest_parts: dest
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>(),
}),
);
}
}
}
pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec<Vec<u8>>) {
(self.root_dir, self.files)
}
}
#[derive(Debug)]
enum VfsEntryRef<'a> {
Dir(&'a VirtualDirectory),
File(&'a VirtualFile),
Symlink(&'a VirtualSymlink),
}
impl<'a> VfsEntryRef<'a> {
pub fn as_fs_stat(&self) -> FsStat {
match self {
VfsEntryRef::Dir(_) => FsStat {
is_directory: true,
is_file: false,
is_symlink: false,
atime: None,
birthtime: None,
mtime: None,
blksize: 0,
size: 0,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
},
VfsEntryRef::File(file) => FsStat {
is_directory: false,
is_file: true,
is_symlink: false,
atime: None,
birthtime: None,
mtime: None,
blksize: 0,
size: file.len,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
},
VfsEntryRef::Symlink(_) => FsStat {
is_directory: false,
is_file: false,
is_symlink: true,
atime: None,
birthtime: None,
mtime: None,
blksize: 0,
size: 0,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
},
}
}
}
// todo(dsherret): we should store this more efficiently in the binary
#[derive(Debug, Serialize, Deserialize)]
pub enum VfsEntry {
Dir(VirtualDirectory),
File(VirtualFile),
Symlink(VirtualSymlink),
}
impl VfsEntry {
pub fn name(&self) -> &str {
match self {
VfsEntry::Dir(dir) => &dir.name,
VfsEntry::File(file) => &file.name,
VfsEntry::Symlink(symlink) => &symlink.name,
}
}
fn as_ref(&self) -> VfsEntryRef {
match self {
VfsEntry::Dir(dir) => VfsEntryRef::Dir(dir),
VfsEntry::File(file) => VfsEntryRef::File(file),
VfsEntry::Symlink(symlink) => VfsEntryRef::Symlink(symlink),
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualDirectory {
pub name: String,
// should be sorted by name
pub entries: Vec<VfsEntry>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VirtualFile {
pub name: String,
pub offset: u64,
pub len: u64,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlink {
pub name: String,
pub dest_parts: Vec<String>,
}
impl VirtualSymlink {
pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf {
let mut dest = root.to_path_buf();
for part in &self.dest_parts {
dest.push(part);
}
dest
}
}
#[derive(Debug)]
pub struct VfsRoot {
pub dir: VirtualDirectory,
pub root_path: PathBuf,
pub start_file_offset: u64,
}
impl VfsRoot {
fn find_entry<'a>(
&'a self,
path: &Path,
) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> {
self.find_entry_inner(path, &mut HashSet::new())
}
fn find_entry_inner<'a>(
&'a self,
path: &Path,
seen: &mut HashSet<PathBuf>,
) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> {
let mut path = Cow::Borrowed(path);
loop {
let (resolved_path, entry) =
self.find_entry_no_follow_inner(&path, seen)?;
match entry {
VfsEntryRef::Symlink(symlink) => {
if !seen.insert(path.to_path_buf()) {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"circular symlinks",
));
}
path = Cow::Owned(symlink.resolve_dest_from_root(&self.root_path));
}
_ => {
return Ok((resolved_path, entry));
}
}
}
}
fn find_entry_no_follow(
&self,
path: &Path,
) -> std::io::Result<(PathBuf, VfsEntryRef)> {
self.find_entry_no_follow_inner(path, &mut HashSet::new())
}
fn find_entry_no_follow_inner<'a>(
&'a self,
path: &Path,
seen: &mut HashSet<PathBuf>,
) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> {
let relative_path = match path.strip_prefix(&self.root_path) {
Ok(p) => p,
Err(_) => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
};
let mut final_path = self.root_path.clone();
let mut current_entry = VfsEntryRef::Dir(&self.dir);
for component in relative_path.components() {
let component = component.as_os_str().to_string_lossy();
let current_dir = match current_entry {
VfsEntryRef::Dir(dir) => {
final_path.push(component.as_ref());
dir
}
VfsEntryRef::Symlink(symlink) => {
let dest = symlink.resolve_dest_from_root(&self.root_path);
let (resolved_path, entry) = self.find_entry_inner(&dest, seen)?;
final_path = resolved_path; // overwrite with the new resolved path
match entry {
VfsEntryRef::Dir(dir) => {
final_path.push(component.as_ref());
dir
}
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
}
}
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
};
match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&component))
{
Ok(index) => {
current_entry = current_dir.entries[index].as_ref();
}
Err(_) => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
}
}
Ok((final_path, current_entry))
}
}
#[derive(Clone)]
struct FileBackedVfsFile {
file: VirtualFile,
pos: Arc<Mutex<u64>>,
vfs: Arc<FileBackedVfs>,
}
impl FileBackedVfsFile {
fn seek(&self, pos: SeekFrom) -> FsResult<u64> {
match pos {
SeekFrom::Start(pos) => {
*self.pos.lock() = pos;
Ok(pos)
}
SeekFrom::End(offset) => {
if offset < 0 && -offset as u64 > self.file.len {
Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "An attempt was made to move the file pointer before the beginning of the file.").into())
} else {
let mut current_pos = self.pos.lock();
*current_pos = if offset >= 0 {
self.file.len - (offset as u64)
} else {
self.file.len + (-offset as u64)
};
Ok(*current_pos)
}
}
SeekFrom::Current(offset) => {
let mut current_pos = self.pos.lock();
if offset >= 0 {
*current_pos += offset as u64;
} else if -offset as u64 > *current_pos {
return Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "An attempt was made to move the file pointer before the beginning of the file.").into());
} else {
*current_pos -= -offset as u64;
}
Ok(*current_pos)
}
}
}
fn read_to_buf(&self, buf: &mut [u8]) -> FsResult<usize> {
let pos = {
let mut pos = self.pos.lock();
let read_pos = *pos;
// advance the position due to the read
*pos = std::cmp::min(self.file.len, *pos + buf.len() as u64);
read_pos
};
self
.vfs
.read_file(&self.file, pos, buf)
.map_err(|err| err.into())
}
fn read_to_end(&self) -> FsResult<Vec<u8>> {
let pos = {
let mut pos = self.pos.lock();
let read_pos = *pos;
// todo(dsherret): should this always set it to the end of the file?
if *pos < self.file.len {
// advance the position due to the read
*pos = self.file.len;
}
read_pos
};
if pos > self.file.len {
return Ok(Vec::new());
}
let size = (self.file.len - pos) as usize;
let mut buf = vec![0; size];
self.vfs.read_file(&self.file, pos, &mut buf)?;
Ok(buf)
}
}
#[async_trait::async_trait(?Send)]
impl deno_io::fs::File for FileBackedVfsFile {
fn read_sync(self: Rc<Self>, buf: &mut [u8]) -> FsResult<usize> {
self.read_to_buf(buf)
}
async fn read_byob(
self: Rc<Self>,
mut buf: BufMutView,
) -> FsResult<(usize, BufMutView)> {
let inner = (*self).clone();
tokio::task::spawn(async move {
let nread = inner.read_to_buf(&mut buf)?;
Ok((nread, buf))
})
.await?
}
fn write_sync(self: Rc<Self>, _buf: &[u8]) -> FsResult<usize> {
Err(FsError::NotSupported)
}
async fn write(
self: Rc<Self>,
_buf: BufView,
) -> FsResult<deno_core::WriteOutcome> {
Err(FsError::NotSupported)
}
fn write_all_sync(self: Rc<Self>, _buf: &[u8]) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn write_all(self: Rc<Self>, _buf: BufView) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn read_all_sync(self: Rc<Self>) -> FsResult<Vec<u8>> {
self.read_to_end()
}
async fn read_all_async(self: Rc<Self>) -> FsResult<Vec<u8>> {
let inner = (*self).clone();
tokio::task::spawn_blocking(move || inner.read_to_end()).await?
}
fn chmod_sync(self: Rc<Self>, _pathmode: u32) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn chmod_async(self: Rc<Self>, _mode: u32) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn seek_sync(self: Rc<Self>, pos: SeekFrom) -> FsResult<u64> {
self.seek(pos)
}
async fn seek_async(self: Rc<Self>, pos: SeekFrom) -> FsResult<u64> {
self.seek(pos)
}
fn datasync_sync(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn datasync_async(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn sync_sync(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn sync_async(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn stat_sync(self: Rc<Self>) -> FsResult<FsStat> {
Err(FsError::NotSupported)
}
async fn stat_async(self: Rc<Self>) -> FsResult<FsStat> {
Err(FsError::NotSupported)
}
fn lock_sync(self: Rc<Self>, _exclusive: bool) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn lock_async(self: Rc<Self>, _exclusive: bool) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn unlock_sync(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn unlock_async(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn truncate_sync(self: Rc<Self>, _len: u64) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn truncate_async(self: Rc<Self>, _len: u64) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn utime_sync(
self: Rc<Self>,
_atime_secs: i64,
_atime_nanos: u32,
_mtime_secs: i64,
_mtime_nanos: u32,
) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn utime_async(
self: Rc<Self>,
_atime_secs: i64,
_atime_nanos: u32,
_mtime_secs: i64,
_mtime_nanos: u32,
) -> FsResult<()> {
Err(FsError::NotSupported)
}
// lower level functionality
fn as_stdio(self: Rc<Self>) -> FsResult<std::process::Stdio> {
Err(FsError::NotSupported)
}
#[cfg(unix)]
fn backing_fd(self: Rc<Self>) -> Option<std::os::unix::prelude::RawFd> {
None
}
#[cfg(windows)]
fn backing_fd(self: Rc<Self>) -> Option<std::os::windows::io::RawHandle> {
None
}
fn try_clone_inner(self: Rc<Self>) -> FsResult<Rc<dyn deno_io::fs::File>> {
Ok(self)
}
}
#[derive(Debug)]
pub struct FileBackedVfs {
file: Mutex<File>,
fs_root: VfsRoot,
}
impl FileBackedVfs {
pub fn new(file: File, fs_root: VfsRoot) -> Self {
Self {
file: Mutex::new(file),
fs_root,
}
}
pub fn root(&self) -> &Path {
&self.fs_root.root_path
}
pub fn is_path_within(&self, path: &Path) -> bool {
path.starts_with(&self.fs_root.root_path)
}
pub fn open_file(
self: &Arc<Self>,
path: &Path,
) -> std::io::Result<Rc<dyn deno_io::fs::File>> {
let file = self.file_entry(path)?;
Ok(Rc::new(FileBackedVfsFile {
file: file.clone(),
vfs: self.clone(),
pos: Default::default(),
}))
}
pub fn read_dir(&self, path: &Path) -> std::io::Result<Vec<FsDirEntry>> {
let dir = self.dir_entry(path)?;
Ok(
dir
.entries
.iter()
.map(|entry| FsDirEntry {
name: entry.name().to_string(),
is_file: matches!(entry, VfsEntry::File(_)),
is_directory: matches!(entry, VfsEntry::Dir(_)),
is_symlink: matches!(entry, VfsEntry::Symlink(_)),
})
.collect(),
)
}
pub fn read_link(&self, path: &Path) -> std::io::Result<PathBuf> {
let (_, entry) = self.fs_root.find_entry_no_follow(path)?;
match entry {
VfsEntryRef::Symlink(symlink) => {
Ok(symlink.resolve_dest_from_root(&self.fs_root.root_path))
}
VfsEntryRef::Dir(_) | VfsEntryRef::File(_) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"not a symlink",
)),
}
}
pub fn lstat(&self, path: &Path) -> std::io::Result<FsStat> {
let (_, entry) = self.fs_root.find_entry_no_follow(path)?;
Ok(entry.as_fs_stat())
}
pub fn stat(&self, path: &Path) -> std::io::Result<FsStat> {
let (_, entry) = self.fs_root.find_entry(path)?;
Ok(entry.as_fs_stat())
}
pub fn canonicalize(&self, path: &Path) -> std::io::Result<PathBuf> {
let (path, _) = self.fs_root.find_entry(path)?;
Ok(path)
}
pub fn read_file_all(&self, file: &VirtualFile) -> std::io::Result<Vec<u8>> {
let mut buf = vec![0; file.len as usize];
self.read_file(file, 0, &mut buf)?;
Ok(buf)
}
pub fn read_file(
&self,
file: &VirtualFile,
pos: u64,
buf: &mut [u8],
) -> std::io::Result<usize> {
let mut fs_file = self.file.lock();
fs_file.seek(SeekFrom::Start(
self.fs_root.start_file_offset + file.offset + pos,
))?;
fs_file.read(buf)
}
pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> {
let (_, entry) = self.fs_root.find_entry(path)?;
match entry {
VfsEntryRef::Dir(dir) => Ok(dir),
VfsEntryRef::Symlink(_) => unreachable!(),
VfsEntryRef::File(_) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"path is a file",
)),
}
}
pub fn file_entry(&self, path: &Path) -> std::io::Result<&VirtualFile> {
let (_, entry) = self.fs_root.find_entry(path)?;
match entry {
VfsEntryRef::Dir(_) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"path is a directory",
)),
VfsEntryRef::Symlink(_) => unreachable!(),
VfsEntryRef::File(file) => Ok(file),
}
}
}
#[cfg(test)]
mod test {
use std::io::Write;
use test_util::TempDir;
use super::*;
fn read_file(vfs: &FileBackedVfs, path: &Path) -> String {
let file = vfs.file_entry(path).unwrap();
String::from_utf8(vfs.read_file_all(file).unwrap()).unwrap()
}
#[test]
fn builds_and_uses_virtual_fs() {
let temp_dir = TempDir::new();
let src_path = temp_dir.path().join("src");
let mut builder = VfsBuilder::new(src_path.clone());
builder.add_file(&src_path.join("a.txt"), "data".into());
builder.add_file(&src_path.join("b.txt"), "data".into());
assert_eq!(builder.files.len(), 1); // because duplicate data
builder.add_file(&src_path.join("c.txt"), "c".into());
builder.add_file(&src_path.join("sub_dir").join("d.txt"), "d".into());
builder.add_file(&src_path.join("e.txt"), "e".into());
builder.add_symlink(
&src_path.join("sub_dir").join("e.txt"),
&src_path.join("e.txt"),
);
// get the virtual fs
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
assert_eq!(read_file(&virtual_fs, &dest_path.join("a.txt")), "data");
assert_eq!(read_file(&virtual_fs, &dest_path.join("b.txt")), "data");
// attempt reading a symlink
assert_eq!(
read_file(&virtual_fs, &dest_path.join("sub_dir").join("e.txt")),
"e",
);
// canonicalize symlink
assert_eq!(
virtual_fs
.canonicalize(&dest_path.join("sub_dir").join("e.txt"))
.unwrap(),
dest_path.join("e.txt"),
);
// metadata
assert!(
virtual_fs
.lstat(&dest_path.join("sub_dir").join("e.txt"))
.unwrap()
.is_symlink
);
assert!(
virtual_fs
.stat(&dest_path.join("sub_dir").join("e.txt"))
.unwrap()
.is_file
);
assert!(
virtual_fs
.stat(&dest_path.join("sub_dir"))
.unwrap()
.is_directory,
);
assert!(virtual_fs.stat(&dest_path.join("e.txt")).unwrap().is_file,);
}
#[test]
fn test_include_dir_recursive() {
let temp_dir = TempDir::new();
temp_dir.create_dir_all("src/nested/sub_dir");
temp_dir.write("src/a.txt", "data");
temp_dir.write("src/b.txt", "data");
util::fs::symlink_dir(
&temp_dir.path().join("src/nested/sub_dir"),
&temp_dir.path().join("src/sub_dir_link"),
)
.unwrap();
temp_dir.write("src/nested/sub_dir/c.txt", "c");
// build and create the virtual fs
let src_path = temp_dir.path().join("src");
let mut builder = VfsBuilder::new(src_path.clone());
builder.add_dir_recursive(&src_path).unwrap();
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
assert_eq!(read_file(&virtual_fs, &dest_path.join("a.txt")), "data",);
assert_eq!(read_file(&virtual_fs, &dest_path.join("b.txt")), "data",);
assert_eq!(
read_file(
&virtual_fs,
&dest_path.join("nested").join("sub_dir").join("c.txt")
),
"c",
);
assert_eq!(
read_file(&virtual_fs, &dest_path.join("sub_dir_link").join("c.txt")),
"c",
);
assert!(
virtual_fs
.lstat(&dest_path.join("sub_dir_link"))
.unwrap()
.is_symlink
);
assert_eq!(
virtual_fs
.canonicalize(&dest_path.join("sub_dir_link").join("c.txt"))
.unwrap(),
dest_path.join("nested").join("sub_dir").join("c.txt"),
);
}
fn into_virtual_fs(
builder: VfsBuilder,
temp_dir: &TempDir,
) -> (PathBuf, FileBackedVfs) {
let virtual_fs_file = temp_dir.path().join("virtual_fs");
let (root_dir, files) = builder.into_dir_and_files();
{
let mut file = std::fs::File::create(&virtual_fs_file).unwrap();
for file_data in &files {
file.write_all(file_data).unwrap();
}
}
let file = std::fs::File::open(&virtual_fs_file).unwrap();
let dest_path = temp_dir.path().join("dest");
(
dest_path.clone(),
FileBackedVfs::new(
file,
VfsRoot {
dir: root_dir,
root_path: dest_path,
start_file_offset: 0,
},
),
)
}
#[test]
fn circular_symlink() {
let temp_dir = TempDir::new();
let src_path = temp_dir.path().join("src");
let mut builder = VfsBuilder::new(src_path.clone());
builder.add_symlink(&src_path.join("a.txt"), &src_path.join("b.txt"));
builder.add_symlink(&src_path.join("b.txt"), &src_path.join("c.txt"));
builder.add_symlink(&src_path.join("c.txt"), &src_path.join("a.txt"));
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
assert_eq!(
virtual_fs
.file_entry(&dest_path.join("a.txt"))
.err()
.unwrap()
.to_string(),
"circular symlinks",
);
assert_eq!(
virtual_fs.read_link(&dest_path.join("a.txt")).unwrap(),
dest_path.join("b.txt")
);
assert_eq!(
virtual_fs.read_link(&dest_path.join("b.txt")).unwrap(),
dest_path.join("c.txt")
);
}
#[tokio::test]
async fn test_open_file() {
let temp_dir = TempDir::new();
let temp_path = temp_dir.path();
let mut builder = VfsBuilder::new(temp_path.to_path_buf());
builder.add_file(
&temp_path.join("a.txt"),
"0123456789".to_string().into_bytes(),
);
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
let virtual_fs = Arc::new(virtual_fs);
let file = virtual_fs.open_file(&dest_path.join("a.txt")).unwrap();
file.clone().seek_sync(SeekFrom::Current(2)).unwrap();
let mut buf = vec![0; 2];
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"23");
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"45");
file.clone().seek_sync(SeekFrom::Current(-4)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"23");
file.clone().seek_sync(SeekFrom::Start(2)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"23");
file.clone().seek_sync(SeekFrom::End(2)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"89");
file.clone().seek_sync(SeekFrom::Current(-8)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"23");
assert_eq!(
file
.clone()
.seek_sync(SeekFrom::Current(-5))
.err()
.unwrap()
.into_io_error()
.to_string(),
"An attempt was made to move the file pointer before the beginning of the file."
);
// go beyond the file length, then back
file.clone().seek_sync(SeekFrom::Current(40)).unwrap();
file.clone().seek_sync(SeekFrom::Current(-38)).unwrap();
let read_buf = file.clone().read(2).await.unwrap();
assert_eq!(read_buf.to_vec(), b"67");
file.clone().seek_sync(SeekFrom::Current(-2)).unwrap();
// read to the end of the file
let all_buf = file.clone().read_all_sync().unwrap();
assert_eq!(all_buf.to_vec(), b"6789");
file.clone().seek_sync(SeekFrom::Current(-9)).unwrap();
// try try_clone_inner and read_all_async
let all_buf = file
.try_clone_inner()
.unwrap()
.read_all_async()
.await
.unwrap();
assert_eq!(all_buf.to_vec(), b"123456789");
}
}

305
cli/tests/integration/compile_tests.rs
View file

@ -5,6 +5,8 @@ use std::process::Command;
use test_util as util;
use test_util::TempDir;
use util::assert_contains;
use util::env_vars_for_npm_tests;
use util::TestContextBuilder;
#[test]
fn compile() {
@ -675,30 +677,40 @@ fn workers_basic() {
#[test]
fn workers_not_in_module_map() {
let _guard = util::http_server();
let dir = TempDir::new();
let context = TestContextBuilder::for_npm()
.use_http_server()
.use_temp_cwd()
.build();
let temp_dir = context.temp_dir();
let exe = if cfg!(windows) {
dir.path().join("not_in_module_map.exe")
temp_dir.path().join("not_in_module_map.exe")
} else {
dir.path().join("not_in_module_map")
temp_dir.path().join("not_in_module_map")
};
let output = util::deno_cmd()
.current_dir(util::root_path())
.arg("compile")
.arg("--output")
.arg(&exe)
.arg(util::testdata_path().join("./compile/workers/not_in_module_map.ts"))
.output()
.unwrap();
assert!(output.status.success());
let main_path =
util::testdata_path().join("./compile/workers/not_in_module_map.ts");
let output = context
.new_command()
.args_vec([
"compile",
"--output",
&exe.to_string_lossy(),
&main_path.to_string_lossy(),
])
.run();
output.assert_exit_code(0);
output.skip_output_check();
let output = Command::new(&exe).env("NO_COLOR", "").output().unwrap();
assert!(!output.status.success());
let stderr = String::from_utf8(output.stderr).unwrap();
assert!(stderr.starts_with(concat!(
"error: Uncaught (in worker \"\") Module not found\n",
"error: Uncaught (in promise) Error: Unhandled error in child worker.\n"
)));
let output = context
.new_command()
.command_name(exe.to_string_lossy())
.env("NO_COLOR", "")
.run();
output.assert_exit_code(1);
output.assert_matches_text(concat!(
"error: Uncaught (in worker \"\") Module not found: [WILDCARD]",
"error: Uncaught (in promise) Error: Unhandled error in child worker.\n[WILDCARD]"
));
}
#[test]
@ -790,3 +802,256 @@ fn dynamic_import_unanalyzable() {
.unwrap();
assert_eq!(String::from_utf8(output.stdout).unwrap(), expected);
}
itest!(npm_specifiers_errors_no_unstable {
args: "compile -A --quiet npm/cached_only/main.ts",
output_str: Some(
concat!(
"error: Using npm specifiers with deno compile requires the --unstable flag.",
"\n\n",
"Caused by:\n",
" npm specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: npm:chalk@5.0.1\n"
)
),
exit_code: 1,
envs: env_vars_for_npm_tests(),
http_server: true,
});
#[test]
fn compile_npm_specifiers() {
let context = TestContextBuilder::for_npm()
.use_sync_npm_download()
.use_temp_cwd()
.build();
let temp_dir = context.temp_dir();
temp_dir.write(
"main.ts",
concat!(
"import path from 'node:path';\n",
"import { getValue, setValue } from 'npm:@denotest/esm-basic';\n",
"import getValueDefault from 'npm:@denotest/esm-import-cjs-default';\n",
"setValue(2);\n",
"console.log(path.join('testing', 'this'));",
"console.log(getValue());",
"console.log(getValueDefault());",
),
);
let binary_path = if cfg!(windows) {
temp_dir.path().join("binary.exe")
} else {
temp_dir.path().join("binary")
};
// try with and without --node-modules-dir
let compile_commands = &[
"compile --unstable --output binary main.ts",
"compile --unstable --node-modules-dir --output binary main.ts",
];
for compile_command in compile_commands {
let output = context.new_command().args(compile_command).run();
output.assert_exit_code(0);
output.skip_output_check();
let output = context
.new_command()
.command_name(binary_path.to_string_lossy())
.run();
output.assert_matches_text(
r#"Node esm importing node cjs
===========================
{
default: [Function (anonymous)],
named: [Function (anonymous)],
MyClass: [class MyClass]
}
{ default: [Function (anonymous)], named: [Function (anonymous)] }
[Module: null prototype] {
MyClass: [class MyClass],
__esModule: true,
default: {
default: [Function (anonymous)],
named: [Function (anonymous)],
MyClass: [class MyClass]
},
named: [Function (anonymous)]
}
[Module: null prototype] {
__esModule: true,
default: { default: [Function (anonymous)], named: [Function (anonymous)] },
named: [Function (anonymous)]
}
===========================
static method
testing[WILDCARD]this
2
5
"#,
);
}
// try with a package.json
temp_dir.remove_dir_all("node_modules");
temp_dir.write(
"main.ts",
concat!(
"import { getValue, setValue } from '@denotest/esm-basic';\n",
"setValue(2);\n",
"console.log(getValue());",
),
);
temp_dir.write(
"package.json",
r#"{ "dependencies": { "@denotest/esm-basic": "1" } }"#,
);
let output = context
.new_command()
.args("compile --unstable --output binary main.ts")
.run();
output.assert_exit_code(0);
output.skip_output_check();
let output = context
.new_command()
.command_name(binary_path.to_string_lossy())
.run();
output.assert_matches_text("2\n");
}
#[test]
fn compile_npm_file_system() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "compile/npm_fs/main.ts",
output_file: "compile/npm_fs/main.out",
node_modules_dir: true,
input_name: Some("binary"),
expected_name: "binary",
run_args: vec![],
});
}
#[test]
fn compile_npm_bin_esm() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:@denotest/bin/cli-esm",
run_args: vec!["this", "is", "a", "test"],
output_file: "npm/deno_run_esm.out",
node_modules_dir: false,
input_name: None,
expected_name: "cli-esm",
});
}
#[test]
fn compile_npm_bin_cjs() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:@denotest/bin/cli-cjs",
run_args: vec!["this", "is", "a", "test"],
output_file: "npm/deno_run_cjs.out",
node_modules_dir: false,
input_name: None,
expected_name: "cli-cjs",
});
}
#[test]
fn compile_npm_cowsay() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:cowsay@1.5.0",
run_args: vec!["Hello"],
output_file: "npm/deno_run_cowsay.out",
node_modules_dir: false,
input_name: None,
expected_name: "cowsay",
});
}
#[test]
fn compile_npm_cowsay_explicit() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:cowsay@1.5.0/cowsay",
run_args: vec!["Hello"],
output_file: "npm/deno_run_cowsay.out",
node_modules_dir: false,
input_name: None,
expected_name: "cowsay",
});
}
#[test]
fn compile_npm_cowthink() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:cowsay@1.5.0/cowthink",
run_args: vec!["Hello"],
output_file: "npm/deno_run_cowthink.out",
node_modules_dir: false,
input_name: None,
expected_name: "cowthink",
});
}
struct RunNpmBinCompileOptions<'a> {
input_specifier: &'a str,
output_file: &'a str,
node_modules_dir: bool,
input_name: Option<&'a str>,
expected_name: &'a str,
run_args: Vec<&'a str>,
}
fn run_npm_bin_compile_test(opts: RunNpmBinCompileOptions) {
let context = TestContextBuilder::for_npm()
.use_sync_npm_download()
.use_temp_cwd()
.build();
let temp_dir = context.temp_dir();
let testdata_path = context.testdata_path();
let main_specifier = if opts.input_specifier.starts_with("npm:") {
opts.input_specifier.to_string()
} else {
testdata_path
.join(opts.input_specifier)
.to_string_lossy()
.to_string()
};
let mut args = vec![
"compile".to_string(),
"-A".to_string(),
"--unstable".to_string(),
];
if opts.node_modules_dir {
args.push("--node-modules-dir".to_string());
}
if let Some(bin_name) = opts.input_name {
args.push("--output".to_string());
args.push(bin_name.to_string());
}
args.push(main_specifier);
// compile
let output = context.new_command().args_vec(args).run();
output.assert_exit_code(0);
output.skip_output_check();
// run
let binary_path = if cfg!(windows) {
temp_dir.path().join(format!("{}.exe", opts.expected_name))
} else {
temp_dir.path().join(opts.expected_name)
};
let output = context
.new_command()
.command_name(binary_path.to_string_lossy())
.args_vec(opts.run_args)
.run();
output.assert_matches_file(opts.output_file);
}

10
cli/tests/integration/npm_tests.rs
View file

@ -855,17 +855,9 @@ fn ensure_registry_files_local() {
}
}
itest!(compile_errors {
args: "compile -A --quiet npm/cached_only/main.ts",
output_str: Some("error: npm specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: npm:chalk@5.0.1\n"),
exit_code: 1,
envs: env_vars_for_npm_tests(),
http_server: true,
});
itest!(bundle_errors {
args: "bundle --quiet npm/esm/main.js",
output_str: Some("error: npm specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: npm:chalk@5.0.1\n"),
output_str: Some("error: npm specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: npm:chalk@5.0.1\n"),
exit_code: 1,
envs: env_vars_for_npm_tests(),
http_server: true,

1
cli/tests/testdata/compile/npm_fs/main.out vendored Normal file
View file

@ -0,0 +1 @@
success

259
cli/tests/testdata/compile/npm_fs/main.ts vendored Normal file
View file

@ -0,0 +1,259 @@
import { url } from "npm:@denotest/esm-basic";
import { fileURLToPath } from "node:url";
import path from "node:path";
import assert from "node:assert/strict";
// will be at node_modules\.deno\@denotest+esm-basic@1.0.0\node_modules\@denotest\esm-basic
const dirPath = path.dirname(fileURLToPath(url));
const nodeModulesPath = path.join(dirPath, "../../../../../");
const packageJsonText = `{
"name": "@denotest/esm-basic",
"version": "1.0.0",
"type": "module",
"main": "main.mjs",
"types": "main.d.mts"
}
`;
const vfsPackageJsonPath = path.join(dirPath, "package.json");
// reading a file in vfs
{
const text = Deno.readTextFileSync(vfsPackageJsonPath);
assert.equal(text, packageJsonText);
}
// reading a file async in vfs
{
const text = await Deno.readTextFile(vfsPackageJsonPath);
assert.equal(text, packageJsonText);
}
// copy file from vfs to real fs
{
Deno.copyFileSync(vfsPackageJsonPath, "package.json");
assert.equal(Deno.readTextFileSync("package.json"), packageJsonText);
}
// copy to vfs
assert.throws(
() => Deno.copyFileSync("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
Deno.removeSync("package.json");
// copy file async from vfs to real fs
{
await Deno.copyFile(vfsPackageJsonPath, "package.json");
assert.equal(Deno.readTextFileSync("package.json"), packageJsonText);
}
// copy to vfs async
await assert.rejects(
() => Deno.copyFile("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
Deno.removeSync("package.json");
// open
{
const file = Deno.openSync(vfsPackageJsonPath);
const bytes = new Uint8Array(10);
file.seekSync(2, Deno.SeekMode.Start);
assert.equal(file.readSync(bytes), 10);
const text = new TextDecoder().decode(bytes);
assert.equal(text, packageJsonText.slice(2, 12));
}
{
const file = await Deno.open(vfsPackageJsonPath);
const bytes = new Uint8Array(10);
await file.seek(2, Deno.SeekMode.Start);
assert.equal(await file.read(bytes), 10);
const text = new TextDecoder().decode(bytes);
assert.equal(text, packageJsonText.slice(2, 12));
}
// chdir
assert.throws(() => Deno.chdir(dirPath), Deno.errors.NotSupported);
// mkdir
assert.throws(
() => Deno.mkdirSync(path.join(dirPath, "subDir")),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.mkdir(path.join(dirPath, "subDir")),
Deno.errors.NotSupported,
);
// chmod
assert.throws(
() => Deno.chmodSync(vfsPackageJsonPath, 0o777),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.chmod(vfsPackageJsonPath, 0o777),
Deno.errors.NotSupported,
);
// chown
assert.throws(
() => Deno.chownSync(vfsPackageJsonPath, 1000, 1000),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.chown(vfsPackageJsonPath, 1000, 1000),
Deno.errors.NotSupported,
);
// remove
assert.throws(
() => Deno.removeSync(vfsPackageJsonPath),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.remove(vfsPackageJsonPath),
Deno.errors.NotSupported,
);
// stat
{
const result = Deno.statSync(vfsPackageJsonPath);
assert(result.isFile);
}
{
const result = await Deno.stat(vfsPackageJsonPath);
assert(result.isFile);
}
// lstat
{
const result = Deno.lstatSync(
path.join(nodeModulesPath, "@denotest", "esm-basic"),
);
assert(result.isSymlink);
}
{
const result = await Deno.lstat(
path.join(nodeModulesPath, "@denotest", "esm-basic"),
);
assert(result.isSymlink);
}
// realpath
{
const result = Deno.realPathSync(
path.join(nodeModulesPath, "@denotest", "esm-basic", "package.json"),
);
assert.equal(result, vfsPackageJsonPath);
}
{
const result = await Deno.realPath(
path.join(nodeModulesPath, "@denotest", "esm-basic", "package.json"),
);
assert.equal(result, vfsPackageJsonPath);
}
// read dir
const readDirNames = ["main.d.mts", "main.mjs", "package.json"];
{
const names = Array.from(Deno.readDirSync(dirPath))
.map((e) => e.name);
assert.deepEqual(readDirNames, names);
}
{
const names = [];
for await (const entry of Deno.readDir(dirPath)) {
names.push(entry.name);
}
assert.deepEqual(readDirNames, names);
}
// rename
assert.throws(
() => Deno.renameSync("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
assert.throws(
() => Deno.renameSync(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.rename("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.rename(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
// link
assert.throws(
() => Deno.linkSync("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
assert.throws(
() => Deno.linkSync(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.link("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.link(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
// symlink
assert.throws(
() => Deno.symlinkSync("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
assert.throws(
() => Deno.symlinkSync(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.symlink("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.symlink(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
// read link
{
const result = Deno.readLinkSync(
path.join(nodeModulesPath, "@denotest", "esm-basic"),
);
assert.equal(result, dirPath);
}
{
const result = await Deno.readLink(
path.join(nodeModulesPath, "@denotest", "esm-basic"),
);
assert.equal(result, dirPath);
}
// truncate
assert.throws(
() => Deno.truncateSync(vfsPackageJsonPath, 0),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.truncate(vfsPackageJsonPath, 0),
Deno.errors.NotSupported,
);
// utime
assert.throws(
() => Deno.utimeSync(vfsPackageJsonPath, 0, 0),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.utime(vfsPackageJsonPath, 0, 0),
Deno.errors.NotSupported,
);
console.log("success");

1
cli/tests/testdata/npm/registry/@denotest/esm-basic/1.0.0/main.d.mts vendored
View file

@ -1,2 +1,3 @@
export declare function setValue(val: number): void;
export declare function getValue(): number;
export declare const url: string;

2
cli/tests/testdata/npm/registry/@denotest/esm-basic/1.0.0/main.mjs vendored
View file

@ -7,3 +7,5 @@ export function setValue(newValue) {
export function getValue() {
return value;
}
export const url = import.meta.url;

2
cli/tests/testdata/package_json/basic/main.info.out vendored
View file

@ -5,4 +5,4 @@ size: [WILDCARD]
file:///[WILDCARD]/main.ts (63B)
└─┬ file:///[WILDCARD]/lib.ts (166B)
└── npm:@denotest/esm-basic@1.0.0 (345B)
└── npm:@denotest/esm-basic@1.0.0 (416B)

15
cli/tools/standalone.rs → cli/tools/compile.rs
View file

@ -5,7 +5,6 @@ use crate::args::Flags;
use crate::factory::CliFactory;
use crate::graph_util::error_for_any_npm_specifier;
use crate::standalone::is_standalone_binary;
use crate::standalone::DenoCompileBinaryWriter;
use crate::util::path::path_has_trailing_slash;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
@ -25,14 +24,9 @@ pub async fn compile(
) -> Result<(), AnyError> {
let factory = CliFactory::from_flags(flags).await?;
let cli_options = factory.cli_options();
let file_fetcher = factory.file_fetcher()?;
let http_client = factory.http_client();
let deno_dir = factory.deno_dir()?;
let module_graph_builder = factory.module_graph_builder().await?;
let parsed_source_cache = factory.parsed_source_cache()?;
let binary_writer =
DenoCompileBinaryWriter::new(file_fetcher, http_client, deno_dir);
let binary_writer = factory.create_compile_binary_writer().await?;
let module_specifier = cli_options.resolve_main_module()?;
let module_roots = {
let mut vec = Vec::with_capacity(compile_flags.include.len() + 1);
@ -56,8 +50,11 @@ pub async fn compile(
)
.unwrap();
// at the moment, we don't support npm specifiers in deno_compile, so show an error
error_for_any_npm_specifier(&graph)?;
if !cli_options.unstable() {
error_for_any_npm_specifier(&graph).context(
"Using npm specifiers with deno compile requires the --unstable flag.",
)?;
}
let parser = parsed_source_cache.as_capturing_parser();
let eszip = eszip::EszipV2::from_graph(graph, &parser, Default::default())?;

2
cli/tools/mod.rs
View file

@ -3,6 +3,7 @@
pub mod bench;
pub mod bundle;
pub mod check;
pub mod compile;
pub mod coverage;
pub mod doc;
pub mod fmt;
@ -12,7 +13,6 @@ pub mod installer;
pub mod lint;
pub mod repl;
pub mod run;
pub mod standalone;
pub mod task;
pub mod test;
pub mod upgrade;

3
cli/tools/task.rs
View file

@ -64,12 +64,13 @@ pub async fn execute_script(
.await;
Ok(exit_code)
} else if let Some(script) = package_json_scripts.get(task_name) {
let package_json_deps_provider = factory.package_json_deps_provider();
let package_json_deps_installer =
factory.package_json_deps_installer().await?;
let npm_resolver = factory.npm_resolver().await?;
let node_resolver = factory.node_resolver().await?;
if let Some(package_deps) = package_json_deps_installer.package_deps() {
if let Some(package_deps) = package_json_deps_provider.deps() {
for (key, value) in package_deps {
if let Err(err) = value {
log::info!(

9
cli/tools/vendor/test.rs vendored
View file

@ -22,7 +22,6 @@ use import_map::ImportMap;
use crate::cache::ParsedSourceCache;
use crate::npm::CliNpmRegistryApi;
use crate::npm::NpmResolution;
use crate::npm::PackageJsonDepsInstaller;
use crate::resolver::CliGraphResolver;
use super::build::VendorEnvironment;
@ -270,18 +269,14 @@ async fn build_test_graph(
None,
None,
));
let deps_installer = Arc::new(PackageJsonDepsInstaller::new(
npm_registry_api.clone(),
npm_resolution.clone(),
None,
));
CliGraphResolver::new(
None,
Some(Arc::new(original_import_map)),
false,
npm_registry_api,
npm_resolution,
deps_installer,
Default::default(),
Default::default(),
)
});
let mut graph = ModuleGraph::default();

9
cli/util/fs.rs
View file

@ -92,12 +92,19 @@ pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
/// subsequently be created along this path by some other code.
pub fn canonicalize_path_maybe_not_exists(
path: &Path,
) -> Result<PathBuf, Error> {
canonicalize_path_maybe_not_exists_with_fs(path, canonicalize_path)
}
pub fn canonicalize_path_maybe_not_exists_with_fs(
path: &Path,
canonicalize: impl Fn(&Path) -> Result<PathBuf, Error>,
) -> Result<PathBuf, Error> {
let path = path.to_path_buf().clean();
let mut path = path.as_path();
let mut names_stack = Vec::new();
loop {
match canonicalize_path(path) {
match canonicalize(path) {
Ok(mut canonicalized_path) => {
for name in names_stack.into_iter().rev() {
canonicalized_path = canonicalized_path.join(name);

2
ext/fs/interface.rs
View file

@ -100,7 +100,7 @@ pub trait FileSystem: std::fmt::Debug + MaybeSend + MaybeSync {
async fn mkdir_async(
&self,
path: PathBuf,
recusive: bool,
recursive: bool,
mode: u32,
) -> FsResult<()>;

9
ext/io/fs.rs
View file

@ -15,6 +15,7 @@ use deno_core::OpState;
use deno_core::ResourceId;
use tokio::task::JoinError;
#[derive(Debug)]
pub enum FsError {
Io(io::Error),
FileBusy,
@ -29,6 +30,14 @@ impl FsError {
Self::NotSupported => io::ErrorKind::Other,
}
}
pub fn into_io_error(self) -> io::Error {
match self {
FsError::Io(err) => err,
FsError::FileBusy => io::Error::new(self.kind(), "file busy"),
FsError::NotSupported => io::Error::new(self.kind(), "not supported"),
}
}
}
impl From<io::Error> for FsError {

1
runtime/build.rs
View file

@ -358,6 +358,7 @@ fn main() {
if env::var_os("DOCS_RS").is_some() {
let snapshot_slice = &[];
#[allow(clippy::needless_borrow)]
#[allow(clippy::disallowed_methods)]
std::fs::write(&runtime_snapshot_path, snapshot_slice).unwrap();
}

45
runtime/clippy.toml Normal file
View file

@ -0,0 +1,45 @@
disallowed-methods = [
{ path = "std::env::current_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::path::Path::canonicalize", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::is_dir", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::is_file", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::is_symlink", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::metadata", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::read_dir", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::read_link", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::symlink_metadata", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::try_exists", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::exists", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::canonicalize", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::is_dir", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::is_file", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::is_symlink", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::metadata", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::read_dir", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::read_link", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::symlink_metadata", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::try_exists", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::env::set_current_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::env::temp_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::canonicalize", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::copy", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::create_dir_all", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::create_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::DirBuilder::new", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::hard_link", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::metadata", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::OpenOptions::new", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::read_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::read_link", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::read_to_string", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::read", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::remove_dir_all", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::remove_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::remove_file", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::rename", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::set_permissions", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::symlink_metadata", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::write", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::path::Path::canonicalize", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::path::Path::exists", reason = "File system operations should be done using FileSystem trait" },
]

6
runtime/examples/hello_runtime.rs
View file

@ -2,6 +2,7 @@
use deno_core::error::AnyError;
use deno_core::FsModuleLoader;
use deno_core::ModuleSpecifier;
use deno_runtime::permissions::PermissionsContainer;
use deno_runtime::worker::MainWorker;
use deno_runtime::worker::WorkerOptions;
@ -14,10 +15,7 @@ deno_core::extension!(hello_runtime, esm = ["hello_runtime_bootstrap.js"]);
async fn main() -> Result<(), AnyError> {
let js_path =
Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/hello_runtime.js");
let main_module = deno_core::resolve_path(
&js_path.to_string_lossy(),
&std::env::current_dir()?,
)?;
let main_module = ModuleSpecifier::from_file_path(js_path).unwrap();
let mut worker = MainWorker::bootstrap_from_options(
main_module.clone(),
PermissionsContainer::allow_all(),

25
runtime/fs_util.rs
View file

@ -3,23 +3,17 @@
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
pub use deno_core::normalize_path;
use std::env::current_dir;
use std::io::Error;
use std::path::Path;
use std::path::PathBuf;
/// Similar to `std::fs::canonicalize()` but strips UNC prefixes on Windows.
pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
Ok(deno_core::strip_unc_prefix(path.canonicalize()?))
}
#[inline]
pub fn resolve_from_cwd(path: &Path) -> Result<PathBuf, AnyError> {
if path.is_absolute() {
Ok(normalize_path(path))
} else {
let cwd =
current_dir().context("Failed to get current working directory")?;
#[allow(clippy::disallowed_methods)]
let cwd = std::env::current_dir()
.context("Failed to get current working directory")?;
Ok(normalize_path(cwd.join(path)))
}
}
@ -28,21 +22,26 @@ pub fn resolve_from_cwd(path: &Path) -> Result<PathBuf, AnyError> {
mod tests {
use super::*;
fn current_dir() -> PathBuf {
#[allow(clippy::disallowed_methods)]
std::env::current_dir().unwrap()
}
#[test]
fn resolve_from_cwd_child() {
let cwd = current_dir().unwrap();
let cwd = current_dir();
assert_eq!(resolve_from_cwd(Path::new("a")).unwrap(), cwd.join("a"));
}
#[test]
fn resolve_from_cwd_dot() {
let cwd = current_dir().unwrap();
let cwd = current_dir();
assert_eq!(resolve_from_cwd(Path::new(".")).unwrap(), cwd);
}
#[test]
fn resolve_from_cwd_parent() {
let cwd = current_dir().unwrap();
let cwd = current_dir();
assert_eq!(resolve_from_cwd(Path::new("a/..")).unwrap(), cwd);
}
@ -66,7 +65,7 @@ mod tests {
#[test]
fn resolve_from_cwd_absolute() {
let expected = Path::new("a");
let cwd = current_dir().unwrap();
let cwd = current_dir();
let absolute_expected = cwd.join(expected);
assert_eq!(resolve_from_cwd(expected).unwrap(), absolute_expected);
}

1
runtime/ops/os/mod.rs
View file

@ -339,6 +339,7 @@ fn rss() -> usize {
(out, idx)
}
#[allow(clippy::disallowed_methods)]
let statm_content = if let Ok(c) = std::fs::read_to_string("/proc/self/statm")
{
c

1
runtime/ops/os/sys_info.rs
View file

@ -48,6 +48,7 @@ pub fn loadavg() -> LoadAvg {
pub fn os_release() -> String {
#[cfg(target_os = "linux")]
{
#[allow(clippy::disallowed_methods)]
match std::fs::read_to_string("/proc/sys/kernel/osrelease") {
Ok(mut s) => {
s.pop(); // pop '\n'

1
test_util/src/builders.rs
View file

@ -341,6 +341,7 @@ impl TestCommandBuilder {
))
}
#[track_caller]
pub fn run(&self) -> TestCommandOutput {
fn read_pipe_to_string(mut pipe: os_pipe::PipeReader) -> String {
let mut output = String::new();

4
test_util/src/temp_dir.rs
View file

@ -58,6 +58,10 @@ impl TempDir {
fs::create_dir_all(self.path().join(path)).unwrap();
}
pub fn remove_dir_all(&self, path: impl AsRef<Path>) {
fs::remove_dir_all(self.path().join(path)).unwrap();
}
pub fn read_to_string(&self, path: impl AsRef<Path>) -> String {
let file_path = self.path().join(path);
fs::read_to_string(&file_path)