foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-21 15:04:11 -05:00
Code Issues Packages 1 Wiki Activity

feat(compile): unstable npm and node specifier support (#19005)

Browse source 
This is the initial support for npm and node specifiers in `deno
compile`. The npm packages are included in the binary and read from it via
a virtual file system. This also supports the `--node-modules-dir` flag,
dependencies specified in a package.json, and npm binary commands (ex.
`deno compile --unstable npm:cowsay`)

Closes #16632
This commit is contained in:
David Sherret 2023-05-10 20:06:59 -04:00 committed by GitHub
parent 5fd74bfa1c
commit 28aa489de9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
44 changed files with 2733 additions and 261 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Cargo.lock generated
View file

@ -2966,9 +2966,9 @@ dependencies = [
[[package]]
name = "monch"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1120c1ab92ab8cdacb3b89ac9a214f512d2e78e90e3b57c00d9551ced19f646f"
checksum = "bb73e1dc7d232e1ab47ef27f45fa1d173a0979b370e763a9d0584556011150e0"
[[package]]
name = "napi-build"

2
cli/Cargo.toml
View file

@ -84,7 +84,7 @@ lazy-regex.workspace = true
libc.workspace = true
log = { workspace = true, features = ["serde"] }
lsp-types.workspace = true
monch = "=0.4.1"
monch = "=0.4.2"
notify.workspace = true
once_cell.workspace = true
os_pipe.workspace = true

7
cli/args/flags.rs
View file

@ -527,8 +527,11 @@ impl Flags {
.ok()
}
Task(_) | Check(_) | Coverage(_) | Cache(_) | Info(_) | Eval(_)
| Test(_) | Bench(_) | Repl(_) => std::env::current_dir().ok(),
_ => None,
| Test(_) | Bench(_) | Repl(_) | Compile(_) => {
std::env::current_dir().ok()
}
Bundle(_) | Completions(_) | Doc(_) | Fmt(_) | Init(_) | Install(_)
| Uninstall(_) | Lsp | Lint(_) | Types | Upgrade(_) | Vendor(_) => None,
}
}

13
cli/args/mod.rs
View file

@ -33,6 +33,7 @@ pub use config_file::TsTypeLib;
pub use flags::*;
pub use lockfile::Lockfile;
pub use lockfile::LockfileError;
pub use package_json::PackageJsonDepsProvider;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::anyhow;
@ -556,7 +557,7 @@ struct CliOptionOverrides {
import_map_specifier: Option<Option<ModuleSpecifier>>,
}
/// Holds the resolved options of many sources used by sub commands
/// Holds the resolved options of many sources used by subcommands
/// and provides some helper function for creating common objects.
pub struct CliOptions {
// the source of the options is a detail the rest of the
@ -1303,6 +1304,16 @@ fn has_flag_env_var(name: &str) -> bool {
matches!(value.as_ref().map(|s| s.as_str()), Ok("1"))
}
pub fn npm_pkg_req_ref_to_binary_command(
req_ref: &NpmPackageReqReference,
) -> String {
let binary_name = req_ref
.sub_path
.as_deref()
.unwrap_or(req_ref.req.name.as_str());
binary_name.to_string()
}
#[cfg(test)]
mod test {
use super::*;

27
cli/args/package_json.rs
View file

@ -28,6 +28,33 @@ pub enum PackageJsonDepValueParseError {
pub type PackageJsonDeps =
BTreeMap<String, Result<NpmPackageReq, PackageJsonDepValueParseError>>;
#[derive(Debug, Default)]
pub struct PackageJsonDepsProvider(Option<PackageJsonDeps>);
impl PackageJsonDepsProvider {
pub fn new(deps: Option<PackageJsonDeps>) -> Self {
Self(deps)
}
pub fn deps(&self) -> Option<&PackageJsonDeps> {
self.0.as_ref()
}
pub fn reqs(&self) -> Vec<&NpmPackageReq> {
match &self.0 {
Some(deps) => {
let mut package_reqs = deps
.values()
.filter_map(|r| r.as_ref().ok())
.collect::<Vec<_>>();
package_reqs.sort(); // deterministic resolution
package_reqs
}
None => Vec::new(),
}
}
}
/// Gets an application level package.json's npm package requirements.
///
/// Note that this function is not general purpose. It is specifically for

50
cli/factory.rs
View file

@ -1,9 +1,11 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use crate::args::npm_pkg_req_ref_to_binary_command;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::Flags;
use crate::args::Lockfile;
use crate::args::PackageJsonDepsProvider;
use crate::args::StorageKeyResolver;
use crate::args::TsConfigType;
use crate::cache::Caches;
@ -30,6 +32,7 @@ use crate::npm::NpmCache;
use crate::npm::NpmResolution;
use crate::npm::PackageJsonDepsInstaller;
use crate::resolver::CliGraphResolver;
use crate::standalone::DenoCompileBinaryWriter;
use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
@ -151,6 +154,7 @@ struct CliFactoryServices {
npm_cache: Deferred<Arc<NpmCache>>,
npm_resolver: Deferred<Arc<CliNpmResolver>>,
npm_resolution: Deferred<Arc<NpmResolution>>,
package_json_deps_provider: Deferred<Arc<PackageJsonDepsProvider>>,
package_json_deps_installer: Deferred<Arc<PackageJsonDepsInstaller>>,
text_only_progress_bar: Deferred<ProgressBar>,
type_checker: Deferred<Arc<TypeChecker>>,
@ -301,8 +305,9 @@ impl CliFactory {
.npm_resolver
.get_or_try_init_async(async {
let npm_resolution = self.npm_resolution().await?;
let fs = self.fs().clone();
let npm_fs_resolver = create_npm_fs_resolver(
self.fs().clone(),
fs.clone(),
self.npm_cache()?.clone(),
self.text_only_progress_bar(),
CliNpmRegistryApi::default_url().to_owned(),
@ -310,6 +315,7 @@ impl CliFactory {
self.options.node_modules_dir_path(),
);
Ok(Arc::new(CliNpmResolver::new(
fs.clone(),
npm_resolution.clone(),
npm_fs_resolver,
self.maybe_lockfile().as_ref().cloned(),
@ -318,6 +324,14 @@ impl CliFactory {
.await
}
pub fn package_json_deps_provider(&self) -> &Arc<PackageJsonDepsProvider> {
self.services.package_json_deps_provider.get_or_init(|| {
Arc::new(PackageJsonDepsProvider::new(
self.options.maybe_package_json_deps(),
))
})
}
pub async fn package_json_deps_installer(
&self,
) -> Result<&Arc<PackageJsonDepsInstaller>, AnyError> {
@ -325,12 +339,10 @@ impl CliFactory {
.services
.package_json_deps_installer
.get_or_try_init_async(async {
let npm_api = self.npm_api()?;
let npm_resolution = self.npm_resolution().await?;
Ok(Arc::new(PackageJsonDepsInstaller::new(
npm_api.clone(),
npm_resolution.clone(),
self.options.maybe_package_json_deps(),
self.package_json_deps_provider().clone(),
self.npm_api()?.clone(),
self.npm_resolution().await?.clone(),
)))
})
.await
@ -365,6 +377,7 @@ impl CliFactory {
self.options.no_npm(),
self.npm_api()?.clone(),
self.npm_resolution().await?.clone(),
self.package_json_deps_provider().clone(),
self.package_json_deps_installer().await?.clone(),
)))
})
@ -535,6 +548,21 @@ impl CliFactory {
self.services.cjs_resolutions.get_or_init(Default::default)
}
pub async fn create_compile_binary_writer(
&self,
) -> Result<DenoCompileBinaryWriter, AnyError> {
Ok(DenoCompileBinaryWriter::new(
self.file_fetcher()?,
self.http_client(),
self.deno_dir()?,
self.npm_api()?,
self.npm_cache()?,
self.npm_resolver().await?,
self.npm_resolution().await?,
self.package_json_deps_provider(),
))
}
/// Gets a function that can be used to create a CliMainWorkerFactory
/// for a file watcher.
pub async fn create_cli_main_worker_factory_func(
@ -572,6 +600,7 @@ impl CliFactory {
NpmModuleLoader::new(
cjs_resolutions.clone(),
node_code_translator.clone(),
fs.clone(),
node_resolver.clone(),
),
)),
@ -587,6 +616,7 @@ impl CliFactory {
&self,
) -> Result<CliMainWorkerFactory, AnyError> {
let node_resolver = self.node_resolver().await?;
let fs = self.fs();
Ok(CliMainWorkerFactory::new(
StorageKeyResolver::from_options(&self.options),
self.npm_resolver().await?.clone(),
@ -603,6 +633,7 @@ impl CliFactory {
NpmModuleLoader::new(
self.cjs_resolutions().clone(),
self.node_code_translator().await?.clone(),
fs.clone(),
node_resolver.clone(),
),
)),
@ -637,11 +668,8 @@ impl CliFactory {
if let Ok(pkg_ref) = NpmPackageReqReference::from_str(&flags.script) {
// if the user ran a binary command, we'll need to set process.argv[0]
// to be the name of the binary command instead of deno
let binary_name = pkg_ref
.sub_path
.as_deref()
.unwrap_or(pkg_ref.req.name.as_str());
maybe_binary_command_name = Some(binary_name.to_string());
maybe_binary_command_name =
Some(npm_pkg_req_ref_to_binary_command(&pkg_ref));
}
}
maybe_binary_command_name

4
cli/graph_util.rs
View file

@ -345,10 +345,10 @@ pub fn error_for_any_npm_specifier(
for module in graph.modules() {
match module {
Module::Npm(module) => {
bail!("npm specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: {}", module.specifier)
bail!("npm specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: {}", module.specifier)
}
Module::Node(module) => {
bail!("Node specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: node:{}", module.module_name)
bail!("Node specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: node:{}", module.module_name)
}
Module::Esm(_) | Module::Json(_) | Module::External(_) => {}
}

6
cli/lsp/documents.rs
View file

@ -46,6 +46,7 @@ use deno_semver::npm::NpmPackageReqReference;
use indexmap::IndexMap;
use lsp::Url;
use once_cell::sync::Lazy;
use package_json::PackageJsonDepsProvider;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
@ -1218,10 +1219,12 @@ impl Documents {
maybe_jsx_config.as_ref(),
maybe_package_json_deps.as_ref(),
);
let deps_provider =
Arc::new(PackageJsonDepsProvider::new(maybe_package_json_deps));
let deps_installer = Arc::new(PackageJsonDepsInstaller::new(
deps_provider.clone(),
npm_registry_api.clone(),
npm_resolution.clone(),
maybe_package_json_deps,
));
self.resolver = Arc::new(CliGraphResolver::new(
maybe_jsx_config,
@ -1229,6 +1232,7 @@ impl Documents {
false,
npm_registry_api,
npm_resolution,
deps_provider,
deps_installer,
));
self.imports = Arc::new(

11
cli/lsp/language_server.rs
View file

@ -457,8 +457,9 @@ fn create_lsp_structs(
));
let resolution =
Arc::new(NpmResolution::from_serialized(api.clone(), None, None));
let fs = Arc::new(deno_fs::RealFs);
let fs_resolver = create_npm_fs_resolver(
Arc::new(deno_fs::RealFs),
fs.clone(),
npm_cache.clone(),
&progress_bar,
registry_url.clone(),
@ -468,7 +469,12 @@ fn create_lsp_structs(
(
api,
npm_cache,
Arc::new(CliNpmResolver::new(resolution.clone(), fs_resolver, None)),
Arc::new(CliNpmResolver::new(
fs,
resolution.clone(),
fs_resolver,
None,
)),
resolution,
)
}
@ -711,6 +717,7 @@ impl Inner {
));
let node_fs = Arc::new(deno_fs::RealFs);
let npm_resolver = Arc::new(CliNpmResolver::new(
node_fs.clone(),
npm_resolution.clone(),
create_npm_fs_resolver(
node_fs.clone(),

3
cli/main.rs
View file

@ -36,7 +36,6 @@ static GLOBAL: Jemalloc = Jemalloc;
use crate::args::flags_from_vec;
use crate::args::DenoSubcommand;
use crate::args::Flags;
use crate::resolver::CliGraphResolver;
use crate::util::display;
use crate::util::v8::get_v8_flags_from_env;
use crate::util::v8::init_v8_flags;
@ -97,7 +96,7 @@ async fn run_subcommand(flags: Flags) -> Result<i32, AnyError> {
Ok(0)
}
DenoSubcommand::Compile(compile_flags) => {
tools::standalone::compile(flags, compile_flags).await?;
tools::compile::compile(flags, compile_flags).await?;
Ok(0)
}
DenoSubcommand::Coverage(coverage_flags) => {

60
cli/module_loader.rs
View file

@ -46,11 +46,13 @@ use deno_graph::JsonModule;
use deno_graph::Module;
use deno_graph::Resolution;
use deno_lockfile::Lockfile;
use deno_runtime::deno_fs;
use deno_runtime::deno_node;
use deno_runtime::deno_node::NodeResolution;
use deno_runtime::deno_node::NodeResolutionMode;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::permissions::PermissionsContainer;
use deno_semver::npm::NpmPackageNvReference;
use deno_semver::npm::NpmPackageReqReference;
use std::borrow::Cow;
use std::cell::RefCell;
@ -417,13 +419,12 @@ impl CliModuleLoader {
} else {
&self.root_permissions
};
let code_source = if let Some(code_source) =
self.shared.npm_module_loader.load_sync(
specifier,
maybe_referrer,
permissions,
)? {
code_source
let code_source = if let Some(result) = self
.shared
.npm_module_loader
.load_sync_if_in_npm_package(specifier, maybe_referrer, permissions)
{
result?
} else {
self
.shared
@ -494,7 +495,7 @@ impl ModuleLoader for CliModuleLoader {
Some(Module::Npm(module)) => self
.shared
.npm_module_loader
.resolve_npm_module(module, permissions),
.resolve_nv_ref(&module.nv_reference, permissions),
Some(Module::Node(module)) => {
deno_node::resolve_builtin_node_module(&module.module_name)
}
@ -547,7 +548,7 @@ impl ModuleLoader for CliModuleLoader {
return self
.shared
.npm_module_loader
.resolve_for_repl(&reference, permissions);
.resolve_req_reference(&reference, permissions);
}
}
}
@ -652,6 +653,7 @@ impl SourceMapGetter for CliSourceMapGetter {
pub struct NpmModuleLoader {
cjs_resolutions: Arc<CjsResolutionStore>,
node_code_translator: Arc<CliNodeCodeTranslator>,
fs: Arc<dyn deno_fs::FileSystem>,
node_resolver: Arc<NodeResolver>,
}
@ -659,11 +661,13 @@ impl NpmModuleLoader {
pub fn new(
cjs_resolutions: Arc<CjsResolutionStore>,
node_code_translator: Arc<CliNodeCodeTranslator>,
fs: Arc<dyn deno_fs::FileSystem>,
node_resolver: Arc<NodeResolver>,
) -> Self {
Self {
cjs_resolutions,
node_code_translator,
fs,
node_resolver,
}
}
@ -693,21 +697,21 @@ impl NpmModuleLoader {
}
}
pub fn resolve_npm_module(
pub fn resolve_nv_ref(
&self,
module: &deno_graph::NpmModule,
nv_ref: &NpmPackageNvReference,
permissions: &PermissionsContainer,
) -> Result<ModuleSpecifier, AnyError> {
self
.handle_node_resolve_result(self.node_resolver.resolve_npm_reference(
&module.nv_reference,
nv_ref,
NodeResolutionMode::Execution,
permissions,
))
.with_context(|| format!("Could not resolve '{}'.", module.nv_reference))
.with_context(|| format!("Could not resolve '{}'.", nv_ref))
}
pub fn resolve_for_repl(
pub fn resolve_req_reference(
&self,
reference: &NpmPackageReqReference,
permissions: &PermissionsContainer,
@ -733,17 +737,31 @@ impl NpmModuleLoader {
}
}
pub fn load_sync(
pub fn load_sync_if_in_npm_package(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
permissions: &PermissionsContainer,
) -> Result<Option<ModuleCodeSource>, AnyError> {
if !self.node_resolver.in_npm_package(specifier) {
return Ok(None);
) -> Option<Result<ModuleCodeSource, AnyError>> {
if self.node_resolver.in_npm_package(specifier) {
Some(self.load_sync(specifier, maybe_referrer, permissions))
} else {
None
}
}
fn load_sync(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
permissions: &PermissionsContainer,
) -> Result<ModuleCodeSource, AnyError> {
let file_path = specifier.to_file_path().unwrap();
let code = std::fs::read_to_string(&file_path).with_context(|| {
let code = self
.fs
.read_to_string(&file_path)
.map_err(AnyError::from)
.with_context(|| {
let mut msg = "Unable to load ".to_string();
msg.push_str(&file_path.to_string_lossy());
if let Some(referrer) = &maybe_referrer {
@ -766,11 +784,11 @@ impl NpmModuleLoader {
.node_code_translator
.esm_code_with_node_globals(specifier, &code)?
};
Ok(Some(ModuleCodeSource {
Ok(ModuleCodeSource {
code: code.into(),
found_url: specifier.clone(),
media_type: MediaType::from_specifier(specifier),
}))
})
}
fn handle_node_resolve_result(

26
cli/npm/installer.rs
View file

@ -10,7 +10,7 @@ use deno_npm::registry::NpmRegistryApi;
use deno_npm::registry::NpmRegistryPackageInfoLoadError;
use deno_semver::npm::NpmPackageReq;
use crate::args::package_json::PackageJsonDeps;
use crate::args::PackageJsonDepsProvider;
use crate::util::sync::AtomicFlag;
use super::CliNpmRegistryApi;
@ -18,23 +18,13 @@ use super::NpmResolution;
#[derive(Debug)]
struct PackageJsonDepsInstallerInner {
deps_provider: Arc<PackageJsonDepsProvider>,
has_installed_flag: AtomicFlag,
npm_registry_api: Arc<CliNpmRegistryApi>,
npm_resolution: Arc<NpmResolution>,
package_deps: PackageJsonDeps,
}
impl PackageJsonDepsInstallerInner {
pub fn reqs(&self) -> Vec<&NpmPackageReq> {
let mut package_reqs = self
.package_deps
.values()
.filter_map(|r| r.as_ref().ok())
.collect::<Vec<_>>();
package_reqs.sort(); // deterministic resolution
package_reqs
}
pub fn reqs_with_info_futures(
&self,
) -> FuturesOrdered<
@ -45,7 +35,7 @@ impl PackageJsonDepsInstallerInner {
>,
>,
> {
let package_reqs = self.reqs();
let package_reqs = self.deps_provider.reqs();
FuturesOrdered::from_iter(package_reqs.into_iter().map(|req| {
let api = self.npm_registry_api.clone();
@ -63,22 +53,18 @@ pub struct PackageJsonDepsInstaller(Option<PackageJsonDepsInstallerInner>);
impl PackageJsonDepsInstaller {
pub fn new(
deps_provider: Arc<PackageJsonDepsProvider>,
npm_registry_api: Arc<CliNpmRegistryApi>,
npm_resolution: Arc<NpmResolution>,
deps: Option<PackageJsonDeps>,
) -> Self {
Self(deps.map(|package_deps| PackageJsonDepsInstallerInner {
Self(Some(PackageJsonDepsInstallerInner {
deps_provider,
has_installed_flag: Default::default(),
npm_registry_api,
npm_resolution,
package_deps,
}))
}
pub fn package_deps(&self) -> Option<&PackageJsonDeps> {
self.0.as_ref().map(|inner| &inner.package_deps)
}
/// Installs the top level dependencies in the package.json file
/// without going through and resolving the descendant dependencies yet.
pub async fn ensure_top_level_install(&self) -> Result<(), AnyError> {

4
cli/npm/resolution.rs
View file

@ -237,6 +237,10 @@ impl NpmResolution {
Ok(nv)
}
pub fn all_packages(&self) -> Vec<NpmResolutionPackage> {
self.snapshot.read().all_packages()
}
pub fn all_packages_partitioned(&self) -> NpmPackagesPartitioned {
self.snapshot.read().all_packages_partitioned()
}

6
cli/npm/resolvers/common.rs
View file

@ -12,6 +12,7 @@ use deno_core::futures;
use deno_core::url::Url;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeResolutionMode;
@ -90,6 +91,7 @@ pub async fn cache_packages(
}
pub fn ensure_registry_read_permission(
fs: &Arc<dyn FileSystem>,
permissions: &dyn NodePermissions,
registry_path: &Path,
path: &Path,
@ -101,8 +103,8 @@ pub fn ensure_registry_read_permission(
.all(|c| !matches!(c, std::path::Component::ParentDir))
{
// todo(dsherret): cache this?
if let Ok(registry_path) = std::fs::canonicalize(registry_path) {
match std::fs::canonicalize(path) {
if let Ok(registry_path) = fs.realpath_sync(registry_path) {
match fs.realpath_sync(path) {
Ok(path) if path.starts_with(registry_path) => {
return Ok(());
}

6
cli/npm/resolvers/global.rs
View file

@ -14,6 +14,7 @@ use deno_npm::resolution::PackageNotFoundFromReferrerError;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeResolutionMode;
@ -28,6 +29,7 @@ use super::common::NpmPackageFsResolver;
/// Resolves packages from the global npm cache.
#[derive(Debug)]
pub struct GlobalNpmPackageResolver {
fs: Arc<dyn FileSystem>,
cache: Arc<NpmCache>,
resolution: Arc<NpmResolution>,
registry_url: Url,
@ -35,11 +37,13 @@ pub struct GlobalNpmPackageResolver {
impl GlobalNpmPackageResolver {
pub fn new(
fs: Arc<dyn FileSystem>,
cache: Arc<NpmCache>,
registry_url: Url,
resolution: Arc<NpmResolution>,
) -> Self {
Self {
fs,
cache,
resolution,
registry_url,
@ -130,7 +134,7 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver {
path: &Path,
) -> Result<(), AnyError> {
let registry_path = self.cache.registry_folder(&self.registry_url);
ensure_registry_read_permission(permissions, &registry_path, path)
ensure_registry_read_permission(&self.fs, permissions, &registry_path, path)
}
}

5
cli/npm/resolvers/local.rs
View file

@ -154,7 +154,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
loop {
current_folder = get_next_node_modules_ancestor(current_folder);
let sub_dir = join_package_name(current_folder, name);
if sub_dir.is_dir() {
if self.fs.is_dir(&sub_dir) {
// if doing types resolution, only resolve the package if it specifies a types property
if mode.is_types() && !name.starts_with("@types/") {
let package_json = PackageJson::load_skip_read_permission(
@ -173,7 +173,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
if mode.is_types() && !name.starts_with("@types/") {
let sub_dir =
join_package_name(current_folder, &types_package_name(name));
if sub_dir.is_dir() {
if self.fs.is_dir(&sub_dir) {
return Ok(sub_dir);
}
}
@ -214,6 +214,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
path: &Path,
) -> Result<(), AnyError> {
ensure_registry_read_permission(
&self.fs,
permissions,
&self.root_node_modules_path,
path,

26
cli/npm/resolvers/mod.rs
View file

@ -18,7 +18,7 @@ use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::resolution::PackageReqNotFoundError;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeResolutionMode;
use deno_runtime::deno_node::NpmResolver;
@ -32,7 +32,7 @@ use serde::Deserialize;
use serde::Serialize;
use crate::args::Lockfile;
use crate::util::fs::canonicalize_path_maybe_not_exists;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::progress_bar::ProgressBar;
use self::common::NpmPackageFsResolver;
@ -49,6 +49,7 @@ pub struct NpmProcessState {
/// Brings together the npm resolution with the file system.
pub struct CliNpmResolver {
fs: Arc<dyn FileSystem>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
resolution: Arc<NpmResolution>,
maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
@ -57,6 +58,7 @@ pub struct CliNpmResolver {
impl std::fmt::Debug for CliNpmResolver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NpmPackageResolver")
.field("fs", &"<omitted>")
.field("fs_resolver", &"<omitted>")
.field("resolution", &"<omitted>")
.field("maybe_lockfile", &"<omitted>")
@ -66,11 +68,13 @@ impl std::fmt::Debug for CliNpmResolver {
impl CliNpmResolver {
pub fn new(
fs: Arc<dyn FileSystem>,
resolution: Arc<NpmResolution>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
) -> Self {
Self {
fs,
fs_resolver,
resolution,
maybe_lockfile,
@ -81,6 +85,10 @@ impl CliNpmResolver {
self.fs_resolver.root_dir_url()
}
pub fn node_modules_path(&self) -> Option<PathBuf> {
self.fs_resolver.node_modules_path()
}
pub fn resolve_pkg_id_from_pkg_req(
&self,
req: &NpmPackageReq,
@ -88,12 +96,17 @@ impl CliNpmResolver {
self.resolution.resolve_pkg_id_from_pkg_req(req)
}
fn resolve_pkg_folder_from_deno_module_at_pkg_id(
pub fn resolve_pkg_folder_from_pkg_id(
&self,
pkg_id: &NpmPackageId,
) -> Result<PathBuf, AnyError> {
let path = self.fs_resolver.package_folder(pkg_id)?;
let path = canonicalize_path_maybe_not_exists(&path)?;
let path = canonicalize_path_maybe_not_exists_with_fs(&path, |path| {
self
.fs
.realpath_sync(path)
.map_err(|err| err.into_io_error())
})?;
log::debug!(
"Resolved package folder of {} to {}",
pkg_id.as_serialized(),
@ -237,7 +250,7 @@ impl NpmResolver for CliNpmResolver {
pkg_nv: &NpmPackageNv,
) -> Result<PathBuf, AnyError> {
let pkg_id = self.resolution.resolve_pkg_id_from_deno_module(pkg_nv)?;
self.resolve_pkg_folder_from_deno_module_at_pkg_id(&pkg_id)
self.resolve_pkg_folder_from_pkg_id(&pkg_id)
}
fn resolve_pkg_id_from_pkg_req(
@ -270,7 +283,7 @@ impl NpmResolver for CliNpmResolver {
}
pub fn create_npm_fs_resolver(
fs: Arc<dyn deno_fs::FileSystem>,
fs: Arc<dyn FileSystem>,
cache: Arc<NpmCache>,
progress_bar: &ProgressBar,
registry_url: Url,
@ -287,6 +300,7 @@ pub fn create_npm_fs_resolver(
resolution,
)),
None => Arc::new(GlobalNpmPackageResolver::new(
fs,
cache,
registry_url,
resolution,

112
cli/resolver.rs
View file

@ -20,16 +20,86 @@ use std::sync::Arc;
use crate::args::package_json::PackageJsonDeps;
use crate::args::JsxImportSourceConfig;
use crate::args::PackageJsonDepsProvider;
use crate::npm::CliNpmRegistryApi;
use crate::npm::NpmResolution;
use crate::npm::PackageJsonDepsInstaller;
use crate::util::sync::AtomicFlag;
/// Result of checking if a specifier is mapped via
/// an import map or package.json.
pub enum MappedResolution {
None,
PackageJson(ModuleSpecifier),
ImportMap(ModuleSpecifier),
}
impl MappedResolution {
pub fn into_specifier(self) -> Option<ModuleSpecifier> {
match self {
MappedResolution::None => Option::None,
MappedResolution::PackageJson(specifier) => Some(specifier),
MappedResolution::ImportMap(specifier) => Some(specifier),
}
}
}
/// Resolver for specifiers that could be mapped via an
/// import map or package.json.
#[derive(Debug)]
pub struct MappedSpecifierResolver {
maybe_import_map: Option<Arc<ImportMap>>,
package_json_deps_provider: Arc<PackageJsonDepsProvider>,
}
impl MappedSpecifierResolver {
pub fn new(
maybe_import_map: Option<Arc<ImportMap>>,
package_json_deps_provider: Arc<PackageJsonDepsProvider>,
) -> Self {
Self {
maybe_import_map,
package_json_deps_provider,
}
}
pub fn resolve(
&self,
specifier: &str,
referrer: &ModuleSpecifier,
) -> Result<MappedResolution, AnyError> {
// attempt to resolve with the import map first
let maybe_import_map_err = match self
.maybe_import_map
.as_ref()
.map(|import_map| import_map.resolve(specifier, referrer))
{
Some(Ok(value)) => return Ok(MappedResolution::ImportMap(value)),
Some(Err(err)) => Some(err),
None => None,
};
// then with package.json
if let Some(deps) = self.package_json_deps_provider.deps() {
if let Some(specifier) = resolve_package_json_dep(specifier, deps)? {
return Ok(MappedResolution::PackageJson(specifier));
}
}
// otherwise, surface the import map error or try resolving when has no import map
if let Some(err) = maybe_import_map_err {
Err(err.into())
} else {
Ok(MappedResolution::None)
}
}
}
/// A resolver that takes care of resolution, taking into account loaded
/// import map, JSX settings.
#[derive(Debug)]
pub struct CliGraphResolver {
maybe_import_map: Option<Arc<ImportMap>>,
mapped_specifier_resolver: MappedSpecifierResolver,
maybe_default_jsx_import_source: Option<String>,
maybe_jsx_import_source_module: Option<String>,
no_npm: bool,
@ -51,7 +121,10 @@ impl Default for CliGraphResolver {
None,
));
Self {
mapped_specifier_resolver: MappedSpecifierResolver {
maybe_import_map: Default::default(),
package_json_deps_provider: Default::default(),
},
maybe_default_jsx_import_source: Default::default(),
maybe_jsx_import_source_module: Default::default(),
no_npm: false,
@ -71,10 +144,14 @@ impl CliGraphResolver {
no_npm: bool,
npm_registry_api: Arc<CliNpmRegistryApi>,
npm_resolution: Arc<NpmResolution>,
package_json_deps_provider: Arc<PackageJsonDepsProvider>,
package_json_deps_installer: Arc<PackageJsonDepsInstaller>,
) -> Self {
Self {
mapped_specifier_resolver: MappedSpecifierResolver {
maybe_import_map,
package_json_deps_provider,
},
maybe_default_jsx_import_source: maybe_jsx_import_source_config
.as_ref()
.and_then(|c| c.default_specifier.clone()),
@ -135,31 +212,20 @@ impl Resolver for CliGraphResolver {
specifier: &str,
referrer: &ModuleSpecifier,
) -> Result<ModuleSpecifier, AnyError> {
// attempt to resolve with the import map first
let maybe_import_map_err = match self
.maybe_import_map
.as_ref()
.map(|import_map| import_map.resolve(specifier, referrer))
use MappedResolution::*;
match self
.mapped_specifier_resolver
.resolve(specifier, referrer)?
{
Some(Ok(value)) => return Ok(value),
Some(Err(err)) => Some(err),
None => None,
};
// then with package.json
if let Some(deps) = self.package_json_deps_installer.package_deps().as_ref()
{
if let Some(specifier) = resolve_package_json_dep(specifier, deps)? {
ImportMap(specifier) => Ok(specifier),
PackageJson(specifier) => {
// found a specifier in the package.json, so mark that
// we need to do an "npm install" later
self.found_package_json_dep_flag.raise();
return Ok(specifier);
Ok(specifier)
}
}
// otherwise, surface the import map error or try resolving when has no import map
if let Some(err) = maybe_import_map_err {
Err(err.into())
} else {
deno_graph::resolve_import(specifier, referrer).map_err(|err| err.into())
None => deno_graph::resolve_import(specifier, referrer)
.map_err(|err| err.into()),
}
}
}

314
cli/standalone/binary.rs
View file

@ -1,10 +1,13 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use std::collections::BTreeMap;
use std::env::current_exe;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::Context;
@ -14,22 +17,112 @@ use deno_core::futures::AsyncReadExt;
use deno_core::futures::AsyncSeekExt;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_npm::registry::PackageDepNpmSchemeValueParseError;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_runtime::permissions::PermissionsOptions;
use deno_semver::npm::NpmPackageReq;
use deno_semver::npm::NpmVersionReqSpecifierParseError;
use log::Level;
use serde::Deserialize;
use serde::Serialize;
use crate::args::package_json::PackageJsonDepValueParseError;
use crate::args::package_json::PackageJsonDeps;
use crate::args::CaData;
use crate::args::CliOptions;
use crate::args::CompileFlags;
use crate::args::PackageJsonDepsProvider;
use crate::cache::DenoDir;
use crate::file_fetcher::FileFetcher;
use crate::http_util::HttpClient;
use crate::npm::CliNpmRegistryApi;
use crate::npm::CliNpmResolver;
use crate::npm::NpmCache;
use crate::npm::NpmResolution;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use super::virtual_fs::FileBackedVfs;
use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VfsRoot;
use super::virtual_fs::VirtualDirectory;
const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd";
#[derive(Serialize, Deserialize)]
enum SerializablePackageJsonDepValueParseError {
SchemeValue(String),
Specifier(String),
Unsupported { scheme: String },
}
impl SerializablePackageJsonDepValueParseError {
pub fn from_err(err: PackageJsonDepValueParseError) -> Self {
match err {
PackageJsonDepValueParseError::SchemeValue(err) => {
Self::SchemeValue(err.value)
}
PackageJsonDepValueParseError::Specifier(err) => {
Self::Specifier(err.source.to_string())
}
PackageJsonDepValueParseError::Unsupported { scheme } => {
Self::Unsupported { scheme }
}
}
}
pub fn into_err(self) -> PackageJsonDepValueParseError {
match self {
SerializablePackageJsonDepValueParseError::SchemeValue(value) => {
PackageJsonDepValueParseError::SchemeValue(
PackageDepNpmSchemeValueParseError { value },
)
}
SerializablePackageJsonDepValueParseError::Specifier(source) => {
PackageJsonDepValueParseError::Specifier(
NpmVersionReqSpecifierParseError {
source: monch::ParseErrorFailureError::new(source),
},
)
}
SerializablePackageJsonDepValueParseError::Unsupported { scheme } => {
PackageJsonDepValueParseError::Unsupported { scheme }
}
}
}
}
#[derive(Serialize, Deserialize)]
pub struct SerializablePackageJsonDeps(
BTreeMap<
String,
Result<NpmPackageReq, SerializablePackageJsonDepValueParseError>,
>,
);
impl SerializablePackageJsonDeps {
pub fn from_deps(deps: PackageJsonDeps) -> Self {
Self(
deps
.into_iter()
.map(|(name, req)| {
let res =
req.map_err(SerializablePackageJsonDepValueParseError::from_err);
(name, res)
})
.collect(),
)
}
pub fn into_deps(self) -> PackageJsonDeps {
self
.0
.into_iter()
.map(|(name, res)| (name, res.map_err(|err| err.into_err())))
.collect()
}
}
#[derive(Deserialize, Serialize)]
pub struct Metadata {
pub argv: Vec<String>,
@ -44,27 +137,74 @@ pub struct Metadata {
pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
pub maybe_import_map: Option<(Url, String)>,
pub entrypoint: ModuleSpecifier,
/// Whether this uses a node_modules directory (true) or the global cache (false).
pub node_modules_dir: bool,
pub npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
pub package_json_deps: Option<SerializablePackageJsonDeps>,
}
pub fn write_binary_bytes(
pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result<FileBackedVfs, AnyError> {
let file_path = current_exe().unwrap();
let mut file = std::fs::File::open(file_path)?;
file.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))?;
let mut trailer = [0; TRAILER_SIZE];
file.read_exact(&mut trailer)?;
let trailer = Trailer::parse(&trailer)?.unwrap();
file.seek(SeekFrom::Start(trailer.npm_vfs_pos))?;
let mut vfs_data = vec![0; trailer.npm_vfs_len() as usize];
file.read_exact(&mut vfs_data)?;
let mut dir: VirtualDirectory = serde_json::from_slice(&vfs_data)?;
// align the name of the directory with the root dir
dir.name = root_dir_path
.file_name()
.unwrap()
.to_string_lossy()
.to_string();
let fs_root = VfsRoot {
dir,
root_path: root_dir_path,
start_file_offset: trailer.npm_files_pos,
};
Ok(FileBackedVfs::new(file, fs_root))
}
fn write_binary_bytes(
writer: &mut impl Write,
original_bin: Vec<u8>,
metadata: &Metadata,
eszip: eszip::EszipV2,
npm_vfs: Option<&VirtualDirectory>,
npm_files: &Vec<Vec<u8>>,
) -> Result<(), AnyError> {
let metadata = serde_json::to_string(metadata)?.as_bytes().to_vec();
let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec();
let eszip_archive = eszip.into_bytes();
let eszip_pos = original_bin.len();
let metadata_pos = eszip_pos + eszip_archive.len();
let mut trailer = MAGIC_TRAILER.to_vec();
trailer.write_all(&eszip_pos.to_be_bytes())?;
trailer.write_all(&metadata_pos.to_be_bytes())?;
writer.write_all(&original_bin)?;
writer.write_all(&eszip_archive)?;
writer.write_all(&metadata)?;
writer.write_all(&trailer)?;
writer.write_all(&npm_vfs)?;
for file in npm_files {
writer.write_all(file)?;
}
// write the trailer, which includes the positions
// of the data blocks in the file
writer.write_all(&{
let eszip_pos = original_bin.len() as u64;
let metadata_pos = eszip_pos + (eszip_archive.len() as u64);
let npm_vfs_pos = metadata_pos + (metadata.len() as u64);
let npm_files_pos = npm_vfs_pos + (npm_vfs.len() as u64);
Trailer {
eszip_pos,
metadata_pos,
npm_vfs_pos,
npm_files_pos,
}
.as_bytes()
})?;
Ok(())
}
@ -73,12 +213,15 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool {
let Ok(mut output_file) = std::fs::File::open(exe_path) else {
return false;
};
if output_file.seek(SeekFrom::End(-24)).is_err() {
if output_file
.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))
.is_err()
{
// This seek may fail because the file is too small to possibly be
// `deno compile` output.
return false;
}
let mut trailer = [0; 24];
let mut trailer = [0; TRAILER_SIZE];
if output_file.read_exact(&mut trailer).is_err() {
return false;
};
@ -88,13 +231,9 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool {
/// This function will try to run this binary as a standalone binary
/// produced by `deno compile`. It determines if this is a standalone
/// binary by checking for the magic trailer string `d3n0l4nd` at EOF-24 (8 bytes * 3).
/// The magic trailer is followed by:
/// - a u64 pointer to the JS bundle embedded in the binary
/// - a u64 pointer to JSON metadata (serialized flags) embedded in the binary
/// These are dereferenced, and the bundle is executed under the configuration
/// specified by the metadata. If no magic trailer is present, this function
/// exits with `Ok(None)`.
/// binary by skipping over the trailer width at the end of the file,
/// then checking for the magic trailer string `d3n0l4nd`. If found,
/// the bundle is executed. If not, this function exits with `Ok(None)`.
pub async fn extract_standalone(
exe_path: &Path,
cli_args: Vec<String>,
@ -104,21 +243,17 @@ pub async fn extract_standalone(
let mut bufreader =
deno_core::futures::io::BufReader::new(AllowStdIo::new(file));
let trailer_pos = bufreader.seek(SeekFrom::End(-24)).await?;
let mut trailer = [0; 24];
let _trailer_pos = bufreader
.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))
.await?;
let mut trailer = [0; TRAILER_SIZE];
bufreader.read_exact(&mut trailer).await?;
let (magic_trailer, rest) = trailer.split_at(8);
if magic_trailer != MAGIC_TRAILER {
return Ok(None);
}
let trailer = match Trailer::parse(&trailer)? {
None => return Ok(None),
Some(trailer) => trailer,
};
let (eszip_archive_pos, rest) = rest.split_at(8);
let metadata_pos = rest;
let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?;
let metadata_pos = u64_from_bytes(metadata_pos)?;
let metadata_len = trailer_pos - metadata_pos;
bufreader.seek(SeekFrom::Start(eszip_archive_pos)).await?;
bufreader.seek(SeekFrom::Start(trailer.eszip_pos)).await?;
let (eszip, loader) = eszip::EszipV2::parse(bufreader)
.await
@ -126,12 +261,14 @@ pub async fn extract_standalone(
let mut bufreader = loader.await.context("Failed to parse eszip archive")?;
bufreader.seek(SeekFrom::Start(metadata_pos)).await?;
bufreader
.seek(SeekFrom::Start(trailer.metadata_pos))
.await?;
let mut metadata = String::new();
bufreader
.take(metadata_len)
.take(trailer.metadata_len())
.read_to_string(&mut metadata)
.await
.context("Failed to read metadata from the current executable")?;
@ -142,6 +279,57 @@ pub async fn extract_standalone(
Ok(Some((metadata, eszip)))
}
const TRAILER_SIZE: usize = std::mem::size_of::<Trailer>() + 8; // 8 bytes for the magic trailer string
struct Trailer {
eszip_pos: u64,
metadata_pos: u64,
npm_vfs_pos: u64,
npm_files_pos: u64,
}
impl Trailer {
pub fn parse(trailer: &[u8]) -> Result<Option<Trailer>, AnyError> {
let (magic_trailer, rest) = trailer.split_at(8);
if magic_trailer != MAGIC_TRAILER {
return Ok(None);
}
let (eszip_archive_pos, rest) = rest.split_at(8);
let (metadata_pos, rest) = rest.split_at(8);
let (npm_vfs_pos, npm_files_pos) = rest.split_at(8);
let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?;
let metadata_pos = u64_from_bytes(metadata_pos)?;
let npm_vfs_pos = u64_from_bytes(npm_vfs_pos)?;
let npm_files_pos = u64_from_bytes(npm_files_pos)?;
Ok(Some(Trailer {
eszip_pos: eszip_archive_pos,
metadata_pos,
npm_vfs_pos,
npm_files_pos,
}))
}
pub fn metadata_len(&self) -> u64 {
self.npm_vfs_pos - self.metadata_pos
}
pub fn npm_vfs_len(&self) -> u64 {
self.npm_files_pos - self.npm_vfs_pos
}
pub fn as_bytes(&self) -> Vec<u8> {
let mut trailer = MAGIC_TRAILER.to_vec();
trailer.write_all(&self.eszip_pos.to_be_bytes()).unwrap();
trailer.write_all(&self.metadata_pos.to_be_bytes()).unwrap();
trailer.write_all(&self.npm_vfs_pos.to_be_bytes()).unwrap();
trailer
.write_all(&self.npm_files_pos.to_be_bytes())
.unwrap();
trailer
}
}
fn u64_from_bytes(arr: &[u8]) -> Result<u64, AnyError> {
let fixed_arr: &[u8; 8] = arr
.try_into()
@ -153,18 +341,34 @@ pub struct DenoCompileBinaryWriter<'a> {
file_fetcher: &'a FileFetcher,
client: &'a HttpClient,
deno_dir: &'a DenoDir,
npm_api: &'a CliNpmRegistryApi,
npm_cache: &'a NpmCache,
npm_resolver: &'a CliNpmResolver,
resolution: &'a NpmResolution,
package_json_deps_provider: &'a PackageJsonDepsProvider,
}
impl<'a> DenoCompileBinaryWriter<'a> {
#[allow(clippy::too_many_arguments)]
pub fn new(
file_fetcher: &'a FileFetcher,
client: &'a HttpClient,
deno_dir: &'a DenoDir,
npm_api: &'a CliNpmRegistryApi,
npm_cache: &'a NpmCache,
npm_resolver: &'a CliNpmResolver,
resolution: &'a NpmResolution,
package_json_deps_provider: &'a PackageJsonDepsProvider,
) -> Self {
Self {
file_fetcher,
client,
deno_dir,
npm_api,
npm_cache,
npm_resolver,
resolution,
package_json_deps_provider,
}
}
@ -284,6 +488,14 @@ impl<'a> DenoCompileBinaryWriter<'a> {
.resolve_import_map(self.file_fetcher)
.await?
.map(|import_map| (import_map.base_url().clone(), import_map.to_json()));
let (npm_snapshot, npm_vfs, npm_files) = if self.resolution.has_packages() {
let (root_dir, files) = self.build_vfs()?.into_dir_and_files();
let snapshot = self.resolution.serialized_snapshot();
(Some(snapshot), Some(root_dir), files)
} else {
(None, None, Vec::new())
};
let metadata = Metadata {
argv: compile_flags.args.clone(),
unstable: cli_options.unstable(),
@ -299,8 +511,44 @@ impl<'a> DenoCompileBinaryWriter<'a> {
ca_data,
entrypoint: entrypoint.clone(),
maybe_import_map,
node_modules_dir: self.npm_resolver.node_modules_path().is_some(),
npm_snapshot,
package_json_deps: self
.package_json_deps_provider
.deps()
.map(|deps| SerializablePackageJsonDeps::from_deps(deps.clone())),
};
write_binary_bytes(writer, original_bin, &metadata, eszip)
write_binary_bytes(
writer,
original_bin,
&metadata,
eszip,
npm_vfs.as_ref(),
&npm_files,
)
}
fn build_vfs(&self) -> Result<VfsBuilder, AnyError> {
if let Some(node_modules_path) = self.npm_resolver.node_modules_path() {
let mut builder = VfsBuilder::new(node_modules_path.clone());
builder.add_dir_recursive(&node_modules_path)?;
Ok(builder)
} else {
// DO NOT include the user's registry url as it may contain credentials,
// but also don't make this dependent on the registry url
let registry_url = self.npm_api.base_url();
let root_path = self.npm_cache.registry_folder(registry_url);
let mut builder = VfsBuilder::new(root_path);
for package in self.resolution.all_packages() {
let folder = self
.npm_resolver
.resolve_pkg_folder_from_pkg_id(&package.pkg_id)?;
builder.add_dir_recursive(&folder)?;
}
// overwrite the root directory's name to obscure the user's registry url
builder.set_root_dir_name("node_modules".to_string());
Ok(builder)
}
}
}

337
cli/standalone/file_system.rs Normal file
View file

@ -0,0 +1,337 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsDirEntry;
use deno_runtime::deno_fs::FsFileType;
use deno_runtime::deno_fs::OpenOptions;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_io::fs::File;
use deno_runtime::deno_io::fs::FsError;
use deno_runtime::deno_io::fs::FsResult;
use deno_runtime::deno_io::fs::FsStat;
use super::virtual_fs::FileBackedVfs;
#[derive(Debug, Clone)]
pub struct DenoCompileFileSystem(Arc<FileBackedVfs>);
impl DenoCompileFileSystem {
pub fn new(vfs: FileBackedVfs) -> Self {
Self(Arc::new(vfs))
}
fn error_if_in_vfs(&self, path: &Path) -> FsResult<()> {
if self.0.is_path_within(path) {
Err(FsError::NotSupported)
} else {
Ok(())
}
}
fn copy_to_real_path(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> {
let old_file = self.0.file_entry(oldpath)?;
let old_file_bytes = self.0.read_file_all(old_file)?;
RealFs.write_file_sync(
newpath,
OpenOptions {
read: false,
write: true,
create: true,
truncate: true,
append: false,
create_new: false,
mode: None,
},
&old_file_bytes,
)
}
}
#[async_trait::async_trait(?Send)]
impl FileSystem for DenoCompileFileSystem {
fn cwd(&self) -> FsResult<PathBuf> {
RealFs.cwd()
}
fn tmp_dir(&self) -> FsResult<PathBuf> {
RealFs.tmp_dir()
}
fn chdir(&self, path: &Path) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.chdir(path)
}
fn umask(&self, mask: Option<u32>) -> FsResult<u32> {
RealFs.umask(mask)
}
fn open_sync(
&self,
path: &Path,
options: OpenOptions,
) -> FsResult<Rc<dyn File>> {
if self.0.is_path_within(path) {
Ok(self.0.open_file(path)?)
} else {
RealFs.open_sync(path, options)
}
}
async fn open_async(
&self,
path: PathBuf,
options: OpenOptions,
) -> FsResult<Rc<dyn File>> {
if self.0.is_path_within(&path) {
Ok(self.0.open_file(&path)?)
} else {
RealFs.open_async(path, options).await
}
}
fn mkdir_sync(
&self,
path: &Path,
recursive: bool,
mode: u32,
) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.mkdir_sync(path, recursive, mode)
}
async fn mkdir_async(
&self,
path: PathBuf,
recursive: bool,
mode: u32,
) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.mkdir_async(path, recursive, mode).await
}
fn chmod_sync(&self, path: &Path, mode: u32) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.chmod_sync(path, mode)
}
async fn chmod_async(&self, path: PathBuf, mode: u32) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.chmod_async(path, mode).await
}
fn chown_sync(
&self,
path: &Path,
uid: Option<u32>,
gid: Option<u32>,
) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.chown_sync(path, uid, gid)
}
async fn chown_async(
&self,
path: PathBuf,
uid: Option<u32>,
gid: Option<u32>,
) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.chown_async(path, uid, gid).await
}
fn remove_sync(&self, path: &Path, recursive: bool) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.remove_sync(path, recursive)
}
async fn remove_async(&self, path: PathBuf, recursive: bool) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.remove_async(path, recursive).await
}
fn copy_file_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> {
self.error_if_in_vfs(newpath)?;
if self.0.is_path_within(oldpath) {
self.copy_to_real_path(oldpath, newpath)
} else {
RealFs.copy_file_sync(oldpath, newpath)
}
}
async fn copy_file_async(
&self,
oldpath: PathBuf,
newpath: PathBuf,
) -> FsResult<()> {
self.error_if_in_vfs(&newpath)?;
if self.0.is_path_within(&oldpath) {
let fs = self.clone();
tokio::task::spawn_blocking(move || {
fs.copy_to_real_path(&oldpath, &newpath)
})
.await?
} else {
RealFs.copy_file_async(oldpath, newpath).await
}
}
fn stat_sync(&self, path: &Path) -> FsResult<FsStat> {
if self.0.is_path_within(path) {
Ok(self.0.stat(path)?)
} else {
RealFs.stat_sync(path)
}
}
async fn stat_async(&self, path: PathBuf) -> FsResult<FsStat> {
if self.0.is_path_within(&path) {
Ok(self.0.stat(&path)?)
} else {
RealFs.stat_async(path).await
}
}
fn lstat_sync(&self, path: &Path) -> FsResult<FsStat> {
if self.0.is_path_within(path) {
Ok(self.0.lstat(path)?)
} else {
RealFs.lstat_sync(path)
}
}
async fn lstat_async(&self, path: PathBuf) -> FsResult<FsStat> {
if self.0.is_path_within(&path) {
Ok(self.0.lstat(&path)?)
} else {
RealFs.lstat_async(path).await
}
}
fn realpath_sync(&self, path: &Path) -> FsResult<PathBuf> {
if self.0.is_path_within(path) {
Ok(self.0.canonicalize(path)?)
} else {
RealFs.realpath_sync(path)
}
}
async fn realpath_async(&self, path: PathBuf) -> FsResult<PathBuf> {
if self.0.is_path_within(&path) {
Ok(self.0.canonicalize(&path)?)
} else {
RealFs.realpath_async(path).await
}
}
fn read_dir_sync(&self, path: &Path) -> FsResult<Vec<FsDirEntry>> {
if self.0.is_path_within(path) {
Ok(self.0.read_dir(path)?)
} else {
RealFs.read_dir_sync(path)
}
}
async fn read_dir_async(&self, path: PathBuf) -> FsResult<Vec<FsDirEntry>> {
if self.0.is_path_within(&path) {
Ok(self.0.read_dir(&path)?)
} else {
RealFs.read_dir_async(path).await
}
}
fn rename_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> {
self.error_if_in_vfs(oldpath)?;
self.error_if_in_vfs(newpath)?;
RealFs.rename_sync(oldpath, newpath)
}
async fn rename_async(
&self,
oldpath: PathBuf,
newpath: PathBuf,
) -> FsResult<()> {
self.error_if_in_vfs(&oldpath)?;
self.error_if_in_vfs(&newpath)?;
RealFs.rename_async(oldpath, newpath).await
}
fn link_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> {
self.error_if_in_vfs(oldpath)?;
self.error_if_in_vfs(newpath)?;
RealFs.link_sync(oldpath, newpath)
}
async fn link_async(
&self,
oldpath: PathBuf,
newpath: PathBuf,
) -> FsResult<()> {
self.error_if_in_vfs(&oldpath)?;
self.error_if_in_vfs(&newpath)?;
RealFs.link_async(oldpath, newpath).await
}
fn symlink_sync(
&self,
oldpath: &Path,
newpath: &Path,
file_type: Option<FsFileType>,
) -> FsResult<()> {
self.error_if_in_vfs(oldpath)?;
self.error_if_in_vfs(newpath)?;
RealFs.symlink_sync(oldpath, newpath, file_type)
}
async fn symlink_async(
&self,
oldpath: PathBuf,
newpath: PathBuf,
file_type: Option<FsFileType>,
) -> FsResult<()> {
self.error_if_in_vfs(&oldpath)?;
self.error_if_in_vfs(&newpath)?;
RealFs.symlink_async(oldpath, newpath, file_type).await
}
fn read_link_sync(&self, path: &Path) -> FsResult<PathBuf> {
if self.0.is_path_within(path) {
Ok(self.0.read_link(path)?)
} else {
RealFs.read_link_sync(path)
}
}
async fn read_link_async(&self, path: PathBuf) -> FsResult<PathBuf> {
if self.0.is_path_within(&path) {
Ok(self.0.read_link(&path)?)
} else {
RealFs.read_link_async(path).await
}
}
fn truncate_sync(&self, path: &Path, len: u64) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.truncate_sync(path, len)
}
async fn truncate_async(&self, path: PathBuf, len: u64) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.truncate_async(path, len).await
}
fn utime_sync(
&self,
path: &Path,
atime_secs: i64,
atime_nanos: u32,
mtime_secs: i64,
mtime_nanos: u32,
) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.utime_sync(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos)
}
async fn utime_async(
&self,
path: PathBuf,
atime_secs: i64,
atime_nanos: u32,
mtime_secs: i64,
mtime_nanos: u32,
) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs
.utime_async(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos)
.await
}
}

255
cli/standalone/mod.rs
View file

@ -1,17 +1,25 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use crate::args::get_root_cert_store;
use crate::args::npm_pkg_req_ref_to_binary_command;
use crate::args::CaData;
use crate::args::CacheSetting;
use crate::args::PackageJsonDepsProvider;
use crate::args::StorageKeyResolver;
use crate::cache::Caches;
use crate::cache::DenoDir;
use crate::cache::NodeAnalysisCache;
use crate::file_fetcher::get_source_from_data_url;
use crate::http_util::HttpClient;
use crate::module_loader::CjsResolutionStore;
use crate::module_loader::NpmModuleLoader;
use crate::node::CliCjsEsmCodeAnalyzer;
use crate::npm::create_npm_fs_resolver;
use crate::npm::CliNpmRegistryApi;
use crate::npm::CliNpmResolver;
use crate::npm::NpmCache;
use crate::npm::NpmResolution;
use crate::resolver::MappedSpecifierResolver;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use crate::util::v8::construct_v8_flags;
@ -19,7 +27,7 @@ use crate::worker::CliMainWorkerFactory;
use crate::worker::CliMainWorkerOptions;
use crate::worker::HasNodeSpecifierChecker;
use crate::worker::ModuleLoaderFactory;
use crate::CliGraphResolver;
use deno_ast::MediaType;
use deno_core::anyhow::Context;
use deno_core::error::type_error;
use deno_core::error::AnyError;
@ -29,31 +37,44 @@ use deno_core::ModuleLoader;
use deno_core::ModuleSpecifier;
use deno_core::ModuleType;
use deno_core::ResolutionKind;
use deno_graph::source::Resolver;
use deno_runtime::deno_fs;
use deno_runtime::deno_node;
use deno_runtime::deno_node::analyze::NodeCodeTranslator;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_tls::rustls::RootCertStore;
use deno_runtime::deno_tls::RootCertStoreProvider;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::permissions::Permissions;
use deno_runtime::permissions::PermissionsContainer;
use deno_semver::npm::NpmPackageReqReference;
use import_map::parse_from_json;
use std::pin::Pin;
use std::rc::Rc;
use std::sync::Arc;
mod binary;
mod file_system;
mod virtual_fs;
pub use binary::extract_standalone;
pub use binary::is_standalone_binary;
pub use binary::DenoCompileBinaryWriter;
use self::binary::load_npm_vfs;
use self::binary::Metadata;
use self::file_system::DenoCompileFileSystem;
struct SharedModuleLoaderState {
eszip: eszip::EszipV2,
mapped_specifier_resolver: MappedSpecifierResolver,
npm_module_loader: Arc<NpmModuleLoader>,
}
#[derive(Clone)]
struct EmbeddedModuleLoader {
eszip: Arc<eszip::EszipV2>,
maybe_import_map_resolver: Option<Arc<CliGraphResolver>>,
shared: Arc<SharedModuleLoaderState>,
root_permissions: PermissionsContainer,
dynamic_permissions: PermissionsContainer,
}
impl ModuleLoader for EmbeddedModuleLoader {
@ -61,10 +82,10 @@ impl ModuleLoader for EmbeddedModuleLoader {
&self,
specifier: &str,
referrer: &str,
_kind: ResolutionKind,
kind: ResolutionKind,
) -> Result<ModuleSpecifier, AnyError> {
// Try to follow redirects when resolving.
let referrer = match self.eszip.get_module(referrer) {
let referrer = match self.shared.eszip.get_module(referrer) {
Some(eszip::Module { ref specifier, .. }) => {
ModuleSpecifier::parse(specifier)?
}
@ -74,27 +95,93 @@ impl ModuleLoader for EmbeddedModuleLoader {
}
};
self
.maybe_import_map_resolver
let permissions = if matches!(kind, ResolutionKind::DynamicImport) {
&self.dynamic_permissions
} else {
&self.root_permissions
};
if let Some(result) = self
.shared
.npm_module_loader
.resolve_if_in_npm_package(specifier, &referrer, permissions)
{
return result;
}
let maybe_mapped = self
.shared
.mapped_specifier_resolver
.resolve(specifier, &referrer)?
.into_specifier();
// npm specifier
let specifier_text = maybe_mapped
.as_ref()
.map(|r| r.resolve(specifier, &referrer))
.unwrap_or_else(|| {
deno_core::resolve_import(specifier, referrer.as_str())
.map_err(|err| err.into())
})
.map(|r| r.as_str())
.unwrap_or(specifier);
if let Ok(reference) = NpmPackageReqReference::from_str(specifier_text) {
return self
.shared
.npm_module_loader
.resolve_req_reference(&reference, permissions);
}
// Built-in Node modules
if let Some(module_name) = specifier_text.strip_prefix("node:") {
return deno_node::resolve_builtin_node_module(module_name);
}
match maybe_mapped {
Some(resolved) => Ok(resolved),
None => deno_core::resolve_import(specifier, referrer.as_str())
.map_err(|err| err.into()),
}
}
fn load(
&self,
module_specifier: &ModuleSpecifier,
_maybe_referrer: Option<&ModuleSpecifier>,
_is_dynamic: bool,
maybe_referrer: Option<&ModuleSpecifier>,
is_dynamic: bool,
) -> Pin<Box<deno_core::ModuleSourceFuture>> {
let is_data_uri = get_source_from_data_url(module_specifier).ok();
let permissions = if is_dynamic {
&self.dynamic_permissions
} else {
&self.root_permissions
};
if let Some(result) =
self.shared.npm_module_loader.load_sync_if_in_npm_package(
module_specifier,
maybe_referrer,
permissions,
)
{
return match result {
Ok(code_source) => Box::pin(deno_core::futures::future::ready(Ok(
deno_core::ModuleSource::new_with_redirect(
match code_source.media_type {
MediaType::Json => ModuleType::Json,
_ => ModuleType::JavaScript,
},
code_source.code,
module_specifier,
&code_source.found_url,
),
))),
Err(err) => Box::pin(deno_core::futures::future::ready(Err(err))),
};
}
let module = self
.shared
.eszip
.get_module(module_specifier.as_str())
.ok_or_else(|| type_error("Module not found"));
.ok_or_else(|| {
type_error(format!("Module not found: {}", module_specifier))
});
// TODO(mmastrac): This clone can probably be removed in the future if ModuleSpecifier is no longer a full-fledged URL
let module_specifier = module_specifier.clone();
@ -128,24 +215,32 @@ impl ModuleLoader for EmbeddedModuleLoader {
}
struct StandaloneModuleLoaderFactory {
loader: EmbeddedModuleLoader,
shared: Arc<SharedModuleLoaderState>,
}
impl ModuleLoaderFactory for StandaloneModuleLoaderFactory {
fn create_for_main(
&self,
_root_permissions: PermissionsContainer,
_dynamic_permissions: PermissionsContainer,
root_permissions: PermissionsContainer,
dynamic_permissions: PermissionsContainer,
) -> Rc<dyn ModuleLoader> {
Rc::new(self.loader.clone())
Rc::new(EmbeddedModuleLoader {
shared: self.shared.clone(),
root_permissions,
dynamic_permissions,
})
}
fn create_for_worker(
&self,
_root_permissions: PermissionsContainer,
_dynamic_permissions: PermissionsContainer,
root_permissions: PermissionsContainer,
dynamic_permissions: PermissionsContainer,
) -> Rc<dyn ModuleLoader> {
Rc::new(self.loader.clone())
Rc::new(EmbeddedModuleLoader {
shared: self.shared.clone(),
root_permissions,
dynamic_permissions,
})
}
fn create_source_map_getter(
@ -183,6 +278,9 @@ pub async fn run(
metadata: Metadata,
) -> Result<(), AnyError> {
let main_module = &metadata.entrypoint;
let current_exe_path = std::env::current_exe().unwrap();
let current_exe_name =
current_exe_path.file_name().unwrap().to_string_lossy();
let dir = DenoDir::new(None)?;
let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider {
ca_stores: metadata.ca_stores,
@ -194,9 +292,14 @@ pub async fn run(
Some(root_cert_store_provider.clone()),
metadata.unsafely_ignore_certificate_errors.clone(),
));
let npm_registry_url = CliNpmRegistryApi::default_url().to_owned();
// use a dummy npm registry url
let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap();
let root_path = std::env::temp_dir()
.join(format!("deno-compile-{}", current_exe_name))
.join("node_modules");
let npm_cache = Arc::new(NpmCache::new(
dir.npm_folder_path(),
root_path.clone(),
CacheSetting::Use,
http_client.clone(),
progress_bar.clone(),
@ -207,44 +310,92 @@ pub async fn run(
http_client.clone(),
progress_bar.clone(),
));
let fs = Arc::new(deno_fs::RealFs);
let npm_resolution =
Arc::new(NpmResolution::from_serialized(npm_api.clone(), None, None));
let (fs, node_modules_path, snapshot) = if let Some(snapshot) =
metadata.npm_snapshot
{
let vfs_root_dir_path = if metadata.node_modules_dir {
root_path
} else {
npm_cache.registry_folder(&npm_registry_url)
};
let vfs =
load_npm_vfs(vfs_root_dir_path).context("Failed to load npm vfs.")?;
let node_modules_path = if metadata.node_modules_dir {
Some(vfs.root().to_path_buf())
} else {
None
};
(
Arc::new(DenoCompileFileSystem::new(vfs)) as Arc<dyn deno_fs::FileSystem>,
node_modules_path,
Some(snapshot.into_valid()?),
)
} else {
(
Arc::new(deno_fs::RealFs) as Arc<dyn deno_fs::FileSystem>,
None,
None,
)
};
let npm_resolution = Arc::new(NpmResolution::from_serialized(
npm_api.clone(),
snapshot,
None,
));
let has_node_modules_dir = node_modules_path.is_some();
let npm_fs_resolver = create_npm_fs_resolver(
fs.clone(),
npm_cache,
&progress_bar,
npm_registry_url,
npm_resolution.clone(),
None,
node_modules_path,
);
let npm_resolver = Arc::new(CliNpmResolver::new(
fs.clone(),
npm_resolution.clone(),
npm_fs_resolver,
None,
));
let node_resolver =
Arc::new(NodeResolver::new(fs.clone(), npm_resolver.clone()));
let cjs_resolutions = Arc::new(CjsResolutionStore::default());
let cache_db = Caches::new(dir.clone());
let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db());
let cjs_esm_code_analyzer = CliCjsEsmCodeAnalyzer::new(node_analysis_cache);
let node_code_translator = Arc::new(NodeCodeTranslator::new(
cjs_esm_code_analyzer,
fs.clone(),
node_resolver.clone(),
npm_resolver.clone(),
));
let package_json_deps_provider = Arc::new(PackageJsonDepsProvider::new(
metadata
.package_json_deps
.map(|serialized| serialized.into_deps()),
));
let maybe_import_map = metadata.maybe_import_map.map(|(base, source)| {
Arc::new(parse_from_json(&base, &source).unwrap().import_map)
});
let module_loader_factory = StandaloneModuleLoaderFactory {
loader: EmbeddedModuleLoader {
eszip: Arc::new(eszip),
maybe_import_map_resolver: metadata.maybe_import_map.map(
|(base, source)| {
Arc::new(CliGraphResolver::new(
None,
Some(Arc::new(
parse_from_json(&base, &source).unwrap().import_map,
)),
false,
npm_api.clone(),
npm_resolution.clone(),
Default::default(),
))
},
shared: Arc::new(SharedModuleLoaderState {
eszip,
mapped_specifier_resolver: MappedSpecifierResolver::new(
maybe_import_map.clone(),
package_json_deps_provider.clone(),
),
},
npm_module_loader: Arc::new(NpmModuleLoader::new(
cjs_resolutions,
node_code_translator,
fs.clone(),
node_resolver.clone(),
)),
}),
};
let permissions = PermissionsContainer::new(Permissions::from_options(
&metadata.permissions,
)?);
let worker_factory = CliMainWorkerFactory::new(
StorageKeyResolver::empty(),
npm_resolver.clone(),
@ -260,14 +411,17 @@ pub async fn run(
debug: false,
coverage_dir: None,
enable_testing_features: false,
has_node_modules_dir: false,
has_node_modules_dir,
inspect_brk: false,
inspect_wait: false,
is_inspecting: false,
is_npm_main: false,
is_npm_main: main_module.scheme() == "npm",
location: metadata.location,
// todo(dsherret): support a binary command being compiled
maybe_binary_npm_command_name: None,
maybe_binary_npm_command_name: NpmPackageReqReference::from_specifier(
main_module,
)
.ok()
.map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref)),
origin_data_folder_path: None,
seed: metadata.seed,
unsafely_ignore_certificate_errors: metadata
@ -278,9 +432,6 @@ pub async fn run(
v8_set_flags(construct_v8_flags(&metadata.v8_flags, vec![]));
let permissions = PermissionsContainer::new(Permissions::from_options(
&metadata.permissions,
)?);
let mut worker = worker_factory
.create_main_worker(main_module.clone(), permissions)
.await?;

983
cli/standalone/virtual_fs.rs Normal file
View file

@ -0,0 +1,983 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fs::File;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::BufMutView;
use deno_core::BufView;
use deno_runtime::deno_fs::FsDirEntry;
use deno_runtime::deno_io;
use deno_runtime::deno_io::fs::FsError;
use deno_runtime::deno_io::fs::FsResult;
use deno_runtime::deno_io::fs::FsStat;
use serde::Deserialize;
use serde::Serialize;
use crate::util;
pub struct VfsBuilder {
root_path: PathBuf,
root_dir: VirtualDirectory,
files: Vec<Vec<u8>>,
current_offset: u64,
file_offsets: HashMap<String, u64>,
}
impl VfsBuilder {
pub fn new(root_path: PathBuf) -> Self {
Self {
root_dir: VirtualDirectory {
name: root_path
.file_stem()
.unwrap()
.to_string_lossy()
.into_owned(),
entries: Vec::new(),
},
root_path,
files: Vec::new(),
current_offset: 0,
file_offsets: Default::default(),
}
}
pub fn set_root_dir_name(&mut self, name: String) {
self.root_dir.name = name;
}
pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> {
self.add_dir(path);
let read_dir = std::fs::read_dir(path)
.with_context(|| format!("Reading {}", path.display()))?;
for entry in read_dir {
let entry = entry?;
let file_type = entry.file_type()?;
let path = entry.path();
if file_type.is_dir() {
self.add_dir_recursive(&path)?;
} else if file_type.is_file() {
let file_bytes = std::fs::read(&path)
.with_context(|| format!("Reading {}", path.display()))?;
self.add_file(&path, file_bytes);
} else if file_type.is_symlink() {
let target = std::fs::read_link(&path)
.with_context(|| format!("Reading symlink {}", path.display()))?;
self.add_symlink(&path, &target);
}
}
Ok(())
}
pub fn add_dir(&mut self, path: &Path) -> &mut VirtualDirectory {
let path = path.strip_prefix(&self.root_path).unwrap();
let mut current_dir = &mut self.root_dir;
for component in path.components() {
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&name))
{
Ok(index) => index,
Err(insert_index) => {
current_dir.entries.insert(
insert_index,
VfsEntry::Dir(VirtualDirectory {
name: name.to_string(),
entries: Vec::new(),
}),
);
insert_index
}
};
match &mut current_dir.entries[index] {
VfsEntry::Dir(dir) => {
current_dir = dir;
}
_ => unreachable!(),
};
}
current_dir
}
pub fn add_file(&mut self, path: &Path, data: Vec<u8>) {
let checksum = util::checksum::gen(&[&data]);
let offset = if let Some(offset) = self.file_offsets.get(&checksum) {
// duplicate file, reuse an old offset
*offset
} else {
self.file_offsets.insert(checksum, self.current_offset);
self.current_offset
};
let dir = self.add_dir(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
let data_len = data.len();
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
Ok(_) => unreachable!(),
Err(insert_index) => {
dir.entries.insert(
insert_index,
VfsEntry::File(VirtualFile {
name: name.to_string(),
offset,
len: data.len() as u64,
}),
);
}
}
// new file, update the list of files
if self.current_offset == offset {
self.files.push(data);
self.current_offset += data_len as u64;
}
}
pub fn add_symlink(&mut self, path: &Path, target: &Path) {
let dest = target.strip_prefix(&self.root_path).unwrap().to_path_buf();
let dir = self.add_dir(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
Ok(_) => unreachable!(),
Err(insert_index) => {
dir.entries.insert(
insert_index,
VfsEntry::Symlink(VirtualSymlink {
name: name.to_string(),
dest_parts: dest
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>(),
}),
);
}
}
}
pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec<Vec<u8>>) {
(self.root_dir, self.files)
}
}
#[derive(Debug)]
enum VfsEntryRef<'a> {
Dir(&'a VirtualDirectory),
File(&'a VirtualFile),
Symlink(&'a VirtualSymlink),
}
impl<'a> VfsEntryRef<'a> {
pub fn as_fs_stat(&self) -> FsStat {
match self {
VfsEntryRef::Dir(_) => FsStat {
is_directory: true,
is_file: false,
is_symlink: false,
atime: None,
birthtime: None,
mtime: None,
blksize: 0,
size: 0,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
},
VfsEntryRef::File(file) => FsStat {
is_directory: false,
is_file: true,
is_symlink: false,
atime: None,
birthtime: None,
mtime: None,
blksize: 0,
size: file.len,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
},
VfsEntryRef::Symlink(_) => FsStat {
is_directory: false,
is_file: false,
is_symlink: true,
atime: None,
birthtime: None,
mtime: None,
blksize: 0,
size: 0,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
},
}
}
}
// todo(dsherret): we should store this more efficiently in the binary
#[derive(Debug, Serialize, Deserialize)]
pub enum VfsEntry {
Dir(VirtualDirectory),
File(VirtualFile),
Symlink(VirtualSymlink),
}
impl VfsEntry {
pub fn name(&self) -> &str {
match self {
VfsEntry::Dir(dir) => &dir.name,
VfsEntry::File(file) => &file.name,
VfsEntry::Symlink(symlink) => &symlink.name,
}
}
fn as_ref(&self) -> VfsEntryRef {
match self {
VfsEntry::Dir(dir) => VfsEntryRef::Dir(dir),
VfsEntry::File(file) => VfsEntryRef::File(file),
VfsEntry::Symlink(symlink) => VfsEntryRef::Symlink(symlink),
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualDirectory {
pub name: String,
// should be sorted by name
pub entries: Vec<VfsEntry>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VirtualFile {
pub name: String,
pub offset: u64,
pub len: u64,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlink {
pub name: String,
pub dest_parts: Vec<String>,
}
impl VirtualSymlink {
pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf {
let mut dest = root.to_path_buf();
for part in &self.dest_parts {
dest.push(part);
}
dest
}
}
#[derive(Debug)]
pub struct VfsRoot {
pub dir: VirtualDirectory,
pub root_path: PathBuf,
pub start_file_offset: u64,
}
impl VfsRoot {
fn find_entry<'a>(
&'a self,
path: &Path,
) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> {
self.find_entry_inner(path, &mut HashSet::new())
}
fn find_entry_inner<'a>(
&'a self,
path: &Path,
seen: &mut HashSet<PathBuf>,
) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> {
let mut path = Cow::Borrowed(path);
loop {
let (resolved_path, entry) =
self.find_entry_no_follow_inner(&path, seen)?;
match entry {
VfsEntryRef::Symlink(symlink) => {
if !seen.insert(path.to_path_buf()) {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"circular symlinks",
));
}
path = Cow::Owned(symlink.resolve_dest_from_root(&self.root_path));
}
_ => {
return Ok((resolved_path, entry));
}
}
}
}
fn find_entry_no_follow(
&self,
path: &Path,
) -> std::io::Result<(PathBuf, VfsEntryRef)> {
self.find_entry_no_follow_inner(path, &mut HashSet::new())
}
fn find_entry_no_follow_inner<'a>(
&'a self,
path: &Path,
seen: &mut HashSet<PathBuf>,
) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> {
let relative_path = match path.strip_prefix(&self.root_path) {
Ok(p) => p,
Err(_) => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
};
let mut final_path = self.root_path.clone();
let mut current_entry = VfsEntryRef::Dir(&self.dir);
for component in relative_path.components() {
let component = component.as_os_str().to_string_lossy();
let current_dir = match current_entry {
VfsEntryRef::Dir(dir) => {
final_path.push(component.as_ref());
dir
}
VfsEntryRef::Symlink(symlink) => {
let dest = symlink.resolve_dest_from_root(&self.root_path);
let (resolved_path, entry) = self.find_entry_inner(&dest, seen)?;
final_path = resolved_path; // overwrite with the new resolved path
match entry {
VfsEntryRef::Dir(dir) => {
final_path.push(component.as_ref());
dir
}
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
}
}
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
};
match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&component))
{
Ok(index) => {
current_entry = current_dir.entries[index].as_ref();
}
Err(_) => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
}
}
Ok((final_path, current_entry))
}
}
#[derive(Clone)]
struct FileBackedVfsFile {
file: VirtualFile,
pos: Arc<Mutex<u64>>,
vfs: Arc<FileBackedVfs>,
}
impl FileBackedVfsFile {
fn seek(&self, pos: SeekFrom) -> FsResult<u64> {
match pos {
SeekFrom::Start(pos) => {
*self.pos.lock() = pos;
Ok(pos)
}
SeekFrom::End(offset) => {
if offset < 0 && -offset as u64 > self.file.len {
Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "An attempt was made to move the file pointer before the beginning of the file.").into())
} else {
let mut current_pos = self.pos.lock();
*current_pos = if offset >= 0 {
self.file.len - (offset as u64)
} else {
self.file.len + (-offset as u64)
};
Ok(*current_pos)
}
}
SeekFrom::Current(offset) => {
let mut current_pos = self.pos.lock();
if offset >= 0 {
*current_pos += offset as u64;
} else if -offset as u64 > *current_pos {
return Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "An attempt was made to move the file pointer before the beginning of the file.").into());
} else {
*current_pos -= -offset as u64;
}
Ok(*current_pos)
}
}
}
fn read_to_buf(&self, buf: &mut [u8]) -> FsResult<usize> {
let pos = {
let mut pos = self.pos.lock();
let read_pos = *pos;
// advance the position due to the read
*pos = std::cmp::min(self.file.len, *pos + buf.len() as u64);
read_pos
};
self
.vfs
.read_file(&self.file, pos, buf)
.map_err(|err| err.into())
}
fn read_to_end(&self) -> FsResult<Vec<u8>> {
let pos = {
let mut pos = self.pos.lock();
let read_pos = *pos;
// todo(dsherret): should this always set it to the end of the file?
if *pos < self.file.len {
// advance the position due to the read
*pos = self.file.len;
}
read_pos
};
if pos > self.file.len {
return Ok(Vec::new());
}
let size = (self.file.len - pos) as usize;
let mut buf = vec![0; size];
self.vfs.read_file(&self.file, pos, &mut buf)?;
Ok(buf)
}
}
#[async_trait::async_trait(?Send)]
impl deno_io::fs::File for FileBackedVfsFile {
fn read_sync(self: Rc<Self>, buf: &mut [u8]) -> FsResult<usize> {
self.read_to_buf(buf)
}
async fn read_byob(
self: Rc<Self>,
mut buf: BufMutView,
) -> FsResult<(usize, BufMutView)> {
let inner = (*self).clone();
tokio::task::spawn(async move {
let nread = inner.read_to_buf(&mut buf)?;
Ok((nread, buf))
})
.await?
}
fn write_sync(self: Rc<Self>, _buf: &[u8]) -> FsResult<usize> {
Err(FsError::NotSupported)
}
async fn write(
self: Rc<Self>,
_buf: BufView,
) -> FsResult<deno_core::WriteOutcome> {
Err(FsError::NotSupported)
}
fn write_all_sync(self: Rc<Self>, _buf: &[u8]) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn write_all(self: Rc<Self>, _buf: BufView) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn read_all_sync(self: Rc<Self>) -> FsResult<Vec<u8>> {
self.read_to_end()
}
async fn read_all_async(self: Rc<Self>) -> FsResult<Vec<u8>> {
let inner = (*self).clone();
tokio::task::spawn_blocking(move || inner.read_to_end()).await?
}
fn chmod_sync(self: Rc<Self>, _pathmode: u32) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn chmod_async(self: Rc<Self>, _mode: u32) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn seek_sync(self: Rc<Self>, pos: SeekFrom) -> FsResult<u64> {
self.seek(pos)
}
async fn seek_async(self: Rc<Self>, pos: SeekFrom) -> FsResult<u64> {
self.seek(pos)
}
fn datasync_sync(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn datasync_async(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn sync_sync(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn sync_async(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn stat_sync(self: Rc<Self>) -> FsResult<FsStat> {
Err(FsError::NotSupported)
}
async fn stat_async(self: Rc<Self>) -> FsResult<FsStat> {
Err(FsError::NotSupported)
}
fn lock_sync(self: Rc<Self>, _exclusive: bool) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn lock_async(self: Rc<Self>, _exclusive: bool) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn unlock_sync(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn unlock_async(self: Rc<Self>) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn truncate_sync(self: Rc<Self>, _len: u64) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn truncate_async(self: Rc<Self>, _len: u64) -> FsResult<()> {
Err(FsError::NotSupported)
}
fn utime_sync(
self: Rc<Self>,
_atime_secs: i64,
_atime_nanos: u32,
_mtime_secs: i64,
_mtime_nanos: u32,
) -> FsResult<()> {
Err(FsError::NotSupported)
}
async fn utime_async(
self: Rc<Self>,
_atime_secs: i64,
_atime_nanos: u32,
_mtime_secs: i64,
_mtime_nanos: u32,
) -> FsResult<()> {
Err(FsError::NotSupported)
}
// lower level functionality
fn as_stdio(self: Rc<Self>) -> FsResult<std::process::Stdio> {
Err(FsError::NotSupported)
}
#[cfg(unix)]
fn backing_fd(self: Rc<Self>) -> Option<std::os::unix::prelude::RawFd> {
None
}
#[cfg(windows)]
fn backing_fd(self: Rc<Self>) -> Option<std::os::windows::io::RawHandle> {
None
}
fn try_clone_inner(self: Rc<Self>) -> FsResult<Rc<dyn deno_io::fs::File>> {
Ok(self)
}
}
#[derive(Debug)]
pub struct FileBackedVfs {
file: Mutex<File>,
fs_root: VfsRoot,
}
impl FileBackedVfs {
pub fn new(file: File, fs_root: VfsRoot) -> Self {
Self {
file: Mutex::new(file),
fs_root,
}
}
pub fn root(&self) -> &Path {
&self.fs_root.root_path
}
pub fn is_path_within(&self, path: &Path) -> bool {
path.starts_with(&self.fs_root.root_path)
}
pub fn open_file(
self: &Arc<Self>,
path: &Path,
) -> std::io::Result<Rc<dyn deno_io::fs::File>> {
let file = self.file_entry(path)?;
Ok(Rc::new(FileBackedVfsFile {
file: file.clone(),
vfs: self.clone(),
pos: Default::default(),
}))
}
pub fn read_dir(&self, path: &Path) -> std::io::Result<Vec<FsDirEntry>> {
let dir = self.dir_entry(path)?;
Ok(
dir
.entries
.iter()
.map(|entry| FsDirEntry {
name: entry.name().to_string(),
is_file: matches!(entry, VfsEntry::File(_)),
is_directory: matches!(entry, VfsEntry::Dir(_)),
is_symlink: matches!(entry, VfsEntry::Symlink(_)),
})
.collect(),
)
}
pub fn read_link(&self, path: &Path) -> std::io::Result<PathBuf> {
let (_, entry) = self.fs_root.find_entry_no_follow(path)?;
match entry {
VfsEntryRef::Symlink(symlink) => {
Ok(symlink.resolve_dest_from_root(&self.fs_root.root_path))
}
VfsEntryRef::Dir(_) | VfsEntryRef::File(_) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"not a symlink",
)),
}
}
pub fn lstat(&self, path: &Path) -> std::io::Result<FsStat> {
let (_, entry) = self.fs_root.find_entry_no_follow(path)?;
Ok(entry.as_fs_stat())
}
pub fn stat(&self, path: &Path) -> std::io::Result<FsStat> {
let (_, entry) = self.fs_root.find_entry(path)?;
Ok(entry.as_fs_stat())
}
pub fn canonicalize(&self, path: &Path) -> std::io::Result<PathBuf> {
let (path, _) = self.fs_root.find_entry(path)?;
Ok(path)
}
pub fn read_file_all(&self, file: &VirtualFile) -> std::io::Result<Vec<u8>> {
let mut buf = vec![0; file.len as usize];
self.read_file(file, 0, &mut buf)?;
Ok(buf)
}
pub fn read_file(
&self,
file: &VirtualFile,
pos: u64,
buf: &mut [u8],
) -> std::io::Result<usize> {
let mut fs_file = self.file.lock();
fs_file.seek(SeekFrom::Start(
self.fs_root.start_file_offset + file.offset + pos,
))?;
fs_file.read(buf)
}
pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> {
let (_, entry) = self.fs_root.find_entry(path)?;
match entry {
VfsEntryRef::Dir(dir) => Ok(dir),
VfsEntryRef::Symlink(_) => unreachable!(),
VfsEntryRef::File(_) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"path is a file",
)),
}
}
pub fn file_entry(&self, path: &Path) -> std::io::Result<&VirtualFile> {
let (_, entry) = self.fs_root.find_entry(path)?;
match entry {
VfsEntryRef::Dir(_) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"path is a directory",
)),
VfsEntryRef::Symlink(_) => unreachable!(),
VfsEntryRef::File(file) => Ok(file),
}
}
}
#[cfg(test)]
mod test {
use std::io::Write;
use test_util::TempDir;
use super::*;
fn read_file(vfs: &FileBackedVfs, path: &Path) -> String {
let file = vfs.file_entry(path).unwrap();
String::from_utf8(vfs.read_file_all(file).unwrap()).unwrap()
}
#[test]
fn builds_and_uses_virtual_fs() {
let temp_dir = TempDir::new();
let src_path = temp_dir.path().join("src");
let mut builder = VfsBuilder::new(src_path.clone());
builder.add_file(&src_path.join("a.txt"), "data".into());
builder.add_file(&src_path.join("b.txt"), "data".into());
assert_eq!(builder.files.len(), 1); // because duplicate data
builder.add_file(&src_path.join("c.txt"), "c".into());
builder.add_file(&src_path.join("sub_dir").join("d.txt"), "d".into());
builder.add_file(&src_path.join("e.txt"), "e".into());
builder.add_symlink(
&src_path.join("sub_dir").join("e.txt"),
&src_path.join("e.txt"),
);
// get the virtual fs
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
assert_eq!(read_file(&virtual_fs, &dest_path.join("a.txt")), "data");
assert_eq!(read_file(&virtual_fs, &dest_path.join("b.txt")), "data");
// attempt reading a symlink
assert_eq!(
read_file(&virtual_fs, &dest_path.join("sub_dir").join("e.txt")),
"e",
);
// canonicalize symlink
assert_eq!(
virtual_fs
.canonicalize(&dest_path.join("sub_dir").join("e.txt"))
.unwrap(),
dest_path.join("e.txt"),
);
// metadata
assert!(
virtual_fs
.lstat(&dest_path.join("sub_dir").join("e.txt"))
.unwrap()
.is_symlink
);
assert!(
virtual_fs
.stat(&dest_path.join("sub_dir").join("e.txt"))
.unwrap()
.is_file
);
assert!(
virtual_fs
.stat(&dest_path.join("sub_dir"))
.unwrap()
.is_directory,
);
assert!(virtual_fs.stat(&dest_path.join("e.txt")).unwrap().is_file,);
}
#[test]
fn test_include_dir_recursive() {
let temp_dir = TempDir::new();
temp_dir.create_dir_all("src/nested/sub_dir");
temp_dir.write("src/a.txt", "data");
temp_dir.write("src/b.txt", "data");
util::fs::symlink_dir(
&temp_dir.path().join("src/nested/sub_dir"),
&temp_dir.path().join("src/sub_dir_link"),
)
.unwrap();
temp_dir.write("src/nested/sub_dir/c.txt", "c");
// build and create the virtual fs
let src_path = temp_dir.path().join("src");
let mut builder = VfsBuilder::new(src_path.clone());
builder.add_dir_recursive(&src_path).unwrap();
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
assert_eq!(read_file(&virtual_fs, &dest_path.join("a.txt")), "data",);
assert_eq!(read_file(&virtual_fs, &dest_path.join("b.txt")), "data",);
assert_eq!(
read_file(
&virtual_fs,
&dest_path.join("nested").join("sub_dir").join("c.txt")
),
"c",
);
assert_eq!(
read_file(&virtual_fs, &dest_path.join("sub_dir_link").join("c.txt")),
"c",
);
assert!(
virtual_fs
.lstat(&dest_path.join("sub_dir_link"))
.unwrap()
.is_symlink
);
assert_eq!(
virtual_fs
.canonicalize(&dest_path.join("sub_dir_link").join("c.txt"))
.unwrap(),
dest_path.join("nested").join("sub_dir").join("c.txt"),
);
}
fn into_virtual_fs(
builder: VfsBuilder,
temp_dir: &TempDir,
) -> (PathBuf, FileBackedVfs) {
let virtual_fs_file = temp_dir.path().join("virtual_fs");
let (root_dir, files) = builder.into_dir_and_files();
{
let mut file = std::fs::File::create(&virtual_fs_file).unwrap();
for file_data in &files {
file.write_all(file_data).unwrap();
}
}
let file = std::fs::File::open(&virtual_fs_file).unwrap();
let dest_path = temp_dir.path().join("dest");
(
dest_path.clone(),
FileBackedVfs::new(
file,
VfsRoot {
dir: root_dir,
root_path: dest_path,
start_file_offset: 0,
},
),
)
}
#[test]
fn circular_symlink() {
let temp_dir = TempDir::new();
let src_path = temp_dir.path().join("src");
let mut builder = VfsBuilder::new(src_path.clone());
builder.add_symlink(&src_path.join("a.txt"), &src_path.join("b.txt"));
builder.add_symlink(&src_path.join("b.txt"), &src_path.join("c.txt"));
builder.add_symlink(&src_path.join("c.txt"), &src_path.join("a.txt"));
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
assert_eq!(
virtual_fs
.file_entry(&dest_path.join("a.txt"))
.err()
.unwrap()
.to_string(),
"circular symlinks",
);
assert_eq!(
virtual_fs.read_link(&dest_path.join("a.txt")).unwrap(),
dest_path.join("b.txt")
);
assert_eq!(
virtual_fs.read_link(&dest_path.join("b.txt")).unwrap(),
dest_path.join("c.txt")
);
}
#[tokio::test]
async fn test_open_file() {
let temp_dir = TempDir::new();
let temp_path = temp_dir.path();
let mut builder = VfsBuilder::new(temp_path.to_path_buf());
builder.add_file(
&temp_path.join("a.txt"),
"0123456789".to_string().into_bytes(),
);
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
let virtual_fs = Arc::new(virtual_fs);
let file = virtual_fs.open_file(&dest_path.join("a.txt")).unwrap();
file.clone().seek_sync(SeekFrom::Current(2)).unwrap();
let mut buf = vec![0; 2];
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"23");
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"45");
file.clone().seek_sync(SeekFrom::Current(-4)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"23");
file.clone().seek_sync(SeekFrom::Start(2)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"23");
file.clone().seek_sync(SeekFrom::End(2)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"89");
file.clone().seek_sync(SeekFrom::Current(-8)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
assert_eq!(buf, b"23");
assert_eq!(
file
.clone()
.seek_sync(SeekFrom::Current(-5))
.err()
.unwrap()
.into_io_error()
.to_string(),
"An attempt was made to move the file pointer before the beginning of the file."
);
// go beyond the file length, then back
file.clone().seek_sync(SeekFrom::Current(40)).unwrap();
file.clone().seek_sync(SeekFrom::Current(-38)).unwrap();
let read_buf = file.clone().read(2).await.unwrap();
assert_eq!(read_buf.to_vec(), b"67");
file.clone().seek_sync(SeekFrom::Current(-2)).unwrap();
// read to the end of the file
let all_buf = file.clone().read_all_sync().unwrap();
assert_eq!(all_buf.to_vec(), b"6789");
file.clone().seek_sync(SeekFrom::Current(-9)).unwrap();
// try try_clone_inner and read_all_async
let all_buf = file
.try_clone_inner()
.unwrap()
.read_all_async()
.await
.unwrap();
assert_eq!(all_buf.to_vec(), b"123456789");
}
}

305
cli/tests/integration/compile_tests.rs
View file

@ -5,6 +5,8 @@ use std::process::Command;
use test_util as util;
use test_util::TempDir;
use util::assert_contains;
use util::env_vars_for_npm_tests;
use util::TestContextBuilder;
#[test]
fn compile() {
@ -675,30 +677,40 @@ fn workers_basic() {
#[test]
fn workers_not_in_module_map() {
let _guard = util::http_server();
let dir = TempDir::new();
let context = TestContextBuilder::for_npm()
.use_http_server()
.use_temp_cwd()
.build();
let temp_dir = context.temp_dir();
let exe = if cfg!(windows) {
dir.path().join("not_in_module_map.exe")
temp_dir.path().join("not_in_module_map.exe")
} else {
dir.path().join("not_in_module_map")
temp_dir.path().join("not_in_module_map")
};
let output = util::deno_cmd()
.current_dir(util::root_path())
.arg("compile")
.arg("--output")
.arg(&exe)
.arg(util::testdata_path().join("./compile/workers/not_in_module_map.ts"))
.output()
.unwrap();
assert!(output.status.success());
let main_path =
util::testdata_path().join("./compile/workers/not_in_module_map.ts");
let output = context
.new_command()
.args_vec([
"compile",
"--output",
&exe.to_string_lossy(),
&main_path.to_string_lossy(),
])
.run();
output.assert_exit_code(0);
output.skip_output_check();
let output = Command::new(&exe).env("NO_COLOR", "").output().unwrap();
assert!(!output.status.success());
let stderr = String::from_utf8(output.stderr).unwrap();
assert!(stderr.starts_with(concat!(
"error: Uncaught (in worker \"\") Module not found\n",
"error: Uncaught (in promise) Error: Unhandled error in child worker.\n"
)));
let output = context
.new_command()
.command_name(exe.to_string_lossy())
.env("NO_COLOR", "")
.run();
output.assert_exit_code(1);
output.assert_matches_text(concat!(
"error: Uncaught (in worker \"\") Module not found: [WILDCARD]",
"error: Uncaught (in promise) Error: Unhandled error in child worker.\n[WILDCARD]"
));
}
#[test]
@ -790,3 +802,256 @@ fn dynamic_import_unanalyzable() {
.unwrap();
assert_eq!(String::from_utf8(output.stdout).unwrap(), expected);
}
itest!(npm_specifiers_errors_no_unstable {
args: "compile -A --quiet npm/cached_only/main.ts",
output_str: Some(
concat!(
"error: Using npm specifiers with deno compile requires the --unstable flag.",
"\n\n",
"Caused by:\n",
" npm specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: npm:chalk@5.0.1\n"
)
),
exit_code: 1,
envs: env_vars_for_npm_tests(),
http_server: true,
});
#[test]
fn compile_npm_specifiers() {
let context = TestContextBuilder::for_npm()
.use_sync_npm_download()
.use_temp_cwd()
.build();
let temp_dir = context.temp_dir();
temp_dir.write(
"main.ts",
concat!(
"import path from 'node:path';\n",
"import { getValue, setValue } from 'npm:@denotest/esm-basic';\n",
"import getValueDefault from 'npm:@denotest/esm-import-cjs-default';\n",
"setValue(2);\n",
"console.log(path.join('testing', 'this'));",
"console.log(getValue());",
"console.log(getValueDefault());",
),
);
let binary_path = if cfg!(windows) {
temp_dir.path().join("binary.exe")
} else {
temp_dir.path().join("binary")
};
// try with and without --node-modules-dir
let compile_commands = &[
"compile --unstable --output binary main.ts",
"compile --unstable --node-modules-dir --output binary main.ts",
];
for compile_command in compile_commands {
let output = context.new_command().args(compile_command).run();
output.assert_exit_code(0);
output.skip_output_check();
let output = context
.new_command()
.command_name(binary_path.to_string_lossy())
.run();
output.assert_matches_text(
r#"Node esm importing node cjs
===========================
{
default: [Function (anonymous)],
named: [Function (anonymous)],
MyClass: [class MyClass]
}
{ default: [Function (anonymous)], named: [Function (anonymous)] }
[Module: null prototype] {
MyClass: [class MyClass],
__esModule: true,
default: {
default: [Function (anonymous)],
named: [Function (anonymous)],
MyClass: [class MyClass]
},
named: [Function (anonymous)]
}
[Module: null prototype] {
__esModule: true,
default: { default: [Function (anonymous)], named: [Function (anonymous)] },
named: [Function (anonymous)]
}
===========================
static method
testing[WILDCARD]this
2
5
"#,
);
}
// try with a package.json
temp_dir.remove_dir_all("node_modules");
temp_dir.write(
"main.ts",
concat!(
"import { getValue, setValue } from '@denotest/esm-basic';\n",
"setValue(2);\n",
"console.log(getValue());",
),
);
temp_dir.write(
"package.json",
r#"{ "dependencies": { "@denotest/esm-basic": "1" } }"#,
);
let output = context
.new_command()
.args("compile --unstable --output binary main.ts")
.run();
output.assert_exit_code(0);
output.skip_output_check();
let output = context
.new_command()
.command_name(binary_path.to_string_lossy())
.run();
output.assert_matches_text("2\n");
}
#[test]
fn compile_npm_file_system() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "compile/npm_fs/main.ts",
output_file: "compile/npm_fs/main.out",
node_modules_dir: true,
input_name: Some("binary"),
expected_name: "binary",
run_args: vec![],
});
}
#[test]
fn compile_npm_bin_esm() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:@denotest/bin/cli-esm",
run_args: vec!["this", "is", "a", "test"],
output_file: "npm/deno_run_esm.out",
node_modules_dir: false,
input_name: None,
expected_name: "cli-esm",
});
}
#[test]
fn compile_npm_bin_cjs() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:@denotest/bin/cli-cjs",
run_args: vec!["this", "is", "a", "test"],
output_file: "npm/deno_run_cjs.out",
node_modules_dir: false,
input_name: None,
expected_name: "cli-cjs",
});
}
#[test]
fn compile_npm_cowsay() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:cowsay@1.5.0",
run_args: vec!["Hello"],
output_file: "npm/deno_run_cowsay.out",
node_modules_dir: false,
input_name: None,
expected_name: "cowsay",
});
}
#[test]
fn compile_npm_cowsay_explicit() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:cowsay@1.5.0/cowsay",
run_args: vec!["Hello"],
output_file: "npm/deno_run_cowsay.out",
node_modules_dir: false,
input_name: None,
expected_name: "cowsay",
});
}
#[test]
fn compile_npm_cowthink() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "npm:cowsay@1.5.0/cowthink",
run_args: vec!["Hello"],
output_file: "npm/deno_run_cowthink.out",
node_modules_dir: false,
input_name: None,
expected_name: "cowthink",
});
}
struct RunNpmBinCompileOptions<'a> {
input_specifier: &'a str,
output_file: &'a str,
node_modules_dir: bool,
input_name: Option<&'a str>,
expected_name: &'a str,
run_args: Vec<&'a str>,
}
fn run_npm_bin_compile_test(opts: RunNpmBinCompileOptions) {
let context = TestContextBuilder::for_npm()
.use_sync_npm_download()
.use_temp_cwd()
.build();
let temp_dir = context.temp_dir();
let testdata_path = context.testdata_path();
let main_specifier = if opts.input_specifier.starts_with("npm:") {
opts.input_specifier.to_string()
} else {
testdata_path
.join(opts.input_specifier)
.to_string_lossy()
.to_string()
};
let mut args = vec![
"compile".to_string(),
"-A".to_string(),
"--unstable".to_string(),
];
if opts.node_modules_dir {
args.push("--node-modules-dir".to_string());
}
if let Some(bin_name) = opts.input_name {
args.push("--output".to_string());
args.push(bin_name.to_string());
}
args.push(main_specifier);
// compile
let output = context.new_command().args_vec(args).run();
output.assert_exit_code(0);
output.skip_output_check();
// run
let binary_path = if cfg!(windows) {
temp_dir.path().join(format!("{}.exe", opts.expected_name))
} else {
temp_dir.path().join(opts.expected_name)
};
let output = context
.new_command()
.command_name(binary_path.to_string_lossy())
.args_vec(opts.run_args)
.run();
output.assert_matches_file(opts.output_file);
}

10
cli/tests/integration/npm_tests.rs
View file

@ -855,17 +855,9 @@ fn ensure_registry_files_local() {
}
}
itest!(compile_errors {
args: "compile -A --quiet npm/cached_only/main.ts",
output_str: Some("error: npm specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: npm:chalk@5.0.1\n"),
exit_code: 1,
envs: env_vars_for_npm_tests(),
http_server: true,
});
itest!(bundle_errors {
args: "bundle --quiet npm/esm/main.js",
output_str: Some("error: npm specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: npm:chalk@5.0.1\n"),
output_str: Some("error: npm specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: npm:chalk@5.0.1\n"),
exit_code: 1,
envs: env_vars_for_npm_tests(),
http_server: true,

1
cli/tests/testdata/compile/npm_fs/main.out vendored Normal file
View file

@ -0,0 +1 @@
success

259
cli/tests/testdata/compile/npm_fs/main.ts vendored Normal file
View file

@ -0,0 +1,259 @@
import { url } from "npm:@denotest/esm-basic";
import { fileURLToPath } from "node:url";
import path from "node:path";
import assert from "node:assert/strict";
// will be at node_modules\.deno\@denotest+esm-basic@1.0.0\node_modules\@denotest\esm-basic
const dirPath = path.dirname(fileURLToPath(url));
const nodeModulesPath = path.join(dirPath, "../../../../../");
const packageJsonText = `{
"name": "@denotest/esm-basic",
"version": "1.0.0",
"type": "module",
"main": "main.mjs",
"types": "main.d.mts"
}
`;
const vfsPackageJsonPath = path.join(dirPath, "package.json");
// reading a file in vfs
{
const text = Deno.readTextFileSync(vfsPackageJsonPath);
assert.equal(text, packageJsonText);
}
// reading a file async in vfs
{
const text = await Deno.readTextFile(vfsPackageJsonPath);
assert.equal(text, packageJsonText);
}
// copy file from vfs to real fs
{
Deno.copyFileSync(vfsPackageJsonPath, "package.json");
assert.equal(Deno.readTextFileSync("package.json"), packageJsonText);
}
// copy to vfs
assert.throws(
() => Deno.copyFileSync("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
Deno.removeSync("package.json");
// copy file async from vfs to real fs
{
await Deno.copyFile(vfsPackageJsonPath, "package.json");
assert.equal(Deno.readTextFileSync("package.json"), packageJsonText);
}
// copy to vfs async
await assert.rejects(
() => Deno.copyFile("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
Deno.removeSync("package.json");
// open
{
const file = Deno.openSync(vfsPackageJsonPath);
const bytes = new Uint8Array(10);
file.seekSync(2, Deno.SeekMode.Start);
assert.equal(file.readSync(bytes), 10);
const text = new TextDecoder().decode(bytes);
assert.equal(text, packageJsonText.slice(2, 12));
}
{
const file = await Deno.open(vfsPackageJsonPath);
const bytes = new Uint8Array(10);
await file.seek(2, Deno.SeekMode.Start);
assert.equal(await file.read(bytes), 10);
const text = new TextDecoder().decode(bytes);
assert.equal(text, packageJsonText.slice(2, 12));
}
// chdir
assert.throws(() => Deno.chdir(dirPath), Deno.errors.NotSupported);
// mkdir
assert.throws(
() => Deno.mkdirSync(path.join(dirPath, "subDir")),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.mkdir(path.join(dirPath, "subDir")),
Deno.errors.NotSupported,
);
// chmod
assert.throws(
() => Deno.chmodSync(vfsPackageJsonPath, 0o777),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.chmod(vfsPackageJsonPath, 0o777),
Deno.errors.NotSupported,
);
// chown
assert.throws(
() => Deno.chownSync(vfsPackageJsonPath, 1000, 1000),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.chown(vfsPackageJsonPath, 1000, 1000),
Deno.errors.NotSupported,
);
// remove
assert.throws(
() => Deno.removeSync(vfsPackageJsonPath),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.remove(vfsPackageJsonPath),
Deno.errors.NotSupported,
);
// stat
{
const result = Deno.statSync(vfsPackageJsonPath);
assert(result.isFile);
}
{
const result = await Deno.stat(vfsPackageJsonPath);
assert(result.isFile);
}
// lstat
{
const result = Deno.lstatSync(
path.join(nodeModulesPath, "@denotest", "esm-basic"),
);
assert(result.isSymlink);
}
{
const result = await Deno.lstat(
path.join(nodeModulesPath, "@denotest", "esm-basic"),
);
assert(result.isSymlink);
}
// realpath
{
const result = Deno.realPathSync(
path.join(nodeModulesPath, "@denotest", "esm-basic", "package.json"),
);
assert.equal(result, vfsPackageJsonPath);
}
{
const result = await Deno.realPath(
path.join(nodeModulesPath, "@denotest", "esm-basic", "package.json"),
);
assert.equal(result, vfsPackageJsonPath);
}
// read dir
const readDirNames = ["main.d.mts", "main.mjs", "package.json"];
{
const names = Array.from(Deno.readDirSync(dirPath))
.map((e) => e.name);
assert.deepEqual(readDirNames, names);
}
{
const names = [];
for await (const entry of Deno.readDir(dirPath)) {
names.push(entry.name);
}
assert.deepEqual(readDirNames, names);
}
// rename
assert.throws(
() => Deno.renameSync("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
assert.throws(
() => Deno.renameSync(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.rename("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.rename(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
// link
assert.throws(
() => Deno.linkSync("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
assert.throws(
() => Deno.linkSync(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.link("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.link(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
// symlink
assert.throws(
() => Deno.symlinkSync("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
assert.throws(
() => Deno.symlinkSync(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.symlink("package.json", vfsPackageJsonPath),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.symlink(vfsPackageJsonPath, "package.json"),
Deno.errors.NotSupported,
);
// read link
{
const result = Deno.readLinkSync(
path.join(nodeModulesPath, "@denotest", "esm-basic"),
);
assert.equal(result, dirPath);
}
{
const result = await Deno.readLink(
path.join(nodeModulesPath, "@denotest", "esm-basic"),
);
assert.equal(result, dirPath);
}
// truncate
assert.throws(
() => Deno.truncateSync(vfsPackageJsonPath, 0),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.truncate(vfsPackageJsonPath, 0),
Deno.errors.NotSupported,
);
// utime
assert.throws(
() => Deno.utimeSync(vfsPackageJsonPath, 0, 0),
Deno.errors.NotSupported,
);
await assert.rejects(
() => Deno.utime(vfsPackageJsonPath, 0, 0),
Deno.errors.NotSupported,
);
console.log("success");

1
cli/tests/testdata/npm/registry/@denotest/esm-basic/1.0.0/main.d.mts vendored
View file

@ -1,2 +1,3 @@
export declare function setValue(val: number): void;
export declare function getValue(): number;
export declare const url: string;

2
cli/tests/testdata/npm/registry/@denotest/esm-basic/1.0.0/main.mjs vendored
View file

@ -7,3 +7,5 @@ export function setValue(newValue) {
export function getValue() {
return value;
}
export const url = import.meta.url;

2
cli/tests/testdata/package_json/basic/main.info.out vendored
View file

@ -5,4 +5,4 @@ size: [WILDCARD]
file:///[WILDCARD]/main.ts (63B)
└─┬ file:///[WILDCARD]/lib.ts (166B)
└── npm:@denotest/esm-basic@1.0.0 (345B)
└── npm:@denotest/esm-basic@1.0.0 (416B)

15
cli/tools/standalone.rs → cli/tools/compile.rs
View file

@ -5,7 +5,6 @@ use crate::args::Flags;
use crate::factory::CliFactory;
use crate::graph_util::error_for_any_npm_specifier;
use crate::standalone::is_standalone_binary;
use crate::standalone::DenoCompileBinaryWriter;
use crate::util::path::path_has_trailing_slash;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
@ -25,14 +24,9 @@ pub async fn compile(
) -> Result<(), AnyError> {
let factory = CliFactory::from_flags(flags).await?;
let cli_options = factory.cli_options();
let file_fetcher = factory.file_fetcher()?;
let http_client = factory.http_client();
let deno_dir = factory.deno_dir()?;
let module_graph_builder = factory.module_graph_builder().await?;
let parsed_source_cache = factory.parsed_source_cache()?;
let binary_writer =
DenoCompileBinaryWriter::new(file_fetcher, http_client, deno_dir);
let binary_writer = factory.create_compile_binary_writer().await?;
let module_specifier = cli_options.resolve_main_module()?;
let module_roots = {
let mut vec = Vec::with_capacity(compile_flags.include.len() + 1);
@ -56,8 +50,11 @@ pub async fn compile(
)
.unwrap();
// at the moment, we don't support npm specifiers in deno_compile, so show an error
error_for_any_npm_specifier(&graph)?;
if !cli_options.unstable() {
error_for_any_npm_specifier(&graph).context(
"Using npm specifiers with deno compile requires the --unstable flag.",
)?;
}
let parser = parsed_source_cache.as_capturing_parser();
let eszip = eszip::EszipV2::from_graph(graph, &parser, Default::default())?;

2
cli/tools/mod.rs
View file

@ -3,6 +3,7 @@
pub mod bench;
pub mod bundle;
pub mod check;
pub mod compile;
pub mod coverage;
pub mod doc;
pub mod fmt;
@ -12,7 +13,6 @@ pub mod installer;
pub mod lint;
pub mod repl;
pub mod run;
pub mod standalone;
pub mod task;
pub mod test;
pub mod upgrade;

3
cli/tools/task.rs
View file

@ -64,12 +64,13 @@ pub async fn execute_script(
.await;
Ok(exit_code)
} else if let Some(script) = package_json_scripts.get(task_name) {
let package_json_deps_provider = factory.package_json_deps_provider();
let package_json_deps_installer =
factory.package_json_deps_installer().await?;
let npm_resolver = factory.npm_resolver().await?;
let node_resolver = factory.node_resolver().await?;
if let Some(package_deps) = package_json_deps_installer.package_deps() {
if let Some(package_deps) = package_json_deps_provider.deps() {
for (key, value) in package_deps {
if let Err(err) = value {
log::info!(

9
cli/tools/vendor/test.rs vendored
View file

@ -22,7 +22,6 @@ use import_map::ImportMap;
use crate::cache::ParsedSourceCache;
use crate::npm::CliNpmRegistryApi;
use crate::npm::NpmResolution;
use crate::npm::PackageJsonDepsInstaller;
use crate::resolver::CliGraphResolver;
use super::build::VendorEnvironment;
@ -270,18 +269,14 @@ async fn build_test_graph(
None,
None,
));
let deps_installer = Arc::new(PackageJsonDepsInstaller::new(
npm_registry_api.clone(),
npm_resolution.clone(),
None,
));
CliGraphResolver::new(
None,
Some(Arc::new(original_import_map)),
false,
npm_registry_api,
npm_resolution,
deps_installer,
Default::default(),
Default::default(),
)
});
let mut graph = ModuleGraph::default();

9
cli/util/fs.rs
View file

@ -92,12 +92,19 @@ pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
/// subsequently be created along this path by some other code.
pub fn canonicalize_path_maybe_not_exists(
path: &Path,
) -> Result<PathBuf, Error> {
canonicalize_path_maybe_not_exists_with_fs(path, canonicalize_path)
}
pub fn canonicalize_path_maybe_not_exists_with_fs(
path: &Path,
canonicalize: impl Fn(&Path) -> Result<PathBuf, Error>,
) -> Result<PathBuf, Error> {
let path = path.to_path_buf().clean();
let mut path = path.as_path();
let mut names_stack = Vec::new();
loop {
match canonicalize_path(path) {
match canonicalize(path) {
Ok(mut canonicalized_path) => {
for name in names_stack.into_iter().rev() {
canonicalized_path = canonicalized_path.join(name);

2
ext/fs/interface.rs
View file

@ -100,7 +100,7 @@ pub trait FileSystem: std::fmt::Debug + MaybeSend + MaybeSync {
async fn mkdir_async(
&self,
path: PathBuf,
recusive: bool,
recursive: bool,
mode: u32,
) -> FsResult<()>;

9
ext/io/fs.rs
View file

@ -15,6 +15,7 @@ use deno_core::OpState;
use deno_core::ResourceId;
use tokio::task::JoinError;
#[derive(Debug)]
pub enum FsError {
Io(io::Error),
FileBusy,
@ -29,6 +30,14 @@ impl FsError {
Self::NotSupported => io::ErrorKind::Other,
}
}
pub fn into_io_error(self) -> io::Error {
match self {
FsError::Io(err) => err,
FsError::FileBusy => io::Error::new(self.kind(), "file busy"),
FsError::NotSupported => io::Error::new(self.kind(), "not supported"),
}
}
}
impl From<io::Error> for FsError {

1
runtime/build.rs
View file

@ -358,6 +358,7 @@ fn main() {
if env::var_os("DOCS_RS").is_some() {
let snapshot_slice = &[];
#[allow(clippy::needless_borrow)]
#[allow(clippy::disallowed_methods)]
std::fs::write(&runtime_snapshot_path, snapshot_slice).unwrap();
}

45
runtime/clippy.toml Normal file
View file

@ -0,0 +1,45 @@
disallowed-methods = [
{ path = "std::env::current_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::path::Path::canonicalize", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::is_dir", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::is_file", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::is_symlink", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::metadata", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::read_dir", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::read_link", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::symlink_metadata", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::Path::try_exists", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::exists", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::canonicalize", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::is_dir", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::is_file", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::is_symlink", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::metadata", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::read_dir", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::read_link", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::symlink_metadata", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::path::PathBuf::try_exists", reason = "File system operations should be done using NodeFs trait" },
{ path = "std::env::set_current_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::env::temp_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::canonicalize", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::copy", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::create_dir_all", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::create_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::DirBuilder::new", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::hard_link", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::metadata", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::OpenOptions::new", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::read_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::read_link", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::read_to_string", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::read", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::remove_dir_all", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::remove_dir", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::remove_file", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::rename", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::set_permissions", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::symlink_metadata", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::fs::write", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::path::Path::canonicalize", reason = "File system operations should be done using FileSystem trait" },
{ path = "std::path::Path::exists", reason = "File system operations should be done using FileSystem trait" },
]

6
runtime/examples/hello_runtime.rs
View file

@ -2,6 +2,7 @@
use deno_core::error::AnyError;
use deno_core::FsModuleLoader;
use deno_core::ModuleSpecifier;
use deno_runtime::permissions::PermissionsContainer;
use deno_runtime::worker::MainWorker;
use deno_runtime::worker::WorkerOptions;
@ -14,10 +15,7 @@ deno_core::extension!(hello_runtime, esm = ["hello_runtime_bootstrap.js"]);
async fn main() -> Result<(), AnyError> {
let js_path =
Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/hello_runtime.js");
let main_module = deno_core::resolve_path(
&js_path.to_string_lossy(),
&std::env::current_dir()?,
)?;
let main_module = ModuleSpecifier::from_file_path(js_path).unwrap();
let mut worker = MainWorker::bootstrap_from_options(
main_module.clone(),
PermissionsContainer::allow_all(),

25
runtime/fs_util.rs
View file

@ -3,23 +3,17 @@
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
pub use deno_core::normalize_path;
use std::env::current_dir;
use std::io::Error;
use std::path::Path;
use std::path::PathBuf;
/// Similar to `std::fs::canonicalize()` but strips UNC prefixes on Windows.
pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
Ok(deno_core::strip_unc_prefix(path.canonicalize()?))
}
#[inline]
pub fn resolve_from_cwd(path: &Path) -> Result<PathBuf, AnyError> {
if path.is_absolute() {
Ok(normalize_path(path))
} else {
let cwd =
current_dir().context("Failed to get current working directory")?;
#[allow(clippy::disallowed_methods)]
let cwd = std::env::current_dir()
.context("Failed to get current working directory")?;
Ok(normalize_path(cwd.join(path)))
}
}
@ -28,21 +22,26 @@ pub fn resolve_from_cwd(path: &Path) -> Result<PathBuf, AnyError> {
mod tests {
use super::*;
fn current_dir() -> PathBuf {
#[allow(clippy::disallowed_methods)]
std::env::current_dir().unwrap()
}
#[test]
fn resolve_from_cwd_child() {
let cwd = current_dir().unwrap();
let cwd = current_dir();
assert_eq!(resolve_from_cwd(Path::new("a")).unwrap(), cwd.join("a"));
}
#[test]
fn resolve_from_cwd_dot() {
let cwd = current_dir().unwrap();
let cwd = current_dir();
assert_eq!(resolve_from_cwd(Path::new(".")).unwrap(), cwd);
}
#[test]
fn resolve_from_cwd_parent() {
let cwd = current_dir().unwrap();
let cwd = current_dir();
assert_eq!(resolve_from_cwd(Path::new("a/..")).unwrap(), cwd);
}
@ -66,7 +65,7 @@ mod tests {
#[test]
fn resolve_from_cwd_absolute() {
let expected = Path::new("a");
let cwd = current_dir().unwrap();
let cwd = current_dir();
let absolute_expected = cwd.join(expected);
assert_eq!(resolve_from_cwd(expected).unwrap(), absolute_expected);
}

1
runtime/ops/os/mod.rs
View file

@ -339,6 +339,7 @@ fn rss() -> usize {
(out, idx)
}
#[allow(clippy::disallowed_methods)]
let statm_content = if let Ok(c) = std::fs::read_to_string("/proc/self/statm")
{
c

1
runtime/ops/os/sys_info.rs
View file

@ -48,6 +48,7 @@ pub fn loadavg() -> LoadAvg {
pub fn os_release() -> String {
#[cfg(target_os = "linux")]
{
#[allow(clippy::disallowed_methods)]
match std::fs::read_to_string("/proc/sys/kernel/osrelease") {
Ok(mut s) => {
s.pop(); // pop '\n'

1
test_util/src/builders.rs
View file

@ -341,6 +341,7 @@ impl TestCommandBuilder {
))
}
#[track_caller]
pub fn run(&self) -> TestCommandOutput {
fn read_pipe_to_string(mut pipe: os_pipe::PipeReader) -> String {
let mut output = String::new();

4
test_util/src/temp_dir.rs
View file

@ -58,6 +58,10 @@ impl TempDir {
fs::create_dir_all(self.path().join(path)).unwrap();
}
pub fn remove_dir_all(&self, path: impl AsRef<Path>) {
fs::remove_dir_all(self.path().join(path)).unwrap();
}
pub fn read_to_string(&self, path: impl AsRef<Path>) -> String {
let file_path = self.path().join(path);
fs::read_to_string(&file_path)