From b67df0f07b37d66deee41550d8f733078e3acd99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 7 Mar 2023 05:13:44 -0400 Subject: [PATCH] refactor: Add "deno_fs" extension crate (#18040) This commit factors out APIs related to file system from "runtime/" to a separate "deno_fs" extension crate. --- Cargo.lock | 18 ++++++ Cargo.toml | 2 + cli/build.rs | 1 + {runtime/js => ext/fs}/30_fs.js | 0 ext/fs/Cargo.toml | 31 +++++++++ ext/fs/README.md | 3 + runtime/ops/fs.rs => ext/fs/lib.rs | 87 +++++++++++++++++++++++--- ext/node/lib.rs | 2 +- ext/node/polyfills/_process/process.ts | 2 +- runtime/Cargo.toml | 2 + runtime/build.rs | 38 ++++++++++- runtime/js/40_process.js | 2 +- runtime/js/90_deno_ns.js | 2 +- runtime/lib.rs | 1 + runtime/ops/mod.rs | 1 - runtime/permissions/mod.rs | 2 +- runtime/web_worker.rs | 2 +- runtime/worker.rs | 2 +- 18 files changed, 179 insertions(+), 19 deletions(-) rename {runtime/js => ext/fs}/30_fs.js (100%) create mode 100644 ext/fs/Cargo.toml create mode 100644 ext/fs/README.md rename runtime/ops/fs.rs => ext/fs/lib.rs (95%) diff --git a/Cargo.lock b/Cargo.lock index 8b9dd991fa..adf3706b70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1115,6 +1115,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "deno_fs" +version = "0.1.0" +dependencies = [ + "deno_core", + "deno_crypto", + "deno_io", + "filetime", + "fs3", + "libc", + "log", + "nix", + "serde", + "tokio", + "winapi", +] + [[package]] name = "deno_graph" version = "0.44.3" @@ -1276,6 +1293,7 @@ dependencies = [ "deno_fetch", "deno_ffi", "deno_flash", + "deno_fs", "deno_http", "deno_io", "deno_napi", diff --git a/Cargo.toml b/Cargo.toml index fdcef9848a..1fbdd5c657 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "ext/fetch", "ext/flash", "ext/ffi", + "ext/fs", "ext/http", "ext/io", "ext/net", @@ -62,6 +63,7 @@ deno_crypto = { version = "0.105.0", path = "./ext/crypto" } deno_fetch = { version = "0.115.0", path = "./ext/fetch" } deno_ffi = { version = "0.78.0", path = "./ext/ffi" } deno_flash = { version = "0.27.0", path = "./ext/flash" } +deno_fs = { version = "0.1.0", path = "./ext/fs" } deno_http = { version = "0.86.0", path = "./ext/http" } deno_io = { version = "0.1.0", path = "./ext/io" } deno_net = { version = "0.83.0", path = "./ext/net" } diff --git a/cli/build.rs b/cli/build.rs index 3baafb522c..9d133d99a4 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -343,6 +343,7 @@ fn create_cli_snapshot(snapshot_path: PathBuf) { false, // No --unstable. ), deno_io::init(Default::default()), + deno_fs::init::(false), deno_node::init::(None), // No --unstable. deno_node::init_polyfill(), deno_ffi::init::(false), diff --git a/runtime/js/30_fs.js b/ext/fs/30_fs.js similarity index 100% rename from runtime/js/30_fs.js rename to ext/fs/30_fs.js diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml new file mode 100644 index 0000000000..279788efa0 --- /dev/null +++ b/ext/fs/Cargo.toml @@ -0,0 +1,31 @@ +# Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +[package] +name = "deno_fs" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "Ops for interacting with the file system" + +[lib] +path = "lib.rs" + +[dependencies] +deno_core.workspace = true +deno_crypto.workspace = true +deno_io.workspace = true +filetime = "0.2.16" +fs3 = "0.5.0" +libc.workspace = true +log.workspace = true +serde.workspace = true +tokio.workspace = true + +[target.'cfg(unix)'.dependencies] +nix.workspace = true + +[target.'cfg(windows)'.dependencies] +winapi = { workspace = true, features = ["winbase"] } diff --git a/ext/fs/README.md b/ext/fs/README.md new file mode 100644 index 0000000000..1fe7c1843d --- /dev/null +++ b/ext/fs/README.md @@ -0,0 +1,3 @@ +# deno_fs + +This crate provides ops for interacting with the file system. diff --git a/runtime/ops/fs.rs b/ext/fs/lib.rs similarity index 95% rename from runtime/ops/fs.rs rename to ext/fs/lib.rs index 546d1d68a7..7551408a94 100644 --- a/runtime/ops/fs.rs +++ b/ext/fs/lib.rs @@ -1,10 +1,10 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. // Some deserializer fields are only used on Unix and Windows build fails without it -use super::utils::into_string; -use crate::fs_util::canonicalize_path; + use deno_core::error::custom_error; use deno_core::error::type_error; use deno_core::error::AnyError; +use deno_core::include_js_files; use deno_core::op; use deno_core::CancelFuture; use deno_core::CancelHandle; @@ -35,6 +35,70 @@ use std::rc::Rc; use std::time::SystemTime; use std::time::UNIX_EPOCH; +/// Similar to `std::fs::canonicalize()` but strips UNC prefixes on Windows. +fn canonicalize_path(path: &Path) -> Result { + let mut canonicalized_path = path.canonicalize()?; + if cfg!(windows) { + canonicalized_path = PathBuf::from( + canonicalized_path + .display() + .to_string() + .trim_start_matches("\\\\?\\"), + ); + } + Ok(canonicalized_path) +} + +/// A utility function to map OsStrings to Strings +fn into_string(s: std::ffi::OsString) -> Result { + s.into_string().map_err(|s| { + let message = format!("File name or path {s:?} is not valid UTF-8"); + custom_error("InvalidData", message) + }) +} + +#[cfg(unix)] +pub fn get_nix_error_class(error: &nix::Error) -> &'static str { + match error { + nix::Error::ECHILD => "NotFound", + nix::Error::EINVAL => "TypeError", + nix::Error::ENOENT => "NotFound", + nix::Error::ENOTTY => "BadResource", + nix::Error::EPERM => "PermissionDenied", + nix::Error::ESRCH => "NotFound", + nix::Error::UnknownErrno => "Error", + &nix::Error::ENOTSUP => unreachable!(), + _ => "Error", + } +} + +struct UnstableChecker { + pub unstable: bool, +} + +impl UnstableChecker { + // NOTE(bartlomieju): keep in sync with `cli/program_state.rs` + pub fn check_unstable(&self, api_name: &str) { + if !self.unstable { + eprintln!( + "Unstable API '{api_name}'. The --unstable flag must be provided." + ); + std::process::exit(70); + } + } +} + +/// Helper for checking unstable features. Used for sync ops. +fn check_unstable(state: &OpState, api_name: &str) { + state.borrow::().check_unstable(api_name) +} + +/// Helper for checking unstable features. Used for async ops. +fn check_unstable2(state: &Rc>, api_name: &str) { + let state = state.borrow(); + state.borrow::().check_unstable(api_name) +} + pub trait FsPermissions { fn check_read(&mut self, p: &Path, api_name: &str) -> Result<(), AnyError>; fn check_read_all(&mut self, api_name: &str) -> Result<(), AnyError>; @@ -53,8 +117,13 @@ use deno_core::error::generic_error; #[cfg(not(unix))] use deno_core::error::not_supported; -pub fn init() -> Extension { +pub fn init(unstable: bool) -> Extension { Extension::builder("deno_fs") + .esm(include_js_files!("30_fs.js",)) + .state(move |state| { + state.put(UnstableChecker { unstable }); + Ok(()) + }) .ops(vec![ op_open_sync::decl::

(), op_open_async::decl::

(), @@ -474,7 +543,7 @@ fn op_flock_sync( exclusive: bool, ) -> Result<(), AnyError> { use fs3::FileExt; - super::check_unstable(state, "Deno.flockSync"); + check_unstable(state, "Deno.flockSync"); StdFileResource::with_file(state, rid, |std_file| { if exclusive { @@ -493,7 +562,7 @@ async fn op_flock_async( exclusive: bool, ) -> Result<(), AnyError> { use fs3::FileExt; - super::check_unstable2(&state, "Deno.flock"); + check_unstable2(&state, "Deno.flock"); StdFileResource::with_file_blocking_task(state, rid, move |std_file| { if exclusive { @@ -512,7 +581,7 @@ fn op_funlock_sync( rid: ResourceId, ) -> Result<(), AnyError> { use fs3::FileExt; - super::check_unstable(state, "Deno.funlockSync"); + check_unstable(state, "Deno.funlockSync"); StdFileResource::with_file(state, rid, |std_file| { std_file.unlock()?; @@ -526,7 +595,7 @@ async fn op_funlock_async( rid: ResourceId, ) -> Result<(), AnyError> { use fs3::FileExt; - super::check_unstable2(&state, "Deno.funlock"); + check_unstable2(&state, "Deno.funlock"); StdFileResource::with_file_blocking_task(state, rid, move |std_file| { std_file.unlock()?; @@ -537,7 +606,7 @@ async fn op_funlock_async( #[op] fn op_umask(state: &mut OpState, mask: Option) -> Result { - super::check_unstable(state, "Deno.umask"); + check_unstable(state, "Deno.umask"); // TODO implement umask for Windows // see https://github.com/nodejs/node/blob/master/src/node_process_methods.cc // and https://docs.microsoft.com/fr-fr/cpp/c-runtime-library/reference/umask?view=vs-2019 @@ -727,7 +796,6 @@ where .check_write(&path, "Deno.chownSync()")?; #[cfg(unix)] { - use crate::errors::get_nix_error_class; use nix::unistd::chown; use nix::unistd::Gid; use nix::unistd::Uid; @@ -768,7 +836,6 @@ where tokio::task::spawn_blocking(move || { #[cfg(unix)] { - use crate::errors::get_nix_error_class; use nix::unistd::chown; use nix::unistd::Gid; use nix::unistd::Uid; diff --git a/ext/node/lib.rs b/ext/node/lib.rs index 468280075d..e1134bd035 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -323,7 +323,7 @@ pub fn init_polyfill() -> Extension { Extension::builder(env!("CARGO_PKG_NAME")) .esm(esm_files) .esm_entry_point("internal:deno_node/module_all.ts") - .dependencies(vec!["deno_io"]) + .dependencies(vec!["deno_io", "deno_fs"]) .ops(vec![ crypto::op_node_create_hash::decl(), crypto::op_node_hash_update::decl(), diff --git a/ext/node/polyfills/_process/process.ts b/ext/node/polyfills/_process/process.ts index bee65f9056..7ed40492dc 100644 --- a/ext/node/polyfills/_process/process.ts +++ b/ext/node/polyfills/_process/process.ts @@ -7,7 +7,7 @@ const core = globalThis.Deno.core; import { nextTick as _nextTick } from "internal:deno_node/_next_tick.ts"; import { _exiting } from "internal:deno_node/_process/exiting.ts"; -import * as fs from "internal:runtime/30_fs.js"; +import * as fs from "internal:deno_fs/30_fs.js"; /** Returns the operating system CPU architecture for which the Deno binary was compiled */ export function arch(): string { diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 5a4028349a..1121562030 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -42,6 +42,7 @@ deno_crypto.workspace = true deno_fetch.workspace = true deno_ffi.workspace = true deno_flash.workspace = true +deno_fs.workspace = true deno_http.workspace = true deno_io.workspace = true deno_net.workspace = true @@ -70,6 +71,7 @@ deno_crypto.workspace = true deno_fetch.workspace = true deno_ffi.workspace = true deno_flash.workspace = true +deno_fs.workspace = true deno_http.workspace = true deno_io.workspace = true deno_napi.workspace = true diff --git a/runtime/build.rs b/runtime/build.rs index 76c0534bf1..166421586a 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -165,6 +165,41 @@ mod startup_snapshot { } } + impl deno_fs::FsPermissions for Permissions { + fn check_read( + &mut self, + _path: &Path, + _api_name: &str, + ) -> Result<(), AnyError> { + unreachable!("snapshotting!") + } + + fn check_read_blind( + &mut self, + _path: &Path, + _display: &str, + _api_name: &str, + ) -> Result<(), AnyError> { + unreachable!("snapshotting!") + } + + fn check_write( + &mut self, + _path: &Path, + _api_name: &str, + ) -> Result<(), AnyError> { + unreachable!("snapshotting!") + } + + fn check_read_all(&mut self, _api_name: &str) -> Result<(), AnyError> { + unreachable!("snapshotting!") + } + + fn check_write_all(&mut self, _api_name: &str) -> Result<(), AnyError> { + unreachable!("snapshotting!") + } + } + fn create_runtime_snapshot( snapshot_path: PathBuf, maybe_additional_extension: Option, @@ -191,6 +226,7 @@ mod startup_snapshot { "deno_http", "deno_flash", "deno_io", + "deno_fs", ]) .esm(include_js_files!( dir "js", @@ -200,7 +236,6 @@ mod startup_snapshot { "10_permissions.js", "11_workers.js", "13_buffer.js", - "30_fs.js", "30_os.js", "40_fs_events.js", "40_http.js", @@ -240,6 +275,7 @@ mod startup_snapshot { deno_napi::init::(), deno_http::init(), deno_io::init(Default::default()), + deno_fs::init::(false), deno_flash::init::(false), // No --unstable runtime_extension, // FIXME(bartlomieju): these extensions are specified last, because they diff --git a/runtime/js/40_process.js b/runtime/js/40_process.js index 28bb2870cd..661f972df0 100644 --- a/runtime/js/40_process.js +++ b/runtime/js/40_process.js @@ -16,7 +16,7 @@ const { SymbolFor, Symbol, } = primordials; -import { FsFile } from "internal:runtime/30_fs.js"; +import { FsFile } from "internal:deno_fs/30_fs.js"; import { readAll } from "internal:deno_io/12_io.js"; import { assert, pathFromURL } from "internal:deno_web/00_infra.js"; import * as abortSignal from "internal:deno_web/03_abort_signal.js"; diff --git a/runtime/js/90_deno_ns.js b/runtime/js/90_deno_ns.js index 45db052923..93d3277870 100644 --- a/runtime/js/90_deno_ns.js +++ b/runtime/js/90_deno_ns.js @@ -15,7 +15,7 @@ import * as version from "internal:runtime/01_version.ts"; import * as permissions from "internal:runtime/10_permissions.js"; import * as io from "internal:deno_io/12_io.js"; import * as buffer from "internal:runtime/13_buffer.js"; -import * as fs from "internal:runtime/30_fs.js"; +import * as fs from "internal:deno_fs/30_fs.js"; import * as os from "internal:runtime/30_os.js"; import * as fsEvents from "internal:runtime/40_fs_events.js"; import * as process from "internal:runtime/40_process.js"; diff --git a/runtime/lib.rs b/runtime/lib.rs index 6bb84698d1..97034bca94 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -8,6 +8,7 @@ pub use deno_crypto; pub use deno_fetch; pub use deno_ffi; pub use deno_flash; +pub use deno_fs; pub use deno_http; pub use deno_io; pub use deno_napi; diff --git a/runtime/ops/mod.rs b/runtime/ops/mod.rs index 48c22ca920..5cb7dcbff2 100644 --- a/runtime/ops/mod.rs +++ b/runtime/ops/mod.rs @@ -1,6 +1,5 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -pub mod fs; pub mod fs_events; pub mod http; pub mod os; diff --git a/runtime/permissions/mod.rs b/runtime/permissions/mod.rs index 5edfbe8f7d..a954387e5f 100644 --- a/runtime/permissions/mod.rs +++ b/runtime/permissions/mod.rs @@ -1915,7 +1915,7 @@ impl deno_websocket::WebSocketPermissions for PermissionsContainer { } } -impl crate::ops::fs::FsPermissions for PermissionsContainer { +impl deno_fs::FsPermissions for PermissionsContainer { fn check_read( &mut self, path: &Path, diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index 06b594dd1d..cce69fabbe 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -425,7 +425,7 @@ impl WebWorker { ), // Extensions providing Deno.* features ops::fs_events::init(), - ops::fs::init::(), + deno_fs::init::(unstable), deno_io::init(options.stdio), deno_tls::init(), deno_net::init::( diff --git a/runtime/worker.rs b/runtime/worker.rs index 66497489d7..1a6b6f2fcb 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -255,7 +255,7 @@ impl MainWorker { options.format_js_error_fn.clone(), ), ops::fs_events::init(), - ops::fs::init::(), + deno_fs::init::(unstable), deno_io::init(options.stdio), deno_tls::init(), deno_net::init::(