foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-24 15:19:26 -05:00
Code Issues Packages 1 Wiki Activity

refactor: Move lockfile to a separate crate (#17503)

Browse source 
Moves the lockfile implementation to a separate crate so other projects
like Deploy can use it as well.
This commit is contained in:
Bartek Iwańczuk 2023-01-23 23:41:02 +01:00 committed by GitHub
parent cd19231306
commit bf237c6241
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 659 additions and 604 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
Cargo.lock generated
View file

@ -775,6 +775,7 @@ dependencies = [
"deno_emit", "deno_emit",
"deno_graph", "deno_graph",
"deno_lint", "deno_lint",
"deno_lockfile",
"deno_runtime", "deno_runtime",
"deno_task_shell", "deno_task_shell",
"dissimilar", "dissimilar",
@ -1112,6 +1113,17 @@ dependencies = [
"serde_json", "serde_json",
] ]
[[package]]
name = "deno_lockfile"
version = "0.1.0"
dependencies = [
"anyhow",
"ring",
"serde",
"serde_json",
"test_util",
]
[[package]] [[package]]
name = "deno_napi" name = "deno_napi"
version = "0.15.0" version = "0.15.0"

2
Cargo.toml
View file

@ -30,6 +30,7 @@ members = [
"ext/websocket", "ext/websocket",
"ext/webstorage", "ext/webstorage",
"ext/napi", "ext/napi",
"lockfile",
] ]
exclude = ["test_util/std/hash/_wasm"] exclude = ["test_util/std/hash/_wasm"]
@ -50,6 +51,7 @@ deno_runtime = { version = "0.93.0", path = "./runtime" }
napi_sym = { version = "0.15.0", path = "./cli/napi/sym" } napi_sym = { version = "0.15.0", path = "./cli/napi/sym" }
deno_bench_util = { version = "0.79.0", path = "./bench_util" } deno_bench_util = { version = "0.79.0", path = "./bench_util" }
test_util = { path = "./test_util" } test_util = { path = "./test_util" }
deno_lockfile = { version = "0.1.0", path = "./lockfile" }
# exts # exts
deno_broadcast_channel = { version = "0.79.0", path = "./ext/broadcast_channel" } deno_broadcast_channel = { version = "0.79.0", path = "./ext/broadcast_channel" }

1
cli/Cargo.toml
View file

@ -48,6 +48,7 @@ deno_doc = "0.52.0"
deno_emit = "0.13.0" deno_emit = "0.13.0"
deno_graph = "0.41.0" deno_graph = "0.41.0"
deno_lint = { version = "0.37.0", features = ["docs"] } deno_lint = { version = "0.37.0", features = ["docs"] }
deno_lockfile.workspace = true
deno_runtime.workspace = true deno_runtime.workspace = true
deno_task_shell = "0.8.1" deno_task_shell = "0.8.1"
napi_sym.workspace = true napi_sym.workspace = true

638
cli/args/lockfile.rs
View file

@ -1,602 +1,84 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use std::collections::BTreeMap;
use std::io::Write;
use std::path::PathBuf;
use deno_core::anyhow::Context;
use deno_core::error::AnyError; use deno_core::error::AnyError;
use deno_core::serde::Deserialize; use std::path::PathBuf;
use deno_core::serde::Serialize;
use deno_core::serde_json;
use log::debug;
use crate::args::config_file::LockConfig; use crate::args::config_file::LockConfig;
use crate::args::ConfigFile; use crate::args::ConfigFile;
use crate::npm::NpmPackageId;
use crate::npm::NpmPackageReq;
use crate::npm::NpmResolutionPackage; use crate::npm::NpmResolutionPackage;
use crate::tools::fmt::format_json;
use crate::util;
use crate::Flags; use crate::Flags;
use super::DenoSubcommand; use super::DenoSubcommand;
#[derive(Debug)] pub use deno_lockfile::Lockfile;
pub struct LockfileError(String); pub use deno_lockfile::LockfileError;
use deno_lockfile::NpmPackageDependencyLockfileInfo;
use deno_lockfile::NpmPackageLockfileInfo;
impl std::fmt::Display for LockfileError { pub fn discover(
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { flags: &Flags,
f.write_str(&self.0) maybe_config_file: Option<&ConfigFile>,
) -> Result<Option<Lockfile>, AnyError> {
if flags.no_lock
|| matches!(
flags.subcommand,
DenoSubcommand::Install(_) | DenoSubcommand::Uninstall(_)
)
{
return Ok(None);
} }
}
impl std::error::Error for LockfileError {} let filename = match flags.lock {
Some(ref lock) => PathBuf::from(lock),
#[derive(Debug, Clone, Serialize, Deserialize)] None => match maybe_config_file {
pub struct NpmPackageInfo { Some(config_file) => {
pub integrity: String, if config_file.specifier.scheme() == "file" {
pub dependencies: BTreeMap<String, String>, match config_file.to_lock_config()? {
} Some(LockConfig::Bool(lock)) if !lock => {
return Ok(None);
#[derive(Clone, Debug, Default, Serialize, Deserialize)] }
pub struct NpmContent { Some(LockConfig::PathBuf(lock)) => config_file
/// Mapping between requests for npm packages and resolved packages, eg. .specifier
/// { .to_file_path()
/// "chalk": "chalk@5.0.0" .unwrap()
/// "react@17": "react@17.0.1" .parent()
/// "foo@latest": "foo@1.0.0" .unwrap()
/// } .join(lock),
pub specifiers: BTreeMap<String, String>, _ => {
/// Mapping between resolved npm specifiers and their associated info, eg. let mut path = config_file.specifier.to_file_path().unwrap();
/// { path.set_file_name("deno.lock");
/// "chalk@5.0.0": { path
/// "integrity": "sha512-...",
/// "dependencies": {
/// "ansi-styles": "ansi-styles@4.1.0",
/// }
/// }
/// }
pub packages: BTreeMap<String, NpmPackageInfo>,
}
impl NpmContent {
fn is_empty(&self) -> bool {
self.specifiers.is_empty() && self.packages.is_empty()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockfileContent {
version: String,
// Mapping between URLs and their checksums for "http:" and "https:" deps
remote: BTreeMap<String, String>,
#[serde(skip_serializing_if = "NpmContent::is_empty")]
#[serde(default)]
pub npm: NpmContent,
}
impl LockfileContent {
fn empty() -> Self {
Self {
version: "2".to_string(),
remote: BTreeMap::new(),
npm: NpmContent::default(),
}
}
}
#[derive(Debug, Clone)]
pub struct Lockfile {
pub overwrite: bool,
pub has_content_changed: bool,
pub content: LockfileContent,
pub filename: PathBuf,
}
impl Lockfile {
pub fn discover(
flags: &Flags,
maybe_config_file: Option<&ConfigFile>,
) -> Result<Option<Lockfile>, AnyError> {
if flags.no_lock
|| matches!(
flags.subcommand,
DenoSubcommand::Install(_) | DenoSubcommand::Uninstall(_)
)
{
return Ok(None);
}
let filename = match flags.lock {
Some(ref lock) => PathBuf::from(lock),
None => match maybe_config_file {
Some(config_file) => {
if config_file.specifier.scheme() == "file" {
match config_file.to_lock_config()? {
Some(LockConfig::Bool(lock)) if !lock => {
return Ok(None);
}
Some(LockConfig::PathBuf(lock)) => config_file
.specifier
.to_file_path()
.unwrap()
.parent()
.unwrap()
.join(lock),
_ => {
let mut path = config_file.specifier.to_file_path().unwrap();
path.set_file_name("deno.lock");
path
}
} }
} else {
return Ok(None);
} }
}
None => return Ok(None),
},
};
let lockfile = Self::new(filename, flags.lock_write)?;
Ok(Some(lockfile))
}
pub fn new(filename: PathBuf, overwrite: bool) -> Result<Lockfile, AnyError> {
// Writing a lock file always uses the new format.
if overwrite {
return Ok(Lockfile {
overwrite,
has_content_changed: false,
content: LockfileContent::empty(),
filename,
});
}
let result = match std::fs::read_to_string(&filename) {
Ok(content) => Ok(content),
Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound {
return Ok(Lockfile {
overwrite,
has_content_changed: false,
content: LockfileContent::empty(),
filename,
});
} else { } else {
Err(e) return Ok(None);
} }
} }
}; None => return Ok(None),
},
};
let s = result.with_context(|| { let lockfile = Lockfile::new(filename, flags.lock_write)?;
format!("Unable to read lockfile: \"{}\"", filename.display()) Ok(Some(lockfile))
})?; }
let value: serde_json::Value =
serde_json::from_str(&s).with_context(|| {
format!(
"Unable to parse contents of the lockfile \"{}\"",
filename.display()
)
})?;
let version = value.get("version").and_then(|v| v.as_str());
let content = if version == Some("2") {
serde_json::from_value::<LockfileContent>(value).with_context(|| {
format!(
"Unable to parse contents of the lockfile \"{}\"",
filename.display()
)
})?
} else {
// If there's no version field, we assume that user is using the old
// version of the lockfile. We'll migrate it in-place into v2 and it
// will be writte in v2 if user uses `--lock-write` flag.
let remote: BTreeMap<String, String> = serde_json::from_value(value)
.with_context(|| {
format!(
"Unable to parse contents of the lockfile \"{}\"",
filename.display()
)
})?;
LockfileContent {
version: "2".to_string(),
remote,
npm: NpmContent::default(),
}
};
Ok(Lockfile { // NOTE(bartlomieju): we don't want a reverse mapping to be possible.
overwrite, #[allow(clippy::from_over_into)]
has_content_changed: false, impl Into<NpmPackageLockfileInfo> for NpmResolutionPackage {
content, fn into(self) -> NpmPackageLockfileInfo {
filename, let dependencies = self
})
}
// Synchronize lock file to disk - noop if --lock-write file is not specified.
pub fn write(&self) -> Result<(), AnyError> {
if !self.has_content_changed && !self.overwrite {
return Ok(());
}
let json_string = serde_json::to_string(&self.content).unwrap();
let format_s = format_json(&json_string, &Default::default())
.ok()
.flatten()
.unwrap_or(json_string);
let mut f = std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(&self.filename)?;
f.write_all(format_s.as_bytes())?;
debug!("lockfile write {}", self.filename.display());
Ok(())
}
// TODO(bartlomieju): this function should return an error instead of a bool,
// but it requires changes to `deno_graph`'s `Locker`.
pub fn check_or_insert_remote(
&mut self,
specifier: &str,
code: &str,
) -> bool {
if !(specifier.starts_with("http:") || specifier.starts_with("https:")) {
return true;
}
if self.overwrite {
// In case --lock-write is specified check always passes
self.insert(specifier, code);
true
} else {
self.check_or_insert(specifier, code)
}
}
pub fn check_or_insert_npm_package(
&mut self,
package: &NpmResolutionPackage,
) -> Result<(), LockfileError> {
if self.overwrite {
// In case --lock-write is specified check always passes
self.insert_npm(package);
Ok(())
} else {
self.check_or_insert_npm(package)
}
}
/// Checks the given module is included, if so verify the checksum. If module
/// is not included, insert it.
fn check_or_insert(&mut self, specifier: &str, code: &str) -> bool {
if let Some(lockfile_checksum) = self.content.remote.get(specifier) {
let compiled_checksum = util::checksum::gen(&[code.as_bytes()]);
lockfile_checksum == &compiled_checksum
} else {
self.insert(specifier, code);
true
}
}
fn insert(&mut self, specifier: &str, code: &str) {
let checksum = util::checksum::gen(&[code.as_bytes()]);
self.content.remote.insert(specifier.to_string(), checksum);
self.has_content_changed = true;
}
fn check_or_insert_npm(
&mut self,
package: &NpmResolutionPackage,
) -> Result<(), LockfileError> {
let specifier = package.id.as_serialized();
if let Some(package_info) = self.content.npm.packages.get(&specifier) {
if package_info.integrity.as_str() != package.dist.integrity().as_str() {
return Err(LockfileError(format!(
"Integrity check failed for npm package: \"{}\". Unable to verify that the package
is the same as when the lockfile was generated.
This could be caused by:
* the lock file may be corrupt
* the source itself may be corrupt
Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".",
package.id.display(), self.filename.display()
)));
}
} else {
self.insert_npm(package);
}
Ok(())
}
fn insert_npm(&mut self, package: &NpmResolutionPackage) {
let dependencies = package
.dependencies .dependencies
.iter() .into_iter()
.map(|(name, id)| (name.to_string(), id.as_serialized())) .map(|(name, id)| NpmPackageDependencyLockfileInfo {
.collect::<BTreeMap<String, String>>(); name,
id: id.as_serialized(),
})
.collect();
self.content.npm.packages.insert( NpmPackageLockfileInfo {
package.id.as_serialized(), display_id: self.id.display(),
NpmPackageInfo { serialized_id: self.id.as_serialized(),
integrity: package.dist.integrity().to_string(), integrity: self.dist.integrity().to_string(),
dependencies, dependencies,
}, }
);
self.has_content_changed = true;
}
pub fn insert_npm_specifier(
&mut self,
package_req: &NpmPackageReq,
package_id: &NpmPackageId,
) {
self
.content
.npm
.specifiers
.insert(package_req.to_string(), package_id.as_serialized());
self.has_content_changed = true;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::npm::NpmPackageId;
use crate::npm::NpmPackageVersionDistInfo;
use crate::npm::NpmVersion;
use deno_core::serde_json;
use deno_core::serde_json::json;
use std::collections::HashMap;
use std::fs::File;
use std::io::prelude::*;
use std::io::Write;
use test_util::TempDir;
fn setup(temp_dir: &TempDir) -> PathBuf {
let file_path = temp_dir.path().join("valid_lockfile.json");
let mut file = File::create(file_path).expect("write file fail");
let value: serde_json::Value = json!({
"version": "2",
"remote": {
"https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4",
"https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a"
},
"npm": {
"specifiers": {},
"packages": {
"nanoid@3.3.4": {
"integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==",
"dependencies": {}
},
"picocolors@1.0.0": {
"integrity": "sha512-foobar",
"dependencies": {}
},
}
}
});
file.write_all(value.to_string().as_bytes()).unwrap();
temp_dir.path().join("valid_lockfile.json")
}
#[test]
fn create_lockfile_for_nonexistent_path() {
let file_path = PathBuf::from("nonexistent_lock_file.json");
assert!(Lockfile::new(file_path, false).is_ok());
}
#[test]
fn new_valid_lockfile() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let result = Lockfile::new(file_path, false).unwrap();
let remote = result.content.remote;
let keys: Vec<String> = remote.keys().cloned().collect();
let expected_keys = vec![
String::from("https://deno.land/std@0.71.0/async/delay.ts"),
String::from("https://deno.land/std@0.71.0/textproto/mod.ts"),
];
assert_eq!(keys.len(), 2);
assert_eq!(keys, expected_keys);
}
#[test]
fn new_lockfile_from_file_and_insert() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let mut lockfile = Lockfile::new(file_path, false).unwrap();
lockfile.insert(
"https://deno.land/std@0.71.0/io/util.ts",
"Here is some source code",
);
let remote = lockfile.content.remote;
let keys: Vec<String> = remote.keys().cloned().collect();
let expected_keys = vec![
String::from("https://deno.land/std@0.71.0/async/delay.ts"),
String::from("https://deno.land/std@0.71.0/io/util.ts"),
String::from("https://deno.land/std@0.71.0/textproto/mod.ts"),
];
assert_eq!(keys.len(), 3);
assert_eq!(keys, expected_keys);
}
#[test]
fn new_lockfile_and_write() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let mut lockfile = Lockfile::new(file_path, true).unwrap();
lockfile.insert(
"https://deno.land/std@0.71.0/textproto/mod.ts",
"Here is some source code",
);
lockfile.insert(
"https://deno.land/std@0.71.0/io/util.ts",
"more source code here",
);
lockfile.insert(
"https://deno.land/std@0.71.0/async/delay.ts",
"this source is really exciting",
);
lockfile.write().expect("unable to write");
let file_path_buf = temp_dir.path().join("valid_lockfile.json");
let file_path = file_path_buf.to_str().expect("file path fail").to_string();
// read the file contents back into a string and check
let mut checkfile = File::open(file_path).expect("Unable to open the file");
let mut contents = String::new();
checkfile
.read_to_string(&mut contents)
.expect("Unable to read the file");
let contents_json =
serde_json::from_str::<serde_json::Value>(&contents).unwrap();
let object = contents_json["remote"].as_object().unwrap();
assert_eq!(
object
.get("https://deno.land/std@0.71.0/textproto/mod.ts")
.and_then(|v| v.as_str()),
// sha-256 hash of the source 'Here is some source code'
Some("fedebba9bb82cce293196f54b21875b649e457f0eaf55556f1e318204947a28f")
);
// confirm that keys are sorted alphabetically
let mut keys = object.keys().map(|k| k.as_str());
assert_eq!(
keys.next(),
Some("https://deno.land/std@0.71.0/async/delay.ts")
);
assert_eq!(keys.next(), Some("https://deno.land/std@0.71.0/io/util.ts"));
assert_eq!(
keys.next(),
Some("https://deno.land/std@0.71.0/textproto/mod.ts")
);
assert!(keys.next().is_none());
}
#[test]
fn check_or_insert_lockfile() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let mut lockfile = Lockfile::new(file_path, false).unwrap();
lockfile.insert(
"https://deno.land/std@0.71.0/textproto/mod.ts",
"Here is some source code",
);
let check_true = lockfile.check_or_insert_remote(
"https://deno.land/std@0.71.0/textproto/mod.ts",
"Here is some source code",
);
assert!(check_true);
let check_false = lockfile.check_or_insert_remote(
"https://deno.land/std@0.71.0/textproto/mod.ts",
"Here is some NEW source code",
);
assert!(!check_false);
// Not present in lockfile yet, should be inserted and check passed.
let check_true = lockfile.check_or_insert_remote(
"https://deno.land/std@0.71.0/http/file_server.ts",
"This is new Source code",
);
assert!(check_true);
}
#[test]
fn check_or_insert_lockfile_npm() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let mut lockfile = Lockfile::new(file_path, false).unwrap();
let npm_package = NpmResolutionPackage {
id: NpmPackageId {
name: "nanoid".to_string(),
version: NpmVersion::parse("3.3.4").unwrap(),
peer_dependencies: Vec::new(),
},
copy_index: 0,
dist: NpmPackageVersionDistInfo::new(
"foo".to_string(),
"shasum".to_string(),
Some("sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==".to_string()),
),
dependencies: HashMap::new(),
};
let check_ok = lockfile.check_or_insert_npm_package(&npm_package);
assert!(check_ok.is_ok());
let npm_package = NpmResolutionPackage {
id: NpmPackageId {
name: "picocolors".to_string(),
version: NpmVersion::parse("1.0.0").unwrap(),
peer_dependencies: Vec::new(),
},
copy_index: 0,
dist: NpmPackageVersionDistInfo::new(
"foo".to_string(),
"shasum".to_string(),
Some("sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==".to_string()),
),
dependencies: HashMap::new(),
};
// Integrity is borked in the loaded lockfile
let check_err = lockfile.check_or_insert_npm_package(&npm_package);
assert!(check_err.is_err());
let npm_package = NpmResolutionPackage {
id: NpmPackageId {
name: "source-map-js".to_string(),
version: NpmVersion::parse("1.0.2").unwrap(),
peer_dependencies: Vec::new(),
},
copy_index: 0,
dist: NpmPackageVersionDistInfo::new(
"foo".to_string(),
"foo".to_string(),
Some("sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==".to_string()),
),
dependencies: HashMap::new(),
};
// Not present in lockfile yet, should be inserted and check passed.
let check_ok = lockfile.check_or_insert_npm_package(&npm_package);
assert!(check_ok.is_ok());
let npm_package = NpmResolutionPackage {
id: NpmPackageId {
name: "source-map-js".to_string(),
version: NpmVersion::parse("1.0.2").unwrap(),
peer_dependencies: Vec::new(),
},
copy_index: 0,
dist: NpmPackageVersionDistInfo::new(
"foo".to_string(),
"foo".to_string(),
Some("sha512-foobar".to_string()),
),
dependencies: HashMap::new(),
};
// Now present in lockfile, should file due to borked integrity
let check_err = lockfile.check_or_insert_npm_package(&npm_package);
assert!(check_err.is_err());
} }
} }

2
cli/args/mod.rs
View file

@ -502,7 +502,7 @@ impl CliOptions {
pub fn from_flags(flags: Flags) -> Result<Self, AnyError> { pub fn from_flags(flags: Flags) -> Result<Self, AnyError> {
let maybe_config_file = ConfigFile::discover(&flags)?; let maybe_config_file = ConfigFile::discover(&flags)?;
let maybe_lock_file = let maybe_lock_file =
Lockfile::discover(&flags, maybe_config_file.as_ref())?; lockfile::discover(&flags, maybe_config_file.as_ref())?;
Ok(Self::new(flags, maybe_config_file, maybe_lock_file)) Ok(Self::new(flags, maybe_config_file, maybe_lock_file))
} }

13
cli/npm/registry.rs
View file

@ -184,19 +184,6 @@ pub struct NpmPackageVersionDistInfo {
} }
impl NpmPackageVersionDistInfo { impl NpmPackageVersionDistInfo {
#[cfg(test)]
pub fn new(
tarball: String,
shasum: String,
integrity: Option<String>,
) -> Self {
Self {
tarball,
shasum,
integrity,
}
}
pub fn integrity(&self) -> Cow<String> { pub fn integrity(&self) -> Cow<String> {
self self
.integrity .integrity

7
cli/npm/resolution/mod.rs
View file

@ -397,10 +397,13 @@ impl NpmResolution {
pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> { pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> {
let snapshot = self.snapshot.read(); let snapshot = self.snapshot.read();
for (package_req, package_id) in snapshot.package_reqs.iter() { for (package_req, package_id) in snapshot.package_reqs.iter() {
lockfile.insert_npm_specifier(package_req, package_id); lockfile.insert_npm_specifier(
package_req.to_string(),
package_id.as_serialized(),
);
} }
for package in snapshot.all_packages() { for package in snapshot.all_packages() {
lockfile.check_or_insert_npm_package(&package)?; lockfile.check_or_insert_npm_package(package.into())?;
} }
Ok(()) Ok(())
} }

35
cli/tests/integration/npm_tests.rs
View file

@ -1116,7 +1116,9 @@ fn lock_file_lock_write() {
"version": "2", "version": "2",
"remote": {}, "remote": {},
"npm": { "npm": {
"specifiers": { "cowsay@1.5.0": "cowsay@1.5.0" }, "specifiers": {
"cowsay@1.5.0": "cowsay@1.5.0"
},
"packages": { "packages": {
"ansi-regex@3.0.1": { "ansi-regex@3.0.1": {
"integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==",
@ -1128,7 +1130,9 @@ fn lock_file_lock_write() {
}, },
"ansi-styles@4.3.0": { "ansi-styles@4.3.0": {
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dependencies": { "color-convert": "color-convert@2.0.1" } "dependencies": {
"color-convert": "color-convert@2.0.1"
}
}, },
"camelcase@5.3.1": { "camelcase@5.3.1": {
"integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
@ -1144,7 +1148,9 @@ fn lock_file_lock_write() {
}, },
"color-convert@2.0.1": { "color-convert@2.0.1": {
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dependencies": { "color-name": "color-name@1.1.4" } "dependencies": {
"color-name": "color-name@1.1.4"
}
}, },
"color-name@1.1.4": { "color-name@1.1.4": {
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
@ -1192,15 +1198,21 @@ fn lock_file_lock_write() {
}, },
"locate-path@5.0.0": { "locate-path@5.0.0": {
"integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
"dependencies": { "p-locate": "p-locate@4.1.0" } "dependencies": {
"p-locate": "p-locate@4.1.0"
}
}, },
"p-limit@2.3.0": { "p-limit@2.3.0": {
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
"dependencies": { "p-try": "p-try@2.2.0" } "dependencies": {
"p-try": "p-try@2.2.0"
}
}, },
"p-locate@4.1.0": { "p-locate@4.1.0": {
"integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
"dependencies": { "p-limit": "p-limit@2.3.0" } "dependencies": {
"p-limit": "p-limit@2.3.0"
}
}, },
"p-try@2.2.0": { "p-try@2.2.0": {
"integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
@ -1239,11 +1251,15 @@ fn lock_file_lock_write() {
}, },
"strip-ansi@4.0.0": { "strip-ansi@4.0.0": {
"integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==",
"dependencies": { "ansi-regex": "ansi-regex@3.0.1" } "dependencies": {
"ansi-regex": "ansi-regex@3.0.1"
}
}, },
"strip-ansi@6.0.1": { "strip-ansi@6.0.1": {
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
"dependencies": { "ansi-regex": "ansi-regex@5.0.1" } "dependencies": {
"ansi-regex": "ansi-regex@5.0.1"
}
}, },
"strip-final-newline@2.0.0": { "strip-final-newline@2.0.0": {
"integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==",
@ -1290,8 +1306,7 @@ fn lock_file_lock_write() {
} }
} }
} }
} }"#;
"#;
temp_dir.write("deno.lock", lock_file_content); temp_dir.write("deno.lock", lock_file_content);
let deno = util::deno_cmd_with_deno_dir(&deno_dir) let deno = util::deno_cmd_with_deno_dir(&deno_dir)

20
lockfile/Cargo.toml Normal file
View file

@ -0,0 +1,20 @@
# Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
[package]
name = "deno_lockfile"
version = "0.1.0"
edition = "2021"
license = "MIT"
description = "An implementation of a lockfile used in Deno"
[lib]
path = "lib.rs"
[dependencies]
anyhow.workspace = true
ring.workspace = true
serde.workspace = true
serde_json.workspace = true
[dev-dependencies]
test_util.workspace = true

3
lockfile/README.md Normal file
View file

@ -0,0 +1,3 @@
# `deno_lockfile`
This crate implements the lockfile format used by Deno.

530
lockfile/lib.rs Normal file
View file

@ -0,0 +1,530 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use std::collections::BTreeMap;
use std::io::Write;
use anyhow::Context;
use anyhow::Error as AnyError;
use ring::digest;
use serde::Deserialize;
use serde::Serialize;
use std::path::PathBuf;
pub struct NpmPackageLockfileInfo {
pub display_id: String,
pub serialized_id: String,
pub integrity: String,
pub dependencies: Vec<NpmPackageDependencyLockfileInfo>,
}
pub struct NpmPackageDependencyLockfileInfo {
pub name: String,
pub id: String,
}
fn gen_checksum(v: &[impl AsRef<[u8]>]) -> String {
let mut ctx = digest::Context::new(&digest::SHA256);
for src in v {
ctx.update(src.as_ref());
}
let digest = ctx.finish();
let out: Vec<String> = digest
.as_ref()
.iter()
.map(|byte| format!("{:02x}", byte))
.collect();
out.join("")
}
#[derive(Debug)]
pub struct LockfileError(String);
impl std::fmt::Display for LockfileError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for LockfileError {}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NpmPackageInfo {
pub integrity: String,
pub dependencies: BTreeMap<String, String>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct NpmContent {
/// Mapping between requests for npm packages and resolved packages, eg.
/// {
/// "chalk": "chalk@5.0.0"
/// "react@17": "react@17.0.1"
/// "foo@latest": "foo@1.0.0"
/// }
pub specifiers: BTreeMap<String, String>,
/// Mapping between resolved npm specifiers and their associated info, eg.
/// {
/// "chalk@5.0.0": {
/// "integrity": "sha512-...",
/// "dependencies": {
/// "ansi-styles": "ansi-styles@4.1.0",
/// }
/// }
/// }
pub packages: BTreeMap<String, NpmPackageInfo>,
}
impl NpmContent {
fn is_empty(&self) -> bool {
self.specifiers.is_empty() && self.packages.is_empty()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockfileContent {
version: String,
// Mapping between URLs and their checksums for "http:" and "https:" deps
remote: BTreeMap<String, String>,
#[serde(skip_serializing_if = "NpmContent::is_empty")]
#[serde(default)]
pub npm: NpmContent,
}
impl LockfileContent {
fn empty() -> Self {
Self {
version: "2".to_string(),
remote: BTreeMap::new(),
npm: NpmContent::default(),
}
}
}
#[derive(Debug, Clone)]
pub struct Lockfile {
pub overwrite: bool,
pub has_content_changed: bool,
pub content: LockfileContent,
pub filename: PathBuf,
}
impl Lockfile {
pub fn new(filename: PathBuf, overwrite: bool) -> Result<Lockfile, AnyError> {
// Writing a lock file always uses the new format.
if overwrite {
return Ok(Lockfile {
overwrite,
has_content_changed: false,
content: LockfileContent::empty(),
filename,
});
}
let result = match std::fs::read_to_string(&filename) {
Ok(content) => Ok(content),
Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound {
return Ok(Lockfile {
overwrite,
has_content_changed: false,
content: LockfileContent::empty(),
filename,
});
} else {
Err(e)
}
}
};
let s = result.with_context(|| {
format!("Unable to read lockfile: \"{}\"", filename.display())
})?;
let value: serde_json::Value =
serde_json::from_str(&s).with_context(|| {
format!(
"Unable to parse contents of the lockfile \"{}\"",
filename.display()
)
})?;
let version = value.get("version").and_then(|v| v.as_str());
let content = if version == Some("2") {
serde_json::from_value::<LockfileContent>(value).with_context(|| {
format!(
"Unable to parse contents of the lockfile \"{}\"",
filename.display()
)
})?
} else {
// If there's no version field, we assume that user is using the old
// version of the lockfile. We'll migrate it in-place into v2 and it
// will be writte in v2 if user uses `--lock-write` flag.
let remote: BTreeMap<String, String> = serde_json::from_value(value)
.with_context(|| {
format!(
"Unable to parse contents of the lockfile \"{}\"",
filename.display()
)
})?;
LockfileContent {
version: "2".to_string(),
remote,
npm: NpmContent::default(),
}
};
Ok(Lockfile {
overwrite,
has_content_changed: false,
content,
filename,
})
}
// Synchronize lock file to disk - noop if --lock-write file is not specified.
pub fn write(&self) -> Result<(), AnyError> {
if !self.has_content_changed && !self.overwrite {
return Ok(());
}
let json_string = serde_json::to_string_pretty(&self.content).unwrap();
let mut f = std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(&self.filename)?;
f.write_all(json_string.as_bytes())?;
Ok(())
}
// TODO(bartlomieju): this function should return an error instead of a bool,
// but it requires changes to `deno_graph`'s `Locker`.
pub fn check_or_insert_remote(
&mut self,
specifier: &str,
code: &str,
) -> bool {
if !(specifier.starts_with("http:") || specifier.starts_with("https:")) {
return true;
}
if self.overwrite {
// In case --lock-write is specified check always passes
self.insert(specifier, code);
true
} else {
self.check_or_insert(specifier, code)
}
}
pub fn check_or_insert_npm_package(
&mut self,
package_info: NpmPackageLockfileInfo,
) -> Result<(), LockfileError> {
if self.overwrite {
// In case --lock-write is specified check always passes
self.insert_npm(package_info);
Ok(())
} else {
self.check_or_insert_npm(package_info)
}
}
/// Checks the given module is included, if so verify the checksum. If module
/// is not included, insert it.
fn check_or_insert(&mut self, specifier: &str, code: &str) -> bool {
if let Some(lockfile_checksum) = self.content.remote.get(specifier) {
let compiled_checksum = gen_checksum(&[code.as_bytes()]);
lockfile_checksum == &compiled_checksum
} else {
self.insert(specifier, code);
true
}
}
fn insert(&mut self, specifier: &str, code: &str) {
let checksum = gen_checksum(&[code.as_bytes()]);
self.content.remote.insert(specifier.to_string(), checksum);
self.has_content_changed = true;
}
fn check_or_insert_npm(
&mut self,
package: NpmPackageLockfileInfo,
) -> Result<(), LockfileError> {
if let Some(package_info) =
self.content.npm.packages.get(&package.serialized_id)
{
if package_info.integrity.as_str() != package.integrity {
return Err(LockfileError(format!(
"Integrity check failed for npm package: \"{}\". Unable to verify that the package
is the same as when the lockfile was generated.
This could be caused by:
* the lock file may be corrupt
* the source itself may be corrupt
Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".",
package.display_id, self.filename.display()
)));
}
} else {
self.insert_npm(package);
}
Ok(())
}
fn insert_npm(&mut self, package_info: NpmPackageLockfileInfo) {
let dependencies = package_info
.dependencies
.iter()
.map(|dep| (dep.name.to_string(), dep.id.to_string()))
.collect::<BTreeMap<String, String>>();
self.content.npm.packages.insert(
package_info.serialized_id.to_string(),
NpmPackageInfo {
integrity: package_info.integrity,
dependencies,
},
);
self.has_content_changed = true;
}
pub fn insert_npm_specifier(
&mut self,
serialized_package_req: String,
serialized_package_id: String,
) {
self
.content
.npm
.specifiers
.insert(serialized_package_req, serialized_package_id);
self.has_content_changed = true;
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use std::fs::File;
use std::io::prelude::*;
use std::io::Write;
use test_util::TempDir;
fn setup(temp_dir: &TempDir) -> PathBuf {
let file_path = temp_dir.path().join("valid_lockfile.json");
let mut file = File::create(file_path).expect("write file fail");
let value: serde_json::Value = json!({
"version": "2",
"remote": {
"https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4",
"https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a"
},
"npm": {
"specifiers": {},
"packages": {
"nanoid@3.3.4": {
"integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==",
"dependencies": {}
},
"picocolors@1.0.0": {
"integrity": "sha512-foobar",
"dependencies": {}
},
}
}
});
file.write_all(value.to_string().as_bytes()).unwrap();
temp_dir.path().join("valid_lockfile.json")
}
#[test]
fn create_lockfile_for_nonexistent_path() {
let file_path = PathBuf::from("nonexistent_lock_file.json");
assert!(Lockfile::new(file_path, false).is_ok());
}
#[test]
fn new_valid_lockfile() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let result = Lockfile::new(file_path, false).unwrap();
let remote = result.content.remote;
let keys: Vec<String> = remote.keys().cloned().collect();
let expected_keys = vec![
String::from("https://deno.land/std@0.71.0/async/delay.ts"),
String::from("https://deno.land/std@0.71.0/textproto/mod.ts"),
];
assert_eq!(keys.len(), 2);
assert_eq!(keys, expected_keys);
}
#[test]
fn new_lockfile_from_file_and_insert() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let mut lockfile = Lockfile::new(file_path, false).unwrap();
lockfile.insert(
"https://deno.land/std@0.71.0/io/util.ts",
"Here is some source code",
);
let remote = lockfile.content.remote;
let keys: Vec<String> = remote.keys().cloned().collect();
let expected_keys = vec![
String::from("https://deno.land/std@0.71.0/async/delay.ts"),
String::from("https://deno.land/std@0.71.0/io/util.ts"),
String::from("https://deno.land/std@0.71.0/textproto/mod.ts"),
];
assert_eq!(keys.len(), 3);
assert_eq!(keys, expected_keys);
}
#[test]
fn new_lockfile_and_write() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let mut lockfile = Lockfile::new(file_path, true).unwrap();
lockfile.insert(
"https://deno.land/std@0.71.0/textproto/mod.ts",
"Here is some source code",
);
lockfile.insert(
"https://deno.land/std@0.71.0/io/util.ts",
"more source code here",
);
lockfile.insert(
"https://deno.land/std@0.71.0/async/delay.ts",
"this source is really exciting",
);
lockfile.write().expect("unable to write");
let file_path_buf = temp_dir.path().join("valid_lockfile.json");
let file_path = file_path_buf.to_str().expect("file path fail").to_string();
// read the file contents back into a string and check
let mut checkfile = File::open(file_path).expect("Unable to open the file");
let mut contents = String::new();
checkfile
.read_to_string(&mut contents)
.expect("Unable to read the file");
let contents_json =
serde_json::from_str::<serde_json::Value>(&contents).unwrap();
let object = contents_json["remote"].as_object().unwrap();
assert_eq!(
object
.get("https://deno.land/std@0.71.0/textproto/mod.ts")
.and_then(|v| v.as_str()),
// sha-256 hash of the source 'Here is some source code'
Some("fedebba9bb82cce293196f54b21875b649e457f0eaf55556f1e318204947a28f")
);
// confirm that keys are sorted alphabetically
let mut keys = object.keys().map(|k| k.as_str());
assert_eq!(
keys.next(),
Some("https://deno.land/std@0.71.0/async/delay.ts")
);
assert_eq!(keys.next(), Some("https://deno.land/std@0.71.0/io/util.ts"));
assert_eq!(
keys.next(),
Some("https://deno.land/std@0.71.0/textproto/mod.ts")
);
assert!(keys.next().is_none());
}
#[test]
fn check_or_insert_lockfile() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let mut lockfile = Lockfile::new(file_path, false).unwrap();
lockfile.insert(
"https://deno.land/std@0.71.0/textproto/mod.ts",
"Here is some source code",
);
let check_true = lockfile.check_or_insert_remote(
"https://deno.land/std@0.71.0/textproto/mod.ts",
"Here is some source code",
);
assert!(check_true);
let check_false = lockfile.check_or_insert_remote(
"https://deno.land/std@0.71.0/textproto/mod.ts",
"Here is some NEW source code",
);
assert!(!check_false);
// Not present in lockfile yet, should be inserted and check passed.
let check_true = lockfile.check_or_insert_remote(
"https://deno.land/std@0.71.0/http/file_server.ts",
"This is new Source code",
);
assert!(check_true);
}
#[test]
fn check_or_insert_lockfile_npm() {
let temp_dir = TempDir::new();
let file_path = setup(&temp_dir);
let mut lockfile = Lockfile::new(file_path, false).unwrap();
let npm_package = NpmPackageLockfileInfo {
display_id: "nanoid@3.3.4".to_string(),
serialized_id: "nanoid@3.3.4".to_string(),
integrity: "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==".to_string(),
dependencies: vec![],
};
let check_ok = lockfile.check_or_insert_npm_package(npm_package);
assert!(check_ok.is_ok());
let npm_package = NpmPackageLockfileInfo {
display_id: "picocolors@1.0.0".to_string(),
serialized_id: "picocolors@1.0.0".to_string(),
integrity: "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==".to_string(),
dependencies: vec![],
};
// Integrity is borked in the loaded lockfile
let check_err = lockfile.check_or_insert_npm_package(npm_package);
assert!(check_err.is_err());
let npm_package = NpmPackageLockfileInfo {
display_id: "source-map-js@1.0.2".to_string(),
serialized_id: "source-map-js@1.0.2".to_string(),
integrity: "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==".to_string(),
dependencies: vec![],
};
// Not present in lockfile yet, should be inserted and check passed.
let check_ok = lockfile.check_or_insert_npm_package(npm_package);
assert!(check_ok.is_ok());
let npm_package = NpmPackageLockfileInfo {
display_id: "source-map-js@1.0.2".to_string(),
serialized_id: "source-map-js@1.0.2".to_string(),
integrity: "sha512-foobar".to_string(),
dependencies: vec![],
};
// Now present in lockfile, should file due to borked integrity
let check_err = lockfile.check_or_insert_npm_package(npm_package);
assert!(check_err.is_err());
}
}