feat(unstable): deno add subcommand (#22520)

This commit adds "deno add" subcommand that has a basic support for
adding "jsr:" packages to "deno.json" file. 

This currently doesn't support "npm:" specifiers and specifying version
constraints.

cli/args/flags.rs

@@ -35,6 +35,11 @@ pub struct FileFlags {
   pub include: Vec<String>,
 }
 
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub struct AddFlags {
+  pub packages: Vec<String>,
+}
+
 #[derive(Clone, Debug, Default, Eq, PartialEq)]
 pub struct BenchFlags {
   pub files: FileFlags,
@@ -307,6 +312,7 @@ pub struct PublishFlags {
 
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum DenoSubcommand {
+  Add(AddFlags),
   Bench(BenchFlags),
   Bundle(BundleFlags),
   Cache(CacheFlags),
@@ -760,9 +766,9 @@ impl Flags {
       | Test(_) | Bench(_) | Repl(_) | Compile(_) | Publish(_) => {
         std::env::current_dir().ok()
       }
-      Bundle(_) | Completions(_) | Doc(_) | Fmt(_) | Init(_) | Install(_)
-      | Uninstall(_) | Jupyter(_) | Lsp | Lint(_) | Types | Upgrade(_)
-      | Vendor(_) => None,
+      Add(_) | Bundle(_) | Completions(_) | Doc(_) | Fmt(_) | Init(_)
+      | Install(_) | Uninstall(_) | Jupyter(_) | Lsp | Lint(_) | Types
+      | Upgrade(_) | Vendor(_) => None,
     }
   }
 
@@ -923,6 +929,7 @@ pub fn flags_from_vec(args: Vec<String>) -> clap::error::Result<Flags> {
 
   if let Some((subcommand, mut m)) = matches.remove_subcommand() {
     match subcommand.as_str() {
+      "add" => add_parse(&mut flags, &mut m),
       "bench" => bench_parse(&mut flags, &mut m),
       "bundle" => bundle_parse(&mut flags, &mut m),
       "cache" => cache_parse(&mut flags, &mut m),
@@ -1078,6 +1085,7 @@ fn clap_root() -> Command {
     .subcommand(run_subcommand())
     .defer(|cmd| {
       cmd
+        .subcommand(add_subcommand())
         .subcommand(bench_subcommand())
         .subcommand(bundle_subcommand())
         .subcommand(cache_subcommand())
@@ -1107,6 +1115,30 @@ fn clap_root() -> Command {
     .after_help(ENV_VARIABLES_HELP)
 }
 
+fn add_subcommand() -> Command {
+  Command::new("add")
+    .about("Add dependencies")
+    .long_about(
+      "Add dependencies to the configuration file.
+
+  deno add @std/path
+
+You can add multiple dependencies at once:
+
+  deno add @std/path @std/assert
+",
+    )
+    .defer(|cmd| {
+      cmd.arg(
+        Arg::new("packages")
+          .help("List of packages to add")
+          .required(true)
+          .num_args(1..)
+          .action(ArgAction::Append),
+      )
+    })
+}
+
 fn bench_subcommand() -> Command {
   Command::new("bench")
     .about("Run benchmarks")
@@ -3218,6 +3250,11 @@ fn unsafely_ignore_certificate_errors_arg() -> Arg {
     .value_parser(flags_net::validator)
 }
 
+fn add_parse(flags: &mut Flags, matches: &mut ArgMatches) {
+  let packages = matches.remove_many::<String>("packages").unwrap().collect();
+  flags.subcommand = DenoSubcommand::Add(AddFlags { packages });
+}
+
 fn bench_parse(flags: &mut Flags, matches: &mut ArgMatches) {
   flags.type_check_mode = TypeCheckMode::Local;
 
@@ -8599,4 +8636,32 @@ mod tests {
       }
     );
   }
+
+  #[test]
+  fn add_subcommand() {
+    let r = flags_from_vec(svec!["deno", "add"]);
+    r.unwrap_err();
+
+    let r = flags_from_vec(svec!["deno", "add", "@david/which"]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Add(AddFlags {
+          packages: svec!["@david/which"],
+        }),
+        ..Flags::default()
+      }
+    );
+
+    let r = flags_from_vec(svec!["deno", "add", "@david/which", "@luca/hello"]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Add(AddFlags {
+          packages: svec!["@david/which", "@luca/hello"],
+        }),
+        ..Flags::default()
+      }
+    );
+  }
 }

cli/lsp/mod.rs

@@ -21,7 +21,7 @@ mod completions;
 mod config;
 mod diagnostics;
 mod documents;
-mod jsr;
+pub mod jsr;
 pub mod language_server;
 mod logging;
 mod lsp_custom;
@@ -32,7 +32,7 @@ mod performance;
 mod refactor;
 mod registries;
 mod repl;
-mod search;
+pub mod search;
 mod semantic_tokens;
 mod testing;
 mod text;

cli/main.rs

@@ -88,6 +88,9 @@ fn spawn_subcommand<F: Future<Output = T> + 'static, T: SubcommandOutput>(
 
 async fn run_subcommand(flags: Flags) -> Result<i32, AnyError> {
   let handle = match flags.subcommand.clone() {
+    DenoSubcommand::Add(add_flags) => spawn_subcommand(async {
+      tools::registry::add(flags, add_flags).await
+    }),
     DenoSubcommand::Bench(bench_flags) => spawn_subcommand(async {
       if bench_flags.watch.is_some() {
         tools::bench::run_benchmarks_with_watch(flags, bench_flags).await

cli/tools/registry/mod.rs

@@ -50,6 +50,7 @@ mod auth;
 mod diagnostics;
 mod graph;
 mod paths;
+mod pm;
 mod provenance;
 mod publish_order;
 mod tar;
@@ -57,6 +58,7 @@ mod unfurl;
 
 use auth::get_auth_method;
 use auth::AuthMethod;
+pub use pm::add;
 use publish_order::PublishOrderGraph;
 pub use unfurl::deno_json_deps;
 use unfurl::SpecifierUnfurler;

cli/tools/registry/pm.rs

@@ -0,0 +1,290 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+use deno_ast::TextChange;
+use deno_config::FmtOptionsConfig;
+use deno_core::anyhow::bail;
+use deno_core::anyhow::Context;
+use deno_core::error::AnyError;
+use deno_core::futures::FutureExt;
+use deno_core::futures::StreamExt;
+use deno_core::serde_json;
+use deno_semver::jsr::JsrPackageReqReference;
+use deno_semver::npm::NpmPackageReqReference;
+use deno_semver::package::PackageReq;
+use jsonc_parser::ast::ObjectProp;
+use jsonc_parser::ast::Value;
+
+use crate::args::AddFlags;
+use crate::args::CacheSetting;
+use crate::args::Flags;
+use crate::factory::CliFactory;
+use crate::file_fetcher::FileFetcher;
+use crate::lsp::jsr::CliJsrSearchApi;
+use crate::lsp::search::PackageSearchApi;
+
+pub async fn add(flags: Flags, add_flags: AddFlags) -> Result<(), AnyError> {
+  let cli_factory = CliFactory::from_flags(flags.clone()).await?;
+  let cli_options = cli_factory.cli_options();
+
+  let Some(config_file) = cli_options.maybe_config_file() else {
+    tokio::fs::write(cli_options.initial_cwd().join("deno.json"), "{}\n")
+      .await
+      .context("Failed to create deno.json file")?;
+    log::info!("Created deno.json configuration file.");
+    return add(flags, add_flags).boxed_local().await;
+  };
+
+  if config_file.specifier.scheme() != "file" {
+    bail!("Can't add dependencies to a remote configuration file");
+  }
+  let config_file_path = config_file.specifier.to_file_path().unwrap();
+
+  let http_client = cli_factory.http_client();
+
+  let mut selected_packages = Vec::with_capacity(add_flags.packages.len());
+  let mut package_reqs = Vec::with_capacity(add_flags.packages.len());
+
+  for package_name in add_flags.packages.iter() {
+    let req = if package_name.starts_with("npm:") {
+      let pkg_req = NpmPackageReqReference::from_str(package_name)
+        .with_context(|| {
+          format!("Failed to parse package required: {}", package_name)
+        })?;
+      AddPackageReq::Npm(pkg_req)
+    } else {
+      let pkg_req = JsrPackageReqReference::from_str(&format!(
+        "jsr:{}",
+        package_name.strip_prefix("jsr:").unwrap_or(package_name)
+      ))
+      .with_context(|| {
+        format!("Failed to parse package required: {}", package_name)
+      })?;
+      AddPackageReq::Jsr(pkg_req)
+    };
+
+    package_reqs.push(req);
+  }
+
+  let deps_http_cache = cli_factory.global_http_cache()?;
+  let mut deps_file_fetcher = FileFetcher::new(
+    deps_http_cache.clone(),
+    CacheSetting::ReloadAll,
+    true,
+    http_client.clone(),
+    Default::default(),
+    None,
+  );
+  deps_file_fetcher.set_download_log_level(log::Level::Trace);
+  let jsr_search_api = CliJsrSearchApi::new(deps_file_fetcher);
+
+  let package_futures = package_reqs
+    .into_iter()
+    .map(|package_req| {
+      find_package_and_select_version_for_req(
+        jsr_search_api.clone(),
+        package_req,
+      )
+      .boxed_local()
+    })
+    .collect::<Vec<_>>();
+
+  let stream_of_futures = deno_core::futures::stream::iter(package_futures);
+  let mut buffered = stream_of_futures.buffer_unordered(10);
+
+  while let Some(package_and_version_result) = buffered.next().await {
+    let package_and_version = package_and_version_result?;
+
+    match package_and_version {
+      PackageAndVersion::NotFound(package_name) => {
+        bail!("{} was not found.", crate::colors::red(package_name));
+      }
+      PackageAndVersion::Selected(selected) => {
+        selected_packages.push(selected);
+      }
+    }
+  }
+
+  let config_file_contents =
+    tokio::fs::read_to_string(&config_file_path).await.unwrap();
+  let ast = jsonc_parser::parse_to_ast(
+    &config_file_contents,
+    &Default::default(),
+    &Default::default(),
+  )?;
+
+  let obj = match ast.value {
+    Some(Value::Object(obj)) => obj,
+    _ => bail!("Failed updating config file due to no object."),
+  };
+
+  let mut existing_imports =
+    if let Some(imports) = config_file.json.imports.clone() {
+      match serde_json::from_value::<HashMap<String, String>>(imports) {
+        Ok(i) => i,
+        Err(_) => bail!("Malformed \"imports\" configuration"),
+      }
+    } else {
+      HashMap::default()
+    };
+
+  for selected_package in selected_packages {
+    log::info!(
+      "Add {} - {}@{}",
+      crate::colors::green(&selected_package.import_name),
+      selected_package.package_name,
+      selected_package.version_req
+    );
+    existing_imports.insert(
+      selected_package.import_name,
+      format!(
+        "{}@{}",
+        selected_package.package_name, selected_package.version_req
+      ),
+    );
+  }
+  let mut import_list: Vec<(String, String)> =
+    existing_imports.into_iter().collect();
+
+  import_list.sort_by(|(k1, _), (k2, _)| k1.cmp(k2));
+  let generated_imports = generate_imports(import_list);
+
+  let fmt_config_options = config_file
+    .to_fmt_config()
+    .ok()
+    .flatten()
+    .map(|config| config.options)
+    .unwrap_or_default();
+
+  let new_text = update_config_file_content(
+    obj,
+    &config_file_contents,
+    generated_imports,
+    fmt_config_options,
+  );
+
+  tokio::fs::write(&config_file_path, new_text)
+    .await
+    .context("Failed to update configuration file")?;
+
+  // TODO(bartlomieju): we should now cache the imports from the config file.
+
+  Ok(())
+}
+
+struct SelectedPackage {
+  import_name: String,
+  package_name: String,
+  version_req: String,
+}
+
+enum PackageAndVersion {
+  NotFound(String),
+  Selected(SelectedPackage),
+}
+
+async fn jsr_find_package_and_select_version(
+  jsr_search_api: CliJsrSearchApi,
+  req: &PackageReq,
+) -> Result<PackageAndVersion, AnyError> {
+  let jsr_prefixed_name = format!("jsr:{}", req.name);
+
+  // TODO(bartlomieju): Need to do semver as well - @luca/flag@^0.14 should use to
+  // highest possible `0.14.x` version.
+  let version_req = req.version_req.version_text();
+  if version_req != "*" {
+    bail!("Specifying version constraints is currently not supported. Package: {}@{}", jsr_prefixed_name, version_req);
+  }
+
+  let Ok(versions) = jsr_search_api.versions(&req.name).await else {
+    return Ok(PackageAndVersion::NotFound(jsr_prefixed_name));
+  };
+
+  let Some(latest_version) = versions.first() else {
+    return Ok(PackageAndVersion::NotFound(jsr_prefixed_name));
+  };
+
+  Ok(PackageAndVersion::Selected(SelectedPackage {
+    import_name: req.name.to_string(),
+    package_name: jsr_prefixed_name,
+    // TODO(bartlomieju): fix it, it should not always be caret
+    version_req: format!("^{}", latest_version),
+  }))
+}
+
+async fn find_package_and_select_version_for_req(
+  jsr_search_api: CliJsrSearchApi,
+  add_package_req: AddPackageReq,
+) -> Result<PackageAndVersion, AnyError> {
+  match add_package_req {
+    AddPackageReq::Jsr(pkg_ref) => {
+      jsr_find_package_and_select_version(jsr_search_api, pkg_ref.req()).await
+    }
+    AddPackageReq::Npm(pkg_req) => {
+      bail!(
+        "Adding npm: packages is currently not supported. Package: npm:{}",
+        pkg_req.req().name
+      );
+    }
+  }
+}
+
+enum AddPackageReq {
+  Jsr(JsrPackageReqReference),
+  Npm(NpmPackageReqReference),
+}
+
+fn generate_imports(packages_to_version: Vec<(String, String)>) -> String {
+  let mut contents = vec![];
+  let len = packages_to_version.len();
+  for (index, (package, version)) in packages_to_version.iter().enumerate() {
+    // TODO(bartlomieju): fix it, once we start support specifying version on the cli
+    contents.push(format!("\"{}\": \"{}\"", package, version));
+    if index != len - 1 {
+      contents.push(",".to_string());
+    }
+  }
+  contents.join("\n")
+}
+
+fn update_config_file_content(
+  obj: jsonc_parser::ast::Object,
+  config_file_contents: &str,
+  generated_imports: String,
+  fmt_options: FmtOptionsConfig,
+) -> String {
+  let mut text_changes = vec![];
+
+  match obj.get("imports") {
+    Some(ObjectProp {
+      value: Value::Object(lit),
+      ..
+    }) => text_changes.push(TextChange {
+      range: (lit.range.start + 1)..(lit.range.end - 1),
+      new_text: generated_imports,
+    }),
+    None => {
+      let insert_position = obj.range.end - 1;
+      text_changes.push(TextChange {
+        range: insert_position..insert_position,
+        new_text: format!("\"imports\": {{ {} }}", generated_imports),
+      })
+    }
+    // we verified the shape of `imports` above
+    Some(_) => unreachable!(),
+  }
+
+  let new_text =
+    deno_ast::apply_text_changes(config_file_contents, text_changes);
+
+  crate::tools::fmt::format_json(
+    &PathBuf::from("deno.json"),
+    &new_text,
+    &fmt_options,
+  )
+  .ok()
+  .map(|formatted_text| formatted_text.unwrap_or_else(|| new_text.clone()))
+  .unwrap_or(new_text)
+}

tests/integration/mod.rs

@@ -50,6 +50,8 @@ mod node_compat_tests;
 mod node_unit_tests;
 #[path = "npm_tests.rs"]
 mod npm;
+#[path = "pm_tests.rs"]
+mod pm;
 #[path = "publish_tests.rs"]
 mod publish;
 

tests/integration/pm_tests.rs

@@ -0,0 +1,108 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+
+use deno_core::serde_json::json;
+use test_util::assert_contains;
+use test_util::env_vars_for_jsr_tests;
+// use test_util::env_vars_for_npm_tests;
+// use test_util::itest;
+use test_util::TestContextBuilder;
+
+#[test]
+fn add_basic() {
+  let starting_deno_json = json!({
+    "name": "@foo/bar",
+    "version": "1.0.0",
+    "exports": "./mod.ts",
+  });
+  let context = pm_context_builder().build();
+  let temp_dir = context.temp_dir().path();
+  temp_dir.join("deno.json").write_json(&starting_deno_json);
+
+  let output = context.new_command().args("add @denotest/add").run();
+  output.assert_exit_code(0);
+  let output = output.combined_output();
+  assert_contains!(output, "Add @denotest/add");
+  temp_dir.join("deno.json").assert_matches_json(json!({
+    "name": "@foo/bar",
+    "version": "1.0.0",
+    "exports": "./mod.ts",
+    "imports": {
+      "@denotest/add": "jsr:@denotest/add@^1.0.0"
+    }
+  }));
+}
+
+#[test]
+fn add_basic_no_deno_json() {
+  let context = pm_context_builder().build();
+  let temp_dir = context.temp_dir().path();
+
+  let output = context.new_command().args("add @denotest/add").run();
+  output.assert_exit_code(0);
+  let output = output.combined_output();
+  assert_contains!(output, "Add @denotest/add");
+  temp_dir.join("deno.json").assert_matches_json(json!({
+    "imports": {
+      "@denotest/add": "jsr:@denotest/add@^1.0.0"
+    }
+  }));
+}
+
+#[test]
+fn add_multiple() {
+  let starting_deno_json = json!({
+    "name": "@foo/bar",
+    "version": "1.0.0",
+    "exports": "./mod.ts",
+  });
+  let context = pm_context_builder().build();
+  let temp_dir = context.temp_dir().path();
+  temp_dir.join("deno.json").write_json(&starting_deno_json);
+
+  let output = context
+    .new_command()
+    .args("add @denotest/add @denotest/subset-type-graph")
+    .run();
+  output.assert_exit_code(0);
+  let output = output.combined_output();
+  assert_contains!(output, "Add @denotest/add");
+  temp_dir.join("deno.json").assert_matches_json(json!({
+    "name": "@foo/bar",
+    "version": "1.0.0",
+    "exports": "./mod.ts",
+    "imports": {
+      "@denotest/add": "jsr:@denotest/add@^1.0.0",
+      "@denotest/subset-type-graph": "jsr:@denotest/subset-type-graph@^0.1.0"
+    }
+  }));
+}
+
+#[test]
+fn add_not_supported_npm() {
+  let context = pm_context_builder().build();
+
+  let output = context
+    .new_command()
+    .args("add @denotest/add npm:express")
+    .run();
+  output.assert_exit_code(1);
+  let output = output.combined_output();
+  assert_contains!(output, "error: Adding npm: packages is currently not supported. Package: npm:express");
+}
+
+#[test]
+fn add_not_supported_version_constraint() {
+  let context = pm_context_builder().build();
+
+  let output = context.new_command().args("add @denotest/add@1").run();
+  output.assert_exit_code(1);
+  let output = output.combined_output();
+  assert_contains!(output, "error: Specifying version constraints is currently not supported. Package: jsr:@denotest/add@1");
+}
+
+fn pm_context_builder() -> TestContextBuilder {
+  TestContextBuilder::new()
+    .use_http_server()
+    .envs(env_vars_for_jsr_tests())
+    .use_temp_cwd()
+}

tests/testdata/jsr/subset_type_graph/main.check.out

@@ -1,16 +1,16 @@
-Download http://127.0.0.1:4250/@denotest/subset_type_graph/meta.json
-Download http://127.0.0.1:4250/@denotest/subset_type_graph_invalid/meta.json
-Download http://127.0.0.1:4250/@denotest/subset_type_graph/0.1.0_meta.json
-Download http://127.0.0.1:4250/@denotest/subset_type_graph_invalid/0.1.0_meta.json
+Download http://127.0.0.1:4250/@denotest/subset-type-graph/meta.json
+Download http://127.0.0.1:4250/@denotest/subset-type-graph-invalid/meta.json
+Download http://127.0.0.1:4250/@denotest/subset-type-graph/0.1.0_meta.json
+Download http://127.0.0.1:4250/@denotest/subset-type-graph-invalid/0.1.0_meta.json
 [UNORDERED_START]
-Download http://127.0.0.1:4250/@denotest/subset_type_graph/0.1.0/mod.ts
-Download http://127.0.0.1:4250/@denotest/subset_type_graph_invalid/0.1.0/mod.ts
+Download http://127.0.0.1:4250/@denotest/subset-type-graph/0.1.0/mod.ts
+Download http://127.0.0.1:4250/@denotest/subset-type-graph-invalid/0.1.0/mod.ts
 [UNORDERED_END]
 Check file:///[WILDCARD]/subset_type_graph/main.ts
 error: TS2322 [ERROR]: Type 'string' is not assignable to type 'number'.
 const invalidTypeCheck: number = "";
       ~~~~~~~~~~~~~~~~
-    at http://127.0.0.1:4250/@denotest/subset_type_graph_invalid/0.1.0/mod.ts:11:7
+    at http://127.0.0.1:4250/@denotest/subset-type-graph-invalid/0.1.0/mod.ts:11:7
 
 TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
 const error1: string = new Foo1().method();
@@ -30,7 +30,7 @@ new Foo1().method2();
     'method' is declared here.
       method(): number {
       ~~~~~~
-        at http://127.0.0.1:4250/@denotest/subset_type_graph/0.1.0/mod.ts:8:3
+        at http://127.0.0.1:4250/@denotest/subset-type-graph/0.1.0/mod.ts:8:3
 
 TS2551 [ERROR]: Property 'method2' does not exist on type 'Foo'. Did you mean 'method'?
 new Foo2().method2();
@@ -40,6 +40,6 @@ new Foo2().method2();
     'method' is declared here.
       method() {
       ~~~~~~
-        at http://127.0.0.1:4250/@denotest/subset_type_graph_invalid/0.1.0/mod.ts:2:3
+        at http://127.0.0.1:4250/@denotest/subset-type-graph-invalid/0.1.0/mod.ts:2:3
 
 Found 5 errors.

tests/testdata/jsr/subset_type_graph/main.ts

@@ -1,5 +1,5 @@
-import { Foo as Foo1 } from "jsr:@denotest/subset_type_graph@0.1.0";
-import { Foo as Foo2 } from "jsr:@denotest/subset_type_graph_invalid@0.1.0";
+import { Foo as Foo1 } from "jsr:@denotest/subset-type-graph@0.1.0";
+import { Foo as Foo2 } from "jsr:@denotest/subset-type-graph-invalid@0.1.0";
 
 // these will both raise type checking errors
 const error1: string = new Foo1().method();

tests/util/server/src/servers/registry.rs

@@ -142,7 +142,12 @@ async fn registry_server_handler(
   // serve the registry package files
   let mut file_path =
     testdata_path().to_path_buf().join("jsr").join("registry");
-  file_path.push(&req.uri().path()[1..].replace("%2f", "/"));
+  file_path.push(
+    &req.uri().path()[1..]
+      .replace("%2f", "/")
+      .replace("%2F", "/"),
+  );
+
   if let Ok(body) = tokio::fs::read(&file_path).await {
     let body = if let Some(version) = file_path
       .file_name()