feat(lsp): npm specifier completions (#20121)

cli/lsp/completions.rs

@@ -5,6 +5,8 @@ use super::config::ConfigSnapshot;
 use super::documents::Documents;
 use super::documents::DocumentsFilter;
 use super::lsp_custom;
+use super::npm::CliNpmSearchApi;
+use super::npm::NpmSearchApi;
 use super::registries::ModuleRegistry;
 use super::tsc;
 
@@ -19,6 +21,7 @@ use deno_core::resolve_path;
 use deno_core::resolve_url;
 use deno_core::serde::Deserialize;
 use deno_core::serde::Serialize;
+use deno_core::serde_json::json;
 use deno_core::url::Position;
 use deno_core::ModuleSpecifier;
 use import_map::ImportMap;
@@ -134,12 +137,14 @@ fn to_narrow_lsp_range(
 /// Given a specifier, a position, and a snapshot, optionally return a
 /// completion response, which will be valid import completions for the specific
 /// context.
+#[allow(clippy::too_many_arguments)]
 pub async fn get_import_completions(
   specifier: &ModuleSpecifier,
   position: &lsp::Position,
   config: &ConfigSnapshot,
   client: &Client,
   module_registries: &ModuleRegistry,
+  npm_search_api: &CliNpmSearchApi,
   documents: &Documents,
   maybe_import_map: Option<Arc<ImportMap>>,
 ) -> Option<lsp::CompletionResponse> {
@@ -161,6 +166,11 @@ pub async fn get_import_completions(
       is_incomplete: false,
       items: get_local_completions(specifier, &text, &range)?,
     }))
+  } else if text.starts_with("npm:") {
+    Some(lsp::CompletionResponse::List(lsp::CompletionList {
+      is_incomplete: false,
+      items: get_npm_completions(&text, &range, npm_search_api).await?,
+    }))
   } else if !text.is_empty() {
     // completion of modules from a module registry or cache
     check_auto_config_registry(&text, config, client, module_registries).await;
@@ -452,6 +462,113 @@ fn get_relative_specifiers(
     .collect()
 }
 
+/// Get completions for `npm:` specifiers.
+async fn get_npm_completions(
+  specifier: &str,
+  range: &lsp::Range,
+  npm_search_api: &impl NpmSearchApi,
+) -> Option<Vec<lsp::CompletionItem>> {
+  debug_assert!(specifier.starts_with("npm:"));
+  let bare_specifier = &specifier[4..];
+
+  // Find the index of the '@' delimiting the package name and version, if any.
+  let v_index = if bare_specifier.starts_with('@') {
+    bare_specifier
+      .find('/')
+      .filter(|idx| !bare_specifier[1..*idx].is_empty())
+      .and_then(|idx| {
+        bare_specifier[idx..]
+          .find('@')
+          .filter(|idx2| !bare_specifier[(idx + 1)..*idx2].is_empty())
+          .filter(|idx2| !bare_specifier[(idx + 1)..*idx2].contains('/'))
+      })
+  } else {
+    bare_specifier
+      .find('@')
+      .filter(|idx| !bare_specifier[..*idx].is_empty())
+      .filter(|idx| !bare_specifier[..*idx].contains('/'))
+  };
+
+  // First try to match `npm:some-package@<version-to-complete>`.
+  if let Some(v_index) = v_index {
+    let package_name = &bare_specifier[..v_index];
+    let v_prefix = &bare_specifier[(v_index + 1)..];
+    let versions = &npm_search_api
+      .package_info(package_name)
+      .await
+      .ok()?
+      .versions;
+    let mut versions = versions.keys().collect::<Vec<_>>();
+    versions.sort();
+    let items = versions
+      .into_iter()
+      .rev()
+      .enumerate()
+      .filter_map(|(idx, version)| {
+        let version = version.to_string();
+        if !version.starts_with(v_prefix) {
+          return None;
+        }
+        let specifier = format!("npm:{}@{}", package_name, &version);
+        let command = Some(lsp::Command {
+          title: "".to_string(),
+          command: "deno.cache".to_string(),
+          arguments: Some(vec![json!([&specifier])]),
+        });
+        let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+          range: *range,
+          new_text: specifier.clone(),
+        }));
+        Some(lsp::CompletionItem {
+          label: specifier,
+          kind: Some(lsp::CompletionItemKind::FILE),
+          detail: Some("(npm)".to_string()),
+          sort_text: Some(format!("{:0>10}", idx + 1)),
+          text_edit,
+          command,
+          commit_characters: Some(
+            IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(),
+          ),
+          ..Default::default()
+        })
+      })
+      .collect();
+    return Some(items);
+  }
+
+  // Otherwise match `npm:<package-to-complete>`.
+  let names = npm_search_api.search(bare_specifier).await.ok()?;
+  let items = names
+    .iter()
+    .enumerate()
+    .map(|(idx, name)| {
+      let specifier = format!("npm:{}", name);
+      let command = Some(lsp::Command {
+        title: "".to_string(),
+        command: "deno.cache".to_string(),
+        arguments: Some(vec![json!([&specifier])]),
+      });
+      let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+        range: *range,
+        new_text: specifier.clone(),
+      }));
+      lsp::CompletionItem {
+        label: specifier,
+        kind: Some(lsp::CompletionItemKind::FILE),
+        detail: Some("(npm)".to_string()),
+        sort_text: Some(format!("{:0>10}", idx + 1)),
+        text_edit,
+        command,
+        commit_characters: Some(
+          IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(),
+        ),
+        ..Default::default()
+      }
+    })
+    .collect();
+  Some(items)
+}
+
 /// Get workspace completions that include modules in the Deno cache which match
 /// the current specifier string.
 fn get_workspace_completions(
@@ -509,12 +626,41 @@ mod tests {
   use crate::cache::HttpCache;
   use crate::lsp::documents::Documents;
   use crate::lsp::documents::LanguageId;
+  use crate::lsp::npm::NpmSearchApi;
+  use crate::AnyError;
+  use async_trait::async_trait;
   use deno_core::resolve_url;
   use deno_graph::Range;
+  use deno_npm::registry::NpmPackageInfo;
+  use deno_npm::registry::NpmRegistryApi;
+  use deno_npm::registry::TestNpmRegistryApi;
   use std::collections::HashMap;
   use std::path::Path;
   use test_util::TempDir;
 
+  #[derive(Default)]
+  struct TestNpmSearchApi(
+    HashMap<String, Arc<Vec<String>>>,
+    TestNpmRegistryApi,
+  );
+
+  #[async_trait]
+  impl NpmSearchApi for TestNpmSearchApi {
+    async fn search(&self, query: &str) -> Result<Arc<Vec<String>>, AnyError> {
+      match self.0.get(query) {
+        Some(names) => Ok(names.clone()),
+        None => Ok(Arc::new(vec![])),
+      }
+    }
+
+    async fn package_info(
+      &self,
+      name: &str,
+    ) -> Result<Arc<NpmPackageInfo>, AnyError> {
+      self.1.package_info(name).await.map_err(|e| e.into())
+    }
+  }
+
   fn mock_documents(
     fixtures: &[(&str, &str, i32, LanguageId)],
     source_fixtures: &[(&str, &str)],
@@ -682,6 +828,231 @@ mod tests {
     );
   }
 
+  #[tokio::test]
+  async fn test_get_npm_completions() {
+    let npm_search_api = TestNpmSearchApi(
+      vec![(
+        "puppe".to_string(),
+        Arc::new(vec![
+          "puppeteer".to_string(),
+          "puppeteer-core".to_string(),
+          "puppeteer-extra-plugin-stealth".to_string(),
+          "puppeteer-extra-plugin".to_string(),
+        ]),
+      )]
+      .into_iter()
+      .collect(),
+      Default::default(),
+    );
+    let range = lsp::Range {
+      start: lsp::Position {
+        line: 0,
+        character: 23,
+      },
+      end: lsp::Position {
+        line: 0,
+        character: 32,
+      },
+    };
+    let actual = get_npm_completions("npm:puppe", &range, &npm_search_api)
+      .await
+      .unwrap();
+    assert_eq!(
+      actual,
+      vec![
+        lsp::CompletionItem {
+          label: "npm:puppeteer".to_string(),
+          kind: Some(lsp::CompletionItemKind::FILE),
+          detail: Some("(npm)".to_string()),
+          sort_text: Some("0000000001".to_string()),
+          text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+            range,
+            new_text: "npm:puppeteer".to_string(),
+          })),
+          command: Some(lsp::Command {
+            title: "".to_string(),
+            command: "deno.cache".to_string(),
+            arguments: Some(vec![json!(["npm:puppeteer"])])
+          }),
+          commit_characters: Some(
+            IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect()
+          ),
+          ..Default::default()
+        },
+        lsp::CompletionItem {
+          label: "npm:puppeteer-core".to_string(),
+          kind: Some(lsp::CompletionItemKind::FILE),
+          detail: Some("(npm)".to_string()),
+          sort_text: Some("0000000002".to_string()),
+          text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+            range,
+            new_text: "npm:puppeteer-core".to_string(),
+          })),
+          command: Some(lsp::Command {
+            title: "".to_string(),
+            command: "deno.cache".to_string(),
+            arguments: Some(vec![json!(["npm:puppeteer-core"])])
+          }),
+          commit_characters: Some(
+            IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect()
+          ),
+          ..Default::default()
+        },
+        lsp::CompletionItem {
+          label: "npm:puppeteer-extra-plugin-stealth".to_string(),
+          kind: Some(lsp::CompletionItemKind::FILE),
+          detail: Some("(npm)".to_string()),
+          sort_text: Some("0000000003".to_string()),
+          text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+            range,
+            new_text: "npm:puppeteer-extra-plugin-stealth".to_string(),
+          })),
+          command: Some(lsp::Command {
+            title: "".to_string(),
+            command: "deno.cache".to_string(),
+            arguments: Some(vec![json!([
+              "npm:puppeteer-extra-plugin-stealth"
+            ])])
+          }),
+          commit_characters: Some(
+            IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect()
+          ),
+          ..Default::default()
+        },
+        lsp::CompletionItem {
+          label: "npm:puppeteer-extra-plugin".to_string(),
+          kind: Some(lsp::CompletionItemKind::FILE),
+          detail: Some("(npm)".to_string()),
+          sort_text: Some("0000000004".to_string()),
+          text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+            range,
+            new_text: "npm:puppeteer-extra-plugin".to_string(),
+          })),
+          command: Some(lsp::Command {
+            title: "".to_string(),
+            command: "deno.cache".to_string(),
+            arguments: Some(vec![json!(["npm:puppeteer-extra-plugin"])])
+          }),
+          commit_characters: Some(
+            IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect()
+          ),
+          ..Default::default()
+        },
+      ]
+    );
+  }
+
+  #[tokio::test]
+  async fn test_get_npm_completions_for_versions() {
+    let npm_search_api = TestNpmSearchApi::default();
+    npm_search_api
+      .1
+      .ensure_package_version("puppeteer", "20.9.0");
+    npm_search_api
+      .1
+      .ensure_package_version("puppeteer", "21.0.0");
+    npm_search_api
+      .1
+      .ensure_package_version("puppeteer", "21.0.1");
+    npm_search_api
+      .1
+      .ensure_package_version("puppeteer", "21.0.2");
+    let range = lsp::Range {
+      start: lsp::Position {
+        line: 0,
+        character: 23,
+      },
+      end: lsp::Position {
+        line: 0,
+        character: 37,
+      },
+    };
+    let actual = get_npm_completions("npm:puppeteer@", &range, &npm_search_api)
+      .await
+      .unwrap();
+    assert_eq!(
+      actual,
+      vec![
+        lsp::CompletionItem {
+          label: "npm:puppeteer@21.0.2".to_string(),
+          kind: Some(lsp::CompletionItemKind::FILE),
+          detail: Some("(npm)".to_string()),
+          sort_text: Some("0000000001".to_string()),
+          text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+            range,
+            new_text: "npm:puppeteer@21.0.2".to_string(),
+          })),
+          command: Some(lsp::Command {
+            title: "".to_string(),
+            command: "deno.cache".to_string(),
+            arguments: Some(vec![json!(["npm:puppeteer@21.0.2"])])
+          }),
+          commit_characters: Some(
+            IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect()
+          ),
+          ..Default::default()
+        },
+        lsp::CompletionItem {
+          label: "npm:puppeteer@21.0.1".to_string(),
+          kind: Some(lsp::CompletionItemKind::FILE),
+          detail: Some("(npm)".to_string()),
+          sort_text: Some("0000000002".to_string()),
+          text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+            range,
+            new_text: "npm:puppeteer@21.0.1".to_string(),
+          })),
+          command: Some(lsp::Command {
+            title: "".to_string(),
+            command: "deno.cache".to_string(),
+            arguments: Some(vec![json!(["npm:puppeteer@21.0.1"])])
+          }),
+          commit_characters: Some(
+            IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect()
+          ),
+          ..Default::default()
+        },
+        lsp::CompletionItem {
+          label: "npm:puppeteer@21.0.0".to_string(),
+          kind: Some(lsp::CompletionItemKind::FILE),
+          detail: Some("(npm)".to_string()),
+          sort_text: Some("0000000003".to_string()),
+          text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+            range,
+            new_text: "npm:puppeteer@21.0.0".to_string(),
+          })),
+          command: Some(lsp::Command {
+            title: "".to_string(),
+            command: "deno.cache".to_string(),
+            arguments: Some(vec![json!(["npm:puppeteer@21.0.0"])])
+          }),
+          commit_characters: Some(
+            IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect()
+          ),
+          ..Default::default()
+        },
+        lsp::CompletionItem {
+          label: "npm:puppeteer@20.9.0".to_string(),
+          kind: Some(lsp::CompletionItemKind::FILE),
+          detail: Some("(npm)".to_string()),
+          sort_text: Some("0000000004".to_string()),
+          text_edit: Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
+            range,
+            new_text: "npm:puppeteer@20.9.0".to_string(),
+          })),
+          command: Some(lsp::Command {
+            title: "".to_string(),
+            command: "deno.cache".to_string(),
+            arguments: Some(vec![json!(["npm:puppeteer@20.9.0"])])
+          }),
+          commit_characters: Some(
+            IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect()
+          ),
+          ..Default::default()
+        },
+      ]
+    );
+  }
+
   #[test]
   fn test_to_narrow_lsp_range() {
     let text_info = SourceTextInfo::from_string(r#""te""#.to_string());

cli/lsp/language_server.rs

@@ -59,6 +59,7 @@ use super::documents::UpdateDocumentConfigOptions;
 use super::logging::lsp_log;
 use super::logging::lsp_warn;
 use super::lsp_custom;
+use super::npm::CliNpmSearchApi;
 use super::parent_process_checker;
 use super::performance::Performance;
 use super::performance::PerformanceMark;
@@ -123,6 +124,8 @@ struct LspNpmServices {
   config_hash: LspNpmConfigHash,
   /// Npm's registry api.
   api: Arc<CliNpmRegistryApi>,
+  /// Npm's search api.
+  search_api: CliNpmSearchApi,
   /// Npm cache
   cache: Arc<NpmCache>,
   /// Npm resolution that is stored in memory.
@@ -556,6 +559,8 @@ impl Inner {
       module_registries_location.clone(),
       http_client.clone(),
     );
+    let npm_search_api =
+      CliNpmSearchApi::new(module_registries.file_fetcher.clone(), None);
     let location = dir.deps_folder_path();
     let deps_http_cache = Arc::new(GlobalHttpCache::new(
       location,
@@ -612,6 +617,7 @@ impl Inner {
       npm: LspNpmServices {
         config_hash: LspNpmConfigHash(0), // this will be updated in initialize
         api: npm_api,
+        search_api: npm_search_api,
         cache: npm_cache,
         resolution: npm_resolution,
         resolver: npm_resolver,
@@ -2345,6 +2351,7 @@ impl Inner {
       &self.config.snapshot(),
       &self.client,
       &self.module_registries,
+      &self.npm.search_api,
       &self.documents,
       self.maybe_import_map.clone(),
     )

cli/lsp/mod.rs

@@ -22,6 +22,7 @@ mod documents;
 pub mod language_server;
 mod logging;
 mod lsp_custom;
+mod npm;
 mod parent_process_checker;
 mod path_to_regex;
 mod performance;

cli/lsp/npm.rs

@@ -0,0 +1,136 @@
+// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use deno_core::anyhow::anyhow;
+use deno_core::error::AnyError;
+use deno_core::parking_lot::Mutex;
+use deno_core::serde_json;
+use deno_core::url::Url;
+use deno_npm::registry::NpmPackageInfo;
+use deno_runtime::permissions::PermissionsContainer;
+use serde::Deserialize;
+
+use crate::file_fetcher::FileFetcher;
+use crate::npm::CliNpmRegistryApi;
+
+#[async_trait::async_trait]
+pub trait NpmSearchApi {
+  async fn search(&self, query: &str) -> Result<Arc<Vec<String>>, AnyError>;
+  async fn package_info(
+    &self,
+    name: &str,
+  ) -> Result<Arc<NpmPackageInfo>, AnyError>;
+}
+
+#[derive(Debug, Clone)]
+pub struct CliNpmSearchApi {
+  base_url: Url,
+  file_fetcher: FileFetcher,
+  info_cache: Arc<Mutex<HashMap<String, Arc<NpmPackageInfo>>>>,
+  search_cache: Arc<Mutex<HashMap<String, Arc<Vec<String>>>>>,
+}
+
+impl CliNpmSearchApi {
+  pub fn new(file_fetcher: FileFetcher, custom_base_url: Option<Url>) -> Self {
+    Self {
+      base_url: custom_base_url
+        .unwrap_or_else(|| CliNpmRegistryApi::default_url().clone()),
+      file_fetcher,
+      info_cache: Default::default(),
+      search_cache: Default::default(),
+    }
+  }
+}
+
+#[async_trait::async_trait]
+impl NpmSearchApi for CliNpmSearchApi {
+  async fn search(&self, query: &str) -> Result<Arc<Vec<String>>, AnyError> {
+    if let Some(names) = self.search_cache.lock().get(query) {
+      return Ok(names.clone());
+    }
+    let mut search_url = self.base_url.clone();
+    search_url
+      .path_segments_mut()
+      .map_err(|_| anyhow!("Custom npm registry URL cannot be a base."))?
+      .pop_if_empty()
+      .extend("-/v1/search".split('/'));
+    search_url
+      .query_pairs_mut()
+      .append_pair("text", &format!("{} boost-exact:false", query));
+    let file = self
+      .file_fetcher
+      .fetch(&search_url, PermissionsContainer::allow_all())
+      .await?;
+    let names = Arc::new(parse_npm_search_response(&file.source)?);
+    self
+      .search_cache
+      .lock()
+      .insert(query.to_string(), names.clone());
+    Ok(names)
+  }
+
+  async fn package_info(
+    &self,
+    name: &str,
+  ) -> Result<Arc<NpmPackageInfo>, AnyError> {
+    if let Some(info) = self.info_cache.lock().get(name) {
+      return Ok(info.clone());
+    }
+    let mut info_url = self.base_url.clone();
+    info_url
+      .path_segments_mut()
+      .map_err(|_| anyhow!("Custom npm registry URL cannot be a base."))?
+      .pop_if_empty()
+      .push(name);
+    let file = self
+      .file_fetcher
+      .fetch(&info_url, PermissionsContainer::allow_all())
+      .await?;
+    let info = Arc::new(serde_json::from_str::<NpmPackageInfo>(&file.source)?);
+    self
+      .info_cache
+      .lock()
+      .insert(name.to_string(), info.clone());
+    Ok(info)
+  }
+}
+
+fn parse_npm_search_response(source: &str) -> Result<Vec<String>, AnyError> {
+  #[derive(Debug, Deserialize)]
+  struct Package {
+    name: String,
+  }
+  #[derive(Debug, Deserialize)]
+  struct Object {
+    package: Package,
+  }
+  #[derive(Debug, Deserialize)]
+  struct Response {
+    objects: Vec<Object>,
+  }
+  let objects = serde_json::from_str::<Response>(source)?.objects;
+  Ok(objects.into_iter().map(|o| o.package.name).collect())
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn test_parse_npm_search_response() {
+    // This is a subset of a realistic response only containing data currently
+    // used by our parser. It's enough to catch regressions.
+    let names = parse_npm_search_response(r#"{"objects":[{"package":{"name":"puppeteer"}},{"package":{"name":"puppeteer-core"}},{"package":{"name":"puppeteer-extra-plugin-stealth"}},{"package":{"name":"puppeteer-extra-plugin"}}]}"#).unwrap();
+    assert_eq!(
+      names,
+      vec![
+        "puppeteer".to_string(),
+        "puppeteer-core".to_string(),
+        "puppeteer-extra-plugin-stealth".to_string(),
+        "puppeteer-extra-plugin".to_string()
+      ]
+    );
+  }
+}

cli/lsp/registries.rs

@@ -415,7 +415,7 @@ enum VariableItems {
 #[derive(Debug, Clone)]
 pub struct ModuleRegistry {
   origins: HashMap<String, Vec<RegistryConfiguration>>,
-  file_fetcher: FileFetcher,
+  pub file_fetcher: FileFetcher,
   http_cache: Arc<GlobalHttpCache>,
 }