foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-11 16:42:21 -05:00
Code Issues Packages 1 Wiki Activity

feat(lsp): add registry import auto-complete (#9934)

Browse source
This commit is contained in:
Kitson Kelly 2021-04-09 11:27:27 +10:00 committed by GitHub
parent 3168fa4ee7
commit d9d4a5d73c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
34 changed files with 2358 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
Cargo.lock generated
View file

@ -518,6 +518,7 @@ dependencies = [
"encoding_rs",
"env_logger",
"exec",
"fancy-regex",
"filetime",
"fwdansi",
"http",
@ -956,6 +957,16 @@ dependencies = [
"libc",
]
[[package]]
name = "fancy-regex"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe09872bd11351a75f22b24c3769fc863e8212d926d6db46b94ad710d14cc5cc"
dependencies = [
"bit-set",
"regex",
]
[[package]]
name = "filetime"
version = "0.2.14"

1
cli/Cargo.toml
View file

@ -49,6 +49,7 @@ dprint-plugin-markdown = "0.6.2"
dprint-plugin-typescript = "0.41.0"
encoding_rs = "0.8.28"
env_logger = "0.8.3"
fancy-regex = "0.5.0"
filetime = "0.2.14"
http = "0.2.3"
indexmap = { version = "1.6.2", features = ["serde"] }

4
cli/file_fetcher.rs
View file

@ -58,7 +58,7 @@ pub struct File {
/// Simple struct implementing in-process caching to prevent multiple
/// fs reads/net fetches for same file.
#[derive(Clone, Default)]
#[derive(Debug, Clone, Default)]
struct FileCache(Arc<Mutex<HashMap<ModuleSpecifier, File>>>);
impl FileCache {
@ -312,7 +312,7 @@ fn strip_shebang(mut value: String) -> String {
}
/// A structure for resolving, fetching and caching source files.
#[derive(Clone)]
#[derive(Debug, Clone)]
pub struct FileFetcher {
auth_tokens: AuthTokens,
allow_remote: bool,

4
cli/lsp/README.md
View file

@ -37,6 +37,10 @@ with Deno:
- `deno/performance` - Requests the return of the timing averages for the
internal instrumentation of Deno.
It does not expect any parameters.
- `deno/reloadImportRegistries` - Reloads any cached responses from import
registries.
It does not expect any parameters.
- `deno/virtualTextDocument` - Requests a virtual text document from the LSP,
which is a read only document that can be displayed in the client. This allows

55
cli/lsp/completions.rs
View file

@ -25,6 +25,7 @@ use swc_ecmascript::visit::VisitWith;
const CURRENT_PATH: &str = ".";
const PARENT_PATH: &str = "..";
const LOCAL_PATHS: &[&str] = &[CURRENT_PATH, PARENT_PATH];
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@ -36,7 +37,7 @@ pub struct CompletionItemData {
/// Given a specifier, a position, and a snapshot, optionally return a
/// completion response, which will be valid import completions for the specific
/// context.
pub fn get_import_completions(
pub async fn get_import_completions(
specifier: &ModuleSpecifier,
position: &lsp::Position,
state_snapshot: &language_server::StateSnapshot,
@ -55,17 +56,52 @@ pub fn get_import_completions(
items: get_local_completions(specifier, &current_specifier, &range)?,
}));
}
// completion of modules within the workspace
// completion of modules from a module registry or cache
if !current_specifier.is_empty() {
return Some(lsp::CompletionResponse::List(lsp::CompletionList {
is_incomplete: false,
items: get_workspace_completions(
let offset = if position.character > range.start.character {
(position.character - range.start.character) as usize
} else {
0
};
let maybe_items = state_snapshot
.module_registries
.get_completions(&current_specifier, offset, &range, state_snapshot)
.await;
let items = maybe_items.unwrap_or_else(|| {
get_workspace_completions(
specifier,
&current_specifier,
&range,
state_snapshot,
),
)
});
return Some(lsp::CompletionResponse::List(lsp::CompletionList {
is_incomplete: false,
items,
}));
} else {
let mut items: Vec<lsp::CompletionItem> = LOCAL_PATHS
.iter()
.map(|s| lsp::CompletionItem {
label: s.to_string(),
kind: Some(lsp::CompletionItemKind::Folder),
detail: Some("(local)".to_string()),
sort_text: Some("1".to_string()),
insert_text: Some(s.to_string()),
..Default::default()
})
.collect();
if let Some(origin_items) = state_snapshot
.module_registries
.get_origin_completions(&current_specifier, &range)
{
items.extend(origin_items);
}
return Some(lsp::CompletionResponse::List(lsp::CompletionList {
is_incomplete: false,
items,
}));
// TODO(@kitsonk) add bare specifiers from import map
}
}
}
@ -738,8 +774,8 @@ mod tests {
}
}
#[test]
fn test_get_import_completions() {
#[tokio::test]
async fn test_get_import_completions() {
let specifier = resolve_url("file:///a/b/c.ts").unwrap();
let position = lsp::Position {
line: 0,
@ -752,7 +788,8 @@ mod tests {
],
&[("https://deno.land/x/a/b/c.ts", "console.log(1);\n")],
);
let actual = get_import_completions(&specifier, &position, &state_snapshot);
let actual =
get_import_completions(&specifier, &position, &state_snapshot).await;
assert_eq!(
actual,
Some(lsp::CompletionResponse::List(lsp::CompletionList {

19
cli/lsp/config.rs
View file

@ -7,6 +7,7 @@ use deno_core::url::Url;
use lspower::jsonrpc::Error as LSPError;
use lspower::jsonrpc::Result as LSPResult;
use lspower::lsp;
use std::collections::HashMap;
#[derive(Debug, Clone, Default)]
pub struct ClientCapabilities {
@ -52,6 +53,8 @@ pub struct CompletionSettings {
pub paths: bool,
#[serde(default)]
pub auto_imports: bool,
#[serde(default)]
pub imports: ImportCompletionSettings,
}
impl Default for CompletionSettings {
@ -61,6 +64,22 @@ impl Default for CompletionSettings {
names: true,
paths: true,
auto_imports: true,
imports: ImportCompletionSettings::default(),
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ImportCompletionSettings {
#[serde(default)]
pub hosts: HashMap<String, bool>,
}
impl Default for ImportCompletionSettings {
fn default() -> Self {
Self {
hosts: HashMap::default(),
}
}
}

201
cli/lsp/language_server.rs
View file

@ -48,6 +48,7 @@ use super::diagnostics;
use super::diagnostics::DiagnosticSource;
use super::documents::DocumentCache;
use super::performance::Performance;
use super::registries;
use super::sources;
use super::sources::Sources;
use super::text;
@ -58,6 +59,9 @@ use super::tsc::Assets;
use super::tsc::TsServer;
use super::urls;
pub const REGISTRIES_PATH: &str = "registries";
const SOURCES_PATH: &str = "deps";
lazy_static::lazy_static! {
static ref ABSTRACT_MODIFIER: Regex = Regex::new(r"\babstract\b").unwrap();
static ref EXPORT_MODIFIER: Regex = Regex::new(r"\bexport\b").unwrap();
@ -71,6 +75,7 @@ pub struct StateSnapshot {
pub assets: Assets,
pub config: Config,
pub documents: DocumentCache,
pub module_registries: registries::ModuleRegistry,
pub performance: Performance,
pub sources: Sources,
}
@ -87,6 +92,10 @@ pub(crate) struct Inner {
diagnostics_server: diagnostics::DiagnosticsServer,
/// The "in-memory" documents in the editor which can be updated and changed.
documents: DocumentCache,
/// Handles module registries, which allow discovery of modules
module_registries: registries::ModuleRegistry,
/// The path to the module registries cache
module_registries_location: PathBuf,
/// An optional URL which provides the location of a TypeScript configuration
/// file which will be used by the Deno LSP.
maybe_config_uri: Option<Url>,
@ -119,8 +128,11 @@ impl Inner {
let maybe_custom_root = env::var("DENO_DIR").map(String::into).ok();
let dir = deno_dir::DenoDir::new(maybe_custom_root)
.expect("could not access DENO_DIR");
let location = dir.root.join("deps");
let sources = Sources::new(&location);
let module_registries_location = dir.root.join(REGISTRIES_PATH);
let module_registries =
registries::ModuleRegistry::new(&module_registries_location);
let sources_location = dir.root.join(SOURCES_PATH);
let sources = Sources::new(&sources_location);
let ts_server = Arc::new(TsServer::new());
let performance = Performance::default();
let diagnostics_server = diagnostics::DiagnosticsServer::new();
@ -134,6 +146,8 @@ impl Inner {
maybe_config_uri: Default::default(),
maybe_import_map: Default::default(),
maybe_import_map_uri: Default::default(),
module_registries,
module_registries_location,
navigation_trees: Default::default(),
performance,
sources,
@ -276,6 +290,7 @@ impl Inner {
assets: self.assets.clone(),
config: self.config.clone(),
documents: self.documents.clone(),
module_registries: self.module_registries.clone(),
performance: self.performance.clone(),
sources: self.sources.clone(),
}
@ -328,6 +343,22 @@ impl Inner {
Ok(())
}
async fn update_registries(&mut self) -> Result<(), AnyError> {
let mark = self.performance.mark("update_registries");
for (registry, enabled) in self.config.settings.suggest.imports.hosts.iter()
{
if *enabled {
info!("Enabling auto complete registry for: {}", registry);
self.module_registries.enable(registry).await?;
} else {
info!("Disabling auto complete registry for: {}", registry);
self.module_registries.disable(registry).await?;
}
}
self.performance.measure(mark);
Ok(())
}
async fn update_tsconfig(&mut self) -> Result<(), AnyError> {
let mark = self.performance.mark("update_tsconfig");
let mut tsconfig = TsConfig::new(json!({
@ -495,6 +526,13 @@ impl Inner {
.show_message(MessageType::Warning, err.to_string())
.await;
}
// Check to see if we need to setup any module registries
if let Err(err) = self.update_registries().await {
self
.client
.show_message(MessageType::Warning, err.to_string())
.await;
}
if self
.config
@ -628,6 +666,12 @@ impl Inner {
.show_message(MessageType::Warning, err.to_string())
.await;
}
if let Err(err) = self.update_registries().await {
self
.client
.show_message(MessageType::Warning, err.to_string())
.await;
}
if let Err(err) = self.update_tsconfig().await {
self
.client
@ -1394,7 +1438,9 @@ impl Inner {
&specifier,
&params.text_document_position.position,
&self.snapshot(),
) {
)
.await
{
Some(response)
} else {
let line_index =
@ -1659,16 +1705,12 @@ impl Inner {
) -> LspResult<Option<Value>> {
match method {
"deno/cache" => match params.map(serde_json::from_value) {
Some(Ok(params)) => Ok(Some(
serde_json::to_value(self.cache(params).await?).map_err(|err| {
error!("Failed to serialize cache response: {}", err);
LspError::internal_error()
})?,
)),
Some(Ok(params)) => self.cache(params).await,
Some(Err(err)) => Err(LspError::invalid_params(err.to_string())),
None => Err(LspError::invalid_params("Missing parameters")),
},
"deno/performance" => Ok(Some(self.get_performance())),
"deno/reloadImportRegistries" => self.reload_import_registries().await,
"deno/virtualTextDocument" => match params.map(serde_json::from_value) {
Some(Ok(params)) => Ok(Some(
serde_json::to_value(self.virtual_text_document(params).await?)
@ -1979,7 +2021,7 @@ struct VirtualTextDocumentParams {
impl Inner {
/// Similar to `deno cache` on the command line, where modules will be cached
/// in the Deno cache, including any of their dependencies.
async fn cache(&mut self, params: CacheParams) -> LspResult<bool> {
async fn cache(&mut self, params: CacheParams) -> LspResult<Option<Value>> {
let mark = self.performance.mark("cache");
let referrer = self.url_map.normalize_url(&params.referrer.uri);
if !params.uris.is_empty() {
@ -2020,7 +2062,7 @@ impl Inner {
LspError::internal_error()
})?;
self.performance.measure(mark);
Ok(true)
Ok(Some(json!(true)))
}
fn get_performance(&self) -> Value {
@ -2028,6 +2070,22 @@ impl Inner {
json!({ "averages": averages })
}
async fn reload_import_registries(&mut self) -> LspResult<Option<Value>> {
fs::remove_dir_all(&self.module_registries_location)
.await
.map_err(|err| {
error!("Unable to remove registries cache: {}", err);
LspError::internal_error()
})?;
self.module_registries =
registries::ModuleRegistry::new(&self.module_registries_location);
self.update_registries().await.map_err(|err| {
error!("Unable to update registries: {}", err);
LspError::internal_error()
})?;
Ok(Some(json!(true)))
}
async fn virtual_text_document(
&mut self,
params: VirtualTextDocumentParams,
@ -3223,6 +3281,127 @@ mod tests {
harness.run().await;
}
#[tokio::test]
async fn test_completions_registry() {
let _g = test_util::http_server();
let mut harness = LspTestHarness::new(vec![
("initialize_request_registry.json", LspResponse::RequestAny),
("initialized_notification.json", LspResponse::None),
(
"did_open_notification_completion_registry.json",
LspResponse::None,
),
(
"completion_request_registry.json",
LspResponse::RequestAssert(|value| {
let response: CompletionResult =
serde_json::from_value(value).unwrap();
let result = response.result.unwrap();
if let CompletionResponse::List(list) = result {
assert_eq!(list.items.len(), 3);
} else {
panic!("unexpected result");
}
}),
),
(
"completion_resolve_request_registry.json",
LspResponse::Request(
4,
json!({
"label": "v2.0.0",
"kind": 19,
"detail": "(version)",
"sortText": "0000000003",
"filterText": "http://localhost:4545/x/a@v2.0.0",
"textEdit": {
"range": {
"start": {
"line": 0,
"character": 20
},
"end": {
"line": 0,
"character": 46
}
},
"newText": "http://localhost:4545/x/a@v2.0.0"
}
}),
),
),
(
"shutdown_request.json",
LspResponse::Request(3, json!(null)),
),
("exit_notification.json", LspResponse::None),
]);
harness.run().await;
}
#[tokio::test]
async fn test_completion_registry_empty_specifier() {
let _g = test_util::http_server();
let mut harness = LspTestHarness::new(vec![
("initialize_request_registry.json", LspResponse::RequestAny),
("initialized_notification.json", LspResponse::None),
(
"did_open_notification_completion_registry_02.json",
LspResponse::None,
),
(
"completion_request_registry_02.json",
LspResponse::Request(
2,
json!({
"isIncomplete": false,
"items": [
{
"label": ".",
"kind": 19,
"detail": "(local)",
"sortText": "1",
"insertText": "."
},
{
"label": "..",
"kind": 19,
"detail": "(local)",
"sortText": "1",
"insertText": ".."
},
{
"label": "http://localhost:4545",
"kind": 19,
"detail": "(registry)",
"sortText": "2",
"textEdit": {
"range": {
"start": {
"line": 0,
"character": 20
},
"end": {
"line": 0,
"character": 20
}
},
"newText": "http://localhost:4545"
}
}
]
}),
),
),
(
"shutdown_request.json",
LspResponse::Request(3, json!(null)),
),
("exit_notification.json", LspResponse::None),
]);
harness.run().await;
}
#[derive(Deserialize)]
struct PerformanceAverages {
averages: Vec<PerformanceAverage>,

4
cli/lsp/mod.rs
View file

@ -9,8 +9,10 @@ mod completions;
mod config;
mod diagnostics;
mod documents;
mod language_server;
pub(crate) mod language_server;
mod path_to_regex;
mod performance;
mod registries;
mod sources;
mod text;
mod tsc;

961
cli/lsp/path_to_regex.rs Normal file
View file

@ -0,0 +1,961 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
// The logic of this module is heavily influenced by path-to-regexp at:
// https://github.com/pillarjs/path-to-regexp/ which is licensed as follows:
// The MIT License (MIT)
//
// Copyright (c) 2014 Blake Embrey (hello@blakeembrey.com)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
use deno_core::error::anyhow;
use deno_core::error::AnyError;
use fancy_regex::Regex as FancyRegex;
use regex::Regex;
use std::collections::HashMap;
use std::fmt;
use std::iter::Peekable;
lazy_static::lazy_static! {
static ref ESCAPE_STRING_RE: Regex =
Regex::new(r"([.+*?=^!:${}()\[\]|/\\])").unwrap();
}
#[derive(Debug, PartialEq, Eq)]
enum TokenType {
Open,
Close,
Pattern,
Name,
Char,
EscapedChar,
Modifier,
End,
}
#[derive(Debug)]
struct LexToken {
token_type: TokenType,
index: usize,
value: String,
}
fn escape_string(s: &str) -> String {
ESCAPE_STRING_RE.replace_all(s, r"\$1").to_string()
}
fn lexer(s: &str) -> Result<Vec<LexToken>, AnyError> {
let mut tokens = Vec::new();
let mut chars = s.chars().peekable();
let mut index = 0_usize;
loop {
match chars.next() {
None => break,
Some(c) if c == '*' || c == '+' || c == '?' => {
tokens.push(LexToken {
token_type: TokenType::Modifier,
index,
value: c.to_string(),
});
index += 1;
}
Some('\\') => {
index += 1;
let value = chars
.next()
.ok_or_else(|| anyhow!("Unexpected end of string at {}.", index))?;
tokens.push(LexToken {
token_type: TokenType::EscapedChar,
index,
value: value.to_string(),
});
index += 1;
}
Some('{') => {
tokens.push(LexToken {
token_type: TokenType::Open,
index,
value: '{'.to_string(),
});
index += 1;
}
Some('}') => {
tokens.push(LexToken {
token_type: TokenType::Close,
index,
value: '}'.to_string(),
});
index += 1;
}
Some(':') => {
let mut name = String::new();
while let Some(c) = chars.peek() {
if (*c >= '0' && *c <= '9')
|| (*c >= 'A' && *c <= 'Z')
|| (*c >= 'a' && *c <= 'z')
|| *c == '_'
{
let ch = chars.next().unwrap();
name.push(ch);
} else {
break;
}
}
if name.is_empty() {
return Err(anyhow!("Missing parameter name at {}", index));
}
let name_len = name.len();
tokens.push(LexToken {
token_type: TokenType::Name,
index,
value: name,
});
index += 1 + name_len;
}
Some('(') => {
let mut count = 1;
let mut pattern = String::new();
if chars.peek() == Some(&'?') {
return Err(anyhow!(
"Pattern cannot start with \"?\" at {}.",
index + 1
));
}
loop {
let next_char = chars.peek();
if next_char.is_none() {
break;
}
if next_char == Some(&'\\') {
pattern.push(chars.next().unwrap());
pattern.push(
chars
.next()
.ok_or_else(|| anyhow!("Unexpected termination of string."))?,
);
continue;
}
if next_char == Some(&')') {
count -= 1;
if count == 0 {
chars.next();
break;
}
} else if next_char == Some(&'(') {
count += 1;
pattern.push(chars.next().unwrap());
if chars.peek() != Some(&'?') {
return Err(anyhow!(
"Capturing groups are not allowed at {}.",
index + pattern.len()
));
}
continue;
}
pattern.push(chars.next().unwrap());
}
if count > 0 {
return Err(anyhow!("Unbalanced pattern at {}.", index));
}
if pattern.is_empty() {
return Err(anyhow!("Missing pattern at {}.", index));
}
let pattern_len = pattern.len();
tokens.push(LexToken {
token_type: TokenType::Pattern,
index,
value: pattern,
});
index += 2 + pattern_len;
}
Some(c) => {
tokens.push(LexToken {
token_type: TokenType::Char,
index,
value: c.to_string(),
});
index += 1;
}
}
}
tokens.push(LexToken {
token_type: TokenType::End,
index,
value: "".to_string(),
});
Ok(tokens)
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum StringOrNumber {
String(String),
Number(usize),
}
impl fmt::Display for StringOrNumber {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match &self {
Self::Number(n) => write!(f, "{}", n),
Self::String(s) => write!(f, "{}", s),
}
}
}
#[derive(Debug, Clone)]
pub enum StringOrVec {
String(String),
Vec(Vec<String>),
}
impl StringOrVec {
pub fn from_str(s: &str, key: &Key) -> StringOrVec {
match &key.modifier {
Some(m) if m == "+" || m == "*" => {
let pat = format!(
"{}{}",
key.prefix.clone().unwrap_or_default(),
key.suffix.clone().unwrap_or_default()
);
s.split(&pat)
.map(String::from)
.collect::<Vec<String>>()
.into()
}
_ => s.into(),
}
}
pub fn to_string(&self, maybe_key: Option<&Key>) -> String {
match self {
Self::String(s) => s.clone(),
Self::Vec(v) => {
let (prefix, suffix) = if let Some(key) = maybe_key {
(
key.prefix.clone().unwrap_or_default(),
key.suffix.clone().unwrap_or_default(),
)
} else {
("/".to_string(), "".to_string())
};
let mut s = String::new();
for segment in v {
s.push_str(&format!("{}{}{}", prefix, segment, suffix));
}
s
}
}
}
}
impl Default for StringOrVec {
fn default() -> Self {
Self::String("".to_string())
}
}
impl<'a> From<&'a str> for StringOrVec {
fn from(s: &'a str) -> Self {
Self::String(s.to_string())
}
}
impl From<Vec<String>> for StringOrVec {
fn from(v: Vec<String>) -> Self {
Self::Vec(v)
}
}
/// Meta data about a key.
#[derive(Debug, Clone)]
pub struct Key {
pub name: StringOrNumber,
pub prefix: Option<String>,
pub suffix: Option<String>,
pub pattern: String,
pub modifier: Option<String>,
}
/// A token is a string (nothing special) or key metadata (capture group).
#[derive(Debug, Clone)]
pub enum Token {
String(String),
Key(Key),
}
#[derive(Debug, Default)]
pub struct ParseOptions {
delimiter: Option<String>,
prefixes: Option<String>,
}
#[derive(Debug)]
pub struct TokensToCompilerOptions {
sensitive: bool,
validate: bool,
}
impl Default for TokensToCompilerOptions {
fn default() -> Self {
Self {
sensitive: false,
validate: true,
}
}
}
#[derive(Debug)]
pub struct TokensToRegexOptions {
sensitive: bool,
strict: bool,
end: bool,
start: bool,
delimiter: Option<String>,
ends_with: Option<String>,
}
impl Default for TokensToRegexOptions {
fn default() -> Self {
Self {
sensitive: false,
strict: false,
end: true,
start: true,
delimiter: None,
ends_with: None,
}
}
}
#[derive(Debug, Default)]
pub struct PathToRegexOptions {
parse_options: Option<ParseOptions>,
token_to_regex_options: Option<TokensToRegexOptions>,
}
fn try_consume(
token_type: &TokenType,
it: &mut Peekable<impl Iterator<Item = LexToken>>,
) -> Option<String> {
if let Some(token) = it.peek() {
if &token.token_type == token_type {
let token = it.next().unwrap();
return Some(token.value);
}
}
None
}
fn must_consume(
token_type: &TokenType,
it: &mut Peekable<impl Iterator<Item = LexToken>>,
) -> Result<String, AnyError> {
try_consume(token_type, it).ok_or_else(|| {
let maybe_token = it.next();
if let Some(token) = maybe_token {
anyhow!(
"Unexpected {:?} at {}, expected {:?}",
token.token_type,
token.index,
token_type
)
} else {
anyhow!("Unexpected end of tokens, expected {:?}", token_type)
}
})
}
fn consume_text(
it: &mut Peekable<impl Iterator<Item = LexToken>>,
) -> Option<String> {
let mut result = String::new();
loop {
if let Some(value) = try_consume(&TokenType::Char, it) {
result.push_str(&value);
}
if let Some(value) = try_consume(&TokenType::EscapedChar, it) {
result.push_str(&value);
} else {
break;
}
}
if result.is_empty() {
None
} else {
Some(result)
}
}
/// Parse a string for the raw tokens.
pub fn parse(
s: &str,
maybe_options: Option<ParseOptions>,
) -> Result<Vec<Token>, AnyError> {
let mut tokens = lexer(s)?.into_iter().peekable();
let options = maybe_options.unwrap_or_default();
let prefixes = options.prefixes.unwrap_or_else(|| "./".to_string());
let default_pattern = if let Some(delimiter) = options.delimiter {
format!("[^{}]+?", escape_string(&delimiter))
} else {
"[^/#?]+?".to_string()
};
let mut result = Vec::new();
let mut key = 0_usize;
let mut path = String::new();
loop {
let char = try_consume(&TokenType::Char, &mut tokens);
let name = try_consume(&TokenType::Name, &mut tokens);
let pattern = try_consume(&TokenType::Pattern, &mut tokens);
if name.is_some() || pattern.is_some() {
let mut prefix = char.unwrap_or_default();
if !prefixes.contains(&prefix) {
path.push_str(&prefix);
prefix = String::new();
}
if !path.is_empty() {
result.push(Token::String(path.clone()));
path = String::new();
}
let name = name.map_or_else(
|| {
let default = StringOrNumber::Number(key);
key += 1;
default
},
StringOrNumber::String,
);
let prefix = if prefix.is_empty() {
None
} else {
Some(prefix)
};
result.push(Token::Key(Key {
name,
prefix,
suffix: None,
pattern: pattern.unwrap_or_else(|| default_pattern.clone()),
modifier: try_consume(&TokenType::Modifier, &mut tokens),
}));
continue;
}
if let Some(value) = char {
path.push_str(&value);
continue;
} else if let Some(value) =
try_consume(&TokenType::EscapedChar, &mut tokens)
{
path.push_str(&value);
continue;
}
if !path.is_empty() {
result.push(Token::String(path.clone()));
path = String::new();
}
if try_consume(&TokenType::Open, &mut tokens).is_some() {
let prefix = consume_text(&mut tokens);
let maybe_name = try_consume(&TokenType::Name, &mut tokens);
let maybe_pattern = try_consume(&TokenType::Pattern, &mut tokens);
let suffix = consume_text(&mut tokens);
must_consume(&TokenType::Close, &mut tokens)?;
let name = maybe_name.clone().map_or_else(
|| {
if maybe_pattern.is_some() {
let default = StringOrNumber::Number(key);
key += 1;
default
} else {
StringOrNumber::String("".to_string())
}
},
StringOrNumber::String,
);
let pattern = if maybe_name.is_some() && maybe_pattern.is_none() {
default_pattern.clone()
} else {
maybe_pattern.unwrap_or_default()
};
result.push(Token::Key(Key {
name,
prefix,
pattern,
suffix,
modifier: try_consume(&TokenType::Modifier, &mut tokens),
}));
continue;
}
must_consume(&TokenType::End, &mut tokens)?;
break;
}
Ok(result)
}
/// Transform a vector of tokens into a regular expression, returning the
/// regular expression and optionally any keys that can be matched as part of
/// the expression.
pub fn tokens_to_regex(
tokens: &[Token],
maybe_options: Option<TokensToRegexOptions>,
) -> Result<(FancyRegex, Option<Vec<Key>>), AnyError> {
let TokensToRegexOptions {
sensitive,
strict,
end,
start,
delimiter,
ends_with,
} = maybe_options.unwrap_or_default();
let has_ends_with = ends_with.is_some();
let ends_with = format!(r"[{}]|$", ends_with.unwrap_or_default());
let delimiter =
format!(r"[{}]", delimiter.unwrap_or_else(|| "/#?".to_string()));
let mut route = if start {
"^".to_string()
} else {
String::new()
};
let maybe_end_token = tokens.iter().last().cloned();
let mut keys: Vec<Key> = Vec::new();
for token in tokens {
let value = match token {
Token::String(s) => s.to_string(),
Token::Key(key) => {
if !key.pattern.is_empty() {
keys.push(key.clone());
}
let prefix = key
.prefix
.clone()
.map_or_else(|| "".to_string(), |s| escape_string(&s));
let suffix = key
.suffix
.clone()
.map_or_else(|| "".to_string(), |s| escape_string(&s));
if !key.pattern.is_empty() {
if !prefix.is_empty() || !suffix.is_empty() {
match &key.modifier {
Some(s) if s == "+" || s == "*" => {
let modifier = if key.modifier == Some("*".to_string()) {
"?"
} else {
""
};
format!(
"(?:{}((?:{})(?:{}{}(?:{}))*){}){}",
prefix,
key.pattern,
suffix,
prefix,
key.pattern,
suffix,
modifier
)
}
_ => {
let modifier = key.modifier.clone().unwrap_or_default();
format!(
r"(?:{}({}){}){}",
prefix, key.pattern, suffix, modifier
)
}
}
} else {
let modifier = key.modifier.clone().unwrap_or_default();
format!(r"({}){}", key.pattern, modifier)
}
} else {
let modifier = key.modifier.clone().unwrap_or_default();
format!(r"(?:{}{}){}", prefix, suffix, modifier)
}
}
};
route.push_str(&value);
}
if end {
if !strict {
route.push_str(&format!(r"{}?", delimiter));
}
if has_ends_with {
route.push_str(&format!(r"(?={})", ends_with));
} else {
route.push('$');
}
} else {
let is_end_deliminated = match maybe_end_token {
Some(Token::String(mut s)) => {
if let Some(c) = s.pop() {
delimiter.contains(c)
} else {
false
}
}
Some(_) => false,
None => true,
};
if !strict {
route.push_str(&format!(r"(?:{}(?={}))?", delimiter, ends_with));
}
if !is_end_deliminated {
route.push_str(&format!(r"(?={}|{})", delimiter, ends_with));
}
}
let flags = if sensitive { "" } else { "(?i)" };
let re = FancyRegex::new(&format!("{}{}", flags, route))?;
let maybe_keys = if keys.is_empty() { None } else { Some(keys) };
Ok((re, maybe_keys))
}
/// Convert a path-like string into a regular expression, returning the regular
/// expression and optionally any keys that can be matched in the string.
pub fn string_to_regex(
path: &str,
maybe_options: Option<PathToRegexOptions>,
) -> Result<(FancyRegex, Option<Vec<Key>>), AnyError> {
let (parse_options, tokens_to_regex_options) =
if let Some(options) = maybe_options {
(options.parse_options, options.token_to_regex_options)
} else {
(None, None)
};
tokens_to_regex(&parse(path, parse_options)?, tokens_to_regex_options)
}
pub struct Compiler {
matches: Vec<Option<Regex>>,
tokens: Vec<Token>,
validate: bool,
}
impl Compiler {
pub fn new(
tokens: &[Token],
maybe_options: Option<TokensToCompilerOptions>,
) -> Self {
let TokensToCompilerOptions {
sensitive,
validate,
} = maybe_options.unwrap_or_default();
let flags = if sensitive { "" } else { "(?i)" };
let matches = tokens
.iter()
.map(|t| {
if let Token::Key(k) = t {
Some(Regex::new(&format!("{}^(?:{})$", flags, k.pattern)).unwrap())
} else {
None
}
})
.collect();
Self {
matches,
tokens: tokens.to_vec(),
validate,
}
}
/// Convert a map of key values into a string.
pub fn to_path(
&self,
params: &HashMap<StringOrNumber, StringOrVec>,
) -> Result<String, AnyError> {
let mut path = String::new();
for (i, token) in self.tokens.iter().enumerate() {
match token {
Token::String(s) => path.push_str(s),
Token::Key(k) => {
let value = params.get(&k.name);
let optional = k.modifier == Some("?".to_string())
|| k.modifier == Some("*".to_string());
let repeat = k.modifier == Some("*".to_string())
|| k.modifier == Some("+".to_string());
match value {
Some(StringOrVec::Vec(v)) => {
if !repeat {
return Err(anyhow!(
"Expected \"{:?}\" to not repeat, but got a vector",
k.name
));
}
if v.is_empty() {
if !optional {
return Err(anyhow!(
"Expected \"{:?}\" to not be empty.",
k.name
));
}
} else {
let prefix = k.prefix.clone().unwrap_or_default();
let suffix = k.suffix.clone().unwrap_or_default();
for segment in v {
if self.validate {
if let Some(re) = &self.matches[i] {
if !re.is_match(segment) {
return Err(anyhow!(
"Expected all \"{:?}\" to match \"{}\", but got {}",
k.name,
k.pattern,
segment
));
}
}
}
path.push_str(&format!("{}{}{}", prefix, segment, suffix));
}
}
}
Some(StringOrVec::String(s)) => {
if self.validate {
if let Some(re) = &self.matches[i] {
if !re.is_match(s) {
return Err(anyhow!(
"Expected \"{:?}\" to match \"{}\", but got \"{}\"",
k.name,
k.pattern,
s
));
}
}
}
let prefix = k.prefix.clone().unwrap_or_default();
let suffix = k.suffix.clone().unwrap_or_default();
path.push_str(&format!("{}{}{}", prefix, s, suffix));
}
None => {
if !optional {
let key_type = if repeat { "an array" } else { "a string" };
return Err(anyhow!(
"Expected \"{:?}\" to be {}",
k.name,
key_type
));
}
}
}
}
}
}
Ok(path)
}
}
#[derive(Debug)]
pub struct MatchResult {
pub path: String,
pub index: usize,
pub params: HashMap<StringOrNumber, StringOrVec>,
}
impl MatchResult {
pub fn get(&self, key: &str) -> Option<&StringOrVec> {
self.params.get(&StringOrNumber::String(key.to_string()))
}
}
#[derive(Debug)]
pub struct Matcher {
maybe_keys: Option<Vec<Key>>,
re: FancyRegex,
}
impl Matcher {
pub fn new(
tokens: &[Token],
maybe_options: Option<TokensToRegexOptions>,
) -> Result<Self, AnyError> {
let (re, maybe_keys) = tokens_to_regex(tokens, maybe_options)?;
Ok(Self { maybe_keys, re })
}
/// Match a string path, optionally returning the match result.
pub fn matches(&self, path: &str) -> Option<MatchResult> {
let caps = self.re.captures(path).ok()??;
let m = caps.get(0)?;
let path = m.as_str().to_string();
let index = m.start();
let mut params = HashMap::new();
if let Some(keys) = &self.maybe_keys {
for (i, key) in keys.iter().enumerate() {
if let Some(m) = caps.get(i + 1) {
let value = if key.modifier == Some("*".to_string())
|| key.modifier == Some("+".to_string())
{
let pat = format!(
"{}{}",
key.prefix.clone().unwrap_or_default(),
key.suffix.clone().unwrap_or_default()
);
m.as_str()
.split(&pat)
.map(String::from)
.collect::<Vec<String>>()
.into()
} else {
m.as_str().into()
};
params.insert(key.name.clone(), value);
}
}
}
Some(MatchResult {
path,
index,
params,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
type FixtureMatch<'a> = (&'a str, usize, usize);
type Fixture<'a> = (&'a str, Option<FixtureMatch<'a>>);
fn test_path(
path: &str,
maybe_options: Option<PathToRegexOptions>,
fixtures: &[Fixture],
) {
let result = string_to_regex(path, maybe_options);
assert!(result.is_ok(), "Could not parse path: \"{}\"", path);
let (re, _) = result.unwrap();
for (fixture, expected) in fixtures {
let result = re.find(*fixture);
assert!(
result.is_ok(),
"Find failure for path \"{}\" and fixture \"{}\"",
path,
fixture
);
let actual = result.unwrap();
if let Some((text, start, end)) = *expected {
assert!(actual.is_some(), "Match failure for path \"{}\" and fixture \"{}\". Expected Some got None", path, fixture);
let actual = actual.unwrap();
assert_eq!(actual.as_str(), text, "Match failure for path \"{}\" and fixture \"{}\". Expected \"{}\" got \"{}\".", path, fixture, text, actual.as_str());
assert_eq!(actual.start(), start);
assert_eq!(actual.end(), end);
} else {
assert!(actual.is_none(), "Match failure for path \"{}\" and fixture \"{}\". Expected None got {:?}", path, fixture, actual);
}
}
}
#[test]
fn test_compiler() {
let tokens = parse("/x/:a@:b/:c*", None).expect("could not parse");
let mut params = HashMap::<StringOrNumber, StringOrVec>::new();
params.insert(
StringOrNumber::String("a".to_string()),
StringOrVec::String("y".to_string()),
);
params.insert(
StringOrNumber::String("b".to_string()),
StringOrVec::String("v1.0.0".to_string()),
);
params.insert(
StringOrNumber::String("c".to_string()),
StringOrVec::Vec(vec!["z".to_string(), "example.ts".to_string()]),
);
let compiler = Compiler::new(&tokens, None);
let actual = compiler.to_path(&params);
assert!(actual.is_ok());
let actual = actual.unwrap();
assert_eq!(actual, "/x/y@v1.0.0/z/example.ts".to_string());
}
#[test]
fn test_string_to_regex() {
test_path("/", None, &[("/test", None), ("/", Some(("/", 0, 1)))]);
test_path(
"/test",
None,
&[
("/test", Some(("/test", 0, 5))),
("/route", None),
("/test/route", None),
("/test/", Some(("/test/", 0, 6))),
],
);
test_path(
"/test/",
None,
&[
("/test", None),
("/test/", Some(("/test/", 0, 6))),
("/test//", Some(("/test//", 0, 7))),
],
);
// case-sensitive paths
test_path(
"/test",
Some(PathToRegexOptions {
parse_options: None,
token_to_regex_options: Some(TokensToRegexOptions {
sensitive: true,
..Default::default()
}),
}),
&[("/test", Some(("/test", 0, 5))), ("/TEST", None)],
);
test_path(
"/TEST",
Some(PathToRegexOptions {
parse_options: None,
token_to_regex_options: Some(TokensToRegexOptions {
sensitive: true,
..Default::default()
}),
}),
&[("/TEST", Some(("/TEST", 0, 5))), ("/test", None)],
);
}
}

855
cli/lsp/registries.rs Normal file
View file

@ -0,0 +1,855 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
use super::language_server;
use super::path_to_regex::parse;
use super::path_to_regex::string_to_regex;
use super::path_to_regex::Compiler;
use super::path_to_regex::Key;
use super::path_to_regex::MatchResult;
use super::path_to_regex::Matcher;
use super::path_to_regex::StringOrNumber;
use super::path_to_regex::StringOrVec;
use super::path_to_regex::Token;
use crate::deno_dir;
use crate::file_fetcher::CacheSetting;
use crate::file_fetcher::FileFetcher;
use crate::http_cache::HttpCache;
use deno_core::error::anyhow;
use deno_core::error::AnyError;
use deno_core::error::Context;
use deno_core::resolve_url;
use deno_core::serde::Deserialize;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::url::Position;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_runtime::deno_file::BlobUrlStore;
use deno_runtime::permissions::Permissions;
use log::error;
use lspower::lsp;
use regex::Regex;
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::Path;
const CONFIG_PATH: &str = "/.well-known/deno-import-intellisense.json";
const COMPONENT: &percent_encoding::AsciiSet = &percent_encoding::CONTROLS
.add(b' ')
.add(b'"')
.add(b'#')
.add(b'<')
.add(b'>')
.add(b'?')
.add(b'`')
.add(b'{')
.add(b'}')
.add(b'/')
.add(b':')
.add(b';')
.add(b'=')
.add(b'@')
.add(b'[')
.add(b'\\')
.add(b']')
.add(b'^')
.add(b'|')
.add(b'$')
.add(b'&')
.add(b'+')
.add(b',');
lazy_static::lazy_static! {
static ref REPLACEMENT_VARIABLE_RE: Regex =
Regex::new(r"\$\{\{?(\w+)\}?\}").unwrap();
}
fn base_url(url: &Url) -> String {
url.origin().ascii_serialization()
}
#[derive(Debug)]
enum CompletorType {
Literal(String),
Key(Key, Option<String>),
}
/// Determine if a completion at a given offset is a string literal or a key/
/// variable.
fn get_completor_type(
offset: usize,
tokens: &[Token],
match_result: &MatchResult,
) -> Option<CompletorType> {
let mut len = 0_usize;
for token in tokens {
match token {
Token::String(s) => {
len += s.chars().count();
if offset < len {
return Some(CompletorType::Literal(s.clone()));
}
}
Token::Key(k) => {
if let Some(prefix) = &k.prefix {
len += prefix.chars().count();
if offset < len {
return Some(CompletorType::Key(k.clone(), Some(prefix.clone())));
}
}
if offset < len {
return None;
}
if let StringOrNumber::String(name) = &k.name {
let value = match_result
.get(name)
.map(|s| s.to_string(Some(&k)))
.unwrap_or_default();
len += value.chars().count();
if offset <= len {
return Some(CompletorType::Key(k.clone(), None));
}
}
if let Some(suffix) = &k.suffix {
len += suffix.chars().count();
if offset <= len {
return Some(CompletorType::Literal(suffix.clone()));
}
}
}
}
}
None
}
/// Convert a completion URL string from a completions configuration into a
/// fully qualified URL which can be fetched to provide the completions.
fn get_completion_endpoint(
url: &str,
tokens: &[Token],
match_result: &MatchResult,
) -> Result<ModuleSpecifier, AnyError> {
let mut url_str = url.to_string();
for (key, value) in match_result.params.iter() {
if let StringOrNumber::String(name) = key {
let maybe_key = tokens.iter().find_map(|t| match t {
Token::Key(k) if k.name == *key => Some(k),
_ => None,
});
url_str =
url_str.replace(&format!("${{{}}}", name), &value.to_string(maybe_key));
url_str = url_str.replace(
&format!("${{{{{}}}}}", name),
&percent_encoding::percent_encode(
value.to_string(maybe_key).as_bytes(),
COMPONENT,
)
.to_string(),
);
}
}
resolve_url(&url_str).map_err(|err| err.into())
}
fn parse_replacement_variables<S: AsRef<str>>(s: S) -> HashSet<String> {
REPLACEMENT_VARIABLE_RE
.captures_iter(s.as_ref())
.filter_map(|c| c.get(1).map(|m| m.as_str().to_string()))
.collect()
}
/// Validate a registry configuration JSON structure.
fn validate_config(config: &RegistryConfigurationJson) -> Result<(), AnyError> {
if config.version != 1 {
return Err(anyhow!(
"Invalid registry configuration. Expected version 1 got {}.",
config.version
));
}
for registry in &config.registries {
let (_, keys) = string_to_regex(&registry.schema, None)?;
let key_names: HashSet<String> = keys.map_or_else(HashSet::new, |keys| {
keys
.iter()
.filter_map(|k| {
if let StringOrNumber::String(s) = &k.name {
Some(s.clone())
} else {
None
}
})
.collect()
});
let mut variable_names = HashSet::<String>::new();
for variable in &registry.variables {
variable_names.insert(variable.key.clone());
if !key_names.contains(&variable.key) {
return Err(anyhow!("Invalid registry configuration. Variable \"{}\" is not present in the schema: \"{}\".", variable.key, registry.schema));
}
for url_var in &parse_replacement_variables(&variable.url) {
if !key_names.contains(url_var) {
return Err(anyhow!("Invalid registry configuration. Variable url \"{}\" is not present in the schema: \"{}\".", url_var, registry.schema));
}
}
}
for key_name in &key_names {
if !variable_names.contains(key_name) {
return Err(anyhow!("Invalid registry configuration. Schema contains key \"{}\" which does not have a defined variable.", key_name));
}
}
}
Ok(())
}
#[derive(Debug, Clone, Deserialize)]
struct RegistryConfigurationVariable {
/// The name of the variable.
key: String,
/// The URL with variable substitutions of the endpoint that will provide
/// completions for the variable.
url: String,
}
#[derive(Debug, Clone, Deserialize)]
struct RegistryConfiguration {
/// A Express-like path which describes how URLs are composed for a registry.
schema: String,
/// The variables denoted in the `schema` should have a variable entry.
variables: Vec<RegistryConfigurationVariable>,
}
/// A structure that represents the configuration of an origin and its module
/// registries.
#[derive(Debug, Deserialize)]
struct RegistryConfigurationJson {
version: u32,
registries: Vec<RegistryConfiguration>,
}
/// A structure which holds the information about currently configured module
/// registries and can provide completion information for URLs that match
/// one of the enabled registries.
#[derive(Debug, Clone)]
pub struct ModuleRegistry {
origins: HashMap<String, Vec<RegistryConfiguration>>,
file_fetcher: FileFetcher,
}
impl Default for ModuleRegistry {
fn default() -> Self {
let custom_root = std::env::var("DENO_DIR").map(String::into).ok();
let dir = deno_dir::DenoDir::new(custom_root).unwrap();
let location = dir.root.join("registries");
let http_cache = HttpCache::new(&location);
let cache_setting = CacheSetting::Use;
let file_fetcher = FileFetcher::new(
http_cache,
cache_setting,
true,
None,
BlobUrlStore::default(),
)
.unwrap();
Self {
origins: HashMap::new(),
file_fetcher,
}
}
}
impl ModuleRegistry {
pub fn new(location: &Path) -> Self {
let http_cache = HttpCache::new(location);
let file_fetcher = FileFetcher::new(
http_cache,
CacheSetting::Use,
true,
None,
BlobUrlStore::default(),
)
.context("Error creating file fetcher in module registry.")
.unwrap();
Self {
origins: HashMap::new(),
file_fetcher,
}
}
fn complete_literal(
&self,
s: String,
completions: &mut HashMap<String, lsp::CompletionItem>,
current_specifier: &str,
offset: usize,
range: &lsp::Range,
) {
let label = if s.starts_with('/') {
s[0..].to_string()
} else {
s.to_string()
};
let full_text = format!(
"{}{}{}",
&current_specifier[..offset],
s,
&current_specifier[offset..]
);
let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
range: *range,
new_text: full_text.clone(),
}));
let filter_text = Some(full_text);
completions.insert(
s,
lsp::CompletionItem {
label,
kind: Some(lsp::CompletionItemKind::Folder),
filter_text,
sort_text: Some("1".to_string()),
text_edit,
..Default::default()
},
);
}
/// Disable a registry, removing its configuration, if any, from memory.
pub async fn disable(&mut self, origin: &str) -> Result<(), AnyError> {
let origin = base_url(&Url::parse(origin)?);
self.origins.remove(&origin);
Ok(())
}
/// Attempt to fetch the configuration for a specific origin.
async fn fetch_config(
&self,
origin: &str,
) -> Result<Vec<RegistryConfiguration>, AnyError> {
let origin_url = Url::parse(origin)?;
let specifier = origin_url.join(CONFIG_PATH)?;
let file = self
.file_fetcher
.fetch(&specifier, &Permissions::allow_all())
.await?;
let config: RegistryConfigurationJson = serde_json::from_str(&file.source)?;
validate_config(&config)?;
Ok(config.registries)
}
/// Enable a registry by attempting to retrieve its configuration and
/// validating it.
pub async fn enable(&mut self, origin: &str) -> Result<(), AnyError> {
let origin = base_url(&Url::parse(origin)?);
#[allow(clippy::map_entry)]
// we can't use entry().or_insert_with() because we can't use async closures
if !self.origins.contains_key(&origin) {
let configs = self.fetch_config(&origin).await?;
self.origins.insert(origin, configs);
}
Ok(())
}
/// For a string specifier from the client, provide a set of completions, if
/// any, for the specifier.
pub async fn get_completions(
&self,
current_specifier: &str,
offset: usize,
range: &lsp::Range,
state_snapshot: &language_server::StateSnapshot,
) -> Option<Vec<lsp::CompletionItem>> {
if let Ok(specifier) = Url::parse(current_specifier) {
let origin = base_url(&specifier);
let origin_len = origin.chars().count();
if offset >= origin_len {
if let Some(registries) = self.origins.get(&origin) {
let path = &specifier[Position::BeforePath..];
let path_offset = offset - origin_len;
let mut completions = HashMap::<String, lsp::CompletionItem>::new();
let mut did_match = false;
for registry in registries {
let tokens = parse(&registry.schema, None)
.map_err(|e| {
error!(
"Error parsing registry schema for origin \"{}\". {}",
origin, e
);
})
.ok()?;
let mut i = tokens.len();
let last_key_name =
StringOrNumber::String(tokens.iter().last().map_or_else(
|| "".to_string(),
|t| {
if let Token::Key(key) = t {
if let StringOrNumber::String(s) = &key.name {
return s.clone();
}
}
"".to_string()
},
));
loop {
let matcher = Matcher::new(&tokens[..i], None)
.map_err(|e| {
error!(
"Error creating matcher for schema for origin \"{}\". {}",
origin, e
);
})
.ok()?;
if let Some(match_result) = matcher.matches(path) {
did_match = true;
let completor_type =
get_completor_type(path_offset, &tokens, &match_result);
match completor_type {
Some(CompletorType::Literal(s)) => self.complete_literal(
s,
&mut completions,
current_specifier,
offset,
range,
),
Some(CompletorType::Key(k, p)) => {
let maybe_url = registry.variables.iter().find_map(|v| {
if k.name == StringOrNumber::String(v.key.clone()) {
Some(v.url.as_str())
} else {
None
}
});
if let Some(url) = maybe_url {
if let Some(items) = self
.get_variable_items(url, &tokens, &match_result)
.await
{
let end = if p.is_some() { i + 1 } else { i };
let compiler = Compiler::new(&tokens[..end], None);
for (idx, item) in items.into_iter().enumerate() {
let label = if let Some(p) = &p {
format!("{}{}", p, item)
} else {
item.clone()
};
let kind = if k.name == last_key_name {
Some(lsp::CompletionItemKind::File)
} else {
Some(lsp::CompletionItemKind::Folder)
};
let mut params = match_result.params.clone();
params.insert(
k.name.clone(),
StringOrVec::from_str(&item, &k),
);
let path =
compiler.to_path(&params).unwrap_or_default();
let mut item_specifier = Url::parse(&origin).ok()?;
item_specifier.set_path(&path);
let full_text = item_specifier.as_str();
let text_edit = Some(lsp::CompletionTextEdit::Edit(
lsp::TextEdit {
range: *range,
new_text: full_text.to_string(),
},
));
let command = if k.name == last_key_name
&& !state_snapshot
.sources
.contains_key(&item_specifier)
{
Some(lsp::Command {
title: "".to_string(),
command: "deno.cache".to_string(),
arguments: Some(vec![json!([item_specifier])]),
})
} else {
None
};
let detail = Some(format!("({})", k.name));
let filter_text = Some(full_text.to_string());
let sort_text = Some(format!("{:0>10}", idx + 1));
completions.insert(
item,
lsp::CompletionItem {
label,
kind,
detail,
filter_text,
sort_text,
text_edit,
command,
..Default::default()
},
);
}
}
}
}
None => (),
}
break;
}
i -= 1;
// If we have fallen though to the first token, and we still
// didn't get a match, but the first token is a string literal, we
// need to suggest the string literal.
if i == 0 {
if let Token::String(s) = &tokens[i] {
if s.starts_with(path) {
let label = s.to_string();
let kind = Some(lsp::CompletionItemKind::Folder);
let mut url = specifier.clone();
url.set_path(s);
let full_text = url.as_str();
let text_edit =
Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
range: *range,
new_text: full_text.to_string(),
}));
let filter_text = Some(full_text.to_string());
completions.insert(
s.to_string(),
lsp::CompletionItem {
label,
kind,
filter_text,
sort_text: Some("1".to_string()),
text_edit,
..Default::default()
},
);
}
}
break;
}
}
}
// If we return None, other sources of completions will be looked for
// but if we did at least match part of a registry, we should send an
// empty vector so that no-completions will be sent back to the client
return if completions.is_empty() && !did_match {
None
} else {
Some(completions.into_iter().map(|(_, i)| i).collect())
};
}
}
}
self.get_origin_completions(current_specifier, range)
}
pub fn get_origin_completions(
&self,
current_specifier: &str,
range: &lsp::Range,
) -> Option<Vec<lsp::CompletionItem>> {
let items = self
.origins
.keys()
.filter_map(|k| {
let mut origin = k.as_str().to_string();
if origin.ends_with('/') {
origin.pop();
}
if origin.starts_with(current_specifier) {
let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
range: *range,
new_text: origin.clone(),
}));
Some(lsp::CompletionItem {
label: origin,
kind: Some(lsp::CompletionItemKind::Folder),
detail: Some("(registry)".to_string()),
sort_text: Some("2".to_string()),
text_edit,
..Default::default()
})
} else {
None
}
})
.collect::<Vec<lsp::CompletionItem>>();
if !items.is_empty() {
Some(items)
} else {
None
}
}
async fn get_variable_items(
&self,
url: &str,
tokens: &[Token],
match_result: &MatchResult,
) -> Option<Vec<String>> {
let specifier = get_completion_endpoint(url, tokens, match_result)
.map_err(|err| {
error!("Internal error mapping endpoint \"{}\". {}", url, err);
})
.ok()?;
let file = self
.file_fetcher
.fetch(&specifier, &Permissions::allow_all())
.await
.map_err(|err| {
error!(
"Internal error fetching endpoint \"{}\". {}",
specifier, err
);
})
.ok()?;
let items: Vec<String> = serde_json::from_str(&file.source)
.map_err(|err| {
error!(
"Error parsing response from endpoint \"{}\". {}",
specifier, err
);
})
.ok()?;
Some(items)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::lsp::documents::DocumentCache;
use crate::lsp::sources::Sources;
use tempfile::TempDir;
fn mock_state_snapshot(
source_fixtures: &[(&str, &str)],
location: &Path,
) -> language_server::StateSnapshot {
let documents = DocumentCache::default();
let sources = Sources::new(location);
let http_cache = HttpCache::new(location);
for (specifier, source) in source_fixtures {
let specifier =
resolve_url(specifier).expect("failed to create specifier");
http_cache
.set(&specifier, HashMap::default(), source.as_bytes())
.expect("could not cache file");
assert!(
sources.get_source(&specifier).is_some(),
"source could not be setup"
);
}
language_server::StateSnapshot {
documents,
sources,
..Default::default()
}
}
fn setup(sources: &[(&str, &str)]) -> language_server::StateSnapshot {
let temp_dir = TempDir::new().expect("could not create temp dir");
let location = temp_dir.path().join("deps");
mock_state_snapshot(sources, &location)
}
#[tokio::test]
async fn test_registry_completions_origin_match() {
let _g = test_util::http_server();
let temp_dir = TempDir::new().expect("could not create tmp");
let location = temp_dir.path().join("registries");
let mut module_registry = ModuleRegistry::new(&location);
module_registry
.enable("http://localhost:4545/")
.await
.expect("could not enable");
let range = lsp::Range {
start: lsp::Position {
line: 0,
character: 20,
},
end: lsp::Position {
line: 0,
character: 21,
},
};
let state_snapshot = setup(&[]);
let completions = module_registry
.get_completions("h", 1, &range, &state_snapshot)
.await;
assert!(completions.is_some());
let completions = completions.unwrap();
assert_eq!(completions.len(), 1);
assert_eq!(completions[0].label, "http://localhost:4545");
assert_eq!(
completions[0].text_edit,
Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
range,
new_text: "http://localhost:4545".to_string()
}))
);
let range = lsp::Range {
start: lsp::Position {
line: 0,
character: 20,
},
end: lsp::Position {
line: 0,
character: 36,
},
};
let completions = module_registry
.get_completions("http://localhost", 16, &range, &state_snapshot)
.await;
assert!(completions.is_some());
let completions = completions.unwrap();
assert_eq!(completions.len(), 1);
assert_eq!(completions[0].label, "http://localhost:4545");
assert_eq!(
completions[0].text_edit,
Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
range,
new_text: "http://localhost:4545".to_string()
}))
);
}
#[tokio::test]
async fn test_registry_completions() {
let _g = test_util::http_server();
let temp_dir = TempDir::new().expect("could not create tmp");
let location = temp_dir.path().join("registries");
let mut module_registry = ModuleRegistry::new(&location);
module_registry
.enable("http://localhost:4545/")
.await
.expect("could not enable");
let state_snapshot = setup(&[]);
let range = lsp::Range {
start: lsp::Position {
line: 0,
character: 20,
},
end: lsp::Position {
line: 0,
character: 41,
},
};
let completions = module_registry
.get_completions("http://localhost:4545", 21, &range, &state_snapshot)
.await;
assert!(completions.is_some());
let completions = completions.unwrap();
assert_eq!(completions.len(), 1);
assert_eq!(completions[0].label, "/x");
assert_eq!(
completions[0].text_edit,
Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
range,
new_text: "http://localhost:4545/x".to_string()
}))
);
let range = lsp::Range {
start: lsp::Position {
line: 0,
character: 20,
},
end: lsp::Position {
line: 0,
character: 42,
},
};
let completions = module_registry
.get_completions("http://localhost:4545/", 22, &range, &state_snapshot)
.await;
assert!(completions.is_some());
let completions = completions.unwrap();
assert_eq!(completions.len(), 1);
assert_eq!(completions[0].label, "/x");
assert_eq!(
completions[0].text_edit,
Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit {
range,
new_text: "http://localhost:4545/x".to_string()
}))
);
let range = lsp::Range {
start: lsp::Position {
line: 0,
character: 20,
},
end: lsp::Position {
line: 0,
character: 44,
},
};
let completions = module_registry
.get_completions("http://localhost:4545/x/", 24, &range, &state_snapshot)
.await;
assert!(completions.is_some());
let completions = completions.unwrap();
assert_eq!(completions.len(), 2);
assert!(completions[0].label == *"a" || completions[0].label == *"b");
assert!(completions[1].label == *"a" || completions[1].label == *"b");
let range = lsp::Range {
start: lsp::Position {
line: 0,
character: 20,
},
end: lsp::Position {
line: 0,
character: 46,
},
};
let completions = module_registry
.get_completions(
"http://localhost:4545/x/a@",
26,
&range,
&state_snapshot,
)
.await;
assert!(completions.is_some());
let completions = completions.unwrap();
assert_eq!(completions.len(), 3);
let range = lsp::Range {
start: lsp::Position {
line: 0,
character: 20,
},
end: lsp::Position {
line: 0,
character: 53,
},
};
let completions = module_registry
.get_completions(
"http://localhost:4545/x/a@v1.0.0/",
33,
&range,
&state_snapshot,
)
.await;
assert!(completions.is_some());
let completions = completions.unwrap();
assert_eq!(completions.len(), 2);
assert_eq!(completions[0].detail, Some("(path)".to_string()));
assert_eq!(completions[0].kind, Some(lsp::CompletionItemKind::File));
assert!(completions[0].command.is_some());
assert_eq!(completions[1].detail, Some("(path)".to_string()));
assert_eq!(completions[0].kind, Some(lsp::CompletionItemKind::File));
assert!(completions[1].command.is_some());
}
#[test]
fn test_parse_replacement_variables() {
let actual = parse_replacement_variables(
"https://deno.land/_vsc1/modules/${module}/v/${{version}}",
);
assert_eq!(actual.iter().count(), 2);
assert!(actual.contains("module"));
assert!(actual.contains("version"));
}
}

10
cli/main.rs
View file

@ -252,11 +252,14 @@ fn print_cache_info(
let deno_dir = &state.dir.root;
let modules_cache = &state.file_fetcher.get_http_cache_location();
let typescript_cache = &state.dir.gen_cache.location;
let registry_cache =
&state.dir.root.join(lsp::language_server::REGISTRIES_PATH);
if json {
let output = json!({
"denoDir": deno_dir,
"modulesCache": modules_cache,
"typescriptCache": typescript_cache,
"registryCache": registry_cache,
});
write_json_to_stdout(&output)
} else {
@ -268,9 +271,14 @@ fn print_cache_info(
);
println!(
"{} {:?}",
colors::bold("TypeScript compiler cache:"),
colors::bold("Emitted modules cache:"),
typescript_cache
);
println!(
"{} {:?}",
colors::bold("Language server registries cache:"),
registry_cache,
);
Ok(())
}
}

3
cli/tests/041_info_flag.out
View file

@ -1,3 +1,4 @@
DENO_DIR location: "[WILDCARD]"
Remote modules cache: "[WILDCARD]deps"
TypeScript compiler cache: "[WILDCARD]gen"
Emitted modules cache: "[WILDCARD]gen"
Language server registries cache: "[WILDCARD]registries"

3
cli/tests/info_json.out
View file

@ -1,5 +1,6 @@
{
"denoDir": "[WILDCARD]",
"modulesCache": "[WILDCARD]deps",
"typescriptCache": "[WILDCARD]gen"
"typescriptCache": "[WILDCARD]gen",
"registryCache": "[WILDCARD]registries"
}

18
cli/tests/lsp/completion_request_registry.json Normal file
View file

@ -0,0 +1,18 @@
{
"jsonrpc": "2.0",
"id": 2,
"method": "textDocument/completion",
"params": {
"textDocument": {
"uri": "file:///a/file.ts"
},
"position": {
"line": 0,
"character": 46
},
"context": {
"triggerKind": 2,
"triggerCharacter": "@"
}
}
}

18
cli/tests/lsp/completion_request_registry_02.json Normal file
View file

@ -0,0 +1,18 @@
{
"jsonrpc": "2.0",
"id": 2,
"method": "textDocument/completion",
"params": {
"textDocument": {
"uri": "file:///a/file.ts"
},
"position": {
"line": 0,
"character": 20
},
"context": {
"triggerKind": 2,
"triggerCharacter": "\""
}
}
}

25
cli/tests/lsp/completion_resolve_request_registry.json Normal file
View file

@ -0,0 +1,25 @@
{
"jsonrpc": "2.0",
"id": 4,
"method": "completionItem/resolve",
"params": {
"label": "v2.0.0",
"kind": 19,
"detail": "(version)",
"sortText": "0000000003",
"filterText": "http://localhost:4545/x/a@v2.0.0",
"textEdit": {
"range": {
"start": {
"line": 0,
"character": 20
},
"end": {
"line": 0,
"character": 46
}
},
"newText": "http://localhost:4545/x/a@v2.0.0"
}
}
}

12
cli/tests/lsp/did_open_notification_completion_registry.json Normal file
View file

@ -0,0 +1,12 @@
{
"jsonrpc": "2.0",
"method": "textDocument/didOpen",
"params": {
"textDocument": {
"uri": "file:///a/file.ts",
"languageId": "typescript",
"version": 1,
"text": "import * as a from \"http://localhost:4545/x/a@\""
}
}
}

12
cli/tests/lsp/did_open_notification_completion_registry_02.json Normal file
View file

@ -0,0 +1,12 @@
{
"jsonrpc": "2.0",
"method": "textDocument/didOpen",
"params": {
"textDocument": {
"uri": "file:///a/file.ts",
"languageId": "typescript",
"version": 1,
"text": "import * as a from \"\""
}
}
}

11
cli/tests/lsp/initialize_request.json
View file

@ -15,8 +15,17 @@
"implementations": true,
"references": true
},
"lint": true,
"importMap": null,
"lint": true,
"suggest": {
"autoImports": true,
"completeFunctionCalls": false,
"names": true,
"paths": true,
"imports": {
"hosts": {}
}
},
"unstable": false
},
"capabilities": {

63
cli/tests/lsp/initialize_request_registry.json Normal file
View file

@ -0,0 +1,63 @@
{
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"processId": 0,
"clientInfo": {
"name": "test-harness",
"version": "1.0.0"
},
"rootUri": null,
"initializationOptions": {
"enable": true,
"codeLens": {
"implementations": true,
"references": true
},
"importMap": null,
"lint": true,
"suggest": {
"autoImports": true,
"completeFunctionCalls": false,
"names": true,
"paths": true,
"imports": {
"hosts": {
"http://localhost:4545/": true
}
}
},
"unstable": false
},
"capabilities": {
"textDocument": {
"codeAction": {
"codeActionLiteralSupport": {
"codeActionKind": {
"valueSet": [
"quickfix"
]
}
},
"isPreferredSupport": true,
"dataSupport": true,
"resolveSupport": {
"properties": [
"edit"
]
}
},
"foldingRange": {
"lineFoldingOnly": true
},
"synchronization": {
"dynamicRegistration": true,
"willSave": true,
"willSaveWaitUntil": true,
"didSave": true
}
}
}
}
}

4
cli/tests/lsp/registries/a_latest.json Normal file
View file

@ -0,0 +1,4 @@
[
"b/c.ts",
"d/e.js"
]

4
cli/tests/lsp/registries/a_v1.0.0.json Normal file
View file

@ -0,0 +1,4 @@
[
"b/c.ts",
"d/e.js"
]

4
cli/tests/lsp/registries/a_v1.0.1.json Normal file
View file

@ -0,0 +1,4 @@
[
"b/c.ts",
"d/e.js"
]

4
cli/tests/lsp/registries/a_v2.0.0.json Normal file
View file

@ -0,0 +1,4 @@
[
"b/c.ts",
"d/e.js"
]

5
cli/tests/lsp/registries/a_versions.json Normal file
View file

@ -0,0 +1,5 @@
[
"v1.0.0",
"v1.0.1",
"v2.0.0"
]

4
cli/tests/lsp/registries/b_latest.json Normal file
View file

@ -0,0 +1,4 @@
[
"b/c.ts",
"d/e.js"
]

4
cli/tests/lsp/registries/b_v0.0.1.json Normal file
View file

@ -0,0 +1,4 @@
[
"b/c.ts",
"d/e.js"
]

4
cli/tests/lsp/registries/b_v0.0.2.json Normal file
View file

@ -0,0 +1,4 @@
[
"b/c.ts",
"d/e.js"
]

4
cli/tests/lsp/registries/b_v0.0.3.json Normal file
View file

@ -0,0 +1,4 @@
[
"b/c.ts",
"d/e.js"
]

5
cli/tests/lsp/registries/b_versions.json Normal file
View file

@ -0,0 +1,5 @@
[
"v0.0.1",
"v0.0.2",
"v0.0.3"
]

35
cli/tests/lsp/registries/deno-import-intellisense.json Normal file
View file

@ -0,0 +1,35 @@
{
"version": 1,
"registries": [
{
"schema": "/x/:module([a-z0-9_]*)@:version?/:path*",
"variables": [
{
"key": "module",
"url": "http://localhost:4545/cli/tests/lsp/registries/modules.json"
},
{
"key": "version",
"url": "http://localhost:4545/cli/tests/lsp/registries/${module}_versions.json"
},
{
"key": "path",
"url": "http://localhost:4545/cli/tests/lsp/registries/${module}_${{version}}.json"
}
]
},
{
"schema": "/x/:module([a-z0-9_]*)/:path*",
"variables": [
{
"key": "module",
"url": "http://localhost:4545/cli/tests/lsp/registries/modules.json"
},
{
"key": "path",
"url": "http://localhost:4545/cli/tests/lsp/registries/${module}_latest.json"
}
]
}
]
}

4
cli/tests/lsp/registries/modules.json Normal file
View file

@ -0,0 +1,4 @@
[
"a",
"b"
]

4
op_crates/file/lib.rs
View file

@ -12,7 +12,7 @@ use std::sync::Arc;
use std::sync::Mutex;
use uuid::Uuid;
#[derive(Clone)]
#[derive(Debug, Clone)]
pub struct Blob {
pub data: Vec<u8>,
pub media_type: String,
@ -20,7 +20,7 @@ pub struct Blob {
pub struct Location(pub Url);
#[derive(Default, Clone)]
#[derive(Debug, Default, Clone)]
pub struct BlobUrlStore(Arc<Mutex<HashMap<Url, Blob>>>);
impl BlobUrlStore {

12
test_util/src/lib.rs
View file

@ -614,6 +614,18 @@ async fn main_server(req: Request<Body>) -> hyper::Result<Response<Body>> {
);
Ok(res)
}
(_, "/.well-known/deno-import-intellisense.json") => {
let file_path = root_path()
.join("cli/tests/lsp/registries/deno-import-intellisense.json");
if let Ok(body) = tokio::fs::read(file_path).await {
Ok(custom_headers(
"/.well-known/deno-import-intellisense.json",
body,
))
} else {
Ok(Response::new(Body::empty()))
}
}
_ => {
let mut file_path = root_path();
file_path.push(&req.uri().path()[1..]);