refactor: use 'data-url' crate to process data URLs in lsp & file_fetcher (#10196)

Closes: #10118

Cargo.lock

@@ -516,6 +516,7 @@ dependencies = [
  "byteorder",
  "chrono",
  "clap",
+ "data-url",
  "deno_core",
  "deno_doc",
  "deno_lint",

cli/Cargo.toml

@@ -43,6 +43,7 @@ atty = "0.2.14"
 base64 = "0.13.0"
 byteorder = "1.4.3"
 clap = "2.33.3"
+data-url = "0.1.0"
 dissimilar = "1.0.2"
 dprint-plugin-json = "0.10.1"
 dprint-plugin-markdown = "0.6.2"

cli/file_fetcher.rs

@@ -10,6 +10,7 @@ use crate::http_util::FetchOnceResult;
 use crate::media_type::MediaType;
 use crate::text_encoding;
 use crate::version::get_user_agent;
+use data_url::DataUrl;
 use deno_core::error::custom_error;
 use deno_core::error::generic_error;
 use deno_core::error::uri_error;
@@ -153,40 +154,6 @@ pub fn get_source_from_bytes(
   Ok(source)
 }
 
-fn get_source_from_data_url(
-  specifier: &ModuleSpecifier,
-) -> Result<(String, MediaType, String), AnyError> {
-  if specifier.scheme() != "data" {
-    return Err(custom_error(
-      "BadScheme",
-      format!("Unexpected scheme of \"{}\"", specifier.scheme()),
-    ));
-  }
-  let path = specifier.path();
-  let mut parts = path.splitn(2, ',');
-  let media_type_part =
-    percent_encoding::percent_decode_str(parts.next().unwrap())
-      .decode_utf8()?;
-  let data_part = if let Some(data) = parts.next() {
-    data
-  } else {
-    return Err(custom_error(
-      "BadUrl",
-      "The data URL is badly formed, missing a comma.",
-    ));
-  };
-  let (media_type, maybe_charset) =
-    map_content_type(specifier, Some(media_type_part.to_string()));
-  let is_base64 = media_type_part.rsplit(';').any(|p| p == "base64");
-  let bytes = if is_base64 {
-    base64::decode(data_part)?
-  } else {
-    percent_encoding::percent_decode_str(data_part).collect()
-  };
-  let source = strip_shebang(get_source_from_bytes(bytes, maybe_charset)?);
-  Ok((source, media_type, media_type_part.to_string()))
-}
-
 /// Return a validated scheme for a given module specifier.
 fn get_validated_scheme(
   specifier: &ModuleSpecifier,
@@ -430,8 +397,18 @@ impl FileFetcher {
       ));
     }
 
-    let (source, media_type, content_type) =
+    let data_url = DataUrl::process(specifier.as_str())
-      get_source_from_data_url(specifier)?;
+      .map_err(|e| uri_error(format!("{:?}", e)))?;
+    let mime = data_url.mime_type();
+    let charset = mime.get_parameter("charset").map(|v| v.to_string());
+    let (bytes, _) = data_url
+      .decode_to_vec()
+      .map_err(|e| uri_error(format!("{:?}", e)))?;
+    let source = strip_shebang(get_source_from_bytes(bytes, charset)?);
+    let content_type = format!("{}", mime);
+    let (media_type, _) =
+      map_content_type(specifier, Some(content_type.clone()));
+
     let local =
       self
         .http_cache
@@ -762,39 +739,6 @@ mod tests {
     assert_eq!(file.source, expected);
   }
 
-  #[test]
-  fn test_get_source_from_data_url() {
-    let fixtures = vec![
-      ("data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=", true, MediaType::TypeScript, "application/typescript;base64", "export const a = \"a\";\n\nexport enum A {\n  A,\n  B,\n  C,\n}\n"),
-      ("data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=?a=b&b=c", true, MediaType::TypeScript, "application/typescript;base64", "export const a = \"a\";\n\nexport enum A {\n  A,\n  B,\n  C,\n}\n"),
-      ("data:text/plain,Hello%2C%20Deno!", true, MediaType::Unknown, "text/plain", "Hello, Deno!"),
-      ("data:,Hello%2C%20Deno!", true, MediaType::Unknown, "", "Hello, Deno!"),
-      ("data:application/javascript,console.log(\"Hello, Deno!\");%0A", true, MediaType::JavaScript, "application/javascript", "console.log(\"Hello, Deno!\");\n"),
-      ("data:text/jsx;base64,ZXhwb3J0IGRlZmF1bHQgZnVuY3Rpb24oKSB7CiAgcmV0dXJuIDxkaXY+SGVsbG8gRGVubyE8L2Rpdj4KfQo=", true, MediaType::Jsx, "text/jsx;base64", "export default function() {\n  return <div>Hello Deno!</div>\n}\n"),
-      ("data:text/tsx;base64,ZXhwb3J0IGRlZmF1bHQgZnVuY3Rpb24oKSB7CiAgcmV0dXJuIDxkaXY+SGVsbG8gRGVubyE8L2Rpdj4KfQo=", true, MediaType::Tsx, "text/tsx;base64", "export default function() {\n  return <div>Hello Deno!</div>\n}\n"),
-    ];
-
-    for (
-      url_str,
-      expected_ok,
-      expected_media_type,
-      expected_media_type_str,
-      expected,
-    ) in fixtures
-    {
-      let specifier = resolve_url(url_str).unwrap();
-      let actual = get_source_from_data_url(&specifier);
-      assert_eq!(actual.is_ok(), expected_ok);
-      if expected_ok {
-        let (actual, actual_media_type, actual_media_type_str) =
-          actual.unwrap();
-        assert_eq!(actual, expected);
-        assert_eq!(actual_media_type, expected_media_type);
-        assert_eq!(actual_media_type_str, expected_media_type_str);
-      }
-    }
-  }
-
   #[test]
   fn test_get_validated_scheme() {
     let fixtures = vec![

cli/lsp/urls.rs

@@ -3,6 +3,8 @@
 use crate::file_fetcher::map_content_type;
 use crate::media_type::MediaType;
 
+use data_url::DataUrl;
+use deno_core::error::uri_error;
 use deno_core::error::AnyError;
 use deno_core::url::Position;
 use deno_core::url::Url;
@@ -49,17 +51,6 @@ fn hash_data_specifier(specifier: &ModuleSpecifier) -> String {
   crate::checksum::gen(&[file_name_str.as_bytes()])
 }
 
-fn data_url_media_type(specifier: &ModuleSpecifier) -> MediaType {
-  let path = specifier.path();
-  let mut parts = path.splitn(2, ',');
-  let media_type_part =
-    percent_encoding::percent_decode_str(parts.next().unwrap())
-      .decode_utf8_lossy();
-  let (media_type, _) =
-    map_content_type(specifier, Some(media_type_part.into()));
-  media_type
-}
-
 /// A bi-directional map of URLs sent to the LSP client and internal module
 /// specifiers.  We need to map internal specifiers into `deno:` schema URLs
 /// to allow the Deno language server to manage these as virtual documents.
@@ -96,7 +87,11 @@ impl LspUrlMap {
         specifier.clone()
       } else {
         let specifier_str = if specifier.scheme() == "data" {
-          let media_type = data_url_media_type(specifier);
+          let data_url = DataUrl::process(specifier.as_str())
+            .map_err(|e| uri_error(format!("{:?}", e)))?;
+          let mime = data_url.mime_type();
+          let (media_type, _) =
+            map_content_type(specifier, Some(format!("{}", mime)));
           let extension = if media_type == MediaType::Unknown {
             ""
           } else {
@@ -159,21 +154,6 @@ mod tests {
     );
   }
 
-  #[test]
-  fn test_data_url_media_type() {
-    let fixture = resolve_url("data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=").unwrap();
-    let actual = data_url_media_type(&fixture);
-    assert_eq!(actual, MediaType::TypeScript);
-
-    let fixture = resolve_url("data:application/javascript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=").unwrap();
-    let actual = data_url_media_type(&fixture);
-    assert_eq!(actual, MediaType::JavaScript);
-
-    let fixture = resolve_url("data:text/plain;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=").unwrap();
-    let actual = data_url_media_type(&fixture);
-    assert_eq!(actual, MediaType::Unknown);
-  }
-
   #[test]
   fn test_lsp_url_map() {
     let mut map = LspUrlMap::default();