foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-08 15:19:40 -05:00
Code Issues Packages 1 Wiki Activity

refactor(cli): remove TextDocument (#7850)

Browse source
This commit is contained in:
Kitson Kelly 2020-10-07 22:43:44 +11:00 committed by GitHub
parent cb3a3a1e95
commit 99aa23b8dd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 126 additions and 227 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
cli/ast.rs
View file

@ -1,6 +1,5 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
use crate::file_fetcher::TextDocument;
use crate::media_type::MediaType; use crate::media_type::MediaType;
use deno_core::error::AnyError; use deno_core::error::AnyError;
@ -247,7 +246,7 @@ impl ParsedModule {
pub fn transpile( pub fn transpile(
self, self,
options: &TranspileOptions, options: &TranspileOptions,
) -> Result<(TextDocument, Option<TextDocument>)> { ) -> Result<(String, Option<String>)> {
let program = Program::Module(self.module); let program = Program::Module(self.module);
let jsx_pass = react::react( let jsx_pass = react::react(
@ -297,7 +296,7 @@ impl ParsedModule {
program.emit_with(&mut emitter)?; program.emit_with(&mut emitter)?;
} }
let mut src = String::from_utf8(buf)?; let mut src = String::from_utf8(buf)?;
let mut map: Option<TextDocument> = None; let mut map: Option<String> = None;
{ {
let mut buf = Vec::new(); let mut buf = Vec::new();
self self
@ -310,10 +309,10 @@ impl ParsedModule {
let encoded_map = base64::encode(buf); let encoded_map = base64::encode(buf);
src.push_str(&encoded_map); src.push_str(&encoded_map);
} else { } else {
map = Some(TextDocument::from(buf)); map = Some(String::from_utf8(buf)?);
} }
} }
Ok((src.into(), map)) Ok((src, map))
} }
} }
@ -439,14 +438,10 @@ mod tests {
let (code, maybe_map) = module let (code, maybe_map) = module
.transpile(&TranspileOptions::default()) .transpile(&TranspileOptions::default())
.expect("could not strip types"); .expect("could not strip types");
assert!(code assert!(code.starts_with("var D;\n(function(D) {\n"));
.to_string() assert!(
.unwrap() code.contains("\n//# sourceMappingURL=data:application/json;base64,")
.starts_with("var D;\n(function(D) {\n")); );
assert!(code
.to_string()
.unwrap()
.contains("\n//# sourceMappingURL=data:application/json;base64,"));
assert!(maybe_map.is_none()); assert!(maybe_map.is_none());
} }
@ -467,10 +462,7 @@ mod tests {
let (code, _) = module let (code, _) = module
.transpile(&TranspileOptions::default()) .transpile(&TranspileOptions::default())
.expect("could not strip types"); .expect("could not strip types");
assert!(code assert!(code.contains("React.createElement(\"div\", null"));
.to_string()
.unwrap()
.contains("React.createElement(\"div\", null"));
} }
#[test] #[test]
@ -501,9 +493,6 @@ mod tests {
let (code, _) = module let (code, _) = module
.transpile(&TranspileOptions::default()) .transpile(&TranspileOptions::default())
.expect("could not strip types"); .expect("could not strip types");
assert!(code assert!(code.contains("_applyDecoratedDescriptor("));
.to_string()
.unwrap()
.contains("_applyDecoratedDescriptor("));
} }
} }

155
cli/file_fetcher.rs
View file

@ -19,7 +19,6 @@ use deno_core::url::Url;
use deno_core::ModuleSpecifier; use deno_core::ModuleSpecifier;
use deno_fetch::reqwest; use deno_fetch::reqwest;
use log::info; use log::info;
use std::borrow::Cow;
use std::collections::HashMap; use std::collections::HashMap;
use std::fs; use std::fs;
use std::future::Future; use std::future::Future;
@ -32,59 +31,6 @@ use std::str;
use std::sync::Arc; use std::sync::Arc;
use std::sync::Mutex; use std::sync::Mutex;
/// Structure representing a text document.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct TextDocument {
bytes: Vec<u8>,
charset: Cow<'static, str>,
}
impl TextDocument {
pub fn new(
bytes: Vec<u8>,
charset: Option<impl Into<Cow<'static, str>>>,
) -> TextDocument {
let charset = charset
.map(|cs| cs.into())
.unwrap_or_else(|| text_encoding::detect_charset(&bytes).into());
TextDocument { bytes, charset }
}
pub fn as_bytes(&self) -> &Vec<u8> {
&self.bytes
}
pub fn into_bytes(self) -> Vec<u8> {
self.bytes
}
pub fn to_str(&self) -> Result<Cow<str>, std::io::Error> {
text_encoding::convert_to_utf8(&self.bytes, &self.charset)
}
pub fn to_string(&self) -> Result<String, std::io::Error> {
self.to_str().map(String::from)
}
}
impl From<Vec<u8>> for TextDocument {
fn from(bytes: Vec<u8>) -> Self {
TextDocument::new(bytes, Option::<&str>::None)
}
}
impl From<String> for TextDocument {
fn from(s: String) -> Self {
TextDocument::new(s.as_bytes().to_vec(), Option::<&str>::None)
}
}
impl From<&str> for TextDocument {
fn from(s: &str) -> Self {
TextDocument::new(s.as_bytes().to_vec(), Option::<&str>::None)
}
}
/// Structure representing local or remote file. /// Structure representing local or remote file.
/// ///
/// In case of remote file `url` might be different than originally requested URL, if so /// In case of remote file `url` might be different than originally requested URL, if so
@ -95,7 +41,7 @@ pub struct SourceFile {
pub filename: PathBuf, pub filename: PathBuf,
pub types_header: Option<String>, pub types_header: Option<String>,
pub media_type: MediaType, pub media_type: MediaType,
pub source_code: TextDocument, pub source_code: String,
} }
/// Simple struct implementing in-process caching to prevent multiple /// Simple struct implementing in-process caching to prevent multiple
@ -242,9 +188,8 @@ impl SourceFileFetcher {
match result { match result {
Ok(mut file) => { Ok(mut file) => {
// TODO: move somewhere? // TODO: move somewhere?
if file.source_code.bytes.starts_with(b"#!") { if file.source_code.starts_with("#!") {
file.source_code = file.source_code = filter_shebang(&file.source_code);
filter_shebang(&file.source_code.to_str().unwrap()[..]).into();
} }
// Cache in-process for subsequent access. // Cache in-process for subsequent access.
@ -369,17 +314,18 @@ impl SourceFileFetcher {
.map_err(|()| uri_error("File URL contains invalid path"))?; .map_err(|()| uri_error("File URL contains invalid path"))?;
permissions.check_read(&filepath)?; permissions.check_read(&filepath)?;
let source_code = match fs::read(filepath.clone()) { let bytes = fs::read(filepath.clone())?;
Ok(c) => c, let source_code = text_encoding::convert_to_utf8(
Err(e) => return Err(e.into()), &bytes,
}; text_encoding::detect_charset(&bytes),
)?
let (media_type, charset) = map_content_type(&filepath, None); .to_string();
let (media_type, _) = map_content_type(&filepath, None);
Ok(SourceFile { Ok(SourceFile {
url: module_url.clone(), url: module_url.clone(),
filename: filepath, filename: filepath,
media_type, media_type,
source_code: TextDocument::new(source_code, charset), source_code,
types_header: None, types_header: None,
}) })
} }
@ -444,12 +390,17 @@ impl SourceFileFetcher {
&fake_filepath, &fake_filepath,
headers.get("content-type").map(|e| e.as_str()), headers.get("content-type").map(|e| e.as_str()),
); );
let source_code = if let Some(charset) = charset {
text_encoding::convert_to_utf8(&source_code, &charset)?.to_string()
} else {
String::from_utf8(source_code)?
};
let types_header = headers.get("x-typescript-types").map(|e| e.to_string()); let types_header = headers.get("x-typescript-types").map(|e| e.to_string());
Ok(Some(SourceFile { Ok(Some(SourceFile {
url: module_url.clone(), url: module_url.clone(),
filename: cache_filename, filename: cache_filename,
media_type, media_type,
source_code: TextDocument::new(source_code, charset), source_code,
types_header, types_header,
})) }))
} }
@ -549,6 +500,11 @@ impl SourceFileFetcher {
&fake_filepath, &fake_filepath,
headers.get("content-type").map(String::as_str), headers.get("content-type").map(String::as_str),
); );
let source_code = if let Some(charset) = charset {
text_encoding::convert_to_utf8(&source, &charset)?.to_string()
} else {
String::from_utf8(source)?
};
let types_header = let types_header =
headers.get("x-typescript-types").map(String::to_string); headers.get("x-typescript-types").map(String::to_string);
@ -557,7 +513,7 @@ impl SourceFileFetcher {
url: module_url.clone(), url: module_url.clone(),
filename: cache_filepath, filename: cache_filepath,
media_type, media_type,
source_code: TextDocument::new(source, charset), source_code,
types_header, types_header,
}; };
@ -631,12 +587,12 @@ fn map_js_like_extension(path: &Path, default: MediaType) -> MediaType {
} }
} }
fn filter_shebang(string: &str) -> Vec<u8> { fn filter_shebang(string: &str) -> String {
if let Some(i) = string.find('\n') { if let Some(i) = string.find('\n') {
let (_, rest) = string.split_at(i); let (_, rest) = string.split_at(i);
rest.as_bytes().to_owned() rest.to_string()
} else { } else {
Vec::new() "".to_string()
} }
} }
@ -811,8 +767,8 @@ mod tests {
assert!(result.is_ok()); assert!(result.is_ok());
let r = result.unwrap(); let r = result.unwrap();
assert_eq!( assert_eq!(
r.source_code.bytes, r.source_code,
&b"export { printHello } from \"./print_hello.ts\";\n"[..] "export { printHello } from \"./print_hello.ts\";\n"
); );
assert_eq!(&(r.media_type), &MediaType::TypeScript); assert_eq!(&(r.media_type), &MediaType::TypeScript);
@ -838,8 +794,8 @@ mod tests {
assert!(result2.is_ok()); assert!(result2.is_ok());
let r2 = result2.unwrap(); let r2 = result2.unwrap();
assert_eq!( assert_eq!(
r2.source_code.bytes, r2.source_code,
&b"export { printHello } from \"./print_hello.ts\";\n"[..] "export { printHello } from \"./print_hello.ts\";\n"
); );
// If get_source_file does not call remote, this should be JavaScript // If get_source_file does not call remote, this should be JavaScript
// as we modified before! (we do not overwrite .headers.json due to no http fetch) // as we modified before! (we do not overwrite .headers.json due to no http fetch)
@ -867,8 +823,8 @@ mod tests {
assert!(result3.is_ok()); assert!(result3.is_ok());
let r3 = result3.unwrap(); let r3 = result3.unwrap();
assert_eq!( assert_eq!(
r3.source_code.bytes, r3.source_code,
&b"export { printHello } from \"./print_hello.ts\";\n"[..] "export { printHello } from \"./print_hello.ts\";\n"
); );
// If get_source_file does not call remote, this should be JavaScript // If get_source_file does not call remote, this should be JavaScript
// as we modified before! (we do not overwrite .headers.json due to no http fetch) // as we modified before! (we do not overwrite .headers.json due to no http fetch)
@ -893,8 +849,8 @@ mod tests {
.await; .await;
assert!(result4.is_ok()); assert!(result4.is_ok());
let r4 = result4.unwrap(); let r4 = result4.unwrap();
let expected4 = &b"export { printHello } from \"./print_hello.ts\";\n"[..]; let expected4 = "export { printHello } from \"./print_hello.ts\";\n";
assert_eq!(r4.source_code.bytes, expected4); assert_eq!(r4.source_code, expected4);
// Resolved back to TypeScript // Resolved back to TypeScript
assert_eq!(&(r4.media_type), &MediaType::TypeScript); assert_eq!(&(r4.media_type), &MediaType::TypeScript);
} }
@ -921,8 +877,8 @@ mod tests {
.await; .await;
assert!(result.is_ok()); assert!(result.is_ok());
let r = result.unwrap(); let r = result.unwrap();
let expected = b"export const loaded = true;\n"; let expected = "export const loaded = true;\n";
assert_eq!(r.source_code.bytes, expected); assert_eq!(r.source_code, expected);
assert_eq!(&(r.media_type), &MediaType::JavaScript); assert_eq!(&(r.media_type), &MediaType::JavaScript);
let (_, headers) = fetcher.http_cache.get(&module_url).unwrap(); let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();
assert_eq!(headers.get("content-type").unwrap(), "text/javascript"); assert_eq!(headers.get("content-type").unwrap(), "text/javascript");
@ -947,8 +903,8 @@ mod tests {
.await; .await;
assert!(result2.is_ok()); assert!(result2.is_ok());
let r2 = result2.unwrap(); let r2 = result2.unwrap();
let expected2 = b"export const loaded = true;\n"; let expected2 = "export const loaded = true;\n";
assert_eq!(r2.source_code.bytes, expected2); assert_eq!(r2.source_code, expected2);
// If get_source_file does not call remote, this should be TypeScript // If get_source_file does not call remote, this should be TypeScript
// as we modified before! (we do not overwrite .headers.json due to no http // as we modified before! (we do not overwrite .headers.json due to no http
// fetch) // fetch)
@ -973,8 +929,8 @@ mod tests {
.await; .await;
assert!(result3.is_ok()); assert!(result3.is_ok());
let r3 = result3.unwrap(); let r3 = result3.unwrap();
let expected3 = b"export const loaded = true;\n"; let expected3 = "export const loaded = true;\n";
assert_eq!(r3.source_code.bytes, expected3); assert_eq!(r3.source_code, expected3);
// Now the old .headers.json file should be overwritten back to JavaScript! // Now the old .headers.json file should be overwritten back to JavaScript!
// (due to http fetch) // (due to http fetch)
assert_eq!(&(r3.media_type), &MediaType::JavaScript); assert_eq!(&(r3.media_type), &MediaType::JavaScript);
@ -1378,7 +1334,7 @@ mod tests {
.await; .await;
assert!(result.is_ok()); assert!(result.is_ok());
let r = result.unwrap(); let r = result.unwrap();
assert_eq!(r.source_code.bytes, b"export const loaded = true;\n"); assert_eq!(r.source_code, "export const loaded = true;\n");
assert_eq!(&(r.media_type), &MediaType::TypeScript); assert_eq!(&(r.media_type), &MediaType::TypeScript);
// Modify .metadata.json, make sure read from local // Modify .metadata.json, make sure read from local
@ -1394,7 +1350,7 @@ mod tests {
let result2 = fetcher.fetch_cached_remote_source(&module_url, 1); let result2 = fetcher.fetch_cached_remote_source(&module_url, 1);
assert!(result2.is_ok()); assert!(result2.is_ok());
let r2 = result2.unwrap().unwrap(); let r2 = result2.unwrap().unwrap();
assert_eq!(r2.source_code.bytes, b"export const loaded = true;\n"); assert_eq!(r2.source_code, "export const loaded = true;\n");
// Not MediaType::TypeScript due to .headers.json modification // Not MediaType::TypeScript due to .headers.json modification
assert_eq!(&(r2.media_type), &MediaType::JavaScript); assert_eq!(&(r2.media_type), &MediaType::JavaScript);
} }
@ -1416,7 +1372,7 @@ mod tests {
.await; .await;
assert!(result.is_ok()); assert!(result.is_ok());
let r = result.unwrap(); let r = result.unwrap();
assert_eq!(r.source_code.bytes, b"export const loaded = true;\n"); assert_eq!(r.source_code, "export const loaded = true;\n");
assert_eq!(&(r.media_type), &MediaType::TypeScript); assert_eq!(&(r.media_type), &MediaType::TypeScript);
let (_, headers) = fetcher.http_cache.get(module_url).unwrap(); let (_, headers) = fetcher.http_cache.get(module_url).unwrap();
assert_eq!(headers.get("content-type").unwrap(), "text/typescript"); assert_eq!(headers.get("content-type").unwrap(), "text/typescript");
@ -1440,7 +1396,7 @@ mod tests {
.await; .await;
assert!(result.is_ok()); assert!(result.is_ok());
let r2 = result.unwrap(); let r2 = result.unwrap();
assert_eq!(r2.source_code.bytes, b"export const loaded = true;\n"); assert_eq!(r2.source_code, "export const loaded = true;\n");
assert_eq!(&(r2.media_type), &MediaType::JavaScript); assert_eq!(&(r2.media_type), &MediaType::JavaScript);
let (_, headers) = fetcher.http_cache.get(module_url).unwrap(); let (_, headers) = fetcher.http_cache.get(module_url).unwrap();
assert_eq!(headers.get("content-type").unwrap(), "text/javascript"); assert_eq!(headers.get("content-type").unwrap(), "text/javascript");
@ -1464,7 +1420,7 @@ mod tests {
.await; .await;
assert!(result.is_ok()); assert!(result.is_ok());
let r3 = result.unwrap(); let r3 = result.unwrap();
assert_eq!(r3.source_code.bytes, b"export const loaded = true;\n"); assert_eq!(r3.source_code, "export const loaded = true;\n");
assert_eq!(&(r3.media_type), &MediaType::TypeScript); assert_eq!(&(r3.media_type), &MediaType::TypeScript);
let (_, headers) = fetcher.http_cache.get(module_url).unwrap(); let (_, headers) = fetcher.http_cache.get(module_url).unwrap();
assert_eq!(headers.get("content-type").unwrap(), "text/typescript"); assert_eq!(headers.get("content-type").unwrap(), "text/typescript");
@ -1559,10 +1515,7 @@ mod tests {
.await; .await;
assert!(r.is_ok()); assert!(r.is_ok());
let fetched_file = r.unwrap(); let fetched_file = r.unwrap();
let source_code = fetched_file.source_code.to_str(); assert_eq!(expected_content, fetched_file.source_code);
assert!(source_code.is_ok());
let actual = source_code.unwrap();
assert_eq!(expected_content, actual);
} }
#[tokio::test] #[tokio::test]
@ -1795,10 +1748,10 @@ mod tests {
#[test] #[test]
fn test_filter_shebang() { fn test_filter_shebang() {
assert_eq!(filter_shebang("#!"), b""); assert_eq!(filter_shebang("#!"), "");
assert_eq!(filter_shebang("#!\n\n"), b"\n\n"); assert_eq!(filter_shebang("#!\n\n"), "\n\n");
let code = "#!/usr/bin/env deno\nconsole.log('hello');\n"; let code = "#!/usr/bin/env deno\nconsole.log('hello');\n";
assert_eq!(filter_shebang(code), b"\nconsole.log('hello');\n"); assert_eq!(filter_shebang(code), "\nconsole.log('hello');\n");
} }
#[tokio::test] #[tokio::test]
@ -1819,7 +1772,7 @@ mod tests {
.await; .await;
assert!(source.is_ok()); assert!(source.is_ok());
let source = source.unwrap(); let source = source.unwrap();
assert_eq!(source.source_code.bytes, b"console.log('etag')"); assert_eq!(source.source_code, "console.log('etag')");
assert_eq!(&(source.media_type), &MediaType::TypeScript); assert_eq!(&(source.media_type), &MediaType::TypeScript);
let (_, headers) = fetcher.http_cache.get(&module_url).unwrap(); let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();
@ -1846,7 +1799,7 @@ mod tests {
) )
.await .await
.unwrap(); .unwrap();
assert_eq!(cached_source.source_code.bytes, b"changed content"); assert_eq!(cached_source.source_code, "changed content");
let modified2 = metadata_path.metadata().unwrap().modified().unwrap(); let modified2 = metadata_path.metadata().unwrap().modified().unwrap();
@ -1871,7 +1824,7 @@ mod tests {
.await; .await;
assert!(source.is_ok()); assert!(source.is_ok());
let source = source.unwrap(); let source = source.unwrap();
assert_eq!(source.source_code.bytes, b"export const foo = 'foo';"); assert_eq!(source.source_code, "export const foo = 'foo';");
assert_eq!(&(source.media_type), &MediaType::JavaScript); assert_eq!(&(source.media_type), &MediaType::JavaScript);
assert_eq!( assert_eq!(
source.types_header, source.types_header,
@ -1941,9 +1894,7 @@ mod tests {
.await; .await;
assert!(source.is_ok()); assert!(source.is_ok());
let source = source.unwrap(); let source = source.unwrap();
assert_eq!(&source.source_code.charset.to_lowercase()[..], charset); assert_eq!(source.source_code, expected_content);
let text = &source.source_code.to_str().unwrap();
assert_eq!(text, expected_content);
assert_eq!(&(source.media_type), &MediaType::TypeScript); assert_eq!(&(source.media_type), &MediaType::TypeScript);
let (_, headers) = fetcher.http_cache.get(&module_url).unwrap(); let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();

2
cli/global_state.rs
View file

@ -247,7 +247,7 @@ impl GlobalState {
} }
} else { } else {
CompiledModule { CompiledModule {
code: out.source_code.to_string()?, code: out.source_code,
name: out.url.to_string(), name: out.url.to_string(),
} }
}; };

17
cli/main.rs
View file

@ -60,7 +60,6 @@ use crate::coverage::CoverageCollector;
use crate::coverage::PrettyCoverageReporter; use crate::coverage::PrettyCoverageReporter;
use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFile;
use crate::file_fetcher::SourceFileFetcher; use crate::file_fetcher::SourceFileFetcher;
use crate::file_fetcher::TextDocument;
use crate::fs as deno_fs; use crate::fs as deno_fs;
use crate::global_state::GlobalState; use crate::global_state::GlobalState;
use crate::media_type::MediaType; use crate::media_type::MediaType;
@ -266,7 +265,7 @@ async fn eval_command(
} else { } else {
MediaType::JavaScript MediaType::JavaScript
}, },
source_code: TextDocument::new(source_code, Some("utf-8")), source_code: String::from_utf8(source_code)?,
}; };
// Save our fake file into file fetcher cache // Save our fake file into file fetcher cache
// to allow module access by TS compiler. // to allow module access by TS compiler.
@ -358,12 +357,7 @@ async fn doc_command(
e.to_string(), e.to_string(),
)) ))
})?; })?;
source_file.source_code.to_string().map_err(|e| { Ok(source_file.source_code)
doc::DocError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
e.to_string(),
))
})
} }
.boxed_local() .boxed_local()
} }
@ -449,7 +443,7 @@ async fn run_from_stdin(flags: Flags) -> Result<(), AnyError> {
url: main_module_url, url: main_module_url,
types_header: None, types_header: None,
media_type: MediaType::TypeScript, media_type: MediaType::TypeScript,
source_code: source.into(), source_code: String::from_utf8(source)?,
}; };
// Save our fake file into file fetcher cache // Save our fake file into file fetcher cache
// to allow module access by TS compiler // to allow module access by TS compiler
@ -575,10 +569,7 @@ async fn test_command(
url: test_file_url.clone(), url: test_file_url.clone(),
types_header: None, types_header: None,
media_type: MediaType::TypeScript, media_type: MediaType::TypeScript,
source_code: TextDocument::new( source_code: test_file.clone(),
test_file.clone().into_bytes(),
Some("utf-8"),
),
}; };
// Save our fake file into file fetcher cache // Save our fake file into file fetcher cache
// to allow module access by TS compiler // to allow module access by TS compiler

2
cli/module_graph.rs
View file

@ -489,7 +489,7 @@ impl ModuleGraphLoader {
&source_file.source_code.as_bytes(), &source_file.source_code.as_bytes(),
version::DENO.as_bytes(), version::DENO.as_bytes(),
]); ]);
let source_code = source_file.source_code.to_string()?; let source_code = source_file.source_code.clone();
if SUPPORTED_MEDIA_TYPES.contains(&source_file.media_type) { if SUPPORTED_MEDIA_TYPES.contains(&source_file.media_type) {
if let Some(types_specifier) = source_file.types_header { if let Some(types_specifier) = source_file.types_header {

79
cli/module_graph2.rs
View file

@ -4,7 +4,6 @@ use crate::ast;
use crate::ast::parse; use crate::ast::parse;
use crate::ast::Location; use crate::ast::Location;
use crate::ast::ParsedModule; use crate::ast::ParsedModule;
use crate::file_fetcher::TextDocument;
use crate::import_map::ImportMap; use crate::import_map::ImportMap;
use crate::lockfile::Lockfile; use crate::lockfile::Lockfile;
use crate::media_type::MediaType; use crate::media_type::MediaType;
@ -37,7 +36,7 @@ use std::sync::Mutex;
use std::time::Instant; use std::time::Instant;
use swc_ecmascript::dep_graph::DependencyKind; use swc_ecmascript::dep_graph::DependencyKind;
pub type BuildInfoMap = HashMap<EmitType, TextDocument>; pub type BuildInfoMap = HashMap<EmitType, String>;
lazy_static! { lazy_static! {
/// Matched the `@deno-types` pragma. /// Matched the `@deno-types` pragma.
@ -151,12 +150,8 @@ fn parse_deno_types(comment: &str) -> Option<String> {
/// A hashing function that takes the source code, version and optionally a /// A hashing function that takes the source code, version and optionally a
/// user provided config and generates a string hash which can be stored to /// user provided config and generates a string hash which can be stored to
/// determine if the cached emit is valid or not. /// determine if the cached emit is valid or not.
fn get_version(source: &TextDocument, version: &str, config: &[u8]) -> String { fn get_version(source: &str, version: &str, config: &[u8]) -> String {
crate::checksum::gen(&[ crate::checksum::gen(&[source.as_bytes(), version.as_bytes(), config])
source.to_str().unwrap().as_bytes(),
version.as_bytes(),
config,
])
} }
/// A logical representation of a module within a graph. /// A logical representation of a module within a graph.
@ -173,7 +168,7 @@ struct Module {
maybe_version: Option<String>, maybe_version: Option<String>,
media_type: MediaType, media_type: MediaType,
specifier: ModuleSpecifier, specifier: ModuleSpecifier,
source: TextDocument, source: String,
} }
impl Default for Module { impl Default for Module {
@ -190,7 +185,7 @@ impl Default for Module {
maybe_version: None, maybe_version: None,
media_type: MediaType::Unknown, media_type: MediaType::Unknown,
specifier: ModuleSpecifier::resolve_url("https://deno.land/x/").unwrap(), specifier: ModuleSpecifier::resolve_url("https://deno.land/x/").unwrap(),
source: TextDocument::new(Vec::new(), Option::<&str>::None), source: "".to_string(),
} }
} }
} }
@ -243,8 +238,7 @@ impl Module {
} }
pub fn parse(&mut self) -> Result<(), AnyError> { pub fn parse(&mut self) -> Result<(), AnyError> {
let parsed_module = let parsed_module = parse(&self.specifier, &self.source, &self.media_type)?;
parse(&self.specifier, &self.source.to_str()?, &self.media_type)?;
// parse out any triple slash references // parse out any triple slash references
for comment in parsed_module.get_leading_comments().iter() { for comment in parsed_module.get_leading_comments().iter() {
@ -470,8 +464,7 @@ impl Graph2 {
let mut lockfile = lf.lock().unwrap(); let mut lockfile = lf.lock().unwrap();
for (ms, module) in self.modules.iter() { for (ms, module) in self.modules.iter() {
let specifier = module.specifier.to_string(); let specifier = module.specifier.to_string();
let code = module.source.to_string()?; let valid = lockfile.check_or_insert(&specifier, &module.source);
let valid = lockfile.check_or_insert(&specifier, &code);
if !valid { if !valid {
return Err( return Err(
InvalidSource(ms.clone(), lockfile.filename.clone()).into(), InvalidSource(ms.clone(), lockfile.filename.clone()).into(),
@ -702,14 +695,9 @@ mod tests {
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub struct MockSpecifierHandler { pub struct MockSpecifierHandler {
pub fixtures: PathBuf, pub fixtures: PathBuf,
pub build_info: HashMap<ModuleSpecifier, TextDocument>, pub build_info: HashMap<ModuleSpecifier, String>,
pub build_info_calls: Vec<(ModuleSpecifier, EmitType, TextDocument)>, pub build_info_calls: Vec<(ModuleSpecifier, EmitType, String)>,
pub cache_calls: Vec<( pub cache_calls: Vec<(ModuleSpecifier, EmitType, String, Option<String>)>,
ModuleSpecifier,
EmitType,
TextDocument,
Option<TextDocument>,
)>,
pub deps_calls: Vec<(ModuleSpecifier, DependencyMap)>, pub deps_calls: Vec<(ModuleSpecifier, DependencyMap)>,
pub types_calls: Vec<(ModuleSpecifier, String)>, pub types_calls: Vec<(ModuleSpecifier, String)>,
pub version_calls: Vec<(ModuleSpecifier, String)>, pub version_calls: Vec<(ModuleSpecifier, String)>,
@ -740,8 +728,7 @@ mod tests {
"jsx" => MediaType::JSX, "jsx" => MediaType::JSX,
_ => MediaType::Unknown, _ => MediaType::Unknown,
}; };
let source = let source = fs::read_to_string(specifier_path)?;
TextDocument::new(fs::read(specifier_path)?, Option::<&str>::None);
Ok(CachedModule { Ok(CachedModule {
source, source,
@ -760,15 +747,15 @@ mod tests {
&self, &self,
specifier: &ModuleSpecifier, specifier: &ModuleSpecifier,
_cache_type: &EmitType, _cache_type: &EmitType,
) -> Result<Option<TextDocument>, AnyError> { ) -> Result<Option<String>, AnyError> {
Ok(self.build_info.get(specifier).cloned()) Ok(self.build_info.get(specifier).cloned())
} }
fn set_cache( fn set_cache(
&mut self, &mut self,
specifier: &ModuleSpecifier, specifier: &ModuleSpecifier,
cache_type: &EmitType, cache_type: &EmitType,
code: TextDocument, code: String,
maybe_map: Option<TextDocument>, maybe_map: Option<String>,
) -> Result<(), AnyError> { ) -> Result<(), AnyError> {
self.cache_calls.push(( self.cache_calls.push((
specifier.clone(), specifier.clone(),
@ -790,7 +777,7 @@ mod tests {
&mut self, &mut self,
specifier: &ModuleSpecifier, specifier: &ModuleSpecifier,
cache_type: &EmitType, cache_type: &EmitType,
build_info: TextDocument, build_info: String,
) -> Result<(), AnyError> { ) -> Result<(), AnyError> {
self self
.build_info .build_info
@ -822,11 +809,9 @@ mod tests {
#[test] #[test]
fn test_get_version() { fn test_get_version() {
let doc_a = let doc_a = "console.log(42);";
TextDocument::new(b"console.log(42);".to_vec(), Option::<&str>::None);
let version_a = get_version(&doc_a, "1.2.3", b""); let version_a = get_version(&doc_a, "1.2.3", b"");
let doc_b = let doc_b = "console.log(42);";
TextDocument::new(b"console.log(42);".to_vec(), Option::<&str>::None);
let version_b = get_version(&doc_b, "1.2.3", b""); let version_b = get_version(&doc_b, "1.2.3", b"");
assert_eq!(version_a, version_b); assert_eq!(version_a, version_b);
@ -845,8 +830,7 @@ mod tests {
#[test] #[test]
fn test_module_emit_valid() { fn test_module_emit_valid() {
let source = let source = "console.log(42);".to_string();
TextDocument::new(b"console.log(42);".to_vec(), Option::<&str>::None);
let maybe_version = Some(get_version(&source, version::DENO, b"")); let maybe_version = Some(get_version(&source, version::DENO, b""));
let module = Module { let module = Module {
source, source,
@ -855,11 +839,9 @@ mod tests {
}; };
assert!(module.emit_valid(b"")); assert!(module.emit_valid(b""));
let source = let source = "console.log(42);".to_string();
TextDocument::new(b"console.log(42);".to_vec(), Option::<&str>::None); let old_source = "console.log(43);";
let old_source = let maybe_version = Some(get_version(old_source, version::DENO, b""));
TextDocument::new(b"console.log(43);".to_vec(), Option::<&str>::None);
let maybe_version = Some(get_version(&old_source, version::DENO, b""));
let module = Module { let module = Module {
source, source,
maybe_version, maybe_version,
@ -867,8 +849,7 @@ mod tests {
}; };
assert!(!module.emit_valid(b"")); assert!(!module.emit_valid(b""));
let source = let source = "console.log(42);".to_string();
TextDocument::new(b"console.log(42);".to_vec(), Option::<&str>::None);
let maybe_version = Some(get_version(&source, "0.0.0", b"")); let maybe_version = Some(get_version(&source, "0.0.0", b""));
let module = Module { let module = Module {
source, source,
@ -877,8 +858,7 @@ mod tests {
}; };
assert!(!module.emit_valid(b"")); assert!(!module.emit_valid(b""));
let source = let source = "console.log(42);".to_string();
TextDocument::new(b"console.log(42);".to_vec(), Option::<&str>::None);
let module = Module { let module = Module {
source, source,
..Module::default() ..Module::default()
@ -888,8 +868,7 @@ mod tests {
#[test] #[test]
fn test_module_set_version() { fn test_module_set_version() {
let source = let source = "console.log(42);".to_string();
TextDocument::new(b"console.log(42);".to_vec(), Option::<&str>::None);
let expected = Some(get_version(&source, version::DENO, b"")); let expected = Some(get_version(&source, version::DENO, b""));
let mut module = Module { let mut module = Module {
source, source,
@ -933,15 +912,11 @@ mod tests {
assert_eq!(h.cache_calls[0].1, EmitType::Cli); assert_eq!(h.cache_calls[0].1, EmitType::Cli);
assert!(h.cache_calls[0] assert!(h.cache_calls[0]
.2 .2
.to_string()
.unwrap()
.contains("# sourceMappingURL=data:application/json;base64,")); .contains("# sourceMappingURL=data:application/json;base64,"));
assert_eq!(h.cache_calls[0].3, None); assert_eq!(h.cache_calls[0].3, None);
assert_eq!(h.cache_calls[1].1, EmitType::Cli); assert_eq!(h.cache_calls[1].1, EmitType::Cli);
assert!(h.cache_calls[1] assert!(h.cache_calls[1]
.2 .2
.to_string()
.unwrap()
.contains("# sourceMappingURL=data:application/json;base64,")); .contains("# sourceMappingURL=data:application/json;base64,"));
assert_eq!(h.cache_calls[0].3, None); assert_eq!(h.cache_calls[0].3, None);
assert_eq!(h.deps_calls.len(), 7); assert_eq!(h.deps_calls.len(), 7);
@ -1002,11 +977,7 @@ mod tests {
assert_eq!(h.cache_calls.len(), 1, "only one file should be emitted"); assert_eq!(h.cache_calls.len(), 1, "only one file should be emitted");
// FIXME(bartlomieju): had to add space in `<div>`, probably a quirk in swc_ecma_codegen // FIXME(bartlomieju): had to add space in `<div>`, probably a quirk in swc_ecma_codegen
assert!( assert!(
h.cache_calls[0] h.cache_calls[0].2.contains("<div >Hello world!</div>"),
.2
.to_string()
.unwrap()
.contains("<div >Hello world!</div>"),
"jsx should have been preserved" "jsx should have been preserved"
); );
} }

45
cli/specifier_handler.rs
View file

@ -3,7 +3,6 @@
use crate::deno_dir::DenoDir; use crate::deno_dir::DenoDir;
use crate::disk_cache::DiskCache; use crate::disk_cache::DiskCache;
use crate::file_fetcher::SourceFileFetcher; use crate::file_fetcher::SourceFileFetcher;
use crate::file_fetcher::TextDocument;
use crate::global_state::GlobalState; use crate::global_state::GlobalState;
use crate::media_type::MediaType; use crate::media_type::MediaType;
use crate::permissions::Permissions; use crate::permissions::Permissions;
@ -23,7 +22,7 @@ use std::pin::Pin;
use std::sync::Arc; use std::sync::Arc;
pub type DependencyMap = HashMap<String, Dependency>; pub type DependencyMap = HashMap<String, Dependency>;
pub type EmitMap = HashMap<EmitType, (TextDocument, Option<TextDocument>)>; pub type EmitMap = HashMap<EmitType, (String, Option<String>)>;
pub type FetchFuture = pub type FetchFuture =
Pin<Box<(dyn Future<Output = Result<CachedModule, AnyError>> + 'static)>>; Pin<Box<(dyn Future<Output = Result<CachedModule, AnyError>> + 'static)>>;
@ -34,7 +33,7 @@ pub struct CachedModule {
pub maybe_types: Option<String>, pub maybe_types: Option<String>,
pub maybe_version: Option<String>, pub maybe_version: Option<String>,
pub media_type: MediaType, pub media_type: MediaType,
pub source: TextDocument, pub source: String,
pub specifier: ModuleSpecifier, pub specifier: ModuleSpecifier,
} }
@ -47,7 +46,7 @@ impl Default for CachedModule {
maybe_types: None, maybe_types: None,
maybe_version: None, maybe_version: None,
media_type: MediaType::Unknown, media_type: MediaType::Unknown,
source: TextDocument::new(Vec::new(), Option::<&str>::None), source: "".to_string(),
specifier: ModuleSpecifier::resolve_url("https://deno.land/x/mod.ts") specifier: ModuleSpecifier::resolve_url("https://deno.land/x/mod.ts")
.unwrap(), .unwrap(),
} }
@ -98,7 +97,7 @@ pub trait SpecifierHandler {
&self, &self,
specifier: &ModuleSpecifier, specifier: &ModuleSpecifier,
emit_type: &EmitType, emit_type: &EmitType,
) -> Result<Option<TextDocument>, AnyError>; ) -> Result<Option<String>, AnyError>;
/// Set the emitted code (and maybe map) for a given module specifier. The /// Set the emitted code (and maybe map) for a given module specifier. The
/// cache type indicates what form the emit is related to. /// cache type indicates what form the emit is related to.
@ -106,8 +105,8 @@ pub trait SpecifierHandler {
&mut self, &mut self,
specifier: &ModuleSpecifier, specifier: &ModuleSpecifier,
emit_type: &EmitType, emit_type: &EmitType,
code: TextDocument, code: String,
maybe_map: Option<TextDocument>, maybe_map: Option<String>,
) -> Result<(), AnyError>; ) -> Result<(), AnyError>;
/// When parsed out of a JavaScript module source, the triple slash reference /// When parsed out of a JavaScript module source, the triple slash reference
@ -123,7 +122,7 @@ pub trait SpecifierHandler {
&mut self, &mut self,
specifier: &ModuleSpecifier, specifier: &ModuleSpecifier,
emit_type: &EmitType, emit_type: &EmitType,
build_info: TextDocument, build_info: String,
) -> Result<(), AnyError>; ) -> Result<(), AnyError>;
/// Set the graph dependencies for a given module specifier. /// Set the graph dependencies for a given module specifier.
@ -245,16 +244,16 @@ impl SpecifierHandler for FetchHandler {
let filename = let filename =
disk_cache.get_cache_filename_with_extension(&url, "js.map"); disk_cache.get_cache_filename_with_extension(&url, "js.map");
let maybe_map: Option<TextDocument> = let maybe_map: Option<String> = if let Ok(map) = disk_cache.get(&filename)
if let Ok(map) = disk_cache.get(&filename) { {
Some(map.into()) Some(String::from_utf8(map)?)
} else { } else {
None None
}; };
let mut emits = HashMap::new(); let mut emits = HashMap::new();
let filename = disk_cache.get_cache_filename_with_extension(&url, "js"); let filename = disk_cache.get_cache_filename_with_extension(&url, "js");
if let Ok(code) = disk_cache.get(&filename) { if let Ok(code) = disk_cache.get(&filename) {
emits.insert(EmitType::Cli, (code.into(), maybe_map)); emits.insert(EmitType::Cli, (String::from_utf8(code)?, maybe_map));
}; };
Ok(CachedModule { Ok(CachedModule {
@ -274,7 +273,7 @@ impl SpecifierHandler for FetchHandler {
&self, &self,
specifier: &ModuleSpecifier, specifier: &ModuleSpecifier,
emit_type: &EmitType, emit_type: &EmitType,
) -> Result<Option<TextDocument>, AnyError> { ) -> Result<Option<String>, AnyError> {
if emit_type != &EmitType::Cli { if emit_type != &EmitType::Cli {
return Err(UnsupportedEmitType(emit_type.clone()).into()); return Err(UnsupportedEmitType(emit_type.clone()).into());
} }
@ -282,7 +281,7 @@ impl SpecifierHandler for FetchHandler {
.disk_cache .disk_cache
.get_cache_filename_with_extension(specifier.as_url(), "buildinfo"); .get_cache_filename_with_extension(specifier.as_url(), "buildinfo");
if let Ok(build_info) = self.disk_cache.get(&filename) { if let Ok(build_info) = self.disk_cache.get(&filename) {
return Ok(Some(build_info.into())); return Ok(Some(String::from_utf8(build_info)?));
} }
Ok(None) Ok(None)
@ -292,7 +291,7 @@ impl SpecifierHandler for FetchHandler {
&mut self, &mut self,
specifier: &ModuleSpecifier, specifier: &ModuleSpecifier,
emit_type: &EmitType, emit_type: &EmitType,
build_info: TextDocument, build_info: String,
) -> Result<(), AnyError> { ) -> Result<(), AnyError> {
if emit_type != &EmitType::Cli { if emit_type != &EmitType::Cli {
return Err(UnsupportedEmitType(emit_type.clone()).into()); return Err(UnsupportedEmitType(emit_type.clone()).into());
@ -310,8 +309,8 @@ impl SpecifierHandler for FetchHandler {
&mut self, &mut self,
specifier: &ModuleSpecifier, specifier: &ModuleSpecifier,
emit_type: &EmitType, emit_type: &EmitType,
code: TextDocument, code: String,
maybe_map: Option<TextDocument>, maybe_map: Option<String>,
) -> Result<(), AnyError> { ) -> Result<(), AnyError> {
if emit_type != &EmitType::Cli { if emit_type != &EmitType::Cli {
return Err(UnsupportedEmitType(emit_type.clone()).into()); return Err(UnsupportedEmitType(emit_type.clone()).into());
@ -414,7 +413,7 @@ pub mod tests {
assert!(cached_module.maybe_dependencies.is_none()); assert!(cached_module.maybe_dependencies.is_none());
assert_eq!(cached_module.media_type, MediaType::TypeScript); assert_eq!(cached_module.media_type, MediaType::TypeScript);
assert_eq!( assert_eq!(
cached_module.source.to_str().unwrap(), cached_module.source,
"export { printHello } from \"./print_hello.ts\";\n" "export { printHello } from \"./print_hello.ts\";\n"
); );
assert_eq!(cached_module.specifier, specifier); assert_eq!(cached_module.specifier, specifier);
@ -431,7 +430,7 @@ pub mod tests {
let cached_module: CachedModule = let cached_module: CachedModule =
file_fetcher.fetch(specifier.clone()).await.unwrap(); file_fetcher.fetch(specifier.clone()).await.unwrap();
assert_eq!(cached_module.emits.len(), 0); assert_eq!(cached_module.emits.len(), 0);
let code = TextDocument::from("some code"); let code = String::from("some code");
file_fetcher file_fetcher
.set_cache(&specifier, &EmitType::Cli, code, None) .set_cache(&specifier, &EmitType::Cli, code, None)
.expect("could not set cache"); .expect("could not set cache");
@ -439,7 +438,7 @@ pub mod tests {
file_fetcher.fetch(specifier.clone()).await.unwrap(); file_fetcher.fetch(specifier.clone()).await.unwrap();
assert_eq!(cached_module.emits.len(), 1); assert_eq!(cached_module.emits.len(), 1);
let actual_emit = cached_module.emits.get(&EmitType::Cli).unwrap(); let actual_emit = cached_module.emits.get(&EmitType::Cli).unwrap();
assert_eq!(actual_emit.0.to_str().unwrap(), "some code"); assert_eq!(actual_emit.0, "some code");
assert_eq!(actual_emit.1, None); assert_eq!(actual_emit.1, None);
} }
} }

22
cli/tsc.rs
View file

@ -735,7 +735,7 @@ impl TsCompiler {
let compiled_source_file = self.get_compiled_source_file(module_url)?; let compiled_source_file = self.get_compiled_source_file(module_url)?;
let compiled_module = CompiledModule { let compiled_module = CompiledModule {
code: compiled_source_file.source_code.to_string()?, code: compiled_source_file.source_code,
name: module_url.to_string(), name: module_url.to_string(),
}; };
@ -760,7 +760,7 @@ impl TsCompiler {
url: module_url.clone(), url: module_url.clone(),
filename: compiled_code_filename, filename: compiled_code_filename,
media_type: MediaType::JavaScript, media_type: MediaType::JavaScript,
source_code: compiled_code.into(), source_code: String::from_utf8(compiled_code)?,
types_header: None, types_header: None,
}; };
@ -817,7 +817,7 @@ impl TsCompiler {
url: module_specifier.as_url().to_owned(), url: module_specifier.as_url().to_owned(),
filename: source_map_filename, filename: source_map_filename,
media_type: MediaType::JavaScript, media_type: MediaType::JavaScript,
source_code: source_code.into(), source_code: String::from_utf8(source_code)?,
types_header: None, types_header: None,
}; };
@ -862,14 +862,12 @@ impl SourceMapGetter for TsCompiler {
fn get_source_line(&self, script_name: &str, line: usize) -> Option<String> { fn get_source_line(&self, script_name: &str, line: usize) -> Option<String> {
self self
.try_resolve_and_get_source_file(script_name) .try_resolve_and_get_source_file(script_name)
.and_then(|out| { .map(|out| {
out.source_code.to_str().ok().map(|v| { // Do NOT use .lines(): it skips the terminating empty line.
// Do NOT use .lines(): it skips the terminating empty line. // (due to internally using .split_terminator() instead of .split())
// (due to internally using .split_terminator() instead of .split()) let lines: Vec<&str> = out.source_code.split('\n').collect();
let lines: Vec<&str> = v.split('\n').collect(); assert!(lines.len() > line);
assert!(lines.len() > line); lines[line].to_string()
lines[line].to_string()
})
}) })
} }
} }
@ -1528,7 +1526,7 @@ mod tests {
url: specifier.as_url().clone(), url: specifier.as_url().clone(),
filename: PathBuf::from(p.to_str().unwrap().to_string()), filename: PathBuf::from(p.to_str().unwrap().to_string()),
media_type: MediaType::TypeScript, media_type: MediaType::TypeScript,
source_code: include_bytes!("./tests/002_hello.ts").to_vec().into(), source_code: include_str!("./tests/002_hello.ts").to_string(),
types_header: None, types_header: None,
}; };
let dir = let dir =