fix(compile): handle TypeScript file included as asset (#27032)

Closes #27024

cli/standalone/binary.rs

@@ -87,6 +87,7 @@ use super::serialization::RemoteModulesStore;
 use super::serialization::RemoteModulesStoreBuilder;
 use super::virtual_fs::FileBackedVfs;
 use super::virtual_fs::VfsBuilder;
+use super::virtual_fs::VfsFileSubDataKind;
 use super::virtual_fs::VfsRoot;
 use super::virtual_fs::VirtualDirectory;
 
@@ -275,7 +276,9 @@ impl StandaloneModules {
     if specifier.scheme() == "file" {
       let path = deno_path_util::url_to_file_path(specifier)?;
       let bytes = match self.vfs.file_entry(&path) {
-        Ok(entry) => self.vfs.read_file_all(entry)?,
+        Ok(entry) => self
+          .vfs
+          .read_file_all(entry, VfsFileSubDataKind::ModuleGraph)?,
         Err(err) if err.kind() == ErrorKind::NotFound => {
           let bytes = match RealFs.read_file_sync(&path, None) {
             Ok(bytes) => bytes,
@@ -691,6 +694,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
               Some(source) => source,
               None => RealFs.read_file_sync(&file_path, None)?,
             },
+            VfsFileSubDataKind::ModuleGraph,
           )
           .with_context(|| {
             format!("Failed adding '{}'", file_path.display())

cli/standalone/file_system.rs

@@ -17,6 +17,7 @@ use deno_runtime::deno_io::fs::FsResult;
 use deno_runtime::deno_io::fs::FsStat;
 
 use super::virtual_fs::FileBackedVfs;
+use super::virtual_fs::VfsFileSubDataKind;
 
 #[derive(Debug, Clone)]
 pub struct DenoCompileFileSystem(Arc<FileBackedVfs>);
@@ -36,7 +37,8 @@ impl DenoCompileFileSystem {
 
   fn copy_to_real_path(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> {
     let old_file = self.0.file_entry(oldpath)?;
-    let old_file_bytes = self.0.read_file_all(old_file)?;
+    let old_file_bytes =
+      self.0.read_file_all(old_file, VfsFileSubDataKind::Raw)?;
     RealFs.write_file_sync(
       newpath,
       OpenOptions {

cli/standalone/mod.rs

@@ -56,6 +56,8 @@ use serialization::DenoCompileModuleSource;
 use std::borrow::Cow;
 use std::rc::Rc;
 use std::sync::Arc;
+use virtual_fs::FileBackedVfs;
+use virtual_fs::VfsFileSubDataKind;
 
 use crate::args::create_default_npmrc;
 use crate::args::get_root_cert_store;
@@ -111,6 +113,7 @@ use self::file_system::DenoCompileFileSystem;
 
 struct SharedModuleLoaderState {
   cjs_tracker: Arc<CjsTracker>,
+  code_cache: Option<Arc<dyn CliCodeCache>>,
   fs: Arc<dyn deno_fs::FileSystem>,
   modules: StandaloneModules,
   node_code_translator: Arc<CliNodeCodeTranslator>,
@@ -118,8 +121,8 @@ struct SharedModuleLoaderState {
   npm_module_loader: Arc<NpmModuleLoader>,
   npm_req_resolver: Arc<CliNpmReqResolver>,
   npm_resolver: Arc<dyn CliNpmResolver>,
+  vfs: Arc<FileBackedVfs>,
   workspace_resolver: WorkspaceResolver,
-  code_cache: Option<Arc<dyn CliCodeCache>>,
 }
 
 impl SharedModuleLoaderState {
@@ -514,7 +517,12 @@ impl NodeRequireLoader for EmbeddedModuleLoader {
     &self,
     path: &std::path::Path,
   ) -> Result<String, AnyError> {
-    Ok(self.shared.fs.read_text_file_lossy_sync(path, None)?)
+    let file_entry = self.shared.vfs.file_entry(path)?;
+    let file_bytes = self
+      .shared
+      .vfs
+      .read_file_all(file_entry, VfsFileSubDataKind::ModuleGraph)?;
+    Ok(String::from_utf8(file_bytes.into_owned())?)
   }
 
   fn is_maybe_cjs(
@@ -817,6 +825,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
   let module_loader_factory = StandaloneModuleLoaderFactory {
     shared: Arc::new(SharedModuleLoaderState {
       cjs_tracker: cjs_tracker.clone(),
+      code_cache: code_cache.clone(),
       fs: fs.clone(),
       modules,
       node_code_translator: node_code_translator.clone(),
@@ -826,10 +835,10 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
         fs.clone(),
         node_code_translator,
       )),
-      code_cache: code_cache.clone(),
       npm_resolver: npm_resolver.clone(),
-      workspace_resolver,
       npm_req_resolver,
+      vfs,
+      workspace_resolver,
     }),
   };
 

cli/standalone/virtual_fs.rs

@@ -32,6 +32,15 @@ use thiserror::Error;
 use crate::util;
 use crate::util::fs::canonicalize_path;
 
+#[derive(Debug, Copy, Clone)]
+pub enum VfsFileSubDataKind {
+  /// Raw bytes of the file.
+  Raw,
+  /// Bytes to use for module loading. For example, for TypeScript
+  /// files this will be the transpiled JavaScript source.
+  ModuleGraph,
+}
+
 #[derive(Error, Debug)]
 #[error(
   "Failed to strip prefix '{}' from '{}'", root_path.display(), target.display()
@@ -141,7 +150,11 @@ impl VfsBuilder {
                 // inline the symlink and make the target file
                 let file_bytes = std::fs::read(&target)
                   .with_context(|| format!("Reading {}", path.display()))?;
-                self.add_file_with_data_inner(&path, file_bytes)?;
+                self.add_file_with_data_inner(
+                  &path,
+                  file_bytes,
+                  VfsFileSubDataKind::Raw,
+                )?;
               } else {
                 log::warn!(
                   "{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.",
@@ -219,25 +232,27 @@ impl VfsBuilder {
   ) -> Result<(), AnyError> {
     let file_bytes = std::fs::read(path)
       .with_context(|| format!("Reading {}", path.display()))?;
-    self.add_file_with_data_inner(path, file_bytes)
+    self.add_file_with_data_inner(path, file_bytes, VfsFileSubDataKind::Raw)
   }
 
   pub fn add_file_with_data(
     &mut self,
     path: &Path,
     data: Vec<u8>,
+    sub_data_kind: VfsFileSubDataKind,
   ) -> Result<(), AnyError> {
     let target_path = canonicalize_path(path)?;
     if target_path != path {
       self.add_symlink(path, &target_path)?;
     }
-    self.add_file_with_data_inner(&target_path, data)
+    self.add_file_with_data_inner(&target_path, data, sub_data_kind)
   }
 
   fn add_file_with_data_inner(
     &mut self,
     path: &Path,
     data: Vec<u8>,
+    sub_data_kind: VfsFileSubDataKind,
   ) -> Result<(), AnyError> {
     log::debug!("Adding file '{}'", path.display());
     let checksum = util::checksum::gen(&[&data]);
@@ -253,8 +268,19 @@ impl VfsBuilder {
     let name = path.file_name().unwrap().to_string_lossy();
     let data_len = data.len();
     match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
-      Ok(_) => {
+      Ok(index) => {
-        // already added, just ignore
+        let entry = &mut dir.entries[index];
+        match entry {
+          VfsEntry::File(virtual_file) => match sub_data_kind {
+            VfsFileSubDataKind::Raw => {
+              virtual_file.offset = offset;
+            }
+            VfsFileSubDataKind::ModuleGraph => {
+              virtual_file.module_graph_offset = offset;
+            }
+          },
+          VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(),
+        }
       }
       Err(insert_index) => {
         dir.entries.insert(
@@ -262,6 +288,7 @@ impl VfsBuilder {
           VfsEntry::File(VirtualFile {
             name: name.to_string(),
             offset,
+            module_graph_offset: offset,
             len: data.len() as u64,
           }),
         );
@@ -454,6 +481,12 @@ pub struct VirtualDirectory {
 pub struct VirtualFile {
   pub name: String,
   pub offset: u64,
+  /// Offset file to use for module loading when it differs from the
+  /// raw file. Often this will be the same offset as above for data
+  /// such as JavaScript files, but for TypeScript files the `offset`
+  /// will be the original raw bytes when included as an asset and this
+  /// offset will be to the transpiled JavaScript source.
+  pub module_graph_offset: u64,
   pub len: u64,
 }
 
@@ -647,7 +680,7 @@ impl FileBackedVfsFile {
       .map_err(|err| err.into())
   }
 
-  fn read_to_end(&self) -> FsResult<Vec<u8>> {
+  fn read_to_end(&self) -> FsResult<Cow<'static, [u8]>> {
     let read_pos = {
       let mut pos = self.pos.lock();
       let read_pos = *pos;
@@ -659,12 +692,20 @@ impl FileBackedVfsFile {
       read_pos
     };
     if read_pos > self.file.len {
-      return Ok(Vec::new());
+      return Ok(Cow::Borrowed(&[]));
     }
+    if read_pos == 0 {
+      Ok(
+        self
+          .vfs
+          .read_file_all(&self.file, VfsFileSubDataKind::Raw)?,
+      )
+    } else {
       let size = (self.file.len - read_pos) as usize;
       let mut buf = vec![0; size];
       self.vfs.read_file(&self.file, read_pos, &mut buf)?;
-    Ok(buf)
+      Ok(Cow::Owned(buf))
+    }
   }
 }
 
@@ -703,11 +744,14 @@ impl deno_io::fs::File for FileBackedVfsFile {
   }
 
   fn read_all_sync(self: Rc<Self>) -> FsResult<Vec<u8>> {
-    self.read_to_end()
+    self.read_to_end().map(|bytes| bytes.into_owned())
   }
   async fn read_all_async(self: Rc<Self>) -> FsResult<Vec<u8>> {
     let inner = (*self).clone();
-    tokio::task::spawn_blocking(move || inner.read_to_end()).await?
+    tokio::task::spawn_blocking(move || {
+      inner.read_to_end().map(|bytes| bytes.into_owned())
+    })
+    .await?
   }
 
   fn chmod_sync(self: Rc<Self>, _pathmode: u32) -> FsResult<()> {
@@ -878,8 +922,9 @@ impl FileBackedVfs {
   pub fn read_file_all(
     &self,
     file: &VirtualFile,
+    sub_data_kind: VfsFileSubDataKind,
   ) -> std::io::Result<Cow<'static, [u8]>> {
-    let read_range = self.get_read_range(file, 0, file.len)?;
+    let read_range = self.get_read_range(file, sub_data_kind, 0, file.len)?;
     match &self.vfs_data {
       Cow::Borrowed(data) => Ok(Cow::Borrowed(&data[read_range])),
       Cow::Owned(data) => Ok(Cow::Owned(data[read_range].to_vec())),
@@ -892,7 +937,12 @@ impl FileBackedVfs {
     pos: u64,
     buf: &mut [u8],
   ) -> std::io::Result<usize> {
-    let read_range = self.get_read_range(file, pos, buf.len() as u64)?;
+    let read_range = self.get_read_range(
+      file,
+      VfsFileSubDataKind::Raw,
+      pos,
+      buf.len() as u64,
+    )?;
     let read_len = read_range.len();
     buf[..read_len].copy_from_slice(&self.vfs_data[read_range]);
     Ok(read_len)
@@ -901,6 +951,7 @@ impl FileBackedVfs {
   fn get_read_range(
     &self,
     file: &VirtualFile,
+    sub_data_kind: VfsFileSubDataKind,
     pos: u64,
     len: u64,
   ) -> std::io::Result<Range<usize>> {
@@ -910,7 +961,11 @@ impl FileBackedVfs {
         "unexpected EOF",
       ));
     }
-    let file_offset = self.fs_root.start_file_offset + file.offset;
+    let offset = match sub_data_kind {
+      VfsFileSubDataKind::Raw => file.offset,
+      VfsFileSubDataKind::ModuleGraph => file.module_graph_offset,
+    };
+    let file_offset = self.fs_root.start_file_offset + offset;
     let start = file_offset + pos;
     let end = file_offset + std::cmp::min(pos + len, file.len);
     Ok(start as usize..end as usize)
@@ -951,7 +1006,13 @@ mod test {
   #[track_caller]
   fn read_file(vfs: &FileBackedVfs, path: &Path) -> String {
     let file = vfs.file_entry(path).unwrap();
-    String::from_utf8(vfs.read_file_all(file).unwrap().into_owned()).unwrap()
+    String::from_utf8(
+      vfs
+        .read_file_all(file, VfsFileSubDataKind::Raw)
+        .unwrap()
+        .into_owned(),
+    )
+    .unwrap()
   }
 
   #[test]
@@ -964,23 +1025,40 @@ mod test {
     let src_path = src_path.to_path_buf();
     let mut builder = VfsBuilder::new(src_path.clone()).unwrap();
     builder
-      .add_file_with_data_inner(&src_path.join("a.txt"), "data".into())
+      .add_file_with_data_inner(
+        &src_path.join("a.txt"),
+        "data".into(),
+        VfsFileSubDataKind::Raw,
+      )
       .unwrap();
     builder
-      .add_file_with_data_inner(&src_path.join("b.txt"), "data".into())
+      .add_file_with_data_inner(
+        &src_path.join("b.txt"),
+        "data".into(),
+        VfsFileSubDataKind::Raw,
+      )
       .unwrap();
     assert_eq!(builder.files.len(), 1); // because duplicate data
     builder
-      .add_file_with_data_inner(&src_path.join("c.txt"), "c".into())
+      .add_file_with_data_inner(
+        &src_path.join("c.txt"),
+        "c".into(),
+        VfsFileSubDataKind::Raw,
+      )
       .unwrap();
     builder
       .add_file_with_data_inner(
         &src_path.join("sub_dir").join("d.txt"),
         "d".into(),
+        VfsFileSubDataKind::Raw,
       )
       .unwrap();
     builder
-      .add_file_with_data_inner(&src_path.join("e.txt"), "e".into())
+      .add_file_with_data_inner(
+        &src_path.join("e.txt"),
+        "e".into(),
+        VfsFileSubDataKind::Raw,
+      )
       .unwrap();
     builder
       .add_symlink(
@@ -1151,6 +1229,7 @@ mod test {
       .add_file_with_data_inner(
         temp_path.join("a.txt").as_path(),
         "0123456789".to_string().into_bytes(),
+        VfsFileSubDataKind::Raw,
       )
       .unwrap();
     let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);

tests/specs/compile/include/self/__test__.jsonc

@@ -0,0 +1,24 @@
+{
+  "tempDir": true,
+  "steps": [{
+    "if": "unix",
+    "args": "compile --allow-read=. --include . --output main main.ts",
+    "output": "[WILDCARD]"
+  }, {
+    "if": "unix",
+    "commandName": "./main",
+    "args": [],
+    "output": "output.out",
+    "exitCode": 0
+  }, {
+    "if": "windows",
+    "args": "compile --allow-read=. --include . --output main main.ts",
+    "output": "[WILDCARD]"
+  }, {
+    "if": "windows",
+    "commandName": "./main.exe",
+    "args": [],
+    "output": "output.out",
+    "exitCode": 0
+  }]
+}

tests/specs/compile/include/self/main.ts

@@ -0,0 +1,6 @@
+function add(a: number, b: number) {
+  return a + b;
+}
+
+console.log(add(1, 2));
+console.log(Deno.readTextFileSync(import.meta.filename!).trim());

tests/specs/compile/include/self/output.out

@@ -0,0 +1,7 @@
+3
+function add(a: number, b: number) {
+  return a + b;
+}
+
+console.log(add(1, 2));
+console.log(Deno.readTextFileSync(import.meta.filename!).trim());