1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-22 15:24:46 -05:00

feat(unstable): Support data: urls (#5157)

This commit is contained in:
Valentin Anger 2020-09-11 22:40:48 +02:00 committed by GitHub
parent a3282aa9ed
commit e3319f34a6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 170 additions and 10 deletions

1
Cargo.lock generated
View file

@ -417,6 +417,7 @@ dependencies = [
"nix",
"notify",
"os_pipe",
"percent-encoding",
"rand 0.7.3",
"regex",
"reqwest",

View file

@ -52,6 +52,7 @@ libc = "0.2.74"
log = "0.4.11"
env_logger = "0.7.1"
notify = "5.0.0-pre.3"
percent-encoding = "2.1.0"
rand = "0.7.3"
regex = "1.3.9"
reqwest = { version = "0.10.7", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli"] }

View file

@ -103,6 +103,9 @@ impl DiskCache {
out = out.join(remaining_components);
}
"data" => {
out.push(crate::checksum::gen(&[url.as_str().as_bytes()]));
}
scheme => {
unimplemented!(
"Don't know how to create cache name for scheme: {}",

View file

@ -100,7 +100,7 @@ impl SourceFileCache {
}
}
const SUPPORTED_URL_SCHEMES: [&str; 3] = ["http", "https", "file"];
const SUPPORTED_URL_SCHEMES: [&str; 4] = ["http", "https", "file", "data"];
#[derive(Clone)]
pub struct SourceFileFetcher {
@ -278,6 +278,7 @@ impl SourceFileFetcher {
) -> Result<Option<SourceFile>, ErrBox> {
let url_scheme = module_url.scheme();
let is_local_file = url_scheme == "file";
let is_data_url = url_scheme == "data";
SourceFileFetcher::check_if_supported_scheme(&module_url)?;
// Local files are always fetched from disk bypassing cache entirely.
@ -285,6 +286,10 @@ impl SourceFileFetcher {
return self.fetch_local_file(&module_url, permissions).map(Some);
}
if is_data_url {
return extract_data_url(module_url).map(Some);
}
self.fetch_cached_remote_source(&module_url, 10)
}
@ -309,6 +314,7 @@ impl SourceFileFetcher {
) -> Result<SourceFile, ErrBox> {
let url_scheme = module_url.scheme();
let is_local_file = url_scheme == "file";
let is_data_url = url_scheme == "data";
SourceFileFetcher::check_if_supported_scheme(&module_url)?;
// Local files are always fetched from disk bypassing cache entirely.
@ -316,6 +322,10 @@ impl SourceFileFetcher {
return self.fetch_local_file(&module_url, permissions);
}
if is_data_url {
return extract_data_url(module_url);
}
// The file is remote, fail if `no_remote` is true.
if no_remote {
let e = std::io::Error::new(
@ -552,6 +562,36 @@ impl SourceFileFetcher {
}
}
fn extract_data_url(url: &Url) -> Result<SourceFile, ErrBox> {
assert_eq!(url.scheme(), "data");
let url_content = &url.as_str()[5..];
let mut part_iterator = url_content.splitn(2, ',');
let media_type_str = part_iterator.next().unwrap();
let data = if let Some(d) = part_iterator.next() {
d
} else {
return Err(ErrBox::new("URIError", "Malformed data url, missing comma"));
};
let filename = PathBuf::new();
let (media_type, charset) = map_content_type(&filename, Some(media_type_str));
let is_base64 = media_type_str.rsplit(';').any(|v| v == "base64");
let bytes = if is_base64 {
base64::decode(data)?
} else {
percent_encoding::percent_decode_str(data).collect::<Vec<u8>>()
};
Ok(SourceFile {
url: url.clone(),
filename,
types_header: None,
media_type,
source_code: TextDocument::new(bytes, charset),
})
}
pub fn map_file_extension(path: &Path) -> msg::MediaType {
match path.extension() {
None => msg::MediaType::Unknown,

View file

@ -118,6 +118,7 @@ impl GlobalState {
self.file_fetcher.clone(),
maybe_import_map,
permissions.clone(),
self.flags.unstable,
is_dyn_import,
false,
);

View file

@ -34,6 +34,7 @@ impl ModuleDepInfo {
global_state.file_fetcher.clone(),
global_state.maybe_import_map.clone(),
Permissions::allow_all(),
global_state.flags.unstable,
false,
true,
);

View file

@ -478,6 +478,7 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<(), ErrBox> {
global_state.file_fetcher.clone(),
global_state.maybe_import_map.clone(),
Permissions::allow_all(),
flags.unstable,
false,
false,
);

View file

@ -7,6 +7,7 @@ use crate::file_fetcher::SourceFileFetcher;
use crate::import_map::ImportMap;
use crate::msg::MediaType;
use crate::permissions::Permissions;
use crate::state::exit_unstable;
use crate::swc_util::Location;
use crate::tsc::pre_process_file;
use crate::tsc::ImportDesc;
@ -44,20 +45,31 @@ fn err_with_location(e: ErrBox, maybe_location: Option<&Location>) -> ErrBox {
}
/// Disallow http:// imports from modules loaded over https://
/// Disallow any imports from modules loaded with data:
fn validate_no_downgrade(
module_specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
maybe_location: Option<&Location>,
) -> Result<(), ErrBox> {
if let Some(referrer) = maybe_referrer.as_ref() {
if let "https" = referrer.as_url().scheme() {
if let "http" = module_specifier.as_url().scheme() {
let e = ErrBox::new("PermissionDenied",
"Modules loaded over https:// are not allowed to import modules over http://"
match referrer.as_url().scheme() {
"https" => {
if let "http" = module_specifier.as_url().scheme() {
let e = ErrBox::new("PermissionDenied",
"Modules loaded over https:// are not allowed to import modules over http://"
);
return Err(err_with_location(e, maybe_location));
};
}
"data" => {
let e = ErrBox::new(
"PermissionDenied",
"Modules loaded using data URL are not allowed to import other modules",
);
return Err(err_with_location(e, maybe_location));
};
};
}
_ => {}
}
};
Ok(())
@ -75,7 +87,7 @@ fn validate_no_file_from_remote(
"http" | "https" => {
let specifier_url = module_specifier.as_url();
match specifier_url.scheme() {
"http" | "https" => {}
"http" | "https" | "data" => {}
_ => {
let e = ErrBox::new(
"PermissionDenied",
@ -257,6 +269,7 @@ pub struct ModuleGraphLoader {
pending_downloads: FuturesUnordered<SourceFileFuture>,
has_downloaded: HashSet<ModuleSpecifier>,
graph: ModuleGraph,
is_unstable: bool,
is_dyn_import: bool,
analyze_dynamic_imports: bool,
}
@ -266,6 +279,7 @@ impl ModuleGraphLoader {
file_fetcher: SourceFileFetcher,
maybe_import_map: Option<ImportMap>,
permissions: Permissions,
is_unstable: bool,
is_dyn_import: bool,
analyze_dynamic_imports: bool,
) -> Self {
@ -276,6 +290,7 @@ impl ModuleGraphLoader {
pending_downloads: FuturesUnordered::new(),
has_downloaded: HashSet::new(),
graph: ModuleGraph::new(),
is_unstable,
is_dyn_import,
analyze_dynamic_imports,
}
@ -405,6 +420,10 @@ impl ModuleGraphLoader {
return Ok(());
}
if !self.is_unstable && module_specifier.as_url().scheme() == "data" {
exit_unstable("data imports");
}
validate_no_downgrade(
&module_specifier,
maybe_referrer.as_ref(),
@ -600,6 +619,7 @@ mod tests {
global_state.file_fetcher.clone(),
None,
Permissions::allow_all(),
global_state.flags.unstable,
false,
false,
);
@ -873,7 +893,7 @@ fn test_pre_process_file() {
let source = r#"
// This comment is placed to make sure that directives are parsed
// even when they start on non-first line
/// <reference lib="dom" />
/// <reference types="./type_reference.d.ts" />
/// <reference path="./type_reference/dep.ts" />
@ -888,7 +908,7 @@ import * as qat from "./type_definitions/qat.ts";
console.log(foo);
console.log(fizz);
console.log(qat.qat);
console.log(qat.qat);
"#;
let (imports, references) =

View file

@ -300,6 +300,7 @@ impl State {
self.check_read(Path::new(&path))?;
Ok(())
}
"data" => Ok(()),
_ => unreachable!(),
}
}

View file

@ -0,0 +1 @@
import _invalid from "data:";

View file

@ -0,0 +1,2 @@
error: Malformed data url, missing comma
Imported from [WILDCARD]

View file

@ -0,0 +1,2 @@
// export default from "https://deno.land/std/version.ts";
import _upgrade from "data:application/javascript;base64,ZXhwb3J0IGRlZmF1bHQgZnJvbSAiaHR0cHM6Ly9kZW5vLmxhbmQvc3RkL3ZlcnNpb24udHMiOw==";

View file

@ -0,0 +1,2 @@
error: Modules loaded using data URL are not allowed to import other modules
Imported from [WILDCARD]

View file

@ -0,0 +1,59 @@
import { assertEquals } from "../../std/testing/asserts.ts";
// export const value = 'Successful import'; export default value;
import data1 from "data:application/javascript;base64,ZXhwb3J0IGNvbnN0IHZhbHVlID0gJ1N1Y2Nlc3NmdWwgaW1wb3J0JzsgZXhwb3J0IGRlZmF1bHQgdmFsdWU7";
Deno.test("static base64 data url import", () => {
assertEquals(data1, "Successful import");
});
Deno.test("dynamic base64 data url import", async () => {
const data2 = await import(
// export const leet = 1337
"data:application/javascript;base64,ZXhwb3J0IGNvbnN0IGxlZXQgPSAxMzM3"
);
assertEquals(data2.leet, 1337);
});
Deno.test("dynamic percent-encoding data url import", async () => {
const data3 = await import(
// export const value = 42;
"data:application/javascript,export%20const%20value%20%3D%2042%3B"
);
assertEquals(data3.value, 42);
});
Deno.test("dynamic base64 typescript data url import", async () => {
const data2 = await import(
// export const leet: number = 1337;
"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGxlZXQ6IG51bWJlciA9IDEzMzc7"
);
assertEquals(data2.leet, 1337);
});
Deno.test("spawn worker with data url", async () => {
let resolve, timeout;
const promise = new Promise((res, rej) => {
resolve = res;
timeout = setTimeout(() => rej("Worker timed out"), 2000);
});
const worker = new Worker(
"data:application/javascript," +
encodeURIComponent("self.onmessage = () => self.postMessage('Worker');"),
{ type: "module" },
);
worker.onmessage = (m) => {
if (m.data === "Worker") {
resolve();
}
};
worker.postMessage();
await promise;
clearTimeout(timeout);
worker.terminate();
});

View file

@ -0,0 +1,3 @@
[WILDCARD]
test result: ok. 5 passed; [WILDCARD]

View file

@ -2394,6 +2394,23 @@ itest!(info_type_import {
output: "info_type_import.out",
});
itest!(data_import {
args: "test --reload --unstable data_import_test.js",
output: "data_import_test.out",
});
itest!(data_import_invalid {
args: "test --reload --unstable data_import_invalid.js",
output: "data_import_invalid.out",
exit_code: 1,
});
itest!(data_import_origin_upgrade {
args: "test --reload --unstable data_import_origin_upgrade.js",
output: "data_import_origin_upgrade.out",
exit_code: 1,
});
#[test]
fn cafile_env_fetch() {
use url::Url;

View file

@ -2,4 +2,5 @@ error: Uncaught TypeError: Unsupported scheme "xxx" for module "xxx:". Supported
"http",
"https",
"file",
"data",
]

View file

@ -679,6 +679,7 @@ impl TsCompiler {
self.file_fetcher.clone(),
global_state.maybe_import_map.clone(),
permissions.clone(),
global_state.flags.unstable,
false,
true,
);
@ -1157,6 +1158,7 @@ async fn create_runtime_module_graph(
global_state.file_fetcher.clone(),
None,
permissions,
global_state.flags.unstable,
false,
false,
);
@ -1676,6 +1678,7 @@ mod tests {
file_fetcher.clone(),
None,
Permissions::allow_all(),
mock_state.flags.unstable,
false,
false,
);
@ -1752,6 +1755,7 @@ mod tests {
file_fetcher.clone(),
None,
Permissions::allow_all(),
mock_state.flags.unstable,
false,
false,
);