foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-21 15:04:11 -05:00
Code Issues Packages 1 Wiki Activity

feat(tls): Optionally support loading native certs (#11491)

Browse source 
This commit adds "DENO_TLS_CA_STORE" env variable to support 
optionally loading certificates from the users local certificate store. 
This will allow them to successfully connect via tls with corporate 
and self signed certs provided they have them installed in their keystore. 
It also allows them to deal with revoked certs by simply updating 
their keystore without having to upgrade Deno.

Currently supported values are "mozilla", "system" or empty value.
This commit is contained in:
Justin Chase 2021-08-07 07:49:38 -05:00 committed by GitHub
parent fddeb4cea2
commit 02c74fb709
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 488 additions and 235 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
Cargo.lock generated
View file

@ -580,6 +580,7 @@ dependencies = [
"deno_net",
"deno_runtime",
"deno_timers",
"deno_tls",
"deno_url",
"deno_web",
"deno_webgpu",
@ -744,8 +745,10 @@ dependencies = [
"bytes",
"data-url",
"deno_core",
"deno_tls",
"deno_web",
"http",
"lazy_static",
"reqwest",
"serde",
"tokio",
@ -803,15 +806,13 @@ name = "deno_net"
version = "0.4.0"
dependencies = [
"deno_core",
"deno_tls",
"lazy_static",
"log",
"rustls",
"serde",
"tokio",
"trust-dns-proto",
"trust-dns-resolver",
"webpki",
"webpki-roots",
]
[[package]]
@ -828,6 +829,7 @@ dependencies = [
"deno_http",
"deno_net",
"deno_timers",
"deno_tls",
"deno_url",
"deno_web",
"deno_webgpu",
@ -871,6 +873,20 @@ dependencies = [
"tokio",
]
[[package]]
name = "deno_tls"
version = "0.1.0"
dependencies = [
"deno_core",
"lazy_static",
"reqwest",
"rustls",
"rustls-native-certs",
"serde",
"webpki",
"webpki-roots",
]
[[package]]
name = "deno_url"
version = "0.13.0"
@ -920,14 +936,13 @@ name = "deno_websocket"
version = "0.18.0"
dependencies = [
"deno_core",
"deno_tls",
"http",
"hyper",
"serde",
"tokio",
"tokio-rustls",
"tokio-tungstenite",
"webpki",
"webpki-roots",
]
[[package]]
@ -2403,6 +2418,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openssl-probe"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
[[package]]
name = "os_pipe"
version = "0.9.2"
@ -3028,6 +3049,18 @@ dependencies = [
"webpki",
]
[[package]]
name = "rustls-native-certs"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092"
dependencies = [
"openssl-probe",
"rustls",
"schannel",
"security-framework",
]
[[package]]
name = "rusty_v8"
version = "0.25.3"
@ -3089,6 +3122,16 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "schannel"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
dependencies = [
"lazy_static",
"winapi 0.3.9",
]
[[package]]
name = "scoped-tls"
version = "1.0.0"
@ -3111,6 +3154,29 @@ dependencies = [
"untrusted",
]
[[package]]
name = "security-framework"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
dependencies = [
"bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "semver"
version = "0.9.0"

1
cli/Cargo.toml
View file

@ -47,6 +47,7 @@ deno_core = { version = "0.95.0", path = "../core" }
deno_doc = "0.9.0"
deno_lint = "0.11.0"
deno_runtime = { version = "0.21.0", path = "../runtime" }
deno_tls = { version = "0.1.0", path = "../extensions/tls" }
atty = "0.2.14"
base64 = "0.13.0"

12
cli/file_fetcher.rs
View file

@ -3,7 +3,6 @@
use crate::auth_tokens::AuthTokens;
use crate::colors;
use crate::http_cache::HttpCache;
use crate::http_util::create_http_client;
use crate::http_util::fetch_once;
use crate::http_util::FetchOnceArgs;
use crate::http_util::FetchOnceResult;
@ -22,6 +21,8 @@ use deno_core::ModuleSpecifier;
use deno_runtime::deno_fetch::reqwest;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::permissions::Permissions;
use deno_tls::create_http_client;
use deno_tls::rustls::RootCertStore;
use log::debug;
use log::info;
use std::borrow::Borrow;
@ -220,7 +221,7 @@ impl FileFetcher {
http_cache: HttpCache,
cache_setting: CacheSetting,
allow_remote: bool,
ca_data: Option<Vec<u8>>,
root_cert_store: Option<RootCertStore>,
blob_store: BlobStore,
) -> Result<Self, AnyError> {
Ok(Self {
@ -229,7 +230,12 @@ impl FileFetcher {
cache: Default::default(),
cache_setting,
http_cache,
http_client: create_http_client(get_user_agent(), ca_data)?,
http_client: create_http_client(
get_user_agent(),
root_cert_store,
None,
None,
)?,
blob_store,
})
}

4
cli/flags.rs
View file

@ -139,6 +139,7 @@ pub struct Flags {
pub allow_read: Option<Vec<PathBuf>>,
pub allow_run: Option<Vec<String>>,
pub allow_write: Option<Vec<PathBuf>>,
pub ca_stores: Option<Vec<String>>,
pub ca_file: Option<String>,
pub cache_blocklist: Vec<String>,
/// This is not exposed as an option in the CLI, it is used internally when
@ -276,6 +277,9 @@ static ENV_VARIABLES_HELP: &str = r#"ENVIRONMENT VARIABLES:
hostnames to use when fetching remote modules from
private repositories
(e.g. "abcde12345@deno.land;54321edcba@github.com")
DENO_TLS_CA_STORE Comma-seperated list of order dependent certificate stores
(system, mozilla)
(defaults to mozilla)
DENO_CERT Load certificate authority from PEM encoded file
DENO_DIR Set the cache directory
DENO_INSTALL_ROOT Set deno install's output directory

98
cli/http_util.rs
View file

@ -1,46 +1,18 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
use crate::auth_tokens::AuthToken;
use deno_core::error::generic_error;
use deno_core::error::AnyError;
use deno_core::url::Url;
use deno_runtime::deno_fetch::reqwest;
use deno_runtime::deno_fetch::reqwest::header::HeaderMap;
use deno_runtime::deno_fetch::reqwest::header::HeaderValue;
use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;
use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH;
use deno_runtime::deno_fetch::reqwest::header::LOCATION;
use deno_runtime::deno_fetch::reqwest::header::USER_AGENT;
use deno_runtime::deno_fetch::reqwest::redirect::Policy;
use deno_runtime::deno_fetch::reqwest::Client;
use deno_runtime::deno_fetch::reqwest::StatusCode;
use log::debug;
use std::collections::HashMap;
/// Create new instance of async reqwest::Client. This client supports
/// proxies and doesn't follow redirects.
pub fn create_http_client(
user_agent: String,
ca_data: Option<Vec<u8>>,
) -> Result<Client, AnyError> {
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, user_agent.parse().unwrap());
let mut builder = Client::builder()
.redirect(Policy::none())
.default_headers(headers)
.use_rustls_tls();
if let Some(ca_data) = ca_data {
let cert = reqwest::Certificate::from_pem(&ca_data)?;
builder = builder.add_root_certificate(cert);
}
builder
.build()
.map_err(|e| generic_error(format!("Unable to build http client: {}", e)))
}
/// Construct the next uri based on base uri and location header fragment
/// See <https://tools.ietf.org/html/rfc3986#section-4.2>
fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {
@ -168,10 +140,12 @@ pub async fn fetch_once(
mod tests {
use super::*;
use crate::version;
use deno_tls::create_http_client;
use deno_tls::rustls::RootCertStore;
use std::fs::read;
fn create_test_client(ca_data: Option<Vec<u8>>) -> Client {
create_http_client("test_client".to_string(), ca_data).unwrap()
create_http_client("test_client".to_string(), None, ca_data, None).unwrap()
}
#[tokio::test]
@ -362,6 +336,7 @@ mod tests {
let client = create_http_client(
version::get_user_agent(),
None,
Some(
read(
test_util::root_path()
@ -371,6 +346,7 @@ mod tests {
)
.unwrap(),
),
None,
)
.unwrap();
let result = fetch_once(FetchOnceArgs {
@ -390,6 +366,64 @@ mod tests {
}
}
#[tokio::test]
async fn test_fetch_with_default_certificate_store() {
let _http_server_guard = test_util::http_server();
// Relies on external http server with a valid mozilla root CA cert.
let url = Url::parse("https://deno.land").unwrap();
let client = create_http_client(
version::get_user_agent(),
None, // This will load mozilla certs by default
None,
None,
)
.unwrap();
let result = fetch_once(FetchOnceArgs {
client,
url,
maybe_etag: None,
maybe_auth_token: None,
})
.await;
println!("{:?}", result);
if let Ok(FetchOnceResult::Code(body, _headers)) = result {
assert!(!body.is_empty());
} else {
panic!();
}
}
// TODO(@justinmchase): Windows should verify certs too and fail to make this request without ca certs
#[cfg(not(windows))]
#[tokio::test]
async fn test_fetch_with_empty_certificate_store() {
let _http_server_guard = test_util::http_server();
// Relies on external http server with a valid mozilla root CA cert.
let url = Url::parse("https://deno.land").unwrap();
let client = create_http_client(
version::get_user_agent(),
Some(RootCertStore::empty()), // no certs loaded at all
None,
None,
)
.unwrap();
let result = fetch_once(FetchOnceArgs {
client,
url,
maybe_etag: None,
maybe_auth_token: None,
})
.await;
if let Ok(FetchOnceResult::Code(_body, _headers)) = result {
// This test is expected to fail since to CA certs have been loaded
panic!();
}
}
#[tokio::test]
async fn test_fetch_with_cafile_gzip() {
let _http_server_guard = test_util::http_server();
@ -400,6 +434,7 @@ mod tests {
.unwrap();
let client = create_http_client(
version::get_user_agent(),
None,
Some(
read(
test_util::root_path()
@ -409,6 +444,7 @@ mod tests {
)
.unwrap(),
),
None,
)
.unwrap();
let result = fetch_once(FetchOnceArgs {
@ -437,6 +473,7 @@ mod tests {
let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap();
let client = create_http_client(
version::get_user_agent(),
None,
Some(
read(
test_util::root_path()
@ -446,6 +483,7 @@ mod tests {
)
.unwrap(),
),
None,
)
.unwrap();
let result = fetch_once(FetchOnceArgs {
@ -488,6 +526,7 @@ mod tests {
.unwrap();
let client = create_http_client(
version::get_user_agent(),
None,
Some(
read(
test_util::root_path()
@ -497,6 +536,7 @@ mod tests {
)
.unwrap(),
),
None,
)
.unwrap();
let result = fetch_once(FetchOnceArgs {

4
cli/main.rs
View file

@ -109,7 +109,7 @@ fn create_web_worker_callback(
.log_level
.map_or(false, |l| l == log::Level::Debug),
unstable: program_state.flags.unstable,
ca_data: program_state.ca_data.clone(),
root_cert_store: program_state.root_cert_store.clone(),
user_agent: version::get_user_agent(),
seed: program_state.flags.seed,
module_loader,
@ -189,7 +189,7 @@ pub fn create_main_worker(
.log_level
.map_or(false, |l| l == log::Level::Debug),
unstable: program_state.flags.unstable,
ca_data: program_state.ca_data.clone(),
root_cert_store: program_state.root_cert_store.clone(),
user_agent: version::get_user_agent(),
seed: program_state.flags.seed,
js_error_create_fn: Some(js_error_create_fn),

59
cli/program_state.rs
View file

@ -30,12 +30,16 @@ use deno_core::resolve_url;
use deno_core::url::Url;
use deno_core::ModuleSource;
use deno_core::ModuleSpecifier;
use deno_tls::rustls::RootCertStore;
use deno_tls::rustls_native_certs::load_native_certs;
use deno_tls::webpki_roots::TLS_SERVER_ROOTS;
use log::debug;
use log::warn;
use std::collections::HashMap;
use std::collections::HashSet;
use std::env;
use std::fs::read;
use std::fs::File;
use std::io::BufReader;
use std::sync::Arc;
/// This structure represents state of single "deno" program.
@ -53,7 +57,7 @@ pub struct ProgramState {
pub maybe_config_file: Option<ConfigFile>,
pub maybe_import_map: Option<ImportMap>,
pub maybe_inspector_server: Option<Arc<InspectorServer>>,
pub ca_data: Option<Vec<u8>>,
pub root_cert_store: Option<RootCertStore>,
pub blob_store: BlobStore,
pub broadcast_channel: InMemoryBroadcastChannel,
pub shared_array_buffer_store: SharedArrayBufferStore,
@ -68,11 +72,50 @@ impl ProgramState {
let dir = deno_dir::DenoDir::new(maybe_custom_root)?;
let deps_cache_location = dir.root.join("deps");
let http_cache = http_cache::HttpCache::new(&deps_cache_location);
let mut root_cert_store = RootCertStore::empty();
let ca_stores: Vec<String> = flags
.ca_stores
.clone()
.or_else(|| {
let env_ca_store = env::var("DENO_TLS_CA_STORE").ok()?;
Some(
env_ca_store
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect(),
)
})
.unwrap_or_else(|| vec!["mozilla".to_string()]);
for store in ca_stores.iter() {
match store.as_str() {
"mozilla" => {
root_cert_store.add_server_trust_anchors(&TLS_SERVER_ROOTS);
}
"system" => {
let roots = load_native_certs()
.expect("could not load platform certs")
.roots;
root_cert_store.roots.extend(roots);
}
_ => {
return Err(anyhow!("Unknown certificate store \"{}\" specified (allowed: \"system,mozilla\")", store));
}
}
}
let ca_file = flags.ca_file.clone().or_else(|| env::var("DENO_CERT").ok());
let ca_data = match &ca_file {
Some(ca_file) => Some(read(ca_file).context("Failed to open ca file")?),
None => None,
};
if let Some(ca_file) = ca_file {
let certfile = File::open(&ca_file)?;
let mut reader = BufReader::new(certfile);
// This function does not return specific errors, if it fails give a generic message.
if let Err(_err) = root_cert_store.add_pem_file(&mut reader) {
return Err(anyhow!("Unable to add pem file to certificate store"));
}
}
let cache_usage = if flags.cached_only {
CacheSetting::Only
@ -92,7 +135,7 @@ impl ProgramState {
http_cache,
cache_usage,
!flags.no_remote,
ca_data.clone(),
Some(root_cert_store.clone()),
blob_store.clone(),
)?;
@ -152,7 +195,7 @@ impl ProgramState {
maybe_config_file,
maybe_import_map,
maybe_inspector_server,
ca_data,
root_cert_store: Some(root_cert_store.clone()),
blob_store,
broadcast_channel,
shared_array_buffer_store,

21
cli/standalone.rs
View file

@ -8,6 +8,7 @@ use crate::ops;
use crate::program_state::ProgramState;
use crate::version;
use data_url::DataUrl;
use deno_core::error::anyhow;
use deno_core::error::type_error;
use deno_core::error::uri_error;
use deno_core::error::AnyError;
@ -29,11 +30,14 @@ use deno_runtime::permissions::Permissions;
use deno_runtime::permissions::PermissionsOptions;
use deno_runtime::worker::MainWorker;
use deno_runtime::worker::WorkerOptions;
use deno_tls::create_default_root_cert_store;
use log::Level;
use std::cell::RefCell;
use std::convert::TryInto;
use std::env::current_exe;
use std::fs::File;
use std::io::BufReader;
use std::io::Cursor;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
@ -51,6 +55,7 @@ pub struct Metadata {
pub location: Option<Url>,
pub v8_flags: Vec<String>,
pub log_level: Option<Level>,
pub ca_stores: Option<Vec<String>>,
pub ca_data: Option<Vec<u8>>,
}
@ -201,6 +206,7 @@ fn metadata_to_flags(metadata: &Metadata) -> Flags {
allow_write: permissions.allow_write,
v8_flags: metadata.v8_flags.clone(),
log_level: metadata.log_level,
ca_stores: metadata.ca_stores.clone(),
..Default::default()
}
}
@ -227,13 +233,26 @@ pub async fn run(
.collect::<Vec<_>>(),
);
let mut root_cert_store = program_state
.root_cert_store
.clone()
.unwrap_or_else(create_default_root_cert_store);
if let Some(cert) = metadata.ca_data {
let reader = &mut BufReader::new(Cursor::new(cert));
// This function does not return specific errors, if it fails give a generic message.
if let Err(_err) = root_cert_store.add_pem_file(reader) {
return Err(anyhow!("Unable to add pem file to certificate store"));
}
}
let options = WorkerOptions {
apply_source_maps: false,
args: metadata.argv,
debug_flag: metadata.log_level.map_or(false, |l| l == log::Level::Debug),
user_agent: version::get_user_agent(),
unstable: metadata.unstable,
ca_data: metadata.ca_data,
root_cert_store: Some(root_cert_store),
seed: metadata.seed,
js_error_create_fn: None,
create_web_worker_cb,

4
cli/tests/integration/mod.rs
View file

@ -2,9 +2,9 @@
use crate::itest;
use deno_core::url;
use deno_runtime::deno_net::ops_tls::rustls;
use deno_runtime::deno_net::ops_tls::webpki;
use deno_runtime::deno_net::ops_tls::TlsStream;
use deno_runtime::deno_tls::rustls;
use deno_runtime::deno_tls::webpki;
use std::fs;
use std::io::BufReader;
use std::io::Cursor;

2
cli/tools/standalone.rs
View file

@ -100,6 +100,7 @@ pub fn create_standalone_binary(
permissions: flags.clone().into(),
v8_flags: flags.v8_flags.clone(),
log_level: flags.log_level,
ca_stores: flags.ca_stores,
ca_data,
};
let mut metadata = serde_json::to_string(&metadata)?.as_bytes().to_vec();
@ -205,6 +206,7 @@ pub fn compile_to_runtime_flags(
allow_read: flags.allow_read,
allow_run: flags.allow_run,
allow_write: flags.allow_write,
ca_stores: flags.ca_stores,
ca_file: flags.ca_file,
cache_blocklist: vec![],
cache_path: None,

2
extensions/fetch/Cargo.toml
View file

@ -17,8 +17,10 @@ path = "lib.rs"
bytes = "1.0.1"
data-url = "0.1.0"
deno_core = { version = "0.95.0", path = "../../core" }
deno_tls = { version = "0.1.0", path = "../tls" }
deno_web = { version = "0.44.0", path = "../web" }
http = "0.2.4"
lazy_static = "1.4.0"
reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli"] }
serde = { version = "1.0.126", features = ["derive"] }
tokio = { version = "1.8.1", features = ["full"] }

76
extensions/fetch/lib.rs
View file

@ -1,7 +1,7 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
use data_url::DataUrl;
use deno_core::error::bad_resource_id;
use deno_core::error::generic_error;
use deno_core::error::null_opbuf;
use deno_core::error::type_error;
use deno_core::error::AnyError;
@ -24,16 +24,14 @@ use deno_core::RcRef;
use deno_core::Resource;
use deno_core::ResourceId;
use deno_core::ZeroCopyBuf;
use data_url::DataUrl;
use deno_tls::create_http_client;
use deno_tls::rustls::RootCertStore;
use deno_tls::Proxy;
use deno_web::BlobStore;
use http::header::CONTENT_LENGTH;
use reqwest::header::HeaderMap;
use reqwest::header::HeaderName;
use reqwest::header::HeaderValue;
use reqwest::header::HOST;
use reqwest::header::USER_AGENT;
use reqwest::redirect::Policy;
use reqwest::Body;
use reqwest::Client;
use reqwest::Method;
@ -59,7 +57,7 @@ pub use reqwest; // Re-export reqwest
pub fn init<P: FetchPermissions + 'static>(
user_agent: String,
ca_data: Option<Vec<u8>>,
root_cert_store: Option<RootCertStore>,
proxy: Option<Proxy>,
request_builder_hook: Option<fn(RequestBuilder) -> RequestBuilder>,
) -> Extension {
@ -84,12 +82,17 @@ pub fn init<P: FetchPermissions + 'static>(
])
.state(move |state| {
state.put::<reqwest::Client>({
create_http_client(user_agent.clone(), ca_data.clone(), proxy.clone())
create_http_client(
user_agent.clone(),
root_cert_store.clone(),
None,
proxy.clone(),
)
.unwrap()
});
state.put::<HttpClientDefaults>(HttpClientDefaults {
ca_data: ca_data.clone(),
user_agent: user_agent.clone(),
root_cert_store: root_cert_store.clone(),
proxy: proxy.clone(),
request_builder_hook,
});
@ -100,7 +103,7 @@ pub fn init<P: FetchPermissions + 'static>(
pub struct HttpClientDefaults {
pub user_agent: String,
pub ca_data: Option<Vec<u8>>,
pub root_cert_store: Option<RootCertStore>,
pub proxy: Option<Proxy>,
pub request_builder_hook: Option<fn(RequestBuilder) -> RequestBuilder>,
}
@ -501,26 +504,12 @@ impl HttpClientResource {
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct CreateHttpClientOptions {
ca_stores: Option<Vec<String>>,
ca_file: Option<String>,
ca_data: Option<ByteString>,
proxy: Option<Proxy>,
}
#[derive(Deserialize, Default, Debug, Clone)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct Proxy {
pub url: String,
pub basic_auth: Option<BasicAuth>,
}
#[derive(Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct BasicAuth {
pub username: String,
pub password: String,
}
pub fn op_create_http_client<FP>(
state: &mut OpState,
args: CreateHttpClientOptions,
@ -541,12 +530,12 @@ where
}
let defaults = state.borrow::<HttpClientDefaults>();
let cert_data =
get_cert_data(args.ca_file.as_deref(), args.ca_data.as_deref())?;
let client = create_http_client(
defaults.user_agent.clone(),
cert_data.or_else(|| defaults.ca_data.clone()),
defaults.root_cert_store.clone(),
cert_data,
args.proxy,
)
.unwrap();
@ -569,36 +558,3 @@ fn get_cert_data(
Ok(None)
}
}
/// Create new instance of async reqwest::Client. This client supports
/// proxies and doesn't follow redirects.
pub fn create_http_client(
user_agent: String,
ca_data: Option<Vec<u8>>,
proxy: Option<Proxy>,
) -> Result<Client, AnyError> {
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, user_agent.parse().unwrap());
let mut builder = Client::builder()
.redirect(Policy::none())
.default_headers(headers)
.use_rustls_tls();
if let Some(ca_data) = ca_data {
let cert = reqwest::Certificate::from_pem(&ca_data)?;
builder = builder.add_root_certificate(cert);
}
if let Some(proxy) = proxy {
let mut reqwest_proxy = reqwest::Proxy::all(&proxy.url)?;
if let Some(basic_auth) = &proxy.basic_auth {
reqwest_proxy =
reqwest_proxy.basic_auth(&basic_auth.username, &basic_auth.password);
}
builder = builder.proxy(reqwest_proxy);
}
builder
.build()
.map_err(|e| generic_error(format!("Unable to build http client: {}", e)))
}

4
extensions/net/Cargo.toml
View file

@ -15,13 +15,11 @@ path = "lib.rs"
[dependencies]
deno_core = { version = "0.95.0", path = "../../core" }
deno_tls = { version = "0.1.0", path = "../tls" }
lazy_static = "1.4.0"
log = "0.4.14"
rustls = "0.19.0"
serde = { version = "1.0.126", features = ["derive"] }
tokio = { version = "1.8.1", features = ["full"] }
trust-dns-proto = "0.20.3"
trust-dns-resolver = { version = "0.20.3", features = ["tokio-runtime", "serde-config"] }
webpki = "0.21.4"
webpki-roots = "0.21.1"

12
extensions/net/lib.rs
View file

@ -11,6 +11,7 @@ use deno_core::error::AnyError;
use deno_core::include_js_files;
use deno_core::Extension;
use deno_core::OpState;
use deno_tls::rustls::RootCertStore;
use std::cell::RefCell;
use std::path::Path;
use std::path::PathBuf;
@ -90,20 +91,17 @@ pub fn get_unstable_declaration() -> PathBuf {
#[derive(Clone)]
pub struct DefaultTlsOptions {
pub ca_data: Option<Vec<u8>>,
pub root_cert_store: Option<RootCertStore>,
}
pub fn init<P: NetPermissions + 'static>(
ca_data: Option<Vec<u8>>,
root_cert_store: Option<RootCertStore>,
unstable: bool,
) -> Extension {
let mut ops_to_register = vec![];
ops_to_register.extend(io::init());
ops_to_register.extend(ops::init::<P>());
ops_to_register.extend(ops_tls::init::<P>());
let default_tls_options = DefaultTlsOptions { ca_data };
Extension::builder()
.js(include_js_files!(
prefix "deno:extensions/net",
@ -113,7 +111,9 @@ pub fn init<P: NetPermissions + 'static>(
))
.ops(ops_to_register)
.state(move |state| {
state.put(default_tls_options.clone());
state.put(DefaultTlsOptions {
root_cert_store: root_cert_store.clone(),
});
state.put(UnstableChecker { unstable });
Ok(())
})

128
extensions/net/ops_tls.rs
View file

@ -1,8 +1,5 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
pub use rustls;
pub use webpki;
use crate::io::TcpStreamResource;
use crate::io::TlsStreamResource;
use crate::ops::IpAddr;
@ -38,30 +35,29 @@ use deno_core::OpState;
use deno_core::RcRef;
use deno_core::Resource;
use deno_core::ResourceId;
use deno_tls::create_client_config;
use deno_tls::rustls::internal::pemfile::certs;
use deno_tls::rustls::internal::pemfile::pkcs8_private_keys;
use deno_tls::rustls::internal::pemfile::rsa_private_keys;
use deno_tls::rustls::Certificate;
use deno_tls::rustls::ClientConfig;
use deno_tls::rustls::ClientSession;
use deno_tls::rustls::NoClientAuth;
use deno_tls::rustls::PrivateKey;
use deno_tls::rustls::ServerConfig;
use deno_tls::rustls::ServerSession;
use deno_tls::rustls::Session;
use deno_tls::webpki::DNSNameRef;
use io::Error;
use io::Read;
use io::Write;
use rustls::internal::pemfile::certs;
use rustls::internal::pemfile::pkcs8_private_keys;
use rustls::internal::pemfile::rsa_private_keys;
use rustls::Certificate;
use rustls::ClientConfig;
use rustls::ClientSession;
use rustls::NoClientAuth;
use rustls::PrivateKey;
use rustls::ServerConfig;
use rustls::ServerSession;
use rustls::Session;
use rustls::StoresClientSessions;
use serde::Deserialize;
use std::borrow::Cow;
use std::cell::RefCell;
use std::collections::HashMap;
use std::convert::From;
use std::fs::File;
use std::io;
use std::io::BufReader;
use std::io::Cursor;
use std::io::ErrorKind;
use std::ops::Deref;
use std::ops::DerefMut;
@ -76,32 +72,6 @@ use tokio::io::ReadBuf;
use tokio::net::TcpListener;
use tokio::net::TcpStream;
use tokio::task::spawn_local;
use webpki::DNSNameRef;
lazy_static::lazy_static! {
static ref CLIENT_SESSION_MEMORY_CACHE: Arc<ClientSessionMemoryCache> =
Arc::new(ClientSessionMemoryCache::default());
}
#[derive(Default)]
struct ClientSessionMemoryCache(Mutex<HashMap<Vec<u8>, Vec<u8>>>);
impl StoresClientSessions for ClientSessionMemoryCache {
fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
self.0.lock().get(key).cloned()
}
fn put(&self, key: Vec<u8>, value: Vec<u8>) -> bool {
let mut sessions = self.0.lock();
// TODO(bnoordhuis) Evict sessions LRU-style instead of arbitrarily.
while sessions.len() >= 1024 {
let key = sessions.keys().next().unwrap().clone();
sessions.remove(&key);
}
sessions.insert(key, value);
true
}
}
#[derive(Debug)]
enum TlsSession {
@ -703,8 +673,6 @@ where
n => n,
};
let cert_file = args.cert_file.as_deref();
let default_tls_options;
{
super::check_unstable2(&state, "Deno.startTls");
let mut s = state.borrow_mut();
@ -713,12 +681,28 @@ where
if let Some(path) = cert_file {
permissions.check_read(Path::new(path))?;
}
default_tls_options = s.borrow::<DefaultTlsOptions>().clone();
}
let ca_data = match cert_file {
Some(path) => {
let mut buf = Vec::new();
File::open(path)?.read_to_end(&mut buf)?;
Some(buf)
}
_ => None,
};
let hostname_dns = DNSNameRef::try_from_ascii_str(hostname)
.map_err(|_| invalid_hostname(hostname))?;
// TODO(@justinmchase): Ideally the certificate store is created once
// and not cloned. The store should be wrapped in Arc<T> to reduce
// copying memory unnecessarily.
let root_cert_store = state
.borrow()
.borrow::<DefaultTlsOptions>()
.root_cert_store
.clone();
let resource_rc = state
.borrow_mut()
.resource_table
@ -732,22 +716,7 @@ where
let local_addr = tcp_stream.local_addr()?;
let remote_addr = tcp_stream.peer_addr()?;
let mut tls_config = ClientConfig::new();
tls_config.set_persistence(CLIENT_SESSION_MEMORY_CACHE.clone());
tls_config
.root_store
.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
if let Some(ca_data) = default_tls_options.ca_data {
let reader = &mut Cursor::new(ca_data);
tls_config.root_store.add_pem_file(reader).unwrap();
};
if let Some(path) = cert_file {
let key_file = File::open(path)?;
let reader = &mut BufReader::new(key_file);
tls_config.root_store.add_pem_file(reader).unwrap();
}
let tls_config = Arc::new(tls_config);
let tls_config = Arc::new(create_client_config(root_cert_store, ca_data)?);
let tls_stream =
TlsStream::new_client_side(tcp_stream, &tls_config, hostname_dns);
@ -786,8 +755,6 @@ where
};
let port = args.port;
let cert_file = args.cert_file.as_deref();
let default_tls_options;
{
let mut s = state.borrow_mut();
let permissions = s.borrow_mut::<NP>();
@ -795,9 +762,22 @@ where
if let Some(path) = cert_file {
permissions.check_read(Path::new(path))?;
}
default_tls_options = s.borrow::<DefaultTlsOptions>().clone();
}
let ca_data = match cert_file {
Some(path) => {
let mut buf = Vec::new();
File::open(path)?.read_to_end(&mut buf)?;
Some(buf)
}
_ => None,
};
let root_cert_store = state
.borrow()
.borrow::<DefaultTlsOptions>()
.root_cert_store
.clone();
let hostname_dns = DNSNameRef::try_from_ascii_str(hostname)
.map_err(|_| invalid_hostname(hostname))?;
@ -808,23 +788,7 @@ where
let tcp_stream = TcpStream::connect(connect_addr).await?;
let local_addr = tcp_stream.local_addr()?;
let remote_addr = tcp_stream.peer_addr()?;
let mut tls_config = ClientConfig::new();
tls_config.set_persistence(CLIENT_SESSION_MEMORY_CACHE.clone());
tls_config
.root_store
.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
if let Some(ca_data) = default_tls_options.ca_data {
let reader = &mut Cursor::new(ca_data);
tls_config.root_store.add_pem_file(reader).unwrap();
};
if let Some(path) = cert_file {
let key_file = File::open(path)?;
let reader = &mut BufReader::new(key_file);
tls_config.root_store.add_pem_file(reader).unwrap();
}
let tls_config = Arc::new(tls_config);
let tls_config = Arc::new(create_client_config(root_cert_store, ca_data)?);
let tls_stream =
TlsStream::new_client_side(tcp_stream, &tls_config, hostname_dns);

24
extensions/tls/Cargo.toml Normal file
View file

@ -0,0 +1,24 @@
# Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
[package]
name = "deno_tls"
version = "0.1.0"
authors = ["the Deno authors"]
edition = "2018"
license = "MIT"
readme = "README.md"
repository = "https://github.com/denoland/deno"
description = "TLS for Deno"
[lib]
path = "lib.rs"
[dependencies]
deno_core = { version = "0.95.0", path = "../../core" }
lazy_static = "1.4.0"
reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli"] }
rustls = "0.19.0"
rustls-native-certs = "0.5.0"
serde = { version = "1.0.126", features = ["derive"] }
webpki = "0.21.4"
webpki-roots = "0.21.1"

129
extensions/tls/lib.rs Normal file
View file

@ -0,0 +1,129 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
pub use reqwest;
pub use rustls;
pub use rustls_native_certs;
pub use webpki;
pub use webpki_roots;
use deno_core::error::anyhow;
use deno_core::error::generic_error;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::Extension;
use reqwest::header::HeaderMap;
use reqwest::header::USER_AGENT;
use reqwest::redirect::Policy;
use reqwest::Client;
use rustls::ClientConfig;
use rustls::RootCertStore;
use rustls::StoresClientSessions;
use serde::Deserialize;
use std::collections::HashMap;
use std::io::BufReader;
use std::io::Cursor;
use std::sync::Arc;
/// This extension has no runtime apis, it only exports some shared native functions.
pub fn init() -> Extension {
Extension::builder().build()
}
#[derive(Deserialize, Default, Debug, Clone)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct Proxy {
pub url: String,
pub basic_auth: Option<BasicAuth>,
}
#[derive(Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct BasicAuth {
pub username: String,
pub password: String,
}
lazy_static::lazy_static! {
static ref CLIENT_SESSION_MEMORY_CACHE: Arc<ClientSessionMemoryCache> =
Arc::new(ClientSessionMemoryCache::default());
}
#[derive(Default)]
struct ClientSessionMemoryCache(Mutex<HashMap<Vec<u8>, Vec<u8>>>);
impl StoresClientSessions for ClientSessionMemoryCache {
fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
self.0.lock().get(key).cloned()
}
fn put(&self, key: Vec<u8>, value: Vec<u8>) -> bool {
let mut sessions = self.0.lock();
// TODO(bnoordhuis) Evict sessions LRU-style instead of arbitrarily.
while sessions.len() >= 1024 {
let key = sessions.keys().next().unwrap().clone();
sessions.remove(&key);
}
sessions.insert(key, value);
true
}
}
pub fn create_default_root_cert_store() -> RootCertStore {
let mut root_cert_store = RootCertStore::empty();
// TODO(@justinmchase): Consider also loading the system keychain here
root_cert_store.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
root_cert_store
}
pub fn create_client_config(
root_cert_store: Option<RootCertStore>,
ca_data: Option<Vec<u8>>,
) -> Result<ClientConfig, AnyError> {
let mut tls_config = ClientConfig::new();
tls_config.set_persistence(CLIENT_SESSION_MEMORY_CACHE.clone());
tls_config.root_store =
root_cert_store.unwrap_or_else(create_default_root_cert_store);
// If a custom cert is specified, add it to the store
if let Some(cert) = ca_data {
let reader = &mut BufReader::new(Cursor::new(cert));
// This function does not return specific errors, if it fails give a generic message.
if let Err(_err) = tls_config.root_store.add_pem_file(reader) {
return Err(anyhow!("Unable to add pem file to certificate store"));
}
}
Ok(tls_config)
}
/// Create new instance of async reqwest::Client. This client supports
/// proxies and doesn't follow redirects.
pub fn create_http_client(
user_agent: String,
root_cert_store: Option<RootCertStore>,
ca_data: Option<Vec<u8>>,
proxy: Option<Proxy>,
) -> Result<Client, AnyError> {
let tls_config = create_client_config(root_cert_store, ca_data)?;
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, user_agent.parse().unwrap());
let mut builder = Client::builder()
.redirect(Policy::none())
.default_headers(headers)
.use_preconfigured_tls(tls_config);
if let Some(proxy) = proxy {
let mut reqwest_proxy = reqwest::Proxy::all(&proxy.url)?;
if let Some(basic_auth) = &proxy.basic_auth {
reqwest_proxy =
reqwest_proxy.basic_auth(&basic_auth.username, &basic_auth.password);
}
builder = builder.proxy(reqwest_proxy);
}
builder
.build()
.map_err(|e| generic_error(format!("Unable to build http client: {}", e)))
}

3
extensions/websocket/Cargo.toml
View file

@ -15,11 +15,10 @@ path = "lib.rs"
[dependencies]
deno_core = { version = "0.95.0", path = "../../core" }
deno_tls = { version = "0.1.0", path = "../tls" }
http = "0.2.4"
hyper = { version = "0.14.9" }
serde = { version = "1.0.126", features = ["derive"] }
tokio = { version = "1.8.1", features = ["full"] }
tokio-rustls = "0.22.0"
tokio-tungstenite = { version = "0.14.0", features = ["rustls-tls"] }
webpki = "0.21.4"
webpki-roots = "0.21.1"

31
extensions/websocket/lib.rs
View file

@ -22,31 +22,31 @@ use deno_core::RcRef;
use deno_core::Resource;
use deno_core::ResourceId;
use deno_core::ZeroCopyBuf;
use deno_tls::create_client_config;
use deno_tls::webpki::DNSNameRef;
use http::{Method, Request, Uri};
use serde::Deserialize;
use serde::Serialize;
use std::borrow::Cow;
use std::cell::RefCell;
use std::io::BufReader;
use std::io::Cursor;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use tokio::net::TcpStream;
use tokio_rustls::{rustls::ClientConfig, TlsConnector};
use tokio_rustls::rustls::RootCertStore;
use tokio_rustls::TlsConnector;
use tokio_tungstenite::tungstenite::{
handshake::client::Response, protocol::frame::coding::CloseCode,
protocol::CloseFrame, Message,
};
use tokio_tungstenite::MaybeTlsStream;
use tokio_tungstenite::{client_async, WebSocketStream};
use webpki::DNSNameRef;
pub use tokio_tungstenite; // Re-export tokio_tungstenite
#[derive(Clone)]
pub struct WsCaData(pub Vec<u8>);
pub struct WsRootStore(pub Option<RootCertStore>);
#[derive(Clone)]
pub struct WsUserAgent(pub String);
@ -197,7 +197,7 @@ where
);
}
let ws_ca_data = state.borrow().try_borrow::<WsCaData>().cloned();
let root_cert_store = state.borrow().borrow::<WsRootStore>().0.clone();
let user_agent = state.borrow().borrow::<WsUserAgent>().0.clone();
let uri: Uri = args.url.parse()?;
let mut request = Request::builder().method(Method::GET).uri(&uri);
@ -221,17 +221,8 @@ where
let socket: MaybeTlsStream<TcpStream> = match uri.scheme_str() {
Some("ws") => MaybeTlsStream::Plain(tcp_socket),
Some("wss") => {
let mut config = ClientConfig::new();
config
.root_store
.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
if let Some(ws_ca_data) = ws_ca_data {
let reader = &mut BufReader::new(Cursor::new(ws_ca_data.0));
config.root_store.add_pem_file(reader).unwrap();
}
let tls_connector = TlsConnector::from(Arc::new(config));
let tls_config = create_client_config(root_cert_store, None)?;
let tls_connector = TlsConnector::from(Arc::new(tls_config));
let dnsname = DNSNameRef::try_from_ascii_str(domain)
.map_err(|_| invalid_hostname(domain))?;
let tls_socket = tls_connector.connect(dnsname, tcp_socket).await?;
@ -385,7 +376,7 @@ pub async fn op_ws_next_event(
pub fn init<P: WebSocketPermissions + 'static>(
user_agent: String,
ca_data: Option<Vec<u8>>,
root_cert_store: Option<RootCertStore>,
) -> Extension {
Extension::builder()
.js(include_js_files!(
@ -404,9 +395,7 @@ pub fn init<P: WebSocketPermissions + 'static>(
])
.state(move |state| {
state.put::<WsUserAgent>(WsUserAgent(user_agent.clone()));
if let Some(ca_data) = ca_data.clone() {
state.put::<WsCaData>(WsCaData(ca_data));
}
state.put::<WsRootStore>(WsRootStore(root_cert_store.clone()));
Ok(())
})
.build()

2
runtime/Cargo.toml
View file

@ -27,6 +27,7 @@ deno_ffi = { version = "0.1.0", path = "../extensions/ffi" }
deno_http = { version = "0.4.0", path = "../extensions/http" }
deno_net = { version = "0.4.0", path = "../extensions/net" }
deno_timers = { version = "0.11.0", path = "../extensions/timers" }
deno_tls = { version = "0.1.0", path = "../extensions/tls" }
deno_url = { version = "0.13.0", path = "../extensions/url" }
deno_web = { version = "0.44.0", path = "../extensions/web" }
deno_webgpu = { version = "0.14.0", path = "../extensions/webgpu" }
@ -48,6 +49,7 @@ deno_ffi = { version = "0.1.0", path = "../extensions/ffi" }
deno_http = { version = "0.4.0", path = "../extensions/http" }
deno_net = { version = "0.4.0", path = "../extensions/net" }
deno_timers = { version = "0.11.0", path = "../extensions/timers" }
deno_tls = { version = "0.1.0", path = "../extensions/tls" }
deno_url = { version = "0.13.0", path = "../extensions/url" }
deno_web = { version = "0.44.0", path = "../extensions/web" }
deno_webgpu = { version = "0.14.0", path = "../extensions/webgpu" }

1
runtime/build.rs
View file

@ -41,6 +41,7 @@ fn create_runtime_snapshot(snapshot_path: &Path, files: Vec<PathBuf>) {
deno_webidl::init(),
deno_console::init(),
deno_url::init(),
deno_tls::init(),
deno_web::init(deno_web::BlobStore::default(), Default::default()),
deno_fetch::init::<deno_fetch::NoFetchPermissions>(
"".to_owned(),

2
runtime/examples/hello_runtime.rs
View file

@ -27,7 +27,7 @@ async fn main() -> Result<(), AnyError> {
args: vec![],
debug_flag: false,
unstable: false,
ca_data: None,
root_cert_store: None,
user_agent: "hello_runtime".to_string(),
seed: None,
js_error_create_fn: None,

1
runtime/lib.rs
View file

@ -8,6 +8,7 @@ pub use deno_ffi;
pub use deno_http;
pub use deno_net;
pub use deno_timers;
pub use deno_tls;
pub use deno_url;
pub use deno_web;
pub use deno_webgpu;

10
runtime/web_worker.rs
View file

@ -29,6 +29,7 @@ use deno_core::ModuleLoader;
use deno_core::ModuleSpecifier;
use deno_core::RuntimeOptions;
use deno_core::SharedArrayBufferStore;
use deno_tls::rustls::RootCertStore;
use deno_web::create_entangled_message_port;
use deno_web::BlobStore;
use deno_web::MessagePort;
@ -252,7 +253,7 @@ pub struct WebWorkerOptions {
pub args: Vec<String>,
pub debug_flag: bool,
pub unstable: bool,
pub ca_data: Option<Vec<u8>>,
pub root_cert_store: Option<RootCertStore>,
pub user_agent: String,
pub seed: Option<u64>,
pub module_loader: Rc<dyn ModuleLoader>,
@ -300,13 +301,13 @@ impl WebWorker {
deno_web::init(options.blob_store.clone(), Some(main_module.clone())),
deno_fetch::init::<Permissions>(
options.user_agent.clone(),
options.ca_data.clone(),
options.root_cert_store.clone(),
None,
None,
),
deno_websocket::init::<Permissions>(
options.user_agent.clone(),
options.ca_data.clone(),
options.root_cert_store.clone(),
),
deno_broadcast_channel::init(
options.broadcast_channel.clone(),
@ -336,8 +337,9 @@ impl WebWorker {
vec![
ops::fs_events::init(),
ops::fs::init(),
deno_tls::init(),
deno_net::init::<Permissions>(
options.ca_data.clone(),
options.root_cert_store.clone(),
options.unstable,
),
ops::os::init(),

15
runtime/worker.rs
View file

@ -22,6 +22,7 @@ use deno_core::ModuleLoader;
use deno_core::ModuleSpecifier;
use deno_core::RuntimeOptions;
use deno_core::SharedArrayBufferStore;
use deno_tls::rustls::RootCertStore;
use deno_web::BlobStore;
use log::debug;
use std::env;
@ -49,7 +50,7 @@ pub struct WorkerOptions {
pub args: Vec<String>,
pub debug_flag: bool,
pub unstable: bool,
pub ca_data: Option<Vec<u8>>,
pub root_cert_store: Option<RootCertStore>,
pub user_agent: String,
pub seed: Option<u64>,
pub module_loader: Rc<dyn ModuleLoader>,
@ -99,13 +100,13 @@ impl MainWorker {
deno_web::init(options.blob_store.clone(), options.location.clone()),
deno_fetch::init::<Permissions>(
options.user_agent.clone(),
options.ca_data.clone(),
options.root_cert_store.clone(),
None,
None,
),
deno_websocket::init::<Permissions>(
options.user_agent.clone(),
options.ca_data.clone(),
options.root_cert_store.clone(),
),
deno_webstorage::init(options.origin_storage_dir.clone()),
deno_crypto::init(options.seed),
@ -126,7 +127,11 @@ impl MainWorker {
ops::fs::init(),
ops::io::init(),
ops::io::init_stdio(),
deno_net::init::<Permissions>(options.ca_data.clone(), options.unstable),
deno_tls::init(),
deno_net::init::<Permissions>(
options.root_cert_store.clone(),
options.unstable,
),
ops::os::init(),
ops::permissions::init(),
ops::process::init(),
@ -295,7 +300,7 @@ mod tests {
args: vec![],
debug_flag: false,
unstable: false,
ca_data: None,
root_cert_store: None,
seed: None,
js_error_create_fn: None,
create_web_worker_cb: Arc::new(|_| unreachable!()),