From 0b0af5c635872132d1c727f13ca05aa9be3d1c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 26 Apr 2024 16:41:53 +0100 Subject: [PATCH] test: update for private npm registry test server (#23572) Factored out from https://github.com/denoland/deno/pull/23560 to make it easier to review. --- Cargo.lock | 3 +- tests/util/server/Cargo.toml | 3 +- tests/util/server/src/https.rs | 96 ++++++++++++-------- tests/util/server/src/npm.rs | 51 ++++++++--- tests/util/server/src/servers/grpc.rs | 2 +- tests/util/server/src/servers/hyper_utils.rs | 3 +- tests/util/server/src/servers/mod.rs | 27 ++++-- 7 files changed, 124 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1bbd41df1..6a3f9fd76e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6555,7 +6555,6 @@ dependencies = [ "base64", "bytes", "console_static_text", - "deno_tls", "deno_unsync", "denokv_proto", "fastwebsockets", @@ -6581,6 +6580,8 @@ dependencies = [ "prost-build", "regex", "reqwest", + "rustls-pemfile", + "rustls-tokio-stream", "semver 1.0.14", "serde", "serde_json", diff --git a/tests/util/server/Cargo.toml b/tests/util/server/Cargo.toml index 641cf59939..a321501b85 100644 --- a/tests/util/server/Cargo.toml +++ b/tests/util/server/Cargo.toml @@ -19,7 +19,6 @@ async-stream = "0.3.3" base64.workspace = true bytes.workspace = true console_static_text.workspace = true -deno_tls.workspace = true deno_unsync = "0" denokv_proto.workspace = true fastwebsockets.workspace = true @@ -44,6 +43,8 @@ pretty_assertions.workspace = true prost.workspace = true regex.workspace = true reqwest.workspace = true +rustls-pemfile.workspace = true +rustls-tokio-stream.workspace = true semver = "=1.0.14" serde.workspace = true serde_json.workspace = true diff --git a/tests/util/server/src/https.rs b/tests/util/server/src/https.rs index 0cc58255d5..8a2524dca9 100644 --- a/tests/util/server/src/https.rs +++ b/tests/util/server/src/https.rs @@ -1,14 +1,12 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use anyhow::anyhow; -use deno_tls::load_certs; -use deno_tls::load_private_keys; -use deno_tls::rustls; -use deno_tls::RootCertStore; -use deno_tls::TlsStream; use futures::Stream; use futures::StreamExt; +use rustls::Certificate; +use rustls::PrivateKey; +use rustls_tokio_stream::rustls; +use rustls_tokio_stream::TlsStream; use std::io; -use std::io::Read; use std::num::NonZeroUsize; use std::result::Result; use std::sync::Arc; @@ -70,43 +68,65 @@ pub fn get_tls_config( let key_file = std::fs::File::open(key_path)?; let ca_file = std::fs::File::open(ca_path)?; - let err_map = |x| io::Error::new(io::ErrorKind::InvalidData, x); - let certs = - load_certs(&mut io::BufReader::new(cert_file)).map_err(err_map)?; + let certs: Vec = { + let mut cert_reader = io::BufReader::new(cert_file); + rustls_pemfile::certs(&mut cert_reader) + .unwrap() + .into_iter() + .map(Certificate) + .collect() + }; let mut ca_cert_reader = io::BufReader::new(ca_file); - let ca_cert = load_certs(&mut ca_cert_reader).map_err(err_map)?.remove(0); + let ca_cert = rustls_pemfile::certs(&mut ca_cert_reader) + .expect("Cannot load CA certificate") + .remove(0); let mut key_reader = io::BufReader::new(key_file); - let mut key = vec![]; - key_reader.read_to_end(&mut key)?; - let key = load_private_keys(&key).map_err(err_map)?.remove(0); - - let mut root_cert_store = RootCertStore::empty(); - root_cert_store.add(&ca_cert).unwrap(); - - // Allow (but do not require) client authentication. - - let mut config = rustls::ServerConfig::builder() - .with_safe_defaults() - .with_client_cert_verifier(Arc::new( - rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new( - root_cert_store, - ), - )) - .with_single_cert(certs, key) - .map_err(|e| anyhow!("Error setting cert: {:?}", e)) - .unwrap(); - - match http_versions { - SupportedHttpVersions::All => { - config.alpn_protocols = vec!["h2".into(), "http/1.1".into()]; + let key = { + let pkcs8_key = rustls_pemfile::pkcs8_private_keys(&mut key_reader) + .expect("Cannot load key file"); + let rsa_key = rustls_pemfile::rsa_private_keys(&mut key_reader) + .expect("Cannot load key file"); + if !pkcs8_key.is_empty() { + Some(pkcs8_key[0].clone()) + } else if !rsa_key.is_empty() { + Some(rsa_key[0].clone()) + } else { + None } - SupportedHttpVersions::Http1Only => {} - SupportedHttpVersions::Http2Only => { - config.alpn_protocols = vec!["h2".into()]; + }; + + match key { + Some(key) => { + let mut root_cert_store = rustls::RootCertStore::empty(); + root_cert_store.add(&rustls::Certificate(ca_cert)).unwrap(); + + // Allow (but do not require) client authentication. + + let mut config = rustls::ServerConfig::builder() + .with_safe_defaults() + .with_client_cert_verifier(Arc::new( + rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new( + root_cert_store, + ), + )) + .with_single_cert(certs, PrivateKey(key)) + .map_err(|e| anyhow!("Error setting cert: {:?}", e)) + .unwrap(); + + match http_versions { + SupportedHttpVersions::All => { + config.alpn_protocols = vec!["h2".into(), "http/1.1".into()]; + } + SupportedHttpVersions::Http1Only => {} + SupportedHttpVersions::Http2Only => { + config.alpn_protocols = vec!["h2".into()]; + } + } + + Ok(Arc::new(config)) } + None => Err(io::Error::new(io::ErrorKind::Other, "Cannot find key")), } - - Ok(Arc::new(config)) } diff --git a/tests/util/server/src/npm.rs b/tests/util/server/src/npm.rs index 7469e9b9e3..62105cebe8 100644 --- a/tests/util/server/src/npm.rs +++ b/tests/util/server/src/npm.rs @@ -16,7 +16,21 @@ use tar::Builder; use crate::testdata_path; pub static CUSTOM_NPM_PACKAGE_CACHE: Lazy = - Lazy::new(CustomNpmPackageCache::default); + Lazy::new(|| { + CustomNpmPackageCache::new(format!( + "http://localhost:{}/npm/registry", + crate::servers::PORT, + )) + }); + +pub static CUSTOM_NPM_PACKAGE_CACHE_FOR_PRIVATE_REGISTRY: Lazy< + CustomNpmPackageCache, +> = Lazy::new(|| { + CustomNpmPackageCache::new(format!( + "http://localhost:{}/npm/registry", + crate::servers::PRIVATE_NPM_REGISTRY_1_PORT + )) +}); struct CustomNpmPackage { pub registry_file: String, @@ -25,10 +39,23 @@ struct CustomNpmPackage { /// Creates tarballs and a registry json file for npm packages /// in the `testdata/npm/registry/@denotest` directory. -#[derive(Default)] -pub struct CustomNpmPackageCache(Mutex>); +pub struct CustomNpmPackageCache { + registry_url: String, + cache: Mutex>, +} impl CustomNpmPackageCache { + pub fn new(registry_url: String) -> Self { + let registry_url = registry_url + .strip_suffix('/') + .unwrap_or(®istry_url) + .to_string(); + Self { + registry_url, + cache: Default::default(), + } + } + pub fn tarball_bytes( &self, name: &str, @@ -51,19 +78,22 @@ impl CustomNpmPackageCache { func: impl FnOnce(&CustomNpmPackage) -> TResult, ) -> Result> { // it's ok if multiple threads race here as they will do the same work twice - if !self.0.lock().contains_key(package_name) { - match get_npm_package(package_name)? { + if !self.cache.lock().contains_key(package_name) { + match get_npm_package(package_name, &self.registry_url)? { Some(package) => { - self.0.lock().insert(package_name.to_string(), package); + self.cache.lock().insert(package_name.to_string(), package); } None => return Ok(None), } } - Ok(self.0.lock().get(package_name).map(func)) + Ok(self.cache.lock().get(package_name).map(func)) } } -fn get_npm_package(package_name: &str) -> Result> { +fn get_npm_package( + package_name: &str, + registry_url: &str, +) -> Result> { let package_folder = testdata_path().join("npm/registry").join(package_name); if !package_folder.exists() { return Ok(None); @@ -111,10 +141,7 @@ fn get_npm_package(package_name: &str) -> Result> { dist.insert("shasum".to_string(), "dummy-value".into()); dist.insert( "tarball".to_string(), - format!( - "http://localhost:4545/npm/registry/{package_name}/{version}.tgz" - ) - .into(), + format!("{registry_url}/{package_name}/{version}.tgz").into(), ); tarballs.insert(version.clone(), tarball_bytes); diff --git a/tests/util/server/src/servers/grpc.rs b/tests/util/server/src/servers/grpc.rs index ff00cae49b..144afc06a3 100644 --- a/tests/util/server/src/servers/grpc.rs +++ b/tests/util/server/src/servers/grpc.rs @@ -1,10 +1,10 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_tls::TlsStream; use futures::StreamExt; use h2; use hyper::header::HeaderName; use hyper::header::HeaderValue; +use rustls_tokio_stream::TlsStream; use tokio::net::TcpStream; use tokio::task::LocalSet; diff --git a/tests/util/server/src/servers/hyper_utils.rs b/tests/util/server/src/servers/hyper_utils.rs index 58b5f0cb9f..ea15bba0e5 100644 --- a/tests/util/server/src/servers/hyper_utils.rs +++ b/tests/util/server/src/servers/hyper_utils.rs @@ -1,7 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use bytes::Bytes; -use deno_tls::TlsStream; use futures::Future; use futures::FutureExt; use futures::Stream; @@ -70,7 +69,7 @@ pub async fn run_server_with_acceptor<'a, A, F, S>( error_msg: &'static str, kind: ServerKind, ) where - A: Stream> + ?Sized, + A: Stream> + ?Sized, F: Fn(Request) -> S + Copy + 'static, S: Future + 'static, { diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs index bc94c668e4..f1ffc124e7 100644 --- a/tests/util/server/src/servers/mod.rs +++ b/tests/util/server/src/servers/mod.rs @@ -51,10 +51,11 @@ use hyper_utils::ServerOptions; use super::https::get_tls_listener_stream; use super::https::SupportedHttpVersions; use super::npm::CUSTOM_NPM_PACKAGE_CACHE; +use super::npm::CUSTOM_NPM_PACKAGE_CACHE_FOR_PRIVATE_REGISTRY; use super::std_path; use super::testdata_path; -const PORT: u16 = 4545; +pub(crate) const PORT: u16 = 4545; const TEST_AUTH_TOKEN: &str = "abcdef123456789"; const TEST_BASIC_AUTH_USERNAME: &str = "testuser123"; const TEST_BASIC_AUTH_PASSWORD: &str = "testpassabc"; @@ -85,7 +86,7 @@ const H2_GRPC_PORT: u16 = 4246; const H2S_GRPC_PORT: u16 = 4247; const REGISTRY_SERVER_PORT: u16 = 4250; const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251; -const PRIVATE_NPM_REGISTRY_1_PORT: u16 = 4252; +pub(crate) const PRIVATE_NPM_REGISTRY_1_PORT: u16 = 4252; // Use the single-threaded scheduler. The hyper server is used as a point of // comparison for the (single-threaded!) benchmarks in cli/bench. We're not @@ -1091,7 +1092,9 @@ async fn main_server( } // serve npm registry files - if let Some(resp) = try_serve_npm_registry(&req, file_path.clone()).await + if let Some(resp) = + try_serve_npm_registry(&req, file_path.clone(), NpmRegistryKind::Public) + .await { return resp; } else if let Some(suffix) = req.uri().path().strip_prefix("/deno_std/") { @@ -1120,6 +1123,11 @@ async fn main_server( const PRIVATE_NPM_REGISTRY_AUTH_TOKEN: &str = "private-reg-token"; +enum NpmRegistryKind { + Public, + Private, +} + async fn wrap_private_npm_registry1(port: u16) { let npm_registry_addr = SocketAddr::from(([127, 0, 0, 1], port)); run_server( @@ -1153,7 +1161,9 @@ async fn private_npm_registry1( let mut file_path = testdata_path().to_path_buf(); file_path.push(&req.uri().path()[1..].replace("%2f", "/")); - if let Some(resp) = try_serve_npm_registry(&req, file_path).await { + if let Some(resp) = + try_serve_npm_registry(&req, file_path, NpmRegistryKind::Private).await + { return resp; } @@ -1165,12 +1175,16 @@ async fn private_npm_registry1( fn handle_custom_npm_registry_path( path: &str, + registry_kind: NpmRegistryKind, ) -> Result>>, anyhow::Error> { let parts = path .split('/') .filter(|p| !p.is_empty()) .collect::>(); - let cache = &CUSTOM_NPM_PACKAGE_CACHE; + let cache = match registry_kind { + NpmRegistryKind::Public => &CUSTOM_NPM_PACKAGE_CACHE, + NpmRegistryKind::Private => &CUSTOM_NPM_PACKAGE_CACHE_FOR_PRIVATE_REGISTRY, + }; let package_name = format!("@denotest/{}", parts[0]); if parts.len() == 2 { if let Some(file_bytes) = @@ -1198,6 +1212,7 @@ fn should_download_npm_packages() -> bool { async fn try_serve_npm_registry( req: &Request, mut file_path: PathBuf, + registry_kind: NpmRegistryKind, ) -> Option>, anyhow::Error>> { if let Some(suffix) = req .uri() @@ -1207,7 +1222,7 @@ async fn try_serve_npm_registry( { // serve all requests to /npm/registry/@deno using the file system // at that path - match handle_custom_npm_registry_path(suffix) { + match handle_custom_npm_registry_path(suffix, registry_kind) { Ok(Some(response)) => return Some(Ok(response)), Ok(None) => {} // ignore, not found Err(err) => {